From 06ca55c532fc9c0f17726fc623a2e11c60370ac4 Mon Sep 17 00:00:00 2001 From: Rickard Bolin Date: Wed, 19 Feb 2025 14:42:34 +0000 Subject: [PATCH 1/4] MLBEDSW-10401: Update documentation to reflect TOSA status - Simplified README file and moved information to other docs - Created new BUILD.md to explain the manual build options Change-Id: I1c834257056e9cc1f11c8257316332e3c6c88379 Signed-off-by: Rickard Bolin --- BUGS.md | 21 +- BUILD.md | 153 ++++++++++++ CONTRIBUTING.md | 65 ++--- OPTIONS.md | 37 ++- PERFORMANCE.md | 17 +- README.md | 432 ++++++++------------------------- SECURITY.md | 13 +- TESTING.md | 149 +++++++----- ethosu/vela/test/test_build.py | 11 +- pyproject.toml | 6 +- 10 files changed, 428 insertions(+), 476 deletions(-) create mode 100644 BUILD.md diff --git a/BUGS.md b/BUGS.md index 0d61cc75..ebb4f874 100644 --- a/BUGS.md +++ b/BUGS.md @@ -15,21 +15,18 @@ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -# Vela Community Bug Reporting +# Vela Bug Reporting -If the Vela community encounters a bug, then we highly encourage the community -to report it. When issuing a report, use the -[Vela issue tracker](https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela/-/issues) -that is part of the Vela project on Arm's GitLab Instance. This site enables any -Vela user to discuss and share any Vela related issue with the community. +If you believe you have identified an issue or bug then we encourage you to +create an issue on the project's [GitLab issue tracker](https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela/-/issues). +However, if it is a security related issue or vulnerability then please use +the process described in [Security Vulnerabilities](SECURITY.md) instead. -The only prerequisite for this process is a GitHub account, which will be used -when creating an account on Arm's GitLab Instance. +The GitLab issue tracker can also be used to ask questions and engage in +discussions with the Vela community. -If you also have a fix for the bug you have identified, then we encourage -you to follow the [contributions guide](CONTRIBUTING.md). - -The bug reporting process in its entirety is detailed below. +The bug reporting process is detailed below. If you also plan to contribute a +fix for a bug then we encourage you to follow the [contributions guide](CONTRIBUTING.md). ## Vela issue tracker diff --git a/BUILD.md b/BUILD.md new file mode 100644 index 00000000..25a5b17c --- /dev/null +++ b/BUILD.md @@ -0,0 +1,153 @@ + +# Vela Building From Source + +This document details how to build Vela manually from its source code. It also +contains some known issues that can occur when the compiler is built. + +### Arm's GitLab Instance + +First obtain the source code by either downloading the desired TGZ file from: + + +Or by cloning the git repository: + +```bash +git clone https://git.gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela.git +``` + +Once you have the source code, Vela can be installed using the following +command from the root directory of the repository: + +```bash +pip3 install . +``` + +#### Advanced Installation for Developers + +If you plan to modify the Vela codebase then it is recommended to install Vela +as an editable package to avoid the need to re-install after every modification. +This is done by adding the `-e` option to the install command like so: + +```bash +pip3 install -e .[dev] +``` + +If you plan to contribute to the Vela project (highly encouraged!) then it is +recommended to install Vela with the development dependencies (see +[Vela Testing](TESTING.md) for more details). + +##### Build options for C++ files (ethosu/regor) ##### + +The C++ part of the the Vela compiler can be configured through the following +environment variables: + +| Variable | Description | +| :------------------------- | :------------------------------------------------------ | +| CMAKE_BUILD_TYPE | Control cmake-build-type (Release or Debug) | +| CMAKE_BUILD_PARALLEL_LEVEL | Control parallel build level | +| CMAKE_GENERATOR | Override the default CMAKE generator | +| CMAKE_ARGS | Provide additional build-time options (see table below) | + +The following build-time options can be provided through CMAKE_ARGS +```bash +# Example (Linux Bash) +CMAKE_ARGS="-DREGOR_ENABLE_LTO=OFF -DREGOR_ENABLE_WERROR=ON" pip3 install -e ".[dev]" +``` +| Option | Description | Arguments | +| :--------------------------------- | :---------------------------------------- | :---------------------------------- | +| REGOR_ENABLE_LTO | Enable Link Time Optimization | ON/OFF | +| REGOR_ENABLE_LDGOLD | Enable Gold linker if available | ON/OFF | +| REGOR_ENABLE_CCACHE | Enable ccache if available | ON/OFF | +| REGOR_ENABLE_WERROR | Enable warnings as errors | ON/OFF | +| REGOR_ENABLE_STD_STATIC | Link libstdc and libgcc statically | ON/OFF | +| REGOR_ENABLE_COVERAGE | Enable Coverage build | ON/OFF | +| REGOR_ENABLE_PROFILING | Enable timer based runtime profiling | ON/OFF | +| REGOR_ENABLE_ASSERT | Enable asserts | ON/OFF | +| REGOR_ENABLE_EXPENSIVE_CHECKS | Enable expensive STL GLICXX asserts | ON/OFF | +| REGOR_ENABLE_RTTI | Enable RTTI (run-time type information) | ON/OFF | +| REGOR_ENABLE_VALGRIND | Enable Valgrind during check target | ON/OFF | +| REGOR_ENABLE_TESTING | Enable unit testing | ON/OFF | +| REGOR_ENABLE_CPPCHECK | Enable CPPCHECK | ON/OFF | +| REGOR_SANITIZE | Sanitizer setting (forwards to fsanitize) | String | +| REGOR_LOG_TRACE_MASK | Log trace enable mask | int (0->7) (See common/logging.hpp) | +| REGOR_PACKAGE_NAME | CPack package name | String | +| REGOR_DEBUG_COMPRESSION | Debug symbol compression | none, zlib, zlib-gnu | +| REGOR_PYTHON_BINDINGS_DESTINATION | Python bindings install destination | String | +| REGOR_PYEXT_VERSION | Python extension version | String | + + +## Known Issues + +### 1. NumPy C API version change + +Once ethos-u-vela is installed, the user might want to install a different NumPy +version that is still within the dependency constraints defined in pyproject.toml. + +In some scenarios, doing so might prevent ethos-u-vela from functioning as +expected due to incompatibilities between the installed NumPy C headers used in +the mlw_codec and the current version of NumPy. + +**Example scenario:** + +In the ethos-u-vela source directory, run: + +```bash +virtualenv -p 3.10 venv +. venv/bin/activate +pip install ethos-u-vela +``` + +Next, install a different NumPy version (e.g. 1.21.3) + +```bash +pip install numpy==1.21.3 --force +``` + +Finally, run ethos-u-vela. You might get an error similar to this: + +``` +ImportError: NumPy C API version mismatch +(Build-time version: 0x10, Run-time version: 0xe) +This is a known issue most likely caused by a change in the API version in +NumPy after installing ethos-u-vela. +``` + +#### Solution + +In order for ethos-u-vela to work with an older version of NumPy that uses +different C APIs, you will need to install the desired NumPy version first, and +then build ethos-u-vela with that specific NumPy version: + +1) Uninstall ethos-u-vela and install the desired version of NumPy + ``` + pip uninstall ethos-u-vela + pip install numpy==1.21.3 --force + ``` + +2) Install required build dependencies + ``` + pip install "setuptools_scm[toml]<6" wheel + ``` + +3) Install ethos-u-vela without build isolation. Not using build isolation + ensures that the correct version of NumPy is used when copying the C headers + in mlw_codec during the build process. + ``` + pip install ethos-u-vela --no-build-isolation --no-cache-dir + ``` \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d78c5228..4867f5f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,39 +21,48 @@ Contributions to Vela are very much welcomed! ## Coding Standard -Vela is written using Python 3.7 language constructs in order to aid -compatibility with other tools. All code must also be run through the -formatting and linting tools described in [Vela Testing](TESTING.md) +All code must run through the developer pre-commit checks described in [Testing](TESTING.md) ## Submitting -In order to submit a contribution to the -[Vela GitLab project](https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela), -please read and follow the -[Arm GitLab contributions guide](https://gitlab.arm.com/documentation/contributions) -on how to create a fork of the project. -To do this you will need to sign-in to the platform using a GitHub account and -add your SSH key under your settings. If there is a problem adding the SSH key -make sure there is a valid email address in the Email Addresses field. -In the commit message please include a Signed-off-by (described below). -When the merge request is created, make sure the target project is -`artificial-intelligence/ethos-u/ethos-u-vela` and the target branch is `main`. - - -## Contribution Guidelines +### One-time Registration +In order to submit a contribution to the project you must first complete the +following one-time process: +* Sign in + * Requires a GitHub ID +* Add SSH key + * This is added under the user settings page + * If there is a problem then make sure there is a valid email address in the + Email Addresses field +* Request permission to create project forks + * See [Requesting Access](https://gitlab.arm.com/documentation/contributions#requesting-access) +* Create a fork of the [Vela project](https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela) + * Click the fork button in the top-right corner on the project's main page + +### Process + +Contributions are made by creating a merge request from your fork to the branch + `main` of the target project `artificial-intelligence/ethos-u/ethos-u-vela`. + Every commit in the merge request must include a Signed-off-by in the commit + message to indicate that you agree to the guidelines below. + +## Guidelines Contributions are only accepted under the following conditions: +* All code passes the developer pre-commit checks +* All code passes a review process conducted on the platform * You certify that the origin of the submission conforms to the [Developer Certificate of Origin (DCO) V1.1](https://developercertificate.org/) -* You give permission according to the [Apache License 2.0](LICENSE.txt). +* You give permission according to the [Apache License 2.0](LICENSE.txt) -To indicate that you agree to these contribution guidelines you must add an -appropriate 'Signed-off-by: Real Name \' line with your -real name and e-mail address to every commit message. This can be done -automatically by adding the `-s` option to your `git commit` command. +To indicate that you agree to the above you need to add a Signed-off-by to your +commit using your real name and e-mail address. +e.g. 'Signed-off-by: Real Name \' to every commit +message. This can be done automatically by adding the `-s` option to your +`git commit` command. -No contributions will be accepted from pseudonyms or anonymous sources. +Contributions are not accepted from pseudonyms or anonymous sources. ## Code Reviews @@ -71,11 +80,5 @@ for more details). ## Bug Resolution In the case that your submission aims to resolve a bug, please follow the -[Vela Community Bug Reporting Process](BUGS.md). The document guides -you through the process of adding a bug ticket to the bug tracker for the -[Vela project](https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela/-/issues). - -The ticket will be visible to the public and will thus help the Vela community -track and find solutions to any bugs that are found during the use of Vela. - -Please include a link to your patch in the issue description. \ No newline at end of file +[Bug Reporting Process](BUGS.md) and include a link to your patch in the issue +description. \ No newline at end of file diff --git a/OPTIONS.md b/OPTIONS.md index d71348ad..642a1836 100644 --- a/OPTIONS.md +++ b/OPTIONS.md @@ -15,17 +15,17 @@ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -# Vela Options +# Vela CLI Options -This file contains a more verbose and detailed description of the Vela -Compiler's CLI options than the built-in help strings. It also defines and -describes Vela's configuration file format. +This file contains a detailed description of the CLI options and the +configuration file format. ## Command Line Interface ### Network (required) -Filename of the network model to compile. The file has to be a `.tflite` file. +Filename of the network model to compile. The file has to be a `.tosa` or +`.tflite` file. **Type: POSIX path** **Default: N/A** @@ -35,8 +35,8 @@ vela path/to/network.tflite ### Help -Displays the help strings of all CLI options. Can be used without the required -Network argument. +Displays the help strings of all CLI options. +This option can be used without the required Network argument. **Type: N/A** **Default: N/A** @@ -46,7 +46,8 @@ vela --help ### Version -Displays the version of the installed Vela Compiler. Can be used without the +Displays the version of the installed Vela Compiler. +This option can be used without the required Network argument. required Network argument. **Type: N/A** **Default: N/A** @@ -57,8 +58,8 @@ vela --version ### API version -Displays the version of the external API. Can be used without the -required Network argument. +Displays the version of the external API. +This option can be used without the required Network argument. **Type: N/A** **Default: N/A** @@ -69,13 +70,11 @@ vela --api-version ### Supported Operator Report Generate the SUPPORTED_OPS.md file in the current working directory. Contains -a summary table for each supported network model format (TFLite/TOSA). The -tables shows all the operators that can be placed on the NPU, and what the -constraints are for that operator to be scheduled on the NPU. If the constraints -are not met for a TFLite operator, then it will be scheduled on the CPU instead. -For TOSA operators there are no fallback to the CPU. Note: There is limited -support for compiling a TOSA neural network (EXPERIMENTAL). Can be used without -the required Network argument. +a summary for TFLite operators which shows all the operators that can be placed +on the NPU, and what the constraints are for that operator to be scheduled on +the NPU. If the constraints are not met, then it will be scheduled on the CPU +instead. For TOSA operators fallback to the CPU is not allowed. +This option can be used without the required Network argument. **Type: N/A** **Default: N/A** @@ -89,8 +88,8 @@ Displays the configuration files in the `ethosu/config_files` directory. All configuration files must have the .ini extension and be placed in an appropriately named directory under `ethosu/config_files`. Note that the file depth from `ethosu/config_files` must be exactly 2 for the file to be -discovered (e.g. `config_files/directory_name/my_config_file.ini`). Can be -used without the required Network argument. +discovered (e.g. `config_files/directory_name/my_config_file.ini`). +This option can be used without the required Network argument. ```bash vela --list-config-files diff --git a/PERFORMANCE.md b/PERFORMANCE.md index 72274693..29d10625 100644 --- a/PERFORMANCE.md +++ b/PERFORMANCE.md @@ -15,10 +15,19 @@ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -# Vela Performance Estimation Summary - -This is a description of the performance estimation summary that Vela prints -after each compilation. This summary is also printed to a csv in the output +# Vela Performance Estimations + +A performance estimation summary is reported after compilation. The cycle and +bandwidth numbers should not be taken as accurate representations of real +performance numbers and they should not be compared against those from other +compilations that use different settings or configurations. The numbers +reported allow the compiler to make its optimisation decisions only. For +accurate performance numbers the network should be run and profiled on an FPGA. +For approximate performance numbers the network can be run on a Fixed +Virtual Platform (FVP) Model. + +The following is a description of the fields reported in the performance +estimation summary. This summary is also printed to a csv in the output directory. The following is an example of the output. diff --git a/README.md b/README.md index 46d2d14c..7d17b963 100644 --- a/README.md +++ b/README.md @@ -17,382 +17,154 @@ limitations under the License. --> # Vela -This tool is used to compile a -[TensorFlow Lite for Microcontrollers](https://www.tensorflow.org/lite/microcontrollers) -neural network model into an optimised version that can run on an embedded -system containing an -[Arm Ethos-U NPU](https://www.arm.com/products/silicon-ip-cpu). - -In order to be accelerated by the Ethos-U NPU the network operators must be -quantised to either 8-bit (unsigned or signed) or 16-bit (signed). - -The optimised model will contain TensorFlow Lite Custom operators for those -parts of the model that can be accelerated by the Ethos-U NPU. Parts of the -model that cannot be accelerated are left unchanged and will instead run on the -Cortex-M series CPU using an appropriate kernel (such as the -[Arm](https://www.arm.com) optimised -[CMSIS-NN](https://github.com/ARM-software/CMSIS_5/tree/develop/CMSIS/NN) -kernels). - -After compilation the optimised model can only be run on an Ethos-U NPU -embedded system. - -The tool will also generate performance estimates (EXPERIMENTAL) for the -compiled model. - -The tool has limited functionality for compiling a -[TOSA](https://git.mlplatform.org/tosa/specification.git/) neural network -(EXPERIMENTAL). - -## TensorFlow Support - -Vela is tested by comparing the bit exact numerical behaviour of the Ethos-U -optimised operators against that of the corresponding TensorFlow Lite reference -kernels (or TensorFlow Lite for Microcontrollers reference kernels in the case -of the UNIDIRECTIONAL_SEQUENCE_LSTM operator). The following list indicates -which version is used for comparison: - -* Vela 4.0.0 to current supports TensorFlow 2.17 -* Vela 3.12.0 supports TensorFlow 2.16 -* Vela 3.11.0 supports TensorFlow 2.15 -* Vela 3.10.0 supports TensorFlow 2.14 -* Vela 3.9.0 supports TensorFlow 2.12 -* Vela 3.8.0 supports TensorFlow 2.11 -* Vela 3.6.0 to 3.7.0 supports TensorFlow 2.10 -* Vela 3.5.0 supports TensorFlow 2.9 -* Vela 3.4.0 supports TensorFlow 2.8 -* Vela 3.3.0 supports TensorFlow 2.7 -* Vela 3.1.0 to 3.2.0 supports TensorFlow 2.5 -* Vela 2.1.0 to 3.0.0 supports TensorFlow 2.4 -* Vela 2.0.0 to 2.0.1 supports TensorFlow 2.3 -* Vela 0.1.0 to 1.2.0 supports TensorFlow 2.1 - -## Python Version Support - -The majority of Vela's testing is done using a single version of Python, as -indicated by the first version in the list below. However, some additional -testing is also performed across a range of newer versions starting at the -minimum version (pyproject.toml:project.requires-python) indicated in the -brackets: - -* Vela 3.10.0 to current supports Python 3.10 (3.9) -* Vela 3.9.0 supports Python 3.10 (3.8) -* Vela 3.8.0 supports Python 3.9 (3.8) -* Vela 3.4.0 to 3.7.0 supports Python 3.7 (3.8) -* Vela 3.3.0 supports Python 3.8 (3.7) -* Vela 0.1.0 to 3.2.0 supports Python 3.6 (3.7) - -## Environment - -Vela runs on Linux and Microsoft Windows 10 operating systems. - -## Prerequisites - -The following should be installed prior to the installation of Vela: - -* Python 3.10 or compatible - - Development version containing the Python/C API header files - - e.g. `apt install python3.10-dev` or `yum install python310-devel` -* Pip3 -* If building from source then C99 and C++17 capable compilers and associated -toolchains are also required - - For Linux operating systems, a GNU toolchain is recommended. - - For Microsoft Windows 10, the Microsoft Visual C++ 14.2 Build Tools are recommended. - See +Vela is a compiler for [Arm Ethos-U NPU](https://www.arm.com/products/silicon-ip-cpu) +edge AI devices. It is used to convert neural networks defined in [TOSA](https://www.mlplatform.org/tosa/) +or [TensorFlow Lite/LiteRT](https://ai.google.dev/edge/litert) format into +Ethos-U command streams. -## Installation +The input network must have quantised activations and quantised weights in order +to be accelerated by the Ethos-U. + +More details about Ethos-U devices: +* Ethos-U55, [product](https://www.arm.com/products/silicon-ip-cpu/ethos/ethos-u55) and [developer](https://developer.arm.com/Processors/Ethos-U55) information +* Ethos-U65, [product](https://www.arm.com/products/silicon-ip-cpu/ethos/ethos-u65) and [developer](https://developer.arm.com/Processors/Ethos-U65) information +* Ethos-U85, [product](https://www.arm.com/products/silicon-ip-cpu/ethos/ethos-u85) and [developer](https://developer.arm.com/Processors/Ethos-U85) information -Vela is available to install as binary wheels or a source distribution from -[PyPi](https://pypi.org/project/ethos-u-vela/), or as -source code from -[Arm's GitLab Instance](https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela). -Both methods will automatically install all the required dependencies. +## Installation -### PyPi +Vela runs on: +* Apple macOS +* Linux +* Microsoft Windows -Install Vela from PyPi using the following command: +The simplest way to obtain Vela is to install it directly from [PyPi](https://pypi.org/project/ethos-u-vela/). +The following will install the latest released version: ```bash pip3 install ethos-u-vela ``` -### Arm's GitLab Instance - -First obtain the source code by either downloading the desired TGZ file from: - - -Or by cloning the git repository: +### Requirements -```bash -git clone https://git.gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela.git -``` +If your system does not match one of the pre-built binary wheels on PyPi then +the installation process will attempt to build Vela from its source code. To do +this it requires the following: +* Development version of Python 3 containing the Python/C API header files + - e.g. `apt install python3.10-dev` +* C99 and C++17 capable compiler and toolchain + - For Apple macOS; XCode or Command Line Tools are recommended, + see or `xcode-select --install` + - For Linux operating systems; a GNU or Clang toolchain is recommended, + e.g. `apt install build-essential` + - For Microsoft Windows 10; the Microsoft Visual C++ 14.2 Build Tools are recommended, + see +* CMake + - See -Once you have the source code, Vela can be installed using the following -command from the root directory of the repository: +### Manually Building -```bash -pip3 install . -``` +The source code is available from [Arm's GitLab Instance](https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela), +see [Building From Source](BUILD.md) for more details. -#### Advanced Installation for Developers +## Getting Started -If you plan to modify the Vela codebase then it is recommended to install Vela -as an editable package to avoid the need to re-install after every modification. -This is done by adding the `-e` option to the install command like so: +Once installed, Vela can be run from the command line by specifying just two +settings: +* Target Ethos-U device; `--accelerator-config` +* Neural network file; `.tflite` or `.tosa` ```bash -pip3 install -e .[dev] +vela --accelerator-config ethos-u55-128 my_network.tflite ``` -If you plan to contribute to the Vela project (highly encouraged!) then it is -recommended to install Vela with the development dependencies (see -[Vela Testing](TESTING.md) for more details). +However, it is highly recommended that you specify some additional settings that +describe the Ethos-U system and what optimisation strategy the compiler should +use. The three additional settings are: +* System configuration; `--system-config` +* Memory mode; `--memory-mode` +* Optimisation strategy; `--optimise` -##### Build options for C++ files (ethosu/regor) ##### +The system configuration and memory mode refer to definitions in a configuration +file. These are specific to the target system. However an example +configuration (`Arm/vela.ini`) containing some generic reference systems is +provided as part of the installation. -The C++ part of the the Vela compiler can be configured through the following environment variables: +The optimisation strategy indicates whether the compiler should minimise +inference time or runtime memory usage. -| Variable | Description | -| :------------------------- | :------------------------------------------------------ | -| CMAKE_BUILD_TYPE | Control cmake-build-type (Release or Debug) | -| CMAKE_BUILD_PARALLEL_LEVEL | Control parallel build level | -| CMAKE_GENERATOR | Override the default CMAKE generator | -| CMAKE_ARGS | Provide additional build-time options (see table below) | +See [CLI Options](OPTIONS.md) for more information. -The following build-time options can be provided through CMAKE_ARGS +Example of how to set the system configuration and memory mode: ```bash -# Example (Linux Bash) -CMAKE_ARGS="-DREGOR_ENABLE_LTO=OFF -DREGOR_ENABLE_WERROR=ON" pip3 install -e .[dev] +vela --config Arm/vela.ini --system-config Ethos_U85_SYS_DRAM_High --memory-mode Dedicated_Sram_384KB --accelerator-config ethos-u85-256 my_network.tosa ``` -| Option | Description | Arguments | -| :--------------------------------- | :---------------------------------------- | :---------------------------------- | -| REGOR_ENABLE_LTO | Enable Link Time Optimization | ON/OFF | -| REGOR_ENABLE_LDGOLD | Enable Gold linker if available | ON/OFF | -| REGOR_ENABLE_CCACHE | Enable ccache if available | ON/OFF | -| REGOR_ENABLE_WERROR | Enable warnings as errors | ON/OFF | -| REGOR_ENABLE_STD_STATIC | Link libstdc and libgcc statically | ON/OFF | -| REGOR_ENABLE_COVERAGE | Enable Coverage build | ON/OFF | -| REGOR_ENABLE_PROFILING | Enable timer based runtime profiling | ON/OFF | -| REGOR_ENABLE_ASSERT | Enable asserts | ON/OFF | -| REGOR_ENABLE_EXPENSIVE_CHECKS | Enable expensive STL GLICXX asserts | ON/OFF | -| REGOR_ENABLE_RTTI | Enable RTTI (run-time type information) | ON/OFF | -| REGOR_ENABLE_VALGRIND | Enable Valgrind during check target | ON/OFF | -| REGOR_ENABLE_TESTING | Enable unit testing | ON/OFF | -| REGOR_ENABLE_CPPCHECK | Enable CPPCHECK | ON/OFF | -| REGOR_SANITIZE | Sanitizer setting (forwards to fsanitize) | String | -| REGOR_LOG_TRACE_MASK | Log trace enable mask | int (0->7) (See common/logging.hpp) | -| REGOR_PACKAGE_NAME | CPack package name | String | -| REGOR_DEBUG_COMPRESSION | Debug symbol compression | none, zlib, zlib-gnu | -| REGOR_PYTHON_BINDINGS_DESTINATION | Python bindings install destination | String | -| REGOR_PYEXT_VERSION | Python extension version | String | - -## Running - -Vela is run with an input `.tflite` or `.tosa` (EXPERIMENTAL) file passed on the -command line. This file contains the neural network to be compiled. The tool then -outputs an optimised `.tflite` file with a `_vela` suffix in the file name, along -with performance estimate (EXPERIMENTAL) CSV files, all to the output directory. -It also prints a performance estimation summary back to the console, see -[Vela Performance Estimation Summary](PERFORMANCE.md). - -Example usage: - -1) Compile the network `my_model.tflite`. The optimised version will be output -to `./output/my_network_vela.tflite`. +Example of how to set the optimisation strategy: ```bash -vela my_model.tflite +vela --optimise Size --accelerator-config ethos-u55-64 my_network.tflite ``` -2) Compile the network `/path/to/my_model.tflite` and specify the output to go -in the directory `./results_dir/`. - -```bash -vela --output-dir ./results_dir /path/to/my_model.tflite -``` - -3) Compile a network targeting a particular Ethos-U NPU. The following command -selects an Ethos-U65 NPU accelerator configured with 512 MAC units. - -```bash -vela --accelerator-config ethos-u65-512 my_model.tflite -``` -4) Compile a network while minimizing peak SRAM usage, prioritising lower SRAM -usage over runtime performance. - -```bash -vela --optimise Size my_model.tflite -``` - -5) Compile a network to have maximum performance, i.e. the fastest inference time. -This prioritises a higher runtime performance over a lower peak SRAM usage. - -```bash -vela --optimise Performance my_model.tflite -``` - -6) Compile a network while optimising for the fastest inference time possible, -with an upper bound for the SRAM usage. The memory limit is set in bytes, i.e. -run the following example if one requires a limit of 300KB. - -```bash -vela --optimise Performance --arena-cache-size 300000 my_model.tflite -``` - -7) Compile a network using a particular embedded system configuration defined in -Vela's configuration file. The following command selects the `My_Sys_Config` -system configuration along with the `My_Mem_Mode` memory mode from the `vela.ini` -configuration file located in the config_files directory. - -```bash -vela --config Arm/vela.ini --system-config My_Sys_Config --memory-mode My_Mem_Mode my_model.tflite -``` - -8) To get a list of all available configuration files in the config_files directory: - +Command to list all known configuration files: ```bash vela --list-config-files ``` -9) To get a list of all available options (see CLI Options section below): - -```bash -vela --help -``` - -## Warnings - -When running the Vela compiler it may report a number of warning messages to the -console. These should all be thoroughly reviewed as they will indicate decisions -that the compiler has made in order to create the optimised network. - -## Example Networks - -Some example networks that contain quantised operators which can be compiled by -Vela to run on the Ethos-U NPU can be found at: - - -## Known Issues - -### 1. NumPy C API version change - -Once ethos-u-vela is installed, the user might want to install a different NumPy -version that is still within the dependency constraints defined in pyproject.toml. - -In some scenarios, doing so might prevent ethos-u-vela from functioning as -expected due to incompatibilities between the installed NumPy C headers used in -the mlw_codec and the current version of NumPy. - -**Example scenario:** - -In the ethos-u-vela source directory, run: - +Command to list all configurations in a configuration file: ```bash -virtualenv -p 3.10 venv -. venv/bin/activate -pip install ethos-u-vela +vela --list-config Arm/vela.ini ``` -Next, install a different NumPy version (e.g. 1.21.3) - -```bash -pip install numpy==1.21.3 --force -``` - -Finally, run ethos-u-vela. You might get an error similar to this: - -``` -ImportError: NumPy C API version mismatch -(Build-time version: 0x10, Run-time version: 0xe) -This is a known issue most likely caused by a change in the API version in -NumPy after installing ethos-u-vela. -``` - -#### Solution - -In order for ethos-u-vela to work with an older version of NumPy that uses -different C APIs, you will need to install the desired NumPy version first, and -then build ethos-u-vela with that specific NumPy version: +### Output -1) Uninstall ethos-u-vela and install the desired version of NumPy - ``` - pip uninstall ethos-u-vela - pip install numpy==1.21.3 --force - ``` +The result of the compilation is an optimised network in either TFLite or Raw +format depending upon the input network. This can be overridden using the +`--output-format option`. -2) Install required build dependencies - ``` - pip install "setuptools_scm[toml]<6" wheel - ``` +TFLite output contains TensorFlow Lite Custom operators for those parts of +the network that can be accelerated by the Ethos-U NPU. Parts of the network +that cannot be accelerated are left unchanged. -3) Install ethos-u-vela without build isolation. Not using build isolation - ensures that the correct version of NumPy is used when copying the C headers - in mlw_codec during the build process. - ``` - pip install ethos-u-vela --no-build-isolation --no-cache-dir - ``` +Raw output contains the command stream and weight data required to run Ethos-U +parts of the optimised network. This is stored in [`.npz`](https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html#module-numpy.lib.format) +format. See the Output format [CLI Option](OPTIONS.md) for more information. +This does not contain CPU parts of the network and so cannot be used for +converting TFLite to TOSA. -## APIs +#### Warnings -Please see [Vela External APIs](API.md). +It is important to thoroughly review and understand all warning messages +generated by the compiler as they could indicate that not all operators were +optimally converted to run on the Ethos-U. -## Bug Reporting +#### Performance information -Please see [Vela Community Bug Reporting](BUGS.md) for a description of how to -report bugs. +A performance estimation summary is reported after compilation. The cycle and +bandwidth numbers should not be taken as accurate representations of real +performance numbers and they should not be compared against those from other +compilations that use different settings or configurations. The numbers +reported allow the compiler to make its optimisation decisions only. For +accurate performance numbers the network should be run and profiled on an FPGA. +For approximate performance numbers the network can be run on a Fixed +Virtual Platform (FVP) Model. -## Contributions - -Please see [Vela Contributions](CONTRIBUTING.md). - -## Debug Database - -Please see [Vela Debug Database](DEBUG_DB.md). +## Additional Vela Information +* [External APIs](API.md) +* [Bug Reporting](BUGS.md) +* [Building From Source](BUILD.md) +* [Contributing](CONTRIBUTING.md) +* [CLI Options](OPTIONS.md) +* [Debug Database](DEBUG_DB.md) +* [Performance Estimations](PERFORMANCE.md) +* [Operator Support](SUPPORTED_OPS.md) +* [Releases](RELEASES.md) +* [Security Vulnerabilities](SECURITY.md) +* [Testing](TESTING.md) ## Inclusive language commitment -This product conforms to Arm’s inclusive language policy and, to the best of -our knowledge, does not contain any non-inclusive language. If you find -something that concerns you, email terms@arm.com. - -## Options - -Please see [Vela CLI Options](OPTIONS.md). This includes a description of the -system configuration file format. - -## Performance - -Please see [Vela Performance Estimation Summary](PERFORMANCE.md). -__NOTE: This is only an estimate. For performance numbers we recommend running -the compiled network on an FVP Model or FPGA.__ - -## Releases - -Please see [Vela Releases](RELEASES.md). - -## Resources - -Additional useful information: - -* [Arm Products: Ethos-U55 NPU](https://www.arm.com/products/silicon-ip-cpu/ethos/ethos-u55) -* [Arm Products: Ethos-U65 NPU](https://www.arm.com/products/silicon-ip-cpu/ethos/ethos-u65) -* [Arm Products: Ethos-U85 NPU](https://www.arm.com/products/silicon-ip-cpu/ethos/ethos-u85) -* [Arm Developer: Ethos-U55 NPU](https://developer.arm.com/Processors/Ethos-U55) -* [Arm Developer: Ethos-U65 NPU](https://developer.arm.com/Processors/Ethos-U65) -* [Arm Developer: Ethos-U85 NPU](https://developer.arm.com/Processors/Ethos-U85) - -## Security - -Please see [Vela Security](SECURITY.md). - -## Supported Operators - -Please see [Vela Supported Operators](SUPPORTED_OPS.md) for the list of -operators supported in this release. - -## Testing - -Please see [Vela Testing](TESTING.md). +This product conforms to Arm’s inclusive language policy and, to the best of our +knowledge, does not contain any non-inclusive language. If you find something +that concerns you then email [terms@arm.com](mailto:terms@arm.com). ## License diff --git a/SECURITY.md b/SECURITY.md index 450546fd..07242523 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,10 +15,11 @@ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -# Vela Security +# Vela Security Vulnerabilities If you believe you have identified a security related issue or vulnerability, then we encourage you to responsibly disclose it to us as soon as possible. +For non-security related issues please see [Bug Reporting](BUGS.md). ## Reporting vulnerabilities @@ -34,16 +35,14 @@ disclosure when notifications and patches/mitigations are available. ### Report -For all security issues, contact Arm by email at -[arm-security@arm.com](mailto:arm-security@arm.com). In the body of the email -include as much information as possible about the issue or vulnerability and any -additional contact details. +For all security issues, contact Arm by email at [arm-security@arm.com](mailto:arm-security@arm.com). +In the body of the email include as much information as possible about the issue +or vulnerability and any additional contact details. ### Secure submission using PGP We support and encourage secure submission of vulnerability reports using PGP -with the key found at -[Report Security Vulnerability](https://developer.arm.com/support/arm-security-updates/report-security-vulnerabilities). +with the key found at [Report Security Vulnerability](https://developer.arm.com/support/arm-security-updates/report-security-vulnerabilities). If you would like replies to be encrypted, please provide your own public key through a secure mechanism. diff --git a/TESTING.md b/TESTING.md index 0e296e39..016355a0 100644 --- a/TESTING.md +++ b/TESTING.md @@ -1,5 +1,5 @@ # Vela Testing -## Tools +Vela is tested in-house by comparing the bit exact numerical behaviour of the +optimised network against that of the corresponding behaviour of the reference +code. In addition, a developer [pre-commit](https://pre-commit.com/) performs +pytest unit tests, along with linting and formatting checks. However, the +Catch2 unit tests need to be run manually. + +## TOSA + +TOSA networks are compared against the [TOSA Reference Model](https://review.mlplatform.org/admin/repos/tosa/reference_model,general). + +## TensorFlow Lite/LiteRT + +TensorFlow Lite/LiteRT networks are compared against the TensorFlow Lite/LiteRT +reference kernels (except for the UNIDIRECTIONAL_SEQUENCE_LSTM operator which +is compared against the TensorFlow Lite/LiteRT for Microcontrollers reference +kernel). The following list indicates which TensorFlow version a particular +Vela version was tested against. It also indicates which version of the +TensorFlow Lite/LiteRT flatbuffer schema was supported. + +* Vela 4.0.0 to current supports TensorFlow 2.17 +* Vela 3.12.0 supports TensorFlow 2.16 +* Vela 3.11.0 supports TensorFlow 2.15 +* Vela 3.10.0 supports TensorFlow 2.14 +* Vela 3.9.0 supports TensorFlow 2.12 +* Vela 3.8.0 supports TensorFlow 2.11 +* Vela 3.6.0 to 3.7.0 supports TensorFlow 2.10 +* Vela 3.5.0 supports TensorFlow 2.9 +* Vela 3.4.0 supports TensorFlow 2.8 +* Vela 3.3.0 supports TensorFlow 2.7 +* Vela 3.1.0 to 3.2.0 supports TensorFlow 2.5 +* Vela 2.1.0 to 3.0.0 supports TensorFlow 2.4 +* Vela 2.0.0 to 2.0.1 supports TensorFlow 2.3 +* Vela 0.1.0 to 1.2.0 supports TensorFlow 2.1 + +## Python Version Support + +The majority of Vela's testing is done using a single version of Python, as +indicated by the non-bracketed version in the list below. However, some +additional testing is also performed across a range of other versions starting +from a minimum version, indicated in the brackets, and going to the latest +released version at the time of testing. + +* Vela 3.10.0 to current supports Python 3.10 (3.9) +* Vela 3.9.0 supports Python 3.10 (3.8) +* Vela 3.8.0 supports Python 3.9 (3.8) +* Vela 3.4.0 to 3.7.0 supports Python 3.7 (3.8) +* Vela 3.3.0 supports Python 3.8 (3.7) +* Vela 0.1.0 to 3.2.0 supports Python 3.6 (3.7) + +## Developer Pre-Commit + +### Installation + +To install the developer pre-commit: +``` bash +pip install -e ".[dev]" +pre-commit install +``` + +### Running + +After installation, all pre-commit stages described below will be automatically +run upon a `git commit` command. + +However, the various stages can also be run manually. + +To run all of the commit stages on all files: +```bash +$ pre-commit run --all-files +``` + +Example of how to run a specific check on a specific file: +```bash +$ pre-commit run pylint --files ethosu/vela/vela.py +``` + +### Linting/Formatting Vela's Python code is PEP8 compliant with the exception of a 120 character line length. The following code formatting and linting tools are run on all the @@ -35,81 +111,24 @@ auto-generated and third-party code see `.pre-commit-config.yaml` for details): * clang-format (code formatter) -All of the above tools can be installed and run using the -[pre-commit](https://pre-commit.com/) framework (see below). +### Unit Tests -In addition, there are both Python and C/C++ unit tests. These use the following -frameworks: +There are both Python and C/C++ unit tests. These use the following frameworks: * pytest (Python) * Catch2 (C/C++) -### Installation - -To install the development dependencies, first install Vela with the development -option: - -``` bash -pip install -e ".[dev]" -``` - -This will install the following tools: - -* pytest -* pytest-cov -* pre-commit -* build -* setuptools_scm - -Then, the remaining tools will be installed automatically upon the first use of -pre-commit. - -### Add pre-commit hook (Automatically running the tools) - -To support code development all the above tools can be configured to run -automatically upon any modified file. This happens when performing a -`git commit` command and is setup using: - -```bash -$ pre-commit install -pre-commit installed at .git/hooks/pre-commit -``` - -When committing a patch, if any of the tools result in a failure (meaning an -issue was found) then the issue will need to be resolved before re-issuing the -commit. - -### Manually running the tools - -All of the tools can be run individually by invoking them using the following -pre-commit framework commands: - -1) Run all of the commit stages on all files in the repository: - -```bash -$ pre-commit run --all-files -``` - -2) Run a specific check on a specific file -```bash -$ pre-commit run pylint --files ethosu/vela/vela.py -``` - -### Manually run the pytest unit tests - -To run all of the pytest unit tests use the following command: +To run all pytest unit tests: ```bash $ pytest ``` -2) Run a specific pytest unit test +Example of how to run a specific pytest unit test: ```bash -$ pytest pytest ethosu/vela/test/test_architecture_allocator.py +$ pytest ethosu/vela/test/test_architecture_allocator.py ``` -### Manually run the Catch2 unit tests - -To run all of the Catch2 unit tests use the following command: +To run all of the Catch2 unit tests: ```bash $ cmake -S ethosu/regor -B build-unit-tests -DCMAKE_BUILD_TYPE=Debug -DREGOR_SANITIZE=address $ cmake --build build-unit-tests -t check diff --git a/ethosu/vela/test/test_build.py b/ethosu/vela/test/test_build.py index f3a9977a..c8581da9 100644 --- a/ethosu/vela/test/test_build.py +++ b/ethosu/vela/test/test_build.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: Copyright 2023-2025 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 # @@ -23,6 +23,8 @@ from tempfile import TemporaryDirectory import build # noreorder import pytest +SAMPLE_VELA_VERSION = "1.2.3" # Vela version used in link substitution and comparisons + @pytest.fixture(scope="module", name="vela_path") def fixture_vela_path(): @@ -36,8 +38,7 @@ def fixture_vela_path(): @pytest.fixture(scope="module", name="built_sdist") def fixture_built_sdist(vela_path: Path) -> TarFile: """Fixture that returns the TarFile with the project sdist.""" - # Set pretend version - os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = "3.7.0" + os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = SAMPLE_VELA_VERSION # Build vela with TemporaryDirectory() as tmp_dir: p_builder = build.ProjectBuilder(vela_path) @@ -59,7 +60,7 @@ def fixture_source_readme(vela_path: Path) -> str: def test_build_correct_readme_links(built_sdist: TarFile, source_readme: str): """Test that PKG-INFO file contains README.md metadata with correct links.""" md_link_pattern = r"(!?\[.+?\]\((.+?)\))" - url = "https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela/-/blob/3.7.0/" + url = "https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela/-/blob/" + SAMPLE_VELA_VERSION + "/" # Extract the name of the package dist_tar_name = str(built_sdist.name).replace(".gz", "") package_name = Path(dist_tar_name).stem @@ -67,6 +68,6 @@ def test_build_correct_readme_links(built_sdist: TarFile, source_readme: str): pkg_info = file.read().decode("utf-8") # Ensure that all links were replaced correctly for match, link in re.findall(md_link_pattern, source_readme): - if "http" not in link and "https" not in link: + if "http" not in link and "https" not in link and "mailto:" not in link: assert match not in pkg_info assert url + link in pkg_info diff --git a/pyproject.toml b/pyproject.toml index d45bece5..52cb9717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,12 +24,12 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Operating System :: Microsoft :: Windows :: Windows 10", - "Programming Language :: C++", "Programming Language :: C", + "Programming Language :: C++", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development :: Compilers", ] -- GitLab From a2f5c6791eea9f3ddd819984337f460988e52178 Mon Sep 17 00:00:00 2001 From: Tim Hall Date: Tue, 25 Feb 2025 21:24:50 +0000 Subject: [PATCH 2/4] MLBEDSW-10489: Add CLI option to list a config file - Added a CLI option --list-configs to list the configurations defined in the specified config file Change-Id: I9a7ffa37d1367e568762aab119d47cf4a9448c40 Signed-off-by: Tim Hall --- OPTIONS.md | 12 ++++++++++++ ethosu/vela/vela.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/OPTIONS.md b/OPTIONS.md index 642a1836..c6a4e6d0 100644 --- a/OPTIONS.md +++ b/OPTIONS.md @@ -95,6 +95,18 @@ This option can be used without the required Network argument. vela --list-config-files ``` +### List Configurations + +Displays a list of configurations (System Configurations and Memory Modes) that +are defined in the specified Vela configuration file. Can be used without the +required Network argument. +**Type: String** +**Default: None** + +```bash +vela --list-configs Arm/vela.ini +``` + ### Output Directory Specifies the output directory of the optimised network model as well as the diff --git a/ethosu/vela/vela.py b/ethosu/vela/vela.py index 1482bada..b5d609a5 100755 --- a/ethosu/vela/vela.py +++ b/ethosu/vela/vela.py @@ -23,6 +23,7 @@ import mmap import os import sys import time +from configparser import ConfigParser from typing import List from typing import Optional @@ -879,6 +880,23 @@ def list_config_files(): print(config[path_length:]) +def list_configs(config_filename): + vela_config_filename = config_filename + if not os.path.isfile(vela_config_filename): + vela_config_filename = os.path.join(architecture_features.CONFIG_FILES_PATH, config_filename) + if not os.path.isfile(vela_config_filename): + assert False, f"Cannot find config file {config_filename}" + + if os.path.splitext(vela_config_filename)[1] != ".ini": + assert False, f"Specified file {config_filename} is not a Vela config file" + + print(f"Configurations defined in {vela_config_filename}:") + vela_config = ConfigParser() + vela_config.read(vela_config_filename) + for section in vela_config.sections(): + print(f" {section}") + + def main(argv: Optional[List[str]] = None) -> int: """Run the main entry point.""" if argv is None: @@ -903,6 +921,11 @@ def main(argv: Optional[List[str]] = None) -> int: "use the --config argument with one of the listed config files (For example: --config Arm/vela.ini )" ), ) + parser.add_argument( + "--list-configs", + type=str, + help=("Display all configurations defined in the specified config file"), + ) # set network nargs to be optional to allow the support-ops-report CLI option to be used standalone parser.add_argument( @@ -1076,6 +1099,10 @@ def main(argv: Optional[List[str]] = None) -> int: list_config_files() return 0 + if args.list_configs: + list_configs(args.list_configs) + return 0 + if args.network is None: parser.error("the following argument is required: NETWORK") @@ -1086,7 +1113,7 @@ def main(argv: Optional[List[str]] = None) -> int: # Make sure the correct separator is used depending on OS config = os.path.normpath(config) - if not config.endswith(".ini"): + if os.path.splitext(config)[1] != ".ini": raise InputFileError(config, "Configuration files must use the .ini extension") if ( -- GitLab From 232e1440cfa19ea54755560af09cf5188586fff6 Mon Sep 17 00:00:00 2001 From: Tim Hall Date: Tue, 25 Feb 2025 21:44:21 +0000 Subject: [PATCH 3/4] MLBEDSW-10490: Change Ethos-U55/65 to use Regor for TOSA - Change the default TOSA behaviour for Ethos-U55/65 to use the Regor compilation core instead of the Python one Change-Id: I325b2f827e84f239dfe6de662dce588fe552c869 Signed-off-by: Tim Hall --- ethosu/vela/vela.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ethosu/vela/vela.py b/ethosu/vela/vela.py index b5d609a5..4458da75 100755 --- a/ethosu/vela/vela.py +++ b/ethosu/vela/vela.py @@ -1173,9 +1173,10 @@ def main(argv: Optional[List[str]] = None) -> int: model_reader_options = model_reader.ModelReaderOptions() - # The default behaviour is to use Vela for Ethos-U55/U65 and Regor for Ethos-U85. However, developers can override - # this by using the --debug-force-regor option - if arch.is_ethos_u85_system or args.debug_force_regor: + # The default behaviour to compile TFLite networks on Ethos-U55/U65 is to use Vela's Python compiler core (no name). + # However, this can be overridden to use Vela's C++ compiler core (Regor) by using the --debug-force-regor option. + # All Ethos-U85 or all TOSA network compilations use Vela's C++ compiler core (Regor). + if arch.is_ethos_u85_system or args.network.lower().endswith(".tosa") or args.debug_force_regor: system_config = "[architecture]\n" system_config += f"macs={arch.num_macs_per_cycle}\n" system_config += f"cores={arch.ncores}\n" -- GitLab From ba9f4572aeafa464e31c20b9776b4d77af4c68e8 Mon Sep 17 00:00:00 2001 From: Tim Hall Date: Thu, 27 Feb 2025 15:38:34 +0000 Subject: [PATCH 4/4] MLBEDSW-10491: MLCE: Align raw output behaviour for Ethos-U55/65 - Fixed bug in selection of raw output - Aligned the raw output format Change-Id: Ia9de278afd4d972b04722abddca1a90fd0e9536b Signed-off-by: Tim Hall --- OPTIONS.md | 12 ++++++------ ethosu/vela/rawdata_writer.py | 13 ++++++++++--- ethosu/vela/vela.py | 30 +++++++++++++++++++++++------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/OPTIONS.md b/OPTIONS.md index c6a4e6d0..c4ebac53 100644 --- a/OPTIONS.md +++ b/OPTIONS.md @@ -128,13 +128,13 @@ action commands attached as well as the required tensor, buffer and operator lists. If any operators fall back to the CPU, they are included in this file as well. -The `raw` format is currently only available for Ethos-U85 and produces an -'.npz' file containing the same information as the `tflite` format, but more -easily accessible. When loaded with NumPy, the arrays can be accessed by name -with the keys listed below. +The `raw` format produces an [`.npz`](https://numpy.org/doc/stable/reference/generated/numpy.lib.format.html#module-numpy.lib.format) +file containing the same information as the `tflite` format, but more easily +accessible. When loaded with NumPy, the arrays can be accessed by name with the +keys listed below. **Raw format keys:** -- **cmd_data** - The command stream. -- **weight_data** - The model weights. +- **cmd_data** - The command stream data and metadata for the driver. +- **weight_data** - The encoded weight, scale & bias, and constant data. - **weight_region** - Memory region identifier for the weights. - **scratch_shape / scratch_fast_shape** - The shape of the scratch and fast scratch buffers. diff --git a/ethosu/vela/rawdata_writer.py b/ethosu/vela/rawdata_writer.py index 2bcc12e9..3ef0e000 100644 --- a/ethosu/vela/rawdata_writer.py +++ b/ethosu/vela/rawdata_writer.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2021, 2024 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: Copyright 2021, 2024-2025 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 # @@ -60,7 +60,7 @@ def write_rawdata_output(nng, arch, filename): ofm_offsets.append(ofm.address) ofm_elem_sizes.append(ofm.element_size()) - filename_sg = f"{filename}_sg{sg_idx}_vela.npz" + filename_sg = f"{filename}_vela_sg{sg_idx}.npz" np.savez( filename_sg, cmd_data=cmd_stream_tensor.values, @@ -68,8 +68,10 @@ def write_rawdata_output(nng, arch, filename): weight_region=weight_region, scratch_shape=scratch_tensor.shape, scratch_region=scratch_region, + scratch_size=int(scratch_tensor.shape[0]), scratch_fast_shape=scratch_fast_tensor.shape, scratch_fast_region=scratch_fast_region, + scratch_fast_size=int(scratch_fast_tensor.shape[0]), input_shape=ifm_shapes, input_elem_size=ifm_elem_sizes, input_region=ifm_regions, @@ -78,13 +80,18 @@ def write_rawdata_output(nng, arch, filename): output_elem_size=ofm_elem_sizes, output_region=ofm_regions, output_offset=ofm_offsets, + variable_shape=[], + variable_elem_size=[], + variable_region=[], + variable_offset=[], ) # Write out a CompiledRawModel to a numpy raw file. def write_rawdata_output_from_model(filename, model): + filename_sg = f"{filename}_vela.npz" np.savez( - filename, + filename_sg, cmd_data=model.command_stream, weight_data=model.read_only.data, weight_region=model.read_only.region, diff --git a/ethosu/vela/vela.py b/ethosu/vela/vela.py index 4458da75..4ef48ce4 100755 --- a/ethosu/vela/vela.py +++ b/ethosu/vela/vela.py @@ -60,7 +60,9 @@ TFLITE_MAGIC = 0x334C4654 TOSA_MAGIC = 0x41534F54 -def process(input_name, enable_debug_db, arch, model_reader_options, compiler_options, scheduler_options): +def process( + input_name, enable_debug_db, arch, model_reader_options, compiler_options, scheduler_options, output_format +): if compiler_options.timing: start = time.time() @@ -95,10 +97,12 @@ def process(input_name, enable_debug_db, arch, model_reader_options, compiler_op ) output_tfl_filename = output_basename + "_vela.tflite" - if input_name.endswith(".tflite"): + if output_format == "tflite": tflite_writer.write_tflite(nng, output_tfl_filename) - if input_name.endswith(".tosa"): + elif output_format == "raw": rawdata_writer.write_rawdata_output(nng, arch, output_basename) + else: + assert False, f"Unsupported output_format = {output_format}" if enable_debug_db: file_offsets = calculate_operator_file_offsets(output_tfl_filename) @@ -124,6 +128,7 @@ def process_regor( options, enable_debug_db, output_dir, + output_format, verbose_weights=False, verbose_cycle_estimate=False, show_cpu_operations=False, @@ -140,12 +145,16 @@ def process_regor( output_basename = os.path.join(output_dir, model_name) - if isinstance(compiled_model, regor.CompiledTFLiteModel): + if output_format == "tflite": + assert isinstance(compiled_model, regor.CompiledTFLiteModel) output_name = output_basename + "_vela.tflite" with open(output_name, "wb") as f: f.write(compiled_model.model) - elif isinstance(compiled_model, regor.CompiledRawModel): - rawdata_writer.write_rawdata_output_from_model(output_basename + "_vela.npz", compiled_model) + elif output_format == "raw": + assert isinstance(compiled_model, regor.CompiledRawModel) + rawdata_writer.write_rawdata_output_from_model(output_basename, compiled_model) + else: + assert False, f"Unsupported output_format = {output_format}" summary_csv_file = "{0}_summary_{1}.csv".format(output_basename, arch.system_config) @@ -1229,6 +1238,7 @@ def main(argv: Optional[List[str]] = None) -> int: options, args.enable_debug_db, args.output_dir, + args.output_format, args.verbose_weights, args.verbose_cycle_estimate, args.show_cpu_operations, @@ -1268,7 +1278,13 @@ def main(argv: Optional[List[str]] = None) -> int: try: nng = process( - args.network, args.enable_debug_db, arch, model_reader_options, compiler_options, scheduler_options + args.network, + args.enable_debug_db, + arch, + model_reader_options, + compiler_options, + scheduler_options, + args.output_format, ) except VelaError as e: -- GitLab