From 08cb0028c37264a47a2ed9d9b78c6f065732df3d Mon Sep 17 00:00:00 2001 From: Alexander Bengtsson Date: Wed, 29 Jan 2025 15:41:53 +0100 Subject: [PATCH] MLBEDSW-9645: TFLite supported operator checks - Add framework for TFLite supported operator checks. - Refactor ArchitectureConstraints to represent what the architecture can handle. Move compiler-related checks into TFLite supported operator framework. Change-Id: Ic50432d42c5fe327c00146ce11a1f0494c6b48d0 Signed-off-by: Alexander Bengtsson --- ethosu/regor/CMakeLists.txt | 3 + ethosu/regor/architecture/architecture.hpp | 4 + .../architecture/architecture_constraints.hpp | 100 +----- .../regor/architecture/ethosu55/ethos_u55.cpp | 7 + .../regor/architecture/ethosu55/ethos_u55.hpp | 3 +- .../ethosu55/ethos_u55_constraints.cpp | 50 +-- .../ethosu55/ethos_u55_constraints.hpp | 16 +- .../regor/architecture/ethosu65/ethos_u65.cpp | 9 +- .../regor/architecture/ethosu65/ethos_u65.hpp | 3 +- .../regor/architecture/ethosu85/ethos_u85.cpp | 8 + .../regor/architecture/ethosu85/ethos_u85.hpp | 3 +- .../ethosu85/ethos_u85_constraints.cpp | 162 +--------- .../ethosu85/ethos_u85_constraints.hpp | 23 +- ethosu/regor/common/data_type.cpp | 4 +- ethosu/regor/compiler/compiler.cpp | 35 ++- ethosu/regor/compiler/graph.hpp | 12 +- ethosu/regor/compiler/scheduler_packing.cpp | 1 - .../regor/compiler/tflite_graph_optimiser.cpp | 147 ++++----- .../regor/compiler/tflite_graph_optimiser.hpp | 2 + ethosu/regor/test/CMakeLists.txt | 1 + ethosu/regor/test/test_graphir_optimiser.cpp | 3 +- ethosu/regor/test/test_tflite_fb.cpp | 3 +- .../test/test_tflite_supported_operators.cpp | 272 ++++++++++++++++ ethosu/regor/test/util.cpp | 7 +- ethosu/regor/tflite/tflite_reader.cpp | 220 +------------ ethosu/regor/tflite/tflite_reader.hpp | 9 +- .../tflite/tflite_supported_operators.cpp | 180 +++++++++++ .../tflite/tflite_supported_operators.hpp | 53 ++++ .../tflite/tflite_supported_operators_u55.cpp | 154 +++++++++ .../tflite/tflite_supported_operators_u55.hpp | 46 +++ .../tflite/tflite_supported_operators_u85.cpp | 296 ++++++++++++++++++ .../tflite/tflite_supported_operators_u85.hpp | 45 +++ 32 files changed, 1253 insertions(+), 628 deletions(-) create mode 100644 ethosu/regor/test/test_tflite_supported_operators.cpp create mode 100644 ethosu/regor/tflite/tflite_supported_operators.cpp create mode 100644 ethosu/regor/tflite/tflite_supported_operators.hpp create mode 100644 ethosu/regor/tflite/tflite_supported_operators_u55.cpp create mode 100644 ethosu/regor/tflite/tflite_supported_operators_u55.hpp create mode 100644 ethosu/regor/tflite/tflite_supported_operators_u85.cpp create mode 100644 ethosu/regor/tflite/tflite_supported_operators_u85.hpp diff --git a/ethosu/regor/CMakeLists.txt b/ethosu/regor/CMakeLists.txt index 8b2c57c3..80aa074f 100644 --- a/ethosu/regor/CMakeLists.txt +++ b/ethosu/regor/CMakeLists.txt @@ -312,6 +312,9 @@ regor_lib( "tflite/tflite_writer.cpp" "tflite/tflite_mapping.cpp" "tflite/tflite_model_semantics.cpp" + "tflite/tflite_supported_operators.cpp" + "tflite/tflite_supported_operators_u85.cpp" + "tflite/tflite_supported_operators_u55.cpp" ) regor_lib( diff --git a/ethosu/regor/architecture/architecture.hpp b/ethosu/regor/architecture/architecture.hpp index 9da3e215..700f6840 100644 --- a/ethosu/regor/architecture/architecture.hpp +++ b/ethosu/regor/architecture/architecture.hpp @@ -34,9 +34,11 @@ #include "compiler/tensor_properties.hpp" #include "mlw_encode.hpp" +#include #include #include #include + namespace regor { @@ -357,6 +359,8 @@ public: virtual AxisMask CanSubdivide(OpType opType, TransposeType transpose, ReverseType reverse) = 0; virtual bool SupportsScalar(OpType opType, DataType dataType, TensorUsage usage) = 0; virtual Flags SupportedWeightFormat(OpType op) = 0; + // helper for arch-dependent callbacks outside of arch + virtual void Call(std::function callBack) = 0; MemArea ReadonlyMemory(); MemArea FeatureMapMemory(); diff --git a/ethosu/regor/architecture/architecture_constraints.hpp b/ethosu/regor/architecture/architecture_constraints.hpp index 7cc9c9f1..07db143f 100644 --- a/ethosu/regor/architecture/architecture_constraints.hpp +++ b/ethosu/regor/architecture/architecture_constraints.hpp @@ -45,50 +45,12 @@ struct ArchFM TensorFormat format = {}; }; -/// -/// Information for querying support for Resize -/// -struct ResizeSupportQuery -{ - ArchResizeMode mode; - GraphApi::FractionND scaleY; - GraphApi::FractionND scaleX; - int offsetY; - int offsetX; - Shape ifmShape; -}; - -const std::array elemWiseMainOps = {OpType::Minimum, OpType::Maximum, OpType::Add, OpType::Mul, OpType::Sub, - OpType::Abs, OpType::Exp, OpType::LeakyRelu, OpType::Rsqrt, OpType::SquaredDifference}; -/// -/// Information for querying whether an operation can be executed by the hardware -/// -struct ExecutionQuery -{ - OpType opType; - OpType targetType; - DataType ifmType; - DataType ifm2Type; - Shape ifmShape; - Shape ifm2Shape; - Shape ofmShape; - DataType ofmType; - TransposeType transposeType; - ReverseType reverseTypeMask; - ResizeSupportQuery resizeQuery; - bool quantScalingInvalidOrUnequal = false; -}; - struct ArchOperatorQuery { ArchFM ifm[2]; ArchFM ofm; ReverseType reverseMask = ReverseType::None; TransposeType transposeMask = TransposeType::None; - struct - { - ResizeSupportQuery resize; - } specific; ~ArchOperatorQuery(){}; }; @@ -145,67 +107,15 @@ class IArchitectureConstraints { public: virtual ~IArchitectureConstraints() = default; - - virtual bool SupportsReverse(OpType opType, ReverseType reverseTypeMask) = 0; + virtual bool SupportsFusedReverse(OpType opType, ReverseType reverseTypeMask) = 0; virtual bool SupportsFusedRescale(OpType opType, TensorUsage tensorUsage, DataType rescaleFromType, DataType rescaleToType, DataType opFromType, DataType opToType, const Quantization &quantization) = 0; - virtual bool SupportsRescale(DataType fromType, DataType toType) = 0; - virtual TransposeSupport SupportsTranspose(OpType opType, TransposeType transposeType) = 0; + virtual TransposeSupport SupportsFusedTranspose(OpType opType, TransposeType transposeType) = 0; virtual bool SupportsAccumulatorSaveRestore() = 0; - virtual bool SupportsLeakyRelu(bool quantized, DataType type) = 0; virtual bool SupportsNegativeStrides() = 0; - virtual bool SupportsNot() = 0; - virtual Flags OperatorQuery(OpType opType, const ArchOperatorQuery *query, ArchRequirements *req = nullptr) = 0; - - bool CanExecute(const ExecutionQuery &query) - { - bool valid = true; - if ( IsFloat(query.ifmType | query.ifm2Type | query.ofmType) ) - { - return false; - } - - switch ( query.opType ) - { - case OpType::MatMul: - valid = SupportsMatMul(query.opType); - break; - case OpType::ReverseV2: - valid = SupportsReverse(query.targetType, query.reverseTypeMask); - break; - case OpType::Gather: - valid = SupportsGather(query.opType); - break; - case OpType::Scatter: - valid = SupportsScatter(query.opType); - break; - case OpType::ArgMax: - valid = SupportsArgMax(query.opType); - break; - case OpType::Cast: - valid = SupportsCast(query.opType, query.ifmType, query.ofmType); - break; - case OpType::Resize: - valid = SupportsResize(query.resizeQuery); - break; - default: - break; - } - if ( std::find(elemWiseMainOps.begin(), elemWiseMainOps.end(), query.opType) != elemWiseMainOps.end() ) - { - valid = SupportsNonMatchingShapes(query.ifmShape, query.ifm2Shape, query.ofmShape); - } - return valid; - } - -protected: - virtual bool SupportsMatMul(OpType opType) = 0; - virtual bool SupportsGather(OpType opType) = 0; - virtual bool SupportsScatter(OpType opType) = 0; - virtual bool SupportsResize(const ResizeSupportQuery &query) = 0; - virtual bool SupportsArgMax(OpType opType) = 0; - virtual bool SupportsCast(OpType opType, DataType ifmType, DataType ofmType) = 0; - virtual bool SupportsNonMatchingShapes(const Shape &ifmShape, const Shape &ifm2Shape, const Shape &ofmShape) = 0; + virtual bool SupportsElementwiseLeakyRelu(bool quantized, DataType type) = 0; + virtual bool SupportsRescale(DataType fromType, DataType toType) = 0; + virtual Flags OperatorQuery(OpType opType, const ArchOperatorQuery *query = nullptr, ArchRequirements *req = nullptr) = 0; }; } // namespace regor diff --git a/ethosu/regor/architecture/ethosu55/ethos_u55.cpp b/ethosu/regor/architecture/ethosu55/ethos_u55.cpp index 71a68f24..a37c15da 100644 --- a/ethosu/regor/architecture/ethosu55/ethos_u55.cpp +++ b/ethosu/regor/architecture/ethosu55/ethos_u55.cpp @@ -34,6 +34,8 @@ #include #include +#include "include/regor.h" + BEGIN_ENUM_TABLE(regor::EthosU55SHRamElements) ADD_ENUM_NAME(SHRAM_IFM8) ADD_ENUM_NAME(SHRAM_IFM16) @@ -652,6 +654,11 @@ EthosU55NpuOp ArchEthosU55::GetHWOp(OpType type) return EthosU55NpuOp::None; } +void ArchEthosU55::Call(std::function callBack) +{ + callBack(REGOR_ARCH_ETHOSU55); +} + int EthosU55OpGroup::Add(const ArchitectureOpGroupQuery &op, const std::vector &dependsOn) { LOG_TRACE1("Trying to add op {}\n", OpTypeToString(op.type)); diff --git a/ethosu/regor/architecture/ethosu55/ethos_u55.hpp b/ethosu/regor/architecture/ethosu55/ethos_u55.hpp index fd39da33..7b87920a 100644 --- a/ethosu/regor/architecture/ethosu55/ethos_u55.hpp +++ b/ethosu/regor/architecture/ethosu55/ethos_u55.hpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2021-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -210,6 +210,7 @@ public: bool SupportsScalar(OpType opType, DataType dataType, TensorUsage usage) override; Flags SupportedWeightFormat(OpType op) override; uint32_t Version() override; + void Call(std::function callBack) override; protected: Shape OfmUBlock() { return _ofmUBlock; } diff --git a/ethosu/regor/architecture/ethosu55/ethos_u55_constraints.cpp b/ethosu/regor/architecture/ethosu55/ethos_u55_constraints.cpp index 55334b5a..31649b43 100644 --- a/ethosu/regor/architecture/ethosu55/ethos_u55_constraints.cpp +++ b/ethosu/regor/architecture/ethosu55/ethos_u55_constraints.cpp @@ -27,6 +27,7 @@ namespace regor static constexpr OpType s_unsupportedU55[] = { OpType::None, OpType::ArgMax, + OpType::Not, OpType::Gather, OpType::Scatter, OpType::Resize, @@ -47,18 +48,12 @@ EthosU55Constraints::EthosU55Constraints(ArchEthosU55 *arch) : _arch(arch) { } -bool EthosU55Constraints::SupportsLeakyRelu(bool quantized, DataType type) +bool EthosU55Constraints::SupportsElementwiseLeakyRelu(bool quantized, DataType type) { return quantized == false && type == DataType::Int16; } -bool EthosU55Constraints::SupportsMatMul(OpType opType) -{ - UNUSED(opType); - return false; -} - -TransposeSupport EthosU55Constraints::SupportsTranspose(OpType opType, TransposeType transposeType) +TransposeSupport EthosU55Constraints::SupportsFusedTranspose(OpType opType, TransposeType transposeType) { if ( IsNone(transposeType) ) return TransposeSupport::Any; @@ -72,7 +67,7 @@ TransposeSupport EthosU55Constraints::SupportsTranspose(OpType opType, Transpose return TransposeSupport::None; } -bool EthosU55Constraints::SupportsReverse(OpType opType, ReverseType reverseTypeMask) +bool EthosU55Constraints::SupportsFusedReverse(OpType opType, ReverseType reverseTypeMask) { UNUSED(opType); return reverseTypeMask == ReverseType::None; @@ -162,43 +157,6 @@ bool EthosU55Constraints::SupportsRescale(DataType fromType, DataType toType) return true; } -bool EthosU55Constraints::SupportsGather(OpType opType) -{ - UNUSED(opType); - return false; -} - -bool EthosU55Constraints::SupportsScatter(OpType opType) -{ - UNUSED(opType); - return false; -} -bool EthosU55Constraints::SupportsResize(const ResizeSupportQuery &query) -{ - UNUSED(query); - return false; -} - -bool EthosU55Constraints::SupportsArgMax(OpType opType) -{ - UNUSED(opType); - return false; -} - -bool EthosU55Constraints::SupportsCast(OpType opType, DataType ifmType, DataType ofmType) -{ - UNUSED(opType); - UNUSED(ifmType); - UNUSED(ofmType); - return false; -} - -bool EthosU55Constraints::SupportsNonMatchingShapes(const Shape &ifmShape, const Shape &ifm2Shape, const Shape &ofmShape) -{ - return (ifmShape == ofmShape) || (ifm2Shape && (ifm2Shape == ofmShape)); -} - - Flags EthosU55Constraints::OperatorQuery(OpType opType, const ArchOperatorQuery *query, ArchRequirements *req) { // Check unsupported operator list before further checks diff --git a/ethosu/regor/architecture/ethosu55/ethos_u55_constraints.hpp b/ethosu/regor/architecture/ethosu55/ethos_u55_constraints.hpp index b9ed473c..a15b3f47 100644 --- a/ethosu/regor/architecture/ethosu55/ethos_u55_constraints.hpp +++ b/ethosu/regor/architecture/ethosu55/ethos_u55_constraints.hpp @@ -30,22 +30,14 @@ private: public: EthosU55Constraints(ArchEthosU55 *arch); - bool SupportsLeakyRelu(bool quantized, DataType type) override; - bool SupportsMatMul(OpType opType) override; - TransposeSupport SupportsTranspose(OpType opType, TransposeType transposeType) override; - bool SupportsReverse(OpType opType, ReverseType reverseTypeMask) override; + bool SupportsFusedReverse(OpType opType, ReverseType reverseTypeMask) override; bool SupportsFusedRescale(OpType opType, TensorUsage tensorUsage, DataType rescaleFromType, DataType rescaleToType, DataType opFromType, DataType opToType, const Quantization &quantization) override; - bool SupportsRescale(DataType fromType, DataType toType) override; + TransposeSupport SupportsFusedTranspose(OpType opType, TransposeType transposeType) override; bool SupportsAccumulatorSaveRestore() override { return false; } - bool SupportsGather(OpType opType) override; - bool SupportsScatter(OpType opType) override; - bool SupportsResize(const ResizeSupportQuery &query) override; - bool SupportsArgMax(OpType opType) override; - bool SupportsCast(OpType opType, DataType ifmType, DataType ofmType) override; - bool SupportsNonMatchingShapes(const Shape &ifmShape, const Shape &ifm2Shape, const Shape &ofmShape) override; bool SupportsNegativeStrides() override { return true; }; - bool SupportsNot() override { return false; }; + bool SupportsElementwiseLeakyRelu(bool quantized, DataType type) override; + bool SupportsRescale(DataType fromType, DataType toType) override; Flags OperatorQuery(OpType opType, const ArchOperatorQuery *query, ArchRequirements *req) override; }; diff --git a/ethosu/regor/architecture/ethosu65/ethos_u65.cpp b/ethosu/regor/architecture/ethosu65/ethos_u65.cpp index 877e1ecc..97ec8b52 100644 --- a/ethosu/regor/architecture/ethosu65/ethos_u65.cpp +++ b/ethosu/regor/architecture/ethosu65/ethos_u65.cpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2021-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -28,6 +28,8 @@ #include #include +#include "include/regor.h" + namespace regor { @@ -88,4 +90,9 @@ std::vector ArchEthosU65::ConfigRegisters() return std::vector(1, ConfigRegister(1)); } +void ArchEthosU65::Call(std::function callBack) +{ + callBack(REGOR_ARCH_ETHOSU65); +} + } // namespace regor diff --git a/ethosu/regor/architecture/ethosu65/ethos_u65.hpp b/ethosu/regor/architecture/ethosu65/ethos_u65.hpp index 057f5d5b..50247dc7 100644 --- a/ethosu/regor/architecture/ethosu65/ethos_u65.hpp +++ b/ethosu/regor/architecture/ethosu65/ethos_u65.hpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2021-2023 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2021-2023, 2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -37,6 +37,7 @@ public: bool ParseConfig(IniReader *reader) override; Address MaxAddress() override { return 1LL << 40; } std::vector ConfigRegisters() override; + void Call(std::function callBack) override; private: int MaxOutstandingDMAOps() override { return 2; } diff --git a/ethosu/regor/architecture/ethosu85/ethos_u85.cpp b/ethosu/regor/architecture/ethosu85/ethos_u85.cpp index 04a3e0b2..45e15874 100644 --- a/ethosu/regor/architecture/ethosu85/ethos_u85.cpp +++ b/ethosu/regor/architecture/ethosu85/ethos_u85.cpp @@ -33,6 +33,9 @@ #include #include +#include "include/regor.h" + + BEGIN_ENUM_TABLE(regor::EthosU85Accumulator) ADD_ENUM_NAME(Acc32) ADD_ENUM_NAME(Acc48) @@ -1360,6 +1363,11 @@ EthosU85NpuOp ArchEthosU85::GetHWOp(OpType type) return EthosU85NpuOp::None; } +void ArchEthosU85::Call(std::function callBack) +{ + callBack(REGOR_ARCH_ETHOSU85); +} + int EthosU85OpGroup::KeyToOpIndex(int key) { if ( key > 0 ) diff --git a/ethosu/regor/architecture/ethosu85/ethos_u85.hpp b/ethosu/regor/architecture/ethosu85/ethos_u85.hpp index 2fb12ab1..ba0bde25 100644 --- a/ethosu/regor/architecture/ethosu85/ethos_u85.hpp +++ b/ethosu/regor/architecture/ethosu85/ethos_u85.hpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2021-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -224,6 +224,7 @@ public: bool SupportsScalar(OpType opType, DataType dataType, TensorUsage usage) override; Flags SupportedWeightFormat(OpType op) override; uint32_t Version() override; + void Call(std::function callBack) override; protected: void ApplyConfig(const AcceleratorConfig *cfg); diff --git a/ethosu/regor/architecture/ethosu85/ethos_u85_constraints.cpp b/ethosu/regor/architecture/ethosu85/ethos_u85_constraints.cpp index 2badd56d..59ec0938 100644 --- a/ethosu/regor/architecture/ethosu85/ethos_u85_constraints.cpp +++ b/ethosu/regor/architecture/ethosu85/ethos_u85_constraints.cpp @@ -37,24 +37,7 @@ static constexpr std::pair s_shortU85[] = { static_assert(is_sorted(s_shortU85, [](const auto &a, const auto &b) { return a.first < b.first; }), "list must be sorted"); - -bool EthosU85Constraints::SupportsLeakyRelu(bool /*quantized*/, DataType /*type*/) -{ - return true; -} - -bool EthosU85Constraints::SupportsMatMul(OpType opType) -{ - EthosU85NpuOp npuOp = ArchEthosU85::GetHWOp(opType); - if ( npuOp == EthosU85NpuOp::None ) - { - return false; - } - - return true; -} - -TransposeSupport EthosU85Constraints::SupportsTranspose(OpType opType, TransposeType transposeType) +TransposeSupport EthosU85Constraints::SupportsFusedTranspose(OpType opType, TransposeType transposeType) { if ( transposeType == TransposeType::None ) return TransposeSupport::Any; @@ -80,7 +63,7 @@ TransposeSupport EthosU85Constraints::SupportsTranspose(OpType opType, Transpose return TransposeSupport::None; } -bool EthosU85Constraints::SupportsReverse(OpType opType, ReverseType reverseTypeMask) +bool EthosU85Constraints::SupportsFusedReverse(OpType opType, ReverseType reverseTypeMask) { Flags reverseMask(reverseTypeMask); // Do not support non-constant axes @@ -177,136 +160,6 @@ bool EthosU85Constraints::SupportsRescale(DataType fromType, DataType toType) return fromType != DataType::UInt16; } -bool EthosU85Constraints::SupportsGather(OpType opType) -{ - EthosU85NpuOp npuOp = ArchEthosU85::GetHWOp(opType); - if ( npuOp == EthosU85NpuOp::None ) - { - return false; - } - - return true; -} - -bool EthosU85Constraints::SupportsScatter(OpType opType) -{ - EthosU85NpuOp npuOp = ArchEthosU85::GetHWOp(opType); - if ( npuOp == EthosU85NpuOp::None ) - { - return false; - } - - return true; -} - -bool EthosU85Constraints::SupportsArgMax(OpType opType) -{ - EthosU85NpuOp npuOp = ArchEthosU85::GetHWOp(opType); - if ( npuOp == EthosU85NpuOp::None ) - { - return false; - } - - return true; -} - -bool EthosU85Constraints::SupportsResize(const ResizeSupportQuery &query) -{ - /* Supported operator checks for resize operations - * - * * Scaling numerators must be less than or equal to 2048 - * * Offsets must be in the range [-numerator, numerator) for each axis - * * The following constraints apply to upscale-factors - * mode REPLICATE: - * Any width and height upscale-factors are supported - * mode NEAREST: - * Any width and height upscale-factors are supported - * mode BILINEAR: - * if IFM W*H == 1*1: - * Any width and height upscale-factors are supported - * else: - * The upscale-factors need to be powers-of-two. - */ - if ( query.ifmShape.Width() == 1 && query.ifmShape.Height() == 1 ) - { - return true; - } - - int n_w = query.scaleX.n; - int d_w = query.scaleX.d; - int n_h = query.scaleY.n; - int d_h = query.scaleY.d; - - if ( n_h > 2048 ) - { - LOG_WARN("Resize height scale numerator ({}) exceeds maximum size (2048).\n", n_h); - return false; - } - if ( n_w > 2048 ) - { - LOG_WARN("Resize width scale numerator ({}) exceeds maximum size (2048).\n", n_w); - return false; - } - if ( query.offsetY >= n_h || query.offsetY < -n_h ) - { - LOG_WARN("Resize height offset: {} is outside the valid range [-height_numerator, height_numerator) = [{}, {})\n", - query.offsetY, -n_h, n_h); - return false; - } - if ( query.offsetX >= n_w || query.offsetX < -n_w ) - { - LOG_WARN("Resize width offset: {} is outside the valid range [-with_numerator, width_numerator) = [{}, {})\n", - query.offsetX, -n_w, n_w); - return false; - } - - if ( query.mode == ArchResizeMode::Bilinear ) - { - if ( d_w == 0 || d_h == 0 ) - { - LOG_WARN("ResizeBilinear w/h divisors can't be zero\n"); - return false; - } - - // Get scale fractions and verify that scale-factor is a power of two. - if ( n_w % d_w != 0 ) - { - LOG_WARN("ResizeBilinear width scale-factor is not an integer: {}/{}\n", n_w, d_w); - return false; - } - if ( n_h % d_h != 0 ) - { - LOG_WARN("ResizeBilinear height scale-factor is not an integer: {}/{}\n", n_h, d_h); - return false; - } - int scale_w = n_w / d_w; - int scale_h = n_h / d_h; - if ( !IsPowerOfTwo(scale_w) ) - { - LOG_WARN("ResizeBilinear width scale-factor is not a power of two: {}\n", double(n_w) / d_w); - return false; - } - if ( !IsPowerOfTwo(scale_h) ) - { - LOG_WARN("ResizeBilinear height scale-factor is not a power of two: {}\n", double(n_h) / d_h); - return false; - } - } - - return true; -} - -bool EthosU85Constraints::SupportsCast(OpType opType, DataType ifmType, DataType ofmType) -{ - return !IsFloat(ifmType | ofmType); -} - -bool EthosU85Constraints::SupportsNonMatchingShapes(const Shape &ifmShape, const Shape &ifm2Shape, const Shape &ofmShape) -{ - return true; -} - - Flags EthosU85Constraints::OperatorQuery(OpType opType, const ArchOperatorQuery *query, ArchRequirements *req) { // Check unsupported operator list first @@ -337,22 +190,17 @@ Flags EthosU85Constraints::OperatorQuery(OpType opType, const ArchO if ( query->transposeMask != TransposeType::None ) { - TransposeSupport tmp = SupportsTranspose(opType, query->transposeMask); + TransposeSupport tmp = SupportsFusedTranspose(opType, query->transposeMask); if ( tmp == TransposeSupport::None ) return QueryResult::Unsupported; } if ( query->reverseMask != ReverseType::None ) { - if ( !SupportsReverse(opType, query->reverseMask) ) return QueryResult::Unsupported; + if ( !SupportsFusedReverse(opType, query->reverseMask) ) return QueryResult::Unsupported; } // Operator specific - if ( opType == OpType::Resize ) - { - if ( !query->specific.resize.ifmShape ) return QueryResult::Unsupported; // TODO: remove from ResizeQuery - if ( !SupportsResize(query->specific.resize) ) return QueryResult::Unsupported; - } - else if ( (opType == OpType::Sigmoid) || (opType == OpType::Tanh) ) + if ( (opType == OpType::Sigmoid) || (opType == OpType::Tanh) ) { if ( req ) { diff --git a/ethosu/regor/architecture/ethosu85/ethos_u85_constraints.hpp b/ethosu/regor/architecture/ethosu85/ethos_u85_constraints.hpp index 784d9266..c193d6d3 100644 --- a/ethosu/regor/architecture/ethosu85/ethos_u85_constraints.hpp +++ b/ethosu/regor/architecture/ethosu85/ethos_u85_constraints.hpp @@ -24,29 +24,20 @@ namespace regor class EthosU85Constraints : public IArchitectureConstraints { +private: + ArchEthosU85 *_arch; + public: EthosU85Constraints(ArchEthosU85 *arch) : _arch(arch) {} - - bool SupportsLeakyRelu(bool quantized, DataType type) override; - bool SupportsMatMul(OpType opType) override; - TransposeSupport SupportsTranspose(OpType opType, TransposeType transposeType) override; - bool SupportsReverse(OpType opType, ReverseType reverseTypeMask) override; + bool SupportsFusedReverse(OpType opType, ReverseType reverseTypeMask) override; bool SupportsFusedRescale(OpType opType, TensorUsage tensorUsage, DataType rescaleFromType, DataType rescaleToType, DataType opFromType, DataType opToType, const Quantization &quantization) override; - bool SupportsRescale(DataType fromType, DataType toType) override; + TransposeSupport SupportsFusedTranspose(OpType opType, TransposeType transposeType) override; bool SupportsAccumulatorSaveRestore() override { return true; } - bool SupportsGather(OpType opType) override; - bool SupportsScatter(OpType opType) override; - bool SupportsResize(const ResizeSupportQuery &query) override; - bool SupportsArgMax(OpType opType) override; - bool SupportsCast(OpType opType, DataType ifmType, DataType ofmType) override; - bool SupportsNonMatchingShapes(const Shape &ifmShape, const Shape &ifm2Shape, const Shape &ofmShape) override; bool SupportsNegativeStrides() override { return false; }; - bool SupportsNot() override { return true; }; + bool SupportsElementwiseLeakyRelu(bool quantized, DataType type) override { return true; }; + bool SupportsRescale(DataType fromType, DataType toType) override; Flags OperatorQuery(OpType opType, const ArchOperatorQuery *query, ArchRequirements *req) override; - -private: - ArchEthosU85 *_arch; }; } // namespace regor diff --git a/ethosu/regor/common/data_type.cpp b/ethosu/regor/common/data_type.cpp index d9c8ac21..a9e37b8a 100644 --- a/ethosu/regor/common/data_type.cpp +++ b/ethosu/regor/common/data_type.cpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2023-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -43,6 +43,7 @@ BEGIN_ENUM_TABLE(regor::DataType) ADD_ENUM_NAME(UInt8) ADD_ENUM_NAME(UInt16) ADD_ENUM_NAME(UInt32) + ADD_ENUM_NAME(UInt48) ADD_ENUM_NAME(UInt64) ADD_ENUM_NAME(QInt) ADD_ENUM_NAME(QInt4) @@ -57,6 +58,7 @@ BEGIN_ENUM_TABLE(regor::DataType) ADD_ENUM_NAME(QUInt16) ADD_ENUM_NAME(QUInt32) ADD_ENUM_NAME(Float) + ADD_ENUM_NAME(BFloat16) ADD_ENUM_NAME(Float16) ADD_ENUM_NAME(Float32) ADD_ENUM_NAME(Float64) diff --git a/ethosu/regor/compiler/compiler.cpp b/ethosu/regor/compiler/compiler.cpp index 8360162a..3927855b 100644 --- a/ethosu/regor/compiler/compiler.cpp +++ b/ethosu/regor/compiler/compiler.cpp @@ -33,9 +33,14 @@ #include "tensor_allocator.hpp" #include "tflite/custom_operator_ethosu.hpp" #include "tflite/tflite_reader.hpp" +#include "tflite/tflite_supported_operators.hpp" +#include "tflite/tflite_supported_operators_u55.hpp" +#include "tflite/tflite_supported_operators_u85.hpp" #include "tflite/tflite_writer.hpp" #include "tosa/tosa_reader.hpp" +#include "include/regor.h" + BEGIN_ENUM_TABLE(regor::OutputFormat) ADD_ENUM_NAME(None) ADD_ENUM_NAME(TFLite) @@ -50,6 +55,24 @@ END_ENUM_TABLE() namespace regor { +namespace +{ + +std::unique_ptr InitSupportedOpsChecker(const std::string &target, IArchitectureConstraints *constraints) +{ + if ( target == REGOR_ARCH_ETHOSU85 ) + { + return std::make_unique(constraints); + } + else + { + assert(target == REGOR_ARCH_ETHOSU55 || target == REGOR_ARCH_ETHOSU65); + return std::make_unique(constraints); + } +} + +} // namespace + Compiler::Compiler(std::unique_ptr &arch) { _architecture = std::move(arch); @@ -179,7 +202,6 @@ bool Compiler::LoadTosa(const void *input, size_t size) return !_builders.empty(); } - bool Compiler::LoadTflite(const void *input, size_t size) { assert(input && size > 0); @@ -188,7 +210,7 @@ bool Compiler::LoadTflite(const void *input, size_t size) if ( _compilerOptions.debugDatabase != !!_optDb ) _optDb = _compilerOptions.debugDatabase ? std::make_unique(&_Db) : nullptr; - TfLiteReader::LoadGraphs(input, size, _graphs, _optDb.get(), _architecture->Constraints()); + TfLiteReader::LoadGraphs(input, size, _graphs, _optDb.get()); return !_graphs.empty(); } @@ -419,6 +441,15 @@ std::unique_ptr Compiler::CompileGraph(std::unique_ptr &graph, { if ( graph->Notation() == GraphNotation::TFLite ) { + // Run TFLite supported-operator checks + std::unique_ptr supportedOps; + _architecture->Call([&](const std::string &target) + { supportedOps = InitSupportedOpsChecker(target, _architecture->Constraints()); }); + + if ( supportedOps ) + { + supportedOps->Process(graph.get()); + } // Run GraphNotation::TFLite Preprocess/optimise step std::unique_ptr optimiser = GraphOptimiser::MakeGraphOptimiser( GraphNotation::TFLite, _architecture->Constraints(), _graphOptimiserOptions, _optDb.get()); diff --git a/ethosu/regor/compiler/graph.hpp b/ethosu/regor/compiler/graph.hpp index 24dcffd7..c4c7bbd3 100644 --- a/ethosu/regor/compiler/graph.hpp +++ b/ethosu/regor/compiler/graph.hpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2021, 2023-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2021, 2023-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -136,6 +136,16 @@ public: }); } + void GetAllOperations(std::vector> &operations) const + { + TraverseGraphFromEnd(Outputs(), + [&](Operation *op) -> bool + { + operations.push_back(op->shared_from_this()); + return true; + }); + } + // Get all operations in the graph, in scheduled order const std::vector &ScheduledOrder() const { return _opsInScheduledOrder; }; diff --git a/ethosu/regor/compiler/scheduler_packing.cpp b/ethosu/regor/compiler/scheduler_packing.cpp index cadef74f..f94e330f 100644 --- a/ethosu/regor/compiler/scheduler_packing.cpp +++ b/ethosu/regor/compiler/scheduler_packing.cpp @@ -538,7 +538,6 @@ std::unique_ptr SchedulerPacking::MakeSchedulerOperation(Ope Set(query.ofm, ofmConn); query.reverseMask = ofmConn->reverse; query.transposeMask = ofmConn->transpose; - query.specific.resize = {}; ArchRequirements req; if ( _arch->Constraints()->OperatorQuery(op->Type(), &query, &req).Any(QueryResult::Native) ) diff --git a/ethosu/regor/compiler/tflite_graph_optimiser.cpp b/ethosu/regor/compiler/tflite_graph_optimiser.cpp index e8fe6443..d47651bf 100644 --- a/ethosu/regor/compiler/tflite_graph_optimiser.cpp +++ b/ethosu/regor/compiler/tflite_graph_optimiser.cpp @@ -939,10 +939,7 @@ Operation *TFLiteGraphOptimiser::ConvertGather(Graph *const graph, Operation *co OpType opType = operation->Type(); - ExecutionQuery query{}; - query.opType = OpType::Gather; - - if ( opType == OpType::GatherV2 && _constraints->CanExecute(query) ) + if ( opType == OpType::GatherV2 ) { auto *paramsConn = operation->Input(TensorUsage::IFM0); auto *idxConn = operation->Input(TensorUsage::IFM1); @@ -1057,10 +1054,7 @@ Operation *TFLiteGraphOptimiser::ConvertScatter(Graph *const graph, Operation *c OpType opType = operation->Type(); - ExecutionQuery query{}; - query.opType = OpType::Scatter; - - if ( opType == OpType::ScatterNd && _constraints->CanExecute(query) ) + if ( opType == OpType::ScatterNd ) { auto *idxConn = operation->Input(TensorUsage::IFM0); auto *updatesConn = operation->Input(TensorUsage::IFM1); @@ -1156,6 +1150,11 @@ Operation *TFLiteGraphOptimiser::ConvertResize(Graph *const graph, Operation *co Operation *returnOp = operation; OpType opType = operation->Type(); + if ( _constraints->OperatorQuery(OpType::Resize).Any(QueryResult::Unsupported) ) + { + // Only run if HW has native Resize support + return returnOp; + } if ( opType == OpType::ResizeBilinear || opType == OpType::ResizeNearestNeighbor ) { auto ifmConn = operation->Input(TensorUsage::IFM); @@ -1230,65 +1229,41 @@ Operation *TFLiteGraphOptimiser::ConvertResize(Graph *const graph, Operation *co heightOffset = (height_d / 2) - (height_n / 2); } - // set up op-support query - ResizeSupportQuery resizeQuery; - resizeQuery.scaleX = {int16_t(width_n), int16_t(width_d)}; - resizeQuery.scaleY = {int16_t(height_n), int16_t(height_d)}; - resizeQuery.offsetX = widthOffset; - resizeQuery.offsetY = heightOffset; - resizeQuery.ifmShape = ifmConn->shape; - - if ( opType == OpType::ResizeBilinear ) - { - resizeQuery.mode = ArchResizeMode::Bilinear; - } - else - { - resizeQuery.mode = ArchResizeMode::Nearest; - } + // Replace ResizeBilinear or ResizeNearestNeighbor with a Resize op + auto resizeOp = std::make_shared(OpType::Resize); + resizeOp->CopyInput(TensorUsage::IFM, *ifmConn); + resizeOp->CopyOutput(TensorUsage::OFM, *ofmConn); + resizeOp->Output(TensorUsage::OFM)->Set(RoundMode::SYMMETRIC); - ExecutionQuery query{}; - query.opType = opType; - query.resizeQuery = resizeQuery; + // write operator attributes + auto *attr = resizeOp->Attribute(); + attr->scaleX = {width_n, width_d}; + attr->scaleY = {height_n, height_d}; + attr->offset = {widthOffset, heightOffset}; + attr->border = {0, 0}; + attr->mode = (opType == OpType::ResizeBilinear) ? tosa::ResizeMode::BILINEAR : tosa::ResizeMode::NEAREST; - if ( _constraints->CanExecute(query) ) + int shift = 0; + if ( opType == OpType::ResizeBilinear && (ifmConn->shape.Width() > 1 || ifmConn->shape.Height() > 1) ) { - // Replace ResizeBilinear or ResizeNearestNeighbor with a Resize op - auto resizeOp = std::make_shared(OpType::Resize); - resizeOp->CopyInput(TensorUsage::IFM, *ifmConn); - resizeOp->CopyOutput(TensorUsage::OFM, *ofmConn); - resizeOp->Output(TensorUsage::OFM)->Set(RoundMode::SYMMETRIC); - - // write operator attributes - auto *attr = resizeOp->Attribute(); - attr->scaleX = {width_n, width_d}; - attr->scaleY = {height_n, height_d}; - attr->offset = {widthOffset, heightOffset}; - attr->border = {0, 0}; - attr->mode = (opType == OpType::ResizeBilinear) ? tosa::ResizeMode::BILINEAR : tosa::ResizeMode::NEAREST; - - int shift = 0; - if ( opType == OpType::ResizeBilinear && (ifmConn->shape.Width() > 1 || ifmConn->shape.Height() > 1) ) - { - // ResizeBilinear is post-scaled with - // 1 / (height_n * width_n) - // as the scale-factor is a power of two, we can use shift - shift = IntLog2(width_n * height_n); - } - - // Set explicit scaling - Quantization ofmQuant = ofmConn->quantization; - ofmQuant.scales.clear(); - ofmQuant.zeroPoints.clear(); - ofmQuant.scales.emplace_back(QuantizedScale(1, shift)); - ofmQuant.zeroPoints.emplace_back(0); - ofmQuant.type = QuantizationType::EXPLICIT; - resizeOp->Output(TensorUsage::OFM)->Set(ofmQuant); - - RecordOptimisation(operation, resizeOp.get()); - returnOp = resizeOp.get(); - operation->Disconnect(); + // ResizeBilinear is post-scaled with + // 1 / (height_n * width_n) + // as the scale-factor is a power of two, we can use shift + shift = IntLog2(width_n * height_n); } + + // Set explicit scaling + Quantization ofmQuant = ofmConn->quantization; + ofmQuant.scales.clear(); + ofmQuant.zeroPoints.clear(); + ofmQuant.scales.emplace_back(QuantizedScale(1, shift)); + ofmQuant.zeroPoints.emplace_back(0); + ofmQuant.type = QuantizationType::EXPLICIT; + resizeOp->Output(TensorUsage::OFM)->Set(ofmQuant); + + RecordOptimisation(operation, resizeOp.get()); + returnOp = resizeOp.get(); + operation->Disconnect(); } return returnOp; } @@ -1407,9 +1382,7 @@ Operation *TFLiteGraphOptimiser::CreateTransposeForMatMul(const std::shared_ptr< Operation *TFLiteGraphOptimiser::RewriteBatchMatMul(Graph *const, Operation *const operation) { Operation *returnOp = operation; - ExecutionQuery query{}; - query.opType = OpType::MatMul; - if ( operation->Type() == OpType::BatchMatMul && _constraints->CanExecute(query) ) + if ( operation->Type() == OpType::BatchMatMul ) { const auto ifm = operation->Input(TensorUsage::IFM0); const auto ifm2 = operation->Input(TensorUsage::IFM1); @@ -1480,9 +1453,7 @@ Operation *TFLiteGraphOptimiser::RewriteFullyConnectDynamic(Graph *const, Operat { Operation *returnOp = operation; auto ifm2 = operation->Input(TensorUsage::Weights); - ExecutionQuery query{}; - query.opType = OpType::MatMul; - if ( operation->Type() == OpType::FullyConnected && !ifm2->tensor->IsConstant() && _constraints->CanExecute(query) ) + if ( operation->Type() == OpType::FullyConnected && !ifm2->tensor->IsConstant() ) { const auto ifm = operation->Input(TensorUsage::IFM0); const auto ofm = operation->Output(TensorUsage::OFM); @@ -2350,7 +2321,8 @@ Operation *TFLiteGraphOptimiser::ConvertLeakyRelu(Graph *const graph, Operation returnOp = Convert8bitLeakyReluToLUT(graph, operation, alpha); RecordOptimisation(operation, returnOp); } - else if ( alpha < 0 || isConvertedPrelu || !_constraints->SupportsLeakyRelu(!IsScalingValidAndEqual(*ifmConn, *ofmConn), ifm->Type()) ) + else if ( alpha < 0 || isConvertedPrelu || + !_constraints->SupportsElementwiseLeakyRelu(!IsScalingValidAndEqual(*ifmConn, *ofmConn), ifm->Type()) ) { // Use 16-bit lowering to Mul + Max or Min + Mul + Relu + Add returnOp = ConvertLeakyRelu16bit(*ifmConn, *ofmConn, operation); @@ -2810,6 +2782,41 @@ SliceConstTensor(const TensorConnection *conn, const Shape &sliceShape, const Sh return std::make_shared(Name, conn->tensor->Type(), sliceShape, std::move(newBuffer)); } +Operation *TFLiteGraphOptimiser::ClampActivations(Graph *const graph, Operation *const operation) +{ + OpType opType = operation->Type(); + auto Quantize = [](float value, const Quantization &quant) + { + float scale = quant.scales.empty() ? 1.0f : float(quant.scales[0].Dequantize()); + int64_t zp = quant.zeroPoints.empty() ? 0 : quant.zeroPoints[0]; + return zp + int64_t(std::round(double(value / scale))); + }; + if ( !IsActivation(opType) ) + { + return operation; + } + Quantization &quant = operation->Output(TensorUsage::OFM)->quantization; + if ( quant.quantMin.size() || quant.quantMax.size() ) + { + return operation; + } + if ( opType == OpType::Relu ) + { + quant.quantMin = {Quantize(0, quant)}; + } + else if ( opType == OpType::Relu6 ) + { + quant.quantMin = {Quantize(0, quant)}; + quant.quantMax = {Quantize(6, quant)}; + } + else if ( opType == OpType::ReluN1To1 ) + { + quant.quantMin = {Quantize(-1, quant)}; + quant.quantMax = {Quantize(1, quant)}; + } + return operation; +} + // Converts a convolution group with N groups into N * Conv2D ops each operating on a 1/N part of // the original channels. Finally, all of the individual results will be concatenated depth-wise into // the OFM tensor. diff --git a/ethosu/regor/compiler/tflite_graph_optimiser.hpp b/ethosu/regor/compiler/tflite_graph_optimiser.hpp index 3cf68bdb..4ca3b92e 100644 --- a/ethosu/regor/compiler/tflite_graph_optimiser.hpp +++ b/ethosu/regor/compiler/tflite_graph_optimiser.hpp @@ -95,6 +95,7 @@ private: Operation *ConvertTanhSigmoidToLUT16(Operation *const op); // Rewrite functions + Operation *ClampActivations(Graph *const graph, Operation *const operation); Operation *ConvertConvolutionGroup(Graph *const graph, Operation *const operation); Operation *ConvertExpToLUT(Graph *const graph, Operation *const operation); Operation *RewritePack(Graph *const graph, Operation *const operation); @@ -196,6 +197,7 @@ public: { {}, { + &TFLiteGraphOptimiser::ClampActivations, &TFLiteGraphOptimiser::ConvertConvolutionGroup, } }, diff --git a/ethosu/regor/test/CMakeLists.txt b/ethosu/regor/test/CMakeLists.txt index 4cd2cd77..0f9391f8 100644 --- a/ethosu/regor/test/CMakeLists.txt +++ b/ethosu/regor/test/CMakeLists.txt @@ -62,6 +62,7 @@ add_catch_test( test_fast_storage_allocator.cpp test_tflite_fb.cpp test_custom_operator_ethosu.cpp + test_tflite_supported_operators.cpp DEPS test_common ) diff --git a/ethosu/regor/test/test_graphir_optimiser.cpp b/ethosu/regor/test/test_graphir_optimiser.cpp index 3812bb4b..567c1d38 100644 --- a/ethosu/regor/test/test_graphir_optimiser.cpp +++ b/ethosu/regor/test/test_graphir_optimiser.cpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2024-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -147,6 +147,7 @@ TEST_CASE("test_graphir_optimiser - ReduceSum") auto ifm = CreateTensor("IFM", Shape(1, 4, 4, 25), DataType::Int8); auto ofm = CreateTensor("OFM", ifm->StorageShape().WithDepth(1), DataType::Int8); auto op = CreateOperation(OpType::ReduceSum, TensorUsage::IFM, ifm, TensorUsage::OFM, ofm); + op->Input(TensorUsage::IFM)->quantization.zeroPoints.clear(); op->Input(TensorUsage::IFM)->quantization.zeroPoints.push_back(ZP); op->Attribute()->axis = ifm->StorageShape().Size() - 1; ops.push_back(std::move(op)); diff --git a/ethosu/regor/test/test_tflite_fb.cpp b/ethosu/regor/test/test_tflite_fb.cpp index 8fa0c6aa..f0d5e2cb 100644 --- a/ethosu/regor/test/test_tflite_fb.cpp +++ b/ethosu/regor/test/test_tflite_fb.cpp @@ -20,6 +20,7 @@ #include "architecture/ethosu85/ethos_u85.hpp" #include "tflite/tflite_reader.hpp" +#include "tflite/tflite_supported_operators_u85.hpp" #include "tflite/tflite_writer.hpp" #include "util.hpp" @@ -114,7 +115,7 @@ TEST_CASE("test_tflite_fb - load/store") TfLiteReader reader; std::vector> readerGraphs; - reader.LoadGraphs(&fb[output_buffer_offset], output_buffer_size, readerGraphs, nullptr, arch->Constraints()); + reader.LoadGraphs(&fb[output_buffer_offset], output_buffer_size, readerGraphs, nullptr); REQUIRE(readerGraphs.size() == 1); std::vector operations; diff --git a/ethosu/regor/test/test_tflite_supported_operators.cpp b/ethosu/regor/test/test_tflite_supported_operators.cpp new file mode 100644 index 00000000..49df5725 --- /dev/null +++ b/ethosu/regor/test/test_tflite_supported_operators.cpp @@ -0,0 +1,272 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "common/common.hpp" + +#include "architecture/ethosu55/ethos_u55.hpp" +#include "architecture/ethosu65/ethos_u65.hpp" +#include "architecture/ethosu85/ethos_u85.hpp" +#include "tflite/tflite_supported_operators.hpp" +#include "tflite/tflite_supported_operators_u55.hpp" +#include "tflite/tflite_supported_operators_u85.hpp" +#include "util.hpp" + +#include + +#include "regor.h" + +using namespace regor; + +namespace +{ +std::unique_ptr MakeSupportedOpsChecker(std::string target, std::shared_ptr &arch) +{ + if ( target == REGOR_ARCH_ETHOSU85 ) + { + return std::make_unique(arch->Constraints()); + } + else + { + return std::make_unique(arch->Constraints()); + } +} + +std::shared_ptr CreateOperation(OpType opType, Shape ifmShape, DataType ifmType, Shape ifm2Shape, + DataType ifm2Type, Shape ofmShape, DataType ofmType) +{ + auto ifm = CreateTensor("IFM", ifmShape, ifmType); + auto ofm = CreateTensor("OFM", ofmShape, ofmType); + std::shared_ptr op = ::CreateOperation(opType, TensorUsage::IFM, ifm, TensorUsage::OFM, ofm); + + if ( ifm2Shape ) + { + auto ifm2 = CreateTensor("IFM2", ifm2Shape, ifm2Type); + op->ConnectInput(TensorUsage::IFM1, ifm2).Set(Quantization::Unit()); + } + if ( opType == OpType::Conv2D ) + { + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + } + return op; +}; + +std::shared_ptr CreateOperation(OpType opType, Shape ifmShape, DataType ifmType, Shape ofmShape, DataType ofmType) +{ + return CreateOperation(opType, ifmShape, ifmType, Shape(), DataType::None, ofmShape, ofmType); +}; + +} // namespace + +TEST_CASE("Supported operators Common") +{ + std::shared_ptr arch = CreateArchDefault(256); + std::string err = "noerror"; + arch->CheckConfiguration(err); + REQUIRE(err == "noerror"); + auto supportedOps = MakeSupportedOpsChecker(REGOR_ARCH_ETHOSU55, arch); + SECTION("ConstraintTensQuantized") + { + auto op = CreateOperation(OpType::Conv2D, Shape(1, 8, 8, 1), DataType::Int8, Shape(1, 8, 8, 1), DataType::Int8); + // Regular op should pass + REQUIRE(supportedOps->Check(op.get()) == true); + auto &quant = op->Output(TensorUsage::OFM)->quantization; + // Removing scales should fail + quant.scales.clear(); + REQUIRE(supportedOps->Check(op.get()) == false); + quant = Quantization::Unit(); + quant.zeroPoints.clear(); + REQUIRE(supportedOps->Check(op.get()) == false); + op->Disconnect(); + } +} + +TEST_CASE("Supported operators EthosU55") +{ + std::shared_ptr arch = CreateArchDefault(256); + std::string err = "noerror"; + arch->CheckConfiguration(err); + REQUIRE(err == "noerror"); + + auto supportedOps = MakeSupportedOpsChecker(REGOR_ARCH_ETHOSU55, arch); + + SECTION("Test positive") + { + // checks are expected to pass + auto op = CreateOperation(OpType::Add, Shape(1, 8, 8, 1), DataType::Int8, Shape(1, 8, 8, 1), DataType::Int8, + Shape(1, 8, 8, 1), DataType::Int8); + REQUIRE(supportedOps->Check(op.get()) == true); + op->Disconnect(); + } + + SECTION("ConstraintOpType") + { + auto op = CreateOperation(OpType::ScatterNd, Shape(1, 8, 8, 1), DataType::Int8, Shape(1, 8, 8, 1), DataType::Int8); + auto op2 = CreateOperation(OpType::GatherV2, Shape(1, 8, 8, 1), DataType::Int8, Shape(1, 8, 8, 1), DataType::Int8); + REQUIRE(supportedOps->Check(op.get()) == false); + REQUIRE(supportedOps->Check(op2.get()) == false); + op->Disconnect(); + op2->Disconnect(); + } + + SECTION("ConstraintTensDtypes") + { + std::set unsupported = { + DataType::Int48, + DataType::Int64, + DataType::UInt48, + DataType::UInt64, + DataType::QInt, + DataType::QInt, + DataType::QInt4, + DataType::QInt8, + DataType::QInt12, + DataType::QInt16, + DataType::QInt32, + DataType::QUInt, + DataType::QUInt4, + DataType::QUInt8, + DataType::QUInt12, + DataType::QUInt16, + DataType::QUInt32, + DataType::Float, + DataType::BFloat16, + DataType::Float16, + DataType::Float32, + DataType::Float64, + DataType::Bool, + DataType::Bool8, + DataType::Complex, + DataType::Complex64, + DataType::Complex128, + DataType::VariablySized, + DataType::String, + DataType::Resource, + DataType::Variant, + }; + std::set supported = { + DataType::UInt8, + DataType::Int8, + DataType::Int16, + DataType::Int32, + }; + for ( auto dtype : unsupported ) + { + auto op = CreateOperation(OpType::Add, Shape(1, 8, 8, 1), dtype, Shape(1, 8, 8, 1), dtype, Shape(1, 8, 8, 1), dtype); + REQUIRE(supportedOps->Check(op.get()) == false); + op->Disconnect(); + } + for ( auto dtype : supported ) + { + auto op = CreateOperation(OpType::Add, Shape(1, 8, 8, 1), dtype, Shape(1, 8, 8, 1), dtype, Shape(1, 8, 8, 1), dtype); + REQUIRE(supportedOps->Check(op.get()) == true); + op->Disconnect(); + } + } + + SECTION("ConstraintBroadCastShapes") + { + auto op = CreateOperation(OpType::Add, Shape(1, 5, 5, 1), DataType::Int8, Shape(1, 2, 2, 1), DataType::Int8, + Shape(1, 8, 8, 1), DataType::Int8); + REQUIRE(supportedOps->Check(op.get()) == false); + op->Disconnect(); + } + + SECTION("ConstraintReverse") + { + auto op = CreateOperation(OpType::ReverseV2, Shape(1, 8, 8, 1), DataType::Int8, Shape(1, 8, 8, 1), DataType::Int8); + // create params + auto params = CreateTensor("axis", Shape(1, 1, 1, 1), DataType::Int32, 1); + op->ConnectInput(TensorUsage::Params, params); + REQUIRE(supportedOps->Check(op.get()) == false); + op->Disconnect(); + } +} + +TEST_CASE("Supported operators EthosU85") +{ + std::shared_ptr arch = CreateArchDefault(256); + std::string err = "noerror"; + arch->CheckConfiguration(err); + REQUIRE(err == "noerror"); + + auto supportedOps = MakeSupportedOpsChecker(REGOR_ARCH_ETHOSU85, arch); + + SECTION("Test positive") + { + // Validate that both inputs broadcasted is supported by Ethos-U85 + auto op = CreateOperation(OpType::Add, Shape(1, 5, 5, 1), DataType::Int8, Shape(1, 2, 2, 1), DataType::Int8, + Shape(1, 8, 8, 1), DataType::Int8); + REQUIRE(supportedOps->Check(op.get()) == true); + op->Disconnect(); + } + + SECTION("ConstraintTensDtypes") + { + std::set unsupported = { + DataType::UInt48, + DataType::UInt64, + DataType::QInt, + DataType::QInt, + DataType::QInt4, + DataType::QInt8, + DataType::QInt12, + DataType::QInt16, + DataType::QInt32, + DataType::QUInt, + DataType::QUInt4, + DataType::QUInt8, + DataType::QUInt12, + DataType::QUInt16, + DataType::QUInt32, + DataType::Float, + DataType::BFloat16, + DataType::Float16, + DataType::Float32, + DataType::Float64, + DataType::Bool8, + DataType::Complex, + DataType::Complex64, + DataType::Complex128, + DataType::VariablySized, + DataType::String, + DataType::Resource, + DataType::Variant, + }; + std::set supported = { + DataType::UInt8, + DataType::Int8, + DataType::Int16, + DataType::Int32, + DataType::Bool, + DataType::Int64, + }; + for ( auto dtype : unsupported ) + { + auto op = CreateOperation(OpType::Add, Shape(1, 8, 8, 1), dtype, Shape(1, 8, 8, 1), dtype, Shape(1, 8, 8, 1), dtype); + REQUIRE(supportedOps->Check(op.get()) == false); + op->Disconnect(); + } + for ( auto dtype : supported ) + { + auto op = CreateOperation(OpType::Add, Shape(1, 8, 8, 1), dtype, Shape(1, 8, 8, 1), dtype, Shape(1, 8, 8, 1), dtype); + REQUIRE(supportedOps->Check(op.get()) == true); + op->Disconnect(); + } + } +} diff --git a/ethosu/regor/test/util.cpp b/ethosu/regor/test/util.cpp index daedb4f8..59423260 100644 --- a/ethosu/regor/test/util.cpp +++ b/ethosu/regor/test/util.cpp @@ -35,6 +35,7 @@ std::string TestConfig(int macs) { std::string config = "[architecture]\n"; config += fmt::format("macs={}\n", macs); + config += "cores=1\n"; // flash config += "[memory.flash]\n"; config += "name=flash\n"; @@ -159,8 +160,8 @@ std::shared_ptr CreateOperation(OpType opType, TensorUsage ifmUsage, { auto op = std::make_shared(opType); op->SetKernel(std::make_unique(Kernel::UnitKernel())); - op->ConnectInput(ifmUsage, ifm); - op->ConnectOutput(ofmUsage, ofm); + op->ConnectInput(ifmUsage, ifm).Set(Quantization::Unit()); + op->ConnectOutput(ofmUsage, ofm).Set(Quantization::Unit()); return op; } @@ -169,7 +170,7 @@ std::shared_ptr CreateOperation(OpType opType, TensorUsage ifmUsage, TensorUsage ifm2Usage, std::shared_ptr &ifm2, TensorUsage ofmUsage, std::shared_ptr &ofm) { auto op = CreateOperation(opType, ifmUsage, ifm, ofmUsage, ofm); - op->ConnectInput(ifm2Usage, ifm2); + op->ConnectInput(ifm2Usage, ifm2).Set(Quantization::Unit()); return op; } diff --git a/ethosu/regor/tflite/tflite_reader.cpp b/ethosu/regor/tflite/tflite_reader.cpp index 8a97ca42..d9e4e15f 100644 --- a/ethosu/regor/tflite/tflite_reader.cpp +++ b/ethosu/regor/tflite/tflite_reader.cpp @@ -44,35 +44,6 @@ namespace regor { - - -static int64_t Quantize(float value, const Quantization &quant) -{ - float scale = quant.scales.empty() ? 1.0f : float(quant.scales[0].Dequantize()); - int64_t zp = quant.zeroPoints.empty() ? 0 : quant.zeroPoints[0]; - return zp + int64_t(std::round(double(value / scale))); -} - -static void ClampActivation(const std::shared_ptr &operation) -{ - OpType opType = operation->Type(); - Quantization &quant = operation->Output(TensorUsage::OFM)->quantization; - if ( opType == OpType::Relu ) - { - quant.quantMin = {Quantize(0, quant)}; - } - else if ( opType == OpType::Relu6 ) - { - quant.quantMin = {Quantize(0, quant)}; - quant.quantMax = {Quantize(6, quant)}; - } - else if ( opType == OpType::ReluN1To1 ) - { - quant.quantMin = {Quantize(-1, quant)}; - quant.quantMax = {Quantize(1, quant)}; - } -} - static void SetKernel(const std::shared_ptr &operation, const Point2i &size, const Point2i &stride, const Point2i &dilation, tflite::Padding padding, int depthMultiplier = 1) { @@ -133,7 +104,7 @@ const tflite::Model *TfLiteReader::LoadModel(const void *input, size_t size) } void TfLiteReader::LoadGraphs(const uint8_t *input, const tflite::Model *model, - std::vector> &graphs, OptimiserDatabase *optDb, IArchitectureConstraints *constraints) + std::vector> &graphs, OptimiserDatabase *optDb) { assert(model); @@ -323,7 +294,7 @@ void TfLiteReader::LoadGraphs(const uint8_t *input, const tflite::Model *model, // Interpretation of operator options may depend on input/output tensor information, // so the operation must be connected to its tensors before parsing operator options. - ParseOperatorOptions(operation, tflite_operator, optDb, constraints); + ParseOperatorOptions(operation, tflite_operator, optDb); // Set rounding according to reference SetOFMRounding(operation); @@ -382,10 +353,9 @@ void TfLiteReader::LoadGraphs(const uint8_t *input, const tflite::Model *model, } } -void TfLiteReader::LoadGraphs(const void *input, size_t size, std::vector> &graphs, - OptimiserDatabase *optDb, IArchitectureConstraints *constraints) +void TfLiteReader::LoadGraphs(const void *input, size_t size, std::vector> &graphs, OptimiserDatabase *optDb) { - LoadGraphs(reinterpret_cast(input), LoadModel(input, size), graphs, optDb, constraints); + LoadGraphs(reinterpret_cast(input), LoadModel(input, size), graphs, optDb); } std::shared_ptr TfLiteReader::ParseTensor(const tflite::Tensor *tflite_tensor, std::shared_ptr &buffer, @@ -480,8 +450,8 @@ static const T *GetBuiltinOptions(const tflite::Operator *tflite_operator) return options; } -void TfLiteReader::ParseOperatorOptions(const std::shared_ptr &operation, - const tflite::Operator *tflite_operator, OptimiserDatabase *optDb, IArchitectureConstraints *constraints) +void TfLiteReader::ParseOperatorOptions( + const std::shared_ptr &operation, const tflite::Operator *tflite_operator, OptimiserDatabase *optDb) { const auto type = tflite_operator->builtin_options_type(); auto activation_function = tflite::ActivationFunctionType::NONE; @@ -853,33 +823,7 @@ void TfLiteReader::ParseOperatorOptions(const std::shared_ptr &operat break; } operation->SetPassthrough(tflite_operator); - - ExecutionQuery query{}; - bool isValidQuery = true; - try - { - query = OperationToExecQuery(*operation); - } - catch ( std::invalid_argument &e ) - { - LOG_WARN("ExecutionQuery not buildable for operation {}: {}\n", EnumToString(operation->Type()), e.what()) - isValidQuery = false; - } - if ( operation->Type() == OpType::None ) - { - operation->SetPassthroughOp(); - } - else if ( operation->Type() != OpType::Passthrough ) - { - if ( !isValidQuery || !(constraints->CanExecute(query)) ) - { - operation->SetPassthroughOp(); - } - else - { - UnFuseActivation(operation, activation_function, optDb); - } - } + UnFuseActivation(operation, activation_function, optDb); } void TfLiteReader::SetOFMRounding(const std::shared_ptr &operation) @@ -922,160 +866,10 @@ void TfLiteReader::UnFuseActivation(const std::shared_ptr &operation, output_tensor->RemoveWriter(operation); operation->ConnectOutput(TensorUsage::OFM, intermediate_tensor).Set(quantization); activation->ConnectInput(TensorUsage::IFM, intermediate_tensor).Set(quantization); - ClampActivation(activation); if ( optDb ) { optDb->AddOptimised(operation.get(), activation.get()); } } -namespace -{ - -ResizeSupportQuery CalculateResizeSupportQuery(const Operation &operation) -{ - auto ifmConn = operation.Input(TensorUsage::IFM); - auto ofmConn = operation.Output(TensorUsage::OFM); - assert(ifmConn); - assert(ofmConn); - - // Get numerators(n) and denominators(d) for the scale fractions - int width_n = ofmConn->shape.Width(); - int width_d = ifmConn->shape.Width(); - int height_n = ofmConn->shape.Height(); - int height_d = ifmConn->shape.Height(); - int heightOffset = 0; - int widthOffset = 0; - - const tflite::Operator *tflite_operator = static_cast(operation.Passthrough()); - assert(tflite_operator); - bool halfPixelCenters = false; - bool alignCorners = false; - if ( operation.Type() == OpType::ResizeBilinear ) - { - const auto *opt = tflite_operator->builtin_options_as_ResizeBilinearOptions(); - assert(opt); - alignCorners = opt->align_corners(); - halfPixelCenters = opt->half_pixel_centers(); - } - else - { - const auto *opt = tflite_operator->builtin_options_as_ResizeNearestNeighborOptions(); - assert(opt); - alignCorners = opt->align_corners(); - // Use half-pixel-centers if align-corners is false. - // This aligns with reference kernels - halfPixelCenters = !alignCorners || opt->half_pixel_centers(); - } - - // Compute scaling fractions - // align-corners use a scale-factor of (n-1)/(d-1) - if ( alignCorners ) - { - if ( width_d > 1 ) - { - width_n -= 1; - width_d -= 1; - } - if ( height_d > 1 ) - { - height_n -= 1; - height_d -= 1; - } - } - - // reduce scaling fractions with gcd - int gcd_w = std::gcd(width_n, width_d); - width_n = (width_n / gcd_w); - width_d = (width_d / gcd_w); - - int gcd_h = std::gcd(height_n, height_d); - height_n = (height_n / gcd_h); - height_d = (height_d / gcd_h); - - if ( halfPixelCenters ) - { - // make sure fractions are evenly divisible by 2 - width_n = width_n * 2; - width_d = width_d * 2; - height_n = height_n * 2; - height_d = height_d * 2; - // adjust offset for half-pixel-centers - widthOffset = (width_d / 2) - (width_n / 2); - heightOffset = (height_d / 2) - (height_n / 2); - } - - // set up op-support query - ResizeSupportQuery resizeQuery; - resizeQuery.scaleX = {int16_t(width_n), int16_t(width_d)}; - resizeQuery.scaleY = {int16_t(height_n), int16_t(height_d)}; - resizeQuery.offsetX = widthOffset; - resizeQuery.offsetY = heightOffset; - resizeQuery.ifmShape = ifmConn->shape; - resizeQuery.mode = (operation.Type() == OpType::ResizeBilinear) ? ArchResizeMode::Bilinear : ArchResizeMode::Nearest; - return resizeQuery; -} - -} // namespace - -ExecutionQuery TfLiteReader::OperationToExecQuery(const Operation &operation) -{ - ExecutionQuery query{}; - query.opType = operation.Type(); - query.ifmType = operation.IFM(0)->Type(); - query.ofmType = operation.OFM()->Type(); - query.ifmShape = operation.Input(TensorUsage::IFM0)->shape; - if ( operation.Input(TensorUsage::IFM1) ) - { - query.ifm2Type = operation.Input(TensorUsage::IFM1)->tensor->Type(); - query.ifm2Shape = operation.Input(TensorUsage::IFM1)->shape; - } - query.ofmShape = operation.Output(TensorUsage::OFM)->shape; - - switch ( query.opType ) - { - case OpType::LeakyRelu: - { - auto *ifmConn = operation.Input(TensorUsage::IFM0); - auto *ofmConn = operation.Output(TensorUsage::OFM); - query.quantScalingInvalidOrUnequal = !IsScalingValidAndEqual(*ifmConn, *ofmConn); - break; - } - case OpType::Transpose: - { - query.ifmShape = operation.Input(TensorUsage::IFM0)->shape; - query.targetType = OpType::MemoryCopy; - query.transposeType = CalculateTransposeType(operation); - break; - } - case OpType::ReverseV2: - { - query.targetType = OpType::MemoryCopy; - auto paramsConn = operation.Input(TensorUsage::Params); - assert(paramsConn); - assert(paramsConn->tensor->Type() == DataType::Int32); - if ( !paramsConn->tensor->IsConstant() ) - { - query.reverseTypeMask = ReverseType::Dynamic; - } - else - { - // non-dynamic reverseType, we convert it to bitmask - auto view = paramsConn->tensor->View(); - Shape axes = Shape(view.Buffer()->Data(), view.ViewShape().Elements()); - query.reverseTypeMask = ToReverseMask(axes, query.ofmShape.Size()); - } - break; - } - case OpType::ResizeBilinear: - case OpType::ResizeNearestNeighbor: - query.opType = OpType::Resize; - query.resizeQuery = CalculateResizeSupportQuery(operation); - break; - default: - break; - } - return query; -} - } // namespace regor diff --git a/ethosu/regor/tflite/tflite_reader.hpp b/ethosu/regor/tflite/tflite_reader.hpp index 8d3d70c7..1104c2e9 100644 --- a/ethosu/regor/tflite/tflite_reader.hpp +++ b/ethosu/regor/tflite/tflite_reader.hpp @@ -39,20 +39,19 @@ public: TfLiteReader() {} static void LoadGraphs(const void *input, size_t size, std::vector> &graphs, - OptimiserDatabase *optDb, IArchitectureConstraints *constraints); // From buffer + OptimiserDatabase *optDb); // From buffer private: static void LoadGraphs(const uint8_t *input, const tflite::Model *model, std::vector> &graphs, - OptimiserDatabase *optDb, IArchitectureConstraints *constraints); // From model + OptimiserDatabase *optDb); // From model static const tflite::Model *LoadModel(const void *input, size_t size); static std::shared_ptr ParseTensor(const tflite::Tensor *tflite_tensor, std::shared_ptr &buffer, std::unordered_map &tensorQuantization); - static void ParseOperatorOptions(const std::shared_ptr &operation, - const tflite::Operator *tflite_operator, OptimiserDatabase *optDb, IArchitectureConstraints *constraints); + static void ParseOperatorOptions( + const std::shared_ptr &operation, const tflite::Operator *tflite_operator, OptimiserDatabase *optDb); static void SetOFMRounding(const std::shared_ptr &operation); static void UnFuseActivation(const std::shared_ptr &operation, tflite::ActivationFunctionType type, OptimiserDatabase *optDb); static void DefaultOperatorOptions(const std::shared_ptr &operation); - static ExecutionQuery OperationToExecQuery(const Operation &operation); }; } // namespace regor diff --git a/ethosu/regor/tflite/tflite_supported_operators.cpp b/ethosu/regor/tflite/tflite_supported_operators.cpp new file mode 100644 index 00000000..3ceb59f8 --- /dev/null +++ b/ethosu/regor/tflite/tflite_supported_operators.cpp @@ -0,0 +1,180 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "tflite_supported_operators.hpp" + +#include "common/common.hpp" +#include "common/logging.hpp" + +#include "compiler/op_type.hpp" + +namespace regor +{ + +bool TfLiteSupportedOperators::ConstraintOpType(const Operation *op) +{ + OpType opType = op->Type(); + if ( _supportedOpTypes.count(opType) == 0 ) + { + Failure(op, "OpType is not supported", ""); + return false; + } + return true; +} + +bool TfLiteSupportedOperators::ConstraintTensDtypes(const Operation *op) +{ + for ( const auto *list : {&op->Inputs(), &op->Outputs()} ) + { + for ( const auto &item : list->pairs() ) + { + auto usage = item.first; + const auto &conn = item.second; + auto type = conn.tensor->Type(); + if ( (IsIFM(usage) || IsOFM(usage)) && _supportedDataTypes.count(type) == 0 ) + { + Failure(op, fmt::format("Operation has tensor with unsupported DataType {}", DataTypeToString(type)), ""); + return false; + } + } + } + return true; +} + +bool TfLiteSupportedOperators::ConstraintTensQuantized(const Operation *op) +{ + const char *constraint = "Input(s), Output and Weight tensors must have quantization parameters"; + // Exceptions for this check + switch ( op->Type() ) + { + case OpType::ArgMax: + case OpType::MirrorPad: + case OpType::Quantize: + case OpType::Shape: + case OpType::Transpose: + case OpType::GatherNd: + case OpType::GatherV2: + return true; + default: + break; + } + for ( const auto *list : {&op->Inputs(), &op->Outputs()} ) + { + for ( const auto &item : list->pairs() ) + { + auto usage = item.first; + const auto &conn = item.second; + if ( IsIFM(usage) || IsOFM(usage) || usage == TensorUsage::Weights ) + { + const Quantization &quant = conn.quantization; + if ( quant.scales.empty() || quant.zeroPoints.empty() ) + { + Failure(op, fmt::format("Operation has tensor {} with missing quantization parameters", conn.tensor->Name()), constraint); + return false; + } + } + } + } + return true; +} + +void TfLiteSupportedOperators::Failure(const Operation *op, const std::string &message, const std::string &constraint) +{ + assert(op); + auto ofmConn = op->Output(TensorUsage::OFM); + const char *name = "N/A"; + if ( ofmConn && ofmConn->tensor ) + { + name = ofmConn->tensor->Name().c_str(); + } + auto tfLiteType = TfLiteMapping::OpTypeToBuiltinOperator(op->Type()); + assert(message.size() || constraint.size()); + LOG_WARN("\nWarning (supported operators) operator:{} ofm:{}\n", TfLiteMapping::BuiltinOperatorToString(tfLiteType), name); + if ( message.size() ) + { + LOG_WARN("Reason: {}\n", message); + } + if ( constraint.size() ) + { + LOG_WARN("Constraint: {}\n", constraint); + } +} + +TfLiteSupportedOperators::TfLiteSupportedOperators(IArchitectureConstraints *constraints) : + _archConstraints(constraints) +{ + _genericChecks = { + &TfLiteSupportedOperators::ConstraintOpType, + &TfLiteSupportedOperators::ConstraintTensDtypes, + &TfLiteSupportedOperators::ConstraintTensQuantized, + }; +} + +namespace +{ +void DisconnectActivation(std::shared_ptr op) +{ + assert(TfLiteMapping::CanFuseActivationFunction(op.get())); + // Op originally had a fused activation + assert(op->Outputs().size() == 1); + assert(op->OFM()->Readers().size() == 1); + auto activation = op->OFM()->Readers().front(); + auto actOfm = activation->Output(TensorUsage::OFM); + assert(actOfm); + // bypass and disconnect the activation + op->CopyOutput(TensorUsage::OFM, *actOfm); + activation->SetPassthroughOp(); + activation->Disconnect(); +} +} // namespace + +void TfLiteSupportedOperators::Process(Graph *graph) +{ + std::vector> operatorList; + graph->GetAllOperations(operatorList); + for ( auto &op : operatorList ) + { + if ( op->Type() == OpType::Passthrough ) + { + // don't check passthrough ops + continue; + } + if ( !Check(op.get()) ) + { + if ( TfLiteMapping::CanFuseActivationFunction(op.get()) ) + { + // op originally had a fused activation + // disconnect it from the graph as it will be handled by CPU + DisconnectActivation(op); + } + else if ( op->IFM(0)->Writers().size() == 1 ) + { + auto pred = op->IFM(0)->Writers().front(); + if ( TfLiteMapping::CanFuseActivationFunction(pred.get()) ) + { + // op is an activation function, disconnect op and set pred to passthrough + DisconnectActivation(pred); + pred->SetPassthroughOp(); + } + } + op->SetPassthroughOp(); + } + } +} + +} // namespace regor diff --git a/ethosu/regor/tflite/tflite_supported_operators.hpp b/ethosu/regor/tflite/tflite_supported_operators.hpp new file mode 100644 index 00000000..6aaacc9a --- /dev/null +++ b/ethosu/regor/tflite/tflite_supported_operators.hpp @@ -0,0 +1,53 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "architecture/architecture_constraints.hpp" +#include "compiler/graph.hpp" +#include "compiler/operation.hpp" +#include "tflite_mapping.hpp" + +namespace regor +{ +class TfLiteSupportedOperators +{ + using OperatorCheck = bool (TfLiteSupportedOperators::*)(const Operation *); + +protected: + std::vector _genericChecks; + IArchitectureConstraints *_archConstraints; + std::unordered_set _supportedOpTypes; + std::unordered_set _supportedDataTypes; + +public: + TfLiteSupportedOperators(IArchitectureConstraints *constraints); + virtual ~TfLiteSupportedOperators() = default; + // process graph and set passthrough for unsupported operators + void Process(Graph *graph); + virtual bool Check(const Operation *) = 0; + +protected: + static void Failure(const Operation *op, const std::string &message, const std::string &constraint); + +private: + bool ConstraintOpType(const Operation *op); + bool ConstraintTensDtypes(const Operation *op); + bool ConstraintTensQuantized(const Operation *op); +}; +} // namespace regor diff --git a/ethosu/regor/tflite/tflite_supported_operators_u55.cpp b/ethosu/regor/tflite/tflite_supported_operators_u55.cpp new file mode 100644 index 00000000..cfedfb3b --- /dev/null +++ b/ethosu/regor/tflite/tflite_supported_operators_u55.cpp @@ -0,0 +1,154 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "tflite_supported_operators_u55.hpp" + +#include "common/common.hpp" +#include "common/logging.hpp" + +#include "compiler/op_type.hpp" + +namespace regor +{ + +TfLiteSupportedOperatorsU55::TfLiteSupportedOperatorsU55(IArchitectureConstraints *constraints) : + TfLiteSupportedOperators(constraints) +{ + _supportedOpTypes = { + // clang-format off + OpType::Add, + OpType::AvgPool, + OpType::Concat, + OpType::Conv2D, + OpType::DepthwiseConv2D, + OpType::FullyConnected, + OpType::Sigmoid, + OpType::MaxPool, + OpType::Mul, + OpType::Relu, + OpType::ReluN1To1, + OpType::Relu6, + OpType::Reshape, + OpType::Softmax, + OpType::Tanh, + OpType::Pad, + OpType::BatchToSpaceND, + OpType::SpaceToBatchND, + OpType::Transpose, + OpType::Mean, + OpType::Sub, + OpType::StridedSlice, + OpType::Exp, + OpType::Split, + OpType::Prelu, + OpType::Maximum, + OpType::ArgMax, + OpType::Minimum, + OpType::PadV2, + OpType::Slice, + OpType::TransposeConv2D, + OpType::Tile, + OpType::ReduceSum, + OpType::Rsqrt, + OpType::Pack, + OpType::Unpack, + OpType::LeakyRelu, + OpType::MirrorPad, + OpType::Abs, + OpType::SplitV, + OpType::Quantize, + OpType::HardSwish, + // clang-format on + }; + _supportedDataTypes = { + // clang-format off + DataType::UInt8, + DataType::Int8, + DataType::Int16, + DataType::Int32 + // clang-format on + }; + _checks = { + &TfLiteSupportedOperatorsU55::ConstraintBroadcastShapes, + &TfLiteSupportedOperatorsU55::ConstraintReverse, + }; +} + +bool TfLiteSupportedOperatorsU55::Check(const Operation *op) +{ + for ( auto &check : _genericChecks ) + { + if ( !((this->*check)(op)) ) return false; + } + for ( auto &check : _checks ) + { + if ( !((this->*check)(op)) ) return false; + } + return true; +} + + +bool TfLiteSupportedOperatorsU55::ConstraintBroadcastShapes(const Operation *op) +{ + const char *constraint = "One input-tensor must match the shape of the output-tensor."; + if ( !IsElementwise(op->Type()) ) + { + // only applied to elementwise ops + return true; + } + auto ifmConn = op->Input(TensorUsage::IFM); + auto ifm2Conn = op->Input(TensorUsage::IFM1); + auto ofmConn = op->Output(TensorUsage::OFM); + assert(ifmConn); + assert(ofmConn); + Shape ifmShape = ifmConn->shape; + Shape ofmShape = ofmConn->shape; + Shape ifm2Shape = ifm2Conn ? ifm2Conn->shape : Shape(); + if ( ifmShape != ofmShape && (ifm2Shape == false || ifm2Shape != ofmShape) ) + { + Failure(op, "Operation has invalid broadcast.", constraint); + return false; + } + return true; +} + +bool TfLiteSupportedOperatorsU55::ConstraintReverse(const Operation *op) +{ + if ( op->Type() != OpType::Reverse && op->Type() != OpType::ReverseV2 ) + { + return true; + } + auto params = op->Input(TensorUsage::Params); + assert(params); + if ( !params->tensor->IsConstant() ) + { + return false; + } + auto ofmConn = op->Output(TensorUsage::OFM); + assert(ofmConn); + auto view = params->tensor->View(); + Shape axes = Shape(view.Buffer()->Data(), view.ViewShape().Elements()); + auto mask = ToReverseMask(axes, ofmConn->shape.Size()); + if ( mask != ReverseType::None ) + { + Failure(op, fmt::format("Reverse is not supported"), ""); + return false; + } + return true; +} +} // namespace regor diff --git a/ethosu/regor/tflite/tflite_supported_operators_u55.hpp b/ethosu/regor/tflite/tflite_supported_operators_u55.hpp new file mode 100644 index 00000000..f3e483d0 --- /dev/null +++ b/ethosu/regor/tflite/tflite_supported_operators_u55.hpp @@ -0,0 +1,46 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "architecture/architecture_constraints.hpp" +#include "compiler/operation.hpp" +#include "tflite_supported_operators.hpp" + +#include +#include + +namespace regor +{ + +class TfLiteSupportedOperatorsU55 : public TfLiteSupportedOperators +{ + using OperatorCheck = bool (TfLiteSupportedOperatorsU55::*)(const Operation *); + +private: + std::vector _checks; + +public: + TfLiteSupportedOperatorsU55(IArchitectureConstraints *constraints); + bool Check(const Operation *) override; + +private: + bool ConstraintBroadcastShapes(const Operation *op); + bool ConstraintReverse(const Operation *op); +}; +} // namespace regor diff --git a/ethosu/regor/tflite/tflite_supported_operators_u85.cpp b/ethosu/regor/tflite/tflite_supported_operators_u85.cpp new file mode 100644 index 00000000..918f81d1 --- /dev/null +++ b/ethosu/regor/tflite/tflite_supported_operators_u85.cpp @@ -0,0 +1,296 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "tflite_supported_operators_u85.hpp" + +#include "common/common.hpp" +#include "common/logging.hpp" + +#include "compiler/op_type.hpp" + +#include + +namespace regor +{ + +TfLiteSupportedOperatorsU85::TfLiteSupportedOperatorsU85(IArchitectureConstraints *constraints) : + TfLiteSupportedOperators(constraints) +{ + _supportedOpTypes = { + // clang-format off + OpType::Add, + OpType::AvgPool, + OpType::Concat, + OpType::Conv2D, + OpType::DepthwiseConv2D, + OpType::FullyConnected, + OpType::Sigmoid, + OpType::MaxPool, + OpType::Mul, + OpType::Relu, + OpType::ReluN1To1, + OpType::Relu6, + OpType::Reshape, + OpType::ResizeBilinear, + OpType::Softmax, + OpType::Tanh, + OpType::Pad, + OpType::GatherV2, + OpType::Transpose, + OpType::Mean, + OpType::Sub, + OpType::Div, + OpType::StridedSlice, + OpType::Exp, + OpType::Split, + OpType::Cast, + OpType::Prelu, + OpType::Maximum, + OpType::ArgMax, + OpType::Minimum, + OpType::PadV2, + OpType::Select, + OpType::Greater, + OpType::GreaterEqual, + OpType::LessEqual, + OpType::Slice, + OpType::TransposeConv2D, + OpType::Tile, + OpType::Equal, + OpType::NotEqual, + OpType::ReduceSum, + OpType::Rsqrt, + OpType::ReduceMax, + OpType::Pack, + OpType::Unpack, + OpType::ReduceMin, + OpType::ReduceAny, + OpType::LogicalOr, + OpType::LogicalAnd, + OpType::LogicalNot, + OpType::ResizeNearestNeighbor, + OpType::LeakyRelu, + OpType::MirrorPad, + OpType::Abs, + OpType::SplitV, + OpType::ReverseV2, + OpType::GatherNd, + OpType::SpaceToBatchND, + OpType::BatchToSpaceND, + OpType::Quantize, + OpType::HardSwish, + OpType::SelectV2, + OpType::ScatterNd, + OpType::SelectV2, + OpType::BatchMatMul, + OpType::ReduceAll, + // clang-format on + }; + _supportedDataTypes = { + // clang-format off + DataType::UInt8, + DataType::Int8, + DataType::Int16, + DataType::Int32, + DataType::Int64, + DataType::Bool + // clang-format on + }; + _checks = { + &TfLiteSupportedOperatorsU85::ConstraintResizeCommon, + &TfLiteSupportedOperatorsU85::ConstraintResizeBilinear, + }; +} + +bool TfLiteSupportedOperatorsU85::Check(const Operation *op) +{ + for ( auto &check : _genericChecks ) + { + if ( !((this->*check)(op)) ) return false; + } + for ( auto &check : _checks ) + { + if ( !((this->*check)(op)) ) return false; + } + return true; +} + +bool TfLiteSupportedOperatorsU85::ConstraintResizeCommon(const Operation *op) +{ + const char *constraint = + "ALIGN_CORNERS and HALF_PIXEL_CENTERS are mutually exclusive.\n" + "if ALIGN_CORNERS:\n" + "\tScale-factor can be maximum 2048\n" + "else if HALF_PIXEL_CENTERS:\n" + "\tScale-factor can be maximum 1024\n"; + OpType opType = op->Type(); + if ( opType != OpType::ResizeBilinear && opType != OpType::ResizeNearestNeighbor ) + { + return true; + } + auto ifmConn = op->Input(TensorUsage::IFM0); + auto ofmConn = op->Output(TensorUsage::OFM); + assert(ifmConn); + assert(ofmConn); + int width_n = ofmConn->shape.Width(); + int width_d = ifmConn->shape.Width(); + int height_n = ofmConn->shape.Height(); + int height_d = ifmConn->shape.Height(); + bool halfPixelCenters = false; + bool alignCorners = false; + const tflite::Operator *passthrough = static_cast(op->Passthrough()); + assert(passthrough); + + if ( opType == OpType::ResizeBilinear ) + { + const auto *opt = passthrough->builtin_options_as_ResizeBilinearOptions(); + assert(opt); + alignCorners = opt->align_corners(); + halfPixelCenters = opt->half_pixel_centers(); + } + else + { + assert(opType == OpType::ResizeNearestNeighbor); + const auto *opt = passthrough->builtin_options_as_ResizeNearestNeighborOptions(); + assert(opt); + alignCorners = opt->align_corners(); + // Use half-pixel-centers if align-corners is false. + // This aligns with reference kernels + halfPixelCenters = !alignCorners || opt->half_pixel_centers(); + } + + if ( alignCorners && halfPixelCenters ) + { + Failure(op, "Operation with both align_corners=true and half_pixel_centers=true (these are mutually exclusive)", constraint); + return false; + } + + if ( alignCorners ) + { + if ( width_d > 1 ) + { + width_n -= 1; + width_d -= 1; + } + if ( height_d > 1 ) + { + height_n -= 1; + height_d -= 1; + } + } + + auto ConstrainScaleFactor = [&](int num, int den, const char *axis) -> bool + { + if ( num == 0 || den == 0 ) + { + Failure(op, fmt::format("unsupported {} scale-factor ({}/{})", axis, num, den), constraint); + return false; + } + int scaleFactor = num / den; + if ( halfPixelCenters && scaleFactor > 1024 ) + { + Failure(op, fmt::format("halfPixelCenters {} scaleFactor exceeds 1024: {}/{}", axis, num, den), constraint); + return false; + } + else if ( scaleFactor > 2048 ) + { + Failure(op, fmt::format("{} scaleFactor exceeds 2048: {}/{}", axis, num, den), constraint); + return false; + } + return true; + }; + if ( !(ConstrainScaleFactor(width_n, width_d, "width") && ConstrainScaleFactor(height_n, height_d, "height")) ) + { + return false; + } + return true; +} + +bool TfLiteSupportedOperatorsU85::ConstraintResizeBilinear(const Operation *op) +{ + const char *constraint = + "if IFM HxW > 1x1\n" + "\tand ALIGN_CORNERS:\n" + "\t\tOFM W-1 and H-1 must be a power-of-two integer-multiple of IFM W-1 and H-1\n" + "\tor HALF_PIXEL_CENTERS:\n" + "\t\tOFM W and H must be a power-of-two integer-multiple of IFM W and H\n"; + OpType opType = op->Type(); + if ( opType != OpType::ResizeBilinear ) + { + return true; + } + auto ifmConn = op->Input(TensorUsage::IFM0); + auto ofmConn = op->Output(TensorUsage::OFM); + assert(ifmConn); + assert(ofmConn); + int width_n = ofmConn->shape.Width(); + int width_d = ifmConn->shape.Width(); + int height_n = ofmConn->shape.Height(); + int height_d = ifmConn->shape.Height(); + bool halfPixelCenters = false; + bool alignCorners = false; + const tflite::Operator *passthrough = static_cast(op->Passthrough()); + assert(passthrough); + + if ( width_d == 1 && height_d == 1 ) + { + return true; + } + + const auto *opt = passthrough->builtin_options_as_ResizeBilinearOptions(); + assert(opt); + alignCorners = opt->align_corners(); + halfPixelCenters = opt->half_pixel_centers(); + + if ( alignCorners ) + { + if ( width_d > 1 ) + { + width_n -= 1; + width_d -= 1; + } + if ( height_d > 1 ) + { + height_n -= 1; + height_d -= 1; + } + } + + auto ConstrainScaleFactor = [&](int num, int den, const char *axis) -> bool + { + assert(num > 0 && den > 0); + int scaleFactor = num / den; + if ( num % den != 0 ) + { + Failure(op, fmt::format("{} scale-factor must be integer. scale-factor: ({}/{})", axis, num, den), constraint); + return false; + } + if ( !IsPowerOfTwo(scaleFactor) ) + { + Failure(op, fmt::format("{} scale-factor must be power of two. scale-factor: ({}/{})", axis, num, den), constraint); + return false; + } + return true; + }; + if ( !(ConstrainScaleFactor(width_n, width_d, "width") && ConstrainScaleFactor(height_n, height_d, "height")) ) + { + return false; + } + return true; +} +} // namespace regor diff --git a/ethosu/regor/tflite/tflite_supported_operators_u85.hpp b/ethosu/regor/tflite/tflite_supported_operators_u85.hpp new file mode 100644 index 00000000..5a76570e --- /dev/null +++ b/ethosu/regor/tflite/tflite_supported_operators_u85.hpp @@ -0,0 +1,45 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "architecture/architecture_constraints.hpp" +#include "compiler/operation.hpp" +#include "tflite_supported_operators.hpp" + +#include + +namespace regor +{ + +class TfLiteSupportedOperatorsU85 : public TfLiteSupportedOperators +{ + using OperatorCheck = bool (TfLiteSupportedOperatorsU85::*)(const Operation *); + +private: + std::vector _checks; + +public: + TfLiteSupportedOperatorsU85(IArchitectureConstraints *constraints); + bool Check(const Operation *) override; + +private: + bool ConstraintResizeCommon(const Operation *op); + bool ConstraintResizeBilinear(const Operation *op); +}; +} // namespace regor -- GitLab