From f0422e7040bfbd4fff8606b7811f5757aa5ef19e Mon Sep 17 00:00:00 2001 From: Alexander Bengtsson Date: Fri, 21 Feb 2025 14:48:07 +0100 Subject: [PATCH] MLBEDSW-10414: Port Vela supported-operator checks (2) Add the following supported operator checks: ConstraintWeightsPrecision Weights must be in 8-bit precision ConstraintWeightSum Validate weight sum to avoid overflow ConstraintBias Bias tensors must be 1D Bias tensors must be constant Bias tensors must have 32- or 64-bit precision Constrain bias range for 64-bit precision Change-Id: Id90dfd7b7f7e6ea614812a5810e20c235229bba2 Signed-off-by: Alexander Bengtsson --- .../test/test_tflite_supported_operators.cpp | 119 ++++++++++++++ ethosu/regor/test/util.cpp | 5 +- ethosu/regor/test/util.hpp | 2 +- .../tflite/tflite_supported_operators.cpp | 145 ++++++++++++++++++ .../tflite/tflite_supported_operators.hpp | 6 + .../tflite/tflite_supported_operators_u55.cpp | 3 + .../tflite/tflite_supported_operators_u85.cpp | 3 + 7 files changed, 281 insertions(+), 2 deletions(-) diff --git a/ethosu/regor/test/test_tflite_supported_operators.cpp b/ethosu/regor/test/test_tflite_supported_operators.cpp index 8edd6fae..3493e3ff 100644 --- a/ethosu/regor/test/test_tflite_supported_operators.cpp +++ b/ethosu/regor/test/test_tflite_supported_operators.cpp @@ -61,6 +61,7 @@ std::shared_ptr CreateOperation(OpType opType, Shape ifmShape, DataTy if ( opType == OpType::Conv2D ) { auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8); + weights->SetAxisOrder(AxisOrder::OHWI); op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); } return op; @@ -86,6 +87,7 @@ TEST_CASE("Supported operators Common") auto op = CreateOperation(OpType::Conv2D, Shape(1, 8, 8, 1), DataType::Int8, Shape(1, 8, 8, 1), DataType::Int8); std::vector values = {1}; auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::OHWI); op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); // Regular op should pass REQUIRE(supportedOps->Check(op.get()) == true); @@ -127,6 +129,7 @@ TEST_CASE("Supported operators Common") auto op = CreateOperation(OpType::FullyConnected, Shape(1, 2, 2, 1), DataType::Int8, Shape(1, 2, 1, 1), DataType::Int8); std::vector values = {1, 1, 1, 1, 1, 1, 1, 1}; auto weights = CreateTensor("weights", Shape(4, 1, 1, 2), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::OHWI); op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); REQUIRE(supportedOps->Check(op.get()) == true); op->Input(TensorUsage::Weights)->tensor->Reshape(Shape(2, 2, 1, 2)); @@ -159,6 +162,122 @@ TEST_CASE("Supported operators Common") REQUIRE(supportedOps->Check(op.get()) == false); op->Disconnect(); } + + SECTION("ConstraintWeightsPrecision") + { + auto op = CreateOperation(OpType::DepthwiseConv2D, Shape(1, 5, 5, 1), DataType::Int8, Shape(1, 5, 5, 1), DataType::Int8); + { + std::vector values(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + std::vector values(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::UInt8, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + std::vector values(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int16, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + std::vector values(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int32, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + op->Disconnect(); + } + + SECTION("ConstraintWeightSum") + { + auto op = CreateOperation(OpType::DepthwiseConv2D, Shape(1, 1, 32768, 2), DataType::Int8, Shape(1, 1, 1, 2), DataType::Int8); + static const int64_t MAX_SUM = (1 << 16) * 127; + { + // Verify supported sum of weights + std::vector values((1 << 16) * 2, 127); + auto weights = CreateTensor("weights", Shape(1, 1, (1 << 16), 2), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + // Verify unsupported sum of weights + std::vector values((1 << 16) * 2, 127); + auto weights = CreateTensor("weights", Shape(1, 1, (1 << 16), 2), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::OHWI); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + op->Disconnect(); + } + + SECTION("ConstraintBias") + { + auto op = CreateOperation(OpType::DepthwiseConv2D, Shape(1, 5, 5, 1), DataType::Int8, Shape(1, 5, 5, 1), DataType::Int8); + std::vector wValues(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8, std::move(wValues)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + { + // Bias must be const + auto bias = CreateTensor("bias", Shape(1), DataType::Int32); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + // Bias must be 1D + std::vector values(1, 1); + auto bias = CreateTensor("bias", Shape(1, 1, 1, 1), DataType::Int32, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + // Bias can't be 8bit + std::vector values(1, 1); + auto bias = CreateTensor("bias", Shape(1), DataType::Int8, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + // Bias can't be 16bit + std::vector values(1, 1); + auto bias = CreateTensor("bias", Shape(1), DataType::Int16, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + // Bias can be 32 bit + std::vector values(1, std::numeric_limits::max()); + auto bias = CreateTensor("bias", Shape(1), DataType::Int32, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + // Bias can be 40 bit + std::vector values(1, (1LL << 40) - 1); + auto bias = CreateTensor("bias", Shape(1), DataType::Int64, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + // Bias can't be >40 bit + std::vector values(1, std::numeric_limits::max()); + auto bias = CreateTensor("bias", Shape(1), DataType::Int64, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + op->Disconnect(); + } } TEST_CASE("Supported operators EthosU55") diff --git a/ethosu/regor/test/util.cpp b/ethosu/regor/test/util.cpp index 59423260..1bd614ca 100644 --- a/ethosu/regor/test/util.cpp +++ b/ethosu/regor/test/util.cpp @@ -132,7 +132,7 @@ std::shared_ptr CreateTensor(const std::string &name, const Shape &stora } // Create a Const Tensor -std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, unsigned value) +std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, int64_t value) { switch ( dtype ) { @@ -148,6 +148,9 @@ std::shared_ptr CreateTensor(const std::string &name, const Shape &stora case DataType::Int32: return CreateTensor(name, storageShape, dtype, std::vector(storageShape.Elements(), int32_t(value))); break; + case DataType::Int64: + return CreateTensor(name, storageShape, dtype, std::vector(storageShape.Elements(), int64_t(value))); + break; default: assert(false); return CreateTensor(name, storageShape, dtype, std::vector(storageShape.Elements(), int8_t(value))); diff --git a/ethosu/regor/test/util.hpp b/ethosu/regor/test/util.hpp index 854740bb..8e605b36 100644 --- a/ethosu/regor/test/util.hpp +++ b/ethosu/regor/test/util.hpp @@ -61,7 +61,7 @@ std::shared_ptr CreateTensor(const std::string &name, const Shape &stora // Create a Const Tensor template std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, std::vector &&values); -std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, unsigned value); +std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, int64_t value); // Create a Operation with unary input std::shared_ptr CreateOperation(OpType opType, TensorUsage ifmUsage, std::shared_ptr &ifm, TensorUsage ofmUsage, std::shared_ptr &ofm); diff --git a/ethosu/regor/tflite/tflite_supported_operators.cpp b/ethosu/regor/tflite/tflite_supported_operators.cpp index ac0e8844..07a5184e 100644 --- a/ethosu/regor/tflite/tflite_supported_operators.cpp +++ b/ethosu/regor/tflite/tflite_supported_operators.cpp @@ -253,6 +253,145 @@ bool TfLiteSupportedOperators::ConstraintMatchingQuantization(const Operation *o return true; } +bool TfLiteSupportedOperators::ConstraintWeightsPrecision(const Operation *op) +{ + const char *constraint = "Weight tensors must be 8-bit precision"; + const auto wconn = op->Input(TensorUsage::Weights); + if ( !wconn ) + { + return true; + } + const auto type = wconn->tensor->Type(); + if ( DataTypeSizeBits(type) != 8 ) + { + Failure(op, fmt::format("Weights tensor with precision: {}", DataTypeToString(type)), constraint); + return false; + } + return true; +} + +bool TfLiteSupportedOperators::ConstraintWeightSum(const Operation *op) +{ + std::string constraint = fmt::format( + "The sum of absolute weights cannot exceed:\n" + "\t{} for 8-bit IFM\n" + "\t{} for 16-bit IFM", + _maxWeightSum8Bit, _maxWeightSum16Bit); + + auto wConn = op->Input(TensorUsage::Weights); + auto ifmConn = op->Input(TensorUsage::IFM); + if ( !wConn || !ifmConn ) + { + return true; + } + if ( !wConn->tensor->IsConstant() ) + { + return true; + } + + auto view = wConn->tensor->View(); + auto zeroPoints = wConn->quantization.zeroPoints; + auto ifmType = ifmConn->tensor->Type(); + int ifmBits = DataTypeSizeBits(ifmType); + int64_t maxWeightSum = ifmBits == 8 ? _maxWeightSum8Bit : _maxWeightSum16Bit; + auto reader = view.Values(wConn->tensor->Type()); + AxisOrder order = wConn->tensor->AxisOrder(); + Shape readShape = wConn->tensor->StorageShape(); + assert(readShape.Size() == 4); + assert(order == AxisOrder::OHWI || order == AxisOrder::IHWO); + + int outChannels = readShape.Depth(); + int inChannels = readShape.Batch(); + if ( order == AxisOrder::OHWI ) + { + std::swap(outChannels, inChannels); + } + // abort early if the readShape of the weights tensor guarantees no overflow. + if ( (255 * readShape.Elements64() / outChannels) < maxWeightSum ) + { + return true; + } + // Accumulate the weights in slices of output-channels + // Fail if any slice overflows maxWeightSum + for ( int out = 0; out < outChannels; out++ ) + { + int64_t zeroPoint = 0; + if ( !zeroPoints.empty() ) + { + zeroPoint = zeroPoints.size() > 1 ? zeroPoints[out] : zeroPoints[0]; + } + int64_t sum = 0; + for ( int in = 0; in < inChannels; in++ ) + { + for ( int h = 0; h < readShape.Height(); h++ ) + { + for ( int w = 0; w < readShape.Width(); w++ ) + { + int64_t v; + if ( order == AxisOrder::OHWI ) + { + v = reader[{out, h, w, in}]; + } + else + { + v = reader[{in, h, w, out}]; + } + sum += std::abs(v - zeroPoint); + } + } + } + if ( sum > maxWeightSum ) + { + Failure(op, fmt::format("The absolute sum of weight-tensor elements: {} exceeds {}", sum, maxWeightSum), constraint); + return false; + } + } + return true; +} + +bool TfLiteSupportedOperators::ConstraintBias(const Operation *op) +{ + auto bConn = op->Input(TensorUsage::Scales); + if ( !bConn ) + { + return true; + } + int biasDim = bConn->shape.Size(); + + if ( biasDim > 1 ) + { + Failure(op, fmt::format("Operation has {}D bias shape.", biasDim), "The bias tensor shape must be 1D"); + return false; + } + if ( !bConn->tensor->IsConstant() ) + { + Failure(op, "Operation has non-constant bias tensor.", "The bias tensor must be constant"); + return false; + } + auto type = bConn->tensor->Type(); + if ( type != DataType::Int32 && type != DataType::Int64 ) + { + Failure(op, fmt::format("Operation has bias with type:{}", DataTypeToString(type)), "The bias tensor precision must be Int32 or Int64"); + return false; + } + if ( type == DataType::Int64 ) + { + // read bias values + auto view = bConn->tensor->View(); + auto values = view.Values(); + for ( int64_t bias : values ) + { + if ( bias > _maxBias ) + { + std::string constraint = fmt::format("Int64 bias must be smaller than {}", _maxBias); + Failure(op, fmt::format("Bias is out of range: {} > {}", bias, _maxBias), constraint); + return false; + } + } + } + return true; +} + void TfLiteSupportedOperators::Failure(const Operation *op, const std::string &message, const std::string &constraint) { assert(op); @@ -278,6 +417,9 @@ void TfLiteSupportedOperators::Failure(const Operation *op, const std::string &m TfLiteSupportedOperators::TfLiteSupportedOperators(IArchitectureConstraints *constraints) : _archConstraints(constraints) { + _maxWeightSum8Bit = 0; + _maxWeightSum16Bit = 0; + _maxBias = 0; _genericChecks = { &TfLiteSupportedOperators::ConstraintOpType, &TfLiteSupportedOperators::ConstraintTensDtypes, @@ -289,6 +431,9 @@ TfLiteSupportedOperators::TfLiteSupportedOperators(IArchitectureConstraints *con &TfLiteSupportedOperators::ConstraintTensQuantized, &TfLiteSupportedOperators::ConstraintPerAxisQuant, &TfLiteSupportedOperators::ConstraintMatchingQuantization, + &TfLiteSupportedOperators::ConstraintWeightsPrecision, + &TfLiteSupportedOperators::ConstraintWeightSum, + &TfLiteSupportedOperators::ConstraintBias, }; } diff --git a/ethosu/regor/tflite/tflite_supported_operators.hpp b/ethosu/regor/tflite/tflite_supported_operators.hpp index 4d062f97..5b3ea01f 100644 --- a/ethosu/regor/tflite/tflite_supported_operators.hpp +++ b/ethosu/regor/tflite/tflite_supported_operators.hpp @@ -34,6 +34,9 @@ protected: IArchitectureConstraints *_archConstraints; std::unordered_set _supportedOpTypes; std::unordered_set _supportedDataTypes; + int64_t _maxWeightSum8Bit; + int64_t _maxWeightSum16Bit; + int64_t _maxBias; public: TfLiteSupportedOperators(IArchitectureConstraints *constraints); @@ -57,5 +60,8 @@ private: bool ConstraintPerAxisQuant(const Operation *op); bool ConstraintMatchingQuantization(const Operation *op); bool ConstraintDepthMultiplier(const Operation *op); + bool ConstraintWeightsPrecision(const Operation *op); + bool ConstraintWeightSum(const Operation *op); + bool ConstraintBias(const Operation *op); }; } // namespace regor diff --git a/ethosu/regor/tflite/tflite_supported_operators_u55.cpp b/ethosu/regor/tflite/tflite_supported_operators_u55.cpp index f3dc242c..33ba97fa 100644 --- a/ethosu/regor/tflite/tflite_supported_operators_u55.cpp +++ b/ethosu/regor/tflite/tflite_supported_operators_u55.cpp @@ -87,6 +87,9 @@ TfLiteSupportedOperatorsU55::TfLiteSupportedOperatorsU55(IArchitectureConstraint DataType::Int32 // clang-format on }; + _maxWeightSum8Bit = 127 * (1 << 16); + _maxWeightSum16Bit = 127 * (1 << 16); + _maxBias = (1LL << 40) - 1; _checks = { &TfLiteSupportedOperatorsU55::ConstraintBroadcastShapes, &TfLiteSupportedOperatorsU55::ConstraintReverse, diff --git a/ethosu/regor/tflite/tflite_supported_operators_u85.cpp b/ethosu/regor/tflite/tflite_supported_operators_u85.cpp index a5db7132..f0575625 100644 --- a/ethosu/regor/tflite/tflite_supported_operators_u85.cpp +++ b/ethosu/regor/tflite/tflite_supported_operators_u85.cpp @@ -115,6 +115,9 @@ TfLiteSupportedOperatorsU85::TfLiteSupportedOperatorsU85(IArchitectureConstraint DataType::Bool8 // clang-format on }; + _maxWeightSum8Bit = 127 * (1 << 16); + _maxWeightSum16Bit = 127 * (1 << 24); + _maxBias = (1LL << 48) - 1; _checks = { &TfLiteSupportedOperatorsU85::ConstraintResizeCommon, &TfLiteSupportedOperatorsU85::ConstraintResizeBilinear, -- GitLab