diff --git a/ethosu/regor/test/test_tflite_supported_operators.cpp b/ethosu/regor/test/test_tflite_supported_operators.cpp index 8edd6fae7a5845db94bc01050d9a6be508870111..3493e3ff85a24ddb684ed5f5c5ce5583a2786452 100644 --- a/ethosu/regor/test/test_tflite_supported_operators.cpp +++ b/ethosu/regor/test/test_tflite_supported_operators.cpp @@ -61,6 +61,7 @@ std::shared_ptr CreateOperation(OpType opType, Shape ifmShape, DataTy if ( opType == OpType::Conv2D ) { auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8); + weights->SetAxisOrder(AxisOrder::OHWI); op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); } return op; @@ -86,6 +87,7 @@ TEST_CASE("Supported operators Common") auto op = CreateOperation(OpType::Conv2D, Shape(1, 8, 8, 1), DataType::Int8, Shape(1, 8, 8, 1), DataType::Int8); std::vector values = {1}; auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::OHWI); op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); // Regular op should pass REQUIRE(supportedOps->Check(op.get()) == true); @@ -127,6 +129,7 @@ TEST_CASE("Supported operators Common") auto op = CreateOperation(OpType::FullyConnected, Shape(1, 2, 2, 1), DataType::Int8, Shape(1, 2, 1, 1), DataType::Int8); std::vector values = {1, 1, 1, 1, 1, 1, 1, 1}; auto weights = CreateTensor("weights", Shape(4, 1, 1, 2), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::OHWI); op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); REQUIRE(supportedOps->Check(op.get()) == true); op->Input(TensorUsage::Weights)->tensor->Reshape(Shape(2, 2, 1, 2)); @@ -159,6 +162,122 @@ TEST_CASE("Supported operators Common") REQUIRE(supportedOps->Check(op.get()) == false); op->Disconnect(); } + + SECTION("ConstraintWeightsPrecision") + { + auto op = CreateOperation(OpType::DepthwiseConv2D, Shape(1, 5, 5, 1), DataType::Int8, Shape(1, 5, 5, 1), DataType::Int8); + { + std::vector values(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + std::vector values(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::UInt8, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + std::vector values(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int16, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + std::vector values(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int32, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + op->Disconnect(); + } + + SECTION("ConstraintWeightSum") + { + auto op = CreateOperation(OpType::DepthwiseConv2D, Shape(1, 1, 32768, 2), DataType::Int8, Shape(1, 1, 1, 2), DataType::Int8); + static const int64_t MAX_SUM = (1 << 16) * 127; + { + // Verify supported sum of weights + std::vector values((1 << 16) * 2, 127); + auto weights = CreateTensor("weights", Shape(1, 1, (1 << 16), 2), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + // Verify unsupported sum of weights + std::vector values((1 << 16) * 2, 127); + auto weights = CreateTensor("weights", Shape(1, 1, (1 << 16), 2), DataType::Int8, std::move(values)); + weights->SetAxisOrder(AxisOrder::OHWI); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + op->Disconnect(); + } + + SECTION("ConstraintBias") + { + auto op = CreateOperation(OpType::DepthwiseConv2D, Shape(1, 5, 5, 1), DataType::Int8, Shape(1, 5, 5, 1), DataType::Int8); + std::vector wValues(1, 1); + auto weights = CreateTensor("weights", Shape(1, 1, 1, 1), DataType::Int8, std::move(wValues)); + weights->SetAxisOrder(AxisOrder::IHWO); + op->ConnectInput(TensorUsage::Weights, weights).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + { + // Bias must be const + auto bias = CreateTensor("bias", Shape(1), DataType::Int32); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + // Bias must be 1D + std::vector values(1, 1); + auto bias = CreateTensor("bias", Shape(1, 1, 1, 1), DataType::Int32, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + // Bias can't be 8bit + std::vector values(1, 1); + auto bias = CreateTensor("bias", Shape(1), DataType::Int8, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + // Bias can't be 16bit + std::vector values(1, 1); + auto bias = CreateTensor("bias", Shape(1), DataType::Int16, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + { + // Bias can be 32 bit + std::vector values(1, std::numeric_limits::max()); + auto bias = CreateTensor("bias", Shape(1), DataType::Int32, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + // Bias can be 40 bit + std::vector values(1, (1LL << 40) - 1); + auto bias = CreateTensor("bias", Shape(1), DataType::Int64, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == true); + } + { + // Bias can't be >40 bit + std::vector values(1, std::numeric_limits::max()); + auto bias = CreateTensor("bias", Shape(1), DataType::Int64, std::move(values)); + op->ConnectInput(TensorUsage::Scales, bias).Set(Quantization::Unit()); + REQUIRE(supportedOps->Check(op.get()) == false); + } + op->Disconnect(); + } } TEST_CASE("Supported operators EthosU55") diff --git a/ethosu/regor/test/util.cpp b/ethosu/regor/test/util.cpp index 59423260c2e8825a6a860960e40fd3954ef21f8b..1bd614caa7b30d02b0e8fc09a7b3535eb2412069 100644 --- a/ethosu/regor/test/util.cpp +++ b/ethosu/regor/test/util.cpp @@ -132,7 +132,7 @@ std::shared_ptr CreateTensor(const std::string &name, const Shape &stora } // Create a Const Tensor -std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, unsigned value) +std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, int64_t value) { switch ( dtype ) { @@ -148,6 +148,9 @@ std::shared_ptr CreateTensor(const std::string &name, const Shape &stora case DataType::Int32: return CreateTensor(name, storageShape, dtype, std::vector(storageShape.Elements(), int32_t(value))); break; + case DataType::Int64: + return CreateTensor(name, storageShape, dtype, std::vector(storageShape.Elements(), int64_t(value))); + break; default: assert(false); return CreateTensor(name, storageShape, dtype, std::vector(storageShape.Elements(), int8_t(value))); diff --git a/ethosu/regor/test/util.hpp b/ethosu/regor/test/util.hpp index 854740bb6b75174447369cbee75cd1e5284d0fee..8e605b36a24f3486b2f176fd4da2d6421f1aa1fe 100644 --- a/ethosu/regor/test/util.hpp +++ b/ethosu/regor/test/util.hpp @@ -61,7 +61,7 @@ std::shared_ptr CreateTensor(const std::string &name, const Shape &stora // Create a Const Tensor template std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, std::vector &&values); -std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, unsigned value); +std::shared_ptr CreateTensor(const std::string &name, const Shape &storageShape, DataType dtype, int64_t value); // Create a Operation with unary input std::shared_ptr CreateOperation(OpType opType, TensorUsage ifmUsage, std::shared_ptr &ifm, TensorUsage ofmUsage, std::shared_ptr &ofm); diff --git a/ethosu/regor/tflite/tflite_supported_operators.cpp b/ethosu/regor/tflite/tflite_supported_operators.cpp index ac0e8844fb83c71953b1de96c0d41f494a28123e..07a5184e9db88577fc199e84428be0c859697e8a 100644 --- a/ethosu/regor/tflite/tflite_supported_operators.cpp +++ b/ethosu/regor/tflite/tflite_supported_operators.cpp @@ -253,6 +253,145 @@ bool TfLiteSupportedOperators::ConstraintMatchingQuantization(const Operation *o return true; } +bool TfLiteSupportedOperators::ConstraintWeightsPrecision(const Operation *op) +{ + const char *constraint = "Weight tensors must be 8-bit precision"; + const auto wconn = op->Input(TensorUsage::Weights); + if ( !wconn ) + { + return true; + } + const auto type = wconn->tensor->Type(); + if ( DataTypeSizeBits(type) != 8 ) + { + Failure(op, fmt::format("Weights tensor with precision: {}", DataTypeToString(type)), constraint); + return false; + } + return true; +} + +bool TfLiteSupportedOperators::ConstraintWeightSum(const Operation *op) +{ + std::string constraint = fmt::format( + "The sum of absolute weights cannot exceed:\n" + "\t{} for 8-bit IFM\n" + "\t{} for 16-bit IFM", + _maxWeightSum8Bit, _maxWeightSum16Bit); + + auto wConn = op->Input(TensorUsage::Weights); + auto ifmConn = op->Input(TensorUsage::IFM); + if ( !wConn || !ifmConn ) + { + return true; + } + if ( !wConn->tensor->IsConstant() ) + { + return true; + } + + auto view = wConn->tensor->View(); + auto zeroPoints = wConn->quantization.zeroPoints; + auto ifmType = ifmConn->tensor->Type(); + int ifmBits = DataTypeSizeBits(ifmType); + int64_t maxWeightSum = ifmBits == 8 ? _maxWeightSum8Bit : _maxWeightSum16Bit; + auto reader = view.Values(wConn->tensor->Type()); + AxisOrder order = wConn->tensor->AxisOrder(); + Shape readShape = wConn->tensor->StorageShape(); + assert(readShape.Size() == 4); + assert(order == AxisOrder::OHWI || order == AxisOrder::IHWO); + + int outChannels = readShape.Depth(); + int inChannels = readShape.Batch(); + if ( order == AxisOrder::OHWI ) + { + std::swap(outChannels, inChannels); + } + // abort early if the readShape of the weights tensor guarantees no overflow. + if ( (255 * readShape.Elements64() / outChannels) < maxWeightSum ) + { + return true; + } + // Accumulate the weights in slices of output-channels + // Fail if any slice overflows maxWeightSum + for ( int out = 0; out < outChannels; out++ ) + { + int64_t zeroPoint = 0; + if ( !zeroPoints.empty() ) + { + zeroPoint = zeroPoints.size() > 1 ? zeroPoints[out] : zeroPoints[0]; + } + int64_t sum = 0; + for ( int in = 0; in < inChannels; in++ ) + { + for ( int h = 0; h < readShape.Height(); h++ ) + { + for ( int w = 0; w < readShape.Width(); w++ ) + { + int64_t v; + if ( order == AxisOrder::OHWI ) + { + v = reader[{out, h, w, in}]; + } + else + { + v = reader[{in, h, w, out}]; + } + sum += std::abs(v - zeroPoint); + } + } + } + if ( sum > maxWeightSum ) + { + Failure(op, fmt::format("The absolute sum of weight-tensor elements: {} exceeds {}", sum, maxWeightSum), constraint); + return false; + } + } + return true; +} + +bool TfLiteSupportedOperators::ConstraintBias(const Operation *op) +{ + auto bConn = op->Input(TensorUsage::Scales); + if ( !bConn ) + { + return true; + } + int biasDim = bConn->shape.Size(); + + if ( biasDim > 1 ) + { + Failure(op, fmt::format("Operation has {}D bias shape.", biasDim), "The bias tensor shape must be 1D"); + return false; + } + if ( !bConn->tensor->IsConstant() ) + { + Failure(op, "Operation has non-constant bias tensor.", "The bias tensor must be constant"); + return false; + } + auto type = bConn->tensor->Type(); + if ( type != DataType::Int32 && type != DataType::Int64 ) + { + Failure(op, fmt::format("Operation has bias with type:{}", DataTypeToString(type)), "The bias tensor precision must be Int32 or Int64"); + return false; + } + if ( type == DataType::Int64 ) + { + // read bias values + auto view = bConn->tensor->View(); + auto values = view.Values(); + for ( int64_t bias : values ) + { + if ( bias > _maxBias ) + { + std::string constraint = fmt::format("Int64 bias must be smaller than {}", _maxBias); + Failure(op, fmt::format("Bias is out of range: {} > {}", bias, _maxBias), constraint); + return false; + } + } + } + return true; +} + void TfLiteSupportedOperators::Failure(const Operation *op, const std::string &message, const std::string &constraint) { assert(op); @@ -278,6 +417,9 @@ void TfLiteSupportedOperators::Failure(const Operation *op, const std::string &m TfLiteSupportedOperators::TfLiteSupportedOperators(IArchitectureConstraints *constraints) : _archConstraints(constraints) { + _maxWeightSum8Bit = 0; + _maxWeightSum16Bit = 0; + _maxBias = 0; _genericChecks = { &TfLiteSupportedOperators::ConstraintOpType, &TfLiteSupportedOperators::ConstraintTensDtypes, @@ -289,6 +431,9 @@ TfLiteSupportedOperators::TfLiteSupportedOperators(IArchitectureConstraints *con &TfLiteSupportedOperators::ConstraintTensQuantized, &TfLiteSupportedOperators::ConstraintPerAxisQuant, &TfLiteSupportedOperators::ConstraintMatchingQuantization, + &TfLiteSupportedOperators::ConstraintWeightsPrecision, + &TfLiteSupportedOperators::ConstraintWeightSum, + &TfLiteSupportedOperators::ConstraintBias, }; } diff --git a/ethosu/regor/tflite/tflite_supported_operators.hpp b/ethosu/regor/tflite/tflite_supported_operators.hpp index 4d062f971406b4066ce1d348cd6713dc6d2d29fd..5b3ea01f66cfb27fe8bcc581e5030f66df44cf85 100644 --- a/ethosu/regor/tflite/tflite_supported_operators.hpp +++ b/ethosu/regor/tflite/tflite_supported_operators.hpp @@ -34,6 +34,9 @@ protected: IArchitectureConstraints *_archConstraints; std::unordered_set _supportedOpTypes; std::unordered_set _supportedDataTypes; + int64_t _maxWeightSum8Bit; + int64_t _maxWeightSum16Bit; + int64_t _maxBias; public: TfLiteSupportedOperators(IArchitectureConstraints *constraints); @@ -57,5 +60,8 @@ private: bool ConstraintPerAxisQuant(const Operation *op); bool ConstraintMatchingQuantization(const Operation *op); bool ConstraintDepthMultiplier(const Operation *op); + bool ConstraintWeightsPrecision(const Operation *op); + bool ConstraintWeightSum(const Operation *op); + bool ConstraintBias(const Operation *op); }; } // namespace regor diff --git a/ethosu/regor/tflite/tflite_supported_operators_u55.cpp b/ethosu/regor/tflite/tflite_supported_operators_u55.cpp index f3dc242cb7c53d14a6c1670de26e3889f417a389..33ba97faa8e3922388860aa9a2b354c3f62b2062 100644 --- a/ethosu/regor/tflite/tflite_supported_operators_u55.cpp +++ b/ethosu/regor/tflite/tflite_supported_operators_u55.cpp @@ -87,6 +87,9 @@ TfLiteSupportedOperatorsU55::TfLiteSupportedOperatorsU55(IArchitectureConstraint DataType::Int32 // clang-format on }; + _maxWeightSum8Bit = 127 * (1 << 16); + _maxWeightSum16Bit = 127 * (1 << 16); + _maxBias = (1LL << 40) - 1; _checks = { &TfLiteSupportedOperatorsU55::ConstraintBroadcastShapes, &TfLiteSupportedOperatorsU55::ConstraintReverse, diff --git a/ethosu/regor/tflite/tflite_supported_operators_u85.cpp b/ethosu/regor/tflite/tflite_supported_operators_u85.cpp index a5db7132f0ca51039ceb64ff6854bb0ccb6ce37d..f0575625f26c441ca11db9e5f3d46d907d8e28b7 100644 --- a/ethosu/regor/tflite/tflite_supported_operators_u85.cpp +++ b/ethosu/regor/tflite/tflite_supported_operators_u85.cpp @@ -115,6 +115,9 @@ TfLiteSupportedOperatorsU85::TfLiteSupportedOperatorsU85(IArchitectureConstraint DataType::Bool8 // clang-format on }; + _maxWeightSum8Bit = 127 * (1 << 16); + _maxWeightSum16Bit = 127 * (1 << 24); + _maxBias = (1LL << 48) - 1; _checks = { &TfLiteSupportedOperatorsU85::ConstraintResizeCommon, &TfLiteSupportedOperatorsU85::ConstraintResizeBilinear,