From 72878c5b74370e3f88c32e7eca6958082396dc27 Mon Sep 17 00:00:00 2001 From: Rickard Bolin Date: Fri, 11 Apr 2025 15:44:48 +0000 Subject: [PATCH] MLBEDSW-10631: Rewrite CONST op TOSA CONST operators does not have any input tensors, only an attribute containing a constant tensor and an output tensor. Until now, we've simply ignored the CONST operators in the TOSA reader since we're only interested in the constant output tensor. This causes issues in the case of a single CONST operator network, where ignoring CONST results in an optimized network without any operators, which our raw mode writer can not handle. This patch replaces CONST operators with IDENTITY operators which gets cleaned up during RemoveReshape to either be removed completely or get converted to a memory copy when it is the last operator remaining in the network. Signed-off-by: Rickard Bolin Change-Id: Icf18f0bad27a1220073574ffa2626f8838b4df69 --- .../regor/architecture/ethosu85/ethos_u85.cpp | 2 +- ethosu/regor/compiler/graphir_optimiser.cpp | 49 ++++++++++++++++--- ethosu/regor/compiler/graphir_optimiser.hpp | 8 +++ ethosu/regor/tflite/tflite_writer.cpp | 22 ++------- ethosu/regor/tosa/tosaValidationGenerator.rb | 3 +- ethosu/regor/tosa/tosa_error_checks.cpp | 7 +-- ethosu/regor/tosa/tosa_reader.cpp | 1 - ethosu/regor/tosa/tosa_validator.cpp | 4 +- ethosu/regor/tosa/tosa_validator.hpp | 4 +- ...sa_validator_version_0_80_0_profile_bi.cpp | 43 +++++++++++++++- 10 files changed, 107 insertions(+), 36 deletions(-) diff --git a/ethosu/regor/architecture/ethosu85/ethos_u85.cpp b/ethosu/regor/architecture/ethosu85/ethos_u85.cpp index 3312c0ba..bf13380a 100644 --- a/ethosu/regor/architecture/ethosu85/ethos_u85.cpp +++ b/ethosu/regor/architecture/ethosu85/ethos_u85.cpp @@ -892,7 +892,7 @@ std::unique_ptr ArchEthosU85::FindBlockConfig(OpType opTyp LOG_INDENT(Logging::Out); constexpr int OFMSplitDepth = 16; // Specific to this architecture - assert(query.ifmBits > 0 && (query.ifmBits <= 32 || (query.ifmBits == 64 && opType == OpType::Rescale))); + assert(query.ifmBits > 0 && (query.ifmBits <= 32 || (query.ifmBits == 64 && (opType == OpType::Rescale || opType == OpType::MemoryCopy)))); assert(query.ofmShape.Size() > 2 && "Insufficient dimensions to search for block config"); assert(query.kernel != nullptr); diff --git a/ethosu/regor/compiler/graphir_optimiser.cpp b/ethosu/regor/compiler/graphir_optimiser.cpp index 251b0536..44a81040 100644 --- a/ethosu/regor/compiler/graphir_optimiser.cpp +++ b/ethosu/regor/compiler/graphir_optimiser.cpp @@ -30,13 +30,22 @@ using namespace GraphOptimisation; Tensor *GraphIrOptimiser::ConvertInt48Tensors(Graph *, Tensor *tensor) { - if ( tensor->Type() == DataType::Int48 && !tensor->IsConstant() ) + if ( DataTypeSizeBits(tensor->Type()) == 48 ) { - tensor->ChangeType(DataType::Int64); - } - else if ( tensor->Type() == DataType::UInt48 && !tensor->IsConstant() ) - { - tensor->ChangeType(DataType::UInt64); + if ( tensor->IsConstant() ) + { + // Unpack 48-bit to 64-bit values + const auto values = tensor->View().Values(); + std::vector unpackedValues(values.begin(), values.end()); + // Replace the tensor's buffer with the new buffer containing the 64-bit values + tensor->SetBuffer(nullptr); + tensor->ChangeType(DataType::Int64); + tensor->SetBuffer(std::make_shared(std::move(unpackedValues))); + } + else + { + tensor->ChangeType(IsSignedInteger(tensor->Type()) ? DataType::Int64 : DataType::UInt64); + } } return tensor; } @@ -371,6 +380,34 @@ Operation *GraphIrOptimiser::ConstPropagation(Graph *const graph, Operation *con return operation; } +/* + * This pass replaces the Const operator with an Identity operator (to be removed in RemoveReshape) and moves the + * "values" attribute tensor to an input tensor instead to enable us to treat it as a normal reshape-like operator. + */ +Operation *GraphIrOptimiser::RewriteConst(Graph *const graph, Operation *const operation) +{ + Operation *returnOp = operation; + OpType opType = operation->Type(); + if ( opType == OpType::Const ) + { + const auto *ofmConn = operation->Output(TensorUsage::OFM); + // Clone tensor to create input tensor with the constant values and remove constant values from output + std::shared_ptr constIfm = ofmConn->tensor->Clone(); + constIfm->SetName("const_values"); + ofmConn->tensor->SetBuffer(nullptr); + + // Create new identity operator (to be removed in RemoveReshape) and set constant values as input + auto identityOp = std::make_shared(OpType::Identity); + identityOp->ConnectInput(TensorUsage::IFM0, constIfm); + identityOp->CopyOutput(TensorUsage::OFM, *ofmConn); + + returnOp = identityOp.get(); + RecordOptimisation(operation, returnOp); + operation->Disconnect(); + } + return returnOp; +} + Operation *GraphIrOptimiser::RewriteFullyConnected(Graph *const graph, Operation *const operation) { UNUSED(graph); diff --git a/ethosu/regor/compiler/graphir_optimiser.hpp b/ethosu/regor/compiler/graphir_optimiser.hpp index 7133f0ac..bc79c703 100644 --- a/ethosu/regor/compiler/graphir_optimiser.hpp +++ b/ethosu/regor/compiler/graphir_optimiser.hpp @@ -41,6 +41,7 @@ class GraphIrOptimiser : public GraphOptimiser private: Operation *ConstPropagation(Graph *const graph, Operation *const operation); + Operation *RewriteConst(Graph *const graph, Operation *const operation); Operation *ConvertAttributes(Graph *const graph, Operation *const operation); Operation *ConvertResizeOffsets(Graph *const graph, Operation *const operation); Tensor *ConvertInt48Tensors(Graph *graph, Tensor *tensor); @@ -94,6 +95,12 @@ private: &GraphOptimiser::RecordOperation } }, + { + {}, + { + &GraphIrOptimiser::RewriteConst, + }, + }, { { &GraphIrOptimiser::ConvertInt48Tensors, @@ -101,6 +108,7 @@ private: &GraphIrOptimiser::ConvertInt4Tensors, }, { + // RemoveReshape must run as a standalone pass &GraphOptimiser::RemoveReshape, } }, diff --git a/ethosu/regor/tflite/tflite_writer.cpp b/ethosu/regor/tflite/tflite_writer.cpp index 18b540cc..584635ee 100644 --- a/ethosu/regor/tflite/tflite_writer.cpp +++ b/ethosu/regor/tflite/tflite_writer.cpp @@ -535,25 +535,11 @@ flatbuffers::Offset TfLiteWriter::SerialiseTensorAddresses(int void TfLiteWriter::SerialiseTensorBuffer(const Tensor *tensor) { - if ( tensor->Type() == DataType::Int48 ) - { // Translate values - const auto values = tensor->View().Values(); - auto v = std::make_unique>(values.begin(), values.end()); - const auto size = v->size() * sizeof(int64_t); - _serialised_buffers.emplace_back(SerialiseBuffer(reinterpret_cast(v->data()), size)); - if ( _useBufferOffset ) - { - _offset_buffers.emplace_back(std::move(v)); - } - } - else + const auto buffer = tensor->View().Buffer(); + _serialised_buffers.emplace_back(SerialiseBuffer(buffer)); + if ( _useBufferOffset ) { - const auto buffer = tensor->View().Buffer(); - _serialised_buffers.emplace_back(SerialiseBuffer(buffer)); - if ( _useBufferOffset ) - { - _offset_buffers.emplace_back(buffer); - } + _offset_buffers.emplace_back(buffer); } } diff --git a/ethosu/regor/tosa/tosaValidationGenerator.rb b/ethosu/regor/tosa/tosaValidationGenerator.rb index d08ccdf5..46440a55 100755 --- a/ethosu/regor/tosa/tosaValidationGenerator.rb +++ b/ethosu/regor/tosa/tosaValidationGenerator.rb @@ -1,6 +1,6 @@ #!/usr/bin/env ruby # -# SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: Copyright 2023-2025 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 # @@ -23,6 +23,7 @@ require 'rexml' REGOR_OP_NAMES = { 'ARGMAX': 'OpType::ArgMax', 'AVG_POOL2D': 'OpType::AvgPool', + 'CONST': 'OpType::Const', 'CONV2D': 'OpType::Conv2D', 'CONV3D': 'OpType::Conv3D', 'DEPTHWISE_CONV2D': 'OpType::DepthwiseConv2D', diff --git a/ethosu/regor/tosa/tosa_error_checks.cpp b/ethosu/regor/tosa/tosa_error_checks.cpp index 3ee9a4b6..26a24b7c 100644 --- a/ethosu/regor/tosa/tosa_error_checks.cpp +++ b/ethosu/regor/tosa/tosa_error_checks.cpp @@ -1520,9 +1520,10 @@ void ErrorIfCheck_3oet4aggtv528(const regor::Operation *op, [[maybe_unused]] con { // Operators: CONST, static constexpr char constraint[] = "ERROR_IF(rankCheck(output, values))"; - const auto &outputShape = op->Output(TensorUsage::OFM)->shape; - const auto &inputShape = op->Input(TensorUsage::IFM)->shape; - if ( outputShape != inputShape ) throw std::invalid_argument(constraint); + const auto &ofmConn = op->Output(TensorUsage::OFM); + const auto bufferSize = ofmConn->tensor->View().Buffer()->Size(); + const auto storageSize = DataTypeStorageSizeBytes(ofmConn->tensor->Type(), ofmConn->shape.Elements()); + if ( bufferSize != storageSize ) throw std::invalid_argument(constraint); } } // namespace checks diff --git a/ethosu/regor/tosa/tosa_reader.cpp b/ethosu/regor/tosa/tosa_reader.cpp index 75afcc69..2fb15473 100644 --- a/ethosu/regor/tosa/tosa_reader.cpp +++ b/ethosu/regor/tosa/tosa_reader.cpp @@ -372,7 +372,6 @@ void TosaReader::LoadGraphs(const tosaFb::TosaGraph *model, std::list input_tensors; diff --git a/ethosu/regor/tosa/tosa_validator.cpp b/ethosu/regor/tosa/tosa_validator.cpp index 2f0a4382..168bd667 100644 --- a/ethosu/regor/tosa/tosa_validator.cpp +++ b/ethosu/regor/tosa/tosa_validator.cpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2023-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// Generated by tosaValidationGenerator for TOSA Specification 0.80.0 +// Automatically generated by tosaValidationGenerator for TOSA Specification 0.80.0 // Do not edit. #include "tosa/tosa_validator.hpp" diff --git a/ethosu/regor/tosa/tosa_validator.hpp b/ethosu/regor/tosa/tosa_validator.hpp index 7a35a058..2ac26814 100644 --- a/ethosu/regor/tosa/tosa_validator.hpp +++ b/ethosu/regor/tosa/tosa_validator.hpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2023-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// Generated by tosaValidationGenerator for TOSA Specification 0.80.0 +// Automatically generated by tosaValidationGenerator for TOSA Specification 0.80.0 // Do not edit. #pragma once diff --git a/ethosu/regor/tosa/tosa_validator_version_0_80_0_profile_bi.cpp b/ethosu/regor/tosa/tosa_validator_version_0_80_0_profile_bi.cpp index a86313e0..7f5dfb80 100644 --- a/ethosu/regor/tosa/tosa_validator_version_0_80_0_profile_bi.cpp +++ b/ethosu/regor/tosa/tosa_validator_version_0_80_0_profile_bi.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2023-2024 Arm Limited. All rights reserved. +// SPDX-FileCopyrightText: Copyright 2023-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// Generated by tosaValidationGenerator for tosa specification 0.80.0 +// Automatically generated by tosaValidationGenerator for TOSA Specification 0.80.0 // Do not edit. #include "compiler/operation.hpp" @@ -2219,6 +2219,42 @@ void ValidateOperator_RESCALE(const regor::Operation *op, const Context &context LevelCheck_1flzmpv6hubzc(op, context); } +void ValidateOperator_CONST(const regor::Operation *op, const Context &context) +{ + const Argument values = {Category::Attribute, "values", "out_t", {0, MAX_RANK}}; /*Constant values shape=shape*/ + const Argument output = {Category::Output, "output", "out_t", {0, MAX_RANK}}; /*Output tensor of the same type, size + as the input tensor shape=shape*/ + const std::vector arguments = { + &values, + &output, + }; + const std::vector typesupports = { + { + {"out_t", "bool_t"}, + }, // Boolean + { + {"out_t", "i4_t"}, + }, // 4-bit + { + {"out_t", "i8_t"}, + }, // 8-bit + { + {"out_t", "i16_t"}, + }, // 16-bit + { + {"out_t", "i32_t"}, + }, // 32-bit + { + {"out_t", "i48_t"}, + }, // 48-bit + { + {"out_t", "shape_t"}, + }, // shape + }; + ValidateArguments(op, arguments, typesupports, context); + ErrorIfCheck_3oet4aggtv528(op, context); +} + void ValidateOperator_IDENTITY(const regor::Operation *op, const Context &context) { const Argument input1 = {Category::Input, "input1", "in_out_t", {0, MAX_RANK}}; /*Input tensor shape=shape*/ @@ -2564,6 +2600,9 @@ void ValidateOperator_Version_0_80_0_Profile_BI(const GraphApi::GraphOperation * case regor::OpType::Rescale: ValidateOperator_RESCALE(op, context); break; + case regor::OpType::Const: + ValidateOperator_CONST(op, context); + break; case regor::OpType::Identity: ValidateOperator_IDENTITY(op, context); break; -- GitLab