diff --git a/ethosu/regor/compiler/attributes.cpp b/ethosu/regor/compiler/attributes.cpp index ad7be88962d68a5ac3fc2d044bd3290492aee57c..7ceda1a78f7b090c5569fd02807659465dac763d 100644 --- a/ethosu/regor/compiler/attributes.cpp +++ b/ethosu/regor/compiler/attributes.cpp @@ -53,6 +53,7 @@ DynamicRef CreateAttribute(uint32_t reducedHash) CASE_MAKE_ATTR_INSTANCE(transpose_attr_t); CASE_MAKE_ATTR_INSTANCE(transpose_conv2d_attr_t); CASE_MAKE_ATTR_INSTANCE(while_attr_t); + CASE_MAKE_ATTR_INSTANCE(mirror_pad_mode_attr_t); default: assert(false && "No attribute has this reduced hash"); // Add a new XXX_attr_t struct to the header then diff --git a/ethosu/regor/compiler/attributes.hpp b/ethosu/regor/compiler/attributes.hpp index badb79619fe1cb01e94dd28359694a9454f56eec..5feb982522f9c67f26c2d8541d32af6acc438eeb 100644 --- a/ethosu/regor/compiler/attributes.hpp +++ b/ethosu/regor/compiler/attributes.hpp @@ -30,6 +30,11 @@ #include #include +namespace tflite +{ +enum class MirrorPadMode : int8_t; +} // namespace tflite + namespace regor { @@ -261,6 +266,14 @@ struct pad_attr_t END_FIELD_TABLE() }; +struct mirror_pad_mode_attr_t +{ + tflite::MirrorPadMode mode; + BEGIN_FIELD_TABLE(mirror_pad_mode_attr_t) + ATTR_FIELD(mode, 0) + END_FIELD_TABLE() +}; + #define REDUCED_HASH(hash) (hash & 0x000FFFFF) DynamicRef CreateAttribute(uint32_t hash); @@ -329,5 +342,4 @@ public: }; - } // namespace regor diff --git a/ethosu/regor/compiler/graphir_optimiser.cpp b/ethosu/regor/compiler/graphir_optimiser.cpp index 7928894b64629a4060b547e586fe8bb54e2a5fc1..24bab1b997f34e88dafa774b3065611fb22ed34b 100644 --- a/ethosu/regor/compiler/graphir_optimiser.cpp +++ b/ethosu/regor/compiler/graphir_optimiser.cpp @@ -1967,10 +1967,10 @@ void GraphIrOptimiser::MoveToConsumer(const Operation *const operation, Operatio Operation *GraphIrOptimiser::MoveSplitSliceToConsumer(Graph *const, Operation *const operation) { auto *ifmConn = operation->Input(TensorUsage::IFM0); + auto *ofmConn = operation->Output(TensorUsage::OFM); - if ( operation->Type() == OpType::MemoryCopy && ifmConn->slice.offset.Size() > 0 ) + if ( operation->Type() == OpType::MemoryCopy && ifmConn->slice.offset.Size() > 0 && (ofmConn->reverse == ReverseType::None) ) { - auto *ofmConn = operation->Output(TensorUsage::OFM); auto *ofm = ofmConn->tensor.get(); // TODO: MLBEDSW-9072: Add check that moving split to consumer is valid diff --git a/ethosu/regor/compiler/tflite_graph_optimiser.cpp b/ethosu/regor/compiler/tflite_graph_optimiser.cpp index 0a0d37f4e468abfb3f4b67969f1fe73ff0c77500..bd973c1264771ac1ed594f028d00d509ab97fa2d 100644 --- a/ethosu/regor/compiler/tflite_graph_optimiser.cpp +++ b/ethosu/regor/compiler/tflite_graph_optimiser.cpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2021-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -2681,6 +2681,95 @@ Operation *TFLiteGraphOptimiser::ConvertPad(Graph *const graph, Operation *const return mainOp.get(); } +void TFLiteGraphOptimiser::MakeMemoryCopyForMirrorPad(const Operation *operation, TensorConnection *ifmConn, const Shape &readShape, + const Shape &readOffset, TensorConnection *ofmConn, const Shape &writeShape, const Shape &writeOffset, ReverseType reverseAxis) +{ + auto op = std::make_shared(OpType::MemoryCopy); + + op->ConnectInput(TensorUsage::IFM0, ifmConn->tensor).Set(ifmConn->shape).Set(ifmConn->quantization).Set({readOffset, readShape}); + + op->ConnectOutput(TensorUsage::OFM, ofmConn->tensor) + .Set(ofmConn->shape) + .Set(ofmConn->quantization) + .Set({writeOffset, writeShape}) + .Set(RoundMode::NATURAL) + .Set(reverseAxis); + + RecordOptimisation(operation, op.get()); +} + +Operation *TFLiteGraphOptimiser::ConvertMirrorPad(Graph *const graph, Operation *const operation) +{ + UNUSED(graph); + if ( operation->Type() != OpType::MirrorPad ) + { + return operation; + } + const auto &ifmConn = operation->Input(TensorUsage::IFM0); + const auto &ifmShape = ifmConn->shape; + const auto &ofmConn = operation->Output(TensorUsage::OFM); + const auto &ofmShape = ofmConn->shape; + const auto ¶msConn = operation->Input(TensorUsage::Params); + BufferReader padValues; + if ( paramsConn->tensor->Type() == DataType::Int32 ) + { + padValues = paramsConn->tensor->View().Values(); + } + else + { + assert(paramsConn->tensor->Type() == DataType::Int64); + padValues = paramsConn->tensor->View().Values(); + } + auto pads = paramsConn->tensor->View().Elements(); + auto padValue = [&](int index) { return (pads > index) ? padValues[index] : 0; }; + int top = padValue(2); + int bottom = padValue(3); + int left = padValue(4); + int right = padValue(5); + + auto *attr = operation->Attribute(); + assert((attr->mode >= tflite::MirrorPadMode::MIN) && (attr->mode <= tflite::MirrorPadMode::MAX)); + int modeOffset = (attr->mode == tflite::MirrorPadMode::REFLECT) ? 1 : 0; + + // Create MemoryCopy op that copies IFM to the right place inside the OFM + Shape zeroShape = ofmShape.WithZeros(); + auto mainOp = MakeMemoryCopyForConcat(ofmConn, ifmConn, zeroShape.WithHeight(top).WithWidth(left)); + RecordOptimisation(operation, mainOp.get()); + + // Add operations that fill the borders of the OFM + if ( top > 0 ) + { + Shape shape = ifmShape.WithHeight(top); + Shape readOffset = zeroShape.WithHeight(modeOffset); + Shape writeOffset = zeroShape.WithWidth(left); + MakeMemoryCopyForMirrorPad(operation, ifmConn, shape, readOffset, ofmConn, shape, writeOffset, ReverseType::H); + } + if ( bottom > 0 ) + { + Shape shape = ifmShape.WithHeight(bottom); + Shape readOffset = zeroShape.WithHeight(ifmShape.Height() - bottom - modeOffset); + Shape writeOffset = zeroShape.WithWidth(left).WithHeight(ofmShape.Height() - bottom); + MakeMemoryCopyForMirrorPad(operation, ifmConn, shape, readOffset, ofmConn, shape, writeOffset, ReverseType::H); + } + if ( left > 0 ) + { + Shape shape = ofmShape.WithWidth(left); + Shape readOffset = zeroShape.WithWidth(left + modeOffset); + Shape writeOffset = zeroShape; + MakeMemoryCopyForMirrorPad(operation, ofmConn, shape, readOffset, ofmConn, shape, writeOffset, ReverseType::W); + } + if ( right > 0 ) + { + Shape shape = ofmShape.WithWidth(right); + Shape readOffset = zeroShape.WithWidth(ifmShape.Width() + left - right - modeOffset); + Shape writeOffset = zeroShape.WithWidth(ofmShape.Width() - right); + MakeMemoryCopyForMirrorPad(operation, ofmConn, shape, readOffset, ofmConn, shape, writeOffset, ReverseType::W); + } + + operation->Disconnect(); + return mainOp.get(); +} + Operation *TFLiteGraphOptimiser::ConvertZeroPoint(Graph *const graph, Operation *const operation) { UNUSED(graph); diff --git a/ethosu/regor/compiler/tflite_graph_optimiser.hpp b/ethosu/regor/compiler/tflite_graph_optimiser.hpp index d87c32bde5853443d37229684537630b6379b9a7..92b6922c655f0924bbeb349a35380256fd341bd2 100644 --- a/ethosu/regor/compiler/tflite_graph_optimiser.hpp +++ b/ethosu/regor/compiler/tflite_graph_optimiser.hpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2021-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -155,6 +155,13 @@ private: // This is done as fall-back for the PAD operators that remain after ReplacePadByExplicitPadding Operation *ConvertPad(Graph *const graph, Operation *const operation); + void MakeMemoryCopyForMirrorPad(const Operation *operation, TensorConnection *ifmConn, const Shape &readShape, + const Shape &readOffset, TensorConnection *ofmConn, const Shape &writeShape, const Shape &writeOffset, ReverseType reverseAxis); + + // Rewrites MIRROR_PAD operator to a MemoryCopy that copies the IFM to the OFM + // followed by up to 4 MemoryCopy operators that append the padding at the borders. + Operation *ConvertMirrorPad(Graph *const graph, Operation *const operation); + // Rewrites zero point as expected by reference Operation *ConvertZeroPoint(Graph *const graph, Operation *const operation); @@ -220,6 +227,7 @@ public: &TFLiteGraphOptimiser::ConvertScatter, &TFLiteGraphOptimiser::ConvertResize, &TFLiteGraphOptimiser::ConvertTranspose, + &TFLiteGraphOptimiser::ConvertMirrorPad, } }, { diff --git a/ethosu/regor/tflite/tflite_reader.cpp b/ethosu/regor/tflite/tflite_reader.cpp index 2da6584a718f14edaa7b0bdaac374eb22abbd61a..d1d1e1899f0ee54169807f9c2a3d0bbb1c23952c 100644 --- a/ethosu/regor/tflite/tflite_reader.cpp +++ b/ethosu/regor/tflite/tflite_reader.cpp @@ -1,5 +1,5 @@ // -// SPDX-FileCopyrightText: Copyright 2021-2024 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2021-2025 Arm Limited and/or its affiliates // // SPDX-License-Identifier: Apache-2.0 // @@ -697,6 +697,13 @@ void TfLiteReader::ParseOperatorOptions(const std::shared_ptr &operat } break; + case tflite::BuiltinOptions::MirrorPadOptions: + { + const auto options = GetBuiltinOptions(tflite_operator); + operation->Attribute()->mode = options->mode(); + } + break; + case tflite::BuiltinOptions::ResizeBilinearOptions: case tflite::BuiltinOptions::ResizeNearestNeighborOptions: break; @@ -772,7 +779,6 @@ void TfLiteReader::ParseOperatorOptions(const std::shared_ptr &operat case tflite::BuiltinOptions::FloorModOptions: case tflite::BuiltinOptions::RangeOptions: case tflite::BuiltinOptions::SquaredDifferenceOptions: - case tflite::BuiltinOptions::MirrorPadOptions: case tflite::BuiltinOptions::AbsOptions: case tflite::BuiltinOptions::UniqueOptions: case tflite::BuiltinOptions::ReverseV2Options: