diff --git a/ethosu/regor/CMakeLists.txt b/ethosu/regor/CMakeLists.txt index 8c52868ab44c6df4d9771400834d0b685e535f83..992eead038817f5f14e4cedc4c3705ec560f7dd2 100644 --- a/ethosu/regor/CMakeLists.txt +++ b/ethosu/regor/CMakeLists.txt @@ -293,6 +293,7 @@ regor_lib( "compiler/tensor_allocator.cpp" "compiler/tflite_graph_optimiser.cpp" "compiler/tflite_graph_optimiser_tp.cpp" + "compiler/tosa_graph_optimiser.cpp" "compiler/cascade_builder.cpp" "compiler/graph_optimiser.cpp" "compiler/graphir_optimiser.cpp" diff --git a/ethosu/regor/compiler/compiler.cpp b/ethosu/regor/compiler/compiler.cpp index 56427ec9c1b17e6a47db115f16226dab1ccfad26..4806ecb7d66d8bb7b91d7a516d81d6490e396335 100644 --- a/ethosu/regor/compiler/compiler.cpp +++ b/ethosu/regor/compiler/compiler.cpp @@ -437,21 +437,12 @@ std::unique_ptr Compiler::CompileGraph(std::unique_ptr &graph, try { - if ( graph->Notation() == GraphNotation::TFLite ) - { - // Run GraphNotation::TFLite Preprocess/optimise step - std::unique_ptr optimiser = GraphOptimiser::MakeGraphOptimiser( - GraphNotation::TFLite, _architecture.get(), _graphOptimiserOptions, _optDb.get()); - if ( optimiser ) - { - optimiser->Process(graph.get()); - } - } + // Create both the specific pre-optimiser and the general post-optimiser + auto graphOptimisers = GraphOptimiser::MakeGraphOptimiser( + graph->Notation(), _architecture.get(), _graphOptimiserOptions, _optDb.get()); - // Run GraphNotation::GraphAPI Preprocess/optimise step - std::unique_ptr optimiser = GraphOptimiser::MakeGraphOptimiser( - GraphNotation::GraphAPI, _architecture.get(), _graphOptimiserOptions, _optDb.get()); - if ( optimiser ) + // Run the graph optimisers + for ( const auto &optimiser : graphOptimisers ) { optimiser->Process(graph.get()); } diff --git a/ethosu/regor/compiler/graph_optimiser.cpp b/ethosu/regor/compiler/graph_optimiser.cpp index 7f0d9cf171a91df09e40fb1497c36bb82d55ac1c..bf64893c1129aeb077b6eb1874211a194c42e9fb 100644 --- a/ethosu/regor/compiler/graph_optimiser.cpp +++ b/ethosu/regor/compiler/graph_optimiser.cpp @@ -29,6 +29,7 @@ #include "tensor.hpp" #include "tflite/tflite_supported_operators.hpp" #include "tflite_graph_optimiser.hpp" +#include "tosa_graph_optimiser.hpp" #include #include @@ -45,9 +46,11 @@ namespace regor using namespace GraphOptimisation; -std::unique_ptr GraphOptimiser::MakeGraphOptimiser( +std::vector> GraphOptimiser::MakeGraphOptimiser( GraphNotation notation, Architecture *arch, const GraphOptimiserOptions &options, OptimiserDatabase *db) { + std::vector> graphOptimisers; + switch ( notation ) { case GraphNotation::TFLite: @@ -55,21 +58,24 @@ std::unique_ptr GraphOptimiser::MakeGraphOptimiser( std::unique_ptr supportedOps; arch->Call([&supportedOps, &arch](const std::string &target) { supportedOps = MakeSupportedOpsChecker(target, arch->Constraints()); }); - return std::unique_ptr( + graphOptimisers.emplace_back( std::make_unique(arch->Constraints(), std::move(supportedOps), options, db)); } + break; case GraphNotation::GraphAPI: - return std::unique_ptr(std::make_unique(arch->Constraints(), options, db)); + graphOptimisers.emplace_back(std::make_unique(arch->Constraints(), options, db)); + break; default: LOG_ERROR("Invalid graph notation"); assert(false); } - - return {}; + graphOptimisers.emplace_back(std::make_unique(arch->Constraints(), options, db)); + return graphOptimisers; } + // Some debug functions #if LOG_TRACE1_ON Operation *GraphOptimiser::VisitOperatorLog(Graph *const graph, Operation *const operation) diff --git a/ethosu/regor/compiler/graph_optimiser.hpp b/ethosu/regor/compiler/graph_optimiser.hpp index 200c4623bc4e38f26735aebb32611c9c649278c8..039e0168243b5230b4c9bf8280c97965a869d72c 100644 --- a/ethosu/regor/compiler/graph_optimiser.hpp +++ b/ethosu/regor/compiler/graph_optimiser.hpp @@ -76,8 +76,9 @@ public: - static std::unique_ptr MakeGraphOptimiser( + static std::vector> MakeGraphOptimiser( GraphNotation notation, Architecture *arch, const GraphOptimiserOptions &options, OptimiserDatabase *db); + static void ParseGraphOptimiserOptions(GraphOptimiserOptions &opt, IniReader &reader); void Process(Graph *graph); diff --git a/ethosu/regor/compiler/graphir_optimiser.cpp b/ethosu/regor/compiler/graphir_optimiser.cpp index 7aa25a8b7ea688666fe05123acbca4566d4ca50c..7ea2812827d276abf0bed1e145371d99b7501746 100644 --- a/ethosu/regor/compiler/graphir_optimiser.cpp +++ b/ethosu/regor/compiler/graphir_optimiser.cpp @@ -295,53 +295,6 @@ Operation *GraphIrOptimiser::ConvertAttributeTensors(Graph *const graph, Operati return operation; } -// Convert compile time constant zero point tensors to quantization zero points -Operation *GraphIrOptimiser::ConvertZeroPointTensors(Graph *const graph, Operation *const operation) -{ - UNUSED(graph); - auto SetZeroPoint = [&](TensorUsage target, TensorUsage param, bool asUnsigned = false) - { - if ( const auto zpConn = operation->Input(param) ) - { - assert(zpConn->tensor->IsConstant()); - const auto targetConn = IsOFM(target) ? operation->Output(target) : operation->Input(target); - assert(targetConn); - auto dataType = asUnsigned ? zpConn->tensor->Type() & ~unsigned(DataType::Signed) : zpConn->tensor->Type(); - auto values = zpConn->tensor->View().Values(dataType); - targetConn->quantization.zeroPoints = {values.begin(), values.end()}; - } - }; - switch ( operation->Type() ) - { - case OpType::AvgPool: - case OpType::Neg: - SetZeroPoint(TensorUsage::IFM, TensorUsage::Params0); - SetZeroPoint(TensorUsage::OFM, TensorUsage::Params1); - break; - case OpType::Conv2D: - case OpType::Conv3D: - case OpType::DepthwiseConv2D: - case OpType::TransposeConv2D: - SetZeroPoint(TensorUsage::IFM, TensorUsage::Params0); - SetZeroPoint(TensorUsage::Weights, TensorUsage::Params1); - break; - case OpType::MatMul: - SetZeroPoint(TensorUsage::IFM0, TensorUsage::Params0); - SetZeroPoint(TensorUsage::IFM1, TensorUsage::Params1); - break; - case OpType::Rescale: - { - const auto signAttr = operation->Attribute(); - SetZeroPoint(TensorUsage::IFM, TensorUsage::Params2, signAttr->input_unsigned); - SetZeroPoint(TensorUsage::OFM, TensorUsage::Params3, signAttr->output_unsigned); - break; - } - default: - break; - } - return operation; -} - Operation *GraphIrOptimiser::ConvertResizeOffsets(Graph *const graph, Operation *const operation) { UNUSED(graph); diff --git a/ethosu/regor/compiler/graphir_optimiser.hpp b/ethosu/regor/compiler/graphir_optimiser.hpp index 9b36d2af30fd2823be8146bdb62ca169be854df5..c7a6b7b1f8d0af7cba47d976f2d7b1426aff6317 100644 --- a/ethosu/regor/compiler/graphir_optimiser.hpp +++ b/ethosu/regor/compiler/graphir_optimiser.hpp @@ -44,7 +44,6 @@ private: Operation *RewriteConst(Graph *const graph, Operation *const operation); Operation *ConvertAttributes(Graph *const graph, Operation *const operation); Operation *ConvertAttributeTensors(Graph *const graph, Operation *const operation); - Operation *ConvertZeroPointTensors(Graph *const graph, Operation *const operation); Operation *ConvertResizeOffsets(Graph *const graph, Operation *const operation); Tensor *ConvertInt48Tensors(Graph *graph, Tensor *tensor); Tensor *ConvertBool8Tensors(Graph *graph, Tensor *tensor); @@ -128,7 +127,6 @@ private: { {}, { - &GraphIrOptimiser::ConvertZeroPointTensors, &GraphIrOptimiser::RewriteRescaleInputs, &GraphIrOptimiser::FuseRescale, // First pass fuse all possible ifm and ofm rescales } diff --git a/ethosu/regor/compiler/tosa_graph_optimiser.cpp b/ethosu/regor/compiler/tosa_graph_optimiser.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b7da0cd6147452b511ceeacbe3e0db085ea90577 --- /dev/null +++ b/ethosu/regor/compiler/tosa_graph_optimiser.cpp @@ -0,0 +1,95 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "compiler/tosa_graph_optimiser.hpp" + +#include "optimiser_utils.hpp" + + +namespace regor +{ + +using namespace GraphOptimisation; + +// Convert compile time constant zero point tensors to quantization zero points +Operation *TosaGraphOptimiser::ConvertZeroPointTensors(Graph *const graph, Operation *const operation) +{ + UNUSED(graph); + auto SetZeroPoint = [&](TensorUsage target, TensorUsage param, bool asUnsigned = false) + { + if ( const auto zpConn = operation->Input(param) ) + { + assert(zpConn->tensor->IsConstant()); + const auto targetConn = IsOFM(target) ? operation->Output(target) : operation->Input(target); + assert(targetConn); + auto dataType = asUnsigned ? zpConn->tensor->Type() & ~unsigned(DataType::Signed) : zpConn->tensor->Type(); + auto values = zpConn->tensor->View().Values(dataType); + targetConn->quantization.zeroPoints = {values.begin(), values.end()}; + } + }; + switch ( operation->Type() ) + { + case OpType::AvgPool: + case OpType::Neg: + SetZeroPoint(TensorUsage::IFM, TensorUsage::Params0); + SetZeroPoint(TensorUsage::OFM, TensorUsage::Params1); + break; + case OpType::Conv2D: + case OpType::Conv3D: + case OpType::DepthwiseConv2D: + case OpType::TransposeConv2D: + SetZeroPoint(TensorUsage::IFM, TensorUsage::Params0); + SetZeroPoint(TensorUsage::Weights, TensorUsage::Params1); + break; + case OpType::MatMul: + SetZeroPoint(TensorUsage::IFM0, TensorUsage::Params0); + SetZeroPoint(TensorUsage::IFM1, TensorUsage::Params1); + break; + case OpType::Rescale: + { + const auto signAttr = operation->Attribute(); + SetZeroPoint(TensorUsage::IFM, TensorUsage::Params2, signAttr->input_unsigned); + SetZeroPoint(TensorUsage::OFM, TensorUsage::Params3, signAttr->output_unsigned); + break; + } + default: + break; + } + return operation; +} + +TosaGraphOptimiser::TosaGraphOptimiser(IArchitectureConstraints *constraints, const GraphOptimiserOptions &options, OptimiserDatabase *db) : + GraphOptimiser(constraints, options, db) +{ +} + +void TosaGraphOptimiser::OptimiseGraph(Graph *graph) +{ + for ( auto iOpt = GraphOptimisationSteps().begin(); iOpt != GraphOptimisationSteps().end(); ++iOpt ) + { + LOG_TRACE1("GraphOptimiser {0}/{1}\n", std::distance(GraphOptimisationSteps().begin(), iOpt) + 1, + GraphOptimisationSteps().size()); + // Check if function lists are empty. Do not call for step that only contain disabled debug functions. + if ( !iOpt->opFunction.empty() || !iOpt->tensorFunction.empty() ) + { + RewriteGraph(graph, *iOpt); + } + } +} + +} // namespace regor diff --git a/ethosu/regor/compiler/tosa_graph_optimiser.hpp b/ethosu/regor/compiler/tosa_graph_optimiser.hpp new file mode 100644 index 0000000000000000000000000000000000000000..0ec35c34b46d130f50fad2736ba4343706519359 --- /dev/null +++ b/ethosu/regor/compiler/tosa_graph_optimiser.hpp @@ -0,0 +1,93 @@ +// +// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the License); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#pragma once + +#include "common/logging.hpp" + +#include "graph.hpp" +#include "graph_optimiser.hpp" +#include "operation.hpp" +#include "tensor.hpp" + +#include + +namespace regor +{ + +/// +/// TOSA Graph optimiser +/// +class TosaGraphOptimiser : public GraphOptimiser +{ + using OpRewriteFunction = Operation *(TosaGraphOptimiser::*)(Graph *, Operation *); + using TensorRewriteFunction = Tensor *(TosaGraphOptimiser::*)(Graph *, Tensor *); + using GraphOptStepArray = std::vector>; + +private: + Operation *ConvertZeroPointTensors(Graph *const graph, Operation *const operation); + +public: + // The graph optimisation steps. + // Order matters, array of rewrites processed in order. + // clang-format off + const GraphOptStepArray _graphOptimisationSteps = + {{ + { + { +#if LOG_TRACE1_ON + &GraphOptimiser::VisitTensorLog +#endif + }, + { +#if LOG_TRACE1_ON + &TosaGraphOptimiser::VisitOperatorLog, +#endif + } + }, + { + {}, + { + &TosaGraphOptimiser::ConvertZeroPointTensors, + } + }, + { + { +#if LOG_TRACE1_ON + &GraphOptimiser::VisitTensorLog +#endif + }, + { +#if LOG_TRACE1_ON + &GraphOptimiser::VisitOperatorLog, +#endif + &GraphOptimiser::RecordOptimisation + } + } + }}; + // clang-format on + +public: + explicit TosaGraphOptimiser(IArchitectureConstraints *constraints, const GraphOptimiserOptions &options, OptimiserDatabase *db); + + const GraphOptStepArray &GraphOptimisationSteps() const { return _graphOptimisationSteps; } + + void OptimiseGraph(Graph *graph); +}; + +} // namespace regor diff --git a/ethosu/regor/test/test_graphir_optimiser.cpp b/ethosu/regor/test/test_graphir_optimiser.cpp index 3adbdfa72e3e394e061c690d7a869286d19d0e52..ce9176ec2c7aa3d54bedf4de3aa0c5110b082e5e 100644 --- a/ethosu/regor/test/test_graphir_optimiser.cpp +++ b/ethosu/regor/test/test_graphir_optimiser.cpp @@ -60,15 +60,15 @@ TEST_CASE("test_graphir_optimiser - constant propagation") }(); GraphOptimiserOptions options; - auto optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); + const auto &optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); std::vector allOps; graph->GetAllOperations(allOps); REQUIRE(allOps.size() == 2); - REQUIRE(bool(optimiser)); - optimiser->Process(graph.get()); + REQUIRE(!optimiser.empty()); + optimiser.back()->Process(graph.get()); allOps.clear(); graph->GetAllOperations(allOps); @@ -106,15 +106,15 @@ TEST_CASE("test_graphir_optimiser - constant propagation") }(); GraphOptimiserOptions options; - auto optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); + const auto &optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); std::vector allOps; graph->GetAllOperations(allOps); REQUIRE(allOps.size() == 3); - REQUIRE(bool(optimiser)); - optimiser->Process(graph.get()); + REQUIRE(!optimiser.empty()); + optimiser.back()->Process(graph.get()); allOps.clear(); graph->GetAllOperations(allOps); @@ -157,10 +157,10 @@ TEST_CASE("test_graphir_optimiser - ReduceSum") }(); GraphOptimiserOptions options; - auto optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); + const auto &optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); - REQUIRE(bool(optimiser)); - optimiser->Process(graph.get()); + REQUIRE(!optimiser.empty()); + optimiser.back()->Process(graph.get()); SchedulerPacking packing(arch.get(), false); auto scheduleOps = packing.Process(graph.get()); @@ -204,9 +204,9 @@ TEST_CASE("test_graphir_optimiser - transpose removal") auto graph = CreateGraph(ops); GraphOptimiserOptions options; - auto optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); + const auto &optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); - optimiser->Process(graph.get()); + optimiser.back()->Process(graph.get()); std::vector allOps; graph->GetAllOperations(allOps); @@ -248,9 +248,9 @@ TEST_CASE("test_graphir_optimiser - transpose merge") auto graph = CreateGraph(ops); GraphOptimiserOptions options; - auto optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); + const auto &optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); - optimiser->Process(graph.get()); + optimiser.back()->Process(graph.get()); // Result Add->Add std::vector allOps; @@ -303,9 +303,9 @@ TEST_CASE("test_graphir_optimiser - replace pad by explicit padding") auto graph = CreateGraph(ops); GraphOptimiserOptions options; - auto optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); + const auto &optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); - optimiser->Process(graph.get()); + optimiser.back()->Process(graph.get()); std::vector allOps; graph->GetAllOperations(allOps); @@ -352,9 +352,9 @@ TEST_CASE("test_graphir_optimiser - fuse rescale with reshape, before") auto graph = CreateGraph(ops); GraphOptimiserOptions options; - auto optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); + const auto &optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); - optimiser->Process(graph.get()); + optimiser.back()->Process(graph.get()); std::vector allOps; graph->GetAllOperations(allOps); @@ -399,9 +399,9 @@ TEST_CASE("test_graphir_optimiser - fuse rescale with reshape, after") auto graph = CreateGraph(ops); GraphOptimiserOptions options; - auto optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); + const auto &optimiser = GraphOptimiser::MakeGraphOptimiser(graph->Notation(), arch.get(), options, nullptr); - optimiser->Process(graph.get()); + optimiser.back()->Process(graph.get()); std::vector allOps; graph->GetAllOperations(allOps);