diff --git a/ethosu/regor/compiler/graphir_optimiser.cpp b/ethosu/regor/compiler/graphir_optimiser.cpp
index 5236b0c3cac8f510cda40edd1abc6e0e9e01b8be..ddcca184caf290d66b68c154034b77bbd2de7358 100644
--- a/ethosu/regor/compiler/graphir_optimiser.cpp
+++ b/ethosu/regor/compiler/graphir_optimiser.cpp
@@ -1002,7 +1002,8 @@ Operation *GraphIrOptimiser::FuseRescale(Graph *const graph, Operation *const op
         auto ofmQuant = ofmConn->quantization;
         ofmQuant.scales = ConvertedScales(ofmConn);
 
-        if ( returnOp == operation && producer && producer->Output(TensorUsage::OFM)->quantization.EqualScales(Quantization::Unit()) &&
+        if ( returnOp == operation && producer && ifmConn->tensor->Readers().size() == 1 &&
+             producer->Output(TensorUsage::OFM)->quantization.EqualScales(Quantization::Unit()) &&
              ifmConn->quantization.zeroPoints == Quantization::Unit().zeroPoints &&
              // fused tensor cannot be in graph-outputs
              !IsTensorInVector(graph->Outputs(), ifmConn->tensor.get()) &&