diff --git a/ethosu/regor/compiler/graphir_optimiser.cpp b/ethosu/regor/compiler/graphir_optimiser.cpp
index 2f80e6ae0817cee39e772a417e6195f6fecc889e..1dea160a9d26f1dedff8650305a77e1ec646aff8 100644
--- a/ethosu/regor/compiler/graphir_optimiser.cpp
+++ b/ethosu/regor/compiler/graphir_optimiser.cpp
@@ -1161,7 +1161,7 @@ Operation *GraphIrOptimiser::FuseRescale(Graph *const graph, Operation *const op
             auto ifmQuant = ifmConn->quantization;
             // Normalize scales to shift 0 if possible
             ifmQuant.scales = ConvertedScales(ofmConn);
-
+            bool replaceInput = false;
             for ( auto ifm : consumer->Inputs().pairs() )
             {
                 if ( ifm.second.tensor == ofmConn->tensor )
@@ -1213,13 +1213,19 @@ Operation *GraphIrOptimiser::FuseRescale(Graph *const graph, Operation *const op
                             // avoid performing this fuse operation.
                             if ( !sameType ) break;
                         }
-                        ReplaceConsumerInput(nullptr, ofmConn->tensor->Readers(), ofmConn->tensor.get(), ifmConn->tensor);
                         ifm.second.quantization = ifmQuant;
                         consumer->Input(ifm.first)->Set(ofmConn->rounding);
                         returnOp = consumer.get();
+                        replaceInput = true;
                     }
                 }
             }
+            if ( replaceInput )
+            {
+                // This is done outside the loop to avoid modifying the consumer inputs while they are being iterated
+                // over.
+                ReplaceConsumerInput(nullptr, ofmConn->tensor->Readers(), ofmConn->tensor.get(), ifmConn->tensor);
+            }
         }
         // If the rescale could not be fused to the consumer of the output of the rescale, check if there
         // is only one producer of the input to the rescale operation. If this input has no zero point