From 1d6f292447387faeda73b3802423530b083a7640 Mon Sep 17 00:00:00 2001
From: Rickard Bolin <rickard.bolin@arm.com>
Date: Mon, 23 Dec 2024 12:36:24 +0000
Subject: [PATCH] MLBEDSW-10206: Add support for Resize when H or W is 1

Allow either height or width dimension to be 1 when the other is
upscaled

Signed-off-by: Rickard Bolin <rickard.bolin@arm.com>
Change-Id: If423338b9f6aeacf2c02cc9437588c13c1949cd6
---
 SUPPORTED_OPS.md                          | 12 ++++----
 ethosu/vela/tflite_graph_optimiser.py     | 35 ++++++++++++++++-------
 ethosu/vela/tflite_supported_operators.py | 25 +++++++++-------
 ethosu/vela/vela.py                       |  4 +--
 4 files changed, 47 insertions(+), 29 deletions(-)

diff --git a/SUPPORTED_OPS.md b/SUPPORTED_OPS.md
index 90707c4c..d8d86678 100644
--- a/SUPPORTED_OPS.md
+++ b/SUPPORTED_OPS.md
@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
+SPDX-FileCopyrightText: Copyright 2020-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
 
 SPDX-License-Identifier: Apache-2.0
 
@@ -19,7 +19,7 @@ limitations under the License.
 # Supported Ops
 
 This file was automatically generated by Vela using the `--supported-ops-report` parameter.  
-Vela version: `4.1.0`
+Vela version: `4.2.0`
 
 This file complies with
 [**Gitiles Markdown syntax**](https://gerrit.googlesource.com/gitiles/+/HEAD/Documentation/markdown.md)
@@ -390,8 +390,8 @@ This is a list of constraints that the RESIZE_BILINEAR operator must satisfy in
 - The width and height of the IFM and OFM must match one of the following criteria:  
         IFM W and H must both be 1  
         IFM must match OFM  
-        W and H scaling must be equal and OFM W-1 and H-1 must be 2x/4x/8x IFM W-1 and H-1, if align_corners is True  
-        W and H scaling must be equal and OFM W and H must be 2x/4x/8x IFM W and H, if align_corners is False
+        W and H scaling must be either 1 or equal and OFM W-1 and H-1 must be 1x/2x/4x/8x IFM W-1 and H-1, if align_corners is True  
+        W and H scaling must be either 1 or equal and OFM W and H must be 1x/2x/4x/8x IFM W and H, if align_corners is False
 - The size tensor must match the output tensor shape
 - Both align_corners and half_pixel_centers can't be True
 - For half_pixel_centers the width and height of the IFM and OFM must match one of the following criteria:  
@@ -405,8 +405,8 @@ This is a list of constraints that the RESIZE_NEAREST_NEIGHBOR operator must sat
 - The width and height of the IFM and OFM must match one of the following criteria:  
         IFM W and H must both be 1  
         IFM must match OFM  
-        W and H scaling must be equal and OFM W-1 and H-1 must be 2x/4x/8x IFM W-1 and H-1, if align_corners is True  
-        W and H scaling must be equal and OFM W and H must be 2x/4x/8x IFM W and H, if align_corners is False
+        W and H scaling must be either 1 or equal and OFM W-1 and H-1 must be 1x/2x/4x/8x IFM W-1 and H-1, if align_corners is True  
+        W and H scaling must be either 1 or equal and OFM W and H must be 1x/2x/4x/8x IFM W and H, if align_corners is False
 - The size tensor must match the output tensor shape
 - Both align_corners and half_pixel_centers can't be True
 
diff --git a/ethosu/vela/tflite_graph_optimiser.py b/ethosu/vela/tflite_graph_optimiser.py
index 62afc009..e80fe1e3 100644
--- a/ethosu/vela/tflite_graph_optimiser.py
+++ b/ethosu/vela/tflite_graph_optimiser.py
@@ -335,7 +335,7 @@ def convert_resize_1x1_to_add(op):
 # Convert ResizeNearestNeighbor with align corners to a depthwise convolution. The IFM will already have been upscaled
 # apart from the final x2 scaling which will be done as part of this operation. The kernel contains a single coefficient
 # to select the appropriate nearest neighbor value
-def convert_resizenn_ac_to_depthwise_conv(op, upscale_factor):
+def convert_resizenn_ac_to_depthwise_conv(op, upscale_factor_h, upscale_factor_w):
     ifm = op.ifm
     ofm = op.ofm
     output_depth = ofm.shape[-1]
@@ -375,7 +375,7 @@ def convert_resizenn_ac_to_depthwise_conv(op, upscale_factor):
         weight_quant.quant_min = -(1 << (ofm_dtype.bits - 1))
         weight_quant.quant_max = (1 << (ofm_dtype.bits - 1)) - 1
 
-    weight_shape = [upscale_factor, upscale_factor, output_depth, output_depth]  # HWIO
+    weight_shape = [upscale_factor_h, upscale_factor_w, output_depth, output_depth]  # HWIO
 
     # the single non-zero coefficient used to select the desired value needs to be placed in the 'centre value', which
     # is calculated by finding the 'centre position' ('*' in the diagram below) and then choosing the 'value' that is
@@ -385,17 +385,15 @@ def convert_resizenn_ac_to_depthwise_conv(op, upscale_factor):
     # 1---*---+
     # | C | D |
     # 2---+---+
-    weight_values = [0] * (upscale_factor * upscale_factor)
-    centre_coeff = (upscale_factor // 2) * upscale_factor + (upscale_factor // 2)
-    weight_values[centre_coeff] = 1
-
+    weight_values = np.zeros(shape=weight_shape)
+    weight_values[upscale_factor_h // 2, upscale_factor_w // 2, :, :] = 1
     # add weight tensor, this will discard the size tensor of the resize op
     op.set_input_tensor(
         create_const_tensor(
             "weights",
             weight_shape,
             ofm_dtype,
-            np.array(weight_values).reshape(weight_shape),
+            weight_values,
             quantization=weight_quant,
         ),
         1,  # inputs tensor weight index
@@ -420,6 +418,7 @@ def convert_resizenn_ac_to_depthwise_conv(op, upscale_factor):
 def convert_resize_to_upscale_and_average_pool(op):
     pre_op = op
     outputs = op.outputs
+    orig_ofm_shape = op.ofm_shapes[0]
     dtype = op.ifm.dtype
 
     op.attrs.update({"strides": (1, 1, 1, 1), "ksize": (1, 1, 1, 1)})
@@ -431,6 +430,12 @@ def convert_resize_to_upscale_and_average_pool(op):
     # Get upscale factor that was calculated in the supported operators check
     upscale_factor = op.attrs["upscale_factor"]
 
+    # Upscale factor can be 1 for one of the dimensions only if the dimension size is 1
+    ifm_h, ifm_w = op.ifm_shapes[0].get_hw_as_list()
+    ofm_h, ofm_w = op.ofm_shapes[0].get_hw_as_list()
+    upscale_factor_h = 1 if ofm_h == ifm_h == 1 else upscale_factor
+    upscale_factor_w = 1 if ofm_w == ifm_w == 1 else upscale_factor
+
     # Calculate how many times 2x2 upscaling needs to be performed
     # Force the result of round to be an integer. This is because the behaviour of rounding numpy.float64 values changed
     # between different versions of numpy. This consistency ensures that the kernel dimensions are kept integral
@@ -444,12 +449,16 @@ def convert_resize_to_upscale_and_average_pool(op):
             scaled_op.inputs[0] = pre_op.outputs[0]
 
         # Nearest neighbor x2 upscaling
-        upscaled_shape = upscaled_shape * 2
+        upscaled_shape[0] *= 2 if upscale_factor_h != 1 else 1
+        upscaled_shape[1] *= 2 if upscale_factor_w != 1 else 1
         shape = op.ofm_shapes[0].as_list()
         shape[1:3] = upscaled_shape
         out_tens = Tensor(shape, dtype, f"{op.outputs[0].name}_{count}")
         out_tens.quantization = op.outputs[0].quantization.clone()
         scaled_op.set_output_tensor(out_tens)
+        # Discard unused data from the 2x2 upscaling when upscaling is 1 in one of the dimensions
+        scaled_op.write_shape = Shape4D(shape)
+        scaled_op.write_offset = Shape4D([0, 0, 0, 0])
         pre_op = scaled_op
 
         scaled_op.set_ifm_ofm_shapes()
@@ -467,20 +476,24 @@ def convert_resize_to_upscale_and_average_pool(op):
         else:
             # padding to the right and bottom (limits average pool to 8x8 kernel)
             scaled_op.attrs["padding"] = Padding.EXPLICIT
-            scaled_op.attrs["explicit_padding"] = [0, 0, upscale_factor - 1, upscale_factor - 1]
+            scaled_op.attrs["explicit_padding"] = [0, 0, upscale_factor_h - 1, upscale_factor_w - 1]
 
         # kernal size dependent on the upscaling factor
-        scaled_op.attrs.update({"ksize": (1, upscale_factor, upscale_factor, 1)})
+        scaled_op.attrs.update({"ksize": (1, upscale_factor_h, upscale_factor_w, 1)})
     else:  # Op.ResizeNearestNeighbor
         if scaled_op.attrs["align_corners"]:
             # use depthwise conv to select the correct value
-            scaled_op = convert_resizenn_ac_to_depthwise_conv(scaled_op, upscale_factor)
+            scaled_op = convert_resizenn_ac_to_depthwise_conv(scaled_op, upscale_factor_h, upscale_factor_w)
         else:
             # Keep 1x1 kernel and average pool, this applies both when
             # half-pixel-centers is True and False. Calculations are the
             # same in the reference.
             pass
 
+    # Set write_shape to OFM size to discard unused data from the 2x2 upscaling when upscaling is 1 in one dimension
+    scaled_op.write_shape = orig_ofm_shape
+    scaled_op.write_offset = Shape4D([0, 0, 0, 0])
+
     scaled_op.outputs = outputs
     scaled_op.outputs[0].ops = [scaled_op]
     scaled_op.set_ifm_ofm_shapes()
diff --git a/ethosu/vela/tflite_supported_operators.py b/ethosu/vela/tflite_supported_operators.py
index 1062e2c6..1fd284b4 100644
--- a/ethosu/vela/tflite_supported_operators.py
+++ b/ethosu/vela/tflite_supported_operators.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
+# SPDX-FileCopyrightText: Copyright 2020-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -735,8 +735,8 @@ class TFLiteSupportedOperators:
         """The width and height of the IFM and OFM must match one of the following criteria:
         IFM W and H must both be 1
         IFM must match OFM
-        W and H scaling must be equal and OFM W-1 and H-1 must be 2x/4x/8x IFM W-1 and H-1, if align_corners is True
-        W and H scaling must be equal and OFM W and H must be 2x/4x/8x IFM W and H, if align_corners is False"""
+        W and H scaling must be either 1 or equal and OFM W-1 and H-1 must be 1x/2x/4x/8x IFM W-1 and H-1, if align_corners is True
+        W and H scaling must be either 1 or equal and OFM W and H must be 1x/2x/4x/8x IFM W and H, if align_corners is False"""
         # Easier to start with False condition as very few cases result in a supported resize
         valid = False
         ifm_shape = op.ifm.shape
@@ -752,18 +752,23 @@ class TFLiteSupportedOperators:
             if ((ifm_shape_h == 1) and (ifm_shape_w == 1)) or (ifm_shape == ofm_shape):
                 valid = True
             else:
-                # Valid if OFM is 2/4/8x IFM (-1 for align corners)
                 if align_corners:
-                    h_upscale_factor = (ofm_shape_h - 1) / (ifm_shape_h - 1)
-                    w_upscale_factor = (ofm_shape_w - 1) / (ifm_shape_w - 1)
+                    # Valid if OFM is 1/2/4/8x IFM (-1 for align corners)
+                    h_upscale_factor = (ofm_shape_h - 1) / (ifm_shape_h - 1) if ifm_shape_h != 1 else 1
+                    w_upscale_factor = (ofm_shape_w - 1) / (ifm_shape_w - 1) if ifm_shape_w != 1 else 1
                 else:
                     h_upscale_factor = ofm_shape_h / ifm_shape_h
                     w_upscale_factor = ofm_shape_w / ifm_shape_w
 
-                # could use either height or width. save as int because it is more usable later in graph optimiser
-                op.attrs["upscale_factor"] = int(h_upscale_factor)
-                valid = h_upscale_factor == w_upscale_factor and h_upscale_factor in (2.0, 4.0, 8.0)
-
+                # Save as int because it is more usable later in graph optimiser
+                op.attrs["upscale_factor"] = int(max(w_upscale_factor, h_upscale_factor))
+                if h_upscale_factor in (1.0, 2.0, 4.0, 8.0) and w_upscale_factor in (1.0, 2.0, 4.0, 8.0):
+                    if (ofm_shape_h != 1) and (ofm_shape_w != 1):
+                        # If neither OFM height or width is 1, upscale factor must be equal
+                        valid = h_upscale_factor == w_upscale_factor
+                    else:
+                        # If one of IFM height or width is 1, allow 1x upscaling in that dimension
+                        valid = (h_upscale_factor == ifm_shape_h == 1) or (w_upscale_factor == ifm_shape_w == 1)
         return valid, f"Op has ifm_shape={ifm_shape}, ofm_shape={ofm_shape} and align_corners={align_corners}"
 
     @staticmethod
diff --git a/ethosu/vela/vela.py b/ethosu/vela/vela.py
index cc28299a..5717683c 100755
--- a/ethosu/vela/vela.py
+++ b/ethosu/vela/vela.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
+# SPDX-FileCopyrightText: Copyright 2020-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -248,7 +248,7 @@ def generate_supported_ops():
     # Add license for supported ops
     lines = [
         "<!--",
-        "SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>",
+        "SPDX-FileCopyrightText: Copyright 2020-2025 Arm Limited and/or its affiliates <open-source-office@arm.com>",
         "",
         "SPDX-License-Identifier: Apache-2.0",
         "",
-- 
GitLab