From 7d3df1c5da91cd1467fb4d634e768b25b103a70a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vincent=20Stehl=C3=A9?= <vincent.stehle@arm.com>
Date: Mon, 28 Nov 2022 17:08:34 +0100
Subject: [PATCH 1/2] dt-parser.yaml: mark no schema as error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Modify the rules to transform "no schema" warnings from dt-validate into
errors, which will block certification. Add a corresponding unit test.

Signed-off-by: Vincent Stehlé <vincent.stehle@arm.com>
---
 dt-parser.yaml                          | 12 +++++++-----
 schemas/common-schema.yaml              |  2 +-
 tests/data/test-dt-parser/no-schema.log |  1 +
 tests/test-dt-parser                    |  4 ++++
 4 files changed, 13 insertions(+), 6 deletions(-)
 create mode 100644 tests/data/test-dt-parser/no-schema.log

diff --git a/dt-parser.yaml b/dt-parser.yaml
index e988774..768dc8e 100644
--- a/dt-parser.yaml
+++ b/dt-parser.yaml
@@ -195,17 +195,19 @@
   update:
     type: ignored
 
-# We categorizethe following common dt-validate warning entries.
-# This is only to ease analysis; those issues still need to be dealt with.
+# We transform the following dt-validate warning into errors as those issues
+# will block certification.
 
 - rule: Categorize all dt-validate warnings about missing schema as `no schema'
-    for easier analysis.
-    This is only to ease analysis; those issues still need to be dealt with.
+    errors, which will block certification.
   criteria:
     type: dt-validate warning
     warning_message: 'failed to match any schema with compatible: '
   update:
-    type: dt-validate warning no schema
+    type: error no schema
+
+# We categorize the following common dt-validate warning entries.
+# This is only to ease analysis; those issues still need to be dealt with.
 
 - rule: Categorize all dt-validate warnings about non-matching names as
     `naming' for easier analysis.
diff --git a/schemas/common-schema.yaml b/schemas/common-schema.yaml
index dc79944..dfb1b68 100644
--- a/schemas/common-schema.yaml
+++ b/schemas/common-schema.yaml
@@ -16,7 +16,7 @@ $defs:
         enum:
             - dt-validate warning missing property
             - dt-validate warning naming
-            - dt-validate warning no schema
             - dt-validate warning not an array
             - dt-validate warning spurious property
+            - error no schema
             - ignored
diff --git a/tests/data/test-dt-parser/no-schema.log b/tests/data/test-dt-parser/no-schema.log
new file mode 100644
index 0000000..f6ac09d
--- /dev/null
+++ b/tests/data/test-dt-parser/no-schema.log
@@ -0,0 +1 @@
+x.dtb:0:0: /i2c@12345678: failed to match any schema with compatible: ['x,y-i2c']
diff --git a/tests/test-dt-parser b/tests/test-dt-parser
index 2c29b93..d62dbb9 100755
--- a/tests/test-dt-parser
+++ b/tests/test-dt-parser
@@ -63,4 +63,8 @@ dt-parser.py --config /dev/null \
 grep -q '1  dtc warning' "$out"
 echo -n 'filter, ' >&3
 
+dt-parser.py "$data/no-schema.log" |& tee "$out"
+grep -q '1  error no schema' "$out"
+echo -n 'no schema, ' >&3
+
 echo 'ok.' >&3
-- 
GitLab


From 6a83ea4e5fb3f69fd9e0923a1ad3c022cfa450ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vincent=20Stehl=C3=A9?= <vincent.stehle@arm.com>
Date: Mon, 28 Nov 2022 17:54:30 +0100
Subject: [PATCH 2/2] dt-parser.py: de-duplicate entries
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flatten all mmio addresses to a single dummy `x' address and de-duplicate
identical entries (we do not consider the `linenum' field for comparison).
We add a `--no-dedupe' command line option to disable this new behaviour
and we update the unit test.

Signed-off-by: Vincent Stehlé <vincent.stehle@arm.com>
---
 dt-parser.py                               | 79 ++++++++++++++++++++++
 tests/data/test-dt-parser/no-dedupe.log    |  4 ++
 tests/data/test-dt-parser/test-config.yaml |  4 +-
 tests/test-dt-parser                       | 15 ++--
 4 files changed, 96 insertions(+), 6 deletions(-)
 create mode 100644 tests/data/test-dt-parser/no-dedupe.log

diff --git a/dt-parser.py b/dt-parser.py
index 0f2213d..68b3fae 100755
--- a/dt-parser.py
+++ b/dt-parser.py
@@ -174,6 +174,79 @@ def parse(filename):
     return r
 
 
+# Cleanup entries
+# We remove all mmio addresses: name@12345678 -> name@x
+# We modify parsed in-place
+def cleanup_entries(parsed):
+    logging.debug('Cleanup')
+    n = 0
+
+    for i, x in enumerate(parsed):
+        m = n
+
+        for k in x.keys():
+            b = x[k]
+
+            # Consider strings only
+            if not isinstance(b, str):
+                continue
+
+            a = re.sub(r'@[0-9A-Fa-f]+', '@x', x[k])
+
+            if a != b:
+                x[k] = a
+                n += 1
+
+        if n > m:
+            logging.debug(
+                f"Cleaned up {n - m} field(s) of entry {i}, "
+                f"`{x['devicetree_node']}'")
+
+    if n:
+        logging.debug(f"Cleaned up {n} entries")
+
+
+# Compare entries for de-duplication
+# We do not compare non-string keys, such as linenum or dt_validate_lines as
+# they do not get cleaned-up anyway
+# We return True if entries should be de-duplicated
+def dupes(a, b):
+    for k in a.keys():
+        if not isinstance(a[k], str):
+            continue
+
+        if a[k] != b[k]:
+            return False
+
+    return True
+
+
+# De-duplicate entries
+def dedupe_entries(parsed):
+    logging.debug('De-dupe')
+    r = []
+    n = 0
+
+    for i, x in enumerate(parsed):
+        d = False
+
+        # TODO! Better way than comparing each entry.
+        for y in r:
+            if dupes(x, y):
+                d = True
+                break
+
+        if d:
+            logging.debug(f"De-duplicated entry {i}, `{x['devicetree_node']}'")
+            n += 1
+            continue
+
+        r.append(x)
+
+    logging.info(f"De-duplicated {n} entries")
+    return r
+
+
 # Evaluate if an entry matches a criteria
 # The criteria is a dict of Key-value pairs.
 # I.E. crit = {"type": "dt-validate warning", "xxx": "yyy", ...}
@@ -346,6 +419,8 @@ if __name__ == '__main__':
     parser.add_argument(
         '--max-message-len', type=int, help='Maximum message width to print',
         default=128)
+    parser.add_argument(
+        '--no-dedupe', action='store_true', help='Turn off de-duplication')
     parser.add_argument(
         '--print', action='store_true', help='Print non-ignored entries')
     parser.add_argument('--yaml', help='YAML output filename')
@@ -361,6 +436,10 @@ if __name__ == '__main__':
 
     # Modify.
 
+    if not args.no_dedupe:
+        cleanup_entries(p)
+        p = dedupe_entries(p)
+
     apply_rules(p, conf)
 
     if args.filter:
diff --git a/tests/data/test-dt-parser/no-dedupe.log b/tests/data/test-dt-parser/no-dedupe.log
new file mode 100644
index 0000000..204d27b
--- /dev/null
+++ b/tests/data/test-dt-parser/no-dedupe.log
@@ -0,0 +1,4 @@
+/home/vinste01/systemreadyir/dt/dump.dtb: platform@c000000: $nodename:0: 'platform@c000000' does not match '^([a-z][a-z0-9\\-]+-bus|bus|soc|axi|ahb|apb)(@[0-9a-f]+)?$'
+	From schema: /home/vinste01/systemreadyir/dt/dt-schema/dtschema/schemas/simple-bus.yaml
+/home/vinste01/systemreadyir/dt/dump.dtb: platform@d000000: $nodename:0: 'platform@d000000' does not match '^([a-z][a-z0-9\\-]+-bus|bus|soc|axi|ahb|apb)(@[0-9a-f]+)?$'
+	From schema: /home/vinste01/systemreadyir/dt/dt-schema/dtschema/schemas/simple-bus.yaml
diff --git a/tests/data/test-dt-parser/test-config.yaml b/tests/data/test-dt-parser/test-config.yaml
index 293789c..6390b1b 100644
--- a/tests/data/test-dt-parser/test-config.yaml
+++ b/tests/data/test-dt-parser/test-config.yaml
@@ -1,7 +1,7 @@
 ---
-- rule: Force one entry as ignored for testing
+- rule: Force two entries as ignored for testing
   criteria:
-    devicetree_node: /pl011@9040000:clocks
+    devicetree_node: /pl011@x:clocks
     dtc_warning_name: clocks_property
     file: dump.dts
     type: dtc warning
diff --git a/tests/test-dt-parser b/tests/test-dt-parser
index d62dbb9..4e68ce8 100755
--- a/tests/test-dt-parser
+++ b/tests/test-dt-parser
@@ -52,13 +52,13 @@ validate.py --schema schemas/dt-parser-yaml-schema.yaml "$yaml"
 echo -n 'valid, ' >&3
 
 dt-parser.py --config "$data/test-config.yaml" "$data/qemu.log" |& tee "$out"
-grep -q '14  dt-validate warning' "$out"
-grep -q '14  dtc warning' "$out"
-grep -q ' 1  ignored' "$out"
+grep -q '12  dt-validate warning' "$out"
+grep -q '13  dtc warning' "$out"
+grep -q ' 2  ignored' "$out"
 echo -n 'rule, ' >&3
 
 dt-parser.py --config /dev/null \
-	--filter "x['devicetree_node'] == '/pl031@9010000:clocks'" \
+	--filter "x['devicetree_node'] == '/pl031@x:clocks'" \
 	"$data/qemu.log" |& tee "$out"
 grep -q '1  dtc warning' "$out"
 echo -n 'filter, ' >&3
@@ -67,4 +67,11 @@ dt-parser.py "$data/no-schema.log" |& tee "$out"
 grep -q '1  error no schema' "$out"
 echo -n 'no schema, ' >&3
 
+dt-parser.py "$data/no-dedupe.log" |& tee "$out"
+grep -q '1  dt-validate warning' "$out"
+
+dt-parser.py --no-dedupe "$data/no-dedupe.log" |& tee "$out"
+grep -q '2  dt-validate warning' "$out"
+echo -n 'no dedupe, ' >&3
+
 echo 'ok.' >&3
-- 
GitLab