commit acc825ff0cb0ee2344aede6fc05337ea7144c4f1
author Srikrishna Iyer <sriyer@google.com> Thu Mar 24 01:15:24 2022 -0700
committer Srikrishna Iyer <46467186+sriyerg@users.noreply.github.com> Thu Mar 24 11:15:42 2022 -0700
tree 79fa295c55570ede085c7f9e431f9dfff9f8f6bd
parent 02df0706c967597c1a77a2109689b802f823cbdb

[dvsim] Treat `tests: ["N/A"]` as an ignored testpoint

This came up in OTP ctrl review.
With this commit, a testplan entry can be discarded by setting `tests: ["N/A"]`.
This is useful for keeping the test entry, but not having it show up in the
mapped simulation results / progress table.

Signed-off-by: Srikrishna Iyer <sriyer@google.com>

util/dvsim/Testplan.py

diff --git a/util/dvsim/Testplan.py b/util/dvsim/Testplan.py
index aada841..2a3e4b7 100644
--- a/util/dvsim/Testplan.py
+++ b/util/dvsim/Testplan.py
@@ -17,6 +17,7 @@
 
 class Result:
     '''The results for a single test'''
+
     def __init__(self, name, passing=0, total=0):
         self.name = name
         self.passing = passing
@@ -138,6 +139,12 @@
         # testpoint.
         self.test_results = []
 
+        # If tests key is set to ["N/A"], then don't map this testpoint to the
+        # simulation results.
+        self.not_mapped = False
+        if self.tests == ["N/A"]:
+            self.not_mapped = True
+
     def __str__(self):
         return super().__str__() + (f"  Milestone: {self.milestone}\n"
                                     f"  Tests: {self.tests}\n")
@@ -192,6 +199,17 @@
         self.tests is an empty list, indicate 0/1 passing so that it is
         factored into the final total.
         """
+        # If no written tests were indicated for this testpoint, then reuse
+        # the testpoint name to count towards "not run".
+        if not self.tests:
+            self.test_results = [Result(name=self.name, passing=0, total=0)]
+            return
+
+        # Skip if this testpoint is not meant to be mapped to the simulation
+        # results.
+        if self.not_mapped:
+            return
+
         for tr in test_results:
             assert isinstance(tr, Result)
             if tr.name in self.tests:
@@ -205,11 +223,6 @@
             if test not in tests_mapped:
                 self.test_results.append(Result(name=test, passing=0, total=0))
 
-        # If no written tests were indicated for this testpoint, then reuse
-        # the testpoint name to count towards "not run".
-        if not self.tests:
-            self.test_results = [Result(name=self.name, passing=0, total=0)]
-
 
 class Testplan:
     """The full testplan
@@ -450,6 +463,8 @@
     def get_milestone_regressions(self):
         regressions = defaultdict(set)
         for tp in self.testpoints:
+            if tp.not_mapped:
+                continue
             if tp.milestone in tp.milestones[1:]:
                 regressions[tp.milestone].update({t for t in tp.tests if t})
 
@@ -542,6 +557,9 @@
             """
             ms = testpoint.milestone
             for tr in testpoint.test_results:
+                if not tr:
+                    continue
+
                 if tr.name in tests_seen:
                     continue
 

util/dvsim/doc/testplanner.md

diff --git a/util/dvsim/doc/testplanner.md b/util/dvsim/doc/testplanner.md
index f431802..cac827b 100644
--- a/util/dvsim/doc/testplanner.md
+++ b/util/dvsim/doc/testplanner.md
@@ -47,6 +47,8 @@
     To cater to these needs, we provide the ability to set a list of written tests for each testpoint.
     It is used to not only indicate the current progress so far into each milestone, but also map the simulation results to the testpoints to generate the final report table.
     This list is initially empty - it is gradually updated as tests are written.
+    Setting this list to `["N/A"]` will prevent this testpoint entry from being mapped to the simulation results.
+    The testpoint will however, still show up in the generated testplan table.
 
 * **tags: list of tags relevant for this testpoint**