[dv/dvsim] Group failures per test in buckets

Within each bucket collect failures per unqualified test name.
Cap the failures shown to 5 unqualified test names and qualified
tests 2 for each.
Show the number of failures per bucket and per test.
Use a trailing backslash to separate the line and log info from the
qualified test name.

Signed-off-by: Guillermo Maturana <maturana@google.com>

util/dvsim/SimCfg.py

diff --git a/util/dvsim/SimCfg.py b/util/dvsim/SimCfg.py
index 56c75f7..41826fe 100644
--- a/util/dvsim/SimCfg.py
+++ b/util/dvsim/SimCfg.py
@@ -5,6 +5,7 @@
 Class describing simulation configuration object
 """
 
+import collections
 import logging as log
 import os
 import shutil
@@ -21,7 +22,10 @@
 from testplanner.testplan_utils import parse_testplan
 from utils import VERBOSE, rm_path
 
-_MAX_TESTS_PER_BUCKET = 10
+
+# This affects the bucketizer failure report.
+_MAX_UNIQUE_TESTS = 5
+_MAX_TEST_RESEEDS = 2
 
 
 def pick_wave_format(fmts):
@@ -539,20 +543,67 @@
         is enabled, then the summary coverage report is also generated. The final
         result is in markdown format.
         '''
+        def indent_by(level):
+            return " " * (4 * level)
+
         def create_failure_message(test, line, context):
-            spaces = " " * 12
-            message = [f"    * {test.qual_name}", ""]
+            message = [f"{indent_by(2)}* {test.qual_name}\\"]
             if line:
                 message.append(
-                    f"{spaces}Line {line}, in log {test.get_log_path()}")
+                    f"{indent_by(2)}  Line {line}, in log " +
+                    test.get_log_path())
             else:
-                message.append(f"{spaces}Log {test.get_log_path()}")
+                message.append(f"{indent_by(2)} Log {test.get_log_path()}")
             if context:
-                lines = [f"{spaces}{c.rstrip()}" for c in context]
+                message.append("")
+                lines = [f"{indent_by(4)}{c.rstrip()}" for c in context]
                 message.extend(lines)
             message.append("")
             return message
 
+        def create_bucket_report(buckets):
+            """Creates a report based on the given buckets.
+
+            The buckets are sorted by descending number of failures. Within
+            buckets this also group tests by unqualified name, and just a few
+            failures are shown per unqualified name.
+
+            Args:
+              buckets: A dictionary by bucket containing triples
+                (test, line, context).
+
+            Returns:
+              A list of text lines for the report.
+            """
+            by_tests = sorted(buckets.items(),
+                              key=lambda i: len(i[1]),
+                              reverse=True)
+            fail_msgs = ["\n## Failure Buckets", ""]
+            for bucket, tests in by_tests:
+                fail_msgs.append(f"* `{bucket}` has {len(tests)} failures:")
+                unique_tests = collections.defaultdict(list)
+                for (test, line, context) in tests:
+                    unique_tests[test.name].append((test, line, context))
+                for name, test_reseeds in list(unique_tests.items())[
+                        :_MAX_UNIQUE_TESTS]:
+                    fail_msgs.append(f"{indent_by(1)}* Test {name} has "
+                                     f"{len(test_reseeds)} failures.")
+                    for test, line, context in test_reseeds[:_MAX_TEST_RESEEDS]:
+                        fail_msgs.extend(
+                            create_failure_message(test, line, context))
+                    if len(test_reseeds) > _MAX_TEST_RESEEDS:
+                        fail_msgs.append(
+                            f"{indent_by(2)}* ... and "
+                            f"{len(test_reseeds) - _MAX_TEST_RESEEDS} "
+                            "more failures.")
+                if len(unique_tests) > _MAX_UNIQUE_TESTS:
+                    fail_msgs.append(
+                        f"{indent_by(1)}* ... and "
+                        f"{len(unique_tests) - _MAX_UNIQUE_TESTS} more tests.")
+
+            fail_msgs.append("")
+            return fail_msgs
+
         deployed_items = self.deploy
         results = SimResults(deployed_items, run_results)
 
@@ -612,26 +663,9 @@
             self.results_summary["Name"] = self._get_results_page_link(
                 self.results_summary["Name"])
 
-        # Append bucketized failures for triage, sorted by descending number
-        # of failures.
         if results.buckets:
             self.errors_seen = True
-            by_tests = sorted(results.buckets.items(),
-                              key=lambda i: len(i[1]),
-                              reverse=True)
-            fail_msgs = ["\n## Failure Buckets", ""]
-            for bucket, tests in by_tests:
-                fail_msgs.append(f"* ```{bucket}```:")
-                # The tests could be sorted by ascending wall clock time.
-                for count, (test, line, context) in enumerate(tests):
-                    if count == _MAX_TESTS_PER_BUCKET:
-                        fail_msgs.append(
-                            f"    * ... {len(tests) - count} more tests.")
-                        break
-                    fail_msgs.extend(
-                        create_failure_message(test, line, context))
-            fail_msgs.append("")
-            results_str += "\n".join(fail_msgs)
+            results_str += "\n".join(create_bucket_report(results.buckets))
 
         self.results_md = results_str