| # Copyright lowRISC contributors. |
| # Licensed under the Apache License, Version 2.0, see LICENSE for details. |
| # SPDX-License-Identifier: Apache-2.0 |
| r"""TestplanEntry and Testplan classes for maintaining testplan entries |
| """ |
| |
| import re |
| import sys |
| from collections import OrderedDict |
| |
| import mistletoe |
| from tabulate import tabulate |
| |
| |
| class TestResult: |
| '''The results for a single test''' |
| def __init__(self, name): |
| self.name = name |
| self.passing = 0 |
| self.total = 0 |
| |
| |
| class TestplanEntry(): |
| """An entry in the testplan |
| |
| A testplan entry has the following information: name of the planned test (testpoint), |
| a brief description indicating intent, stimulus and checking procedure, targeted milestone |
| and the list of actual developed tests. |
| """ |
| name = "" |
| desc = "" |
| milestone = "" |
| tests = [] |
| |
| fields = ("name", "desc", "milestone", "tests") |
| milestones = ("N.A.", "V1", "V2", "V3") |
| |
| def __init__(self, name, desc, milestone, tests, substitutions=[]): |
| self.name = name |
| self.desc = desc |
| self.milestone = milestone |
| self.tests = tests |
| if not self.do_substitutions(substitutions): |
| sys.exit(1) |
| |
| @staticmethod |
| def is_valid_entry(kv_pairs): |
| '''Pass a list of key=value pairs to check if testplan entries can be extracted |
| from it. |
| ''' |
| for field in TestplanEntry.fields: |
| if field not in kv_pairs.keys(): |
| print( |
| "Error: input key-value pairs does not contain all of the ", |
| "required fields to create an entry:\n", kv_pairs, |
| "\nRequired fields:\n", TestplanEntry.fields) |
| return False |
| if type(kv_pairs[field]) is str and kv_pairs[field] == "": |
| print("Error: field \'", field, "\' is an empty string\n:", |
| kv_pairs) |
| return False |
| if field == "milestone" and kv_pairs[ |
| field] not in TestplanEntry.milestones: |
| print("Error: milestone \'", kv_pairs[field], |
| "\' is invalid. Legal values:\n", |
| TestplanEntry.milestones) |
| return False |
| return True |
| |
| def do_substitutions(self, substitutions): |
| '''Substitute {wildcards} in tests |
| |
| If tests have {wildcards}, they are substituted with the 'correct' values using |
| key=value pairs provided by the substitutions arg. If wildcards are present but no |
| replacement is available, then the wildcards are replaced with an empty string. |
| ''' |
| if substitutions == []: |
| return True |
| for kv_pair in substitutions: |
| resolved_tests = [] |
| [(k, v)] = kv_pair.items() |
| for test in self.tests: |
| match = re.findall(r"{([A-Za-z0-9\_]+)}", test) |
| if len(match) > 0: |
| # match is a list of wildcards used in test |
| for item in match: |
| if item == k: |
| if type(v) is list: |
| if v == []: |
| resolved_test = test.replace( |
| "{" + item + "}", "") |
| resolved_tests.append(resolved_test) |
| else: |
| for subst_item in v: |
| resolved_test = test.replace( |
| "{" + item + "}", subst_item) |
| resolved_tests.append(resolved_test) |
| elif type(v) is str: |
| resolved_test = test.replace( |
| "{" + item + "}", v) |
| resolved_tests.append(resolved_test) |
| else: |
| print( |
| "Error: wildcard", item, "in test", test, |
| "has no viable", |
| "replacement value (need str or list):\n", |
| kv_pair) |
| return False |
| else: |
| resolved_tests.append(test) |
| if resolved_tests != []: |
| self.tests = resolved_tests |
| |
| # if wildcards have no available replacements in substitutions arg, then |
| # replace with empty string |
| resolved_tests = [] |
| for test in self.tests: |
| match = re.findall(r"{([A-Za-z0-9\_]+)}", test) |
| if len(match) > 0: |
| for item in match: |
| resolved_tests.append(test.replace("{" + item + "}", "")) |
| if resolved_tests != []: |
| self.tests = resolved_tests |
| return True |
| |
| def map_regr_results(self, regr_results, map_full_testplan=True): |
| '''map regression results to tests in this entry |
| |
| Given a list of regression results (a tuple containing {test name, # passing and |
| # total} find if the name of the test in the results list matches the written tests |
| in this testplan entry. If there is a match, then append the passing / total |
| information. If no match is found, or if self.tests is an empty list, indicate 0/1 |
| passing so that it is factored into the final total. |
| ''' |
| test_results = [] |
| for test in self.tests: |
| found = False |
| for regr_result in regr_results: |
| if test == regr_result["name"]: |
| test_results.append(regr_result) |
| regr_result["mapped"] = True |
| found = True |
| break |
| |
| # if a test was not found in regr results, indicate 0/1 passing |
| if map_full_testplan and not found: |
| test_results.append({"name": test, "passing": 0, "total": 0}) |
| |
| # if no written tests were indicated in the testplan, reuse planned |
| # test name and indicate 0/1 passing |
| if map_full_testplan and self.tests == []: |
| test_results.append({"name": self.name, "passing": 0, "total": 0}) |
| |
| # replace tests with test results |
| self.tests = test_results |
| return regr_results |
| |
| def display(self): |
| print("testpoint: ", self.name) |
| print("description: ", self.desc) |
| print("milestone: ", self.milestone) |
| print("tests: ", self.tests) |
| |
| |
| class Testplan(): |
| """The full testplan |
| |
| This comprises of TestplanEntry entries |
| """ |
| |
| name = "" |
| entries = [] |
| |
| def __init__(self, name): |
| self.name = name |
| self.entries = [] |
| self.results_summary = OrderedDict() |
| self.results = "" |
| |
| if name == "": |
| print("Error: testplan name cannot be empty") |
| sys.exit(1) |
| |
| def entry_exists(self, entry): |
| '''check if new entry has the same name as one of the existing entries |
| ''' |
| for existing_entry in self.entries: |
| if entry.name == existing_entry.name: |
| print("Error: found a testplan entry with name = ", entry.name) |
| print("existing entry:\n", existing_entry) |
| print("new entry:\n", entry) |
| return True |
| return False |
| |
| def add_entry(self, entry): |
| '''add a new entry into the testplan |
| ''' |
| if self.entry_exists(entry): |
| sys.exit(1) |
| self.entries.append(entry) |
| |
| def sort(self): |
| '''sort entries by milestone |
| ''' |
| self.entries = sorted(self.entries, key=lambda entry: entry.milestone) |
| |
| def map_regr_results(self, regr_results, map_full_testplan=True): |
| '''map regression results to testplan entries |
| ''' |
| def sum_results(totals, entry): |
| '''function to generate milestone and grand totals |
| ''' |
| ms = entry.milestone |
| for test in entry.tests: |
| # Create dummy tests entry for milestone total |
| if totals[ms].tests == []: |
| totals[ms].tests = [{ |
| "name": "**TOTAL**", |
| "passing": 0, |
| "total": 0 |
| }] |
| # Sum milestone total |
| totals[ms].tests[0]["passing"] += test["passing"] |
| totals[ms].tests[0]["total"] += test["total"] |
| # Sum grand total |
| if ms != "N.A.": |
| totals["N.A."].tests[0]["passing"] += test["passing"] |
| totals["N.A."].tests[0]["total"] += test["total"] |
| return totals |
| |
| totals = {} |
| # Create entry for total in each milestone; & the grand total. |
| for ms in TestplanEntry.milestones: |
| totals[ms] = TestplanEntry(name="N.A.", |
| desc="Total tests", |
| milestone=ms, |
| tests=[{ |
| "name": "**TOTAL**", |
| "passing": 0, |
| "total": 0 |
| }]) |
| if ms != "N.A.": |
| totals[ms].tests = [] |
| |
| for entry in self.entries: |
| regr_results = entry.map_regr_results(regr_results, |
| map_full_testplan) |
| totals = sum_results(totals, entry) |
| |
| # extract unmapped tests from regr_results and create 'unmapped' entry |
| unmapped_regr_results = [] |
| for regr_result in regr_results: |
| if "mapped" not in regr_result.keys(): |
| unmapped_regr_results.append(regr_result) |
| |
| unmapped = TestplanEntry( |
| name="Unmapped tests", |
| desc="""A list of tests in the regression result that are not |
| mapped to testplan entries.""", |
| milestone="N.A.", |
| tests=unmapped_regr_results) |
| totals = sum_results(totals, unmapped) |
| |
| # add total back into 'entries' |
| for ms in TestplanEntry.milestones[1:]: |
| self.entries.append(totals[ms]) |
| self.sort() |
| self.entries.append(unmapped) |
| self.entries.append(totals["N.A."]) |
| |
| def display(self): |
| '''display the complete testplan for debug |
| ''' |
| print("name: ", self.name) |
| for entry in self.entries: |
| entry.display() |
| |
| def get_milestone_regressions(self): |
| regressions = {} |
| for entry in self.entries: |
| # Skip if milestone is "n.a." |
| if entry.milestone not in entry.milestones[1:]: |
| continue |
| # if ms key doesnt exist, create one |
| if entry.milestone not in regressions.keys(): |
| regressions[entry.milestone] = [] |
| # Append new tests to the list |
| for test in entry.tests: |
| if test not in regressions[entry.milestone] and test != "": |
| regressions[entry.milestone].append(test) |
| |
| # Build regressions dict into a hjson like data structure |
| output = [] |
| for ms in regressions.keys(): |
| ms_dict = {} |
| ms_dict["name"] = ms |
| ms_dict["tests"] = regressions[ms] |
| output.append(ms_dict) |
| return output |
| |
| def testplan_table(self, fmt="pipe"): |
| '''Generate testplan table from hjson entries in the format specified |
| by the 'fmt' arg. |
| ''' |
| table = [["Milestone", "Name", "Description", "Tests"]] |
| colalign = ("center", "center", "left", "left") |
| for entry in self.entries: |
| tests = "" |
| for test in entry.tests: |
| tests += test + "<br>\n" |
| desc = entry.desc.strip() |
| if fmt == "html": |
| desc = mistletoe.markdown(desc) |
| table.append([entry.milestone, entry.name, desc, tests]) |
| result = tabulate(table, |
| headers="firstrow", |
| tablefmt=fmt, |
| colalign=colalign) |
| result = result.replace("<", "<") |
| result = result.replace(">", ">") |
| return result |
| |
| def results_table(self, test_results, map_full_testplan=True, fmt="pipe"): |
| '''Print the mapped regression results into a table in the format |
| specified by the 'fmt' arg. |
| |
| test_results should be a list of TestResult objects, one for each named |
| test. |
| |
| ''' |
| # Generate a list of dictionaries as expected by map_regr_results. In |
| # future, that code could be tightened up to use proper classes too, |
| # but let's put a shim here for now. |
| regr_results = [{"name": tr.name, "passing": tr.passing, "total": tr.total} |
| for tr in test_results] |
| self.map_regr_results(regr_results, map_full_testplan) |
| table = [[ |
| "Milestone", "Name", "Tests", "Passing", "Total", "Pass Rate" |
| ]] |
| colalign = ("center", "center", "left", "center", "center", "center") |
| for entry in self.entries: |
| milestone = entry.milestone |
| entry_name = entry.name |
| if milestone == "N.A.": |
| milestone = "" |
| if entry_name == "N.A.": |
| entry_name = "" |
| for test in entry.tests: |
| if test["total"] == 0: |
| pass_rate = "-- %" |
| else: |
| pass_rate = test["passing"] / test["total"] * 100 |
| pass_rate = "{0:.2f} %".format(round(pass_rate, 2)) |
| table.append([ |
| milestone, entry_name, test["name"], test["passing"], |
| test["total"], pass_rate |
| ]) |
| milestone = "" |
| entry_name = "" |
| if entry.milestone == "N.A." and entry.name == "N.A.": |
| self.results_summary["Name"] = self.name.upper() |
| self.results_summary["Passing"] = test["passing"] |
| self.results_summary["Total"] = test["total"] |
| self.results_summary["Pass Rate"] = pass_rate |
| |
| self.results = tabulate(table, |
| headers="firstrow", |
| tablefmt="pipe", |
| colalign=colalign) |
| return self.results |
