| #!/usr/bin/env python3 |
| |
| # Copyright lowRISC contributors. |
| # Licensed under the Apache License, Version 2.0, see LICENSE for details. |
| # SPDX-License-Identifier: Apache-2.0 |
| |
| import unittest |
| import subprocess |
| from collections import namedtuple |
| from pathlib import ( |
| Path, |
| PurePath, |
| ) |
| from unittest.mock import ( |
| Mock, |
| mock_open, |
| patch, |
| ) |
| |
| import common |
| |
| |
| class AnyStringWith(str): |
| |
| def __eq__(self, other): |
| return self in other |
| |
| |
| class AnyDict(dict): |
| |
| def __eq__(self, other): |
| return True |
| |
| |
| class TestCommon(unittest.TestCase): |
| |
| RUN_ARGS = { |
| "check": True, |
| "encoding": "ascii", |
| "errors": "ignore", |
| "env": AnyDict(), |
| "stderr": subprocess.PIPE, |
| "stdout": subprocess.PIPE, |
| } |
| |
| @patch("common.subprocess.run") |
| def test_run_single_arg_error(self, mock_run): |
| mock_run.side_effect = Exception("foo") |
| with self.assertRaisesRegex(Exception, "foo"): |
| common.run("ls") |
| mock_run.assert_called_once_with(("ls", ), **self.RUN_ARGS) |
| |
| @patch("common.subprocess.run") |
| def test_run_multi_arg_error(self, mock_run): |
| mock_run.side_effect = Exception("foo") |
| with self.assertRaisesRegex(Exception, "foo"): |
| common.run("ls", "-la", "./") |
| mock_run.assert_called_once_with(("ls", "-la", "./"), **self.RUN_ARGS) |
| |
| @patch("common.subprocess.run") |
| def test_run_single_arg_success(self, mock_run): |
| MockStreams = namedtuple("MockStdout", ["stdout", "stderr"]) |
| mock_run.side_effect = (MockStreams( |
| stdout="\nline 1\n\nline 2\nline3\n\n", stderr=""), ) |
| self.assertEqual(common.run("ls"), ["line 1", "line 2", "line3"]) |
| mock_run.assert_called_once_with(("ls", ), **self.RUN_ARGS) |
| |
| @patch("common.subprocess.run") |
| def test_run_multi_arg_success(self, mock_run): |
| common.run("ls", "-la", "./") |
| mock_run.assert_called_once_with(("ls", "-la", "./"), **self.RUN_ARGS) |
| |
| # Use `autospec=True` to mock the unbounded `Path.mkdir` and `Path.__truediv__` |
| # methods so that we can assert the values of both the `self` and the `other` |
| # arguments of calls. |
| @patch("pathlib.Path.mkdir", autospec=True) |
| @patch("pathlib.Path.__truediv__", autospec=True) |
| @patch("common.run") |
| def test_create_out_dir(self, mock_run, mock_path_div, mock_path_mkdir): |
| out_root_dir = Path("foo") |
| head_hash = ["123abc"] |
| head_timestamp = ["123456"] |
| out_sub_dir = PurePath("unittest") |
| dir_1 = Path(f"{out_root_dir}/{head_timestamp[0]}-{head_hash[0]}/") |
| dir_2 = Path(f"{dir_1}/{out_sub_dir}/") |
| |
| mock_run.side_effect = (head_hash, head_timestamp) |
| mock_path_div.side_effect = (dir_1, dir_2) |
| |
| self.assertEqual(common.create_out_dir(out_root_dir, out_sub_dir), |
| dir_2) |
| |
| calls_run = mock_run.call_args_list |
| self.assertEqual(len(calls_run), 2) |
| self.assertEqual(calls_run[0][0], ("git", "rev-parse", "HEAD")) |
| self.assertEqual(calls_run[1][0], |
| ("git", "show", "-s", "--format=%ct", "HEAD")) |
| |
| calls_path_div = mock_path_div.call_args_list |
| self.assertEqual(len(calls_path_div), 2) |
| self.assertEqual(calls_path_div[0][0], (out_root_dir, "123456-123abc")) |
| self.assertEqual(calls_path_div[1][0], (Path(dir_1), out_sub_dir)) |
| |
| mock_path_mkdir.assert_called_once_with(dir_2, |
| parents=True, |
| exist_ok=True) |
| |
| @patch("common.run") |
| def test_instrument_device_libs(self, mock_run): |
| device_libs = ["//foo:bar", "//foo:baz"] |
| obj_files = ["foo.o", "bar.o", "baz.o"] |
| |
| mock_run.side_effect = (None, obj_files) |
| |
| self.assertEqual(common.instrument_device_libs(device_libs, 'qux'), |
| obj_files) |
| calls_run = mock_run.call_args_list |
| self.assertEqual(len(calls_run), 2) |
| self.assertEqual(calls_run[0][0], |
| (common.BAZEL, "build", "--config=qux", *device_libs)) |
| starlark_list = "[f.path for f in target.output_groups.compilation_outputs.to_list()]" |
| self.assertEqual(calls_run[1][0], |
| (common.BAZEL, "cquery", "--config=qux", |
| "(//foo:bar + //foo:baz)", '--output=starlark', |
| f"--starlark:expr='\\n'.join({starlark_list})")) |
| |
| @patch("common.run") |
| def test_get_test_log_dirs(self, mock_run): |
| test_targets = ["//foo:bar", "//foo:baz"] |
| test_log_dir = ["/qux/testlogs"] |
| |
| mock_run.side_effect = (test_log_dir, ) |
| |
| self.assertEqual( |
| common.get_test_log_dirs(test_targets), |
| [Path("/qux/testlogs/foo/bar"), |
| Path("/qux/testlogs/foo/baz")]) |
| mock_run.assert_called_once_with(common.BAZEL, "info", |
| "bazel-testlogs") |
| |
| @patch("common.print") |
| @patch("pathlib.Path.open", new_callable=mock_open) |
| @patch("pathlib.Path.__truediv__", |
| side_effect=Path.__truediv__, |
| autospec=True) |
| @patch("common.run") |
| def test_generate_report(self, mock_run, mock_path_div, mock_path_open, |
| mock_print): |
| for print_text_report in (True, False): |
| for m in [mock_run, mock_path_div, mock_path_open, mock_print]: |
| m.reset_mock() |
| with self.subTest(f"print_text_report:{print_text_report}"): |
| out_dir = Path("foo") |
| merged_profile = Path("merged.profdata") |
| merged_library = Path("merged.so") |
| report_title = "My Report" |
| text_report_content = [ |
| "<contents of coverage report in text format>" |
| ] |
| # (1) html report, (2) text report, (3) print text report (if requested) |
| mock_run.side_effect = ([], text_report_content, |
| text_report_content) |
| |
| common.generate_report(out_dir, merged_profile, merged_library, |
| report_title, print_text_report) |
| |
| calls_run = mock_run.call_args_list |
| self.assertEqual( |
| calls_run[0][0], |
| (common.LLVM_COV, "show", "--show-line-counts", |
| "--show-regions", f"--project-title={report_title}", |
| "--format=html", "--output-dir=./foo", "--instr-profile", |
| str(merged_profile), str(merged_library))) |
| self.assertEqual(calls_run[1][0], |
| (common.LLVM_COV, "report", "--instr-profile", |
| str(merged_profile), str(merged_library))) |
| mock_path_div.assert_called_once_with(out_dir, "report.txt") |
| mock_path_open.assert_called_once_with("w") |
| mock_path_open().write.assert_called_once_with( |
| *text_report_content) |
| |
| if print_text_report: |
| self.assertEqual(len(calls_run), 3) |
| self.assertEqual(calls_run[2][0], |
| (common.LLVM_COV, "report", "--use-color", |
| "--instr-profile", str(merged_profile), |
| str(merged_library))) |
| |
| calls_print = mock_print.call_args_list |
| self.assertEqual(len(calls_print), 2) |
| self.assertEqual(calls_print[0][0], |
| (*text_report_content, )) |
| self.assertEqual(calls_print[1][0], |
| AnyStringWith("coverage artifacts in")) |
| else: |
| self.assertEqual(len(calls_run), 2) |
| mock_print.assert_called_once_with( |
| AnyStringWith("coverage artifacts in")) |
| |
| @patch("common.generate_report") |
| @patch("common.get_test_log_dirs") |
| @patch("common.instrument_device_libs") |
| @patch("common.run") |
| @patch("common.create_out_dir") |
| def test_measure_coverage(self, mock_create_out_dir, mock_run, |
| mock_instrument_device_libs, |
| mock_get_test_logs_dir, mock_generate_report): |
| out_root_dir = Path("foo") |
| out_sub_dir = PurePath("bar") |
| out_dir = Path("{out_root_dir}/123456-cafecafe/{out_sub_dir}") |
| device_libs_all = ["//foo:bar1", "//foo:bar2", "//foo:bar3"] |
| device_libs = ["//for:bar2", "//foo:bar3"] |
| obj_files = ["//foo:bar2.o", "//foo:bar3.o"] |
| merged_library = out_dir / "merged.so" |
| bazel_test_type = common.BazelTestType.CC_TEST |
| config = "my_config" |
| test_targets_all = ["//foo:test1", "//foo:test2", "//foo:test3"] |
| test_targets = ["//foo:test1", "//foo:test3"] |
| test_log_dirs = ["/testlogs/foo/test1", "/testlogs/foo/test3"] |
| profile_files = [ |
| "/testlogs/foo/test1/coverage.profdata", |
| "/testlogs/foo/test3/coverage.profdata" |
| ] |
| merged_profile = out_dir / "merged.profdata" |
| report_title = "My Report" |
| print_text_report = True |
| |
| mock_libs_fn = Mock(side_effect=(device_libs, )) |
| mock_objs_fn = Mock(side_effect=(obj_files, )) |
| mock_test_targets_fn = Mock(side_effect=(test_targets, )) |
| mock_test_log_dirs_fn = Mock(side_effect=(profile_files, )) |
| |
| mock_create_out_dir.side_effect = (out_dir, ) |
| mock_run.side_effect = ( |
| device_libs_all, |
| test_targets_all, |
| None, |
| None, |
| ) |
| mock_instrument_device_libs.side_effect = (obj_files, ) |
| mock_get_test_logs_dir.side_effect = (test_log_dirs, ) |
| |
| common.measure_coverage( |
| bazel_test_type=bazel_test_type, |
| config=config, |
| libs_fn=mock_libs_fn, |
| log_level=common.LogLevel.NONE, |
| objs_fn=mock_objs_fn, |
| out_root_dir=out_root_dir, |
| out_sub_dir=out_sub_dir, |
| print_text_report=print_text_report, |
| report_title=report_title, |
| test_log_dirs_fn=mock_test_log_dirs_fn, |
| test_targets_fn=mock_test_targets_fn, |
| ) |
| |
| mock_create_out_dir.assert_called_once_with(out_root_dir, out_sub_dir) |
| calls_run = mock_run.call_args_list |
| self.assertEqual(len(calls_run), 4) |
| self.assertEqual(calls_run[0][0], |
| (common.BAZEL, "query", common.DEVICE_LIBS_QUERY)) |
| mock_libs_fn.assert_called_once_with(device_libs_all) |
| mock_instrument_device_libs.assert_called_once_with( |
| device_libs, config) |
| mock_objs_fn.assert_called_once_with(out_dir / "merged.so", obj_files) |
| self.assertEqual( |
| calls_run[1][0], |
| (common.BAZEL, "query", AnyStringWith(bazel_test_type.value))) |
| mock_test_targets_fn.assert_called_once_with(test_targets_all) |
| self.assertEqual( |
| calls_run[2][0], |
| (common.BAZEL, "coverage", "--define", "bitstream=skip", |
| f"--config={config}", "--test_output=all", *test_targets)) |
| mock_get_test_logs_dir.assert_called_once_with(test_targets) |
| mock_test_log_dirs_fn.assert_called_once_with(test_log_dirs) |
| self.assertEqual(calls_run[3][0], |
| (common.LLVM_PROFDATA, "merge", "--sparse", |
| "--output", str(merged_profile), *profile_files)) |
| mock_generate_report.assert_called_once_with(out_dir, merged_profile, |
| merged_library, |
| report_title, |
| print_text_report) |
| |
| |
| if __name__ == "__main__": |
| unittest.main() |
