Add util/fpga/bitstream_bisect.py This tool enables us to bisect over cached bitstreams, and optionally narrow down ambiguous results in a second phase. Proposal: https://github.com/lowRISC/opentitan/issues/16406 Signed-off-by: Dan McArdle <dmcardle@google.com>
diff --git a/rules/scripts/BUILD b/rules/scripts/BUILD index 5941449..43824ed 100644 --- a/rules/scripts/BUILD +++ b/rules/scripts/BUILD
@@ -20,10 +20,19 @@ ], ) +py_library( + name = "bitstreams_workspace", + srcs = [ + "bitstreams_workspace.py", + ], +) + py_test( name = "bitstreams_workspace_test", srcs = [ - "bitstreams_workspace.py", "bitstreams_workspace_test.py", ], + deps = [ + ":bitstreams_workspace", + ], )
diff --git a/rules/scripts/bitstreams_workspace.py b/rules/scripts/bitstreams_workspace.py index 21de475..1aa6c16 100755 --- a/rules/scripts/bitstreams_workspace.py +++ b/rules/scripts/bitstreams_workspace.py
@@ -63,6 +63,7 @@ class BitstreamCache(object): + def __init__(self, bucket_url, cachedir, latest_update, offline=False): """Initialize the Bitstream Cache Manager.""" if bucket_url[-1] != '/': @@ -76,6 +77,14 @@ self.offline = offline self.available = {} + @staticmethod + def MakeWithDefaults() -> 'BitstreamCache': + """Create a BitstreamCache with default parameters.""" + args = parser.parse_args([]) + cache = BitstreamCache(args.bucket_url, args.cache, args.latest_update, + args.offline) + return cache + def InitRepository(self): """Create the cache directory and symlink it into the bazel repository dir.""" os.makedirs(self.cachedir, exist_ok=True)
diff --git a/rules/scripts/bitstreams_workspace_test.py b/rules/scripts/bitstreams_workspace_test.py index 7e00782..09abb00 100644 --- a/rules/scripts/bitstreams_workspace_test.py +++ b/rules/scripts/bitstreams_workspace_test.py
@@ -11,6 +11,11 @@ class TestBitstreamCache(unittest.TestCase): + def test_make_with_default(self): + # Changes to command-line argument defaults could break this method, so + # it's important to at least have code coverage. + BitstreamCache.MakeWithDefaults() + def test_get_from_cache(self): BITSTREAM_ORIG = 'lowrisc_systems_chip_earlgrey_cw310_0.1.bit.orig' BITSTREAM_SPLICE = 'lowrisc_systems_chip_earlgrey_cw310_0.1.bit.splice'
diff --git a/util/fpga/BUILD b/util/fpga/BUILD new file mode 100644 index 0000000..9ce1891 --- /dev/null +++ b/util/fpga/BUILD
@@ -0,0 +1,32 @@ +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_python//python:defs.bzl", "py_binary") +load("@ot_python_deps//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "bitstream_bisect_support", + deps = [ + "//rules/scripts:bitstreams_workspace", + requirement("typer"), + requirement("rich"), + ], +) + +py_binary( + name = "bitstream_bisect", + srcs = ["bitstream_bisect.py"], + deps = [":bitstream_bisect_support"], +) + +py_test( + name = "bitstream_bisect_test", + srcs = [ + "bitstream_bisect.py", + "bitstream_bisect_test.py", + ], + deps = [":bitstream_bisect_support"], +)
diff --git a/util/fpga/bitstream_bisect.py b/util/fpga/bitstream_bisect.py new file mode 100644 index 0000000..2ec6e22 --- /dev/null +++ b/util/fpga/bitstream_bisect.py
@@ -0,0 +1,413 @@ +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +r"""Efficiently bisect FPGA regressions with the bitstream cache. + +This tool bisects while minimizing the number of calls to the slow command. +Under the hood, it actually runs up to two git bisects. The first bisect only +tests commits that have cached bitstreams. The optional second bisect narrows +down ambiguous results. (For more information on bisecting, see `man 1 +git-bisect` or <https://git-scm.com/docs/git-bisect>.) + +In the first phase, the tool classifies commits as good/bad/skip with the +user-provided `--fast-command`. In the second phase, it uses `--slow-command`. +These commands classify the current commit via their exit status. Exiting with 0 +indicates that the commit is good. Exiting with 125 indicates the commit cannot +be tested. Exiting with any other status indicates that the commit is bad. + +Run bitstream_bisect.py with Bazel like so: + +./bazelisk.sh run //util/fpga:bitstream_bisect -- \ + --good HEAD~30 \ + --bad HEAD \ + --fast-command "./bazelisk.sh test //sw/device/tests:uart_smoketest_fpga_cw310_rom" \ + --slow-command "./bazelisk.sh test --define bitstream=vivado \ + //sw/device/tests:uart_smoketest_fpga_cw310_rom" + +""" + +import abc +import enum +import os +import re +import shlex +import string +import subprocess +from typing import Dict, List, Optional, Set, Tuple + +import rich +import rich.console +import typer + +from rules.scripts.bitstreams_workspace import BitstreamCache + + +class CommitHash(str): + """A string representation of a Git commit hash. + + The given string must contain only hex digits. It is canonicalized to + lowercase, but not canonicalized relative to the current git repository. As + such, two CommitHash objects may fail to compare equal even though git would + say they refer to the same commit. + """ + + def __new__(cls, s): + t = super().__new__(cls, s).lower() + if len(t) == 0 or len(set(t) - set(string.hexdigits)) > 0: + raise Exception("Invalid CommitHash input: '{}'".format(t)) + return t + + +class CommitJudgment(enum.Enum): + GOOD = enum.auto() + BAD = enum.auto() + SKIP = enum.auto() + + +class BisectLog: + """A container and parser for the results of `git bisect log`. + """ + + class ParserBug(Exception): + pass + + def __init__(self, log: str): + """Parse `log` and initialize the object.""" + lines = [s.rstrip() for s in log.splitlines()] + self.judgments = BisectLog._extract_judgments(lines) + self.candidates = BisectLog._extract_candidates(lines) + + @staticmethod + def _extract_judgments( + lines: List[str]) -> List[Tuple[CommitHash, CommitJudgment]]: + out = [] + for line in lines: + if line == '' or line.startswith("#"): + continue + if line == 'git bisect start': + continue + match = re.match("^git bisect (good|bad|skip) ([0-9a-f]+)$", line) + if match is None: + raise BisectLog.ParserBug( + "Failed to parse line: '{}'".format(line)) + assert len(match.groups()) == 2 + judgment, commit = match.groups() + if judgment == 'good': + out.append((CommitHash(commit), CommitJudgment.GOOD)) + elif judgment == 'bad': + out.append((CommitHash(commit), CommitJudgment.BAD)) + elif judgment == 'skip': + out.append((CommitHash(commit), CommitJudgment.SKIP)) + else: + raise BisectLog.ParserBug("Unreachable") + return out + + @staticmethod + def _extract_candidates(lines: List[str]) -> List[CommitHash]: + # Scan through the log to find a line indicating that there is a single + # first bad commit. + re_first_bad = re.compile(r"^# first bad commit: \[([0-9a-f]+)\] .*$") + for i, line in enumerate(lines): + match = re_first_bad.match(line) + if match is None: + continue + assert len(match.groups()) == 1 + first_bad = match.group(1) + return [CommitHash(first_bad)] + + # Scan through the log to find a sequence of possible first bad commits. + LOG_LINE_ONLY_SKIPPED = "# only skipped commits left to test" + candidate_lines = [] + for i, line in reversed(list(enumerate(lines))): + # When only skipped commits remain, expect a sequence of "possible + # first bad commit" lines. + if line == LOG_LINE_ONLY_SKIPPED: + candidate_lines = lines[i + 1:] + break + + re_possible_first_bad = re.compile( + r"^# possible first bad commit: \[([0-9a-f]+)\] .*$") + candidates = [] + for line in candidate_lines: + match = re_possible_first_bad.match(line) + if match is None: + continue + assert len(match.groups()) == 1 + commit = match.group(1) + candidates.append(CommitHash(commit)) + return candidates + + +class GitControllerABC(abc.ABC): + """An abstract Git interface that will never run the system `git`.""" + + @abc.abstractmethod + def _git(self, *args: str) -> Tuple[int, str, str]: + """Runs git with `args`. Returns (returncode, stdout, stderr).""" + pass + + @abc.abstractmethod + def _git_no_capture_output(self, *args: str) -> int: + """Like `_git()`, but doesn't capture stdout/stderr.""" + pass + + def _git_checked(self, *args: str) -> Tuple[str, str]: + """Like _git(), but raises an exception when the returncode != 0.""" + returncode, stdout, stderr = self._git(*args) + if returncode != 0: + lines = [ + "Git exited with {}".format(returncode), + "---stdout---", + stdout, + "---stderr---", + stderr, + ] + raise Exception("\n".join(lines)) + return (stdout, stderr) + + def rev_parse(self, commitish: str) -> CommitHash: + stdout, _ = self._git_checked("rev-parse", commitish) + return CommitHash(stdout.rstrip()) + + def bisect_start(self): + self._git_checked("bisect", "start") + + def bisect_reset(self): + self._git_checked("bisect", "reset") + + def bisect_good(self, rev: CommitHash): + self._git_checked("bisect", "good", rev) + + def bisect_bad(self, rev: CommitHash): + self._git_checked("bisect", "bad", rev) + + def bisect_skip(self, rev: CommitHash): + self._git_checked("bisect", "skip", rev) + + def bisect_run(self, *args: str) -> BisectLog: + """Executes `git bisect run` without capturing stdout/stderr.""" + # Git's exit status is non-zero when bisect results are ambiguous. + _ = self._git_no_capture_output("bisect", "run", *args) + return self.bisect_log() + + def bisect_log(self) -> BisectLog: + stdout, _ = self._git_checked("bisect", "log") + return BisectLog(stdout) + + def bisect_view(self) -> List[Tuple[CommitHash, str]]: + stdout, _ = self._git_checked("bisect", "view", "--oneline", + "--no-abbrev-commit") + out = [] + for line in stdout.splitlines(): + commit, tail = line.split(" ", 1) + out.append((CommitHash(commit), tail)) + return out + + +class GitController(GitControllerABC): + """A concrete git controller that actually executes git commands.""" + + def __init__(self, git_binary: str, verbose: bool): + self._git_binary = git_binary + self._verbose = verbose + self._console = rich.console.Console() + self._env: Dict[str, str] = dict(os.environ) + # Prevent `git bisect view` from launching the `gitk` GUI. + self._env.pop("DISPLAY", None) + + def _git(self, *args: str) -> Tuple[int, str, str]: + command = [self._git_binary] + list(args) + proc = subprocess.run(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self._env, + encoding="utf-8") + if self._verbose or proc.returncode != 0: + print() + command_str = " ".join(command) + rich.print( + "Command [green]{}[/green] exited with status {}".format( + rich.markup.escape(command_str), proc.returncode)) + rich.print( + rich.panel.Panel(rich.markup.escape(proc.stdout), + title="git stdout")) + rich.print( + rich.panel.Panel(rich.markup.escape(proc.stderr), + title="git stderr")) + return (proc.returncode, proc.stdout, proc.stderr) + + def _git_no_capture_output(self, *args: str) -> int: + command = [self._git_binary] + list(args) + command_str = " ".join(command) + command_str = rich.markup.escape(command_str) + self._console.rule( + "Streaming output from [bold]{}".format(command_str)) + proc = subprocess.run(["git"] + list(args), env=self._env) + self._console.rule("Git exited with status {}".format(proc.returncode)) + self._console.print() + return proc.returncode + + +class BisectSession: + + def __init__(self, + git: GitControllerABC, + cache_keys: Set[CommitHash], + visualize: bool = False): + self._git = git + self._cache_keys: Set[CommitHash] = cache_keys + self._console = rich.console.Console() + self._visualize = visualize + + def run(self, + good_commit: CommitHash, + bad_commit: CommitHash, + fast_command: List[str], + slow_command: Optional[List[str]] = None) -> BisectLog: + """Run an unattended bisect session in one or two phases. + + This method is conceptually equivalent to `git bisect run`. Unlike plain + `git bisect`, it runs in two phases. The first phase only operates on + cached bitstreams. This will only get us so far, so it optionally can + fall back to a slower command, e.g. one that builds the bitstreams it + tests. + + Args: + good_commit: + Commit where `fast_command` and `slow_command` are known to succeed. + bad_commit: + Commit where `fast_command` and `slow_command` are known to fail. + fast_command: + A list of strings that represent a git-bisect script and args. These + are passed to `git-bisect run` in the first phase. + slow_command: + Just like `fast_command`, but for the second phase. If this arg is + None, we skip the second phase. + + Returns: + Parsed output from `git bisect log`, run after the final bisect phase. + """ + + self._git.bisect_start() + self._git.bisect_good(good_commit) + self._git.bisect_bad(bad_commit) + + # Skip commits that aren't in the bitstream cache. + for commit, _ in self._git.bisect_view(): + if commit not in self._cache_keys: + self._git.bisect_skip(commit) + + self._maybe_visualize("Only testing commits with cached bitstreams") + bisect_log = self._git.bisect_run(*fast_command) + + # Skip the second phase when results are unambiguous. + if len(bisect_log.candidates) == 1: + self._maybe_visualize("Final results") + self._git.bisect_reset() + return bisect_log + + # Prepare a new git bisect session where we will run the slow script + # without the benefit of the bitstream cache. Replay all the "good" and + # "bad" commands, but none of the "skip" commands. + if slow_command is None: + self._git.bisect_reset() + return bisect_log + + self._git.bisect_reset() + self._git.bisect_start() + + # Replay only "good" and "bad" decisions from the previous phase. It's + # tempting to unconditionally skip cached commits, but it's better to + # simply reuse judgments from the previous phase. It's possible that the + # user-supplied `fast_command` chose to skip some cached commits for + # reasons outside the scope of this tool, and we now need to test them + # in the second phase. + for commit, judgment in bisect_log.judgments: + if judgment == CommitJudgment.GOOD: + self._git.bisect_good(commit) + elif judgment == CommitJudgment.BAD: + self._git.bisect_bad(commit) + + self._maybe_visualize("Narrowing down ambiguous results") + bisect_log = self._git.bisect_run(*slow_command) + + self._maybe_visualize("Final results") + self._git.bisect_reset() + return bisect_log + + def _maybe_visualize(self, label: str): + """Prints a table summarizing the current range of commits.""" + if not self._visualize: + return + + table = rich.table.Table( + title="Bitstream Bisect Status: [bold]{}".format(label)) + table.add_column("Commit") + table.add_column("Description") + table.add_column("Cached") + table.add_column("Judgment") + + bisect_log = self._git.bisect_log() + judgment_dict: Dict[CommitHash, + CommitJudgment] = dict(bisect_log.judgments) + + for commit, description in self._git.bisect_view(): + judgment = judgment_dict.get(commit) + table.add_row(commit, rich.markup.escape(description), + "Cached" if commit in self._cache_keys else "", + judgment.name if judgment else "") + + self._console.print(table) + + +app = typer.Typer(pretty_exceptions_enable=False, + rich_markup_mode="rich", + add_completion=False) + + +@app.command(help=__doc__) +def main( + good: str = typer.Option(..., help="Commit-ish for known-good revision."), + bad: str = typer.Option(..., help="Commit-ish for known-bad revision."), + fast_command: str = typer.Option( + ..., help="Command for testing commits with cached bitstreams."), + slow_command: Optional[str] = typer.Option( + None, help="Command that tests remaining commits in second phase."), + git_binary: str = typer.Option("/usr/bin/git", help="Path to Git binary."), + verbose: bool = typer.Option( + False, help="Log stdout/stderr of every git command."), +): + # Escape the Bazel sandbox. Without this step, git would not find the .git/ + # directory. Note that this works nicely with git worktrees. + workspace_dir = os.environ['BUILD_WORKSPACE_DIRECTORY'] + os.chdir(workspace_dir) + + # Create a Git controller capable of invoking the system's `git` binary. Use + # it to canonicalize potentially-symbolic "commitish" strings so we don't + # misinterpret them when a different commit is checked out. + git = GitController(git_binary, verbose) + good_commit = git.rev_parse(good) + bad_commit = git.rev_parse(bad) + print("Canonicalized good revision:", good_commit) + print("Canonicalized bad revision: ", bad_commit) + + # NOTE: This will generate network traffic! + # Fetch a listing of the current bitstream cache. + bitstream_cache = BitstreamCache.MakeWithDefaults() + bitstream_cache.GetBitstreamsAvailable(refresh=True) + bitstream_cache_keys = set(bitstream_cache.available.keys()) + + fast_command_pieces = shlex.split(fast_command) + slow_command_pieces = shlex.split( + slow_command) if slow_command is not None else None + + session = BisectSession(git, bitstream_cache_keys, visualize=True) + session.run(good_commit, + bad_commit, + fast_command_pieces, + slow_command=slow_command_pieces) + return 0 + + +if __name__ == '__main__': + app()
diff --git a/util/fpga/bitstream_bisect_test.py b/util/fpga/bitstream_bisect_test.py new file mode 100644 index 0000000..ca222d0 --- /dev/null +++ b/util/fpga/bitstream_bisect_test.py
@@ -0,0 +1,342 @@ +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from typing import Tuple +from unittest.mock import MagicMock, call + +from bitstream_bisect import (BisectLog, BisectSession, CommitHash, + CommitJudgment, GitControllerABC) + + +class MockableGitController(GitControllerABC): + """Provides stub implementations of GitControllerABC's abstract methods.""" + + def _git(self, *args: str) -> Tuple[int, str, str]: + raise Exception("_git unexpectedly called") + + def _git_no_capture_output(self, *args: str) -> int: + returncode, _, _ = self._git(*args) + return returncode + + +class TestCommitHash(unittest.TestCase): + """Test that CommitHash validates its inputs.""" + + def test_simple(self): + c = CommitHash("c1") + self.assertEqual(str(c), "c1") + + c = CommitHash("deadbeef") + self.assertEqual(str(c), "deadbeef") + self.assertEqual(str(c), CommitHash("deadbeef")) + + def test_non_lowercase(self): + c = CommitHash("C1") + self.assertEqual(str(c), "c1") + self.assertEqual(str(c), CommitHash("C1")) + self.assertEqual(str(c), CommitHash("c1")) + + c = CommitHash("DEADBEEF") + self.assertEqual(str(c), "deadbeef") + self.assertEqual(str(c), CommitHash("DEADBEEF")) + self.assertEqual(str(c), CommitHash("deadbeef")) + + c = CommitHash("DeAdBeEf") + self.assertEqual(str(c), "deadbeef") + self.assertEqual(str(c), CommitHash("DeAdBeEf")) + self.assertEqual(str(c), CommitHash("deadbeef")) + + def test_invalid(self): + with self.assertRaises(Exception): + CommitHash("") + with self.assertRaises(Exception): + CommitHash("hello") + with self.assertRaises(Exception): + CommitHash("HEAD") + + +class TestParseBisectLog(unittest.TestCase): + + def test_one_line(self): + bisect_log = BisectLog("git bisect good c123") + self.assertEqual(bisect_log.judgments, [('c123', CommitJudgment.GOOD)]) + self.assertEqual(bisect_log.candidates, []) + + bisect_log = BisectLog("git bisect bad c123") + self.assertEqual(bisect_log.judgments, [('c123', CommitJudgment.BAD)]) + self.assertEqual(bisect_log.candidates, []) + + bisect_log = BisectLog("git bisect skip c123") + self.assertEqual(bisect_log.judgments, [('c123', CommitJudgment.SKIP)]) + self.assertEqual(bisect_log.candidates, []) + + bisect_log = BisectLog("") + self.assertEqual(bisect_log.judgments, []) + self.assertEqual(bisect_log.candidates, []) + + bisect_log = BisectLog("git bisect start") + self.assertEqual(bisect_log.judgments, []) + self.assertEqual(bisect_log.candidates, []) + + def test_unrecognized_actions_rejected(self): + """The parser should reject non-comment lines it doesn't understand.""" + # Unexpected whitespace. + with self.assertRaises(BisectLog.ParserBug): + _ = BisectLog(" git bisect skip c123") + with self.assertRaises(BisectLog.ParserBug): + _ = BisectLog("git bisect skip c123") + with self.assertRaises(BisectLog.ParserBug): + _ = BisectLog("git bisect skip c123") + with self.assertRaises(BisectLog.ParserBug): + _ = BisectLog(" # git bisect skip c123") + # Unrecognized action with commit. + with self.assertRaises(BisectLog.ParserBug): + _ = BisectLog("git bisect foo c123") + # Unrecognized action without commit. + with self.assertRaises(BisectLog.ParserBug): + _ = BisectLog("git bisect bar") + + def test_unrecognized_comments_ignored(self): + bisect_log = BisectLog("# foo") + self.assertEqual(bisect_log.judgments, []) + self.assertEqual(bisect_log.candidates, []) + + def test_candidates_simple(self): + bisect_log = BisectLog("") + self.assertEqual(bisect_log.judgments, []) + self.assertEqual(bisect_log.candidates, []) + + bisect_log = BisectLog( + "# first bad commit: [c42] Commit description...") + self.assertEqual(bisect_log.judgments, []) + self.assertEqual(bisect_log.candidates, ["c42"]) + + bisect_log = BisectLog(""" +# only skipped commits left to test +# possible first bad commit: [c123] Foo description +# possible first bad commit: [c234] Bar description +""") + self.assertEqual(bisect_log.judgments, []) + self.assertEqual(bisect_log.candidates, ["c123", "c234"]) + + def test_realistic_only_skipped_remain(self): + # This output was produced by `git` on a temporary repo. + BISECT_LOG = """# status: waiting for both good and bad commits +# good: [0b47ff5f2ac78589cab7d9b93c4cabb9e4a1f736] First commit +git bisect good 0b47ff5f2ac78589cab7d9b93c4cabb9e4a1f736 +# status: waiting for bad commit, 1 good commit known +# bad: [4f166216806f44b1ddf67c4649d5e646cf9bbf22] Fourth commit +git bisect bad 4f166216806f44b1ddf67c4649d5e646cf9bbf22 +# skip: [132c4e1f8773cd3e14e2f4de8a8b77422b74e28b] Third commit +git bisect skip 132c4e1f8773cd3e14e2f4de8a8b77422b74e28b +# skip: [8fe76ab59c4f7d43237292a57f1d84b774436224] Second commit +git bisect skip 8fe76ab59c4f7d43237292a57f1d84b774436224 +# only skipped commits left to test +# possible first bad commit: [4f166216806f44b1ddf67c4649d5e646cf9bbf22] Fourth commit +# possible first bad commit: [132c4e1f8773cd3e14e2f4de8a8b77422b74e28b] Third commit +# possible first bad commit: [8fe76ab59c4f7d43237292a57f1d84b774436224] Second commit +# good: [8fe76ab59c4f7d43237292a57f1d84b774436224] Second commit +git bisect good 8fe76ab59c4f7d43237292a57f1d84b774436224 +# only skipped commits left to test +# possible first bad commit: [4f166216806f44b1ddf67c4649d5e646cf9bbf22] Fourth commit +# possible first bad commit: [132c4e1f8773cd3e14e2f4de8a8b77422b74e28b] Third commit +""" + bisect_log = BisectLog(BISECT_LOG) + self.assertEqual(bisect_log.candidates, [ + "4f166216806f44b1ddf67c4649d5e646cf9bbf22", + "132c4e1f8773cd3e14e2f4de8a8b77422b74e28b" + ]) + self.assertEqual(bisect_log.judgments, [ + ("0b47ff5f2ac78589cab7d9b93c4cabb9e4a1f736", CommitJudgment.GOOD), + ("4f166216806f44b1ddf67c4649d5e646cf9bbf22", CommitJudgment.BAD), + ("132c4e1f8773cd3e14e2f4de8a8b77422b74e28b", CommitJudgment.SKIP), + ("8fe76ab59c4f7d43237292a57f1d84b774436224", CommitJudgment.SKIP), + ("8fe76ab59c4f7d43237292a57f1d84b774436224", CommitJudgment.GOOD), + ]) + + def test_realistic_has_first_bad_commit(self): + # This output was produced by `git` on a temporary repo. + BISECT_LOG = """git bisect start +# status: waiting for both good and bad commits +# good: [00722da469712cd917e59082601d189fe0c6507e] First commit +git bisect good 00722da469712cd917e59082601d189fe0c6507e +# status: waiting for bad commit, 1 good commit known +# bad: [3e2031c1cb7b27e8bf36c85756c301279b8f9f30] Fourth commit +git bisect bad 3e2031c1cb7b27e8bf36c85756c301279b8f9f30 +# bad: [dc5320f6d974d3fce7bcc03fd18d920c8b5e3cbf] Third commit +git bisect bad dc5320f6d974d3fce7bcc03fd18d920c8b5e3cbf +# bad: [d72b7491e6fdadb2ef742370410d5488a7bbfdba] Second commit +git bisect bad d72b7491e6fdadb2ef742370410d5488a7bbfdba +# first bad commit: [d72b7491e6fdadb2ef742370410d5488a7bbfdba] Second commit +""" + bisect_log = BisectLog(BISECT_LOG) + self.assertEqual(bisect_log.candidates, + ["d72b7491e6fdadb2ef742370410d5488a7bbfdba"]) + self.assertEqual(bisect_log.judgments, [ + ("00722da469712cd917e59082601d189fe0c6507e", CommitJudgment.GOOD), + ("3e2031c1cb7b27e8bf36c85756c301279b8f9f30", CommitJudgment.BAD), + ("dc5320f6d974d3fce7bcc03fd18d920c8b5e3cbf", CommitJudgment.BAD), + ("d72b7491e6fdadb2ef742370410d5488a7bbfdba", CommitJudgment.BAD), + ]) + + +class TestBisectSession(unittest.TestCase): + + def test_two_cached_only_fast(self): + """Simple case with two cached commits and only the fast command.""" + + BISECT_LOG = """git bisect start +# status: waiting for both good and bad commits +# good: [c1] Fake good commit +git bisect good c1 +# status: waiting for bad commit, 1 good commit known +# bad: [c2] Fake bad commit +git bisect bad c2 +# first bad commit: [c2] Fake bad commit +""" + + git = MockableGitController() + git._git = MagicMock(return_value=(0, "", "")) + git.bisect_view = MagicMock(return_value=[ + (CommitHash("c1"), "Commit description"), + (CommitHash("c2"), "Commit description"), + ]) + git.bisect_log = MagicMock(return_value=BisectLog(BISECT_LOG)) + + session = BisectSession(git, cache_keys=set(["c1", "c2"])) + result: BisectLog = session.run("c1", "c2", ["fast_script.sh"]) + + self.assertEqual(result.candidates, ["c2"]) + git._git.assert_has_calls([ + call("bisect", "start"), + call("bisect", "good", "c1"), + call("bisect", "bad", "c2"), + call('bisect', 'run', 'fast_script.sh'), + # We would expect `call('bisect', 'log')` here, but mocking + # `git.bisect_log()` prevented it from calling `git._git()`. + call('bisect', 'reset'), + ]) + git.bisect_log.assert_called_once_with() + git.bisect_view.assert_called_once_with() + + def test_only_fast(self): + """Mix of cached and uncached with only the fast command.""" + + BISECT_LOG = """git bisect start +# status: waiting for both good and bad commits +# skip: [c2] Skipped commit +git bisect skip c2 +# good: [c1] Fake good commit +git bisect good c1 +# status: waiting for bad commit, 1 good commit known +# bad: [c3] Fake bad commit +git bisect bad c3 +# only skipped commits left to test +# possible first bad commit: [c2] Skipped commit +# possible first bad commit: [c3] Fake bad commit +""" + + git = MockableGitController() + git._git = MagicMock(return_value=(0, "", "")) + git.bisect_view = MagicMock(return_value=[ + (CommitHash("c1"), "Fake good commit"), + (CommitHash("c2"), "Skipped commit"), + (CommitHash("c3"), "Fake bad commit"), + ]) + + git.bisect_log = MagicMock(return_value=BisectLog(BISECT_LOG)) + + session = BisectSession(git, cache_keys=set(["c1", "c3"])) + bisect_log = session.run("c1", "c3", ["fast_script.sh"]) + + self.assertEqual(bisect_log.candidates, ["c2", "c3"]) + git._git.assert_has_calls([ + call("bisect", "start"), + call("bisect", "good", "c1"), + call("bisect", "bad", "c3"), + call("bisect", "skip", "c2"), + call('bisect', 'run', 'fast_script.sh'), + # We would expect `call('bisect', 'log')` here, but mocking + # `git.bisect_log()` prevented it from calling `git._git()`. + ]) + git.bisect_log.assert_called_once_with() + git.bisect_view.assert_called_once_with() + + def test_fast_and_slow(self): + """Mix of cached and uncached with fast and slow command.""" + + BISECT_LOG_1 = """git bisect start +# status: waiting for both good and bad commits +# good: [c1] Commit 1 +git bisect good c1 +# status: waiting for bad commit, 1 good commit known +# bad: [c5] Commit 5 +git bisect bad c5 +# skip: [c3] Commit 3 +git bisect skip c3 +# skip: [c4] Commit 4 +git bisect skip c4 +# good: [c2] Commit 2 +git bisect good c2 +# only skipped commits left to test +# possible first bad commit: [c3] Commit 3 +# possible first bad commit: [c4] Commit 4 +""" + BISECT_LOG_2 = """git bisect start +# status: waiting for both good and bad commits +# good: [c1] Commit 1 +git bisect good c1 +# status: waiting for bad commit, 1 good commit known +# bad: [c5] Commit 5 +git bisect bad c5 +# good: [c2] Commit 2 +git bisect good c2 +# good: [c3] Commit 3 +git bisect good c3 +# bad: [c4] Commit 4 +git bisect bad c4 +# first bad commit: [c4] Commit 4 +""" + parsed_bisect_logs = [BisectLog(BISECT_LOG_1), BisectLog(BISECT_LOG_2)] + + git = MockableGitController() + git._git = MagicMock(return_value=(0, "", "")) + git.bisect_log = MagicMock(side_effect=parsed_bisect_logs) + + git.bisect_view = MagicMock(return_value=[ + (CommitHash("c1"), "Commit 1"), + (CommitHash("c2"), "Commit 2"), + (CommitHash("c3"), "Commit 3"), + (CommitHash("c4"), "Commit 4"), + (CommitHash("c5"), "Commit 5"), + ]) + + session = BisectSession(git, cache_keys=set(["c1", "c2"])) + bisect_log = session.run("c1", "c5", ["fast_script.sh"], + ["slow_script.sh"]) + + self.assertEqual(bisect_log.candidates, ["c4"]) + + git._git.assert_has_calls([ + call("bisect", "start"), + call("bisect", "good", "c1"), + call("bisect", "bad", "c5"), + call("bisect", "skip", "c3"), + call("bisect", "skip", "c4"), + call("bisect", "skip", "c5"), + call('bisect', 'run', 'fast_script.sh'), + call('bisect', 'reset'), + call('bisect', 'start'), + call('bisect', 'good', 'c1'), + call('bisect', 'bad', 'c5'), + call('bisect', 'good', 'c2'), + call('bisect', 'run', 'slow_script.sh'), + ]) + git.bisect_log.assert_has_calls([call(), call()]) + git.bisect_view.assert_called_once_with() + + +if __name__ == '__main__': + unittest.main()