| #!/usr/bin/env python3 |
| # Copyright 2021 The IREE Authors |
| # |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. |
| # See https://llvm.org/LICENSE.txt for license information. |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| """Generates benchmark results as pull request comments. |
| |
| This script is meant to be used by CI and uses pip package "markdown_strings". |
| """ |
| |
| import sys |
| import pathlib |
| |
| # Add build_tools python dir to the search path. |
| sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) |
| |
| import argparse |
| import collections |
| import dataclasses |
| import json |
| from typing import Any, Dict, Optional, Set, Tuple |
| |
| import markdown_strings as md |
| import requests |
| |
| from common import benchmark_definition, benchmark_presentation, common_arguments |
| from reporting import benchmark_comment |
| |
| GITHUB_IREE_REPO_PREFIX = "https://github.com/openxla/iree" |
| IREE_DASHBOARD_URL = "https://perf.iree.dev/apis/v2" |
| IREE_PROJECT_ID = "IREE" |
| # The maximal numbers of trials when querying base commit benchmark results. |
| MAX_BASE_COMMIT_QUERY_COUNT = 10 |
| # The max number of rows to show per table. |
| TABLE_SIZE_CUT = 3 |
| THIS_DIRECTORY = pathlib.Path(__file__).resolve().parent |
| |
| |
| @dataclasses.dataclass(frozen=True) |
| class CommentDef(object): |
| title: str |
| type_id: str |
| |
| |
| # Map from comment type to comment definition. |
| COMMENT_DEF_MAP = { |
| "android-benchmark-summary": CommentDef( |
| title="Abbreviated Android Benchmark Summary", |
| type_id="bf8cdf94-a992-466d-b11c-778cbd805a22", |
| ), |
| "linux-benchmark-summary": CommentDef( |
| title="Abbreviated Linux Benchmark Summary", |
| type_id="37549014-3c67-4e74-8d88-8e929231abe3", |
| ), |
| "benchmark-summary": CommentDef( |
| title="Abbreviated Benchmark Summary", |
| type_id="5b42cbfe-26a0-4164-a51c-07f06762e2dc", |
| ), |
| } |
| |
| |
| def get_git_total_commit_count(commit: str, verbose: bool = False) -> int: |
| """Gets the total commit count in history ending with the given commit.""" |
| # TODO(#11703): Should use --first-parent here. See issue for the required |
| # work. |
| count = benchmark_definition.execute_cmd_and_get_stdout( |
| ["git", "rev-list", "--count", commit], cwd=THIS_DIRECTORY, verbose=verbose |
| ) |
| return int(count) |
| |
| |
| def get_from_dashboard( |
| url: str, payload: Dict[str, Any], verbose: bool = False |
| ) -> Dict[str, Dict[str, Any]]: |
| headers = {"Content-type": "application/json"} |
| data = json.dumps(payload) |
| |
| if verbose: |
| print(f"API request payload: {data}") |
| |
| response = requests.get(url, data=data, headers=headers) |
| code = response.status_code |
| if code != 200: |
| raise requests.RequestException( |
| f"Failed to get from dashboard server with status code {code}" |
| ) |
| |
| data = response.json() |
| if verbose: |
| print(f"Queried base benchmark data: {data}") |
| return data |
| |
| |
| BenchmarkQueryResults = Dict[str, Dict[str, Any]] |
| |
| |
| def query_base_benchmark_results( |
| commit: str, verbose: bool = False |
| ) -> BenchmarkQueryResults: |
| """Queries the benchmark results for the given commit.""" |
| build_id = get_git_total_commit_count(commit, verbose) |
| payload = {"projectId": IREE_PROJECT_ID, "buildId": build_id} |
| return get_from_dashboard( |
| f"{IREE_DASHBOARD_URL}/getBuild", payload, verbose=verbose |
| ) |
| |
| |
| @dataclasses.dataclass(frozen=True) |
| class ComparableBenchmarkResults(object): |
| commit_sha: str |
| benchmark_results: BenchmarkQueryResults |
| |
| |
| def _find_comparable_benchmark_results( |
| start_commit: str, required_benchmark_keys: Set[str], verbose: bool = False |
| ) -> Optional[ComparableBenchmarkResults]: |
| cmds = [ |
| "git", |
| "rev-list", |
| "--first-parent", |
| f"--max-count={MAX_BASE_COMMIT_QUERY_COUNT}", |
| start_commit, |
| ] |
| output = benchmark_definition.execute_cmd_and_get_stdout( |
| cmds, cwd=THIS_DIRECTORY, verbose=verbose |
| ) |
| previous_commits = output.splitlines() |
| # Try to query some base benchmark to diff against, from the top of the |
| # tree. Bail out if the maximal trial number is exceeded. |
| for base_commit in previous_commits: |
| base_benchmarks = query_base_benchmark_results( |
| commit=base_commit, verbose=verbose |
| ) |
| base_benchmark_keys = set(base_benchmarks.keys()) |
| if required_benchmark_keys <= base_benchmark_keys: |
| return ComparableBenchmarkResults( |
| commit_sha=base_commit, benchmark_results=base_benchmarks |
| ) |
| |
| return None |
| |
| |
| def _get_git_commit_hash(ref: str, verbose: bool = False) -> str: |
| """Gets the commit hash for the given commit.""" |
| return benchmark_definition.execute_cmd_and_get_stdout( |
| ["git", "rev-parse", ref], cwd=THIS_DIRECTORY, verbose=verbose |
| ) |
| |
| |
| def _get_git_merge_base_commit( |
| pr_commit: str, target_branch: str, verbose: bool = False |
| ) -> str: |
| return benchmark_definition.execute_cmd_and_get_stdout( |
| args=["git", "merge-base", target_branch, pr_commit], |
| cwd=THIS_DIRECTORY, |
| verbose=verbose, |
| ) |
| |
| |
| def _get_experimental_dt_comparison_markdown( |
| execution_benchmarks: Dict[str, benchmark_presentation.AggregateBenchmarkLatency], |
| ) -> Optional[str]: |
| """Get the comparison table to compare different data-tiling options.""" |
| |
| dt_tags = {"no-dt": "No-DT (baseline)", "dt-only": "DT-Only", "dt-uk": "DT-UK"} |
| latency_map = collections.defaultdict(dict) |
| for bench_id, latency in execution_benchmarks.items(): |
| dt_tag = next((tag for tag in dt_tags if tag in latency.name), None) |
| if dt_tag is None: |
| continue |
| # See build_tools/python/e2e_test_framework/definitions/iree_definitions.py |
| # for how benchmark names are constructed. |
| # Format: model_name gen_tags exec_tags ... |
| model, gen_tags, remaining = latency.name.split(" ", maxsplit=2) |
| # Format: [compile targets][tags] |
| compile_targets = gen_tags.split("][")[0] + "]" |
| key_name = " ".join([model, compile_targets, remaining]) |
| latency_map[key_name][dt_tag] = (bench_id, latency.mean_time / 1e6) |
| |
| if len(latency_map) == 0: |
| return None |
| |
| # Compute speedup vs. the baseline. |
| table = {} |
| for key_name, data in latency_map.items(): |
| baseline = data.get("no-dt") |
| baseline = None if baseline is None else baseline[1] |
| row = {} |
| for dt_tag in dt_tags: |
| pair = data.get(dt_tag) |
| if pair is None: |
| continue |
| bench_id, mean_time = pair |
| text = f"{mean_time:.03f}" |
| if baseline is not None: |
| text += f" ({(baseline / mean_time):.01f}X)" |
| row[dt_tag] = (bench_id, text) |
| table[key_name] = row |
| |
| table_columns = [["Name"] + list(table.keys())] |
| for dt_tag, dt_name in dt_tags.items(): |
| column = [dt_name] |
| for key_name, data in table.items(): |
| pair = data.get(dt_tag) |
| if pair is None: |
| column.append("N/A") |
| continue |
| bench_id, text = pair |
| column.append(benchmark_presentation.make_series_link(text, bench_id)) |
| table_columns.append(column) |
| |
| return md.table(table_columns) |
| |
| |
| def _get_benchmark_result_markdown( |
| execution_benchmarks: Dict[str, benchmark_presentation.AggregateBenchmarkLatency], |
| compilation_metrics: Dict[str, benchmark_presentation.CompilationMetrics], |
| pr_url: str, |
| build_url: str, |
| comment_def: CommentDef, |
| commit_info_md: str, |
| ) -> Tuple[str, str]: |
| """Gets the full/abbreviated markdown summary of all benchmarks in files.""" |
| |
| pr_info = md.link("Pull request", pr_url) |
| build_info = md.link("Build", build_url) |
| |
| # Compose the full benchmark tables. |
| full_table = [md.header("Full Benchmark Summary", 2)] |
| full_table.append(md.unordered_list([commit_info_md, pr_info, build_info])) |
| |
| # Compose the abbreviated benchmark tables. |
| abbr_table = [md.header(comment_def.title, 2)] |
| abbr_table.append(commit_info_md) |
| |
| # The temporary table to help compare different data-tiling options. |
| dt_cmp_table = _get_experimental_dt_comparison_markdown( |
| execution_benchmarks=execution_benchmarks |
| ) |
| if dt_cmp_table is not None: |
| dt_cmp_header = md.header("Data-Tiling Comparison Table", 3) |
| full_table += [dt_cmp_header, dt_cmp_table] |
| abbr_table += [ |
| dt_cmp_header, |
| "<details>", |
| "<summary>Click to show</summary>", |
| dt_cmp_table, |
| "</details>", |
| ] |
| |
| if len(execution_benchmarks) > 0: |
| full_table.append( |
| benchmark_presentation.categorize_benchmarks_into_tables( |
| execution_benchmarks |
| ) |
| ) |
| |
| abbr_benchmarks_tables = ( |
| benchmark_presentation.categorize_benchmarks_into_tables( |
| execution_benchmarks, TABLE_SIZE_CUT |
| ) |
| ) |
| if len(abbr_benchmarks_tables) == 0: |
| abbr_table.append("No improved or regressed benchmarks 🏖️") |
| else: |
| abbr_table.append(abbr_benchmarks_tables) |
| |
| # Compose the full compilation metrics tables. |
| if len(compilation_metrics) > 0: |
| full_table.append( |
| benchmark_presentation.categorize_compilation_metrics_into_tables( |
| compilation_metrics |
| ) |
| ) |
| |
| abbr_compilation_metrics_tables = ( |
| benchmark_presentation.categorize_compilation_metrics_into_tables( |
| compilation_metrics, TABLE_SIZE_CUT |
| ) |
| ) |
| if len(abbr_compilation_metrics_tables) == 0: |
| abbr_table.append("No improved or regressed compilation metrics 🏖️") |
| else: |
| abbr_table.append(abbr_compilation_metrics_tables) |
| |
| abbr_table.append("For more information:") |
| # We don't know until a Gist is really created. Use a placeholder for now and |
| # replace later. |
| full_result_info = md.link( |
| "Full benchmark result tables", benchmark_comment.GIST_LINK_PLACEHORDER |
| ) |
| abbr_table.append(md.unordered_list([full_result_info, build_info])) |
| |
| # Append the unique comment type id to help identify and update the existing |
| # comment. |
| abbr_table.append(f"<!--Comment type id: {comment_def.type_id}-->") |
| |
| return "\n\n".join(full_table), "\n\n".join(abbr_table) |
| |
| |
| def parse_arguments(): |
| """Parses command-line options.""" |
| |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| "--benchmark_files", |
| metavar="<benchmark-json-files>", |
| default=[], |
| action="append", |
| help=( |
| "Paths to the JSON files containing benchmark results, " "accepts wildcards" |
| ), |
| ) |
| parser.add_argument( |
| "--compile_stats_files", |
| metavar="<compile-stats-json-files>", |
| default=[], |
| action="append", |
| help=( |
| "Paths to the JSON files containing compilation statistics, " |
| "accepts wildcards" |
| ), |
| ) |
| parser.add_argument("--pr_number", required=True, type=int, help="PR number") |
| parser.add_argument( |
| "--pr_committish", type=str, default="HEAD", help="PR commit hash or ref" |
| ) |
| parser.add_argument( |
| "--pr_base_branch", type=str, default=None, help="Base branch to merge the PR." |
| ) |
| parser.add_argument( |
| "--comment_type", |
| required=True, |
| choices=COMMENT_DEF_MAP.keys(), |
| help="Type of summary comment", |
| ) |
| parser.add_argument( |
| "--build_url", |
| required=True, |
| type=str, |
| help="CI build page url to show in the report", |
| ) |
| parser.add_argument("--output", type=pathlib.Path, default=None) |
| parser.add_argument( |
| "--verbose", |
| action="store_true", |
| help="Print internal information during execution", |
| ) |
| |
| return parser.parse_args() |
| |
| |
| def main(args): |
| benchmark_files = common_arguments.expand_and_check_file_paths(args.benchmark_files) |
| compile_stats_files = common_arguments.expand_and_check_file_paths( |
| args.compile_stats_files |
| ) |
| |
| pr_commit = _get_git_commit_hash(ref=args.pr_committish, verbose=args.verbose) |
| execution_benchmarks = benchmark_presentation.aggregate_all_benchmarks( |
| benchmark_files=benchmark_files, expected_pr_commit=pr_commit |
| ) |
| compilation_metrics = benchmark_presentation.collect_all_compilation_metrics( |
| compile_stats_files=compile_stats_files, expected_pr_commit=pr_commit |
| ) |
| |
| if args.pr_base_branch is None: |
| pr_base_commit = None |
| else: |
| pr_base_commit = _get_git_merge_base_commit( |
| pr_commit=pr_commit, target_branch=args.pr_base_branch, verbose=args.verbose |
| ) |
| |
| if pr_base_commit is None: |
| comparable_results = None |
| else: |
| required_benchmark_keys = set(execution_benchmarks.keys()) |
| for target_id in compilation_metrics: |
| for mapper in benchmark_presentation.COMPILATION_METRICS_TO_TABLE_MAPPERS: |
| required_benchmark_keys.add(mapper.get_series_id(target_id)) |
| |
| comparable_results = _find_comparable_benchmark_results( |
| start_commit=pr_base_commit, |
| required_benchmark_keys=required_benchmark_keys, |
| verbose=args.verbose, |
| ) |
| |
| if comparable_results is None: |
| comparable_commit = None |
| else: |
| comparable_commit = comparable_results.commit_sha |
| # Update the execution benchmarks with base numbers. |
| for bench in execution_benchmarks: |
| base_benchmark = comparable_results.benchmark_results[bench] |
| if base_benchmark["sampleUnit"] != "ns": |
| raise ValueError("Only support nanoseconds for latency sample.") |
| execution_benchmarks[bench].base_mean_time = base_benchmark["sample"] |
| |
| # Update the compilation metrics with base numbers. |
| for target_id, metrics in compilation_metrics.items(): |
| updated_metrics = metrics |
| for mapper in benchmark_presentation.COMPILATION_METRICS_TO_TABLE_MAPPERS: |
| base_benchmark = comparable_results.benchmark_results[ |
| mapper.get_series_id(target_id) |
| ] |
| if base_benchmark["sampleUnit"] != mapper.get_unit(): |
| raise ValueError("Unit of the queried sample is mismatched.") |
| updated_metrics = mapper.update_base_value( |
| updated_metrics, base_benchmark["sample"] |
| ) |
| compilation_metrics[target_id] = updated_metrics |
| |
| pr_commit_link = md.link(pr_commit, f"{GITHUB_IREE_REPO_PREFIX}/commit/{pr_commit}") |
| commit_info_md = f"@ commit {pr_commit_link}" |
| if comparable_commit is not None: |
| baseline_commit_link = md.link( |
| comparable_commit, f"{GITHUB_IREE_REPO_PREFIX}/commit/{comparable_commit}" |
| ) |
| commit_info_md += f" (vs. base {baseline_commit_link})" |
| elif pr_base_commit is not None: |
| commit_info_md += " (no previous benchmark results to compare)" |
| |
| comment_def = COMMENT_DEF_MAP[args.comment_type] |
| full_md, abbr_md = _get_benchmark_result_markdown( |
| execution_benchmarks=execution_benchmarks, |
| compilation_metrics=compilation_metrics, |
| pr_url=f"{GITHUB_IREE_REPO_PREFIX}/pull/{args.pr_number}", |
| build_url=args.build_url, |
| comment_def=comment_def, |
| commit_info_md=commit_info_md, |
| ) |
| |
| comment_data = benchmark_comment.CommentData( |
| type_id=comment_def.type_id, |
| abbr_md=abbr_md, |
| full_md=full_md, |
| unverified_pr_number=args.pr_number, |
| ) |
| comment_json_data = json.dumps(dataclasses.asdict(comment_data), indent=2) |
| if args.output is None: |
| print(comment_json_data) |
| else: |
| args.output.write_text(comment_json_data) |
| |
| |
| if __name__ == "__main__": |
| main(parse_arguments()) |
