build_tools/benchmarks/diff_local_benchmarks.py

#!/usr/bin/env python3
# Copyright 2021 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Diffs two local benchmark result JSON files.

Example usage:
  python3 diff_local_benchmarks.py --base=/path/to/base_benchmarks.json
                                   --target=/path/to/target_benchmarks.json
"""

import pathlib
import sys

# Add build_tools python dir to the search path.
sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python")))

import argparse

from typing import Optional

from common.benchmark_presentation import *


def get_benchmark_result_markdown(
    base_benchmark_file: Optional[pathlib.Path],
    target_benchmark_file: Optional[pathlib.Path],
    base_compile_stats_file: Optional[pathlib.Path],
    target_compile_stats_file: Optional[pathlib.Path],
    verbose: bool = False,
) -> str:
    """Gets the full markdown summary of all benchmarks in files."""
    base_benchmarks = {}
    target_benchmarks = {}
    base_compilation_metrics = {}
    target_compilation_metrics = {}
    if base_benchmark_file and target_benchmark_file:
        base_benchmarks = aggregate_all_benchmarks([base_benchmark_file])
        target_benchmarks = aggregate_all_benchmarks([target_benchmark_file])
    if base_compile_stats_file and target_compile_stats_file:
        base_compilation_metrics = collect_all_compilation_metrics(
            [base_compile_stats_file]
        )
        target_compilation_metrics = collect_all_compilation_metrics(
            [target_compile_stats_file]
        )

    # Update the target benchmarks with their corresponding base numbers.
    for bench in base_benchmarks:
        if bench in target_benchmarks:
            target_benchmarks[bench].base_mean_time = base_benchmarks[bench].mean_time

    for target_name, base_metrics in base_compilation_metrics.items():
        updated_metrics = base_metrics
        for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS:
            metric_key = mapper.get_series_name(target_name)
            base_value, _ = mapper.get_current_and_base_value(base_metrics)
            updated_metrics = mapper.update_base_value(updated_metrics, base_value)
        target_compilation_metrics[target_name] = updated_metrics

    # Compose the full benchmark tables.
    full_table = [md.header("Full Benchmark Summary", 2)]
    full_table.append(categorize_benchmarks_into_tables(target_benchmarks))

    # Compose the full compilation metrics tables.
    full_table.append(
        categorize_compilation_metrics_into_tables(target_compilation_metrics)
    )

    return "\n\n".join(full_table)


def parse_arguments():
    """Parses command-line options."""

    def check_file_path(path):
        path = pathlib.Path(path)
        if path.is_file():
            return path
        else:
            raise ValueError(path)

    parser = argparse.ArgumentParser()
    parser.add_argument("--base", type=check_file_path, help="Base benchmark results")
    parser.add_argument(
        "--target", type=check_file_path, help="Target benchmark results"
    )
    parser.add_argument(
        "--base-compile-stats", type=check_file_path, help="Base compilation statistics"
    )
    parser.add_argument(
        "--target-compile-stats",
        type=check_file_path,
        help="Target compilation statistics",
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        help="Print internal information during execution",
    )
    args = parser.parse_args()

    return args


if __name__ == "__main__":
    args = parse_arguments()
    if args.base or args.target:
        if not args.base or not args.target:
            raise ValueError("--base and --target must be used together.")
    if args.base_compile_stats or args.target_compile_stats:
        if not args.base_compile_stats or not args.target_compile_stats:
            raise ValueError(
                "--base-compile-stats and --target-compile-stats must "
                "be used together."
            )

    print(
        get_benchmark_result_markdown(
            args.base,
            args.target,
            args.base_compile_stats,
            args.target_compile_stats,
            verbose=args.verbose,
        )
    )