build_tools/benchmarks/common/benchmark_presentation.py

# Copyright 2021 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import (
    Any,
    Callable,
    Dict,
    Generic,
    List,
    Optional,
    Sequence,
    Tuple,
    TypeVar,
    Union,
)
import pathlib
import dataclasses
import json
import urllib.parse
import markdown_strings as md
import math

from common import benchmark_definition, benchmark_thresholds
from common.benchmark_thresholds import (
    BENCHMARK_THRESHOLDS,
    COMPILATION_TIME_THRESHOLDS,
    TOTAL_ARTIFACT_SIZE_THRESHOLDS,
    TOTAL_DISPATCH_SIZE_THRESHOLDS,
    BenchmarkThreshold,
    ThresholdUnit,
)

GetMetricFunc = Callable[[Any], Tuple[int, Optional[int]]]

PERFBOARD_SERIES_PREFIX = "https://perf.iree.dev/serie?IREE?"
BENCHMARK_RESULTS_HEADERS = [
    "Benchmark Name",
    "Average Latency (ms)",
    "Median Latency (ms)",
    "Latency Standard Deviation (ms)",
]
# Since We don't have a structural way to store metric data yet, each metric is
# assigned with a fixed id generated from uuid.uuid4(), to identify the series.
COMPILATION_TIME_METRIC_ID = "e54cd682-c079-4c42-b4ad-d92c4bedea13"
COMPILATION_TIME_SERIES_SUFFIX = "compilation:module:compilation-time"
TOTAL_DISPATCH_SIZE_METRIC_ID = "9e15f7e6-383c-47ec-bd38-ecba55a5f10a"
TOTAL_DISPATCH_SIZE_SERIES_SUFFIX = (
    "compilation:module:component-size:total-dispatch-size"
)
TOTAL_ARTIFACT_SIZE_METRIC_ID = "2c8a9198-c01c-45b9-a7da-69c82cf749f7"
TOTAL_ARTIFACT_SIZE_SERIES_SUFFIX = "compilation:module:total-artifact-size"
STREAM_IR_DISPATCH_COUNT_METRIC_ID = "7b72cd9e-43ed-4078-b6d3-20b810f9e4ad"
STREAM_IR_DISPATCH_COUNT_SERIES_SUFFIX = "compilation:ir:stream-dispatch-count"


@dataclass
class AggregateBenchmarkLatency:
    """An object for describing aggregate latency numbers for a benchmark."""

    name: str
    benchmark_info: benchmark_definition.BenchmarkInfo
    mean_time: int
    median_time: int
    stddev_time: int
    # The average latency time for the base commit to compare against.
    base_mean_time: Optional[int] = None

    def __str__(self) -> str:
        return self.name


@dataclass(frozen=True)
class CompilationMetrics:
    """An object for describing the summary of statistics and the reference."""

    name: str
    compilation_info: benchmark_definition.CompilationInfo
    compilation_time_ms: int
    total_dispatch_component_bytes: int
    total_artifact_bytes: int
    stream_ir_dispatch_count: int
    base_compilation_time_ms: Optional[int] = None
    base_total_artifact_bytes: Optional[int] = None
    base_total_dispatch_component_bytes: Optional[int] = None
    base_stream_ir_dispatch_count: Optional[int] = None

    def __str__(self) -> str:
        return self.name


T = TypeVar("T")


class MetricsToTableMapper(ABC, Generic[T]):
    """Abstract class to help map benchmark metrics to table.

    It contains a set of methods to help table generator get the required
    information for a metric. For example, extract the current and base metric
    value, the metric thresholds, the table header of the metrics, ...
    """

    @abstractmethod
    def update_base_value(self, obj: T, base_value: Any) -> T:
        """Sets the base value and returns the updated metric object."""
        raise NotImplementedError()

    @abstractmethod
    def get_current_and_base_value(self, obj: T) -> Tuple[int, Optional[int]]:
        """Returns the current and base (can be None) value."""
        raise NotImplementedError()

    def get_series_id(self, benchmark_id: str) -> str:
        """Returns the dashboard series id."""
        return f"{benchmark_id}-{self.get_metric_id()}"

    @abstractmethod
    def get_metric_id(self) -> str:
        """Returns the dashboard series id."""
        raise NotImplementedError()

    @abstractmethod
    def get_series_name(self, name: str) -> str:
        """Returns the dashboard series name."""
        raise NotImplementedError()

    @abstractmethod
    def get_unit(self) -> str:
        """Returns the unit of the metric value."""
        raise NotImplementedError()

    @abstractmethod
    def get_table_header(self) -> str:
        """Returns the header of the table."""
        raise NotImplementedError()

    @staticmethod
    @abstractmethod
    def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
        raise NotImplementedError()

    @staticmethod
    @abstractmethod
    def get_table_title() -> str:
        raise NotImplementedError()


class CompilationTimeToTable(MetricsToTableMapper[CompilationMetrics]):
    """Helper to map CompilationMetrics to compilation time column."""

    def update_base_value(
        self, compile_metrics: CompilationMetrics, base_value: Any
    ) -> CompilationMetrics:
        return dataclasses.replace(compile_metrics, base_compilation_time_ms=base_value)

    def get_current_and_base_value(
        self, compile_metrics: CompilationMetrics
    ) -> Tuple[int, Optional[int]]:
        return (
            compile_metrics.compilation_time_ms,
            compile_metrics.base_compilation_time_ms,
        )

    def get_metric_id(self) -> str:
        return COMPILATION_TIME_METRIC_ID

    def get_series_name(self, name: str) -> str:
        return f"{name} [{COMPILATION_TIME_SERIES_SUFFIX}]"

    def get_unit(self) -> str:
        return "ms"

    def get_table_header(self) -> str:
        return f"Compilation Time ({self.get_unit()})"

    @staticmethod
    def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
        return COMPILATION_TIME_THRESHOLDS

    @staticmethod
    def get_table_title() -> str:
        return "Compilation Times"


class TotalDispatchSizeToTable(MetricsToTableMapper[CompilationMetrics]):
    """Helper to map CompilationMetrics to total dispatch size column."""

    def update_base_value(
        self, compile_metrics: CompilationMetrics, base_value: Any
    ) -> CompilationMetrics:
        return dataclasses.replace(
            compile_metrics, base_total_dispatch_component_bytes=base_value
        )

    def get_current_and_base_value(
        self, compile_metrics: CompilationMetrics
    ) -> Tuple[int, Optional[int]]:
        return (
            compile_metrics.total_dispatch_component_bytes,
            compile_metrics.base_total_dispatch_component_bytes,
        )

    def get_metric_id(self) -> str:
        return TOTAL_DISPATCH_SIZE_METRIC_ID

    def get_series_name(self, name: str) -> str:
        return f"{name} [{TOTAL_DISPATCH_SIZE_SERIES_SUFFIX}]"

    def get_unit(self) -> str:
        return "bytes"

    def get_table_header(self) -> str:
        return f"Total Dispatch Size ({self.get_unit()})"

    @staticmethod
    def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
        return TOTAL_DISPATCH_SIZE_THRESHOLDS

    @staticmethod
    def get_table_title() -> str:
        return "Total Dispatch Sizes"


class TotalArtifactSizeToTable(MetricsToTableMapper[CompilationMetrics]):
    """Helper to map CompilationMetrics to total artifact size column."""

    def update_base_value(
        self, compile_metrics: CompilationMetrics, base_value: Any
    ) -> CompilationMetrics:
        return dataclasses.replace(
            compile_metrics, base_total_artifact_bytes=base_value
        )

    def get_current_and_base_value(
        self, compile_metrics: CompilationMetrics
    ) -> Tuple[int, Optional[int]]:
        return (
            compile_metrics.total_artifact_bytes,
            compile_metrics.base_total_artifact_bytes,
        )

    def get_metric_id(self) -> str:
        return TOTAL_ARTIFACT_SIZE_METRIC_ID

    def get_series_name(self, name: str) -> str:
        return f"{name} [{TOTAL_ARTIFACT_SIZE_SERIES_SUFFIX}]"

    def get_unit(self) -> str:
        return "bytes"

    def get_table_header(self) -> str:
        return f"Total Artifact Size ({self.get_unit()})"

    @staticmethod
    def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
        return TOTAL_ARTIFACT_SIZE_THRESHOLDS

    @staticmethod
    def get_table_title() -> str:
        return "Total Artifact Sizes"


class StreamIRDispatchCountToTable(MetricsToTableMapper[CompilationMetrics]):
    """Helper to map CompilationMetrics to Stream IR Dispatch Count column."""

    def update_base_value(
        self, compile_metrics: CompilationMetrics, base_value: Any
    ) -> CompilationMetrics:
        return dataclasses.replace(
            compile_metrics, base_stream_ir_dispatch_count=base_value
        )

    def get_current_and_base_value(
        self, compile_metrics: CompilationMetrics
    ) -> Tuple[int, Optional[int]]:
        return (
            compile_metrics.stream_ir_dispatch_count,
            compile_metrics.base_stream_ir_dispatch_count,
        )

    def get_metric_id(self) -> str:
        return STREAM_IR_DISPATCH_COUNT_METRIC_ID

    def get_series_name(self, name: str) -> str:
        return f"{name} [{STREAM_IR_DISPATCH_COUNT_SERIES_SUFFIX}]"

    def get_unit(self) -> str:
        return "number"

    def get_table_header(self) -> str:
        return f"Stream IR Dispatch Count (# of cmd.dispatch ops)"

    @staticmethod
    def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
        return benchmark_thresholds.STREAM_IR_DISPATCH_COUNT_THRESHOLDS

    @staticmethod
    def get_table_title() -> str:
        return "Stream IR Dispatch Count (# of cmd.dispatch ops)"


COMPILATION_METRICS_TO_TABLE_MAPPERS: List[MetricsToTableMapper[CompilationMetrics]] = [
    CompilationTimeToTable(),
    TotalDispatchSizeToTable(),
    TotalArtifactSizeToTable(),
    StreamIRDispatchCountToTable(),
]


def aggregate_all_benchmarks(
    benchmark_files: Sequence[pathlib.Path], expected_pr_commit: Optional[str] = None
) -> Dict[str, AggregateBenchmarkLatency]:
    """Aggregates all benchmarks in the given files.

    Args:
    - benchmark_files: A list of JSON files, each can be decoded as a
      BenchmarkResults.
    - expected_pr_commit: An optional Git commit SHA to match against.

    Returns:
    - A dict of benchmark names to AggregateBenchmarkLatency numbers.
    """

    aggregate_results = {}
    benchmark_names = set()
    for benchmark_file in benchmark_files:
        file_results = benchmark_definition.BenchmarkResults.from_json_str(
            benchmark_file.read_text()
        )

        if (expected_pr_commit is not None) and (
            file_results.commit != expected_pr_commit
        ):
            raise ValueError("Inconsistent pull request commit")

        for benchmark_index in range(len(file_results.benchmarks)):
            benchmark_run = file_results.benchmarks[benchmark_index]

            series_name = str(benchmark_run.info)
            # Make sure each benchmark has a unique name.
            if series_name in benchmark_names:
                raise ValueError(f"Duplicated benchmark name: {series_name}")
            benchmark_names.add(series_name)

            series_id = benchmark_run.info.run_config_id
            if series_id in aggregate_results:
                raise ValueError(f"Duplicated benchmark id: {series_id}")

            aggregate_results[series_id] = AggregateBenchmarkLatency(
                name=series_name,
                benchmark_info=benchmark_run.info,
                mean_time=benchmark_run.metrics.real_time.mean,
                median_time=benchmark_run.metrics.real_time.median,
                stddev_time=benchmark_run.metrics.real_time.stddev,
            )

    return aggregate_results


def collect_all_compilation_metrics(
    compile_stats_files: Sequence[pathlib.Path],
    expected_pr_commit: Optional[str] = None,
) -> Dict[str, CompilationMetrics]:
    """Collects all compilation statistics in the given files.

    Args:
      compile_stats_files: A list of JSON files, each can be decoded as a
        CompilationResults.
      expected_pr_commit: An optional Git commit SHA to match against.

    Returns:
      A dict of benchmark names to CompilationMetrics.
    """
    compile_metrics = {}
    target_names = set()
    for compile_stats_file in compile_stats_files:
        with compile_stats_file.open("r") as f:
            file_results = benchmark_definition.CompilationResults.from_json_object(
                json.load(f)
            )

        if (expected_pr_commit is not None) and (
            file_results.commit != expected_pr_commit
        ):
            raise ValueError("Inconsistent pull request commit")

        for compile_stats in file_results.compilation_statistics:
            component_sizes = compile_stats.module_component_sizes
            stream_dispatch_count = compile_stats.ir_stats.stream_dispatch_count

            target_name = str(compile_stats.compilation_info)
            if target_name in target_names:
                raise ValueError(f"Duplicated target name: {target_name}")
            target_names.add(target_name)

            target_id = compile_stats.compilation_info.gen_config_id
            if target_id in compile_metrics:
                raise ValueError(f"Duplicated target id: {target_id}")

            compile_metrics[target_id] = CompilationMetrics(
                name=target_name,
                compilation_info=compile_stats.compilation_info,
                compilation_time_ms=compile_stats.compilation_time_ms,
                total_artifact_bytes=component_sizes.file_bytes,
                total_dispatch_component_bytes=component_sizes.total_dispatch_component_bytes,
                stream_ir_dispatch_count=stream_dispatch_count,
            )

    return compile_metrics


def _make_series_link(name: str, series_id: str) -> str:
    """Add link to the given benchmark name.

    Args:
      name: the text to show on the link.
      series_id: the dashboard series id.
    """
    url = PERFBOARD_SERIES_PREFIX + urllib.parse.quote(series_id, safe="()[]@,")
    return md.link(name, url)


def _add_header_and_get_markdown_table(
    headers: Sequence[str], rows: Sequence[Tuple], size_cut: Optional[int] = None
) -> str:
    """Generates a markdown table with headers.

    Args:
      headers: list of table headers.
      rows: list of rows. Each row is a tuple with the same length as headers.
      size_cut: If not None, only show the top N results for each table.
    """

    total_size = len(rows)
    if size_cut is not None:
        rows = rows[0:size_cut]

    columns = [[header] for header in headers]
    for row in rows:
        for column, item in zip(columns, row):
            column.append(item)

    table_str = md.table(columns)
    if size_cut is not None and size_cut < total_size:
        table_str += "\n\n"
        table_str += md.italics(f"[Top {size_cut} out of {total_size} results showed]")
    return table_str


T = TypeVar("T")


def _categorize_on_single_metric(
    metrics_map: Dict[str, T],
    metric_func: GetMetricFunc,
    thresholds: Sequence[BenchmarkThreshold],
    metric_unit: str,
) -> Tuple[Dict[str, T], Dict[str, T], Dict[str, T], Dict[str, T]]:
    """Categorize the metrics object into regressed, improved, similar, and the
    raw group (the group with no base to compare to).

    Args:
      metrics_map: map of (series_id, metrics object).
      metric_func: the function returns current and base value of the metric.
      thresholds: list of threshold settings to match for categorizing.
    Returns:
      A tuple of (regressed, improved, similar, raw) groups.
    """

    regressed_map = {}
    improved_map = {}
    similar_map = {}
    raw_map = {}
    for series_id, metrics_obj in metrics_map.items():
        current, base = metric_func(metrics_obj)
        if base is None:
            raw_map[series_id] = metrics_obj
            continue

        series_name = str(metrics_obj)
        similar_threshold = None
        for threshold in thresholds:
            if threshold.regex.match(series_name):
                similar_threshold = threshold
                break
        if similar_threshold is None:
            raise ValueError(f"No matched threshold setting for: {series_name}")

        if similar_threshold.unit == ThresholdUnit.PERCENTAGE:
            ratio = abs(current - base) / base * 100
        elif similar_threshold.unit.value == metric_unit:
            ratio = abs(current - base)
        else:
            raise ValueError(
                f"Mismatch between metric unit '{metric_unit}' and threshold unit '{similar_threshold.unit.value}'"
            )

        if ratio <= similar_threshold.threshold:
            similar_map[series_id] = metrics_obj
        elif current > base:
            regressed_map[series_id] = metrics_obj
        else:
            improved_map[series_id] = metrics_obj

    return (regressed_map, improved_map, similar_map, raw_map)


def _get_fixed_point_str(value: Union[int, float], digits=3) -> str:
    if isinstance(value, int) or value.is_integer():
        return str(math.floor(value))
    return f"{{:.{digits}f}}".format(value)


def _get_compare_text(current: float, base: Optional[int]) -> str:
    """Generates the text of comparison between current and base value. Returns
    the current value if the base value is None.
    """
    # If base is None, don't need to do compare.
    if base is None:
        return f"{_get_fixed_point_str(current)}"

    ratio = abs(current - base) / base
    direction = "↑" if current > base else ("↓" if current < base else "")
    return f"{_get_fixed_point_str(current)} (vs. {_get_fixed_point_str(base)}, {ratio:.2%}{direction})"


def _sort_benchmarks_and_get_table(
    benchmarks: Dict[str, AggregateBenchmarkLatency], size_cut: Optional[int] = None
) -> str:
    """Sorts all benchmarks according to the improvement/regression ratio and
    returns a markdown table for it.

    Args:
      benchmarks_map: map of (series_id, benchmark object).
      size_cut: If not None, only show the top N results for each table.
    """
    sorted_rows = []
    for series_id, benchmark in benchmarks.items():
        current = benchmark.mean_time / 1e6
        base = benchmark.base_mean_time / 1e6
        ratio = abs(current - base) / base
        str_mean = _get_compare_text(current, base)
        clickable_name = _make_series_link(benchmark.name, series_id)
        sorted_rows.append(
            (
                ratio,
                (
                    clickable_name,
                    str_mean,
                    f"{_get_fixed_point_str(benchmark.median_time / 1e6)}",
                    f"{_get_fixed_point_str(benchmark.stddev_time / 1e6)}",
                ),
            )
        )
    sorted_rows.sort(key=lambda row: row[0], reverse=True)

    return _add_header_and_get_markdown_table(
        headers=BENCHMARK_RESULTS_HEADERS,
        rows=[row[1] for row in sorted_rows],
        size_cut=size_cut,
    )


def categorize_benchmarks_into_tables(
    benchmarks: Dict[str, AggregateBenchmarkLatency], size_cut: Optional[int] = None
) -> str:
    """Splits benchmarks into regressed/improved/similar/raw categories and
    returns their markdown tables.

    If size_cut is None, the table includes regressed/improved/similar/raw
    categories; otherwise, the table includes regressed/improved/raw categories.

    Args:
      benchmarks: A dictionary of benchmark names to its aggregate info.
      size_cut: If not None, only show the top N results for each table.
    """
    regressed, improved, similar, raw = _categorize_on_single_metric(
        benchmarks,
        lambda results: (results.mean_time, results.base_mean_time),
        BENCHMARK_THRESHOLDS,
        "ns",
    )

    tables = []
    if regressed:
        tables.append(md.header("Regressed Latencies 🚩", 3))
        tables.append(_sort_benchmarks_and_get_table(regressed, size_cut))
    if improved:
        tables.append(md.header("Improved Latencies 🎉", 3))
        tables.append(_sort_benchmarks_and_get_table(improved, size_cut))
    # If we want to abbreviate, similar results won't be interesting.
    if similar and size_cut is None:
        tables.append(md.header("Similar Latencies", 3))
        tables.append(_sort_benchmarks_and_get_table(similar, size_cut))
    if raw:
        tables.append(md.header("Raw Latencies", 3))
        raw_list = [
            (
                _make_series_link(name=v.name, series_id=k),
                f"{_get_fixed_point_str(v.mean_time / 1e6)}",
                f"{_get_fixed_point_str(v.median_time / 1e6)}",
                f"{_get_fixed_point_str(v.stddev_time / 1e6)}",
            )
            for k, v in raw.items()
        ]
        tables.append(
            _add_header_and_get_markdown_table(
                BENCHMARK_RESULTS_HEADERS, raw_list, size_cut=size_cut
            )
        )
    return "\n\n".join(tables)


def _sort_metrics_objects_and_get_table(
    metrics_objs: Dict[str, T],
    mapper: MetricsToTableMapper[T],
    headers: Sequence[str],
    size_cut: Optional[int] = None,
) -> str:
    """Sorts all metrics objects according to the improvement/regression ratio and
    returns a markdown table for it.

    Args:
      metrics_objs: map of (target_id, CompilationMetrics). All objects must
        contain base value.
      mapper: MetricsToTableMapper for metrics_objs.
      headers: list of table headers.
      size_cut: If not None, only show the top N results for each table.
    """
    sorted_rows = []
    for target_id, metrics_obj in metrics_objs.items():
        current, base = mapper.get_current_and_base_value(metrics_obj)
        if base is None:
            raise AssertionError("Base can't be None for sorting.")
        ratio = abs(current - base) / base
        sorted_rows.append(
            (
                ratio,
                (
                    _make_series_link(
                        str(metrics_obj), mapper.get_series_id(target_id)
                    ),
                    _get_compare_text(current, base),
                ),
            )
        )
    sorted_rows.sort(key=lambda row: row[0], reverse=True)

    return _add_header_and_get_markdown_table(
        headers=headers, rows=[row[1] for row in sorted_rows], size_cut=size_cut
    )


def categorize_compilation_metrics_into_tables(
    compile_metrics_map: Dict[str, CompilationMetrics], size_cut: Optional[int] = None
) -> str:
    """Splits compilation metrics into regressed/improved/all categories
    and returns their markdown tables.

    If size_cut is None, the table includes regressed/improved/all categories;
    otherwise, the table includes regressed/improved categories.

    Args:
      compile_metrics_map: A dictionary of benchmark names to its compilation
        metrics.
      size_cut: If not None, only show the top N results for each table.
    """

    tables = []
    for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS:
        regressed, improved, _, _ = _categorize_on_single_metric(
            compile_metrics_map,
            mapper.get_current_and_base_value,
            mapper.get_metric_thresholds(),
            mapper.get_unit(),
        )

        table_title = mapper.get_table_title()
        table_header = mapper.get_table_header()
        if regressed:
            tables.append(md.header(f"Regressed {table_title} 🚩", 3))
            tables.append(
                _sort_metrics_objects_and_get_table(
                    metrics_objs=regressed,
                    mapper=mapper,
                    headers=["Benchmark Name", table_header],
                    size_cut=size_cut,
                )
            )
        if improved:
            tables.append(md.header(f"Improved {table_title} 🎉", 3))
            tables.append(
                _sort_metrics_objects_and_get_table(
                    metrics_objs=improved,
                    mapper=mapper,
                    headers=["Benchmark Name", table_header],
                    size_cut=size_cut,
                )
            )

    # If we want to abbreviate, similar results won't be interesting.
    if size_cut is None and compile_metrics_map:
        tables.append(md.header("All Compilation Metrics", 3))
        headers = ["Benchmark Name"] + [
            mapper.get_table_header() for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS
        ]
        rows = []
        for target_id, metrics in compile_metrics_map.items():
            row = [metrics.name]
            for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS:
                current, base = mapper.get_current_and_base_value(metrics)
                row.append(
                    _make_series_link(
                        _get_compare_text(current, base),
                        mapper.get_series_id(target_id),
                    )
                )
            rows.append(tuple(row))

        tables.append(
            _add_header_and_get_markdown_table(headers, rows, size_cut=size_cut)
        )

    return "\n\n".join(tables)