# Copyright 2021 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import (Any, Callable, Dict, Generic, List, Optional, Sequence,
                    Tuple, TypeVar, Union)
import pathlib
import dataclasses
import json
import urllib.parse
import markdown_strings as md
import math

from common import benchmark_definition, benchmark_thresholds
from common.benchmark_thresholds import (BENCHMARK_THRESHOLDS,
                                         COMPILATION_TIME_THRESHOLDS,
                                         TOTAL_ARTIFACT_SIZE_THRESHOLDS,
                                         TOTAL_DISPATCH_SIZE_THRESHOLDS,
                                         BenchmarkThreshold, ThresholdUnit)

GetMetricFunc = Callable[[Any], Tuple[int, Optional[int]]]

PERFBOARD_SERIES_PREFIX = "https://perf.iree.dev/serie?IREE?"
BENCHMARK_RESULTS_HEADERS = [
    "Benchmark Name",
    "Average Latency (ms)",
    "Median Latency (ms)",
    "Latency Standard Deviation (ms)",
]
# Since We don't have a structural way to store metric data yet, each metric is
# assigned with a fixed id generated from uuid.uuid4(), to identify the series.
COMPILATION_TIME_METRIC_ID = "e54cd682-c079-4c42-b4ad-d92c4bedea13"
COMPILATION_TIME_SERIES_SUFFIX = "compilation:module:compilation-time"
TOTAL_DISPATCH_SIZE_METRIC_ID = "9e15f7e6-383c-47ec-bd38-ecba55a5f10a"
TOTAL_DISPATCH_SIZE_SERIES_SUFFIX = "compilation:module:component-size:total-dispatch-size"
TOTAL_ARTIFACT_SIZE_METRIC_ID = "2c8a9198-c01c-45b9-a7da-69c82cf749f7"
TOTAL_ARTIFACT_SIZE_SERIES_SUFFIX = "compilation:module:total-artifact-size"
STREAM_IR_DISPATCH_COUNT_METRIC_ID = "7b72cd9e-43ed-4078-b6d3-20b810f9e4ad"
STREAM_IR_DISPATCH_COUNT_SERIES_SUFFIX = "compilation:ir:stream-dispatch-count"


@dataclass
class AggregateBenchmarkLatency:
  """An object for describing aggregate latency numbers for a benchmark."""
  name: str
  benchmark_info: benchmark_definition.BenchmarkInfo
  mean_time: int
  median_time: int
  stddev_time: int
  # The average latency time for the base commit to compare against.
  base_mean_time: Optional[int] = None

  def __str__(self) -> str:
    return self.name


@dataclass(frozen=True)
class CompilationMetrics:
  """An object for describing the summary of statistics and the reference."""
  name: str
  compilation_info: benchmark_definition.CompilationInfo
  compilation_time_ms: int
  total_dispatch_component_bytes: int
  total_artifact_bytes: int
  stream_ir_dispatch_count: int
  base_compilation_time_ms: Optional[int] = None
  base_total_artifact_bytes: Optional[int] = None
  base_total_dispatch_component_bytes: Optional[int] = None
  base_stream_ir_dispatch_count: Optional[int] = None

  def __str__(self) -> str:
    return self.name


T = TypeVar("T")


class MetricsToTableMapper(ABC, Generic[T]):
  """Abstract class to help map benchmark metrics to table.

    It contains a set of methods to help table generator get the required
    information for a metric. For example, extract the current and base metric
    value, the metric thresholds, the table header of the metrics, ...
  """

  @abstractmethod
  def update_base_value(self, obj: T, base_value: Any) -> T:
    """Sets the base value and returns the updated metric object."""
    raise NotImplementedError()

  @abstractmethod
  def get_current_and_base_value(self, obj: T) -> Tuple[int, Optional[int]]:
    """Returns the current and base (can be None) value."""
    raise NotImplementedError()

  def get_series_id(self, benchmark_id: str) -> str:
    """Returns the dashboard series id."""
    return f"{benchmark_id}-{self.get_metric_id()}"

  @abstractmethod
  def get_metric_id(self) -> str:
    """Returns the dashboard series id."""
    raise NotImplementedError()

  @abstractmethod
  def get_series_name(self, name: str) -> str:
    """Returns the dashboard series name."""
    raise NotImplementedError()

  @abstractmethod
  def get_unit(self) -> str:
    """Returns the unit of the metric value."""
    raise NotImplementedError()

  @abstractmethod
  def get_table_header(self) -> str:
    """Returns the header of the table."""
    raise NotImplementedError()

  @staticmethod
  @abstractmethod
  def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
    raise NotImplementedError()

  @staticmethod
  @abstractmethod
  def get_table_title() -> str:
    raise NotImplementedError()


class CompilationTimeToTable(MetricsToTableMapper[CompilationMetrics]):
  """Helper to map CompilationMetrics to compilation time column."""

  def update_base_value(self, compile_metrics: CompilationMetrics,
                        base_value: Any) -> CompilationMetrics:
    return dataclasses.replace(compile_metrics,
                               base_compilation_time_ms=base_value)

  def get_current_and_base_value(
      self, compile_metrics: CompilationMetrics) -> Tuple[int, Optional[int]]:
    return (compile_metrics.compilation_time_ms,
            compile_metrics.base_compilation_time_ms)

  def get_metric_id(self) -> str:
    return COMPILATION_TIME_METRIC_ID

  def get_series_name(self, name: str) -> str:
    return f"{name} [{COMPILATION_TIME_SERIES_SUFFIX}]"

  def get_unit(self) -> str:
    return "ms"

  def get_table_header(self) -> str:
    return f"Compilation Time ({self.get_unit()})"

  @staticmethod
  def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
    return COMPILATION_TIME_THRESHOLDS

  @staticmethod
  def get_table_title() -> str:
    return "Compilation Times"


class TotalDispatchSizeToTable(MetricsToTableMapper[CompilationMetrics]):
  """Helper to map CompilationMetrics to total dispatch size column."""

  def update_base_value(self, compile_metrics: CompilationMetrics,
                        base_value: Any) -> CompilationMetrics:
    return dataclasses.replace(compile_metrics,
                               base_total_dispatch_component_bytes=base_value)

  def get_current_and_base_value(
      self, compile_metrics: CompilationMetrics) -> Tuple[int, Optional[int]]:
    return (compile_metrics.total_dispatch_component_bytes,
            compile_metrics.base_total_dispatch_component_bytes)

  def get_metric_id(self) -> str:
    return TOTAL_DISPATCH_SIZE_METRIC_ID

  def get_series_name(self, name: str) -> str:
    return f"{name} [{TOTAL_DISPATCH_SIZE_SERIES_SUFFIX}]"

  def get_unit(self) -> str:
    return "bytes"

  def get_table_header(self) -> str:
    return f"Total Dispatch Size ({self.get_unit()})"

  @staticmethod
  def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
    return TOTAL_DISPATCH_SIZE_THRESHOLDS

  @staticmethod
  def get_table_title() -> str:
    return "Total Dispatch Sizes"


class TotalArtifactSizeToTable(MetricsToTableMapper[CompilationMetrics]):
  """Helper to map CompilationMetrics to total artifact size column."""

  def update_base_value(self, compile_metrics: CompilationMetrics,
                        base_value: Any) -> CompilationMetrics:
    return dataclasses.replace(compile_metrics,
                               base_total_artifact_bytes=base_value)

  def get_current_and_base_value(
      self, compile_metrics: CompilationMetrics) -> Tuple[int, Optional[int]]:
    return (compile_metrics.total_artifact_bytes,
            compile_metrics.base_total_artifact_bytes)

  def get_metric_id(self) -> str:
    return TOTAL_ARTIFACT_SIZE_METRIC_ID

  def get_series_name(self, name: str) -> str:
    return f"{name} [{TOTAL_ARTIFACT_SIZE_SERIES_SUFFIX}]"

  def get_unit(self) -> str:
    return "bytes"

  def get_table_header(self) -> str:
    return f"Total Artifact Size ({self.get_unit()})"

  @staticmethod
  def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
    return TOTAL_ARTIFACT_SIZE_THRESHOLDS

  @staticmethod
  def get_table_title() -> str:
    return "Total Artifact Sizes"


class StreamIRDispatchCountToTable(MetricsToTableMapper[CompilationMetrics]):
  """Helper to map CompilationMetrics to Stream IR Dispatch Count column."""

  def update_base_value(self, compile_metrics: CompilationMetrics,
                        base_value: Any) -> CompilationMetrics:
    return dataclasses.replace(compile_metrics,
                               base_stream_ir_dispatch_count=base_value)

  def get_current_and_base_value(
      self, compile_metrics: CompilationMetrics) -> Tuple[int, Optional[int]]:
    return (compile_metrics.stream_ir_dispatch_count,
            compile_metrics.base_stream_ir_dispatch_count)

  def get_metric_id(self) -> str:
    return STREAM_IR_DISPATCH_COUNT_METRIC_ID

  def get_series_name(self, name: str) -> str:
    return f"{name} [{STREAM_IR_DISPATCH_COUNT_SERIES_SUFFIX}]"

  def get_unit(self) -> str:
    return "number"

  def get_table_header(self) -> str:
    return f"Stream IR Dispatch Count (# of cmd.dispatch ops)"

  @staticmethod
  def get_metric_thresholds() -> Sequence[BenchmarkThreshold]:
    return benchmark_thresholds.STREAM_IR_DISPATCH_COUNT_THRESHOLDS

  @staticmethod
  def get_table_title() -> str:
    return "Stream IR Dispatch Count (# of cmd.dispatch ops)"


COMPILATION_METRICS_TO_TABLE_MAPPERS: List[
    MetricsToTableMapper[CompilationMetrics]] = [
        CompilationTimeToTable(),
        TotalDispatchSizeToTable(),
        TotalArtifactSizeToTable(),
        StreamIRDispatchCountToTable(),
    ]


def aggregate_all_benchmarks(
    benchmark_files: Sequence[pathlib.Path],
    expected_pr_commit: Optional[str] = None
) -> Dict[str, AggregateBenchmarkLatency]:
  """Aggregates all benchmarks in the given files.

  Args:
  - benchmark_files: A list of JSON files, each can be decoded as a
    BenchmarkResults.
  - expected_pr_commit: An optional Git commit SHA to match against.

  Returns:
  - A dict of benchmark names to AggregateBenchmarkLatency numbers.
  """

  aggregate_results = {}
  benchmark_names = set()
  for benchmark_file in benchmark_files:
    file_results = benchmark_definition.BenchmarkResults.from_json_str(
        benchmark_file.read_text())

    if ((expected_pr_commit is not None) and
        (file_results.commit != expected_pr_commit)):
      raise ValueError("Inconsistent pull request commit")

    for benchmark_index in range(len(file_results.benchmarks)):
      benchmark_run = file_results.benchmarks[benchmark_index]

      series_name = str(benchmark_run.info)
      # Make sure each benchmark has a unique name.
      if series_name in benchmark_names:
        raise ValueError(f"Duplicated benchmark name: {series_name}")
      benchmark_names.add(series_name)

      series_id = benchmark_run.info.run_config_id
      if series_id in aggregate_results:
        raise ValueError(f"Duplicated benchmark id: {series_id}")

      aggregate_results[series_id] = AggregateBenchmarkLatency(
          name=series_name,
          benchmark_info=benchmark_run.info,
          mean_time=benchmark_run.metrics.real_time.mean,
          median_time=benchmark_run.metrics.real_time.median,
          stddev_time=benchmark_run.metrics.real_time.stddev)

  return aggregate_results


def collect_all_compilation_metrics(
    compile_stats_files: Sequence[pathlib.Path],
    expected_pr_commit: Optional[str] = None) -> Dict[str, CompilationMetrics]:
  """Collects all compilation statistics in the given files.

    Args:
      compile_stats_files: A list of JSON files, each can be decoded as a
        CompilationResults.
      expected_pr_commit: An optional Git commit SHA to match against.

    Returns:
      A dict of benchmark names to CompilationMetrics.
  """
  compile_metrics = {}
  target_names = set()
  for compile_stats_file in compile_stats_files:
    with compile_stats_file.open("r") as f:
      file_results = benchmark_definition.CompilationResults.from_json_object(
          json.load(f))

    if ((expected_pr_commit is not None) and
        (file_results.commit != expected_pr_commit)):
      raise ValueError("Inconsistent pull request commit")

    for compile_stats in file_results.compilation_statistics:
      component_sizes = compile_stats.module_component_sizes
      stream_dispatch_count = compile_stats.ir_stats.stream_dispatch_count

      target_name = str(compile_stats.compilation_info)
      if target_name in target_names:
        raise ValueError(f"Duplicated target name: {target_name}")
      target_names.add(target_name)

      target_id = compile_stats.compilation_info.gen_config_id
      if target_id in compile_metrics:
        raise ValueError(f"Duplicated target id: {target_id}")

      compile_metrics[target_id] = CompilationMetrics(
          name=target_name,
          compilation_info=compile_stats.compilation_info,
          compilation_time_ms=compile_stats.compilation_time_ms,
          total_artifact_bytes=component_sizes.file_bytes,
          total_dispatch_component_bytes=component_sizes.
          total_dispatch_component_bytes,
          stream_ir_dispatch_count=stream_dispatch_count)

  return compile_metrics


def _make_series_link(name: str, series_id: str) -> str:
  """Add link to the given benchmark name.

    Args:
      name: the text to show on the link.
      series_id: the dashboard series id.
  """
  url = PERFBOARD_SERIES_PREFIX + urllib.parse.quote(series_id, safe="()[]@,")
  return md.link(name, url)


def _add_header_and_get_markdown_table(headers: Sequence[str],
                                       rows: Sequence[Tuple],
                                       size_cut: Optional[int] = None) -> str:
  """Generates a markdown table with headers.

  Args:
    headers: list of table headers.
    rows: list of rows. Each row is a tuple with the same length as headers.
    size_cut: If not None, only show the top N results for each table.
  """

  total_size = len(rows)
  if size_cut is not None:
    rows = rows[0:size_cut]

  columns = [[header] for header in headers]
  for row in rows:
    for column, item in zip(columns, row):
      column.append(item)

  table_str = md.table(columns)
  if size_cut is not None and size_cut < total_size:
    table_str += "\n\n"
    table_str += md.italics(
        f"[Top {size_cut} out of {total_size} results showed]")
  return table_str


T = TypeVar("T")


def _categorize_on_single_metric(
    metrics_map: Dict[str, T],
    metric_func: GetMetricFunc,
    thresholds: Sequence[BenchmarkThreshold],
    metric_unit: str,
) -> Tuple[Dict[str, T], Dict[str, T], Dict[str, T], Dict[str, T]]:
  """Categorize the metrics object into regressed, improved, similar, and the
    raw group (the group with no base to compare to).

    Args:
      metrics_map: map of (series_id, metrics object).
      metric_func: the function returns current and base value of the metric.
      thresholds: list of threshold settings to match for categorizing.
    Returns:
      A tuple of (regressed, improved, similar, raw) groups.
  """

  regressed_map = {}
  improved_map = {}
  similar_map = {}
  raw_map = {}
  for series_id, metrics_obj in metrics_map.items():
    current, base = metric_func(metrics_obj)
    if base is None:
      raw_map[series_id] = metrics_obj
      continue

    series_name = str(metrics_obj)
    similar_threshold = None
    for threshold in thresholds:
      if threshold.regex.match(series_name):
        similar_threshold = threshold
        break
    if similar_threshold is None:
      raise ValueError(f"No matched threshold setting for: {series_name}")

    if similar_threshold.unit == ThresholdUnit.PERCENTAGE:
      ratio = abs(current - base) / base * 100
    elif similar_threshold.unit.value == metric_unit:
      ratio = abs(current - base)
    else:
      raise ValueError(
          f"Mismatch between metric unit '{metric_unit}' and threshold unit '{similar_threshold.unit.value}'"
      )

    if ratio <= similar_threshold.threshold:
      similar_map[series_id] = metrics_obj
    elif current > base:
      regressed_map[series_id] = metrics_obj
    else:
      improved_map[series_id] = metrics_obj

  return (regressed_map, improved_map, similar_map, raw_map)


def _get_fixed_point_str(value: Union[int, float], digits=3) -> str:
  if isinstance(value, int) or value.is_integer():
    return str(math.floor(value))
  return f"{{:.{digits}f}}".format(value)


def _get_compare_text(current: float, base: Optional[int]) -> str:
  """Generates the text of comparison between current and base value. Returns
    the current value if the base value is None.
  """
  # If base is None, don't need to do compare.
  if base is None:
    return f"{_get_fixed_point_str(current)}"

  ratio = abs(current - base) / base
  direction = "↑" if current > base else ("↓" if current < base else "")
  return f"{_get_fixed_point_str(current)} (vs. {_get_fixed_point_str(base)}, {ratio:.2%}{direction})"


def _sort_benchmarks_and_get_table(benchmarks: Dict[str,
                                                    AggregateBenchmarkLatency],
                                   size_cut: Optional[int] = None) -> str:
  """Sorts all benchmarks according to the improvement/regression ratio and
    returns a markdown table for it.

    Args:
      benchmarks_map: map of (series_id, benchmark object).
      size_cut: If not None, only show the top N results for each table.
  """
  sorted_rows = []
  for series_id, benchmark in benchmarks.items():
    current = benchmark.mean_time / 1e6
    base = benchmark.base_mean_time / 1e6
    ratio = abs(current - base) / base
    str_mean = _get_compare_text(current, base)
    clickable_name = _make_series_link(benchmark.name, series_id)
    sorted_rows.append(
        (ratio, (clickable_name, str_mean,
                 f"{_get_fixed_point_str(benchmark.median_time / 1e6)}",
                 f"{_get_fixed_point_str(benchmark.stddev_time / 1e6)}")))
  sorted_rows.sort(key=lambda row: row[0], reverse=True)

  return _add_header_and_get_markdown_table(
      headers=BENCHMARK_RESULTS_HEADERS,
      rows=[row[1] for row in sorted_rows],
      size_cut=size_cut)


def categorize_benchmarks_into_tables(benchmarks: Dict[
    str, AggregateBenchmarkLatency],
                                      size_cut: Optional[int] = None) -> str:
  """Splits benchmarks into regressed/improved/similar/raw categories and
    returns their markdown tables.

    If size_cut is None, the table includes regressed/improved/similar/raw
    categories; otherwise, the table includes regressed/improved/raw categories.

    Args:
      benchmarks: A dictionary of benchmark names to its aggregate info.
      size_cut: If not None, only show the top N results for each table.
  """
  regressed, improved, similar, raw = _categorize_on_single_metric(
      benchmarks, lambda results: (results.mean_time, results.base_mean_time),
      BENCHMARK_THRESHOLDS, "ns")

  tables = []
  if regressed:
    tables.append(md.header("Regressed Latencies 🚩", 3))
    tables.append(_sort_benchmarks_and_get_table(regressed, size_cut))
  if improved:
    tables.append(md.header("Improved Latencies 🎉", 3))
    tables.append(_sort_benchmarks_and_get_table(improved, size_cut))
  # If we want to abbreviate, similar results won't be interesting.
  if similar and size_cut is None:
    tables.append(md.header("Similar Latencies", 3))
    tables.append(_sort_benchmarks_and_get_table(similar, size_cut))
  if raw:
    tables.append(md.header("Raw Latencies", 3))
    raw_list = [(_make_series_link(name=v.name, series_id=k),
                 f"{_get_fixed_point_str(v.mean_time / 1e6)}",
                 f"{_get_fixed_point_str(v.median_time / 1e6)}",
                 f"{_get_fixed_point_str(v.stddev_time / 1e6)}")
                for k, v in raw.items()]
    tables.append(
        _add_header_and_get_markdown_table(BENCHMARK_RESULTS_HEADERS,
                                           raw_list,
                                           size_cut=size_cut))
  return "\n\n".join(tables)


def _sort_metrics_objects_and_get_table(metrics_objs: Dict[str, T],
                                        mapper: MetricsToTableMapper[T],
                                        headers: Sequence[str],
                                        size_cut: Optional[int] = None) -> str:
  """Sorts all metrics objects according to the improvement/regression ratio and
    returns a markdown table for it.

    Args:
      metrics_objs: map of (target_id, CompilationMetrics). All objects must
        contain base value.
      mapper: MetricsToTableMapper for metrics_objs.
      headers: list of table headers.
      size_cut: If not None, only show the top N results for each table.
  """
  sorted_rows = []
  for target_id, metrics_obj in metrics_objs.items():
    current, base = mapper.get_current_and_base_value(metrics_obj)
    if base is None:
      raise AssertionError("Base can't be None for sorting.")
    ratio = abs(current - base) / base
    sorted_rows.append((ratio, (
        _make_series_link(str(metrics_obj), mapper.get_series_id(target_id)),
        _get_compare_text(current, base),
    )))
  sorted_rows.sort(key=lambda row: row[0], reverse=True)

  return _add_header_and_get_markdown_table(
      headers=headers, rows=[row[1] for row in sorted_rows], size_cut=size_cut)


def categorize_compilation_metrics_into_tables(
    compile_metrics_map: Dict[str, CompilationMetrics],
    size_cut: Optional[int] = None) -> str:
  """Splits compilation metrics into regressed/improved/all categories
    and returns their markdown tables.

    If size_cut is None, the table includes regressed/improved/all categories;
    otherwise, the table includes regressed/improved categories.

    Args:
      compile_metrics_map: A dictionary of benchmark names to its compilation
        metrics.
      size_cut: If not None, only show the top N results for each table.
  """

  tables = []
  for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS:
    regressed, improved, _, _ = _categorize_on_single_metric(
        compile_metrics_map, mapper.get_current_and_base_value,
        mapper.get_metric_thresholds(), mapper.get_unit())

    table_title = mapper.get_table_title()
    table_header = mapper.get_table_header()
    if regressed:
      tables.append(md.header(f"Regressed {table_title} 🚩", 3))
      tables.append(
          _sort_metrics_objects_and_get_table(
              metrics_objs=regressed,
              mapper=mapper,
              headers=["Benchmark Name", table_header],
              size_cut=size_cut))
    if improved:
      tables.append(md.header(f"Improved {table_title} 🎉", 3))
      tables.append(
          _sort_metrics_objects_and_get_table(
              metrics_objs=improved,
              mapper=mapper,
              headers=["Benchmark Name", table_header],
              size_cut=size_cut))

  # If we want to abbreviate, similar results won't be interesting.
  if size_cut is None and compile_metrics_map:
    tables.append(md.header("All Compilation Metrics", 3))
    headers = ["Benchmark Name"] + [
        mapper.get_table_header()
        for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS
    ]
    rows = []
    for target_id, metrics in compile_metrics_map.items():
      row = [metrics.name]
      for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS:
        current, base = mapper.get_current_and_base_value(metrics)
        row.append(
            _make_series_link(_get_compare_text(current, base),
                              mapper.get_series_id(target_id)))
      rows.append(tuple(row))

    tables.append(
        _add_header_and_get_markdown_table(headers, rows, size_cut=size_cut))

  return "\n\n".join(tables)