| # Copyright 2022 The IREE Authors |
| # |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. |
| # See https://llvm.org/LICENSE.txt for license information. |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| r"""Script to generate a HTML summary of SHARK Tank benchmarks. |
| |
| Example usage: |
| |
| python parse_shark_benchmarks.py \ |
| --cpu_shark_csv=icelake_shark_bench_results.csv \ |
| --cpu_iree_csv=icelake_iree_bench_results.csv \ |
| --cpu_baseline_csv=cpu_baseline.csv \ |
| --gpu_shark_csv=a100_shark_bench_results.csv \ |
| --gpu_iree_csv=a100_iree_bench_results.csv \ |
| --gpu_baseline_csv=a100_baseline.csv \ |
| --version_info iree=version_info.txt \ |
| --output_path=/tmp/summary.html |
| """ |
| |
| import argparse |
| import pandas as pd |
| import pathlib |
| import sys |
| |
| from datetime import date |
| |
| # Add build_tools python dir to the search path. |
| sys.path.insert(0, str(pathlib.Path(__file__).parent.parent.parent / "python")) |
| from reporting.common import html_utils |
| |
| # Column headers. |
| _MODEL = "model" |
| _DIALECT = "dialect" |
| _DATA_TYPE = "data_type" |
| _BASELINE = "baseline" |
| _DEVICE = "device" |
| _BASELINE_LATENCY = "baseline latency (ms)" |
| _IREE_LATENCY = "IREE latency (ms)" |
| _SHARK_LATENCY = "SHARK latency (ms)" |
| _IREE_VS_BASELINE = "IREE vs baseline" |
| _SHARK_VS_BASELINE = "SHARK vs baseline" |
| _IREE_VS_SHARK = "IREE vs SHARK" |
| _BASELINE_MEMORY = "Baseline Peak Device Memory (MB)" |
| _IREE_MEMORY = "IREE Peak Device Memory (MB)" |
| _SHARK_MEMORY = "SHARK Peak Device Memory (MB)" |
| |
| _PERF_COLUMNS = [_IREE_VS_BASELINE, _SHARK_VS_BASELINE, _IREE_VS_SHARK] |
| _LATENCY_COLUMNS = [_BASELINE_LATENCY, _IREE_LATENCY, _SHARK_LATENCY] |
| _MEMORY_COLUMNS = [_BASELINE_MEMORY, _IREE_MEMORY, _SHARK_MEMORY] |
| |
| |
| def _generate_table(df_iree, df_shark, df_baseline, title): |
| """Generates a table comparing latencies between IREE, SHARK and a baseline.""" |
| summary = pd.DataFrame(columns=[ |
| _MODEL, _BASELINE, _DATA_TYPE, _DIALECT, _DEVICE, _BASELINE_LATENCY, |
| _IREE_LATENCY, _SHARK_LATENCY, _IREE_VS_BASELINE, _SHARK_VS_BASELINE, |
| _IREE_VS_SHARK, _BASELINE_MEMORY, _IREE_MEMORY, _SHARK_MEMORY |
| ]) |
| |
| models = df_iree.model.unique() |
| for model in models: |
| iree_results_per_model = df_iree.loc[df_iree.model == model] |
| dialects = iree_results_per_model.dialect.unique() |
| for dialect in dialects: |
| iree_results_per_dialect = iree_results_per_model.loc[ |
| iree_results_per_model.dialect == dialect] |
| data_types = iree_results_per_dialect.data_type.unique() |
| for data_type in data_types: |
| iree_results_per_datatype = iree_results_per_dialect.loc[ |
| iree_results_per_dialect.data_type == data_type] |
| device_types = iree_results_per_datatype.device.unique() |
| for device in device_types: |
| iree_results = iree_results_per_datatype.loc[ |
| iree_results_per_datatype.device == device] |
| if len(iree_results) != 3: |
| print(f"Warning! Expected number of results to be 3. Got" |
| f" {len(iree_results)}") |
| print(iree_results) |
| continue |
| |
| baseline_results = df_baseline.loc[(df_baseline.model == model) & |
| (df_baseline.dialect == dialect) & |
| (df_baseline.data_type |
| == data_type) & |
| (df_baseline.device == device)] |
| |
| if baseline_results.empty: |
| # We use snapshots of latencies for baseline. If it is a new |
| # benchmark that is not included in the snapshot yet, emit a |
| # warning. |
| print( |
| f"Warning: No baseline results found for {model}, {dialect}," |
| f" {data_type}, {device}. Using IREE version as baseline. Please" |
| f" update baseline csv.") |
| engine = iree_results.engine.iloc[0] |
| baseline_df = iree_results.loc[iree_results.engine == engine] |
| baseline_latency = baseline_df.iloc[0]["ms/iter"] |
| baseline_device_mb = baseline_df.iloc[0]["device_memory_mb"] |
| else: |
| engine = baseline_results.engine.iloc[0] |
| baseline_df = baseline_results.loc[baseline_results.engine == |
| engine] |
| baseline_latency = baseline_df.iloc[0]["ms/iter"] |
| baseline_device_mb = baseline_df.iloc[0]["device_memory_mb"] |
| |
| iree_df = iree_results.loc[iree_results.engine == "shark_iree_c"] |
| iree_latency = iree_df.iloc[0]["ms/iter"] |
| iree_device_mb = iree_df.iloc[0]["device_memory_mb"] |
| iree_vs_baseline = html_utils.format_latency_comparison( |
| iree_latency, baseline_latency) |
| |
| if df_shark is not None: |
| shark_results = df_shark.loc[(df_shark.model == model) & |
| (df_shark.dialect == dialect) & |
| (df_shark.data_type == data_type) & |
| (df_shark.device == device)] |
| if shark_results.empty: |
| print( |
| f"Warning: No SHARK results for {model}, {dialect}, {data_type}, {device}." |
| ) |
| continue |
| |
| shark_df = shark_results.loc[shark_results.engine == "shark_iree_c"] |
| shark_latency = shark_df.iloc[0]["ms/iter"] |
| shark_device_mb = shark_df.iloc[0]["device_memory_mb"] |
| shark_vs_baseline = html_utils.format_latency_comparison( |
| shark_latency, baseline_latency) |
| iree_vs_shark = html_utils.format_latency_comparison( |
| iree_latency, shark_latency) |
| else: |
| # If there are no SHARK benchmarks available, use default values. |
| # These columns will be hidden later. |
| shark_latency = 0 |
| shark_vs_baseline = "<missing_comparison>" |
| iree_vs_shark = "<missing_comparison>" |
| |
| summary.loc[len(summary)] = [ |
| model, |
| engine, |
| data_type, |
| dialect, |
| device, |
| f"{baseline_latency:.1f}", |
| f"{iree_latency:.1f}", |
| f"{shark_latency:.1f}", |
| iree_vs_baseline, |
| shark_vs_baseline, |
| iree_vs_shark, |
| f"{baseline_device_mb:.3f}", |
| f"{iree_device_mb:.3f}", |
| f"{shark_device_mb:.3f}", |
| ] |
| |
| summary = summary.round(2) |
| |
| st = summary.style.set_table_styles(html_utils.get_table_css()) |
| st = st.hide(axis="index") |
| if df_shark is None: |
| st = st.hide_columns( |
| subset=[_SHARK_LATENCY, _SHARK_VS_BASELINE, _IREE_VS_SHARK]) |
| st = st.set_caption(title) |
| st = st.applymap(html_utils.style_performance, subset=_PERF_COLUMNS) |
| st = st.set_properties(subset=[_MODEL], |
| **{ |
| "width": "300px", |
| "text-align": "left", |
| }) |
| st = st.set_properties(subset=[_BASELINE], |
| **{ |
| "width": "140", |
| "text-align": "center", |
| }) |
| st = st.set_properties(subset=[_DIALECT, _DATA_TYPE, _DEVICE], |
| **{ |
| "width": "100", |
| "text-align": "center", |
| }) |
| st = st.set_properties(subset=_LATENCY_COLUMNS, |
| **{ |
| "width": "100", |
| "text-align": "right", |
| }) |
| st = st.set_properties(subset=_PERF_COLUMNS, |
| **{ |
| "width": "150px", |
| "text-align": "right", |
| "color": "#ffffff" |
| }) |
| st = st.set_properties(subset=_MEMORY_COLUMNS, |
| **{ |
| "width": "100", |
| "text-align": "right", |
| }) |
| |
| return st.to_html() + "<br/>" |
| |
| |
| def generate_table(iree_csv, |
| baseline_csv, |
| shark_csv=None, |
| shape_type="static", |
| device="cpu", |
| title="Benchmarks"): |
| """Generates a table comparing latencies between IREE, SHARK and a baseline. |
| |
| Args: |
| iree_csv: Path to the csv file containing IREE latencies. |
| baseline_csv: Path to the csv file containing baseline latencies. |
| shark_csv: Path to the csv file containing SHARK-Runtime latencies. This is optional. |
| shape_type: Currently either `static` or `dynamic`. |
| device: Device used to run the benchmarks. |
| title: The title of the generated table. |
| |
| Returns: |
| An HTML string containing the summarized report. |
| """ |
| shark_df = None |
| if shark_csv is not None: |
| shark_df = pd.read_csv(shark_csv) |
| shark_df = shark_df.loc[(shark_df.shape_type == shape_type) & |
| (shark_df.device == device)] |
| |
| iree_df = pd.read_csv(iree_csv) |
| iree_df = iree_df.loc[(iree_df.shape_type == shape_type) & |
| (iree_df.device == device)] |
| |
| baseline_df = pd.read_csv(baseline_csv) |
| baseline_df = baseline_df.loc[(baseline_df.shape_type == shape_type) & |
| (baseline_df.device == device)] |
| |
| return _generate_table(iree_df, shark_df, baseline_df, title) |
| |
| |
| def main(args): |
| """Summarizes benchmark results generated by the SHARK Tank.""" |
| version_html = f"<i>last updated: {date.today().isoformat()}</i><br/><br/>" |
| version_html += "<i><b>Version Info</b></i><br/>" |
| with open(args.version_info) as f: |
| version_info = dict(l.strip().split("=", 1) for l in f) |
| for key, value in version_info.items(): |
| version_html += f"<i>{key}: {value}</i><br/>" |
| version_html += "<br/>" |
| |
| html = html_utils.generate_header_and_legend(version_html) |
| |
| # Generate Server CPU Static. |
| if args.cpu_iree_csv is not None: |
| html += generate_table(args.cpu_iree_csv, |
| args.cpu_baseline_csv, |
| shark_csv=args.cpu_shark_csv, |
| shape_type="static", |
| device="cpu", |
| title="Server Intel Ice Lake CPU (Static Shapes)") |
| |
| # Generate Server GPU Static. |
| if args.gpu_iree_csv is not None: |
| html += generate_table(args.gpu_iree_csv, |
| args.gpu_baseline_csv, |
| shark_csv=args.gpu_shark_csv, |
| shape_type="static", |
| device="cuda", |
| title="Server NVIDIA Tesla A100 GPU (Static Shapes)") |
| |
| # Generate Server CPU Dynamic. |
| if args.cpu_iree_csv is not None: |
| html += generate_table(args.cpu_iree_csv, |
| args.cpu_baseline_csv, |
| shark_csv=args.cpu_shark_csv, |
| shape_type="dynamic", |
| device="cpu", |
| title="Server Intel Ice Lake CPU (Dynamic Shapes)") |
| |
| # Generate Server GPU Dynamic. |
| if args.gpu_iree_csv is not None: |
| html += generate_table( |
| args.gpu_iree_csv, |
| args.gpu_baseline_csv, |
| shark_csv=args.gpu_shark_csv, |
| shape_type="dynamic", |
| device="cuda", |
| title="Server NVIDIA Tesla A100 GPU (Dynamic Shapes)") |
| |
| args.output_path.write_text(html) |
| |
| |
| def parse_args(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| "--cpu_shark_csv", |
| type=str, |
| default=None, |
| help="The path to the csv file with CPU benchmarking results from the " |
| "SHARK runtime.") |
| parser.add_argument( |
| "--cpu_iree_csv", |
| type=str, |
| default=None, |
| help="The path to the csv file with CPU benchmarking results from IREE.") |
| parser.add_argument( |
| "--cpu_baseline_csv", |
| type=str, |
| default="data/icelake_baseline_2022-09-19.csv", |
| help="The path to the csv file containing baseline CPU results.") |
| parser.add_argument( |
| "--gpu_shark_csv", |
| type=str, |
| default=None, |
| help="The path to the csv file with GPU benchmarking results from the " |
| "SHARK runtime.") |
| parser.add_argument( |
| "--gpu_iree_csv", |
| type=str, |
| default=None, |
| help="The path to the csv file with CPU benchmarking results from IREE.") |
| parser.add_argument( |
| "--gpu_baseline_csv", |
| type=str, |
| default="data/a100_baseline_2022-09-19.csv", |
| help="The path to the csv file containing baseline GPU results.") |
| parser.add_argument( |
| "--version_info", |
| type=pathlib.Path, |
| default=None, |
| help= |
| "The path to a text file containing version information of the frameworks tested." |
| ) |
| parser.add_argument( |
| "--output_path", |
| type=pathlib.Path, |
| default="/tmp/summary.html", |
| help="The path to the output html file that summarizes results.") |
| return parser.parse_args() |
| |
| |
| if __name__ == "__main__": |
| main(parse_args()) |
