Clean up code of legacy benchmark suite (#14081)
Clean up code of the legacy benchmark suite under `build_tools/benchmarks` and the related CMake rules.
Fix #11076
diff --git a/.github/workflows/benchmark_compilation.yml b/.github/workflows/benchmark_compilation.yml
index 616f945..32975c6 100644
--- a/.github/workflows/benchmark_compilation.yml
+++ b/.github/workflows/benchmark_compilation.yml
@@ -93,7 +93,7 @@
COMPILE_STATS_RESULTS: benchmark-results/compile-stats-results.json
run: |
mkdir -p benchmark-results
- ./build_tools/benchmarks/collect_compilation_statistics.py alpha \
+ ./build_tools/benchmarks/collect_compilation_statistics.py \
--e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR}" \
--build_log="${E2E_TEST_ARTIFACTS_BUILD_LOG}" \
--compilation_benchmark_config="${BENCHMARK_CONFIG}" \
diff --git a/build_tools/benchmarks/collect_compilation_statistics.py b/build_tools/benchmarks/collect_compilation_statistics.py
index a57a5e2..19c80e9 100755
--- a/build_tools/benchmarks/collect_compilation_statistics.py
+++ b/build_tools/benchmarks/collect_compilation_statistics.py
@@ -6,8 +6,8 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Collect compilation statistics from benchmark suites.
-The benchmark suites need to be built with ninja and enable the CMake option
-IREE_ENABLE_LEGACY_COMPILATION_BENCHMARKS.
+See docs/developers/developing_iree/benchmark_suites.md for how to build the
+benchmark suites.
"""
import pathlib
@@ -27,7 +27,6 @@
from common import benchmark_definition
from common.benchmark_definition import CompilationInfo, CompilationResults, CompilationStatistics, ModuleComponentSizes, get_git_commit_hash
-from common.benchmark_suite import BenchmarkSuite
from common import benchmark_config
from e2e_test_artifacts import iree_artifacts
from e2e_test_framework import serialization
@@ -39,6 +38,7 @@
NINJA_LOG_HEADER = "ninja log v5"
NINJA_BUILD_LOG = ".ninja_log"
COMPILATION_STATS_MODULE_SUFFIX = "compile-stats"
+E2E_TEST_ARTIFACTS_REL_PATH = "e2e_test_artifacts"
VM_COMPONENT_NAME = "module.fb"
CONST_COMPONENT_NAME = "_const.bin"
@@ -53,24 +53,19 @@
@dataclass(frozen=True)
class ModuleInfo(object):
module_path: pathlib.Path
- stream_stats_path: Optional[pathlib.Path]
+ stream_stats_path: pathlib.Path
def match_module_cmake_target(module_path: pathlib.PurePath) -> Optional[str]:
- if module_path.match(f"{benchmark_config.E2E_TEST_ARTIFACTS_REL_PATH}/iree_*/"
+ if module_path.match(f"{E2E_TEST_ARTIFACTS_REL_PATH}/iree_*/"
f"{iree_artifacts.MODULE_FILENAME}"):
# <e2e test artifacts dir>/iree_<module dir>/<module filename>
path_parts = module_path.parts[-3:]
- elif module_path.match(
- f"{benchmark_config.BENCHMARK_SUITE_REL_PATH}/*/{MODULE_DIR}/"
- f"*.{MODULE_FILE_EXTENSION}"):
- # <benchmark_suites dir>/<category>/vmfb/<module filename>.vmfb
- path_parts = module_path.parts[-4:]
- else:
- return None
- # Join to get the CMake target name. This is *not* a filesystem path, so we
- # don't want \ separators on Windows that we would get with os.path.join().
- return '/'.join(path_parts)
+ # Join to get the CMake target name. This is *not* a filesystem path, so we
+ # don't want \ separators on Windows that we would get with os.path.join().
+ return '/'.join(path_parts)
+
+ return None
def parse_compilation_time_from_ninja_log(log: TextIO) -> Dict[str, int]:
@@ -136,20 +131,6 @@
total_dispatch_component_bytes=total_dispatch_component_bytes)
-def get_module_path(flag_file: TextIO) -> Optional[str]:
- """Retrieve the module path for compilation statistics from the flag file."""
-
- module_path = None
- for line in flag_file:
- match = re.match("--module=(.+)", line.strip())
- if match:
- module_name, module_ext = os.path.splitext(match.group(1))
- module_path = f"{module_name}-{COMPILATION_STATS_MODULE_SUFFIX}{module_ext}"
- break
-
- return module_path
-
-
def get_module_map_from_compilation_benchmark_config(
compilation_benchmark_config_data: TextIO,
e2e_test_artifacts_dir: pathlib.PurePath
@@ -188,53 +169,6 @@
return module_map
-def get_module_map_from_benchmark_suite(
- benchmark_suite_dir: pathlib.Path) -> Dict[CompilationInfo, ModuleInfo]:
- benchmark_suite = BenchmarkSuite.load_from_benchmark_suite_dir(
- benchmark_suite_dir)
- module_map = {}
- for category, _ in benchmark_suite.list_categories():
- benchmark_cases = benchmark_suite.filter_benchmarks_for_category(
- category=category)
- for benchmark_case in benchmark_cases:
- if benchmark_case.benchmark_case_dir is None:
- raise ValueError("benchmark_case_dir can't be None.")
- benchmark_case_dir = benchmark_case.benchmark_case_dir
-
- flag_file_path = benchmark_case_dir / BENCHMARK_FLAGFILE
- with flag_file_path.open("r") as flag_file:
- module_path = get_module_path(flag_file)
-
- if module_path is None:
- raise RuntimeError(
- f"Can't find the module file in the flagfile: {flag_file_path}")
- compilation_info = CompilationInfo.build_with_legacy_name(
- model_name=benchmark_case.model_name,
- model_tags=tuple(benchmark_case.model_tags),
- model_source=category,
- target_arch=benchmark_case.target_arch,
- compile_tags=tuple(benchmark_case.bench_mode))
- module_map[compilation_info] = ModuleInfo(
- module_path=(benchmark_case_dir / module_path).resolve(),
- stream_stats_path=None)
-
- return module_map
-
-
-def _legacy_get_module_map_and_build_log(args: argparse.Namespace):
- module_map = get_module_map_from_benchmark_suite(
- args.build_dir / benchmark_config.BENCHMARK_SUITE_REL_PATH)
- return module_map, args.build_dir / NINJA_BUILD_LOG
-
-
-def _alpha_get_module_map_and_build_log(args: argparse.Namespace):
- config_data = args.compilation_benchmark_config.open("r")
- module_map = get_module_map_from_compilation_benchmark_config(
- compilation_benchmark_config_data=config_data,
- e2e_test_artifacts_dir=args.e2e_test_artifacts_dir)
- return module_map, args.build_log
-
-
def _check_dir_path(path_str: str) -> pathlib.Path:
path = pathlib.Path(path_str)
if not path.is_dir():
@@ -252,55 +186,35 @@
def _parse_arguments():
"""Returns an argument parser with common options."""
- # Makes global options come *after* command.
- # See https://stackoverflow.com/q/23296695
- subparser_base = argparse.ArgumentParser(add_help=False)
- subparser_base.add_argument("--output",
- type=pathlib.Path,
- help="Path to output JSON file.")
- subparser_base.add_argument(
- "--verbose",
- action="store_true",
- help="Print internal information during execution.")
-
parser = argparse.ArgumentParser(
description="Collect compilation statistics from benchmark suites.")
-
- subparser = parser.add_subparsers(title="tool version", required=True)
- legacy_parser = subparser.add_parser("legacy",
- parents=[subparser_base],
- help="Use with legacy benchmark suites.")
- legacy_parser.set_defaults(
- get_module_map_and_build_log=_legacy_get_module_map_and_build_log)
- legacy_parser.add_argument(
- "build_dir",
- type=_check_dir_path,
- help="Path to the build directory containing benchmark suites.")
-
- alpha_parser = subparser.add_parser("alpha",
- parents=[subparser_base],
- help="Use with e2e test artifacts.")
- alpha_parser.set_defaults(
- get_module_map_and_build_log=_alpha_get_module_map_and_build_log)
- alpha_parser.add_argument(
+ parser.add_argument(
"--compilation_benchmark_config",
type=_check_file_path,
required=True,
help="Exported compilation benchmark config of e2e test artifacts.")
- alpha_parser.add_argument("--build_log",
- type=_check_file_path,
- required=True,
- help="Path to the ninja build log.")
- alpha_parser.add_argument("--e2e_test_artifacts_dir",
- type=_check_dir_path,
- required=True,
- help="Path to the e2e test artifacts directory.")
+ parser.add_argument("--build_log",
+ type=_check_file_path,
+ required=True,
+ help="Path to the ninja build log.")
+ parser.add_argument("--e2e_test_artifacts_dir",
+ type=_check_dir_path,
+ required=True,
+ help="Path to the e2e test artifacts directory.")
+ parser.add_argument("--output",
+ type=pathlib.Path,
+ help="Path to output JSON file.")
return parser.parse_args()
def main(args: argparse.Namespace):
- module_map, build_log_path = args.get_module_map_and_build_log(args)
+ config_data = args.compilation_benchmark_config.open("r")
+ module_map = get_module_map_from_compilation_benchmark_config(
+ compilation_benchmark_config_data=config_data,
+ e2e_test_artifacts_dir=args.e2e_test_artifacts_dir)
+ build_log_path = args.build_log
+
with build_log_path.open("r") as log_file:
target_build_time_map = parse_compilation_time_from_ninja_log(log_file)
@@ -318,15 +232,10 @@
f"Module path isn't a module cmake target: {module_path}")
compilation_time_ms = target_build_time_map[cmake_target]
- if module_info.stream_stats_path is None:
- # TODO(#11076): Set dummy data as the legacy benchmark suites don't
- # support IR statistics. Will be removed during the cleanup.
- ir_stats = benchmark_definition.IRStatistics(stream_dispatch_count=-1)
- else:
- stream_stats_json = json.loads(module_info.stream_stats_path.read_text())
- exec_stats_json = stream_stats_json["stream-aggregate"]["execution"]
- ir_stats = benchmark_definition.IRStatistics(
- stream_dispatch_count=exec_stats_json["dispatch-count"])
+ stream_stats_json = json.loads(module_info.stream_stats_path.read_text())
+ exec_stats_json = stream_stats_json["stream-aggregate"]["execution"]
+ ir_stats = benchmark_definition.IRStatistics(
+ stream_dispatch_count=exec_stats_json["dispatch-count"])
compilation_statistics = CompilationStatistics(
compilation_info=compilation_info,
diff --git a/build_tools/benchmarks/collect_compilation_statistics_test.py b/build_tools/benchmarks/collect_compilation_statistics_test.py
index cb1f405..6328757 100644
--- a/build_tools/benchmarks/collect_compilation_statistics_test.py
+++ b/build_tools/benchmarks/collect_compilation_statistics_test.py
@@ -12,7 +12,7 @@
import zipfile
from common.benchmark_definition import ModuleComponentSizes
-from collect_compilation_statistics import CONST_COMPONENT_NAME, VM_COMPONENT_NAME, get_module_component_info, get_module_path, parse_compilation_time_from_ninja_log
+from collect_compilation_statistics import CONST_COMPONENT_NAME, VM_COMPONENT_NAME, get_module_component_info, parse_compilation_time_from_ninja_log
from e2e_test_artifacts import iree_artifacts
from e2e_test_framework import serialization
from e2e_test_framework.definitions import common_definitions, iree_definitions
@@ -28,22 +28,15 @@
self.assertEqual(target, "e2e_test_artifacts/iree_abcd/module.vmfb")
- def test_match_module_cmake_target_with_benchmark_suites(self):
- target = collect_compilation_statistics.match_module_cmake_target(
- pathlib.PurePath(
- "iree/iree-build/benchmark_suites/TFLite/vmfb/test.vmfb"))
-
- self.assertEqual(target, "benchmark_suites/TFLite/vmfb/test.vmfb")
-
def test_match_module_cmake_target_not_match(self):
target = collect_compilation_statistics.match_module_cmake_target(
- pathlib.PurePath("benchmark_suites/TFLite/vmfb/test.mlir"))
+ pathlib.PurePath("other/target.vmfb"))
self.assertIsNone(target)
def test_parse_compilation_time_from_ninja_log(self):
- target1 = "benchmark_suites/TFLite/vmfb/deeplabv3.vmfb"
- target2 = "benchmark_suites/TFLite/vmfb/mobilessd.vmfb"
+ target1 = "e2e_test_artifacts/iree_deeplabv3/module.vmfb"
+ target2 = "e2e_test_artifacts/iree_mobilessd/module.vmfb"
ninja_log = StringIO("# ninja log v5\n"
f"0\t100\taaa\tbuild/{target1}\taaa\n"
f"130\t200\tbbb\tbuild/{target2}\tbbb\n")
@@ -85,13 +78,6 @@
RuntimeError, lambda: get_module_component_info(
BytesIO(module_file_data), len(module_file_data)))
- def test_get_module_path(self):
- flag_file = StringIO(f"--module=/abcd.vmfb\n--inputs=1x2x3xf32")
-
- moduel_path = get_module_path(flag_file)
-
- self.assertEqual(moduel_path, "/abcd-compile-stats.vmfb")
-
def test_get_module_map_from_compilation_benchmark_config(self):
model_a = common_definitions.Model(
id="1234",
diff --git a/build_tools/benchmarks/common/benchmark_config.py b/build_tools/benchmarks/common/benchmark_config.py
index dc944d2..2d08d4e 100644
--- a/build_tools/benchmarks/common/benchmark_config.py
+++ b/build_tools/benchmarks/common/benchmark_config.py
@@ -10,10 +10,8 @@
from typing import Optional
import pathlib
-BENCHMARK_SUITE_REL_PATH = "benchmark_suites"
BENCHMARK_RESULTS_REL_PATH = "benchmark-results"
CAPTURES_REL_PATH = "captures"
-E2E_TEST_ARTIFACTS_REL_PATH = "e2e_test_artifacts"
@dataclass
@@ -112,17 +110,7 @@
capture_tarball=args.capture_tarball.resolve(),
capture_tmp_dir=per_commit_tmp_dir / CAPTURES_REL_PATH)
- if args.e2e_test_artifacts_dir is not None:
- root_benchmark_dir = args.e2e_test_artifacts_dir
- else:
- # TODO(#11076): Remove legacy path.
- build_dir = args.build_dir.resolve()
- if args.execution_benchmark_config is not None:
- root_benchmark_dir = build_dir / E2E_TEST_ARTIFACTS_REL_PATH
- else:
- root_benchmark_dir = build_dir / BENCHMARK_SUITE_REL_PATH
-
- return BenchmarkConfig(root_benchmark_dir=root_benchmark_dir,
+ return BenchmarkConfig(root_benchmark_dir=args.e2e_test_artifacts_dir,
benchmark_results_dir=per_commit_tmp_dir /
BENCHMARK_RESULTS_REL_PATH,
git_commit_hash=git_commit_hash,
diff --git a/build_tools/benchmarks/common/benchmark_config_test.py b/build_tools/benchmarks/common/benchmark_config_test.py
index 0f8dbb7..32d2387 100644
--- a/build_tools/benchmarks/common/benchmark_config_test.py
+++ b/build_tools/benchmarks/common/benchmark_config_test.py
@@ -17,32 +17,42 @@
class BenchmarkConfigTest(unittest.TestCase):
def setUp(self):
- self._build_dir_manager = tempfile.TemporaryDirectory()
self._tmp_dir_manager = tempfile.TemporaryDirectory()
- self.build_dir = pathlib.Path(self._build_dir_manager.name).resolve()
self.tmp_dir = pathlib.Path(self._tmp_dir_manager.name).resolve()
+ self._build_dir_manager = tempfile.TemporaryDirectory()
+ self.build_dir = pathlib.Path(self._build_dir_manager.name).resolve()
+ self.e2e_test_artifacts_dir = self.build_dir / "e2e_test_artifacts"
+ self.e2e_test_artifacts_dir.mkdir()
self.normal_tool_dir = self.build_dir / "normal_tool"
self.normal_tool_dir.mkdir()
self.traced_tool_dir = self.build_dir / "traced_tool"
self.traced_tool_dir.mkdir()
- self.trace_capture_tool = tempfile.NamedTemporaryFile()
- os.chmod(self.trace_capture_tool.name, stat.S_IEXEC)
+ self.trace_capture_tool = self.build_dir / "tracy_capture"
+ # Create capture tool with executable file mode.
+ self.trace_capture_tool.touch(mode=0o755)
+ self.execution_config = self.build_dir / "execution_config.json"
+ self.execution_config.touch()
def tearDown(self):
- self.trace_capture_tool.close()
- self._tmp_dir_manager.cleanup()
self._build_dir_manager.cleanup()
+ self._tmp_dir_manager.cleanup()
def test_build_from_args(self):
args = common_arguments.Parser().parse_args([
f"--tmp_dir={self.tmp_dir}",
f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
f"--traced_benchmark_tool_dir={self.traced_tool_dir}",
- f"--trace_capture_tool={self.trace_capture_tool.name}",
- f"--capture_tarball=capture.tar", f"--driver_filter_regex=a",
- f"--model_name_regex=b", f"--mode_regex=c", f"--keep_going",
- f"--benchmark_min_time=10", f"--compatible_only",
- str(self.build_dir)
+ f"--trace_capture_tool={self.trace_capture_tool}",
+ f"--capture_tarball=capture.tar",
+ f"--driver_filter_regex=a",
+ f"--model_name_regex=b",
+ f"--mode_regex=c",
+ f"--keep_going",
+ f"--benchmark_min_time=10",
+ f"--compatible_only",
+ f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
+ f"--execution_benchmark_config={self.execution_config}",
+ "--target_device=test",
])
config = benchmark_config.BenchmarkConfig.build_from_args(
@@ -51,11 +61,11 @@
per_commit_tmp_dir = self.tmp_dir / "abcd"
expected_trace_capture_config = benchmark_config.TraceCaptureConfig(
traced_benchmark_tool_dir=self.traced_tool_dir,
- trace_capture_tool=pathlib.Path(self.trace_capture_tool.name).resolve(),
+ trace_capture_tool=pathlib.Path(self.trace_capture_tool).resolve(),
capture_tarball=pathlib.Path("capture.tar").resolve(),
capture_tmp_dir=per_commit_tmp_dir / "captures")
expected_config = benchmark_config.BenchmarkConfig(
- root_benchmark_dir=self.build_dir / "benchmark_suites",
+ root_benchmark_dir=self.e2e_test_artifacts_dir,
benchmark_results_dir=per_commit_tmp_dir / "benchmark-results",
git_commit_hash="abcd",
normal_benchmark_tool_dir=self.normal_tool_dir,
@@ -72,7 +82,9 @@
args = common_arguments.Parser().parse_args([
f"--tmp_dir={self.tmp_dir}",
f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
- str(self.build_dir)
+ f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
+ f"--execution_benchmark_config={self.execution_config}",
+ "--target_device=test",
])
config = benchmark_config.BenchmarkConfig.build_from_args(
@@ -85,52 +97,15 @@
f"--tmp_dir={self.tmp_dir}",
f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
f"--traced_benchmark_tool_dir={self.traced_tool_dir}",
- str(self.build_dir)
+ f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
+ f"--execution_benchmark_config={self.execution_config}",
+ "--target_device=test",
])
self.assertRaises(
ValueError, lambda: benchmark_config.BenchmarkConfig.build_from_args(
args=args, git_commit_hash="abcd"))
- def test_build_from_args_with_e2e_test_artifacts_dir(self):
- with tempfile.TemporaryDirectory() as e2e_test_artifacts_dir:
- exec_bench_config = pathlib.Path(
- e2e_test_artifacts_dir) / "exec_bench_config.json"
- exec_bench_config.touch()
- args = common_arguments.Parser().parse_args([
- f"--tmp_dir={self.tmp_dir}",
- f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
- f"--e2e_test_artifacts_dir={e2e_test_artifacts_dir}",
- f"--execution_benchmark_config={exec_bench_config}",
- f"--target_device_name=device_a",
- ])
-
- config = benchmark_config.BenchmarkConfig.build_from_args(
- args=args, git_commit_hash="abcd")
-
- self.assertEqual(config.root_benchmark_dir,
- pathlib.Path(e2e_test_artifacts_dir))
-
- def test_build_from_args_with_execution_benchmark_config_and_build_dir(self):
- with tempfile.TemporaryDirectory() as e2e_test_artifacts_dir:
- exec_bench_config = pathlib.Path(
- e2e_test_artifacts_dir) / "exec_bench_config.json"
- exec_bench_config.touch()
- args = common_arguments.Parser().parse_args([
- f"--tmp_dir={self.tmp_dir}",
- f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
- f"--execution_benchmark_config={exec_bench_config}",
- f"--target_device_name=device_a",
- str(self.build_dir)
- ])
-
- config = benchmark_config.BenchmarkConfig.build_from_args(
- args=args, git_commit_hash="abcd")
-
- self.assertEqual(
- config.root_benchmark_dir,
- self.build_dir / benchmark_config.E2E_TEST_ARTIFACTS_REL_PATH)
-
if __name__ == "__main__":
unittest.main()
diff --git a/build_tools/benchmarks/common/benchmark_definition.py b/build_tools/benchmarks/common/benchmark_definition.py
index 6101271..1bd29ea 100644
--- a/build_tools/benchmarks/common/benchmark_definition.py
+++ b/build_tools/benchmarks/common/benchmark_definition.py
@@ -21,25 +21,6 @@
from e2e_test_framework.definitions import common_definitions
-# A map from CPU ABI to IREE's legacy benchmark target architecture.
-CPU_ABI_TO_LEGACY_TARGET_ARCH_MAP = {
- "arm64-v8a": "cpu-arm64-v8a",
- "x86_64-cascadeLake": "cpu-x86_64-cascadelake",
-}
-
-# A map from GPU name to IREE's legacy benchmark target architecture.
-GPU_NAME_TO_LEGACY_TARGET_ARCH_MAP = {
- "adreno-640": "gpu-adreno",
- "adreno-650": "gpu-adreno",
- "adreno-660": "gpu-adreno",
- "adreno-730": "gpu-adreno",
- "mali-g77": "gpu-mali-valhall",
- "mali-g78": "gpu-mali-valhall",
- "tesla-v100-sxm2-16gb": "gpu-cuda-sm_70",
- "nvidia-a100-sxm4-40gb": "gpu-cuda-sm_80",
- "nvidia-geforce-rtx-3090": "gpu-cuda-sm_80",
-}
-
# A map from CPU ABI to IREE's benchmark target architecture.
CPU_ABI_TO_TARGET_ARCH_MAP = {
"arm64-v8a":
@@ -252,29 +233,16 @@
params = ", ".join(params)
return f"{self.platform_type.value} device <{params}>"
- def get_iree_cpu_arch_name(self,
- use_legacy_name: bool = False) -> Optional[str]:
+ def get_cpu_arch(self) -> Optional[common_definitions.DeviceArchitecture]:
name = self.cpu_abi.lower()
if self.cpu_uarch:
name += f"-{self.cpu_uarch.lower()}"
- if use_legacy_name:
- return CPU_ABI_TO_LEGACY_TARGET_ARCH_MAP.get(name)
+ return CPU_ABI_TO_TARGET_ARCH_MAP.get(name)
- arch = CPU_ABI_TO_TARGET_ARCH_MAP.get(name)
- # TODO(#11076): Return common_definitions.DeviceArchitecture instead after
- # removing the legacy path.
- return None if arch is None else str(arch)
-
- def get_iree_gpu_arch_name(self,
- use_legacy_name: bool = False) -> Optional[str]:
+ def get_gpu_arch(self) -> Optional[common_definitions.DeviceArchitecture]:
name = self.gpu_name.lower()
-
- if use_legacy_name:
- return GPU_NAME_TO_LEGACY_TARGET_ARCH_MAP.get(name)
-
- arch = GPU_NAME_TO_TARGET_ARCH_MAP.get(name)
- return None if arch is None else str(arch)
+ return GPU_NAME_TO_TARGET_ARCH_MAP.get(name)
def get_detailed_cpu_arch_name(self) -> str:
"""Returns the detailed architecture name."""
@@ -341,10 +309,12 @@
- model_source: the source of the model, e.g., 'TensorFlow'
- bench_mode: a list of tags for benchmark mode,
e.g., ['1-thread', 'big-core', 'full-inference']
+ - device_info: an DriverInfo object describing the IREE runtime dirver.
+ - device_info: an DeviceInfo object describing the device where benchmarks run
- compile_tags: an optional list of tags to describe the compile configs,
e.g., ['fuse-padding']
- runner: which runner is used for benchmarking, e.g., 'iree_vulkan', 'tflite'
- - device_info: an DeviceInfo object describing the device where benchmarks run
+ - run_config_id: ID of the corresponding iree_definitions.E2EModelRunConfig.
"""
name: str
@@ -360,48 +330,6 @@
def __str__(self):
return self.name
- @classmethod
- def build_with_legacy_name(cls, model_name: str, model_tags: Sequence[str],
- model_source: str, bench_mode: Sequence[str],
- driver_info: DriverInfo, device_info: DeviceInfo):
- """Build legacy name by combining the components of the BenchmarkInfo.
-
- This is the legacy way to construct the name and still used as primary key
- in the legacy benchmark system. It's deprecated and the new benchmark suites
- use a human-defined name which can be more concise.
- """
- # TODO(#11076): Remove when we drop the legacy path in
- # BenchmarkDriver.__get_benchmark_info_from_case
-
- # Get the target architecture and better driver name depending on the runner.
- target_arch = None
- if driver_info.device_type == 'GPU':
- target_arch = "GPU-" + device_info.gpu_name
- elif driver_info.device_type == 'CPU':
- target_arch = "CPU-" + device_info.get_detailed_cpu_arch_name()
- else:
- raise ValueError(f"Unrecognized device type '{driver_info.device_type}' "
- f"of the driver '{driver_info.pretty_name}'")
-
- if model_tags:
- tags = ",".join(model_tags)
- model_part = f"{model_name} [{tags}] ({model_source})"
- else:
- model_part = f"{model_name} ({model_source})"
- device_part = f"{device_info.model} ({target_arch})"
-
- mode_tags = ",".join(bench_mode)
- name = (f"{model_part} {mode_tags} with {driver_info.pretty_name} "
- f"@ {device_part}")
-
- return cls(name=name,
- model_name=model_name,
- model_tags=model_tags,
- model_source=model_source,
- bench_mode=bench_mode,
- driver_info=driver_info,
- device_info=device_info)
-
def to_json_object(self) -> Dict[str, Any]:
return {
"name": self.name,
@@ -658,32 +586,6 @@
def __str__(self):
return self.name
- @classmethod
- def build_with_legacy_name(cls, model_name: str, model_tags: Sequence[str],
- model_source: str, target_arch: str,
- compile_tags: Sequence[str]):
- """Build legacy name by combining the components of the CompilationInfo.
-
- This is the legacy way to construct the name and still used as primary key
- in the legacy benchmark system. It's deprecated and the new benchmark suites
- use a human-defined name which can be more concise.
- """
- # TODO(#11076): Remove when we drop
- # collect_compilation_statistics.get_module_map_from_benchmark_suite
- if model_tags:
- tags = ",".join(model_tags)
- model_part = f"{model_name} [{tags}] ({model_source})"
- else:
- model_part = f"{model_name} ({model_source})"
- compile_tags_str = ",".join(compile_tags)
- name = f"{model_part} {target_arch} {compile_tags_str}"
- return cls(name=name,
- model_name=model_name,
- model_tags=tuple(model_tags),
- model_source=model_source,
- target_arch=target_arch,
- compile_tags=tuple(compile_tags))
-
@staticmethod
def from_json_object(json_object: Dict[str, Any]):
return CompilationInfo(name=json_object["name"],
diff --git a/build_tools/benchmarks/common/benchmark_driver.py b/build_tools/benchmarks/common/benchmark_driver.py
index c22afdf..efa168c 100644
--- a/build_tools/benchmarks/common/benchmark_driver.py
+++ b/build_tools/benchmarks/common/benchmark_driver.py
@@ -55,10 +55,9 @@
"""Execute the benchmark flow.
It performs the following steps:
- 1. Enumerate all categories in the benchmark suites.
- 2. For each category, enumerate and filter benchmark cases.
- 3. Call 'run_benchmark_case' for each benchmark case.
- 4. Collect the benchmark results and captures.
+ 1. Enumerate and filter benchmark cases.
+ 2. Call 'run_benchmark_case' for each benchmark case.
+ 3. Collect the benchmark results and captures.
"""
self.config.benchmark_results_dir.mkdir(parents=True, exist_ok=True)
@@ -66,10 +65,8 @@
self.config.trace_capture_config.capture_tmp_dir.mkdir(parents=True,
exist_ok=True)
- use_legacy_name = self.benchmark_suite.legacy_suite
-
- cpu_target_arch = self.device_info.get_iree_cpu_arch_name(use_legacy_name)
- gpu_target_arch = self.device_info.get_iree_gpu_arch_name(use_legacy_name)
+ cpu_target_arch = self.device_info.get_cpu_arch()
+ gpu_target_arch = self.device_info.get_gpu_arch()
detected_architectures = [
arch for arch in [cpu_target_arch, gpu_target_arch] if arch is not None
]
@@ -87,76 +84,74 @@
drivers, loaders = self.__get_available_drivers_and_loaders()
- for category, _ in self.benchmark_suite.list_categories():
- benchmark_cases = self.benchmark_suite.filter_benchmarks_for_category(
- category=category,
- available_drivers=drivers,
- available_loaders=loaders,
- target_architectures=compatible_arch_filter,
- driver_filter=self.config.driver_filter,
- mode_filter=self.config.mode_filter,
- model_name_filter=self.config.model_name_filter)
+ benchmark_cases = self.benchmark_suite.filter_benchmarks(
+ available_drivers=drivers,
+ available_loaders=loaders,
+ target_architectures=compatible_arch_filter,
+ driver_filter=self.config.driver_filter,
+ mode_filter=self.config.mode_filter,
+ model_name_filter=self.config.model_name_filter)
- for benchmark_case in benchmark_cases:
- benchmark_info = self.__get_benchmark_info_from_case(
- category=category, benchmark_case=benchmark_case)
- benchmark_name = str(benchmark_info)
+ for benchmark_case in benchmark_cases:
+ benchmark_info = self.__get_benchmark_info_from_case(
+ benchmark_case=benchmark_case)
+ benchmark_name = str(benchmark_info)
- if benchmark_case.target_arch not in detected_architectures:
- print(f"WARNING: Benchmark '{benchmark_name}' may be incompatible"
- f" with the detected architectures '{detected_architectures}'"
- f" on the device. Pass --compatible-only to skip incompatible"
- f" benchmarks.")
+ if benchmark_case.target_arch not in detected_architectures:
+ print(f"WARNING: Benchmark '{benchmark_name}' may be incompatible"
+ f" with the detected architectures '{detected_architectures}'"
+ f" on the device. Pass --compatible-only to skip incompatible"
+ f" benchmarks.")
- # Sanity check for the uniqueness of benchmark names.
- if benchmark_name in self._seen_benchmark_names:
- raise ValueError(
- f"Found duplicate benchmark {benchmark_name} in the suites.")
- self._seen_benchmark_names.add(benchmark_name)
+ # Sanity check for the uniqueness of benchmark names.
+ if benchmark_name in self._seen_benchmark_names:
+ raise ValueError(
+ f"Found duplicate benchmark {benchmark_name} in the suites.")
+ self._seen_benchmark_names.add(benchmark_name)
- results_path, capture_path = self.__get_output_paths(benchmark_name)
- # If we continue from the previous results, check and skip if the result
- # files exist.
- if self.config.continue_from_previous:
- if results_path is not None and results_path.exists():
- self.finished_benchmarks.append((benchmark_info, results_path))
- results_path = None
-
- if capture_path is not None and capture_path.exists():
- self.finished_captures.append(capture_path)
- capture_path = None
-
- # Skip if no need to benchmark and capture.
- if results_path is None and capture_path is None:
- continue
-
- print(f"--> Benchmark started: {benchmark_name} <--")
-
- try:
- self.run_benchmark_case(benchmark_case, results_path, capture_path)
- except Exception as e:
- # Delete unfinished results if they exist.
- if results_path is not None:
- results_path.unlink(missing_ok=True)
- if capture_path is not None:
- capture_path.unlink(missing_ok=True)
-
- if not self.config.keep_going:
- raise e
-
- print(f"Processing of benchmark failed with: {e}")
- self.benchmark_errors.append(e)
- continue
- finally:
- # Some grace time.
- time.sleep(self.benchmark_grace_time)
-
- print("Benchmark completed")
-
- if results_path:
+ results_path, capture_path = self.__get_output_paths(benchmark_name)
+ # If we continue from the previous results, check and skip if the result
+ # files exist.
+ if self.config.continue_from_previous:
+ if results_path is not None and results_path.exists():
self.finished_benchmarks.append((benchmark_info, results_path))
- if capture_path:
+ results_path = None
+
+ if capture_path is not None and capture_path.exists():
self.finished_captures.append(capture_path)
+ capture_path = None
+
+ # Skip if no need to benchmark and capture.
+ if results_path is None and capture_path is None:
+ continue
+
+ print(f"--> Benchmark started: {benchmark_name} <--")
+
+ try:
+ self.run_benchmark_case(benchmark_case, results_path, capture_path)
+ except Exception as e:
+ # Delete unfinished results if they exist.
+ if results_path is not None:
+ results_path.unlink(missing_ok=True)
+ if capture_path is not None:
+ capture_path.unlink(missing_ok=True)
+
+ if not self.config.keep_going:
+ raise e
+
+ print(f"Processing of benchmark failed with: {e}")
+ self.benchmark_errors.append(e)
+ continue
+ finally:
+ # Some grace time.
+ time.sleep(self.benchmark_grace_time)
+
+ print("Benchmark completed")
+
+ if results_path:
+ self.finished_benchmarks.append((benchmark_info, results_path))
+ if capture_path:
+ self.finished_captures.append(capture_path)
def get_benchmark_results(self) -> BenchmarkResults:
"""Returns the finished benchmark results."""
@@ -203,24 +198,16 @@
return (benchmark_results_filename, capture_filename)
def __get_benchmark_info_from_case(
- self, category: str, benchmark_case: BenchmarkCase) -> BenchmarkInfo:
+ self, benchmark_case: BenchmarkCase) -> BenchmarkInfo:
run_config = benchmark_case.run_config
- if run_config is None:
- # TODO(#11076): Remove legacy path.
- return BenchmarkInfo.build_with_legacy_name(
- model_name=benchmark_case.model_name,
- model_tags=benchmark_case.model_tags,
- model_source=category,
- bench_mode=benchmark_case.bench_mode,
- driver_info=benchmark_case.driver_info,
- device_info=self.device_info)
-
run_tags = run_config.module_execution_config.tags
- compile_tags = run_config.module_generation_config.compile_config.tags
+ gen_config = run_config.module_generation_config
+ model_source = str(gen_config.imported_model.model.source_type)
+ compile_tags = gen_config.compile_config.tags
return BenchmarkInfo(name=run_config.name,
model_name=benchmark_case.model_name,
model_tags=benchmark_case.model_tags,
- model_source=category,
+ model_source=model_source,
bench_mode=run_tags,
compile_tags=compile_tags,
driver_info=benchmark_case.driver_info,
diff --git a/build_tools/benchmarks/common/benchmark_driver_test.py b/build_tools/benchmarks/common/benchmark_driver_test.py
index d12ac1d..106cb55 100644
--- a/build_tools/benchmarks/common/benchmark_driver_test.py
+++ b/build_tools/benchmarks/common/benchmark_driver_test.py
@@ -139,19 +139,20 @@
model_name="model_tflite",
model_tags=[],
bench_mode=["sync"],
- target_arch="x86_64-cascadelake",
+ target_arch=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE,
driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"],
benchmark_case_dir=pathlib.Path("case1"),
benchmark_tool_name="tool",
run_config=run_config_a)
- self.case2 = BenchmarkCase(model_name="model_tflite",
- model_tags=[],
- bench_mode=["task"],
- target_arch="x86_64-cascadelake",
- driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"],
- benchmark_case_dir=pathlib.Path("case2"),
- benchmark_tool_name="tool",
- run_config=run_config_b)
+ self.case2 = BenchmarkCase(
+ model_name="model_tflite",
+ model_tags=[],
+ bench_mode=["task"],
+ target_arch=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE,
+ driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"],
+ benchmark_case_dir=pathlib.Path("case2"),
+ benchmark_tool_name="tool",
+ run_config=run_config_b)
compile_target_rv64 = iree_definitions.CompileTarget(
target_backend=iree_definitions.TargetBackend.LLVM_CPU,
@@ -178,16 +179,16 @@
model_name="model_tflite",
model_tags=[],
bench_mode=["task"],
- target_arch="riscv_64-generic",
+ target_arch=common_definitions.DeviceArchitecture.RV64_GENERIC,
driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"],
benchmark_case_dir=pathlib.Path("incompatible_case"),
benchmark_tool_name="tool",
run_config=run_config_incompatible)
- self.benchmark_suite = BenchmarkSuite({
- pathlib.Path("suite/TFLite"): [
- self.case1, self.case2, self.incompatible_case
- ],
- })
+ self.benchmark_suite = BenchmarkSuite([
+ self.case1,
+ self.case2,
+ self.incompatible_case,
+ ])
def tearDown(self) -> None:
self._tmp_dir_obj.cleanup()
diff --git a/build_tools/benchmarks/common/benchmark_presentation.py b/build_tools/benchmarks/common/benchmark_presentation.py
index 1d74e18..9cab2a7 100644
--- a/build_tools/benchmarks/common/benchmark_presentation.py
+++ b/build_tools/benchmarks/common/benchmark_presentation.py
@@ -99,12 +99,6 @@
def get_series_id(self, benchmark_id: str) -> str:
"""Returns the dashboard series id."""
- # TODO(#11076): Remove legacy path.
- # Whitespace is used in the legacy benchmark id as the delimiter, while not
- # used in the new benchmark id. This is a temporary solution to generate
- # both ids during the migration.
- if " " in benchmark_id:
- return self.get_series_name(benchmark_id)
return f"{benchmark_id}-{self.get_metric_id()}"
@abstractmethod
@@ -317,11 +311,7 @@
raise ValueError(f"Duplicated benchmark name: {series_name}")
benchmark_names.add(series_name)
- # TODO(#11076): Remove legacy path.
series_id = benchmark_run.info.run_config_id
- if series_id is None:
- series_id = series_name
-
if series_id in aggregate_results:
raise ValueError(f"Duplicated benchmark id: {series_id}")
@@ -369,10 +359,6 @@
target_names.add(target_name)
target_id = compile_stats.compilation_info.gen_config_id
- # TODO(#11076): Remove legacy path.
- if target_id is None:
- target_id = target_name
-
if target_id in compile_metrics:
raise ValueError(f"Duplicated target id: {target_id}")
diff --git a/build_tools/benchmarks/common/benchmark_suite.py b/build_tools/benchmarks/common/benchmark_suite.py
index 1a80969..d673909 100644
--- a/build_tools/benchmarks/common/benchmark_suite.py
+++ b/build_tools/benchmarks/common/benchmark_suite.py
@@ -5,28 +5,8 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Utilities for handling the benchmark suite.
-Benchmark artifacts should be generated by building the `iree-benchmark-suites`
-CMake target, which put them in the following directory structure:
-
-<root-build-dir>/benchmark_suites
-└── <benchmark-category> (e.g., TFLite)
- ├── <benchmark-suite> (e.g., MobileBertSquad-fp32)
- │ ├── <benchmark-case> (e.g., iree-vulkan__GPU-Mali-Valhall__kernel-execution)
- │ │ ├── compilation_statistics.json
- │ │ ├── tool
- │ │ └── flagfile
- │ ├── ...
- │ │ ├── compilation_statistics.json
- │ │ ├── tool
- │ │ └── flagfile
- │ └── <benchmark_case>
- │ │ ├── compilation_statistics.json
- │ ├── tool
- │ └── flagfile
- └── vmfb
- ├── <compiled-iree-model>-<sha1>.vmfb
- ├── ...
- └── <compiled-iree-model>-<sha1>.vmfb
+See docs/developers/developing_iree/benchmark_suites.md for how to build the
+benchmark suite.
"""
import collections
@@ -37,10 +17,7 @@
from typing import Dict, List, Optional, Sequence, Tuple
from common.benchmark_definition import IREE_DRIVERS_INFOS, DriverInfo
from e2e_test_artifacts import iree_artifacts
-from e2e_test_framework.definitions import iree_definitions
-
-# All benchmarks' relative path against root build directory.
-BENCHMARK_SUITE_REL_PATH = "benchmark_suites"
+from e2e_test_framework.definitions import common_definitions, iree_definitions
MODEL_FLAGFILE_NAME = "flagfile"
MODEL_TOOLFILE_NAME = "tool"
@@ -53,22 +30,21 @@
model_name: the source model, e.g., 'MobileSSD'.
model_tags: the source model tags, e.g., ['f32'].
bench_mode: the benchmark mode, e.g., '1-thread,big-core'.
- target_arch: the target CPU/GPU architature, e.g., 'GPU-Adreno'.
+ target_arch: the target CPU/GPU architature.
driver_info: the IREE driver configuration.
benchmark_tool_name: the benchmark tool, e.g., 'iree-benchmark-module'.
benchmark_case_dir: the path to benchmark case directory.
- run_config: the run config from e2e test framework. This overrides the
- `benchmark_case_dir`.
+ run_config: the run config from e2e test framework.
"""
model_name: str
model_tags: Sequence[str]
bench_mode: Sequence[str]
- target_arch: str
+ target_arch: common_definitions.DeviceArchitecture
driver_info: DriverInfo
benchmark_tool_name: str
benchmark_case_dir: pathlib.Path
- run_config: Optional[iree_definitions.E2EModelRunConfig] = None
+ run_config: iree_definitions.E2EModelRunConfig
# A map from execution config to driver info. This is temporary during migration
@@ -93,46 +69,25 @@
class BenchmarkSuite(object):
"""Represents the benchmarks in benchmark suite directory."""
- def __init__(self,
- suite_map: Dict[pathlib.Path, List[BenchmarkCase]],
- legacy_suite: bool = False):
+ def __init__(self, benchmark_cases: Sequence[BenchmarkCase]):
"""Construct a benchmark suite.
Args:
- suites: the map of benchmark cases keyed by category directories.
- legacy_suite: true if this is a legacy benchmark suite.
+ benchmark_cases: list of benchmark cases.
"""
- self.suite_map = suite_map
- self.category_map = dict((category_dir.name, category_dir)
- for category_dir in self.suite_map.keys())
- self.legacy_suite = legacy_suite
+ self.benchmark_cases = list(benchmark_cases)
- def list_categories(self) -> List[Tuple[str, pathlib.Path]]:
- """Returns all categories and their directories.
-
- Returns:
- A tuple of (category name, category dir).
- """
- category_list = [(name, path) for name, path in self.category_map.items()]
- # Fix the order of category list.
- category_list.sort(key=lambda category: category[0])
- return category_list
-
- # TODO(#11076): target_architectures should be a list of
- # common_definitions.DeviceArchitecture instead of string, after removing the
- # legacy path.
- def filter_benchmarks_for_category(
+ def filter_benchmarks(
self,
- category: str,
available_drivers: Optional[Sequence[str]] = None,
available_loaders: Optional[Sequence[str]] = None,
- target_architectures: Optional[Sequence[str]] = None,
+ target_architectures: Optional[Sequence[
+ common_definitions.DeviceArchitecture]] = None,
driver_filter: Optional[str] = None,
mode_filter: Optional[str] = None,
model_name_filter: Optional[str] = None) -> Sequence[BenchmarkCase]:
- """Filters benchmarks in a specific category for the given device.
+ """Filters benchmarks.
Args:
- category: the specific benchmark category.
available_drivers: list of drivers supported by the tools. None means to
match any driver.
available_loaders: list of executable loaders supported by the tools.
@@ -146,12 +101,8 @@
A list of matched benchmark cases.
"""
- category_dir = self.category_map.get(category)
- if category_dir is None:
- return []
-
chosen_cases = []
- for benchmark_case in self.suite_map[category_dir]:
+ for benchmark_case in self.benchmark_cases:
driver_info = benchmark_case.driver_info
driver_name = driver_info.driver_name
@@ -164,11 +115,10 @@
matched_loader = not driver_info.loader_name or available_loaders is None or (
driver_info.loader_name in available_loaders)
- target_arch = benchmark_case.target_arch.lower()
if target_architectures is None:
matched_arch = True
else:
- matched_arch = target_arch in target_architectures
+ matched_arch = benchmark_case.target_arch in target_architectures
bench_mode = ','.join(benchmark_case.bench_mode)
matched_mode = (mode_filter is None or
@@ -177,16 +127,8 @@
model_name_with_tags = benchmark_case.model_name
if len(benchmark_case.model_tags) > 0:
model_name_with_tags += f"-{','.join(benchmark_case.model_tags)}"
- if self.legacy_suite:
- # For backward compatibility, model_name_filter matches against the string:
- # <model name with tags>/<benchmark case name>
- model_and_case_name = f"{model_name_with_tags}/{benchmark_case.benchmark_case_dir.name}"
- else:
- # For the new run option, we drop the obscure old semantic and only
- # search on model name and its tags.
- model_and_case_name = model_name_with_tags
matched_model_name = (model_name_filter is None or re.match(
- model_name_filter, model_and_case_name) is not None)
+ model_name_filter, model_name_with_tags) is not None)
if (matched_driver and matched_loader and matched_arch and
matched_model_name and matched_mode):
@@ -206,7 +148,7 @@
A benchmark suite.
"""
- suite_map = collections.defaultdict(list)
+ benchmark_cases = []
for run_config in run_configs:
module_gen_config = run_config.module_generation_config
module_exec_config = run_config.module_execution_config
@@ -219,7 +161,7 @@
f"Can't map execution config to driver info: {module_exec_config}.")
driver_info = IREE_DRIVERS_INFOS[driver_info_key]
- target_arch = str(target_device_spec.architecture)
+ target_arch = target_device_spec.architecture
model = module_gen_config.imported_model.model
module_dir_path = iree_artifacts.get_module_dir_path(
@@ -235,50 +177,6 @@
benchmark_tool_name=run_config.tool.value,
benchmark_case_dir=module_dir_path,
run_config=run_config)
- category = pathlib.Path(model.source_type.value)
- suite_map[category].append(benchmark_case)
+ benchmark_cases.append(benchmark_case)
- return BenchmarkSuite(suite_map=suite_map)
-
- @staticmethod
- def load_from_benchmark_suite_dir(benchmark_suite_dir: pathlib.Path):
- """Scans and loads the benchmarks under the directory."""
-
- suite_map: Dict[pathlib.Path,
- List[BenchmarkCase]] = collections.defaultdict(list)
- for benchmark_case_dir in benchmark_suite_dir.glob("**"):
- model_dir = benchmark_case_dir.parent
- benchmark_name = benchmark_case_dir.name
- # Take the benchmark directory name and see if it matches the benchmark
- # naming convention:
- # <iree-driver>__<target-architecture>__<benchmark_mode>
- segments = benchmark_name.split("__")
- if len(segments) != 3 or not segments[0].startswith("iree-"):
- continue
-
- config, target_arch, bench_mode = segments
- bench_mode = bench_mode.split(",")
-
- # The path of model_dir is expected to be:
- # <benchmark_suite_dir>/<category>/<model_name>-<model_tags>
- category_dir = model_dir.parent
- model_name_with_tags = model_dir.name
- model_name_parts = model_name_with_tags.split("-", 1)
- model_name = model_name_parts[0]
- if len(model_name_parts) == 2:
- model_tags = model_name_parts[1].split(",")
- else:
- model_tags = []
-
- tool_name = (benchmark_case_dir / MODEL_TOOLFILE_NAME).read_text().strip()
-
- suite_map[category_dir].append(
- BenchmarkCase(model_name=model_name,
- model_tags=model_tags,
- bench_mode=bench_mode,
- target_arch=target_arch,
- driver_info=IREE_DRIVERS_INFOS[config.lower()],
- benchmark_case_dir=benchmark_case_dir,
- benchmark_tool_name=tool_name))
-
- return BenchmarkSuite(suite_map=suite_map, legacy_suite=True)
+ return BenchmarkSuite(benchmark_cases=benchmark_cases)
diff --git a/build_tools/benchmarks/common/benchmark_suite_test.py b/build_tools/benchmarks/common/benchmark_suite_test.py
index 8354547..7a8d69f 100644
--- a/build_tools/benchmarks/common/benchmark_suite_test.py
+++ b/build_tools/benchmarks/common/benchmark_suite_test.py
@@ -6,9 +6,7 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
import pathlib
-import tempfile
import unittest
-from typing import Sequence
from common.benchmark_definition import IREE_DRIVERS_INFOS
from common.benchmark_suite import BenchmarkCase, BenchmarkSuite
from e2e_test_framework.definitions import common_definitions, iree_definitions
@@ -17,114 +15,99 @@
class BenchmarkSuiteTest(unittest.TestCase):
- def test_list_categories(self):
- suite = BenchmarkSuite({
- pathlib.Path("suite/TFLite"): [],
- pathlib.Path("suite/PyTorch"): [],
- })
+ def test_filter_benchmarks(self):
+ model = common_definitions.Model(
+ id="model",
+ name="model",
+ tags=[],
+ source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR,
+ source_url="",
+ entry_function="predict",
+ input_types=["1xf32"])
+ exec_config = iree_definitions.ModuleExecutionConfig.build(
+ id="exec",
+ tags=[],
+ loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF,
+ driver=iree_definitions.RuntimeDriver.LOCAL_SYNC)
+ device_spec = common_definitions.DeviceSpec.build(
+ id="dev",
+ device_name="dev",
+ architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
+ host_environment=common_definitions.HostEnvironment.LINUX_X86_64,
+ device_parameters=[],
+ tags=[])
+ compile_target = iree_definitions.CompileTarget(
+ target_backend=iree_definitions.TargetBackend.LLVM_CPU,
+ target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC,
+ target_abi=iree_definitions.TargetABI.LINUX_GNU)
+ dummy_run_config = iree_definitions.E2EModelRunConfig.build(
+ module_generation_config=iree_definitions.ModuleGenerationConfig.build(
+ imported_model=iree_definitions.ImportedModel.from_model(model),
+ compile_config=iree_definitions.CompileConfig.build(
+ id="1", tags=[], compile_targets=[compile_target])),
+ module_execution_config=exec_config,
+ target_device_spec=device_spec,
+ input_data=common_definitions.ZEROS_MODEL_INPUT_DATA,
+ tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE)
- self.assertEqual(suite.list_categories(),
- [("PyTorch", pathlib.Path("suite/PyTorch")),
- ("TFLite", pathlib.Path("suite/TFLite"))])
+ case1 = BenchmarkCase(
+ model_name="deepnet",
+ model_tags=[],
+ bench_mode=["1-thread", "full-inference"],
+ target_arch=common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC,
+ driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"],
+ benchmark_case_dir=pathlib.Path("case1"),
+ benchmark_tool_name="tool",
+ run_config=dummy_run_config)
+ case2 = BenchmarkCase(
+ model_name="deepnetv2",
+ model_tags=["f32"],
+ bench_mode=["full-inference"],
+ target_arch=common_definitions.DeviceArchitecture.ARM_VALHALL,
+ driver_info=IREE_DRIVERS_INFOS["iree-vulkan"],
+ benchmark_case_dir=pathlib.Path("case2"),
+ benchmark_tool_name="tool",
+ run_config=dummy_run_config)
+ case3 = BenchmarkCase(
+ model_name="deepnetv3",
+ model_tags=["f32"],
+ bench_mode=["full-inference"],
+ target_arch=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE,
+ driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"],
+ benchmark_case_dir=pathlib.Path("case3"),
+ benchmark_tool_name="tool",
+ run_config=dummy_run_config)
+ suite = BenchmarkSuite([case1, case2, case3])
- def test_filter_benchmarks_for_category(self):
- case1 = BenchmarkCase(model_name="deepnet",
- model_tags=[],
- bench_mode=["1-thread", "full-inference"],
- target_arch="CPU-ARMv8",
- driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"],
- benchmark_case_dir=pathlib.Path("case1"),
- benchmark_tool_name="tool")
- case2 = BenchmarkCase(model_name="deepnetv2",
- model_tags=["f32"],
- bench_mode=["full-inference"],
- target_arch="GPU-Mali",
- driver_info=IREE_DRIVERS_INFOS["iree-vulkan"],
- benchmark_case_dir=pathlib.Path("case2"),
- benchmark_tool_name="tool")
- case3 = BenchmarkCase(model_name="deepnetv3",
- model_tags=["f32"],
- bench_mode=["full-inference"],
- target_arch="CPU-x86_64",
- driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"],
- benchmark_case_dir=pathlib.Path("case3"),
- benchmark_tool_name="tool")
- suite = BenchmarkSuite({
- pathlib.Path("suite/TFLite"): [case1, case2, case3],
- })
-
- cpu_and_gpu_benchmarks = suite.filter_benchmarks_for_category(
- category="TFLite",
+ cpu_and_gpu_benchmarks = suite.filter_benchmarks(
available_drivers=["local-task", "vulkan"],
available_loaders=["embedded-elf"],
- target_architectures=["cpu-armv8", "gpu-mali"],
+ target_architectures=[
+ common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC,
+ common_definitions.DeviceArchitecture.ARM_VALHALL,
+ ],
driver_filter=None,
mode_filter=".*full-inference.*",
model_name_filter="deepnet.*")
- gpu_benchmarks = suite.filter_benchmarks_for_category(
- category="TFLite",
+ gpu_benchmarks = suite.filter_benchmarks(
available_drivers=["local-task", "vulkan"],
available_loaders=["embedded-elf"],
- target_architectures=["gpu-mali"],
+ target_architectures=[
+ common_definitions.DeviceArchitecture.ARM_VALHALL,
+ ],
driver_filter="vulkan",
mode_filter=".*full-inference.*",
model_name_filter="deepnet.*")
- all_benchmarks = suite.filter_benchmarks_for_category(
- category="TFLite",
- available_drivers=None,
- target_architectures=None,
- driver_filter=None,
- mode_filter=None,
- model_name_filter=None)
+ all_benchmarks = suite.filter_benchmarks(available_drivers=None,
+ target_architectures=None,
+ driver_filter=None,
+ mode_filter=None,
+ model_name_filter=None)
self.assertEqual(cpu_and_gpu_benchmarks, [case1, case2])
self.assertEqual(gpu_benchmarks, [case2])
self.assertEqual(all_benchmarks, [case1, case2, case3])
- def test_filter_benchmarks_for_nonexistent_category(self):
- suite = BenchmarkSuite({
- pathlib.Path("suite/TFLite"): [],
- })
-
- benchmarks = suite.filter_benchmarks_for_category(
- category="PyTorch",
- available_drivers=[],
- available_loaders=[],
- target_architectures=["ARMv8", "Mali-G78"])
-
- self.assertEqual(benchmarks, [])
-
- def test_load_from_benchmark_suite_dir(self):
- with tempfile.TemporaryDirectory() as tmp_dir:
- tmp_dir = pathlib.Path(tmp_dir)
- tflite_dir = tmp_dir / "TFLite"
- pytorch_dir = tmp_dir / "PyTorch"
- BenchmarkSuiteTest.__create_bench(tflite_dir,
- model_name="DeepNet",
- model_tags=["f32"],
- bench_mode=["4-thread", "full"],
- target_arch="CPU-ARMv8",
- config="iree-llvm-cpu",
- tool="run-cpu-bench")
- case2 = BenchmarkSuiteTest.__create_bench(pytorch_dir,
- model_name="DeepNetv2",
- model_tags=[],
- bench_mode=["full-inference"],
- target_arch="GPU-Mali",
- config="iree-vulkan",
- tool="run-gpu-bench")
-
- suite = BenchmarkSuite.load_from_benchmark_suite_dir(tmp_dir)
-
- self.assertEqual(suite.list_categories(), [("PyTorch", pytorch_dir),
- ("TFLite", tflite_dir)])
- self.assertEqual(
- suite.filter_benchmarks_for_category(
- category="PyTorch",
- available_drivers=["vulkan"],
- available_loaders=[],
- target_architectures=["cpu-armv8", "gpu-mali"]), [case2])
-
def test_load_from_run_configs(self):
model_tflite = common_definitions.Model(
id="tflite",
@@ -204,79 +187,34 @@
suite = BenchmarkSuite.load_from_run_configs(run_configs=run_configs,
root_benchmark_dir=root_dir)
- self.assertEqual(
- suite.list_categories(),
- [("exported_stablehlo_mlir", pathlib.Path("exported_stablehlo_mlir")),
- ("exported_tflite", pathlib.Path("exported_tflite"))])
- run_config_a_case_dir = pathlib.Path(
- iree_artifacts.get_module_dir_path(
- run_config_a.module_generation_config, root_dir))
- run_config_b_case_dir = pathlib.Path(
- iree_artifacts.get_module_dir_path(
- run_config_b.module_generation_config, root_dir))
+
+ loaded_run_configs = [case.run_config for case in suite.filter_benchmarks()]
+ self.assertEqual(loaded_run_configs, [
+ run_config_a,
+ run_config_b,
+ run_config_c,
+ ])
run_config_c_case_dir = pathlib.Path(
iree_artifacts.get_module_dir_path(
run_config_c.module_generation_config, root_dir))
self.assertEqual(
- suite.filter_benchmarks_for_category(category="exported_tflite"), [
- BenchmarkCase(model_name=model_tflite.name,
- model_tags=model_tflite.tags,
- bench_mode=exec_config_a.tags,
- target_arch="riscv_32-generic",
- driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"],
- benchmark_tool_name="iree-benchmark-module",
- benchmark_case_dir=run_config_a_case_dir,
- run_config=run_config_a),
- BenchmarkCase(model_name=model_tflite.name,
- model_tags=model_tflite.tags,
- bench_mode=exec_config_b.tags,
- target_arch="riscv_64-generic",
- driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"],
- benchmark_tool_name="iree-benchmark-module",
- benchmark_case_dir=run_config_b_case_dir,
- run_config=run_config_b)
- ])
- self.assertEqual(
- suite.filter_benchmarks_for_category(
- category="exported_stablehlo_mlir",
- target_architectures=["riscv_32-generic"],
+ suite.filter_benchmarks(
+ target_architectures=[
+ common_definitions.DeviceArchitecture.RV32_GENERIC
+ ],
model_name_filter="model_tf.*fp32",
- mode_filter="defaults"),
- [
- BenchmarkCase(model_name=model_tf.name,
- model_tags=model_tf.tags,
- bench_mode=exec_config_a.tags,
- target_arch="riscv_32-generic",
- driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"],
- benchmark_tool_name="iree-benchmark-module",
- benchmark_case_dir=run_config_c_case_dir,
- run_config=run_config_c)
+ mode_filter="defaults",
+ ), [
+ BenchmarkCase(
+ model_name=model_tf.name,
+ model_tags=model_tf.tags,
+ bench_mode=exec_config_a.tags,
+ target_arch=common_definitions.DeviceArchitecture.RV32_GENERIC,
+ driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"],
+ benchmark_tool_name="iree-benchmark-module",
+ benchmark_case_dir=run_config_c_case_dir,
+ run_config=run_config_c)
])
- self.assertEqual(
- suite.filter_benchmarks_for_category(
- category="exported_stablehlo_mlir",
- target_architectures=["cpu-riscv_32-generic"],
- mode_filter="experimental"), [])
-
- @staticmethod
- def __create_bench(dir_path: pathlib.Path, model_name: str,
- model_tags: Sequence[str], bench_mode: Sequence[str],
- target_arch: str, config: str, tool: str):
- case_name = f"{config}__{target_arch}__{','.join(bench_mode)}"
- model_name_with_tags = model_name
- if len(model_tags) > 0:
- model_name_with_tags += f"-{','.join(model_tags)}"
- bench_path = dir_path / model_name_with_tags / case_name
- bench_path.mkdir(parents=True)
- (bench_path / "tool").write_text(tool)
-
- return BenchmarkCase(model_name=model_name,
- model_tags=model_tags,
- bench_mode=bench_mode,
- target_arch=target_arch,
- driver_info=IREE_DRIVERS_INFOS[config],
- benchmark_case_dir=bench_path,
- benchmark_tool_name=tool)
if __name__ == "__main__":
diff --git a/build_tools/benchmarks/common/common_arguments.py b/build_tools/benchmarks/common/common_arguments.py
index 73a58a2..258265f 100644
--- a/build_tools/benchmarks/common/common_arguments.py
+++ b/build_tools/benchmarks/common/common_arguments.py
@@ -42,24 +42,11 @@
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- artifacts_dir_group = self.add_mutually_exclusive_group(required=True)
- # TODO(#11076): Replace build-dir argument with e2e-test-artifacts-dir.
- artifacts_dir_group.add_argument(
- "build_dir",
- metavar="<build-dir>",
- type=_check_dir_path,
- default=None,
- nargs="?",
- help="Path to the build directory containing benchmark suites")
- artifacts_dir_group.add_argument(
- "--e2e_test_artifacts_dir",
- metavar="<e2e-test-artifacts-dir>",
- type=_check_dir_path,
- default=None,
- help=(
- "Path to the IREE e2e test artifacts directory. This will override "
- "<build-dir> and eventually replace it. For now must use with "
- "--execution_benchmark_config"))
+ self.add_argument("--e2e_test_artifacts_dir",
+ metavar="<e2e-test-artifacts-dir>",
+ type=_check_dir_path,
+ required=True,
+ help="Path to the IREE e2e test artifacts directory.")
self.add_argument(
"--normal_benchmark_tool_dir",
@@ -160,34 +147,13 @@
"information")
self.add_argument("--execution_benchmark_config",
type=_check_file_path,
- default=None,
+ required=True,
help="JSON config for the execution benchmarks")
self.add_argument("--target_device_name",
type=str,
- default=None,
+ required=True,
help="Target device in benchmark config to run")
- def parse_args(
- self, arg_strs: Optional[Sequence[str]] = None) -> argparse.Namespace:
- args = super().parse_args(arg_strs)
-
- # TODO(#11076): Remove these checks and make --execution_benchmark_config
- # and --target_device_name required args.
- use_new_benchmark_suite = (args.execution_benchmark_config is not None or
- args.target_device_name is not None)
- if use_new_benchmark_suite:
- if (args.execution_benchmark_config is None or
- args.target_device_name is None):
- self.error(
- "--execution_benchmark_config and --target_device_name must be set together."
- )
- elif args.e2e_test_artifacts_dir is not None:
- self.error(
- "--e2e_test_artifacts_dir requires --execution_benchmark_config and --target_device_name."
- )
-
- return args
-
def expand_and_check_file_paths(paths: Sequence[str]) -> List[pathlib.Path]:
"""Expands the wildcards in the paths and check if they are files.
diff --git a/build_tools/benchmarks/common/common_arguments_test.py b/build_tools/benchmarks/common/common_arguments_test.py
index 1d9a3a6..1469261 100644
--- a/build_tools/benchmarks/common/common_arguments_test.py
+++ b/build_tools/benchmarks/common/common_arguments_test.py
@@ -5,55 +5,72 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-import unittest
-import shutil
+import pathlib
import tempfile
+import unittest
import common.common_arguments
class CommonArgumentsTest(unittest.TestCase):
- def test_parser(self):
- with tempfile.TemporaryDirectory() as tempdir:
- common.common_arguments.Parser().parse_args([
- "--normal_benchmark_tool_dir=" + tempdir,
- "--traced_benchmark_tool_dir=" + tempdir,
- "--trace_capture_tool=" + shutil.which("ls"), "."
- ])
+ def setUp(self):
+ self._build_dir_manager = tempfile.TemporaryDirectory()
+ self.build_dir = pathlib.Path(self._build_dir_manager.name).resolve()
+ self.e2e_test_artifacts_dir = self.build_dir / "e2e_test_artifacts"
+ self.e2e_test_artifacts_dir.mkdir()
+ self.normal_tool_dir = self.build_dir / "normal_tool"
+ self.normal_tool_dir.mkdir()
+ self.traced_tool_dir = self.build_dir / "traced_tool"
+ self.traced_tool_dir.mkdir()
+ self.trace_capture_tool = self.build_dir / "tracy_capture"
+ # Create capture tool with executable file mode.
+ self.trace_capture_tool.touch(mode=0o755)
+ self.execution_config = self.build_dir / "execution_config.json"
+ self.execution_config.touch()
- def test_parser_check_build_dir(self):
- arg_parser = common.common_arguments.Parser()
- with self.assertRaises(SystemExit):
- arg_parser.parse_args(["nonexistent"])
+ def tearDown(self):
+ self._build_dir_manager.cleanup()
+
+ def test_parser(self):
+ common.common_arguments.Parser().parse_args([
+ f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
+ f"--traced_benchmark_tool_dir={self.traced_tool_dir}",
+ f"--trace_capture_tool={self.trace_capture_tool}",
+ f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
+ f"--execution_benchmark_config={self.execution_config}",
+ "--target_device=test",
+ ])
def test_parser_check_normal_benchmark_tool(self):
arg_parser = common.common_arguments.Parser()
with self.assertRaises(SystemExit):
- arg_parser.parse_args(["--normal_benchmark_tool_dir=nonexistent", "."])
+ arg_parser.parse_args([
+ "--normal_benchmark_tool_dir=nonexistent",
+ f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
+ f"--execution_benchmark_config={self.execution_config}",
+ "--target_device=test",
+ ])
def test_parser_check_traced_benchmark_tool(self):
arg_parser = common.common_arguments.Parser()
with self.assertRaises(SystemExit):
- arg_parser.parse_args(["--traced_benchmark_tool_dir=nonexistent", "."])
+ arg_parser.parse_args([
+ "--traced_benchmark_tool_dir=nonexistent",
+ f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
+ f"--execution_benchmark_config={self.execution_config}",
+ "--target_device=test",
+ ])
def test_parser_check_trace_capture_tool(self):
arg_parser = common.common_arguments.Parser()
with self.assertRaises(SystemExit):
- arg_parser.parse_args(["--trace_capture_tool=nonexistent", "."])
-
- def test_parser_e2e_test_artifacts_dir_needs_execution_benchmark_config(self):
- arg_parser = common.common_arguments.Parser()
- with tempfile.TemporaryDirectory() as tempdir:
- with self.assertRaises(SystemExit):
- arg_parser.parse_args([f"--e2e_test_artifacts_dir={tempdir}"])
-
- def test_parser_only_execution_benchmark_config_or_target_device_name(self):
- arg_parser = common.common_arguments.Parser()
- with self.assertRaises(SystemExit):
- arg_parser.parse_args([f"--execution_benchmark_config"])
- with self.assertRaises(SystemExit):
- arg_parser.parse_args([f"--target_device_name"])
+ arg_parser.parse_args([
+ "--trace_capture_tool=nonexistent",
+ f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
+ f"--execution_benchmark_config={self.execution_config}",
+ "--target_device=test",
+ ])
if __name__ == "__main__":
diff --git a/build_tools/benchmarks/run_benchmarks_on_android.py b/build_tools/benchmarks/run_benchmarks_on_android.py
index 1d9fc04..5ffccd0 100755
--- a/build_tools/benchmarks/run_benchmarks_on_android.py
+++ b/build_tools/benchmarks/run_benchmarks_on_android.py
@@ -196,19 +196,11 @@
benchmark_case_dir.relative_to(self.config.root_benchmark_dir))
run_config = benchmark_case.run_config
- if run_config is None:
- # TODO(#11076): Remove legacy path.
- self.__push_vmfb_file(benchmark_case_dir)
- self.__check_and_push_file(benchmark_case_dir / MODEL_FLAGFILE_NAME,
- android_case_dir)
- taskset = self.__deduce_taskset(benchmark_case.bench_mode)
- run_args = [f"--flagfile={MODEL_FLAGFILE_NAME}"]
- else:
- self.__check_and_push_file(
- benchmark_case_dir / iree_artifacts.MODULE_FILENAME, android_case_dir)
- taskset = self.__deduce_taskset_from_run_config(run_config)
- run_args = run_config.materialize_run_flags()
- run_args.append(f"--module={iree_artifacts.MODULE_FILENAME}")
+ self.__check_and_push_file(
+ benchmark_case_dir / iree_artifacts.MODULE_FILENAME, android_case_dir)
+ taskset = self.__deduce_taskset_from_run_config(run_config)
+ run_args = run_config.materialize_run_flags()
+ run_args.append(f"--module={iree_artifacts.MODULE_FILENAME}")
if benchmark_results_filename is not None:
self.__run_benchmark(android_case_dir=android_case_dir,
@@ -286,7 +278,6 @@
stdout_redirect = None if self.verbose else subprocess.DEVNULL
execute_cmd(capture_cmd, verbose=self.verbose, stdout=stdout_redirect)
-
# TODO(#13187): These logics are inherited from the legacy benchmark suites,
# which only work for a few specific phones. We should define the topology
# in their device specs.
@@ -308,22 +299,6 @@
raise ValueError(f"Unsupported config to deduce taskset: '{run_config}'.")
- # TODO(#11076): Remove legacy path.
- def __deduce_taskset(self, bench_mode: Sequence[str]) -> str:
- """Deduces the CPU affinity taskset mask according to benchmark modes."""
- # TODO: we actually should check the number of cores the phone have.
- if "big-core" in bench_mode:
- return "80" if "1-thread" in bench_mode else "f0"
- if "little-core" in bench_mode:
- return "08" if "1-thread" in bench_mode else "0f"
- # Not specified: use the 7th core.
- return "80"
-
- def __push_vmfb_file(self, benchmark_case_dir: pathlib.Path):
- vmfb_path = get_vmfb_full_path_for_benchmark_case(benchmark_case_dir)
- vmfb_rel_dir = vmfb_path.parent.relative_to(self.config.root_benchmark_dir)
- self.__check_and_push_file(vmfb_path, pathlib.PurePosixPath(vmfb_rel_dir))
-
def __check_and_push_file(self, host_path: pathlib.Path,
relative_dir: pathlib.PurePosixPath):
"""Checks if the file has been pushed and pushes it if not."""
@@ -369,21 +344,16 @@
commit = get_git_commit_hash("HEAD")
benchmark_config = BenchmarkConfig.build_from_args(args, commit)
- if args.execution_benchmark_config is None:
- # TODO(#11076): Remove legacy path.
- benchmark_suite = BenchmarkSuite.load_from_benchmark_suite_dir(
- benchmark_config.root_benchmark_dir)
- else:
- benchmark_groups = json.loads(args.execution_benchmark_config.read_text())
- benchmark_group = benchmark_groups.get(args.target_device_name)
- if benchmark_group is None:
- raise ValueError("Target device not found in the benchmark config.")
- run_configs = serialization.unpack_and_deserialize(
- data=benchmark_group["run_configs"],
- root_type=List[iree_definitions.E2EModelRunConfig])
- benchmark_suite = BenchmarkSuite.load_from_run_configs(
- run_configs=run_configs,
- root_benchmark_dir=benchmark_config.root_benchmark_dir)
+ benchmark_groups = json.loads(args.execution_benchmark_config.read_text())
+ benchmark_group = benchmark_groups.get(args.target_device_name)
+ if benchmark_group is None:
+ raise ValueError("Target device not found in the benchmark config.")
+ run_configs = serialization.unpack_and_deserialize(
+ data=benchmark_group["run_configs"],
+ root_type=List[iree_definitions.E2EModelRunConfig])
+ benchmark_suite = BenchmarkSuite.load_from_run_configs(
+ run_configs=run_configs,
+ root_benchmark_dir=benchmark_config.root_benchmark_dir)
benchmark_driver = AndroidBenchmarkDriver(device_info=device_info,
benchmark_config=benchmark_config,
diff --git a/build_tools/benchmarks/run_benchmarks_on_linux.py b/build_tools/benchmarks/run_benchmarks_on_linux.py
index cd33256..9554ccc 100755
--- a/build_tools/benchmarks/run_benchmarks_on_linux.py
+++ b/build_tools/benchmarks/run_benchmarks_on_linux.py
@@ -21,8 +21,7 @@
import tarfile
from common.benchmark_driver import BenchmarkDriver
-from common.benchmark_suite import (MODEL_FLAGFILE_NAME, BenchmarkCase,
- BenchmarkSuite)
+from common.benchmark_suite import BenchmarkCase, BenchmarkSuite
from common.benchmark_config import BenchmarkConfig
from common.benchmark_definition import (execute_cmd,
execute_cmd_and_get_output,
@@ -57,35 +56,9 @@
self.__run_capture(benchmark_case=benchmark_case,
capture_filename=capture_filename)
- def __parse_flagfile(self, case_dir: pathlib.Path) -> List[str]:
- return [
- line.strip()
- for line in (case_dir / MODEL_FLAGFILE_NAME).read_text().splitlines()
- ]
-
def __build_tool_cmds(self, benchmark_case: BenchmarkCase,
tool_path: pathlib.Path) -> List[Any]:
run_config = benchmark_case.run_config
- if run_config is None:
- # TODO(#11076): Remove legacy path.
- if benchmark_case.benchmark_case_dir is None:
- raise ValueError(
- "benchmark_case_dir can't be None if run_config is None.")
-
- # TODO(pzread): Taskset should be derived from CPU topology.
- # Only use the low 8 cores.
- cmds = ["taskset", "0xFF", tool_path]
-
- run_flags = self.__parse_flagfile(benchmark_case.benchmark_case_dir)
- # Replace the CUDA device flag with the specified GPU.
- if benchmark_case.driver_info.driver_name == "cuda":
- run_flags = [
- flag for flag in run_flags if not flag.startswith("--device")
- ]
- run_flags.append(f"--device=cuda://{self.gpu_id}")
-
- return cmds + run_flags
-
cmds: List[Any] = run_module_utils.build_linux_wrapper_cmds_for_device_spec(
run_config.target_device_spec)
cmds.append(tool_path)
@@ -113,11 +86,8 @@
driver_info=benchmark_case.driver_info,
benchmark_min_time=self.config.benchmark_min_time))
- # TODO(#11076): Legacy mode need to switch CWD, remove it in the cleanup.
- cwd = (benchmark_case.benchmark_case_dir
- if benchmark_case.run_config is None else None)
benchmark_stdout, benchmark_stderr = execute_cmd_and_get_output(
- cmd, cwd=cwd, verbose=self.verbose)
+ cmd, verbose=self.verbose)
benchmark_metrics = parse_iree_benchmark_metrics(benchmark_stdout,
benchmark_stderr)
if self.verbose:
@@ -135,12 +105,8 @@
cmd = self.__build_tool_cmds(benchmark_case=benchmark_case,
tool_path=tool_path)
- # TODO(#11076): Legacy mode need to switch CWD, remove it in the cleanup.
- cwd = (benchmark_case.benchmark_case_dir
- if benchmark_case.run_config is None else None)
process = subprocess.Popen(cmd,
env={"TRACY_NO_EXIT": "1"},
- cwd=cwd,
stdout=subprocess.PIPE,
text=True)
@@ -162,21 +128,16 @@
commit = get_git_commit_hash("HEAD")
benchmark_config = BenchmarkConfig.build_from_args(args, commit)
- if args.execution_benchmark_config is None:
- # TODO(#11076): Remove legacy path.
- benchmark_suite = BenchmarkSuite.load_from_benchmark_suite_dir(
- benchmark_config.root_benchmark_dir)
- else:
- benchmark_groups = json.loads(args.execution_benchmark_config.read_text())
- benchmark_group = benchmark_groups.get(args.target_device_name)
- if benchmark_group is None:
- raise ValueError("Target device not found in the benchmark config.")
- run_configs = serialization.unpack_and_deserialize(
- data=benchmark_group["run_configs"],
- root_type=typing.List[iree_definitions.E2EModelRunConfig])
- benchmark_suite = BenchmarkSuite.load_from_run_configs(
- run_configs=run_configs,
- root_benchmark_dir=benchmark_config.root_benchmark_dir)
+ benchmark_groups = json.loads(args.execution_benchmark_config.read_text())
+ benchmark_group = benchmark_groups.get(args.target_device_name)
+ if benchmark_group is None:
+ raise ValueError("Target device not found in the benchmark config.")
+ run_configs = serialization.unpack_and_deserialize(
+ data=benchmark_group["run_configs"],
+ root_type=typing.List[iree_definitions.E2EModelRunConfig])
+ benchmark_suite = BenchmarkSuite.load_from_run_configs(
+ run_configs=run_configs,
+ root_benchmark_dir=benchmark_config.root_benchmark_dir)
benchmark_driver = LinuxBenchmarkDriver(gpu_id=args.gpu_id,
device_info=device_info,
diff --git a/build_tools/cmake/iree_benchmark_suite.cmake b/build_tools/cmake/iree_benchmark_suite.cmake
index 33cad0b..3d11e6a 100644
--- a/build_tools/cmake/iree_benchmark_suite.cmake
+++ b/build_tools/cmake/iree_benchmark_suite.cmake
@@ -55,7 +55,6 @@
COMMENT
"Importing ${_MODEL_BASENAME} into MLIR"
)
- add_dependencies(iree-benchmark-import-models "${_RULE_TARGET_NAME}")
endif()
endfunction()
@@ -110,381 +109,5 @@
COMMENT
"Importing ${_MODEL_BASENAME} into MLIR"
)
- add_dependencies(iree-benchmark-import-models "${_RULE_TARGET_NAME}")
endif()
endfunction()
-
-# iree_benchmark_suite()
-#
-# Generates benchmark suites for MLIR input modules. The generated artifacts
-# will be placed in the "<binary-root>/benchmark_suites/<category>" directory,
-# where "<category>" is the name of the immediate directory containing the
-# CMakeLists.txt. The generated artifacts are expected to be executed with
-# `iree-benchmark-module`.
-#
-# Parameters:
-# GROUP_NAME: A group name this benchmark will join. Each group has its own
-# CMake's benchmark suite target: "iree-benchmark-suites-<GROUP_NAME>".
-# MODULES: A list for model specification. Due to CMake's lack of data
-# structures, each module is represented as a list suitable to be parsed
-# by cmake_parse_arguments:
-# - NAME: The input module's name.
-# - TAGS: comma-separated tags for the input module.
-# - SOURCE: The input file for the input module. Supported formats are
-# MLIR files in the IREE input format (which should have a .mlir
-# extension) or TFLite FlatBuffers (with a .tflite extension). In
-# addition to permitting a source file, this can be a URL ("http://"
-# or "https://") from which to download the file. This URL should
-# point to a file in one of the appropriate input formats, optionally
-# compressed in the gzip format, in which case it should have a
-# trailing ".gz" extension in addition to other extensions.
-# - ENTRY_FUNCTION: The entry function name for the input module.
-# - FUNCTION_INPUT: A list of comma-separated entry function inputs for
-# the input module.
-# BENCHMARK_MODES: A list strings, where ech one of them is a comma-
-# separated list of benchmark mode tags.
-# TARGET_BACKEND: The compiler target backend.
-# TARGET_ARCHITECTURE: The detailed target backend's architecture.
-# COMPILATION_FLAGS: A list of command-line options and their values to
-# pass to the IREE compiler tool for artifact generation.
-# CONFIG: Benchmark runner configuration name.
-# DRIVER: The runtime driver.
-# RUNTIME_FLAGS: A list of command-line options and their values to pass
-# to the IREE runtime during benchmark exectuion.
-#
-# The above parameters largely fall into two categories: 1) for specifying
-# the MLIR input module and its metadata, 2) for specifying the compilation/
-# runtime configuration.
-#
-# 1)
-#
-# The MODULES provide information about the input module and its metadata. For
-# example, we can generate modules with idential names from different sources
-# (TensorFlow, TFLite, PyTorch, etc.), and we can transform the same input
-# module differently for benchmarking different aspects like fp32 vs fp16.
-#
-# 2)
-#
-# TARGET_BACKEND and COMPILATION_FLAGS control how the input module will be
-# converted into the final IREE deployable module format. DRIVER and
-# RUNTIME_FLAGS specify how the module will be executed. BENCHMARK_MODES
-# can be used to give descriptions of the compilation/runtime configuration
-# (e.g., full-inference vs. kernel-execution) and specify more contextual
-# requirements (e.g., big-core vs. little-core).
-#
-function(iree_benchmark_suite)
- if(NOT IREE_BUILD_LEGACY_BENCHMARKS)
- return()
- endif()
-
- cmake_parse_arguments(
- PARSE_ARGV 0
- _RULE
- ""
- "GROUP_NAME;CONFIG;DRIVER;TARGET_BACKEND;TARGET_ARCHITECTURE"
- "BENCHMARK_MODES;BENCHMARK_TOOL;MODULES;COMPILATION_FLAGS;RUNTIME_FLAGS"
- )
-
- iree_validate_required_arguments(
- _RULE
- "GROUP_NAME;CONFIG;DRIVER;TARGET_BACKEND;TARGET_ARCHITECTURE"
- "BENCHMARK_MODES;BENCHMARK_TOOL;MODULES"
- )
-
- # Try to check if the compiler supports the TARGET_BACKEND. If
- # IREE_HOST_BIN_DIR is set, we are using a compiler binary, in which
- # case we can't check its supported backends just by looking at this build
- # dir's cmake variables --- we would have to implement a configure-check
- # executing `iree-compile --iree-hal-list-target-backends`.
- if(NOT IREE_HOST_BIN_DIR)
- string(TOUPPER ${_RULE_TARGET_BACKEND} _UPPERCASE_TARGET_BACKEND)
- string(REPLACE "-" "_" _NORMALIZED_TARGET_BACKEND ${_UPPERCASE_TARGET_BACKEND})
- if(NOT IREE_TARGET_BACKEND_${_NORMALIZED_TARGET_BACKEND})
- return()
- endif()
- endif()
-
- iree_package_name(_PACKAGE_NAME)
-
- # Add the benchmark suite target.
- set(SUITE_SUB_TARGET "iree-benchmark-suites-${_RULE_GROUP_NAME}")
- if(NOT TARGET "${SUITE_SUB_TARGET}")
- add_custom_target("${SUITE_SUB_TARGET}")
- endif()
-
- foreach(_MODULE IN LISTS _RULE_MODULES)
- cmake_parse_arguments(
- _MODULE
- ""
- "NAME;TAGS;SOURCE;ENTRY_FUNCTION;FUNCTION_INPUTS"
- "IMPORT_FLAGS"
- ${_MODULE}
- )
- iree_validate_required_arguments(
- _MODULE
- "NAME;TAGS;SOURCE;ENTRY_FUNCTION;FUNCTION_INPUTS"
- ""
- )
-
- get_filename_component(_CATEGORY "${CMAKE_CURRENT_SOURCE_DIR}" NAME)
- set(_ROOT_ARTIFACTS_DIR "${IREE_BINARY_DIR}/benchmark_suites/${_CATEGORY}")
- set(_VMFB_ARTIFACTS_DIR "${_ROOT_ARTIFACTS_DIR}/vmfb")
- file(MAKE_DIRECTORY ${_VMFB_ARTIFACTS_DIR})
-
- # The name of any custom target that drives creation of the final source
- # MLIR file. Depending on the format of the source, this will get updated.
- set(_MODULE_SOURCE_TARGET "")
-
- # If the source file is from the web, create a custom command to download
- # it and wrap that with a custom target so later we can use for dependency.
- if("${_MODULE_SOURCE}" MATCHES "^https?://")
- set(_SOURCE_URL "${_MODULE_SOURCE}")
- # Update the source file to the downloaded-to place.
- string(REPLACE "/" ";" _SOURCE_URL_SEGMENTS "${_SOURCE_URL}")
- list(POP_BACK _SOURCE_URL_SEGMENTS _LAST_URL_SEGMENT)
- set(_DOWNLOAD_TARGET_NAME "iree-download-benchmark-source-${_LAST_URL_SEGMENT}")
-
- # Strip off gzip/tar suffix if present (downloader unpacks if necessary)
- string(REGEX REPLACE "(\.gz)|(\.tar\.gz)$" "" _SOURCE_FILE_BASENAME "${_LAST_URL_SEGMENT}")
- set(_MODULE_SOURCE "${_ROOT_ARTIFACTS_DIR}/${_SOURCE_FILE_BASENAME}")
- if(NOT TARGET "${_PACKAGE_NAME}_${_DOWNLOAD_TARGET_NAME}")
- iree_fetch_artifact(
- NAME
- "${_DOWNLOAD_TARGET_NAME}"
- SOURCE_URL
- "${_SOURCE_URL}"
- OUTPUT
- "${_MODULE_SOURCE}"
- UNPACK
- )
- endif()
- set(_MODULE_SOURCE_TARGET "${_PACKAGE_NAME}_${_DOWNLOAD_TARGET_NAME}")
- endif()
-
- # If the source is a TFLite file, import it.
- if("${_MODULE_SOURCE}" MATCHES "\.tflite$")
- cmake_path(GET _MODULE_SOURCE FILENAME _MODEL_BASENAME)
- set(_MODULE_SOURCE_TARGET "${_PACKAGE_NAME}_iree-import-tf-${_MODEL_BASENAME}")
- iree_import_tflite_model(
- TARGET_NAME "${_MODULE_SOURCE_TARGET}"
- SOURCE "${_MODULE_SOURCE}"
- IMPORT_FLAGS
- "--output-format=mlir-bytecode"
- ${_MODULE_IMPORT_FLAGS}
- OUTPUT_MLIR_FILE "${_MODULE_SOURCE}.mlir"
- )
- set(_MODULE_SOURCE "${_MODULE_SOURCE}.mlir")
- endif()
-
- # If the source is a TensorFlow SavedModel directory, import it.
- if("${_MODULE_SOURCE}" MATCHES "-tf-model$")
- cmake_path(GET _MODULE_SOURCE FILENAME _MODEL_BASENAME)
- set(_MODULE_SOURCE_TARGET "${_PACKAGE_NAME}_iree-import-tf-${_MODEL_BASENAME}")
- iree_import_tf_model(
- TARGET_NAME "${_MODULE_SOURCE_TARGET}"
- SOURCE "${_MODULE_SOURCE}"
- IMPORT_FLAGS
- "--output-format=mlir-bytecode"
- ${_MODULE_IMPORT_FLAGS}
- OUTPUT_MLIR_FILE "${_MODULE_SOURCE}.mlir"
- )
- set(_MODULE_SOURCE "${_MODULE_SOURCE}.mlir")
- endif()
-
- # Next create the command and target for compiling the input module into
- # IREE deployable format for each benchmark mode.
- string(JOIN "-" _MODULE_DIR_NAME "${_MODULE_NAME}" "${_MODULE_TAGS}")
- foreach(_BENCHMARK_MODE IN LISTS _RULE_BENCHMARK_MODES)
- set(_BENCHMARK_DIR_NAME
- "${_RULE_CONFIG}__${_RULE_TARGET_ARCHITECTURE}__${_BENCHMARK_MODE}")
-
- # A list of name segments for composing unique CMake target names.
- set(_COMMON_NAME_SEGMENTS "${_MODULE_NAME}")
- string(REPLACE "," "-" _TAGS "${_MODULE_TAGS}")
- string(REPLACE "," "-" _MODE "${_BENCHMARK_MODE}")
- list(APPEND _COMMON_NAME_SEGMENTS
- "${_TAGS}" "${_MODE}" "${_RULE_TARGET_BACKEND}"
- "${_RULE_TARGET_ARCHITECTURE}")
-
- # Add a friendly target name to drive this benchmark and any others that
- # share the same easily-describable properties.
- set(_FRIENDLY_TARGET_NAME_LIST "iree-generate-benchmark-artifact")
- list(APPEND _FRIENDLY_TARGET_NAME_LIST ${_COMMON_NAME_SEGMENTS})
- list(JOIN _FRIENDLY_TARGET_NAME_LIST "__" _FRIENDLY_TARGET_NAME)
-
- # The full list of compilation flags.
- set(_COMPILATION_ARGS "")
- list(APPEND _COMPILATION_ARGS "--mlir-print-op-on-diagnostic=false")
- list(APPEND _COMPILATION_ARGS "--iree-hal-target-backends=${_RULE_TARGET_BACKEND}")
- list(SORT _RULE_COMPILATION_FLAGS)
- list(APPEND _COMPILATION_ARGS ${_RULE_COMPILATION_FLAGS})
-
- # Get a unique identifier for this IREE module file by hashing the command
- # line flags and input file. We will also use this for the CMake target.
- # Note that this is NOT A SECURE HASHING ALGORITHM. We just want
- # uniqueness and MD5 is fast. If that changes, switch to something much
- # better (like SHA256).
- string(MD5 _VMFB_HASH "${_COMPILATION_ARGS};${_MODULE_SOURCE}")
- get_filename_component(_MODULE_SOURCE_BASENAME "${_MODULE_SOURCE}" NAME)
- set(_MODULE_SOURCE_BASENAME_WITH_HASH "${_MODULE_SOURCE_BASENAME}-${_VMFB_HASH}")
- set(_VMFB_FILE "${_VMFB_ARTIFACTS_DIR}/${_MODULE_SOURCE_BASENAME_WITH_HASH}.vmfb")
-
- # Register the target once and share across all benchmarks having the same
- # MLIR source and compilation flags.
- set(_COMPILATION_NAME
- "iree-generate-benchmark-artifact-${_MODULE_SOURCE_BASENAME_WITH_HASH}"
- )
- set(_COMPILATION_TARGET_NAME "${_PACKAGE_NAME}_${_COMPILATION_NAME}")
- if(NOT TARGET "${_COMPILATION_TARGET_NAME}")
- iree_bytecode_module(
- NAME
- "${_COMPILATION_NAME}"
- MODULE_FILE_NAME
- "${_VMFB_FILE}"
- SRC
- "${_MODULE_SOURCE}"
- FLAGS
- ${_COMPILATION_ARGS}
- DEPENDS
- "${_MODULE_SOURCE_TARGET}"
- FRIENDLY_NAME
- "${_FRIENDLY_TARGET_NAME}"
- )
-
- # Mark dependency so that we have one target to drive them all.
- add_dependencies(iree-benchmark-suites "${_COMPILATION_TARGET_NAME}")
- add_dependencies("${SUITE_SUB_TARGET}" "${_COMPILATION_TARGET_NAME}")
- endif()
-
- set(_COMPILE_STATS_COMPILATION_NAME
- "${_COMPILATION_NAME}-compile-stats"
- )
- set(_COMPILE_STATS_COMPILATION_TARGET_NAME
- "${_PACKAGE_NAME}_${_COMPILE_STATS_COMPILATION_NAME}"
- )
- set(_COMPILE_STATS_VMFB_FILE
- "${_VMFB_ARTIFACTS_DIR}/${_MODULE_SOURCE_BASENAME_WITH_HASH}-compile-stats.vmfb"
- )
- if(IREE_ENABLE_LEGACY_COMPILATION_BENCHMARKS AND NOT TARGET "${_COMPILE_STATS_COMPILATION_TARGET_NAME}")
- iree_bytecode_module(
- NAME
- "${_COMPILE_STATS_COMPILATION_NAME}"
- MODULE_FILE_NAME
- "${_COMPILE_STATS_VMFB_FILE}"
- SRC
- "${_MODULE_SOURCE}"
- FLAGS
- # Enable zip polyglot to provide component sizes.
- "--iree-vm-emit-polyglot-zip=true"
- # Disable debug symbols to provide correct component sizes.
- "--iree-llvmcpu-debug-symbols=false"
- ${_COMPILATION_ARGS}
- DEPENDS
- "${_MODULE_SOURCE_TARGET}"
- FRIENDLY_NAME
- "${_FRIENDLY_TARGET_NAME}"
- )
-
- # Mark dependency so that we have one target to drive them all.
- add_dependencies(iree-benchmark-suites
- "${_COMPILE_STATS_COMPILATION_TARGET_NAME}"
- )
- add_dependencies("${SUITE_SUB_TARGET}"
- "${_COMPILE_STATS_COMPILATION_TARGET_NAME}"
- )
- endif()
-
- if(NOT TARGET "${_FRIENDLY_TARGET_NAME}")
- add_custom_target("${_FRIENDLY_TARGET_NAME}")
- endif()
- add_dependencies("${_FRIENDLY_TARGET_NAME}" "${_COMPILATION_TARGET_NAME}")
- if(IREE_ENABLE_LEGACY_COMPILATION_BENCHMARKS)
- add_dependencies("${_FRIENDLY_TARGET_NAME}"
- "${_COMPILE_STATS_COMPILATION_TARGET_NAME}")
- endif()
-
- set(_RUN_SPEC_DIR "${_ROOT_ARTIFACTS_DIR}/${_MODULE_DIR_NAME}/${_BENCHMARK_DIR_NAME}")
- list(JOIN _COMMON_NAME_SEGMENTS "__" _RUN_SPEC_TARGET_SUFFIX)
-
- # Create the command and target for the flagfile spec used to execute
- # the generated artifacts.
- set(_FLAG_FILE "${_RUN_SPEC_DIR}/flagfile")
- set(_ADDITIONAL_ARGS "${_RULE_RUNTIME_FLAGS}")
- list(APPEND _ADDITIONAL_ARGS "--device_allocator=caching")
- set(_ADDITIONAL_ARGS_CL "--additional_args=\"${_ADDITIONAL_ARGS}\"")
- file(RELATIVE_PATH _MODULE_FILE_FLAG "${_RUN_SPEC_DIR}" "${_VMFB_FILE}")
- add_custom_command(
- OUTPUT "${_FLAG_FILE}"
- COMMAND
- "${Python3_EXECUTABLE}" "${IREE_ROOT_DIR}/build_tools/scripts/generate_flagfile.py"
- --module="${_MODULE_FILE_FLAG}"
- --device=${_RULE_DRIVER}
- --function=${_MODULE_ENTRY_FUNCTION}
- --inputs=${_MODULE_FUNCTION_INPUTS}
- "${_ADDITIONAL_ARGS_CL}"
- -o "${_FLAG_FILE}"
- DEPENDS
- "${IREE_ROOT_DIR}/build_tools/scripts/generate_flagfile.py"
- WORKING_DIRECTORY "${_RUN_SPEC_DIR}"
- COMMENT "Generating ${_FLAG_FILE}"
- )
-
- set(_FLAGFILE_GEN_TARGET_NAME
- "${_PACKAGE_NAME}_iree-generate-benchmark-flagfile__${_RUN_SPEC_TARGET_SUFFIX}")
- add_custom_target("${_FLAGFILE_GEN_TARGET_NAME}"
- DEPENDS "${_FLAG_FILE}"
- )
-
- # Create the command and target for the toolfile spec used to execute
- # the generated artifacts.
- set(_TOOL_FILE "${_RUN_SPEC_DIR}/tool")
- add_custom_command(
- OUTPUT "${_TOOL_FILE}"
- COMMAND ${CMAKE_COMMAND} -E echo ${_RULE_BENCHMARK_TOOL} > "${_TOOL_FILE}"
- WORKING_DIRECTORY "${_RUN_SPEC_DIR}"
- COMMENT "Generating ${_TOOL_FILE}"
- )
-
- set(_TOOLFILE_GEN_TARGET_NAME
- "${_PACKAGE_NAME}_iree-generate-benchmark-toolfile__${_RUN_SPEC_TARGET_SUFFIX}")
- add_custom_target("${_TOOLFILE_GEN_TARGET_NAME}"
- DEPENDS "${_TOOL_FILE}"
- )
-
- # Generate a flagfile containing command-line options used to compile the
- # generated artifacts.
- set(_COMPILATION_FLAGFILE "${_RUN_SPEC_DIR}/compilation_flagfile")
- # Generate the flagfile with python command. We can't use "file" because
- # it can't be part of a target's dependency and generated lazily. And
- # "cmake -E echo" doesn't work with newlines.
- add_custom_command(
- OUTPUT "${_COMPILATION_FLAGFILE}"
- COMMAND
- "${Python3_EXECUTABLE}" "${IREE_ROOT_DIR}/build_tools/scripts/generate_compilation_flagfile.py"
- --output "${_COMPILATION_FLAGFILE}"
- -- ${_COMPILATION_ARGS}
- WORKING_DIRECTORY "${_RUN_SPEC_DIR}"
- COMMENT "Generating ${_COMPILATION_FLAGFILE}"
- )
-
- set(_COMPILATION_FLAGFILE_GEN_TARGET_NAME
- "${_PACKAGE_NAME}_iree-generate-benchmark-compilation-flagfile__${_RUN_SPEC_TARGET_SUFFIX}")
- add_custom_target("${_COMPILATION_FLAGFILE_GEN_TARGET_NAME}"
- DEPENDS "${_COMPILATION_FLAGFILE}"
- )
-
- # Mark dependency so that we have one target to drive them all.
- add_dependencies(iree-benchmark-suites
- "${_COMPILATION_FLAGFILE_GEN_TARGET_NAME}"
- "${_FLAGFILE_GEN_TARGET_NAME}"
- "${_TOOLFILE_GEN_TARGET_NAME}"
- )
- add_dependencies("${SUITE_SUB_TARGET}"
- "${_COMPILATION_FLAGFILE_GEN_TARGET_NAME}"
- "${_FLAGFILE_GEN_TARGET_NAME}"
- "${_TOOLFILE_GEN_TARGET_NAME}"
- )
- endforeach(_BENCHMARK_MODE IN LISTS _RULE_BENCHMARK_MODES)
-
- endforeach(_MODULE IN LISTS _RULE_MODULES)
-endfunction(iree_benchmark_suite)
diff --git a/docs/developers/developing_iree/benchmark_suites.md b/docs/developers/developing_iree/benchmark_suites.md
index 567211c..aefb788 100644
--- a/docs/developers/developing_iree/benchmark_suites.md
+++ b/docs/developers/developing_iree/benchmark_suites.md
@@ -119,7 +119,6 @@
```sh
build_tools/benchmarks/collect_compilation_statistics.py \
- alpha \
--compilation_benchmark_config=comp_config.json \
--e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR?}" \
--build_log="${IREE_BUILD_DIR?}/.ninja_log" \
