| # Copyright 2022 The IREE Authors |
| # |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. |
| # See https://llvm.org/LICENSE.txt for license information. |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| |
| import dataclasses |
| import json |
| import pathlib |
| import tempfile |
| from typing import Optional |
| import unittest |
| |
| from common import benchmark_config |
| from common.benchmark_suite import BenchmarkCase, BenchmarkSuite |
| from common.benchmark_driver import BenchmarkDriver |
| from common.benchmark_definition import ( |
| IREE_DRIVERS_INFOS, |
| DeviceInfo, |
| PlatformType, |
| BenchmarkLatency, |
| BenchmarkMemory, |
| BenchmarkMetrics, |
| ) |
| from e2e_test_framework.definitions import common_definitions, iree_definitions |
| |
| |
| class FakeBenchmarkDriver(BenchmarkDriver): |
| def __init__( |
| self, *args, raise_exception_on_case: Optional[BenchmarkCase] = None, **kwargs |
| ): |
| super().__init__(*args, **kwargs) |
| self.raise_exception_on_case = raise_exception_on_case |
| self.run_benchmark_cases = [] |
| |
| def run_benchmark_case( |
| self, |
| benchmark_case: BenchmarkCase, |
| benchmark_results_filename: Optional[pathlib.Path], |
| capture_filename: Optional[pathlib.Path], |
| ) -> None: |
| if self.raise_exception_on_case == benchmark_case: |
| raise Exception("fake exception") |
| |
| self.run_benchmark_cases.append(benchmark_case) |
| |
| if benchmark_results_filename: |
| fake_benchmark_metrics = BenchmarkMetrics( |
| real_time=BenchmarkLatency(0, 0, 0, "ns"), |
| cpu_time=BenchmarkLatency(0, 0, 0, "ns"), |
| host_memory=BenchmarkMemory(0, 0, 0, 0, "bytes"), |
| device_memory=BenchmarkMemory(0, 0, 0, 0, "bytes"), |
| raw_data={}, |
| ) |
| benchmark_results_filename.write_text( |
| json.dumps(fake_benchmark_metrics.to_json_object()) |
| ) |
| if capture_filename: |
| capture_filename.write_text("{}") |
| |
| |
| class BenchmarkDriverTest(unittest.TestCase): |
| def setUp(self): |
| self._tmp_dir_obj = tempfile.TemporaryDirectory() |
| self._root_dir_obj = tempfile.TemporaryDirectory() |
| |
| self.tmp_dir = pathlib.Path(self._tmp_dir_obj.name) |
| (self.tmp_dir / "build_config.txt").write_text( |
| "IREE_HAL_DRIVER_LOCAL_SYNC=ON\n" |
| "IREE_HAL_DRIVER_LOCAL_TASK=ON\n" |
| "IREE_HAL_EXECUTABLE_LOADER_EMBEDDED_ELF=ON\n" |
| ) |
| |
| self.benchmark_results_dir = ( |
| self.tmp_dir / benchmark_config.BENCHMARK_RESULTS_REL_PATH |
| ) |
| self.captures_dir = self.tmp_dir / benchmark_config.CAPTURES_REL_PATH |
| self.benchmark_results_dir.mkdir() |
| self.captures_dir.mkdir() |
| |
| self.config = benchmark_config.BenchmarkConfig( |
| root_benchmark_dir=pathlib.Path(self._root_dir_obj.name), |
| benchmark_results_dir=self.benchmark_results_dir, |
| git_commit_hash="abcd", |
| normal_benchmark_tool_dir=self.tmp_dir, |
| trace_capture_config=benchmark_config.TraceCaptureConfig( |
| traced_benchmark_tool_dir=self.tmp_dir, |
| trace_capture_tool=self.tmp_dir / "capture_tool", |
| capture_tarball=self.tmp_dir / "captures.tar", |
| capture_tmp_dir=self.captures_dir, |
| ), |
| use_compatible_filter=True, |
| ) |
| |
| self.device_info = DeviceInfo( |
| platform_type=PlatformType.LINUX, |
| model="Unknown", |
| cpu_abi="x86_64", |
| cpu_uarch="CascadeLake", |
| cpu_features=[], |
| gpu_name="unknown", |
| ) |
| |
| model_tflite = common_definitions.Model( |
| id="tflite", |
| name="model_tflite", |
| tags=[], |
| source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, |
| source_url="", |
| entry_function="predict", |
| input_types=["1xf32"], |
| ) |
| device_spec = common_definitions.DeviceSpec.build( |
| id="dev", |
| device_name="test_dev", |
| architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, |
| host_environment=common_definitions.HostEnvironment.LINUX_X86_64, |
| device_parameters=[], |
| tags=[], |
| ) |
| compile_target = iree_definitions.CompileTarget( |
| target_backend=iree_definitions.TargetBackend.LLVM_CPU, |
| target_architecture=( |
| common_definitions.DeviceArchitecture.X86_64_CASCADELAKE |
| ), |
| target_abi=iree_definitions.TargetABI.LINUX_GNU, |
| ) |
| gen_config = iree_definitions.ModuleGenerationConfig.build( |
| imported_model=iree_definitions.ImportedModel.from_model(model_tflite), |
| compile_config=iree_definitions.CompileConfig.build( |
| id="comp_a", tags=[], compile_targets=[compile_target] |
| ), |
| ) |
| exec_config_a = iree_definitions.ModuleExecutionConfig.build( |
| id="exec_a", |
| tags=["sync"], |
| loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, |
| driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, |
| ) |
| run_config_a = iree_definitions.E2EModelRunConfig.build( |
| module_generation_config=gen_config, |
| module_execution_config=exec_config_a, |
| target_device_spec=device_spec, |
| input_data=common_definitions.ZEROS_MODEL_INPUT_DATA, |
| tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, |
| ) |
| exec_config_b = iree_definitions.ModuleExecutionConfig.build( |
| id="exec_b", |
| tags=["task"], |
| loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, |
| driver=iree_definitions.RuntimeDriver.LOCAL_TASK, |
| ) |
| run_config_b = iree_definitions.E2EModelRunConfig.build( |
| module_generation_config=gen_config, |
| module_execution_config=exec_config_b, |
| target_device_spec=device_spec, |
| input_data=common_definitions.ZEROS_MODEL_INPUT_DATA, |
| tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, |
| ) |
| self.case1 = BenchmarkCase( |
| model_name="model_tflite", |
| model_tags=[], |
| bench_mode=["sync"], |
| target_arch=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, |
| driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"], |
| benchmark_case_dir=pathlib.Path("case1"), |
| benchmark_tool_name="tool", |
| run_config=run_config_a, |
| ) |
| self.case2 = BenchmarkCase( |
| model_name="model_tflite", |
| model_tags=[], |
| bench_mode=["task"], |
| target_arch=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, |
| driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"], |
| benchmark_case_dir=pathlib.Path("case2"), |
| benchmark_tool_name="tool", |
| run_config=run_config_b, |
| ) |
| |
| compile_target_rv64 = iree_definitions.CompileTarget( |
| target_backend=iree_definitions.TargetBackend.LLVM_CPU, |
| target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, |
| target_abi=iree_definitions.TargetABI.LINUX_GNU, |
| ) |
| gen_config_rv64 = iree_definitions.ModuleGenerationConfig.build( |
| imported_model=iree_definitions.ImportedModel.from_model(model_tflite), |
| compile_config=iree_definitions.CompileConfig.build( |
| id="comp_rv64", tags=[], compile_targets=[compile_target_rv64] |
| ), |
| ) |
| device_spec_rv64 = common_definitions.DeviceSpec.build( |
| id="rv64_dev", |
| device_name="rv64_dev", |
| architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, |
| host_environment=common_definitions.HostEnvironment.LINUX_X86_64, |
| device_parameters=[], |
| tags=[], |
| ) |
| run_config_incompatible = iree_definitions.E2EModelRunConfig.build( |
| module_generation_config=gen_config_rv64, |
| module_execution_config=exec_config_b, |
| target_device_spec=device_spec_rv64, |
| input_data=common_definitions.ZEROS_MODEL_INPUT_DATA, |
| tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, |
| ) |
| self.incompatible_case = BenchmarkCase( |
| model_name="model_tflite", |
| model_tags=[], |
| bench_mode=["task"], |
| target_arch=common_definitions.DeviceArchitecture.RV64_GENERIC, |
| driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"], |
| benchmark_case_dir=pathlib.Path("incompatible_case"), |
| benchmark_tool_name="tool", |
| run_config=run_config_incompatible, |
| ) |
| self.benchmark_suite = BenchmarkSuite( |
| [ |
| self.case1, |
| self.case2, |
| self.incompatible_case, |
| ] |
| ) |
| |
| def tearDown(self) -> None: |
| self._tmp_dir_obj.cleanup() |
| self._root_dir_obj.cleanup() |
| |
| def test_run(self): |
| driver = FakeBenchmarkDriver( |
| self.device_info, self.config, self.benchmark_suite |
| ) |
| |
| driver.run() |
| |
| self.assertEqual(driver.get_benchmark_results().commit, "abcd") |
| self.assertEqual(len(driver.get_benchmark_results().benchmarks), 2) |
| self.assertEqual( |
| driver.get_benchmark_results().benchmarks[0].metrics.raw_data, {} |
| ) |
| self.assertEqual( |
| driver.get_benchmark_result_filenames(), |
| [ |
| self.benchmark_results_dir / f"{self.case1.run_config}.json", |
| self.benchmark_results_dir / f"{self.case2.run_config}.json", |
| ], |
| ) |
| self.assertEqual( |
| driver.get_capture_filenames(), |
| [ |
| self.captures_dir / f"{self.case1.run_config}.tracy", |
| self.captures_dir / f"{self.case2.run_config}.tracy", |
| ], |
| ) |
| self.assertEqual(driver.get_benchmark_errors(), []) |
| |
| def test_run_disable_compatible_filter(self): |
| self.config.use_compatible_filter = False |
| driver = FakeBenchmarkDriver( |
| self.device_info, self.config, self.benchmark_suite |
| ) |
| |
| driver.run() |
| |
| self.assertEqual(len(driver.get_benchmark_results().benchmarks), 3) |
| |
| def test_run_with_no_capture(self): |
| self.config.trace_capture_config = None |
| driver = FakeBenchmarkDriver( |
| self.device_info, self.config, self.benchmark_suite |
| ) |
| |
| driver.run() |
| |
| self.assertEqual(len(driver.get_benchmark_result_filenames()), 2) |
| self.assertEqual(driver.get_capture_filenames(), []) |
| |
| def test_run_with_exception_and_keep_going(self): |
| self.config.keep_going = True |
| driver = FakeBenchmarkDriver( |
| self.device_info, |
| self.config, |
| self.benchmark_suite, |
| raise_exception_on_case=self.case1, |
| ) |
| |
| driver.run() |
| |
| self.assertEqual(len(driver.get_benchmark_errors()), 1) |
| self.assertEqual(len(driver.get_benchmark_result_filenames()), 1) |
| |
| def test_run_with_previous_benchmarks_and_captures(self): |
| benchmark_filename = ( |
| self.benchmark_results_dir / f"{self.case1.run_config}.json" |
| ) |
| benchmark_filename.touch() |
| capture_filename = self.captures_dir / f"{self.case1.run_config}.tracy" |
| capture_filename.touch() |
| config = dataclasses.replace(self.config, continue_from_previous=True) |
| driver = FakeBenchmarkDriver( |
| device_info=self.device_info, |
| benchmark_config=config, |
| benchmark_suite=self.benchmark_suite, |
| ) |
| |
| driver.run() |
| |
| self.assertEqual(len(driver.run_benchmark_cases), 1) |
| self.assertEqual(len(driver.get_benchmark_result_filenames()), 2) |
| self.assertEqual(len(driver.get_capture_filenames()), 2) |
| |
| |
| if __name__ == "__main__": |
| unittest.main() |
