Dump run and compile flags into benchmark JSON config (#12397)
diff --git a/build_tools/benchmarks/collect_compilation_statistics.py b/build_tools/benchmarks/collect_compilation_statistics.py index 6abacd1..bc91dd2 100755 --- a/build_tools/benchmarks/collect_compilation_statistics.py +++ b/build_tools/benchmarks/collect_compilation_statistics.py
@@ -155,7 +155,7 @@ model_source=model.source_type.value, target_arch=f"[{','.join(target_archs)}]", compile_tags=tuple(compile_config.tags), - gen_config_id=gen_config.composite_id()) + gen_config_id=gen_config.composite_id) module_dir_path = iree_artifacts.get_module_dir_path( module_generation_config=gen_config, root_path=e2e_test_artifacts_dir) module_path = module_dir_path / iree_artifacts.MODULE_FILENAME
diff --git a/build_tools/benchmarks/collect_compilation_statistics_test.py b/build_tools/benchmarks/collect_compilation_statistics_test.py index d645ed6..e03e4f2 100644 --- a/build_tools/benchmarks/collect_compilation_statistics_test.py +++ b/build_tools/benchmarks/collect_compilation_statistics_test.py
@@ -120,16 +120,11 @@ target_architecture=common_definitions.DeviceArchitecture. RV64_GENERIC, target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU), - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture. - RV32_GENERIC, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, target_abi=iree_definitions.TargetABI.LINUX_GNU) ]) - gen_config_a = iree_definitions.ModuleGenerationConfig( + gen_config_a = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_a, compile_config=compile_config_a) - gen_config_b = iree_definitions.ModuleGenerationConfig( + gen_config_b = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_a, compile_config=compile_config_b) serialized_gen_config = json.dumps( serialization.serialize_and_pack([gen_config_a, gen_config_b])) @@ -145,17 +140,16 @@ model_source=model_a.source_type.value, target_arch=f"[cpu-x86_64-cascadelake-linux-gnu]", compile_tags=tuple(gen_config_a.compile_config.tags), - gen_config_id=gen_config_a.composite_id()) + gen_config_id=gen_config_a.composite_id) module_a_path = iree_artifacts.get_module_dir_path( gen_config_a, root_dir) / iree_artifacts.MODULE_FILENAME compile_info_b = common.benchmark_definition.CompilationInfo( model_name=model_a.name, model_tags=tuple(model_a.tags), model_source=model_a.source_type.value, - target_arch= - f"[cpu-riscv_64-generic-linux-gnu,cpu-riscv_32-generic-linux-gnu]", + target_arch=f"[cpu-riscv_64-generic-linux-gnu]", compile_tags=tuple(gen_config_a.compile_config.tags), - gen_config_id=gen_config_b.composite_id()) + gen_config_id=gen_config_b.composite_id) module_b_path = iree_artifacts.get_module_dir_path( gen_config_b, root_dir) / iree_artifacts.MODULE_FILENAME self.assertEqual(module_map, {
diff --git a/build_tools/benchmarks/common/benchmark_driver.py b/build_tools/benchmarks/common/benchmark_driver.py index 35a85a4..646fd69 100644 --- a/build_tools/benchmarks/common/benchmark_driver.py +++ b/build_tools/benchmarks/common/benchmark_driver.py
@@ -219,7 +219,7 @@ compile_tags=compile_tags, driver_info=benchmark_case.driver_info, device_info=self.device_info, - run_config_id=benchmark_case.run_config.composite_id()) + run_config_id=benchmark_case.run_config.composite_id) def __get_available_drivers_and_loaders( self) -> Tuple[Sequence[str], Sequence[str]]:
diff --git a/build_tools/benchmarks/common/benchmark_suite_test.py b/build_tools/benchmarks/common/benchmark_suite_test.py index d4e2290..71c2136 100644 --- a/build_tools/benchmarks/common/benchmark_suite_test.py +++ b/build_tools/benchmarks/common/benchmark_suite_test.py
@@ -169,32 +169,36 @@ architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, host_environment=common_definitions.HostEnvironment.LINUX_X86_64, device_parameters=[]) - run_config_a = iree_definitions.E2EModelRunConfig( - module_generation_config=iree_definitions.ModuleGenerationConfig( + compile_target = iree_definitions.CompileTarget( + target_backend=iree_definitions.TargetBackend.LLVM_CPU, + target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, + target_abi=iree_definitions.TargetABI.LINUX_GNU) + run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation( + module_generation_config=iree_definitions.ModuleGenerationConfig. + with_flag_generation( imported_model=iree_definitions.ImportedModel.from_model( model_tflite), - compile_config=iree_definitions.CompileConfig(id="1", - tags=[], - compile_targets=[])), + compile_config=iree_definitions.CompileConfig( + id="1", tags=[], compile_targets=[compile_target])), module_execution_config=exec_config_a, target_device_spec=device_spec_a, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA) - run_config_b = iree_definitions.E2EModelRunConfig( - module_generation_config=iree_definitions.ModuleGenerationConfig( + run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation( + module_generation_config=iree_definitions.ModuleGenerationConfig. + with_flag_generation( imported_model=iree_definitions.ImportedModel.from_model( model_tflite), - compile_config=iree_definitions.CompileConfig(id="2", - tags=[], - compile_targets=[])), + compile_config=iree_definitions.CompileConfig( + id="2", tags=[], compile_targets=[compile_target])), module_execution_config=exec_config_b, target_device_spec=device_spec_b, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA) - run_config_c = iree_definitions.E2EModelRunConfig( - module_generation_config=iree_definitions.ModuleGenerationConfig( + run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation( + module_generation_config=iree_definitions.ModuleGenerationConfig. + with_flag_generation( imported_model=iree_definitions.ImportedModel.from_model(model_tf), - compile_config=iree_definitions.CompileConfig(id="3", - tags=[], - compile_targets=[])), + compile_config=iree_definitions.CompileConfig( + id="3", tags=[], compile_targets=[compile_target])), module_execution_config=exec_config_a, target_device_spec=device_spec_a, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA)
diff --git a/build_tools/benchmarks/export_benchmark_config_test.py b/build_tools/benchmarks/export_benchmark_config_test.py index a463668..81578c4 100644 --- a/build_tools/benchmarks/export_benchmark_config_test.py +++ b/build_tools/benchmarks/export_benchmark_config_test.py
@@ -18,11 +18,18 @@ source_url="", entry_function="predict", input_types=["1xf32"]) -COMMON_GEN_CONFIG = iree_definitions.ModuleGenerationConfig( +COMMON_GEN_CONFIG = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=iree_definitions.ImportedModel.from_model(COMMON_MODEL), - compile_config=iree_definitions.CompileConfig(id="1", - tags=[], - compile_targets=[])) + compile_config=iree_definitions.CompileConfig( + id="1", + tags=[], + compile_targets=[ + iree_definitions.CompileTarget( + target_backend=iree_definitions.TargetBackend.LLVM_CPU, + target_architecture=common_definitions.DeviceArchitecture. + RV64_GENERIC, + target_abi=iree_definitions.TargetABI.LINUX_GNU) + ])) COMMON_EXEC_CONFIG = iree_definitions.ModuleExecutionConfig( id="exec", tags=[], @@ -48,17 +55,17 @@ device_name="dev_c", architecture=common_definitions.DeviceArchitecture.CUDA_SM80, host_environment=common_definitions.HostEnvironment.LINUX_X86_64) - matched_run_config_a = iree_definitions.E2EModelRunConfig( + matched_run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=COMMON_GEN_CONFIG, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_a, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA) - unmatched_run_config_b = iree_definitions.E2EModelRunConfig( + unmatched_run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=COMMON_GEN_CONFIG, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_b, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA) - matched_run_config_c = iree_definitions.E2EModelRunConfig( + matched_run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=COMMON_GEN_CONFIG, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_c, @@ -96,17 +103,17 @@ device_name="dev_a_gpu", architecture=common_definitions.DeviceArchitecture.ADRENO_GENERIC, host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A) - run_config_a = iree_definitions.E2EModelRunConfig( + run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=COMMON_GEN_CONFIG, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_a, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA) - run_config_b = iree_definitions.E2EModelRunConfig( + run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=COMMON_GEN_CONFIG, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_b, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA) - run_config_c = iree_definitions.E2EModelRunConfig( + run_config_c = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=COMMON_GEN_CONFIG, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_c, @@ -133,12 +140,12 @@ device_name="dev_b", architecture=common_definitions.DeviceArchitecture.VALHALL_MALI, host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A) - run_config_a = iree_definitions.E2EModelRunConfig( + run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=COMMON_GEN_CONFIG, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_a, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA) - run_config_b = iree_definitions.E2EModelRunConfig( + run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=COMMON_GEN_CONFIG, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_b, @@ -170,13 +177,16 @@ source_url="", entry_function="predict", input_types=["1xf32"]) - compile_config = iree_definitions.CompileConfig(id="1", - tags=[], - compile_targets=[]) - small_gen_config = iree_definitions.ModuleGenerationConfig( + compile_target = iree_definitions.CompileTarget( + target_backend=iree_definitions.TargetBackend.LLVM_CPU, + target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, + target_abi=iree_definitions.TargetABI.LINUX_GNU) + compile_config = iree_definitions.CompileConfig( + id="1", tags=[], compile_targets=[compile_target]) + small_gen_config = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=iree_definitions.ImportedModel.from_model(small_model), compile_config=compile_config) - big_gen_config = iree_definitions.ModuleGenerationConfig( + big_gen_config = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=iree_definitions.ImportedModel.from_model(big_model), compile_config=compile_config) device_spec_a = common_definitions.DeviceSpec( @@ -189,12 +199,12 @@ device_name="dev_b", architecture=common_definitions.DeviceArchitecture.VALHALL_MALI, host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A) - run_config_a = iree_definitions.E2EModelRunConfig( + run_config_a = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=small_gen_config, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_a, input_data=common_definitions.ZEROS_MODEL_INPUT_DATA) - run_config_b = iree_definitions.E2EModelRunConfig( + run_config_b = iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=big_gen_config, module_execution_config=COMMON_EXEC_CONFIG, target_device_spec=device_spec_b,
diff --git a/build_tools/benchmarks/run_benchmarks_on_linux.py b/build_tools/benchmarks/run_benchmarks_on_linux.py index 92fe8b4..e8a022c 100755 --- a/build_tools/benchmarks/run_benchmarks_on_linux.py +++ b/build_tools/benchmarks/run_benchmarks_on_linux.py
@@ -88,12 +88,7 @@ run_config.module_generation_config, root_path=self.config.root_benchmark_dir) cmds += [f"--module={module_dir_path / iree_artifacts.MODULE_FILENAME}"] - cmds += run_module_utils.build_run_flags_for_model( - model=run_config.module_generation_config.imported_model.model, - model_input_data=run_config.input_data) - cmds += run_module_utils.build_run_flags_for_execution_config( - module_execution_config=run_config.module_execution_config, - gpu_id=self.gpu_id) + cmds += run_config.materialize_run_flags(gpu_id=self.gpu_id) return cmds
diff --git a/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py b/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py index 988e121..70cadba 100644 --- a/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py
@@ -51,19 +51,19 @@ tflite_models.MOBILENET_V3SMALL, ] default_gen_configs = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.DEFAULT_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in default_models ] fuse_padding_gen_configs = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.FUSE_PADDING_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in default_models ] fuse_padding_repeated_kernel_gen_configs = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.FUSE_PADDING_REPEATED_KERNEL_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in [
diff --git a/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py b/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py index 9e6ea7a..53eed51 100644 --- a/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py
@@ -71,18 +71,18 @@ ] default_gen_confings = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.DEFAULT_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in self.NONQUANT_MODELS + self.QUANT_MODELS ] experimental_gen_confings = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.MMT4D_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in self.NONQUANT_MODELS ] + [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.MMT4D_AND_DOTPROD_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in self.QUANT_MODELS
diff --git a/build_tools/python/benchmark_suites/iree/benchmark_collections.py b/build_tools/python/benchmark_suites/iree/benchmark_collections.py index 1c595e8..2ce0aa7 100644 --- a/build_tools/python/benchmark_suites/iree/benchmark_collections.py +++ b/build_tools/python/benchmark_suites/iree/benchmark_collections.py
@@ -54,7 +54,7 @@ "--iree-llvm-debug-symbols=false" ]) compile_stats_gen_configs.append( - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=gen_config.imported_model, compile_config=compile_stats_config)) all_gen_configs += compile_stats_gen_configs
diff --git a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py index 75094da..3cc67a6 100644 --- a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py
@@ -33,7 +33,7 @@ """Generates IREE compile and run configs.""" gen_configs = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.SM_80_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in model_groups.LARGE
diff --git a/build_tools/python/benchmark_suites/iree/mali_benchmarks.py b/build_tools/python/benchmark_suites/iree/mali_benchmarks.py index 600426c..1e1c710 100644 --- a/build_tools/python/benchmark_suites/iree/mali_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/mali_benchmarks.py
@@ -114,17 +114,17 @@ extra_flags=compile_config.extra_flags + ["--iree-flow-demote-f32-to-f16"]) return [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=compile_config, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in fp32_models ] + [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=demote_compile_config, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in fp16_models ] + [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=demote_compile_config, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in quant_models
diff --git a/build_tools/python/benchmark_suites/iree/riscv_benchmarks.py b/build_tools/python/benchmark_suites/iree/riscv_benchmarks.py index b8372de..b482b5a 100644 --- a/build_tools/python/benchmark_suites/iree/riscv_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/riscv_benchmarks.py
@@ -37,7 +37,7 @@ List[iree_definitions.E2EModelRunConfig]]: """Generates IREE compile and run configs.""" gen_configs = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.DEFAULT_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in self.MODELS @@ -68,7 +68,7 @@ List[iree_definitions.E2EModelRunConfig]]: """Generates IREE compile and run configs.""" gen_configs = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.DEFAULT_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in self.MODELS
diff --git a/build_tools/python/benchmark_suites/iree/utils.py b/build_tools/python/benchmark_suites/iree/utils.py index 6df1039..c92c2e6 100644 --- a/build_tools/python/benchmark_suites/iree/utils.py +++ b/build_tools/python/benchmark_suites/iree/utils.py
@@ -19,7 +19,7 @@ """Generates the run specs from the product of compile specs and run configs. """ return [ - iree_definitions.E2EModelRunConfig( + iree_definitions.E2EModelRunConfig.with_flag_generation( module_generation_config=module_generation_config, module_execution_config=module_execution_config, target_device_spec=device_spec,
diff --git a/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py b/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py index b4bc6f9..b94caaa 100644 --- a/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py
@@ -33,7 +33,7 @@ """Generates IREE compile and run configs.""" gen_configs = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.DEFAULT_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in [tflite_models.MOBILENET_V2, tflite_models.MOBILENET_V3SMALL]
diff --git a/build_tools/python/benchmark_suites/iree/x86_64_benchmarks.py b/build_tools/python/benchmark_suites/iree/x86_64_benchmarks.py index eb93873..8f85bdc 100644 --- a/build_tools/python/benchmark_suites/iree/x86_64_benchmarks.py +++ b/build_tools/python/benchmark_suites/iree/x86_64_benchmarks.py
@@ -43,7 +43,7 @@ """Generates IREE compile and run configs.""" gen_configs = [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.CASCADELAKE_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in model_groups.SMALL + model_groups.LARGE @@ -51,7 +51,7 @@ # TODO(#11174): Excludes ResNet50 excluded_models_for_experiments = [tf_models.RESNET50_TF_FP32] gen_configs += [ - iree_definitions.ModuleGenerationConfig( + iree_definitions.ModuleGenerationConfig.with_flag_generation( compile_config=self.CASCADELAKE_FUSE_PADDING_COMPILE_CONFIG, imported_model=iree_definitions.ImportedModel.from_model(model)) for model in model_groups.SMALL + model_groups.LARGE
diff --git a/build_tools/python/e2e_model_tests/cmake_generator.py b/build_tools/python/e2e_model_tests/cmake_generator.py index e4dc571..e890f44 100644 --- a/build_tools/python/e2e_model_tests/cmake_generator.py +++ b/build_tools/python/e2e_model_tests/cmake_generator.py
@@ -7,7 +7,7 @@ from typing import List -from e2e_model_tests import test_definitions, run_module_utils +from e2e_model_tests import test_definitions from e2e_test_artifacts import iree_artifacts from e2e_test_framework.definitions import iree_definitions import cmake_builder.rules @@ -24,7 +24,7 @@ for gen_config in module_generation_configs: module_path = iree_artifacts.get_module_dir_path( gen_config) / iree_artifacts.MODULE_FILENAME - all_module_path_map[(gen_config.imported_model.composite_id(), + all_module_path_map[(gen_config.imported_model.composite_id, gen_config.compile_config.id)] = module_path cmake_rules = [] @@ -37,20 +37,20 @@ compile_config = test_definitions.PLATFORM_COMPILE_CONFIG_MAP[platform] module_path = all_module_path_map.get( - (imported_model.composite_id(), compile_config.id)) + (imported_model.composite_id, compile_config.id)) if module_path is None: raise ValueError( f"Module for {test_config.name} on {platform} not found.") platform_module_map[platform.value] = module_path - runner_args = run_module_utils.build_run_flags_for_model( - model=imported_model.model, - model_input_data=test_config.input_data) + test_config.extra_test_flags # TODO(#11136): Currently the DRIVER is a separate field in the CMake rule ( # and has effect on test labels). Rules should be generated in another way # to avoid that. Generates the flags without the driver for now. - runner_args += run_module_utils.build_run_flags_for_execution_config( - test_config.execution_config, with_driver=False) + runner_args = iree_definitions.generate_run_flags( + imported_model=imported_model, + input_data=test_config.input_data, + module_execution_config=test_config.execution_config, + with_driver=False) + test_config.extra_test_flags cmake_rule = cmake_builder.rules.build_iree_benchmark_suite_module_test( target_name=test_config.name, driver=test_config.execution_config.driver.value,
diff --git a/build_tools/python/e2e_model_tests/run_module_utils.py b/build_tools/python/e2e_model_tests/run_module_utils.py index bd88c08..724a972 100644 --- a/build_tools/python/e2e_model_tests/run_module_utils.py +++ b/build_tools/python/e2e_model_tests/run_module_utils.py
@@ -5,50 +5,12 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception """Utils that help launch iree run module tools.""" -from typing import Any, List +from typing import List from e2e_test_framework.definitions import common_definitions -from e2e_test_framework.definitions.iree_definitions import ModuleExecutionConfig, RuntimeDriver from e2e_test_framework.device_specs import device_parameters -def build_run_flags_for_model( - model: common_definitions.Model, - model_input_data: common_definitions.ModelInputData) -> List[str]: - """Returns the IREE run module flags for the model and its inputs.""" - - run_flags = [f"--function={model.entry_function}"] - if model_input_data != common_definitions.ZEROS_MODEL_INPUT_DATA: - raise ValueError("Currently only support all-zeros data.") - run_flags += [f"--input={input_type}=0" for input_type in model.input_types] - return run_flags - - -def build_run_flags_for_execution_config( - module_execution_config: ModuleExecutionConfig, - gpu_id: str = "0", - with_driver: bool = True) -> List[str]: - """Returns the IREE run module flags of the execution config. - - Args: - module_execution_config: execution config. - gpu_id: target gpu id, if runs on GPUs. - with_driver: populate the driver flags if true. False can be used for - generating flags for some CMake rules with a separate DRIVER arg. - Returns: - List of flags. - """ - - run_flags = module_execution_config.extra_flags.copy() - if with_driver: - driver = module_execution_config.driver - if driver == RuntimeDriver.CUDA: - run_flags.append(f"--device=cuda://{gpu_id}") - else: - run_flags.append(f"--device={driver.value}") - return run_flags - - def build_linux_wrapper_cmds_for_device_spec( device_spec: common_definitions.DeviceSpec) -> List[str]: """Builds the commands with tools to create the execution environment."""
diff --git a/build_tools/python/e2e_model_tests/run_module_utils_test.py b/build_tools/python/e2e_model_tests/run_module_utils_test.py index 5a255da..b4fa578 100644 --- a/build_tools/python/e2e_model_tests/run_module_utils_test.py +++ b/build_tools/python/e2e_model_tests/run_module_utils_test.py
@@ -7,66 +7,11 @@ import unittest from e2e_model_tests import run_module_utils -from e2e_test_framework.definitions import common_definitions, iree_definitions +from e2e_test_framework.definitions import common_definitions from e2e_test_framework.device_specs import device_parameters -class RunModuleTuilsTest(unittest.TestCase): - - def test_build_run_flags_for_model(self): - model = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32", "2x2xf32"]) - - flags = run_module_utils.build_run_flags_for_model( - model, common_definitions.ZEROS_MODEL_INPUT_DATA) - - self.assertEqual( - flags, ["--function=main", "--input=1xf32=0", "--input=2x2xf32=0"]) - - def test_build_run_flags_for_execution_config(self): - execution_config = iree_definitions.ModuleExecutionConfig( - id="123", - tags=["test"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - extra_flags=["--task=10"]) - - flags = run_module_utils.build_run_flags_for_execution_config( - execution_config) - - self.assertEqual(flags, ["--task=10", "--device=local-task"]) - - def test_build_run_flags_for_execution_config_with_cuda(self): - execution_config = iree_definitions.ModuleExecutionConfig( - id="123", - tags=["test"], - loader=iree_definitions.RuntimeLoader.NONE, - driver=iree_definitions.RuntimeDriver.CUDA, - extra_flags=[]) - - flags = run_module_utils.build_run_flags_for_execution_config( - execution_config, gpu_id="3") - - self.assertEqual(flags, ["--device=cuda://3"]) - - def test_build_run_flags_for_execution_config_without_driver(self): - execution_config = iree_definitions.ModuleExecutionConfig( - id="123", - tags=["test"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - extra_flags=["--task=10"]) - - flags = run_module_utils.build_run_flags_for_execution_config( - execution_config, with_driver=False) - - self.assertEqual(flags, ["--task=10"]) +class RunModuleUtilsTest(unittest.TestCase): def test_build_linux_wrapper_cmds_for_device_spec(self): device_spec = common_definitions.DeviceSpec(
diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py index 04a4efb..cd00cf5 100644 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py +++ b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py
@@ -59,7 +59,7 @@ cmake_rules=[]) # Import target name: iree-imported-model-<imported_model_id> - target_name = f"iree-imported-model-{imported_model.composite_id()}" + target_name = f"iree-imported-model-{imported_model.composite_id}" import_flags = import_config.materialize_import_flags(model) if import_config.tool == iree_definitions.ImportTool.TFLITE_IMPORTER: @@ -92,15 +92,10 @@ module_generation_config: iree_definitions.ModuleGenerationConfig, output_file_path: pathlib.PurePath) -> IreeModuleCompileRule: - imported_model = module_generation_config.imported_model - compile_config = module_generation_config.compile_config - compile_flags = self._generate_compile_flags( - compile_config=compile_config, - mlir_dialect_type=imported_model.import_config.dialect_type.value - ) + compile_config.extra_flags + compile_flags = module_generation_config.materialize_compile_flags() # Module target name: iree-module-<gen_config_id> - target_name = f"iree-module-{module_generation_config.composite_id()}" + target_name = f"iree-module-{module_generation_config.composite_id}" cmake_rules = [ cmake_builder.rules.build_iree_bytecode_module( @@ -122,66 +117,6 @@ """ return f"{self._package_name}_{target_name}" - def _generate_compile_flags(self, - compile_config: iree_definitions.CompileConfig, - mlir_dialect_type: str) -> List[str]: - if len(compile_config.compile_targets) != 1: - raise ValueError(f"Only one compile target is supported. Got:" - f" {compile_config.compile_targets}") - - compile_target = compile_config.compile_targets[0] - flags = [ - f"--iree-hal-target-backends={compile_target.target_backend.value}", - f"--iree-input-type={mlir_dialect_type}" - ] - flags.extend(self._generate_compile_target_flags(compile_target)) - return flags - - def _generate_compile_target_flags( - self, target: iree_definitions.CompileTarget) -> List[str]: - arch_info = target.target_architecture - if target.target_backend == iree_definitions.TargetBackend.VULKAN_SPIRV: - return [ - f"--iree-vulkan-target-triple={arch_info.architecture}-unknown-{target.target_abi.value}", - ] - - if arch_info.architecture == "x86_64": - flags = [ - f"--iree-llvm-target-triple=x86_64-unknown-{target.target_abi.value}", - f"--iree-llvm-target-cpu={arch_info.microarchitecture.lower()}" - ] - elif arch_info.architecture == "riscv_64": - flags = [ - f"--iree-llvm-target-triple=riscv64-pc-{target.target_abi.value}", - "--iree-llvm-target-cpu=generic-rv64", "--iree-llvm-target-abi=lp64d", - "--iree-llvm-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v", - "--riscv-v-fixed-length-vector-lmul-max=8" - ] - elif arch_info.architecture == "riscv_32": - # TODO(llvm-project/60463): Replace 'zve32f' with 'zve32x'. - flags = [ - f"--iree-llvm-target-triple=riscv32-pc-{target.target_abi.value}", - "--iree-llvm-target-cpu=generic-rv32", "--iree-llvm-target-abi=ilp32", - "--iree-llvm-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f", - "--riscv-v-fixed-length-vector-lmul-max=8" - ] - elif arch_info.architecture == "armv8.2-a": - flags = [ - f"--iree-llvm-target-triple=aarch64-none-{target.target_abi.value}", - ] - elif arch_info.architecture == "cuda": - if target.target_abi != iree_definitions.TargetABI.LINUX_GNU: - raise ValueError( - f"Unsupported target ABI for CUDA backend: `{target.target_abi}`") - flags = [ - f"--iree-hal-cuda-llvm-target-arch=sm_80", - ] - elif arch_info.architecture == "vmvx": - flags = [] - else: - raise ValueError(f"Unsupported architecture: '{arch_info.architecture}'") - return flags - def generate_rules( package_name: str, root_path: pathlib.PurePath, @@ -203,7 +138,7 @@ rule_builder = IreeRuleBuilder(package_name=package_name) all_imported_models = dict( - (config.imported_model.composite_id(), config.imported_model) + (config.imported_model.composite_id, config.imported_model) for config in module_generation_configs) cmake_rules = [] @@ -226,7 +161,7 @@ compile_stats_module_target_names = [] for gen_config in module_generation_configs: model_import_rule = model_import_rule_map[ - gen_config.imported_model.composite_id()] + gen_config.imported_model.composite_id] module_dir_path = iree_artifacts.get_module_dir_path( module_generation_config=gen_config, root_path=root_path) module_compile_rule = rule_builder.build_module_compile_rule(
diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator_test.py b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator_test.py index 941a9dd..82a831d 100644 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator_test.py +++ b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator_test.py
@@ -42,7 +42,7 @@ self.assertEqual( rule.target_name, - f"iree-imported-model-{tflite_imported_model.composite_id()}") + f"iree-imported-model-{tflite_imported_model.composite_id}") self.assertEqual(rule.output_file_path, output_file_path) def test_build_model_import_rule_linalg(self): @@ -90,7 +90,7 @@ target_backend=iree_definitions.TargetBackend.LLVM_CPU, target_abi=iree_definitions.TargetABI.LINUX_GNU) ]) - gen_config = iree_definitions.ModuleGenerationConfig( + gen_config = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model, compile_config=compile_config) model_import_rule = iree_rule_generator.IreeModelImportRule( target_name=f"iree-import-model-abcd", @@ -103,8 +103,7 @@ module_generation_config=gen_config, output_file_path=output_file_path) - self.assertEqual(rule.target_name, - f"iree-module-{gen_config.composite_id()}") + self.assertEqual(rule.target_name, f"iree-module-{gen_config.composite_id}") self.assertEqual(rule.output_module_path, output_file_path) def test_build_target_path(self): @@ -165,13 +164,13 @@ target_backend=iree_definitions.TargetBackend.LLVM_CPU, target_abi=iree_definitions.TargetABI.LINUX_GNU) ]) - gen_config_a = iree_definitions.ModuleGenerationConfig( + gen_config_a = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_a, compile_config=compile_config_a) - gen_config_b = iree_definitions.ModuleGenerationConfig( + gen_config_b = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_b, compile_config=compile_config_a) - gen_config_c = iree_definitions.ModuleGenerationConfig( + gen_config_c = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_b, compile_config=compile_config_b) - gen_config_d = iree_definitions.ModuleGenerationConfig( + gen_config_d = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_c, compile_config=compile_config_b) model_rule_map = { model_a.id: @@ -201,19 +200,19 @@ concated_cmake_rules = "\n".join(cmake_rules) self.assertRegex(concated_cmake_rules, - f"iree-imported-model-{imported_model_a.composite_id()}") + f"iree-imported-model-{imported_model_a.composite_id}") self.assertRegex(concated_cmake_rules, - f"iree-imported-model-{imported_model_b.composite_id()}") + f"iree-imported-model-{imported_model_b.composite_id}") self.assertRegex(concated_cmake_rules, - f"iree-imported-model-{imported_model_c.composite_id()}") + f"iree-imported-model-{imported_model_c.composite_id}") self.assertRegex(concated_cmake_rules, - f"iree-module-{gen_config_a.composite_id()}") + f"iree-module-{gen_config_a.composite_id}") self.assertRegex(concated_cmake_rules, - f"iree-module-{gen_config_b.composite_id()}") + f"iree-module-{gen_config_b.composite_id}") self.assertRegex(concated_cmake_rules, - f"iree-module-{gen_config_c.composite_id()}") + f"iree-module-{gen_config_c.composite_id}") self.assertRegex(concated_cmake_rules, - f"iree-module-{gen_config_d.composite_id()}") + f"iree-module-{gen_config_d.composite_id}") if __name__ == "__main__":
diff --git a/build_tools/python/e2e_test_artifacts/iree_artifacts.py b/build_tools/python/e2e_test_artifacts/iree_artifacts.py index ca63e62..aaf4a0c 100644 --- a/build_tools/python/e2e_test_artifacts/iree_artifacts.py +++ b/build_tools/python/e2e_test_artifacts/iree_artifacts.py
@@ -45,7 +45,7 @@ model_prefix = _get_model_prefix(imported_model) # Imported model path: <root_path>/<model_prefix>_<imported_model_id>.mlir - return (root_path / f"{model_prefix}_{imported_model.composite_id()}.mlir") + return (root_path / f"{model_prefix}_{imported_model.composite_id}.mlir") def get_module_dir_path( @@ -65,7 +65,7 @@ model_prefix = _get_model_prefix(module_generation_config.imported_model) # Module dir path: <root_path>/<model_prefix>_module_<gen_config_id> return (root_path / - f"{model_prefix}_module_{module_generation_config.composite_id()}") + f"{model_prefix}_module_{module_generation_config.composite_id}") def get_dependent_model_map(
diff --git a/build_tools/python/e2e_test_artifacts/iree_artifacts_test.py b/build_tools/python/e2e_test_artifacts/iree_artifacts_test.py index f83f15a..c440bf3 100644 --- a/build_tools/python/e2e_test_artifacts/iree_artifacts_test.py +++ b/build_tools/python/e2e_test_artifacts/iree_artifacts_test.py
@@ -30,7 +30,7 @@ self.assertEqual( path, root_path / f"{iree_artifacts.IREE_ARTIFACT_PREFIX}_{model.name}_" - f"{imported_model.composite_id()}.mlir") + f"{imported_model.composite_id}.mlir") def test_get_imported_model_path_with_mlir_model(self): model = common_definitions.Model( @@ -70,7 +70,7 @@ target_backend=iree_definitions.TargetBackend.LLVM_CPU, target_abi=iree_definitions.TargetABI.LINUX_GNU) ]) - gen_config = iree_definitions.ModuleGenerationConfig( + gen_config = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model, compile_config=compile_config) root_path = pathlib.PurePath("root") @@ -79,7 +79,7 @@ self.assertEqual( path, root_path / f"{iree_artifacts.IREE_ARTIFACT_PREFIX}_{model.name}_" - f"module_{gen_config.composite_id()}") + f"module_{gen_config.composite_id}") def test_get_dependent_model_map(self): model_a = common_definitions.Model( @@ -120,11 +120,11 @@ target_backend=iree_definitions.TargetBackend.LLVM_CPU, target_abi=iree_definitions.TargetABI.LINUX_GNU) ]) - gen_config_a = iree_definitions.ModuleGenerationConfig( + gen_config_a = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_a, compile_config=compile_config_a) - gen_config_b = iree_definitions.ModuleGenerationConfig( + gen_config_b = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_b, compile_config=compile_config_a) - gen_config_c = iree_definitions.ModuleGenerationConfig( + gen_config_c = iree_definitions.ModuleGenerationConfig.with_flag_generation( imported_model=imported_model_b, compile_config=compile_config_b) models = iree_artifacts.get_dependent_model_map(
diff --git a/build_tools/python/e2e_test_framework/CMakeLists.txt b/build_tools/python/e2e_test_framework/CMakeLists.txt index 2abba74..54a1a95 100644 --- a/build_tools/python/e2e_test_framework/CMakeLists.txt +++ b/build_tools/python/e2e_test_framework/CMakeLists.txt
@@ -18,4 +18,4 @@ "unique_ids_test.py" ) -add_subdirectory(device_specs) +iree_add_all_subdirs()
diff --git a/build_tools/python/e2e_test_framework/definitions/CMakeLists.txt b/build_tools/python/e2e_test_framework/definitions/CMakeLists.txt new file mode 100644 index 0000000..3781cfe --- /dev/null +++ b/build_tools/python/e2e_test_framework/definitions/CMakeLists.txt
@@ -0,0 +1,12 @@ +# Copyright 2023 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +iree_build_tools_py_test( + NAME + iree_definitions_test + SRC + "iree_definitions_test.py" +)
diff --git a/build_tools/python/e2e_test_framework/definitions/iree_definitions.py b/build_tools/python/e2e_test_framework/definitions/iree_definitions.py index b16c921..a9c1719 100644 --- a/build_tools/python/e2e_test_framework/definitions/iree_definitions.py +++ b/build_tools/python/e2e_test_framework/definitions/iree_definitions.py
@@ -161,49 +161,198 @@ } -@serialization.serializable +@serialization.serializable(type_key="iree_imported_models", + id_field="composite_id") @dataclass(frozen=True) class ImportedModel(object): """Describes an imported MLIR model.""" + composite_id: str model: common_definitions.Model import_config: ImportConfig - def composite_id(self): - return unique_ids.hash_composite_id([self.model.id, self.import_config.id]) - @staticmethod def from_model(model: common_definitions.Model): config = MODEL_SOURCE_TO_DEFAULT_IMPORT_CONFIG_MAP.get(model.source_type) if config is None: raise ValueError(f"Unsupported model source type: {model.source_type}.") - return ImportedModel(model=model, import_config=config) + composite_id = unique_ids.hash_composite_id([model.id, config.id]) + return ImportedModel(composite_id=composite_id, + model=model, + import_config=config) -@serialization.serializable +@serialization.serializable(type_key="iree_module_generation_configs", + id_field="composite_id") @dataclass(frozen=True) class ModuleGenerationConfig(object): """Describes a compile target to generate the module.""" + composite_id: str imported_model: ImportedModel compile_config: CompileConfig + # Full list of flags to compile with, derived from sub-components, with + # unmaterialized placeholders. Allows the compile flags to be persisted and + # decouple from the generation code. Also serves as useful information in the + # serialized JSON. + compile_flags: List[str] - def composite_id(self): - return unique_ids.hash_composite_id( - [self.imported_model.composite_id(), self.compile_config.id]) + def materialize_compile_flags(self): + """Materialize flags with dependent values.""" + return self.compile_flags + + @staticmethod + def with_flag_generation(imported_model: ImportedModel, + compile_config: CompileConfig): + composite_id = unique_ids.hash_composite_id( + [imported_model.composite_id, compile_config.id]) + return ModuleGenerationConfig( + composite_id=composite_id, + imported_model=imported_model, + compile_config=compile_config, + compile_flags=_generate_compile_flags( + compile_config, imported_model.import_config.dialect_type)) -@serialization.serializable +# Placeholder to be replaced with gpu id when outputting the actual flag list. +E2E_MODEL_RUN_CONFIG_GPU_ID_PLACEHOLDER = r"${GPU_ID_PLACEHOLDER}" + + +@serialization.serializable(type_key="iree_e2e_model_run_configs", + id_field="composite_id") @dataclass(frozen=True) class E2EModelRunConfig(object): """Describes an e2e run.""" + composite_id: str module_generation_config: ModuleGenerationConfig module_execution_config: ModuleExecutionConfig target_device_spec: common_definitions.DeviceSpec input_data: common_definitions.ModelInputData + # Full list of flags to run with, derived from sub-components, with + # unmaterialized placeholders. Allows the run flags to be persisted and + # decouple from the generation code. Also serves as useful information in the + # serialized JSON. + run_flags: List[str] - def composite_id(self): - return unique_ids.hash_composite_id([ - self.module_generation_config.composite_id(), - self.module_execution_config.id, self.target_device_spec.id, - self.input_data.id + def materialize_run_flags(self, gpu_id: str = "0"): + """Materialize flags with dependent values.""" + return [ + flag.replace(E2E_MODEL_RUN_CONFIG_GPU_ID_PLACEHOLDER, gpu_id) + for flag in self.run_flags + ] + + @staticmethod + def with_flag_generation(module_generation_config: ModuleGenerationConfig, + module_execution_config: ModuleExecutionConfig, + target_device_spec: common_definitions.DeviceSpec, + input_data: common_definitions.ModelInputData): + composite_id = unique_ids.hash_composite_id([ + module_generation_config.composite_id, module_execution_config.id, + target_device_spec.id, input_data.id ]) + run_flags = generate_run_flags( + imported_model=module_generation_config.imported_model, + input_data=input_data, + module_execution_config=module_execution_config, + gpu_id=E2E_MODEL_RUN_CONFIG_GPU_ID_PLACEHOLDER) + return E2EModelRunConfig(composite_id=composite_id, + module_generation_config=module_generation_config, + module_execution_config=module_execution_config, + target_device_spec=target_device_spec, + input_data=input_data, + run_flags=run_flags) + + +def generate_run_flags(imported_model: ImportedModel, + input_data: common_definitions.ModelInputData, + module_execution_config: ModuleExecutionConfig, + gpu_id: str = "0", + with_driver: bool = True) -> List[str]: + """Returns the IREE run module flags of the input model and execution config. + Args: + model: source model. + input_data: model input data. + module_execution_config: execution config. + gpu_id: target gpu id, if runs on GPUs. + with_driver: populate the driver flags if true. False can be used for + generating flags for some CMake rules with a separate DRIVER arg. + Returns: + List of flags. + """ + + model = imported_model.model + run_flags = [f"--function={model.entry_function}"] + if input_data != common_definitions.ZEROS_MODEL_INPUT_DATA: + raise ValueError("Currently only support all-zeros data.") + run_flags += [f"--input={input_type}=0" for input_type in model.input_types] + + exec_config = module_execution_config + run_flags += exec_config.extra_flags.copy() + if with_driver: + driver = exec_config.driver + if driver == RuntimeDriver.CUDA: + run_flags.append(f"--device=cuda://{gpu_id}") + else: + run_flags.append(f"--device={driver.value}") + + return run_flags + + +def _generate_compile_flags(compile_config: CompileConfig, + dialect_type: MLIRDialectType) -> List[str]: + if len(compile_config.compile_targets) != 1: + raise ValueError(f"Only one compile target is supported. Got:" + f" {compile_config.compile_targets}") + + compile_target = compile_config.compile_targets[0] + flags = [ + f"--iree-hal-target-backends={compile_target.target_backend.value}", + f"--iree-input-type={dialect_type.value}" + ] + flags += _generate_compile_target_flags(compile_target) + flags += compile_config.extra_flags + return flags + + +def _generate_compile_target_flags(target: CompileTarget) -> List[str]: + arch_info = target.target_architecture + if target.target_backend == TargetBackend.VULKAN_SPIRV: + return [ + f"--iree-vulkan-target-triple={arch_info.architecture}-unknown-{target.target_abi.value}", + ] + + if arch_info.architecture == "x86_64": + flags = [ + f"--iree-llvm-target-triple=x86_64-unknown-{target.target_abi.value}", + f"--iree-llvm-target-cpu={arch_info.microarchitecture.lower()}" + ] + elif arch_info.architecture == "riscv_64": + flags = [ + f"--iree-llvm-target-triple=riscv64-pc-{target.target_abi.value}", + "--iree-llvm-target-cpu=generic-rv64", "--iree-llvm-target-abi=lp64d", + "--iree-llvm-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v", + "--riscv-v-fixed-length-vector-lmul-max=8" + ] + elif arch_info.architecture == "riscv_32": + # TODO(llvm-project/60463): Replace 'zve32f' with 'zve32x'. + flags = [ + f"--iree-llvm-target-triple=riscv32-pc-{target.target_abi.value}", + "--iree-llvm-target-cpu=generic-rv32", "--iree-llvm-target-abi=ilp32", + "--iree-llvm-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f", + "--riscv-v-fixed-length-vector-lmul-max=8" + ] + elif arch_info.architecture == "armv8.2-a": + flags = [ + f"--iree-llvm-target-triple=aarch64-none-{target.target_abi.value}", + ] + elif arch_info.architecture == "cuda": + if target.target_abi != TargetABI.LINUX_GNU: + raise ValueError( + f"Unsupported target ABI for CUDA backend: `{target.target_abi}`") + flags = [ + f"--iree-hal-cuda-llvm-target-arch={arch_info.microarchitecture}", + ] + elif arch_info.architecture == "vmvx": + flags = [] + else: + raise ValueError(f"Unsupported architecture: '{arch_info.architecture}'") + return flags
diff --git a/build_tools/python/e2e_test_framework/definitions/iree_definitions_test.py b/build_tools/python/e2e_test_framework/definitions/iree_definitions_test.py new file mode 100644 index 0000000..083c197 --- /dev/null +++ b/build_tools/python/e2e_test_framework/definitions/iree_definitions_test.py
@@ -0,0 +1,94 @@ +## Copyright 2023 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import unittest + +from e2e_test_framework.definitions import common_definitions, iree_definitions + + +class IreeDefinitionsTest(unittest.TestCase): + + def test_generate_run_flags(self): + imported_model = iree_definitions.ImportedModel.from_model( + common_definitions.Model( + id="1234", + name="tflite_m", + tags=[], + source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, + source_url="https://example.com/xyz.tflite", + entry_function="main", + input_types=["1xf32", "2x2xf32"])) + execution_config = iree_definitions.ModuleExecutionConfig( + id="123", + tags=["test"], + loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, + driver=iree_definitions.RuntimeDriver.LOCAL_TASK, + extra_flags=["--task=10"]) + + flags = iree_definitions.generate_run_flags( + imported_model=imported_model, + input_data=common_definitions.ZEROS_MODEL_INPUT_DATA, + module_execution_config=execution_config) + + self.assertEqual(flags, [ + "--function=main", "--input=1xf32=0", "--input=2x2xf32=0", "--task=10", + "--device=local-task" + ]) + + def test_generate_run_flags_with_cuda(self): + imported_model = iree_definitions.ImportedModel.from_model( + common_definitions.Model( + id="1234", + name="tflite_m", + tags=[], + source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, + source_url="https://example.com/xyz.tflite", + entry_function="main", + input_types=["1xf32"])) + execution_config = iree_definitions.ModuleExecutionConfig( + id="123", + tags=["test"], + loader=iree_definitions.RuntimeLoader.NONE, + driver=iree_definitions.RuntimeDriver.CUDA, + extra_flags=[]) + + flags = iree_definitions.generate_run_flags( + imported_model=imported_model, + input_data=common_definitions.ZEROS_MODEL_INPUT_DATA, + module_execution_config=execution_config, + gpu_id="3") + + self.assertEqual( + flags, ["--function=main", "--input=1xf32=0", "--device=cuda://3"]) + + def test_generate_run_flags_without_driver(self): + imported_model = iree_definitions.ImportedModel.from_model( + common_definitions.Model( + id="1234", + name="tflite_m", + tags=[], + source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, + source_url="https://example.com/xyz.tflite", + entry_function="main", + input_types=["1xf32"])) + execution_config = iree_definitions.ModuleExecutionConfig( + id="123", + tags=["test"], + loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, + driver=iree_definitions.RuntimeDriver.LOCAL_TASK, + extra_flags=["--task=10"]) + + flags = iree_definitions.generate_run_flags( + imported_model=imported_model, + input_data=common_definitions.ZEROS_MODEL_INPUT_DATA, + module_execution_config=execution_config, + with_driver=False) + + self.assertEqual(flags, ["--function=main", "--input=1xf32=0", "--task=10"]) + + +if __name__ == "__main__": + unittest.main()
diff --git a/tests/e2e/models/generated_e2e_model_tests.cmake b/tests/e2e/models/generated_e2e_model_tests.cmake index 1973823..daf6550 100644 --- a/tests/e2e/models/generated_e2e_model_tests.cmake +++ b/tests/e2e/models/generated_e2e_model_tests.cmake
@@ -51,8 +51,8 @@ RUNNER_ARGS "--function=main" "--input=1x257x257x3xf32=0" - "--expected_f32_threshold=0.001" "--device_allocator=caching" + "--expected_f32_threshold=0.001" ) iree_benchmark_suite_module_test(