Tool to generate the cmake file of benchmark suite (#10308)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1f76ba1..5df8be9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt

@@ -85,6 +85,7 @@
 option(IREE_BUILD_EXPERIMENTAL_REMOTING "Builds experimental remoting support." OFF)
 option(IREE_BUILD_EXPERIMENTAL_VMVX_MMT4D "Enables MMT4D methods in the VMVX module." OFF)
 option(IREE_BUILD_EXPERIMENTAL_WEB_SAMPLES "Builds experimental web samples." OFF)
+option(IREE_BUILD_EXPERIMENTAL_PYTHON_GENERATED_BENCHMARKS "Builds IREE benchmark suites generated by the python benchmark framework." OFF)
 
 #-------------------------------------------------------------------------------
 # Runtime HAL Driver Options
@@ -703,7 +704,7 @@
 # IREE top-level targets
 #-------------------------------------------------------------------------------
 
-if(IREE_BUILD_BENCHMARKS)
+if(IREE_BUILD_BENCHMARKS OR IREE_BUILD_EXPERIMENTAL_PYTHON_GENERATED_BENCHMARKS)
   # Add top-level custom targets to drive generating benchmark suites.
 
   # iree-benchmark-import-models imports benchmark models from their source
@@ -746,7 +747,7 @@
 # Note: Test deps are not built as part of all (use the iree-test-deps target).
 add_subdirectory(tests EXCLUDE_FROM_ALL)
 
-if(IREE_BUILD_BENCHMARKS)
+if(IREE_BUILD_BENCHMARKS OR IREE_BUILD_EXPERIMENTAL_PYTHON_GENERATED_BENCHMARKS)
   find_program(IREE_IMPORT_TFLITE_PATH iree-import-tflite)
   if(IREE_IMPORT_TFLITE_PATH)
     message(STATUS "Found ${IREE_IMPORT_TFLITE_PATH} to generate benchmark artifacts")

diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
index 0e9c88b..08986b4 100644
--- a/benchmarks/CMakeLists.txt
+++ b/benchmarks/CMakeLists.txt

@@ -1 +1,5 @@
 iree_add_all_subdirs()
+
+if (IREE_BUILD_EXPERIMENTAL_PYTHON_GENERATED_BENCHMARKS)
+  include(generated_benchmark_suites.cmake)
+endif()

diff --git a/benchmarks/generated_benchmark_suites.cmake b/benchmarks/generated_benchmark_suites.cmake
new file mode 100644
index 0000000..abf9456
--- /dev/null
+++ b/benchmarks/generated_benchmark_suites.cmake

@@ -0,0 +1,59 @@
+################################################################################
+# Autogenerated by build_tools/benchmarks/generate_cmake_benchmark_suites.py   #
+# To update the benchmarks, modify the files in build_tools/benchmarks/suites/ #
+# and regenerate this file.                                                    #
+################################################################################
+
+################################################################################
+# Defines the required variables                                               #
+################################################################################
+iree_package_name(_PACKAGE_NAME)
+set(_ROOT_ARTIFACTS_DIR "${IREE_BINARY_DIR}/benchmark_suites")
+set(_MODEL_ARTIFACTS_DIR "${_ROOT_ARTIFACTS_DIR}/models")
+set(_IREE_ARTIFACTS_DIR "${_ROOT_ARTIFACTS_DIR}/iree")
+
+################################################################################
+# Below is generated by build_tools/benchmarks/suites/cmake_rule_generator.py  #
+################################################################################
+# Fetch the model from "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_1.0_224.tflite"
+add_custom_command(
+  OUTPUT "${_MODEL_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2.tflite"
+  COMMAND
+    "${Python3_EXECUTABLE}" "${IREE_ROOT_DIR}/build_tools/scripts/download_file.py"
+    "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_1.0_224.tflite" -o "${_MODEL_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2.tflite"
+  DEPENDS
+    "${IREE_ROOT_DIR}/build_tools/scripts/download_file.py"
+  COMMENT "Downloading https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_1.0_224.tflite"
+)
+add_custom_target(
+    "${_PACKAGE_NAME}_model-7d45f8e5-bb5e-48d0-928d-8f125104578f"
+  DEPENDS
+    "${_MODEL_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2.tflite"
+)
+
+# Import the TFLite model "${_MODEL_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2.tflite"
+iree_import_tflite_model(
+  TARGET_NAME "${_PACKAGE_NAME}_iree-import-model-7d45f8e5-bb5e-48d0-928d-8f125104578f"
+  SOURCE "${_MODEL_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2.tflite"
+  OUTPUT_MLIR_FILE "${_IREE_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2/mobilenet_v2.mlir"
+)
+# Mark dependency so users can import models without compiling them.
+add_dependencies(iree-benchmark-import-models "${_PACKAGE_NAME}_iree-import-model-7d45f8e5-bb5e-48d0-928d-8f125104578f")
+
+# Compile the module "${_IREE_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2/e7e18b0f-c72d-4f1c-89b1-5afee70df6e9.vmfb"
+iree_bytecode_module(
+  NAME
+    "iree-module-7d45f8e5-bb5e-48d0-928d-8f125104578f-e7e18b0f-c72d-4f1c-89b1-5afee70df6e9"
+  MODULE_FILE_NAME
+    "${_IREE_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2/e7e18b0f-c72d-4f1c-89b1-5afee70df6e9.vmfb"
+  SRC
+    "${_IREE_ARTIFACTS_DIR}/7d45f8e5-bb5e-48d0-928d-8f125104578f_mobilenet_v2/mobilenet_v2.mlir"
+  FLAGS
+    --iree-hal-target-backends=llvm-cpu;--iree-input-type=tosa;--iree-llvm-target-triple=x86_64-unknown-linux-gnu;--iree-llvm-target-cpu=cascadelake
+  DEPENDS
+    "${_PACKAGE_NAME}_iree-import-model-7d45f8e5-bb5e-48d0-928d-8f125104578f"
+)
+# Mark dependency so that we have one target to drive them all.
+add_dependencies(iree-benchmark-suites "${_PACKAGE_NAME}_iree-module-7d45f8e5-bb5e-48d0-928d-8f125104578f-e7e18b0f-c72d-4f1c-89b1-5afee70df6e9")
+
+################################################################################

diff --git a/build_tools/benchmarks/generate_cmake_benchmark_suites.py b/build_tools/benchmarks/generate_cmake_benchmark_suites.py
new file mode 100755
index 0000000..0bbe938
--- /dev/null
+++ b/build_tools/benchmarks/generate_cmake_benchmark_suites.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+## Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Generates a CMake file to build the benchmark suites."""
+
+import sys
+import pathlib
+import argparse
+
+# Add build_tools python dir to the search path.
+sys.path.insert(0, str(pathlib.Path(__file__).parent / ".." / "python"))
+
+import benchmarks.iree.definitions
+from e2e_test_framework import cmake_rule_generator
+
+TEMPLATE_DIR = pathlib.Path(__file__).parent
+GENERATED_BENCHMARK_SUITES_CMAKE_TEMPLATE = cmake_rule_generator.read_template_from_file(
+    TEMPLATE_DIR / "iree_generated_benchmark_suites_template.cmake")
+
+
+def parse_arguments():
+  """Parses command-line options."""
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--output",
+                      required=True,
+                      help="Path to write the generated cmake file.")
+
+  return parser.parse_args()
+
+
+def main(args: argparse.Namespace):
+  compile_specs, _ = benchmarks.iree.definitions.generate()
+  benchmark_rules = cmake_rule_generator.generate_rules(
+      model_artifacts_dir="${_MODEL_ARTIFACTS_DIR}",
+      iree_artifacts_dir="${_IREE_ARTIFACTS_DIR}",
+      iree_compile_specs=compile_specs)
+  cmake_file = GENERATED_BENCHMARK_SUITES_CMAKE_TEMPLATE.substitute(
+      __BENCHMARK_RULES='\n'.join(benchmark_rules))
+  with open(args.output, "w") as output_file:
+    output_file.write(cmake_file)
+
+
+if __name__ == "__main__":
+  main(parse_arguments())

diff --git a/build_tools/benchmarks/iree_generated_benchmark_suites_template.cmake b/build_tools/benchmarks/iree_generated_benchmark_suites_template.cmake
new file mode 100644
index 0000000..f0d2ea4
--- /dev/null
+++ b/build_tools/benchmarks/iree_generated_benchmark_suites_template.cmake

@@ -0,0 +1,19 @@
+################################################################################
+# Autogenerated by build_tools/benchmarks/generate_cmake_benchmark_suites.py   #
+# To update the benchmarks, modify the files in build_tools/benchmarks/suites/ #
+# and regenerate this file.                                                    #
+################################################################################
+
+################################################################################
+# Defines the required variables                                               #
+################################################################################
+iree_package_name(_PACKAGE_NAME)
+set(_ROOT_ARTIFACTS_DIR "$${IREE_BINARY_DIR}/benchmark_suites")
+set(_MODEL_ARTIFACTS_DIR "$${_ROOT_ARTIFACTS_DIR}/models")
+set(_IREE_ARTIFACTS_DIR "$${_ROOT_ARTIFACTS_DIR}/iree")
+
+################################################################################
+# Below is generated by build_tools/benchmarks/suites/cmake_rule_generator.py  #
+################################################################################
+$__BENCHMARK_RULES
+################################################################################

diff --git a/build_tools/python/benchmarks/iree/benchmarks.py b/build_tools/python/benchmarks/iree/definitions.py
similarity index 93%
rename from build_tools/python/benchmarks/iree/benchmarks.py
rename to build_tools/python/benchmarks/iree/definitions.py
index bece168..fdd8144 100644
--- a/build_tools/python/benchmarks/iree/benchmarks.py
+++ b/build_tools/python/benchmarks/iree/definitions.py

@@ -80,3 +80,9 @@
               tool=MODULE_BENCHMARK_TOOL,
               extra_flags=[f"--task_topology_group_count={thread_num}"]))
     return run_configs
+
+
+def generate(
+) -> Tuple[List[iree_definitions.CompileSpec], List[iree_definitions.RunSpec]]:
+  """Generates all compile and run specs for IREE benchmarks."""
+  return Linux_x86_64_Benchmarks.generate()

diff --git a/build_tools/python/e2e_test_framework/cmake_rule_generator.py b/build_tools/python/e2e_test_framework/cmake_rule_generator.py
index 9a2a7c0..9dc8014 100644
--- a/build_tools/python/e2e_test_framework/cmake_rule_generator.py
+++ b/build_tools/python/e2e_test_framework/cmake_rule_generator.py

@@ -17,22 +17,20 @@
 
 from e2e_test_framework.definitions import common_definitions, iree_definitions
 
+
+def read_template_from_file(template_path: pathlib.Path) -> string.Template:
+  return string.Template(template_path.read_text())
+
+
 TEMPLATE_DIR = pathlib.Path(__file__).parent
-
-
-def read_template_from_file(template_name: str) -> string.Template:
-  with open(TEMPLATE_DIR / template_name, "r") as f:
-    return string.Template(f.read())
-
-
 DOWNLOAD_ARTIFACT_CMAKE_TEMPLATE = read_template_from_file(
-    "iree_download_artifact_template.cmake")
+    TEMPLATE_DIR / "iree_download_artifact_template.cmake")
 TFLITE_IMPORT_CMAKE_TEMPLATE = read_template_from_file(
-    "iree_tflite_import_template.cmake")
+    TEMPLATE_DIR / "iree_tflite_import_template.cmake")
 TF_IMPORT_CMAKE_TEMPLATE = read_template_from_file(
-    "iree_tf_import_template.cmake")
+    TEMPLATE_DIR / "iree_tf_import_template.cmake")
 IREE_BYTECODE_MODULE_CMAKE_TEMPLATE = read_template_from_file(
-    "iree_bytecode_module_template.cmake")
+    TEMPLATE_DIR / "iree_bytecode_module_template.cmake")
 
 
 @dataclass

diff --git a/build_tools/python/e2e_test_framework/iree_tf_import_template.cmake b/build_tools/python/e2e_test_framework/iree_tf_import_template.cmake
index c8b33d0..364e448 100644
--- a/build_tools/python/e2e_test_framework/iree_tf_import_template.cmake
+++ b/build_tools/python/e2e_test_framework/iree_tf_import_template.cmake

@@ -5,3 +5,5 @@
   ENTRY_FUNCTION "$__ENTRY_FUNCTION"
   OUTPUT_MLIR_FILE "$__OUTPUT_PATH"
 )
+# Mark dependency so users can import models without compiling them.
+add_dependencies(iree-benchmark-import-models "$${_PACKAGE_NAME}_$__TARGET_NAME")

diff --git a/build_tools/python/e2e_test_framework/iree_tflite_import_template.cmake b/build_tools/python/e2e_test_framework/iree_tflite_import_template.cmake
index 0e05e42..82616f9 100644
--- a/build_tools/python/e2e_test_framework/iree_tflite_import_template.cmake
+++ b/build_tools/python/e2e_test_framework/iree_tflite_import_template.cmake

@@ -4,3 +4,5 @@
   SOURCE "$__SOURCE_MODEL_PATH"
   OUTPUT_MLIR_FILE "$__OUTPUT_PATH"
 )
+# Mark dependency so users can import models without compiling them.
+add_dependencies(iree-benchmark-import-models "$${_PACKAGE_NAME}_$__TARGET_NAME")