Retire Mako benchmarking infra. (#6834)
We have benchmarking pipeline v2, which uses Dana to replace Mako.
https://github.com/google/dana
- There are no patches to Mako repo for several months.
- We have more control on Dana since we have access to Dana.
- The new pipeline includes building models with cmake targets.
- It's easier to add new targets in https://github.com/google/iree/blob/main/benchmarks/TensorFlow/CMakeLists.txt
- etc
The new pipeline is more flexible than Mako, so we're going to retire Mako benchmarking infra.
The dashboard is moved to https://perf.iree.dev/IREE/statusOverall
diff --git a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
deleted file mode 100644
index 6f908ac..0000000
--- a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
+++ /dev/null
@@ -1,85 +0,0 @@
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-steps:
- - label: "build"
- commands:
- - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:7d780787608474301e74e1b5cc2a1bfd1304a79ed9e0774c7ed422c0e4a38625 build_tools/kokoro/gcp_ubuntu/cmake/android/build.sh arm64-v8a"
- - "python3 build_tools/mako/prepare_benchmark_files.py"
- - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:7d780787608474301e74e1b5cc2a1bfd1304a79ed9e0774c7ed422c0e4a38625 python3 build_tools/mako/compile_android_modules.py"
- - "tar -czvf model-Pixel4-artifacts.tgz build-android/iree/tools/iree-benchmark-module *Pixel4*.vmfb"
- - "tar -czvf model-S20-artifacts.tgz build-android/iree/tools/iree-benchmark-module *S20*.vmfb"
- - "tar -czvf flagfiles.tgz *_flagfile"
- agents:
- - "queue=build"
- env:
- IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
- artifact_paths:
- - "model-Pixel4-artifacts.tgz"
- - "model-S20-artifacts.tgz"
- - "flagfiles.tgz"
-
- - wait
-
- - label: "benchmark on Pixel 4 (snapdragon-855, adreno-640)"
- commands:
- - "buildkite-agent artifact download --step build model-Pixel4-artifacts.tgz ./"
- - "buildkite-agent artifact download --step build flagfiles.tgz ./"
- - "tar xzvf model-Pixel4-artifacts.tgz"
- - "tar xzvf flagfiles.tgz"
- - "adb shell mkdir -p /data/local/tmp/benchmark_tmpdir"
- - "adb push build-android/iree/tools/iree-benchmark-module /data/local/tmp/benchmark_tmpdir"
- - "adb push *.vmfb /data/local/tmp/benchmark_tmpdir"
- - "adb push *_flagfile /data/local/tmp/benchmark_tmpdir"
- - "python3 build_tools/mako/benchmark_modules_on_android.py --git_hash=$(git rev-parse HEAD) Pixel4"
- - "adb shell rm -rf /data/local/tmp/benchmark_tmpdir"
- agents:
- - "android-soc=snapdragon-855"
- - "android-version=11"
- - "queue=benchmark-android"
- env:
- IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
- artifact_paths: "mako-*.log"
- timeout_in_minutes: "15"
-
- - label: "benchmark on Galaxy S20 (exynos-990, mali-g77)"
- commands:
- - "buildkite-agent artifact download --step build model-S20-artifacts.tgz ./"
- - "buildkite-agent artifact download --step build flagfiles.tgz ./"
- - "tar xzvf model-S20-artifacts.tgz"
- - "tar xzvf flagfiles.tgz"
- - "adb shell mkdir -p /data/local/tmp/benchmark_tmpdir"
- - "adb push build-android/iree/tools/iree-benchmark-module /data/local/tmp/benchmark_tmpdir"
- - "adb push *.vmfb /data/local/tmp/benchmark_tmpdir"
- - "adb push *_flagfile /data/local/tmp/benchmark_tmpdir"
- - "python3 build_tools/mako/benchmark_modules_on_android.py --git_hash=$(git rev-parse HEAD) S20"
- - "adb shell rm -rf /data/local/tmp/benchmark_tmpdir"
- agents:
- - "android-soc=exynos-990"
- - "android-version=11"
- - "queue=benchmark-android"
- env:
- IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
- artifact_paths: "mako-*.log"
- timeout_in_minutes: "15"
-
- - wait
-
- - label: "upload data to Mako"
- commands:
- - "gsutil cp gs://iree-benchmark-artifacts/uploader ."
- - "chmod +x uploader"
- - "buildkite-agent artifact download mako-*.log ./"
- - "ls mako-*.log | xargs -n 1 ./uploader --file_path"
- - "rm mako-*.log"
- agents:
- - "test-android=false"
- - "build=false"
- - "mako-uploader=true"
- - "test-desktop-gpu=false"
- branches: "main"
- soft_fail:
- - exit_status: 123
diff --git a/build_tools/mako/benchmark_modules_on_android.py b/build_tools/mako/benchmark_modules_on_android.py
deleted file mode 100644
index 2f5059d..0000000
--- a/build_tools/mako/benchmark_modules_on_android.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-# Benchmarks modules generated by compile_android_modules.py and writes
-# performance data to a text proto file, e.g., mako-phone-mako_tag-git_hash.
-#
-# The script benchmarks modules on 7-th big core, ie, run with `taskset 80`.
-
-import argparse
-import datetime
-import subprocess
-import os
-import re
-
-import configuration
-
-PATTERN = re.compile(r"BM_(\w+)(.+)/real_time(\s+) (?P<ms>[.0-9]+) ms")
-DEVICE_ROOT = "/data/local/tmp/benchmark_tmpdir"
-
-
-def parse_arguments():
- parser = argparse.ArgumentParser()
- parser.add_argument("--git_hash", default="UNKNOWN")
- parser.add_argument("phone")
- args = parser.parse_args()
- return args
-
-
-def get_mako_sample(value, tag) -> str:
- return f"""
-samples: {{
- time: {value}
- target: "{tag}"
-}}""".strip()
-
-
-def get_mako_metadata(git_hash, timestamp, benchmark_key) -> str:
- return f"""
-metadata: {{
- git_hash: "{git_hash}"
- timestamp_ms: {timestamp}
- benchmark_key: "{benchmark_key}"
-}}
-""".strip()
-
-
-def benchmark(module_name, flagfile_name, target) -> str:
- samples = []
- cmd = [
- "adb", "shell", "LD_LIBRARY_PATH=/data/local/tmp", "taskset",
- target.taskset, f"{DEVICE_ROOT}/iree-benchmark-module",
- f"--flagfile={DEVICE_ROOT}/{flagfile_name}",
- f"--module_file={DEVICE_ROOT}/{module_name}", f"--driver={target.driver}",
- "--benchmark_repetitions=10"
- ] + target.runtime_flags
- print(f"Running cmd: {' '.join(cmd)}")
- output = subprocess.run(cmd,
- check=True,
- capture_output=True,
- universal_newlines=True).stdout
- for line in output.split("\n"):
- m = PATTERN.match(line)
- if m is not None:
- samples.append(get_mako_sample(m.group("ms"), target.mako_tag))
- return "\n".join(samples)
-
-
-def main(args) -> None:
- timestamp = int(datetime.datetime.now().timestamp() * 1000)
- for model_benchmark in configuration.MODEL_BENCHMARKS:
- for phone in model_benchmark.phones:
- if phone.name != args.phone:
- continue
- mako_log = []
- for target in phone.targets:
- module_name = configuration.get_module_name(model_benchmark.name,
- phone.name, target.mako_tag)
- flagfile_name = configuration.get_flagfile_name(model_benchmark.name)
- mako_log.append(benchmark(module_name, flagfile_name, target))
- mako_log.append(
- get_mako_metadata(args.git_hash, timestamp, phone.benchmark_key))
- mako_log = "\n".join(mako_log)
- filename = f"mako-{model_benchmark.name}-{phone.name}-{args.git_hash}.log"
- open(filename, "w").write(mako_log)
- print(mako_log)
-
-
-if __name__ == "__main__":
- main(parse_arguments())
diff --git a/build_tools/mako/compile_android_modules.py b/build_tools/mako/compile_android_modules.py
deleted file mode 100644
index 1efc9bf..0000000
--- a/build_tools/mako/compile_android_modules.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-# Compiles the given model (which is configured in configuration.py) to modules.
-#
-# The scripts is used for benchmarking automation, and it assumes:
-# 1) ANDROID_NDK env is set.
-# 2) IREE is built for the host in `build-host`, e.g. build with
-# build_tools/cmake/build_android.sh script.
-
-import subprocess
-
-import configuration
-
-IREE_TRANSLATE_PATH = "build-host/iree/tools/iree-translate"
-
-
-def main() -> None:
- for model_benchmark in configuration.MODEL_BENCHMARKS:
- for phone in model_benchmark.phones:
- for target in phone.targets:
- module_name = configuration.get_module_name(model_benchmark.name,
- phone.name, target.mako_tag)
- print(f"Generating {module_name} ...")
- subprocess.run(args=[
- IREE_TRANSLATE_PATH, model_benchmark.model_path,
- "--iree-mlir-to-vm-bytecode-module",
- f"--iree-hal-target-backends={target.hal_target_backend}", "-o",
- module_name
- ] + target.compilation_flags,
- check=True)
-
-
-if __name__ == "__main__":
- main()
diff --git a/build_tools/mako/config/README.md b/build_tools/mako/config/README.md
deleted file mode 100644
index e5ee9ee..0000000
--- a/build_tools/mako/config/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-This directory collects all Mako related files. The `uploader` is located at
-[iree-mako](https://github.com/hanhanW/iree-mako/tree/iree). The Mako repo is
-not pull in IREE because setting deps is complicated, and IREE only needs simple
-features to upload performance data to Mako server.
-
-To create a new benchmark, see
-[Mako guide](https://github.com/google/mako/blob/master/docs/GUIDE.md) for more
-details.
diff --git a/build_tools/mako/config/benchmark_template.config b/build_tools/mako/config/benchmark_template.config
deleted file mode 100644
index c0a8f91..0000000
--- a/build_tools/mako/config/benchmark_template.config
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-# This template is used to define/update Mako benchmarks. The input_value_info
-# and metric_info_list should be updated according to the purpose of the
-# benchmark. The benchmark_name and description must be updated. The
-# benchmark_key should not be added before creating the benchmark. You have to
-# fill the server-generated key which is returned by the Mako tool.
-#
-# For more details, see:
-# https://github.com/google/mako/blob/master/docs/GUIDE.md#preparing-your-benchmark
-
-# benchmark_key: "TBD"
-benchmark_name: "BENCHMARK_NAME (PHONE) (UNIT)"
-project_name: "iree"
-
-# Only owners can write to the benchmark
-owner_list: "hanchung@google.com"
-owner_list: "buildkite-agent@iree-oss.iam.gserviceaccount.com"
-
-# Define the name and type for x-axis of run charts
-input_value_info: {
- value_key: "t"
- label: "time"
- type: NUMERIC
-}
-
-# Three metrics, define the names for y-axis values of both run and aggregate
-# charts.
-metric_info_list: {
- value_key: "cpu"
- label: "DYLib_AOT"
-}
-metric_info_list: {
- value_key: "vmvx3t"
- label: "VMVX"
-}
-metric_info_list: {
- value_key: "vlk"
- label: "Vulkan-SPIRV"
-}
-
-description: "IREE Benchmark Template"
diff --git a/build_tools/mako/config/mobile-bert-f16-s20.config b/build_tools/mako/config/mobile-bert-f16-s20.config
deleted file mode 100644
index f7b24c9..0000000
--- a/build_tools/mako/config/mobile-bert-f16-s20.config
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-benchmark_key: "4636549841944576"
-benchmark_name: "MobileBert f16 version (S20) (Time Unit: ms)"
-project_name: "iree"
-
-# Only owners can write to the benchmark
-owner_list: "hanchung@google.com"
-owner_list: "buildkite-agent@iree-oss.iam.gserviceaccount.com"
-
-# Define the name and type for x-axis of run charts
-input_value_info: {
- value_key: "t"
- label: "time"
- type: NUMERIC
-}
-
-# Three metrics, define the names for y-axis values of both run and aggregate
-# charts.
-metric_info_list: {
- value_key: "vlk"
- label: "Vulkan-SPIRV"
-}
-
-description: "Mobile BERT Q&A model f16 version"
diff --git a/build_tools/mako/config/mobile-bert-pixel4.config b/build_tools/mako/config/mobile-bert-pixel4.config
deleted file mode 100644
index 45c4d96..0000000
--- a/build_tools/mako/config/mobile-bert-pixel4.config
+++ /dev/null
@@ -1,36 +0,0 @@
-# Copyright 2020 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-benchmark_key: "5538704950034432"
-benchmark_name: "MobileBert (Pixel 4) (Time Unit: ms)"
-project_name: "iree"
-
-# Only owners can write to the benchmark
-owner_list: "hanchung@google.com"
-owner_list: "buildkite-agent@iree-oss.iam.gserviceaccount.com"
-
-# Define the name and type for x-axis of run charts
-input_value_info: {
- value_key: "t"
- label: "time"
- type: NUMERIC
-}
-
-# Define the names for y-axis values of both run and aggregate charts.
-metric_info_list: {
- value_key: "cpu"
- label: "DYLib_AOT-1-thread"
-}
-metric_info_list: {
- value_key: "cpu3t"
- label: "DYLib_AOT-3-thread"
-}
-metric_info_list: {
- value_key: "vlk"
- label: "Vulkan-SPIRV"
-}
-
-description: "Mobile BERT Q&A model"
diff --git a/build_tools/mako/config/mobile-bert-s20.config b/build_tools/mako/config/mobile-bert-s20.config
deleted file mode 100644
index 9d141bf..0000000
--- a/build_tools/mako/config/mobile-bert-s20.config
+++ /dev/null
@@ -1,36 +0,0 @@
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-benchmark_key: "4699630718681088"
-benchmark_name: "MobileBert (S20) (Time Unit: ms)"
-project_name: "iree"
-
-# Only owners can write to the benchmark
-owner_list: "hanchung@google.com"
-owner_list: "buildkite-agent@iree-oss.iam.gserviceaccount.com"
-
-# Define the name and type for x-axis of run charts
-input_value_info: {
- value_key: "t"
- label: "time"
- type: NUMERIC
-}
-
-# Define the names for y-axis values of both run and aggregate charts.
-metric_info_list: {
- value_key: "cpu"
- label: "DYLib_AOT-1-thread"
-}
-metric_info_list: {
- value_key: "cpu3t"
- label: "DYLib_AOT-3-thread"
-}
-metric_info_list: {
- value_key: "vlk"
- label: "Vulkan-SPIRV"
-}
-
-description: "Mobile BERT Q&A model"
diff --git a/build_tools/mako/config/mobilenet-v2-pixel4.config b/build_tools/mako/config/mobilenet-v2-pixel4.config
deleted file mode 100644
index 9f8534e..0000000
--- a/build_tools/mako/config/mobilenet-v2-pixel4.config
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-benchmark_key: "6338759231537152"
-benchmark_name: "MobileNet V2 (Pixel 4) (Time Unit: ms)"
-project_name: "iree"
-
-# Only owners can write to the benchmark
-owner_list: "hanchung@google.com"
-owner_list: "buildkite-agent@iree-oss.iam.gserviceaccount.com"
-
-# Define the name and type for x-axis of run charts
-input_value_info: {
- value_key: "t"
- label: "time"
- type: NUMERIC
-}
-
-# Define the names for y-axis values of both run and aggregate charts.
-metric_info_list: {
- value_key: "cpu"
- label: "DYLib_AOT-1-thread"
-}
-metric_info_list: {
- value_key: "cpu3t"
- label: "DYLib_AOT-3-thread"
-}
-metric_info_list: {
- value_key: "vmvx3t"
- label: "VMVX"
-}
-metric_info_list: {
- value_key: "vlk"
- label: "Vulkan-SPIRV"
-}
-metric_info_list: {
- value_key: "cpu2"
- label: "DYLib_AOT_Linalg-On-Tensors"
-}
-
-description: "Keras Applications: MobileNet V2"
diff --git a/build_tools/mako/config/mobilenet-v2-s20.config b/build_tools/mako/config/mobilenet-v2-s20.config
deleted file mode 100644
index e357726..0000000
--- a/build_tools/mako/config/mobilenet-v2-s20.config
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-benchmark_key: "5618403088793600"
-benchmark_name: "MobileNet V2 (S20) (Time Unit: ms)"
-project_name: "iree"
-
-# Only owners can write to the benchmark
-owner_list: "hanchung@google.com"
-owner_list: "buildkite-agent@iree-oss.iam.gserviceaccount.com"
-
-# Define the name and type for x-axis of run charts
-input_value_info: {
- value_key: "t"
- label: "time"
- type: NUMERIC
-}
-
-# Define the names for y-axis values of both run and aggregate charts.
-metric_info_list: {
- value_key: "cpu"
- label: "DYLib_AOT-1-thread"
-}
-metric_info_list: {
- value_key: "cpu3t"
- label: "DYLib_AOT-3-thread"
-}
-metric_info_list: {
- value_key: "vmvx3t"
- label: "VMVX"
-}
-metric_info_list: {
- value_key: "vlk"
- label: "Vulkan-SPIRV"
-}
-metric_info_list: {
- value_key: "cpu2"
- label: "DYLib_AOT_Linalg-On-Tensors"
-}
-metric_info_list: {
- value_key: "vlk2"
- label: "Vulkan-SPIRV_Linalg-On-Tensors"
-}
-
-description: "Keras Applications: MobileNet V2"
diff --git a/build_tools/mako/config/project.config b/build_tools/mako/config/project.config
deleted file mode 100644
index fae05ce..0000000
--- a/build_tools/mako/config/project.config
+++ /dev/null
@@ -1,25 +0,0 @@
-# Copyright 2020 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-# project_name is always evaluated as lowercase and must be unique
-project_name: "iree"
-
-# project_alias is the display name of the project and will be shown on
-# mako.dev. It does not have to be unique.
-project_alias: "IREE"
-
-# TODO(hanchung): Add other IREE members to owner_list.
-owner_list: "hanchung@google.com"
-
-# By default any automation creating Buganizer work items will be opened under
-# the root buganizer component for benchmarks in this Project.
-# TODO(hanchung): Create a specific component for this, or make it file issues
-# to Github.
-default_issue_tracker : {
- buganizer_config : {
- component_id : "646953"
- }
-}
diff --git a/build_tools/mako/configuration.py b/build_tools/mako/configuration.py
deleted file mode 100644
index 4816ac1..0000000
--- a/build_tools/mako/configuration.py
+++ /dev/null
@@ -1,292 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-
-class TargetInfo:
- """Information of a target backend.
-
- Attributes:
- driver: The driver used in iree-benchmark-module, e.g., vulkan.
- hal_target_backend: The target name used in iree-translate, e.g., vulkan-spirv.
- taskset: The value used for taskset when benchmarking the IREE module.
- mako_tag: The value_key in Mako config. This will be used in Mako metric
- info, which should match to the config.
- compilation_flags: Addition compilation flags. This is useful to target
- different hardware.
- runtime_flags: Addition runtime flags. This is useful when benchmarking.
- E.g., CPU can run with single thread or multi thread.
- """
-
- def __init__(self,
- driver,
- hal_target_backend,
- taskset,
- mako_tag,
- compilation_flags=None,
- runtime_flags=None):
- if compilation_flags is None:
- compilation_flags = []
- if runtime_flags is None:
- runtime_flags = []
- self.driver = driver
- self.hal_target_backend = hal_target_backend
- self.taskset = taskset
- self.mako_tag = mako_tag
- # This setup is going to be deprecated. Now we only test mhlo inputs for
- # Mako setup, hardcoded would make it eaiser.
- self.compilation_flags = compilation_flags + ['--iree-input-type=mhlo']
- self.runtime_flags = runtime_flags
-
- def add_batch_flag(self, size):
- self.compilation_flags.append(
- f"--iree-hal-benchmark-dispatch-repeat-count={size}")
- self.runtime_flags.append(f"--batch_size={size}")
-
-
-class PhoneBenchmarkInfo:
- """Information of a phone.
-
- Attributes:
- name: The name of the phone.
- targets: A list of TargetInfo which indicates the target config to benchmark
- on the phone.
- benchmark_key: Mako benchmark key, which can be found in config/
- """
-
- def __init__(self, name, targets, benchmark_key):
- if "_" in name:
- raise ValueError("The target name contains invalid char '_'")
- self.name = name
- self.targets = targets
- self.benchmark_key = benchmark_key
-
-
-class ModelBenchmarkInfo:
- """Information of a model.
-
- Attributes:
- name: The name of the model.
- model_artifacts_name: The filename of model artifacts which locates in
- Google bucket iree-model-artifacts.
- model_path: A path to MLIR input file. This can be a relative path.
- flagfile_path: A path to flagfile. This can be a relative path.
- phones: A list of PhoneBenchmarkInfo that the benchmarking targets on.
- """
-
- def __init__(self, name, model_artifacts_name, model_path, flagfile_path,
- phones):
- if "_" in name:
- raise ValueError("The target name contains invalid char '_'")
- self.name = name
- self.model_artifacts_name = model_artifacts_name
- self.model_path = model_path
- self.flagfile_path = flagfile_path
- self.phones = phones
-
-
-def get_pixel4_default_target_list(skipped_target=None,
- batch_config=None,
- compilation_flags=None):
- if skipped_target is None:
- skipped_target = []
- if batch_config is None:
- batch_config = []
- if compilation_flags is None:
- compilation_flags = []
- targets = [
- TargetInfo(driver="dylib-sync",
- hal_target_backend="dylib-llvm-aot",
- taskset="80",
- mako_tag="cpu",
- compilation_flags=[
- "--iree-llvm-target-triple=aarch64-none-linux-android29",
- "--iree-flow-inline-constants-max-byte-length=2048",
- ]),
- TargetInfo(driver="dylib",
- hal_target_backend="dylib-llvm-aot",
- taskset="f0",
- mako_tag="cpu3t",
- compilation_flags=[
- "--iree-llvm-target-triple=aarch64-none-linux-android29",
- "--iree-flow-inline-constants-max-byte-length=2048",
- ],
- runtime_flags=[
- "--task_topology_group_count=3",
- ]),
- TargetInfo(driver="vmvx",
- hal_target_backend="vmvx",
- taskset="f0",
- mako_tag="vmvx3t",
- compilation_flags=[
- "--iree-flow-inline-constants-max-byte-length=2048",
- ],
- runtime_flags=[
- "--task_topology_group_count=3",
- ]),
- TargetInfo(driver="vulkan",
- hal_target_backend="vulkan-spirv",
- taskset="80",
- mako_tag="vlk",
- compilation_flags=[
- "--iree-vulkan-target-triple=adreno-a640-android11",
- "--iree-flow-inline-constants-max-byte-length=2048",
- "--iree-flow-dispatch-formation-enable-operand-fusion",
- "--iree-enable-fusion-with-reduction-ops",
- ])
- ]
- targets = [elem for elem in targets if elem.mako_tag not in skipped_target]
- for target in targets:
- if target.mako_tag in batch_config:
- target.add_batch_flag(batch_config[target.mako_tag])
- if target.mako_tag in compilation_flags:
- target.compilation_flags += compilation_flags[target.mako_tag]
- return targets
-
-
-def get_s20_default_target_list(skipped_target=None,
- batch_config=None,
- compilation_flags=None):
- if skipped_target is None:
- skipped_target = []
- if batch_config is None:
- batch_config = []
- if compilation_flags is None:
- compilation_flags = []
- targets = [
- TargetInfo(driver="dylib-sync",
- hal_target_backend="dylib-llvm-aot",
- taskset="80",
- mako_tag="cpu",
- compilation_flags=[
- "--iree-llvm-target-triple=aarch64-none-linux-android29",
- "--iree-flow-inline-constants-max-byte-length=2048",
- ]),
- TargetInfo(driver="dylib",
- hal_target_backend="dylib-llvm-aot",
- taskset="f0",
- mako_tag="cpu3t",
- compilation_flags=[
- "--iree-llvm-target-triple=aarch64-none-linux-android29",
- "--iree-flow-inline-constants-max-byte-length=2048",
- ],
- runtime_flags=[
- "--task_topology_group_count=3",
- ]),
- TargetInfo(driver="vmvx",
- hal_target_backend="vmvx",
- taskset="f0",
- mako_tag="vmvx3t",
- compilation_flags=[
- "--iree-flow-inline-constants-max-byte-length=2048",
- ],
- runtime_flags=[
- "--task_topology_group_count=3",
- ]),
- TargetInfo(
- driver="vulkan",
- hal_target_backend="vulkan-spirv",
- taskset="80",
- mako_tag="vlk",
- compilation_flags=[
- "--iree-vulkan-target-triple=valhall-g77-android11",
- # TODO(GH-5330): Revisit the number or delete the flag.
- "--iree-flow-inline-constants-max-byte-length=16",
- "--iree-flow-dispatch-formation-enable-operand-fusion"
- ])
- ]
- targets = [elem for elem in targets if elem.mako_tag not in skipped_target]
- for target in targets:
- if target.mako_tag in batch_config:
- target.add_batch_flag(batch_config[target.mako_tag])
- if target.mako_tag in compilation_flags:
- target.compilation_flags += compilation_flags[target.mako_tag]
- return targets
-
-
-# The batch numbers are roughly computed to let it benchmark more than 3
-# seconds.
-# Do not set batch size on Pixel 4 for GPU targets, because it will get killed
-# after 2 seconds. See https://github.com/google/iree/issues/5052
-MODEL_BENCHMARKS = [
- ModelBenchmarkInfo(
- name="mobile-bert",
- model_artifacts_name=
- "iree-mobile-bert-artifacts-6fe4616e0ab9958eb18f368960a31276f1362029.tar.gz",
- model_path="tmp/iree/modules/MobileBertSquad/iree_input.mlir",
- flagfile_path=
- "tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile",
- phones=[
- PhoneBenchmarkInfo(name="Pixel4",
- benchmark_key="5538704950034432",
- targets=get_pixel4_default_target_list(
- skipped_target=["cpu2", "vmvx3t", "vlk2"],)),
- PhoneBenchmarkInfo(name="S20",
- benchmark_key="4699630718681088",
- targets=get_s20_default_target_list(
- skipped_target=["cpu2", "vmvx3t", "vlk2"],)),
- ]),
- ModelBenchmarkInfo(
- name="mobilenet-v2",
- model_artifacts_name="mobilenet-v2.tar.gz",
- model_path="mobilenet-v2/iree_input.mlir",
- flagfile_path="mobilenet-v2/flagfile",
- phones=[
- PhoneBenchmarkInfo(
- name="Pixel4",
- benchmark_key="6338759231537152",
- targets=get_pixel4_default_target_list(
- skipped_target=["vlk2"],
- compilation_flags={
- 'cpu': [
- # TODO(GH-5857): Enable this after fixing segfault.
- #"--iree-flow-dispatch-formation-enable-operand-fusion",
- "-iree-llvm-loop-unrolling=true"
- ],
- 'cpu3t': [
- # TODO(GH-5857): Enable this after fixing segfault.
- #"--iree-flow-dispatch-formation-enable-operand-fusion",
- "-iree-llvm-loop-unrolling=true"
- ]
- })),
- PhoneBenchmarkInfo(
- name="S20",
- benchmark_key="5618403088793600",
- targets=get_s20_default_target_list(
- compilation_flags={
- 'cpu': [
- # TODO(GH-5857): Enable this after fixing segfault.
- #"--iree-flow-dispatch-formation-enable-operand-fusion",
- "-iree-llvm-loop-unrolling=true"
- ],
- 'cpu3t': [
- # TODO(GH-5857): Enable this after fixing segfault.
- #"--iree-flow-dispatch-formation-enable-operand-fusion",
- "-iree-llvm-loop-unrolling=true"
- ]
- })),
- ]),
- #ModelBenchmarkInfo(
- # name="mobilebert-f16",
- # model_artifacts_name="mobilebert-f16.tar.gz",
- # model_path="mobilebert-f16/mobilebert-f16.mlir",
- # flagfile_path="mobilebert-f16/flagfile",
- # phones=[
- # PhoneBenchmarkInfo(
- # name="S20",
- # benchmark_key="4636549841944576",
- # targets=get_s20_default_target_list(
- # skipped_target=['cpu', 'vmvx3t', 'cpu2', 'vlk2'])),
- # ])
-]
-
-
-def get_flagfile_name(model_name):
- return f"{model_name}_flagfile"
-
-
-def get_module_name(model_name, phone_name, mako_tag):
- return f"{model_name}_{phone_name}_{mako_tag}.vmfb"
diff --git a/build_tools/mako/prepare_benchmark_files.py b/build_tools/mako/prepare_benchmark_files.py
deleted file mode 100644
index 58b5b05..0000000
--- a/build_tools/mako/prepare_benchmark_files.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2021 The IREE Authors
-#
-# Licensed under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-# Prepares files for
-# * compile_android_modules.py
-# * benchmark_modules_on_android.py
-#
-# The script assumes model artifacts are in google bucket and they are zipped in
-# tar.gz format.
-
-import subprocess
-
-import configuration
-
-
-def main() -> None:
- for model_benchmark in configuration.MODEL_BENCHMARKS:
- print(f"Preparing benchmark files for {model_benchmark.name}")
- subprocess.run(args=[
- "gsutil", "cp",
- f"gs://iree-model-artifacts/{model_benchmark.model_artifacts_name}", "."
- ],
- check=True)
- subprocess.run(args=["tar", "-xvf", model_benchmark.model_artifacts_name],
- check=True)
- subprocess.run(args=[
- "cp", model_benchmark.flagfile_path,
- configuration.get_flagfile_name(model_benchmark.name)
- ],
- check=True)
-
-
-if __name__ == "__main__":
- main()
