Add support for benchmarking on targets with different setup. (#5029)
This rewrites the `compile_android_modules.sh` with python, and adds support for setting up different compilation flags for different phones.
The performance of MobileBert on S20 on GPU goes down to 490 ms.
diff --git a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
index ecf516a..47d6281 100644
--- a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
+++ b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
@@ -18,25 +18,31 @@
- "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec build_tools/kokoro/gcp_ubuntu/cmake/android/build.sh arm64-v8a"
- "gsutil cp gs://iree-model-artifacts/iree-mobile-bert-artifacts-6fe4616e0ab9958eb18f368960a31276f1362029.tar.gz ."
- "tar -xzf iree-mobile-bert-artifacts-6fe4616e0ab9958eb18f368960a31276f1362029.tar.gz"
- - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec build_tools/mako/compile_android_modules.sh --model=tmp/iree/modules/MobileBertSquad/iree_input.mlir --targets=vulkan-spirv,vmla,dylib-llvm-aot --prefix=mobile-bert"
- - "tar -czvf model-artifacts.tgz build-android/iree/tools/iree-benchmark-module mobile-bert-*.vmfb tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile"
+ - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec python3 build_tools/mako/compile_android_modules.py"
+ - "cp tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile mobile-bert_flagfile"
+ - "tar -czvf model-Pixel4-artifacts.tgz build-android/iree/tools/iree-benchmark-module mobile-bert_Pixel4*.vmfb"
+ - "tar -czvf model-S20-artifacts.tgz build-android/iree/tools/iree-benchmark-module mobile-bert_S20*.vmfb"
agents:
- "build=true"
env:
IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
- artifact_paths: "model-artifacts.tgz"
+ artifact_paths:
+ - "model-Pixel4-artifacts.tgz"
+ - "model-S20-artifacts.tgz"
+ - "mobile-bert_flagfile"
- wait
- label: "benchmark on Pixel 4 (snapdragon-855, adreno-640)"
commands:
- - "buildkite-agent artifact download --step build model-artifacts.tgz ./"
- - "tar xzvf model-artifacts.tgz"
+ - "buildkite-agent artifact download --step build model-Pixel4-artifacts.tgz ./"
+ - "buildkite-agent artifact download --step build mobile-bert_flagfile ./"
+ - "tar xzvf model-Pixel4-artifacts.tgz"
- "adb shell mkdir -p /data/local/tmp/benchmark_tmpdir"
- "adb push build-android/iree/tools/iree-benchmark-module /data/local/tmp/benchmark_tmpdir"
- - "adb push mobile-bert-*.vmfb /data/local/tmp/benchmark_tmpdir"
- - "adb push tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile /data/local/tmp/benchmark_tmpdir"
- - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=5538704950034432 --model=mobile-bert --git_hash=$(git rev-parse HEAD)"
+ - "adb push *.vmfb /data/local/tmp/benchmark_tmpdir"
+ - "adb push mobile-bert_flagfile /data/local/tmp/benchmark_tmpdir"
+ - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=5538704950034432 --model=mobile-bert --git_hash=$(git rev-parse HEAD) --phone=Pixel4"
- "adb shell rm -rf /data/local/tmp/benchmark_tmpdir"
agents:
- "android-soc=snapdragon-855"
@@ -49,13 +55,14 @@
- label: "benchmark on Galaxy S20 (exynos-990, mali-g77)"
commands:
- - "buildkite-agent artifact download --step build model-artifacts.tgz ./"
- - "tar xzvf model-artifacts.tgz"
+ - "buildkite-agent artifact download --step build model-S20-artifacts.tgz ./"
+ - "buildkite-agent artifact download --step build mobile-bert_flagfile ./"
+ - "tar xzvf model-S20-artifacts.tgz"
- "adb shell mkdir -p /data/local/tmp/benchmark_tmpdir"
- "adb push build-android/iree/tools/iree-benchmark-module /data/local/tmp/benchmark_tmpdir"
- - "adb push mobile-bert-*.vmfb /data/local/tmp/benchmark_tmpdir"
- - "adb push tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile /data/local/tmp/benchmark_tmpdir"
- - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=4699630718681088 --model=mobile-bert --git_hash=$(git rev-parse HEAD)"
+ - "adb push *.vmfb /data/local/tmp/benchmark_tmpdir"
+ - "adb push mobile-bert_flagfile /data/local/tmp/benchmark_tmpdir"
+ - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=4699630718681088 --model=mobile-bert --git_hash=$(git rev-parse HEAD) --phone=S20"
- "adb shell rm -rf /data/local/tmp/benchmark_tmpdir"
agents:
- "android-soc=exynos-990"
diff --git a/build_tools/mako/benchmark_modules_on_android.sh b/build_tools/mako/benchmark_modules_on_android.sh
index 77d6305..1f7adda 100755
--- a/build_tools/mako/benchmark_modules_on_android.sh
+++ b/build_tools/mako/benchmark_modules_on_android.sh
@@ -31,6 +31,10 @@
model="${1#*=}"
shift
;;
+ --phone=*)
+ phone="${1#*=}"
+ shift
+ ;;
--targets=*)
targets="${1#*=}"
shift
@@ -63,6 +67,11 @@
exit 1
fi
+if [[ -z "${phone}" ]]; then
+ echo "Must set --phone flag.";
+ exit 1
+fi
+
function append_mako_metadata {
local mako_log="$1"
cat >> "${mako_log}" << EOF
@@ -111,11 +120,11 @@
esac
driver="$(echo ${target} | cut -d '-' -f1)"
- test_out="${OUTPUT_DIR}/${model}-${target}_output.txt"
+ test_out="${OUTPUT_DIR}/${model}_${phone}_${target}_${TAG}_output.txt"
adb shell LD_LIBRARY_PATH=/data/local/tmp taskset 80 \
"${DEVICE_ROOT}/iree-benchmark-module" \
- "--flagfile=${DEVICE_ROOT}/flagfile" \
- "--module_file=${DEVICE_ROOT}/${model}-${target}.vmfb" \
+ "--flagfile=${DEVICE_ROOT}/${model}_flagfile" \
+ "--module_file=${DEVICE_ROOT}/${model}_${phone}_${target}_${TAG}.vmfb" \
"--driver=${driver}" \
"${extra_flags[@]}" \
--benchmark_repetitions=10 | tee "${test_out}"
diff --git a/build_tools/mako/compile_android_modules.py b/build_tools/mako/compile_android_modules.py
new file mode 100644
index 0000000..855380f
--- /dev/null
+++ b/build_tools/mako/compile_android_modules.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+
+class TargetInfo:
+ """Information of a target backend.
+
+ Attributes:
+ name: The target name used in iree-translate, e.g., vulkan-spirv.
+ mako_tag: The value_key in Mako config. This will be used in Mako metric
+ info, which should match to the config.
+ extra_flags: Addition compilation flags. This is useful to target different
+ hardware.
+ """
+
+ def __init__(self, name, mako_tag, extra_flags=[]):
+ self.name = name
+ self.mako_tag = mako_tag
+ self.extra_flags = extra_flags
+
+
+class PhoneInfo:
+ """Information of a phone.
+
+ Attributes:
+ name: The name of the phone.
+ targets: A list of TargetInfo which indicates the target config to benchmark
+ on the phone.
+ """
+
+ def __init__(self, name, targets):
+ self.name = name
+ self.targets = targets
+
+
+class ModelInfo:
+ """Information of a model.
+
+ Attributes:
+ name: The name of the model.
+ model_path: A path to MLIR input file. This can be a relative path.
+ """
+
+ def __init__(self, name, model_path):
+ self.name = name
+ self.model_path = model_path
+
+
+PHONES = [
+ PhoneInfo(
+ name="Pixel4",
+ targets=[
+ TargetInfo(name="vmla", mako_tag="vmla"),
+ TargetInfo(
+ name="dylib-llvm-aot",
+ mako_tag="cpu",
+ extra_flags=[
+ "--iree-llvm-target-triple=aarch64-none-linux-android29"
+ ]),
+ TargetInfo(
+ name="vulkan-spirv",
+ mako_tag="vlk",
+ extra_flags=[
+ "--iree-spirv-enable-vectorization",
+ "--iree-vulkan-target-triple=qualcomm-adreno640-unknown-android10"
+ ])
+ ]),
+ PhoneInfo(
+ name="S20",
+ targets=[
+ TargetInfo(name="vmla", mako_tag="vmla"),
+ TargetInfo(
+ name="dylib-llvm-aot",
+ mako_tag="cpu",
+ extra_flags=[
+ "--iree-llvm-target-triple=aarch64-none-linux-android29"
+ ]),
+ TargetInfo(
+ name="vulkan-spirv",
+ mako_tag="vlk",
+ extra_flags=[
+ "--iree-spirv-enable-vectorization",
+ "--iree-vulkan-target-triple=valhall-g77-unknown-android10"
+ ])
+ ])
+]
+
+MODELS = [
+ ModelInfo(
+ name="mobile-bert",
+ model_path="tmp/iree/modules/MobileBertSquad/iree_input.mlir",
+ )
+]
+
+
+def main() -> None:
+ IREE_TRANSLATE_PATH = "build-host/iree/tools/iree-translate"
+ for model in MODELS:
+ for phone in PHONES:
+ for target in phone.targets:
+ module_name = "{}_{}_{}_{}.vmfb".format(model.name, phone.name,
+ target.name, target.mako_tag)
+ if module_name.count("_") != 3:
+ raise ValueError(
+ "Expect model name, phone name and target name do not contain '_'"
+ )
+ print("Generating {} ...".format(module_name))
+ subprocess.run([
+ IREE_TRANSLATE_PATH, model.model_path,
+ "--iree-mlir-to-vm-bytecode-module", "--iree-hal-target-backends={}"
+ .format(target.name), "-o", module_name
+ ] + target.extra_flags)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/build_tools/mako/compile_android_modules.sh b/build_tools/mako/compile_android_modules.sh
deleted file mode 100755
index 32d4a5f..0000000
--- a/build_tools/mako/compile_android_modules.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/bin/bash
-
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Compiles the given model to modules. The targets are expected to be a
-# comma-separated list, e.g., --targets=vmla,vulkan-spirv,dylib-llvm-aot
-#
-# The scripts is used for benchmarking automation, and it assumes:
-# 1) ANDROID_NDK env is set.
-# 2) IREE is built for the host in `build-host`, e.g. build with
-# build_tools/cmake/build_android.sh script.
-
-set -e
-set -o pipefail
-set -o xtrace
-
-prefix="module"
-while [[ $# -gt 0 ]]; do
- token="$1"
- case $token in
- --model=*)
- model=${1#*=}
- shift
- ;;
- --targets=*)
- targets=${1#*=}
- shift
- ;;
- --prefix=*)
- prefix=${1#*=}
- shift
- ;;
- esac
-done
-
-if [[ -z "${model}" ]]; then
- echo "Must set --model flag.";
- exit 1
-fi
-
-IFS=',' read -ra targets_array <<< "$targets"
-for target in "${targets_array[@]}"
-do
- echo "Compile the module for ${target}..."
- module_name="${prefix}-${target}.vmfb"
- extra_flags=()
- case "${target}" in
- "dylib-llvm-aot")
- # Note: this should match the phones the benchmarks will be run on.
- # We could pass it in as an option or query the min/max in $ANDROID_NDK,
- # but that will only be worth it if this script is used in different
- # environments or for benchmarking on different phones.
- extra_flags+=('--iree-llvm-target-triple=aarch64-none-linux-android29')
- ;;
- *)
- ;;
- esac
- build-host/iree/tools/iree-translate \
- --iree-mlir-to-vm-bytecode-module \
- --iree-hal-target-backends="${target}" \
- "${extra_flags[@]}" \
- "${model}" \
- -o "${module_name}"
-done
