Add MobileNetV2 to benchmarking targets. (#5064)
diff --git a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
index 47d6281..ec76d03 100644
--- a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
+++ b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
@@ -17,11 +17,15 @@
commands:
- "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec build_tools/kokoro/gcp_ubuntu/cmake/android/build.sh arm64-v8a"
- "gsutil cp gs://iree-model-artifacts/iree-mobile-bert-artifacts-6fe4616e0ab9958eb18f368960a31276f1362029.tar.gz ."
+ - "gsutil cp gs://iree-model-artifacts/mobilenet-v2.tar.gz ."
- "tar -xzf iree-mobile-bert-artifacts-6fe4616e0ab9958eb18f368960a31276f1362029.tar.gz"
+ - "tar -xzf mobilenet-v2.tar.gz"
- "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec python3 build_tools/mako/compile_android_modules.py"
- "cp tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile mobile-bert_flagfile"
- - "tar -czvf model-Pixel4-artifacts.tgz build-android/iree/tools/iree-benchmark-module mobile-bert_Pixel4*.vmfb"
- - "tar -czvf model-S20-artifacts.tgz build-android/iree/tools/iree-benchmark-module mobile-bert_S20*.vmfb"
+ - "cp mobilenet-v2/flagfile mobilenet-v2_flagfile"
+ - "tar -czvf model-Pixel4-artifacts.tgz build-android/iree/tools/iree-benchmark-module *Pixel4*.vmfb"
+ - "tar -czvf model-S20-artifacts.tgz build-android/iree/tools/iree-benchmark-module *S20*.vmfb"
+ - "tar -czvf flagfiles.tgz *_flagfile"
agents:
- "build=true"
env:
@@ -29,20 +33,22 @@
artifact_paths:
- "model-Pixel4-artifacts.tgz"
- "model-S20-artifacts.tgz"
- - "mobile-bert_flagfile"
+ - "flagfiles.tgz"
- wait
- label: "benchmark on Pixel 4 (snapdragon-855, adreno-640)"
commands:
- "buildkite-agent artifact download --step build model-Pixel4-artifacts.tgz ./"
- - "buildkite-agent artifact download --step build mobile-bert_flagfile ./"
+ - "buildkite-agent artifact download --step build flagfiles.tgz ./"
- "tar xzvf model-Pixel4-artifacts.tgz"
+ - "tar xzvf flagfiles.tgz"
- "adb shell mkdir -p /data/local/tmp/benchmark_tmpdir"
- "adb push build-android/iree/tools/iree-benchmark-module /data/local/tmp/benchmark_tmpdir"
- "adb push *.vmfb /data/local/tmp/benchmark_tmpdir"
- - "adb push mobile-bert_flagfile /data/local/tmp/benchmark_tmpdir"
+ - "adb push *_flagfile /data/local/tmp/benchmark_tmpdir"
- "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=5538704950034432 --model=mobile-bert --git_hash=$(git rev-parse HEAD) --phone=Pixel4"
+ - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=6338759231537152 --model=mobilenet-v2 --git_hash=$(git rev-parse HEAD) --phone=Pixel4"
- "adb shell rm -rf /data/local/tmp/benchmark_tmpdir"
agents:
- "android-soc=snapdragon-855"
@@ -56,13 +62,15 @@
- label: "benchmark on Galaxy S20 (exynos-990, mali-g77)"
commands:
- "buildkite-agent artifact download --step build model-S20-artifacts.tgz ./"
- - "buildkite-agent artifact download --step build mobile-bert_flagfile ./"
+ - "buildkite-agent artifact download --step build flagfiles.tgz ./"
- "tar xzvf model-S20-artifacts.tgz"
+ - "tar xzvf flagfiles.tgz"
- "adb shell mkdir -p /data/local/tmp/benchmark_tmpdir"
- "adb push build-android/iree/tools/iree-benchmark-module /data/local/tmp/benchmark_tmpdir"
- "adb push *.vmfb /data/local/tmp/benchmark_tmpdir"
- - "adb push mobile-bert_flagfile /data/local/tmp/benchmark_tmpdir"
+ - "adb push *_flagfile /data/local/tmp/benchmark_tmpdir"
- "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=4699630718681088 --model=mobile-bert --git_hash=$(git rev-parse HEAD) --phone=S20"
+ - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=5618403088793600 --model=mobilenet-v2 --git_hash=$(git rev-parse HEAD) --phone=S20"
- "adb shell rm -rf /data/local/tmp/benchmark_tmpdir"
agents:
- "android-soc=exynos-990"
diff --git a/build_tools/mako/benchmark_modules_on_android.sh b/build_tools/mako/benchmark_modules_on_android.sh
index 1f7adda..169e73b 100755
--- a/build_tools/mako/benchmark_modules_on_android.sh
+++ b/build_tools/mako/benchmark_modules_on_android.sh
@@ -133,8 +133,7 @@
done < <(sed -En -e "${SED_EXPR}" "${test_out}")
}
-phone_serial_number="$(adb get-serialno)"
-mako_log="mako-${phone_serial_number}-${git_hash}.log"
+mako_log="mako-${model}-${phone}-${git_hash}.log"
: > "${mako_log}" # Empty the log file
IFS=',' read -ra targets_array <<< "$targets"
diff --git a/build_tools/mako/compile_android_modules.py b/build_tools/mako/compile_android_modules.py
index 855380f..5ce6e8d 100644
--- a/build_tools/mako/compile_android_modules.py
+++ b/build_tools/mako/compile_android_modules.py
@@ -104,6 +104,10 @@
ModelInfo(
name="mobile-bert",
model_path="tmp/iree/modules/MobileBertSquad/iree_input.mlir",
+ ),
+ ModelInfo(
+ name="mobilenet-v2",
+ model_path="mobilenet-v2/iree_input.mlir",
)
]
diff --git a/build_tools/mako/config/mobilenet-v2-pixel4.config b/build_tools/mako/config/mobilenet-v2-pixel4.config
new file mode 100644
index 0000000..f701d72
--- /dev/null
+++ b/build_tools/mako/config/mobilenet-v2-pixel4.config
@@ -0,0 +1,45 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+benchmark_key: "6338759231537152"
+benchmark_name: "MobileNet V2 (Pixel 4) (Time Unit: ms)"
+project_name: "iree"
+
+# Only owners can write to the benchmark
+owner_list: "hanchung@google.com"
+owner_list: "buildkite-agent@iree-oss.iam.gserviceaccount.com"
+
+# Define the name and type for x-axis of run charts
+input_value_info: {
+ value_key: "t"
+ label: "time"
+ type: NUMERIC
+}
+
+# Three metrics, define the names for y-axis values of both run and aggregate
+# charts.
+metric_info_list: {
+ value_key: "cpu"
+ label: "DYLib_AOT"
+}
+metric_info_list: {
+ value_key: "vmla"
+ label: "VMLA"
+}
+metric_info_list: {
+ value_key: "vlk"
+ label: "Vulkan-SPIRV"
+}
+
+description: "Keras Applications: MobileNet V2"
diff --git a/build_tools/mako/config/mobilenet-v2-s20.config b/build_tools/mako/config/mobilenet-v2-s20.config
new file mode 100644
index 0000000..8d1fd3f
--- /dev/null
+++ b/build_tools/mako/config/mobilenet-v2-s20.config
@@ -0,0 +1,45 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+benchmark_key: "5618403088793600"
+benchmark_name: "MobileNet V2 (S20) (Time Unit: ms)"
+project_name: "iree"
+
+# Only owners can write to the benchmark
+owner_list: "hanchung@google.com"
+owner_list: "buildkite-agent@iree-oss.iam.gserviceaccount.com"
+
+# Define the name and type for x-axis of run charts
+input_value_info: {
+ value_key: "t"
+ label: "time"
+ type: NUMERIC
+}
+
+# Three metrics, define the names for y-axis values of both run and aggregate
+# charts.
+metric_info_list: {
+ value_key: "cpu"
+ label: "DYLib_AOT"
+}
+metric_info_list: {
+ value_key: "vmla"
+ label: "VMLA"
+}
+metric_info_list: {
+ value_key: "vlk"
+ label: "Vulkan-SPIRV"
+}
+
+description: "Keras Applications: MobileNet V2"
