diff --git a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
index ecf516a..47d6281 100644
--- a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
+++ b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark.yml
@@ -18,25 +18,31 @@
       - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec build_tools/kokoro/gcp_ubuntu/cmake/android/build.sh arm64-v8a"
       - "gsutil cp gs://iree-model-artifacts/iree-mobile-bert-artifacts-6fe4616e0ab9958eb18f368960a31276f1362029.tar.gz ."
       - "tar -xzf iree-mobile-bert-artifacts-6fe4616e0ab9958eb18f368960a31276f1362029.tar.gz"
-      - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec build_tools/mako/compile_android_modules.sh --model=tmp/iree/modules/MobileBertSquad/iree_input.mlir --targets=vulkan-spirv,vmla,dylib-llvm-aot --prefix=mobile-bert"
-      - "tar -czvf model-artifacts.tgz build-android/iree/tools/iree-benchmark-module mobile-bert-*.vmfb tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile"
+      - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/cmake-android@sha256:15d3266ae4865f7642a4ef4d76e5181f0dc3482a7cfba9021b6b55be524208ec python3 build_tools/mako/compile_android_modules.py"
+      - "cp tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile mobile-bert_flagfile"
+      - "tar -czvf model-Pixel4-artifacts.tgz build-android/iree/tools/iree-benchmark-module mobile-bert_Pixel4*.vmfb"
+      - "tar -czvf model-S20-artifacts.tgz build-android/iree/tools/iree-benchmark-module mobile-bert_S20*.vmfb"
     agents:
       - "build=true"
     env:
       IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
-    artifact_paths: "model-artifacts.tgz"
+    artifact_paths:
+      - "model-Pixel4-artifacts.tgz"
+      - "model-S20-artifacts.tgz"
+      - "mobile-bert_flagfile"
 
   - wait
 
   - label: "benchmark on Pixel 4 (snapdragon-855, adreno-640)"
     commands:
-      - "buildkite-agent artifact download --step build model-artifacts.tgz ./"
-      - "tar xzvf model-artifacts.tgz"
+      - "buildkite-agent artifact download --step build model-Pixel4-artifacts.tgz ./"
+      - "buildkite-agent artifact download --step build mobile-bert_flagfile ./"
+      - "tar xzvf model-Pixel4-artifacts.tgz"
       - "adb shell mkdir -p /data/local/tmp/benchmark_tmpdir"
       - "adb push build-android/iree/tools/iree-benchmark-module /data/local/tmp/benchmark_tmpdir"
-      - "adb push mobile-bert-*.vmfb /data/local/tmp/benchmark_tmpdir"
-      - "adb push tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile /data/local/tmp/benchmark_tmpdir"
-      - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=5538704950034432 --model=mobile-bert --git_hash=$(git rev-parse HEAD)"
+      - "adb push *.vmfb /data/local/tmp/benchmark_tmpdir"
+      - "adb push mobile-bert_flagfile /data/local/tmp/benchmark_tmpdir"
+      - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=5538704950034432 --model=mobile-bert --git_hash=$(git rev-parse HEAD) --phone=Pixel4"
       - "adb shell rm -rf /data/local/tmp/benchmark_tmpdir"
     agents:
       - "android-soc=snapdragon-855"
@@ -49,13 +55,14 @@
 
   - label: "benchmark on Galaxy S20 (exynos-990, mali-g77)"
     commands:
-      - "buildkite-agent artifact download --step build model-artifacts.tgz ./"
-      - "tar xzvf model-artifacts.tgz"
+      - "buildkite-agent artifact download --step build model-S20-artifacts.tgz ./"
+      - "buildkite-agent artifact download --step build mobile-bert_flagfile ./"
+      - "tar xzvf model-S20-artifacts.tgz"
       - "adb shell mkdir -p /data/local/tmp/benchmark_tmpdir"
       - "adb push build-android/iree/tools/iree-benchmark-module /data/local/tmp/benchmark_tmpdir"
-      - "adb push mobile-bert-*.vmfb /data/local/tmp/benchmark_tmpdir"
-      - "adb push tmp/iree/modules/MobileBertSquad/iree_vmla/traces/serving_default/flagfile /data/local/tmp/benchmark_tmpdir"
-      - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=4699630718681088 --model=mobile-bert --git_hash=$(git rev-parse HEAD)"
+      - "adb push *.vmfb /data/local/tmp/benchmark_tmpdir"
+      - "adb push mobile-bert_flagfile /data/local/tmp/benchmark_tmpdir"
+      - "build_tools/mako/benchmark_modules_on_android.sh --targets=vulkan-spirv,vmla,dylib-llvm-aot --benchmark_key=4699630718681088 --model=mobile-bert --git_hash=$(git rev-parse HEAD) --phone=S20"
       - "adb shell rm -rf /data/local/tmp/benchmark_tmpdir"
     agents:
       - "android-soc=exynos-990"
diff --git a/build_tools/mako/benchmark_modules_on_android.sh b/build_tools/mako/benchmark_modules_on_android.sh
index 77d6305..1f7adda 100755
--- a/build_tools/mako/benchmark_modules_on_android.sh
+++ b/build_tools/mako/benchmark_modules_on_android.sh
@@ -31,6 +31,10 @@
       model="${1#*=}"
       shift
       ;;
+    --phone=*)
+      phone="${1#*=}"
+      shift
+      ;;
     --targets=*)
       targets="${1#*=}"
       shift
@@ -63,6 +67,11 @@
   exit 1
 fi
 
+if [[ -z "${phone}" ]]; then
+  echo "Must set --phone flag.";
+  exit 1
+fi
+
 function append_mako_metadata {
   local mako_log="$1"
   cat >> "${mako_log}" << EOF
@@ -111,11 +120,11 @@
   esac
 
   driver="$(echo ${target} | cut -d '-' -f1)"
-  test_out="${OUTPUT_DIR}/${model}-${target}_output.txt"
+  test_out="${OUTPUT_DIR}/${model}_${phone}_${target}_${TAG}_output.txt"
   adb shell LD_LIBRARY_PATH=/data/local/tmp taskset 80 \
     "${DEVICE_ROOT}/iree-benchmark-module" \
-    "--flagfile=${DEVICE_ROOT}/flagfile" \
-    "--module_file=${DEVICE_ROOT}/${model}-${target}.vmfb" \
+    "--flagfile=${DEVICE_ROOT}/${model}_flagfile" \
+    "--module_file=${DEVICE_ROOT}/${model}_${phone}_${target}_${TAG}.vmfb" \
     "--driver=${driver}" \
     "${extra_flags[@]}" \
     --benchmark_repetitions=10 | tee "${test_out}"
diff --git a/build_tools/mako/compile_android_modules.py b/build_tools/mako/compile_android_modules.py
new file mode 100644
index 0000000..855380f
--- /dev/null
+++ b/build_tools/mako/compile_android_modules.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+
+class TargetInfo:
+  """Information of a target backend.
+
+  Attributes:
+    name: The target name used in iree-translate, e.g., vulkan-spirv.
+    mako_tag: The value_key in Mako config. This will be used in Mako metric
+      info, which should match to the config.
+    extra_flags: Addition compilation flags. This is useful to target different
+      hardware.
+  """
+
+  def __init__(self, name, mako_tag, extra_flags=[]):
+    self.name = name
+    self.mako_tag = mako_tag
+    self.extra_flags = extra_flags
+
+
+class PhoneInfo:
+  """Information of a phone.
+
+  Attributes:
+    name: The name of the phone.
+    targets: A list of TargetInfo which indicates the target config to benchmark
+      on the phone.
+  """
+
+  def __init__(self, name, targets):
+    self.name = name
+    self.targets = targets
+
+
+class ModelInfo:
+  """Information of a model.
+
+  Attributes:
+    name: The name of the model.
+    model_path: A path to MLIR input file. This can be a relative path.
+  """
+
+  def __init__(self, name, model_path):
+    self.name = name
+    self.model_path = model_path
+
+
+PHONES = [
+    PhoneInfo(
+        name="Pixel4",
+        targets=[
+            TargetInfo(name="vmla", mako_tag="vmla"),
+            TargetInfo(
+                name="dylib-llvm-aot",
+                mako_tag="cpu",
+                extra_flags=[
+                    "--iree-llvm-target-triple=aarch64-none-linux-android29"
+                ]),
+            TargetInfo(
+                name="vulkan-spirv",
+                mako_tag="vlk",
+                extra_flags=[
+                    "--iree-spirv-enable-vectorization",
+                    "--iree-vulkan-target-triple=qualcomm-adreno640-unknown-android10"
+                ])
+        ]),
+    PhoneInfo(
+        name="S20",
+        targets=[
+            TargetInfo(name="vmla", mako_tag="vmla"),
+            TargetInfo(
+                name="dylib-llvm-aot",
+                mako_tag="cpu",
+                extra_flags=[
+                    "--iree-llvm-target-triple=aarch64-none-linux-android29"
+                ]),
+            TargetInfo(
+                name="vulkan-spirv",
+                mako_tag="vlk",
+                extra_flags=[
+                    "--iree-spirv-enable-vectorization",
+                    "--iree-vulkan-target-triple=valhall-g77-unknown-android10"
+                ])
+        ])
+]
+
+MODELS = [
+    ModelInfo(
+        name="mobile-bert",
+        model_path="tmp/iree/modules/MobileBertSquad/iree_input.mlir",
+    )
+]
+
+
+def main() -> None:
+  IREE_TRANSLATE_PATH = "build-host/iree/tools/iree-translate"
+  for model in MODELS:
+    for phone in PHONES:
+      for target in phone.targets:
+        module_name = "{}_{}_{}_{}.vmfb".format(model.name, phone.name,
+                                                target.name, target.mako_tag)
+        if module_name.count("_") != 3:
+          raise ValueError(
+              "Expect model name, phone name and target name do not contain '_'"
+          )
+        print("Generating {} ...".format(module_name))
+        subprocess.run([
+            IREE_TRANSLATE_PATH, model.model_path,
+            "--iree-mlir-to-vm-bytecode-module", "--iree-hal-target-backends={}"
+            .format(target.name), "-o", module_name
+        ] + target.extra_flags)
+
+
+if __name__ == "__main__":
+  main()
diff --git a/build_tools/mako/compile_android_modules.sh b/build_tools/mako/compile_android_modules.sh
deleted file mode 100755
index 32d4a5f..0000000
--- a/build_tools/mako/compile_android_modules.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/bin/bash
-
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Compiles the given model to modules. The targets are expected to be a
-# comma-separated list, e.g., --targets=vmla,vulkan-spirv,dylib-llvm-aot
-#
-# The scripts is used for benchmarking automation, and it assumes:
-#   1) ANDROID_NDK env is set.
-#   2) IREE is built for the host in `build-host`, e.g. build with
-#      build_tools/cmake/build_android.sh script.
-
-set -e
-set -o pipefail
-set -o xtrace
-
-prefix="module"
-while [[ $# -gt 0 ]]; do
-  token="$1"
-  case $token in
-    --model=*)
-      model=${1#*=}
-      shift
-      ;;
-    --targets=*)
-      targets=${1#*=}
-      shift
-      ;;
-    --prefix=*)
-      prefix=${1#*=}
-      shift
-      ;;
-  esac
-done
-
-if [[ -z "${model}" ]]; then
-  echo "Must set --model flag.";
-  exit 1
-fi
-
-IFS=',' read -ra targets_array <<< "$targets"
-for target in "${targets_array[@]}"
-do
-  echo "Compile the module for ${target}..."
-  module_name="${prefix}-${target}.vmfb"
-  extra_flags=()
-  case "${target}" in
-    "dylib-llvm-aot")
-      # Note: this should match the phones the benchmarks will be run on.
-      # We could pass it in as an option or query the min/max in $ANDROID_NDK,
-      # but that will only be worth it if this script is used in different
-      # environments or for benchmarking on different phones.
-      extra_flags+=('--iree-llvm-target-triple=aarch64-none-linux-android29')
-      ;;
-    *)
-      ;;
-  esac
-  build-host/iree/tools/iree-translate \
-    --iree-mlir-to-vm-bytecode-module \
-    --iree-hal-target-backends="${target}" \
-    "${extra_flags[@]}" \
-    "${model}" \
-    -o "${module_name}"
-done