Enable benchmarks from TFLite flatbuffers on benchmark pipeline (#7494)
This replaces the three benchmarks derived from TFLite flatbuffers to
use the original flatbuffer source rather than a pre-compiled MLIR
file. These benchmarks are updated in place since the original source
is exactly the same. Using flatbuffers avoids a few issues:
1. The MLIR IR formats do not offer any backward compatibility
guarantees. When MLIR syntax changes in a backward-incompatible way,
the benchmark pipeline breaks.
2. Updates to the frontend compilation are not captured in real time.
They are only incorporated as part of the updates above, which means
that they are realized only when updating the MLIR files due to
syntax changes.
This adds a dependency on the TFLIte frontend to benchmark generation.
In the automated pipeline, the additional Bazel compilation uses the
remote cache and should therefore usually be fast (as it shares the
cache with the normal CI run). In local generation, users will have to
either build `iree-import-tflite` from source or fetch it as part of a
release. Given that we would like to move all benchmarks to originate
from stable formats, I've made trying to run a TFLite benchmark without
`IREE_BUILD_TFLITE_COMPILER` set an error rather than a warning that
skips the benchmark. This explicit check just fails slightly earlier
and more clearly, as the `TARGET_FILE` lookup for `iree-import-tflite`
would fail subsequently.
diff --git a/benchmarks/TFLite/CMakeLists.txt b/benchmarks/TFLite/CMakeLists.txt
index 358f117..6576533 100644
--- a/benchmarks/TFLite/CMakeLists.txt
+++ b/benchmarks/TFLite/CMakeLists.txt
@@ -22,20 +22,8 @@
TAGS
"fp32"
SOURCE
- "https://storage.googleapis.com/iree-model-artifacts/DeepLabV3-2bcafb1.mlir.gz"
- ENTRY_FUNCTION
- "main"
- FUNCTION_INPUTS
- "1x257x257x3xf32"
-)
-
-set(DEEPLABV3_FP32_TFLITE_MODULE
- NAME
- "DeepLabV3"
- TAGS
- "fp32,tflitefb"
- SOURCE
- "https://storage.googleapis.com/iree-model-artifacts/deeplabv3.tflite.gz"
+ # No significant compression of tflite files by gzip
+ "https://storage.googleapis.com/iree-model-artifacts/deeplabv3.tflite"
ENTRY_FUNCTION
"main"
FUNCTION_INPUTS
@@ -48,7 +36,8 @@
TAGS
"fp32"
SOURCE
- "https://storage.googleapis.com/iree-model-artifacts/MobileSSD-2bcafb1.mlir.gz"
+ # No significant compression of tflite files by gzip
+ "https://storage.googleapis.com/iree-model-artifacts/mobile_ssd_v2_float_coco.tflite"
ENTRY_FUNCTION
"main"
FUNCTION_INPUTS
@@ -61,7 +50,8 @@
TAGS
"fp32"
SOURCE
- "https://storage.googleapis.com/iree-model-artifacts/PoseNet-2bcafb1.mlir.gz"
+ # No significant compression of tflite files by gzip
+ "https://storage.googleapis.com/iree-model-artifacts/posenet.tflite"
ENTRY_FUNCTION
"main"
FUNCTION_INPUTS
@@ -82,7 +72,6 @@
iree_benchmark_suite(
MODULES
"${DEEPLABV3_FP32_MODULE}"
- "${DEEPLABV3_FP32_TFLITE_MODULE}"
"${MOBILESSD_FP32_MODULE}"
"${POSENET_FP32_MODULE}"
@@ -106,7 +95,6 @@
iree_benchmark_suite(
MODULES
"${DEEPLABV3_FP32_MODULE}"
- "${DEEPLABV3_FP32_TFLITE_MODULE}"
"${MOBILESSD_FP32_MODULE}"
"${POSENET_FP32_MODULE}"
@@ -132,7 +120,6 @@
iree_benchmark_suite(
MODULES
"${DEEPLABV3_FP32_MODULE}"
- "${DEEPLABV3_FP32_TFLITE_MODULE}"
"${MOBILESSD_FP32_MODULE}"
"${POSENET_FP32_MODULE}"
@@ -155,7 +142,6 @@
iree_benchmark_suite(
MODULES
"${DEEPLABV3_FP32_MODULE}"
- "${DEEPLABV3_FP32_TFLITE_MODULE}"
"${MOBILESSD_FP32_MODULE}"
"${POSENET_FP32_MODULE}"
@@ -178,7 +164,6 @@
iree_benchmark_suite(
MODULES
"${DEEPLABV3_FP32_MODULE}"
- "${DEEPLABV3_FP32_TFLITE_MODULE}"
"${MOBILESSD_FP32_MODULE}"
"${POSENET_FP32_MODULE}"
diff --git a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark2.yml b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark2.yml
index 80042a2..a6e8e35 100644
--- a/build_tools/buildkite/cmake/android/arm64-v8a/benchmark2.yml
+++ b/build_tools/buildkite/cmake/android/arm64-v8a/benchmark2.yml
@@ -9,7 +9,7 @@
steps:
- label: "Build"
commands:
- - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/android@sha256:58adb4131cfc7b08cd5767c577420f3479ca2f46bb67d9fac6c0797984627758 build_tools/cmake/build_android_benchmark.sh"
+ - "docker run --user=$(id -u):$(id -g) --volume=\\${HOME?}:\\${HOME?} --volume=/etc/passwd:/etc/passwd:ro --volume=/etc/group:/etc/group:ro --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/frontends@sha256:58adb4131cfc7b08cd5767c577420f3479ca2f46bb67d9fac6c0797984627758 build_tools/cmake/build_android_benchmark.sh"
- "tar --exclude='*.tar.gz' --exclude='*.tgz' --exclude='*.mlir' -czvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz build-host/benchmark_suites"
- "tar -czvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz build-android/iree/tools/iree-benchmark-module build-android-trace/iree/tools/iree-benchmark-module"
if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark')"
diff --git a/build_tools/cmake/build_android.sh b/build_tools/cmake/build_android.sh
index dcf07f4..2f52513 100755
--- a/build_tools/cmake/build_android.sh
+++ b/build_tools/cmake/build_android.sh
@@ -11,8 +11,7 @@
# Host binaries (e.g. compiler tools) will be built and installed in build-host/
# Android binaries (e.g. tests) will be built in build-android/.
-set -x
-set -e
+set -xeuo pipefail
if [ "$#" -ne 1 ]; then
echo "usage: $0 <android-abi>"
@@ -21,18 +20,31 @@
ANDROID_ABI=$1
-ROOT_DIR=$(git rev-parse --show-toplevel)
-
CMAKE_BIN=${CMAKE_BIN:-$(which cmake)}
-"${CMAKE_BIN?}" --version
+"${CMAKE_BIN}" --version
+"${CC}" --version
+"${CXX}" --version
ninja --version
+echo "Android NDK path: ${ANDROID_NDK}"
-cd ${ROOT_DIR?}
+ROOT_DIR=$(git rev-parse --show-toplevel)
+cd ${ROOT_DIR}
+
+# BUILD the iree-import-tflite binary for importing models to benchmark from
+# TFLite flatbuffers.
+cd "${ROOT_DIR}/integrations/tensorflow"
+BAZEL_CMD=(bazel --noworkspace_rc --bazelrc=build_tools/bazel/iree-tf.bazelrc)
+BAZEL_BINDIR="$(${BAZEL_CMD[@]} info bazel-bin)"
+"${BAZEL_CMD[@]}" build //iree_tf_compiler:iree-import-tflite \
+ --config=generic_clang \
+ --config=remote_cache_tf_integrations
# --------------------------------------------------------------------------- #
# Build for the host.
+cd "${ROOT_DIR}"
+
if [ -d "build-host" ]
then
echo "build-host directory already exists. Will use cached results there."
@@ -43,23 +55,24 @@
cd build-host
# Configure, build, install.
-"${CMAKE_BIN?}" -G Ninja .. \
+"${CMAKE_BIN}" -G Ninja .. \
-DCMAKE_INSTALL_PREFIX=./install \
-DIREE_ENABLE_ASSERTIONS=ON \
-DIREE_BUILD_COMPILER=ON \
-DIREE_BUILD_TESTS=OFF \
-DIREE_BUILD_BENCHMARKS=ON \
+ -DIREE_BUILD_TFLITE_COMPILER=ON \
-DIREE_BUILD_SAMPLES=OFF
-"${CMAKE_BIN?}" --build . --target install
+"${CMAKE_BIN}" --build . --target install
# Also make sure that we can generate artifacts for benchmarking on Android.
-"${CMAKE_BIN?}" --build . --target iree-benchmark-suites
+"${CMAKE_BIN}" --build . --target iree-benchmark-suites
# --------------------------------------------------------------------------- #
-cd ${ROOT_DIR?}
-
# --------------------------------------------------------------------------- #
# Build for the target (Android).
+cd ${ROOT_DIR}
+
if [ -d "build-android" ]
then
echo "build-android directory already exists. Will use cached results there."
@@ -70,13 +83,13 @@
cd build-android
# Configure towards 64-bit Android 10, then build.
-"${CMAKE_BIN?}" -G Ninja .. \
- -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK?}/build/cmake/android.toolchain.cmake \
- -DANDROID_ABI="${ANDROID_ABI?}" \
+"${CMAKE_BIN}" -G Ninja .. \
+ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \
+ -DANDROID_ABI="${ANDROID_ABI}" \
-DANDROID_PLATFORM=android-29 \
- -DIREE_HOST_BINARY_ROOT=$PWD/../build-host/install \
+ -DIREE_HOST_BINARY_ROOT="${PWD}/../build-host/install" \
-DIREE_ENABLE_ASSERTIONS=ON \
-DIREE_BUILD_COMPILER=OFF \
-DIREE_BUILD_TESTS=ON \
-DIREE_BUILD_SAMPLES=OFF
-"${CMAKE_BIN?}" --build .
+"${CMAKE_BIN}" --build .
diff --git a/build_tools/cmake/build_android_benchmark.sh b/build_tools/cmake/build_android_benchmark.sh
index f26287c..c9733aa 100755
--- a/build_tools/cmake/build_android_benchmark.sh
+++ b/build_tools/cmake/build_android_benchmark.sh
@@ -12,31 +12,39 @@
# Host binaries (e.g. compiler tools) will be built and installed in build-host/
# Android binaries (e.g. tests) will be built in build-android/.
-set -x
-set -e
+set -xeuo pipefail
# Print the UTC time when set -x is on.
export PS4='[$(date -u "+%T %Z")] '
# Check these exist and print the versions for later debugging.
-export CMAKE_BIN="$(which cmake)"
-"${CMAKE_BIN?}" --version
-"${CC?}" --version
-"${CXX?}" --version
+CMAKE_BIN=${CMAKE_BIN:-$(which cmake)}
+"${CMAKE_BIN}" --version
+"${CC}" --version
+"${CXX}" --version
ninja --version
python3 --version
-echo "Android NDK path: ${ANDROID_NDK?}"
+echo "Android NDK path: ${ANDROID_NDK}"
ROOT_DIR=$(git rev-parse --show-toplevel)
-cd ${ROOT_DIR?}
+cd "${ROOT_DIR}"
echo "Initializing submodules"
./scripts/git/submodule_versions.py init
+# BUILD the iree-import-tflite binary for importing models to benchmark from
+# TFLite flatbuffers.
+cd "${ROOT_DIR}/integrations/tensorflow"
+BAZEL_CMD=(bazel --noworkspace_rc --bazelrc=build_tools/bazel/iree-tf.bazelrc)
+BAZEL_BINDIR="$(${BAZEL_CMD[@]} info bazel-bin)"
+"${BAZEL_CMD[@]}" build //iree_tf_compiler:iree-import-tflite \
+ --config=generic_clang \
+ --config=remote_cache_tf_integrations
+
# --------------------------------------------------------------------------- #
# Build for the host.
-cd ${ROOT_DIR?}
+cd "${ROOT_DIR}"
if [ -d "build-host" ]
then
@@ -48,21 +56,22 @@
cd build-host
# Configure, build, install.
-"${CMAKE_BIN?}" -G Ninja .. \
+"${CMAKE_BIN}" -G Ninja .. \
-DCMAKE_INSTALL_PREFIX=./install \
-DIREE_BUILD_COMPILER=ON \
-DIREE_BUILD_TESTS=OFF \
-DIREE_BUILD_BENCHMARKS=ON \
+ -DIREE_BUILD_TFLITE_COMPILER=ON \
-DIREE_BUILD_SAMPLES=OFF
-"${CMAKE_BIN?}" --build . --target install
+"${CMAKE_BIN}" --build . --target install
# Also generate artifacts for benchmarking on Android.
-"${CMAKE_BIN?}" --build . --target iree-benchmark-suites
+"${CMAKE_BIN}" --build . --target iree-benchmark-suites
# --------------------------------------------------------------------------- #
# --------------------------------------------------------------------------- #
# Build for the target (Android).
-cd ${ROOT_DIR?}
+cd "${ROOT_DIR}"
if [ -d "build-android" ]
then
@@ -74,20 +83,20 @@
cd build-android
# Configure towards 64-bit Android 10, then build.
-"${CMAKE_BIN?}" -G Ninja .. \
- -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK?}/build/cmake/android.toolchain.cmake \
+"${CMAKE_BIN}" -G Ninja .. \
+ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \
-DANDROID_ABI=arm64-v8a \
-DANDROID_PLATFORM=android-29 \
- -DIREE_HOST_BINARY_ROOT=$PWD/../build-host/install \
+ -DIREE_HOST_BINARY_ROOT="${PWD}/../build-host/install" \
-DIREE_BUILD_COMPILER=OFF \
-DIREE_BUILD_TESTS=ON \
-DIREE_BUILD_SAMPLES=OFF
-"${CMAKE_BIN?}" --build . --target iree-benchmark-module
+"${CMAKE_BIN}" --build . --target iree-benchmark-module
# --------------------------------------------------------------------------- #
# Build for the target (Android) with tracing.
-cd ${ROOT_DIR?}
+cd "${ROOT_DIR}"
if [ -d "build-android-trace" ]
then
@@ -99,13 +108,13 @@
cd build-android-trace
# Configure towards 64-bit Android 10, then build.
-"${CMAKE_BIN?}" -G Ninja .. \
- -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK?}/build/cmake/android.toolchain.cmake \
+"${CMAKE_BIN}" -G Ninja .. \
+ -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \
-DANDROID_ABI=arm64-v8a \
-DANDROID_PLATFORM=android-29 \
- -DIREE_HOST_BINARY_ROOT=$PWD/../build-host/install \
+ -DIREE_HOST_BINARY_ROOT="${PWD}/../build-host/install" \
-DIREE_ENABLE_RUNTIME_TRACING=ON \
-DIREE_BUILD_COMPILER=OFF \
-DIREE_BUILD_TESTS=ON \
-DIREE_BUILD_SAMPLES=OFF
-"${CMAKE_BIN?}" --build . --target iree-benchmark-module
+"${CMAKE_BIN}" --build . --target iree-benchmark-module
diff --git a/build_tools/cmake/iree_benchmark_suite.cmake b/build_tools/cmake/iree_benchmark_suite.cmake
index 454fc62..a241954 100644
--- a/build_tools/cmake/iree_benchmark_suite.cmake
+++ b/build_tools/cmake/iree_benchmark_suite.cmake
@@ -132,9 +132,8 @@
# If the source is a TFLite file, import it.
if("${_MODULE_SOURCE}" MATCHES "\.tflite$")
if (NOT IREE_BUILD_TFLITE_COMPILER)
- message(WARNING "Skipping benchmark for ${_MODULE_SOURCE} because"
- " IREE_BUILD_TFLITE_COMPILER if OFF")
- continue()
+ message(SEND_ERROR "Benchmarks of ${_MODULE_SOURCE} require"
+ " IREE_BUILD_TFLITE_COMPILER to be ON")
endif()
set(_TFLITE_FILE "${_MODULE_SOURCE}")
set(_MODULE_SOURCE "${_TFLITE_FILE}.mlir")
@@ -204,6 +203,7 @@
COMMAND
"$<TARGET_FILE:iree::tools::iree-translate>"
${_TRANSLATION_ARGS}
+ "--mlir-print-op-on-diagnostic=false"
"${_MODULE_SOURCE}"
-o "${_VMFB_FILE}"
WORKING_DIRECTORY "${_VMFB_ARTIFACTS_DIR}"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/android/arm64-v8a/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake/android/arm64-v8a/build_kokoro.sh
index b1a092f..06595f7 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake/android/arm64-v8a/build_kokoro.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake/android/arm64-v8a/build_kokoro.sh
@@ -23,8 +23,10 @@
# Sets DOCKER_RUN_ARGS
docker_setup
+# Need to use frontends image (which also has Android toolchain) to build the
+# TFLite compiler for generating benchmarks.
docker run "${DOCKER_RUN_ARGS[@]?}" \
- gcr.io/iree-oss/android@sha256:58adb4131cfc7b08cd5767c577420f3479ca2f46bb67d9fac6c0797984627758 \
+ gcr.io/iree-oss/frontends@sha256:f9490f31ff09c2053a9972a002f6af65fcace41d834583754ea93c6df35ff1f4 \
build_tools/kokoro/gcp_ubuntu/cmake/android/build.sh arm64-v8a
# Kokoro will rsync this entire directory back to the executor orchestrating the
