| # Copyright 2021 The IREE Authors |
| # |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. |
| # See https://llvm.org/LICENSE.txt for license information. |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| |
| # Pipeline steps for https://buildkite.com/iree/iree-benchmark |
| |
| steps: |
| - label: "Build" |
| commands: | |
| docker run --user=$(id -u):$(id -g) \ |
| --volume="$${HOME?}:$${HOME?}" \ |
| --volume=/etc/passwd:/etc/passwd:ro \ |
| --volume=/etc/group:/etc/group:ro \ |
| --volume="$$PWD:$$IREE_DOCKER_WORKDIR" \ |
| --workdir="$$IREE_DOCKER_WORKDIR" \ |
| --rm \ |
| gcr.io/iree-oss/frontends@sha256:e1d45e60ab427717433e38310c8a04282c1d89e43f7b106b53e472d8d51966af \ |
| build_tools/cmake/build_android_benchmark.sh |
| tar --exclude='*.tar.gz' \ |
| --exclude='*.tgz' \ |
| --exclude='*.mlir' \ |
| --exclude='*.tflite' \ |
| --exclude='*tf-model' \ |
| -czvf \ |
| "benchmark-suites-$${BUILDKITE_BUILD_NUMBER}.tgz" \ |
| "build-host/benchmark_suites" \ |
| "build-host/.ninja_log" |
| find "build-host/benchmark_suites" -name '*.mlir' | \ |
| tar -czvf "source-mlir-models-$${BUILDKITE_BUILD_NUMBER}.tgz" -T - |
| tar -czvf "iree-android-tools-$${BUILDKITE_BUILD_NUMBER}.tgz" \ |
| "build-android/tools/iree-benchmark-module" \ |
| "build-android-trace/tools/iree-benchmark-module" \ |
| "build-android/tools/build_config.txt" |
| if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark' || build.pull_request.labels includes 'buildkite:benchmark-android')" |
| agents: |
| - "queue=build" |
| env: |
| IREE_DOCKER_WORKDIR: "/usr/src/github/iree" |
| artifact_paths: |
| - "benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz" |
| - "source-mlir-models-${BUILDKITE_BUILD_NUMBER}.tgz" |
| - "iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz" |
| |
| - wait |
| |
| - label: "Benchmark on Pixel 4 (snapdragon-855, adreno-640)" |
| commands: |
| - "git clean -fdx" |
| - "buildkite-agent artifact download --step Build benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz ./" |
| - "buildkite-agent artifact download --step Build iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz ./" |
| - "wget https://storage.googleapis.com/iree-shared-files/tracy-capture-52b6af88.tgz" |
| - "tar -xzvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz" |
| - "tar -xzvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz" |
| - "tar -xzvf tracy-capture-52b6af88.tgz" |
| - "python3 build_tools/benchmarks/run_benchmarks_on_android.py --pin-cpu-freq --pin-gpu-freq --normal_benchmark_tool_dir=build-android/tools/ --traced_benchmark_tool_dir=build-android-trace/tools/ --trace_capture_tool=tracy-capture -o benchmark-results-pixel-4-${BUILDKITE_BUILD_NUMBER}.json --capture_tarball=trace-captures-pixel-4-${BUILDKITE_BUILD_NUMBER}.tgz --driver_filter_regex='(?!vulkan).*' --verbose build-host/" |
| if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark' || build.pull_request.labels includes 'buildkite:benchmark-android')" |
| agents: |
| - "android-soc=snapdragon-855" |
| - "android-version=12" |
| - "queue=benchmark-android" |
| artifact_paths: |
| - "benchmark-results-pixel-4-${BUILDKITE_BUILD_NUMBER}.json" |
| - "trace-captures-pixel-4-${BUILDKITE_BUILD_NUMBER}.tgz" |
| timeout_in_minutes: "60" |
| |
| - label: "Benchmark on Pixel 6 Pro (google-tensor, mali-g78)" |
| commands: |
| - "git clean -fdx" |
| - "buildkite-agent artifact download --step Build benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz ./" |
| - "buildkite-agent artifact download --step Build iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz ./" |
| - "wget https://storage.googleapis.com/iree-shared-files/tracy-capture-52b6af88.tgz" |
| - "tar -xzvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz" |
| - "tar -xzvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz" |
| - "tar -xzvf tracy-capture-52b6af88.tgz" |
| - "python3 build_tools/benchmarks/run_benchmarks_on_android.py --pin-cpu-freq --pin-gpu-freq --normal_benchmark_tool_dir=build-android/tools/ --traced_benchmark_tool_dir=build-android-trace/tools/ --trace_capture_tool=tracy-capture -o benchmark-results-pixel6-pro-${BUILDKITE_BUILD_NUMBER}.json --capture_tarball=trace-captures-pixel6-pro-${BUILDKITE_BUILD_NUMBER}.tgz --verbose build-host/" |
| if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark' || build.pull_request.labels includes 'buildkite:benchmark-android')" |
| agents: |
| - "android-soc=google-tensor" |
| - "android-version=12" |
| - "queue=benchmark-android" |
| artifact_paths: |
| - "benchmark-results-pixel6-pro-${BUILDKITE_BUILD_NUMBER}.json" |
| - "trace-captures-pixel6-pro-${BUILDKITE_BUILD_NUMBER}.tgz" |
| timeout_in_minutes: "60" |
| |
| - label: "Benchmark on Moto Edge X30 (snapdragon-8gen1, adreno-730)" |
| commands: |
| - "git clean -fdx" |
| - "buildkite-agent artifact download --step Build benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz ./" |
| - "buildkite-agent artifact download --step Build iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz ./" |
| - "wget https://storage.googleapis.com/iree-shared-files/tracy-capture-52b6af88.tgz" |
| - "tar -xzvf benchmark-suites-${BUILDKITE_BUILD_NUMBER}.tgz" |
| - "tar -xzvf iree-android-tools-${BUILDKITE_BUILD_NUMBER}.tgz" |
| - "tar -xzvf tracy-capture-52b6af88.tgz" |
| - "python3 build_tools/benchmarks/run_benchmarks_on_android.py --pin-cpu-freq --pin-gpu-freq --normal_benchmark_tool_dir=build-android/tools/ --traced_benchmark_tool_dir=build-android-trace/tools/ --trace_capture_tool=tracy-capture -o benchmark-results-moto-edge-x30-${BUILDKITE_BUILD_NUMBER}.json --capture_tarball=trace-captures-moto-edge-x30-${BUILDKITE_BUILD_NUMBER}.tgz --driver_filter_regex='vulkan' --verbose build-host/" |
| if: "build.pull_request.id == null || (build.pull_request.labels includes 'buildkite:benchmark' || build.pull_request.labels includes 'buildkite:benchmark-android')" |
| agents: |
| - "android-soc=snapdragon-8gen1" |
| - "android-version=12" |
| - "queue=benchmark-android" |
| artifact_paths: |
| - "benchmark-results-moto-edge-x30-${BUILDKITE_BUILD_NUMBER}.json" |
| - "trace-captures-moto-edge-x30-${BUILDKITE_BUILD_NUMBER}.tgz" |
| timeout_in_minutes: "60" |
| |
| - wait |
| |
| - label: "Comment benchmark results on pull request" |
| commands: | |
| git clean -fdx |
| # Fetch base branch for comparison. |
| git fetch -v --prune --no-recurse-submodules -- origin \ |
| "$${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" |
| buildkite-agent artifact download "benchmark-results-*.json" ./ |
| # The script will handle the wildcard patterns. |
| # --comment_type_id is a random-generated id and shouldn't be changed ( |
| # even we change the title); otherwise the script will post a new comment |
| # instead of updating the existing one. |
| python3 build_tools/benchmarks/generate_benchmark_comment.py \ |
| --verbose \ |
| --benchmark_files="benchmark-results-*.json" \ |
| --pr_number="$${BUILDKITE_PULL_REQUEST}" \ |
| --pr_committish="$${BUILDKITE_COMMIT}" \ |
| --pr_base_branch="origin/$${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" \ |
| --build_url="$${BUILDKITE_BUILD_URL}" \ |
| --comment_type="android-benchmark-summary" \ |
| --output="benchmark-comment.json" |
| export COMMENT_BOT_USER="iree-github-actions-bot" |
| export GIST_BOT_TOKEN="$${GITHUB_TOKEN}" |
| python3 build_tools/benchmarks/post_benchmark_comment.py \ |
| --verbose \ |
| --no_verify_pr \ |
| "benchmark-comment.json" |
| key: "post-on-pr" |
| if: "build.pull_request.id != null && (build.pull_request.labels includes 'buildkite:benchmark' || build.pull_request.labels includes 'buildkite:benchmark-android')" |
| agents: |
| - "queue=report" |
| |
| - label: "Push benchmark results to dashboard" |
| commands: | |
| git clean -fdx |
| buildkite-agent artifact download benchmark-results-*.json ./ |
| python3 build_tools/benchmarks/upload_benchmarks_to_dashboard.py \ |
| --verbose \ |
| --benchmark_files benchmark-results-*.json |
| key: "upload-to-dashboard" |
| branches: "main" |
| agents: |
| - "queue=report" |
