build_tools/benchmarks/comparisons/run_benchmarks.py

# Copyright 2022 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
""" Runs benchmarks and saves results to a .csv file

Expects a directory structure of:
<root-benchmark-dir>/
  └── ./benchmark_model (TFLite benchmark binary)
      ./iree-benchmark-module (IREE benchmark binary)
  ├── setup/
        ├── set_adreno_gpu_scaling_policy.sh
        ├── set_android_scaling_governor.sh
        └── set_pixel6_gpu_scaling_policy.sh
  ├── test_data/
  └── models/
        ├── tflite/*.tflite
        └── iree/
              └── <target>/*.vmfb e.g. llvm-cpu, vulkan, cuda.

"""

import argparse
import os

from common.benchmark_runner import *
from common.utils import *
from mobilebert_fp32_commands import *
from mobilebert_int8_commands import *
from simple_commands import *


def benchmark_desktop_cpu(
    device_name: str,
    command_factories: list[BenchmarkCommandFactory],
    results_path: str,
):
    benchmarks = []
    for factory in command_factories:
        benchmarks.extend(factory.generate_benchmark_commands("desktop", "cpu"))

    for num_threads in [1, 2, 4, 8]:
        for benchmark in benchmarks:
            results_array = [
                device_name,
                benchmark.model_name,
                benchmark.runtime,
                benchmark.driver,
                num_threads,
            ]
            benchmark.num_threads = num_threads
            results_array.extend(run_command(benchmark))
            write_benchmark_result(results_array, results_path)


def benchmark_desktop_gpu(
    device_name: str,
    command_factories: list[BenchmarkCommandFactory],
    results_path: str,
):
    benchmarks = []
    for factory in command_factories:
        benchmarks.extend(factory.generate_benchmark_commands("desktop", "gpu"))
    for benchmark in benchmarks:
        results_array = [
            device_name,
            benchmark.model_name,
            benchmark.runtime,
            benchmark.driver,
            benchmark.num_threads,
        ]
        results_array.extend(run_command(benchmark))
        write_benchmark_result(results_array, results_path)


def benchmark_mobile_cpu(
    device_name: str,
    command_factories: list[BenchmarkCommandFactory],
    results_path: str,
):
    benchmarks = []
    for factory in command_factories:
        benchmarks.extend(factory.generate_benchmark_commands("mobile", "cpu"))

    for _, tuple in enumerate([("80", 1), ("C0", 2), ("F0", 4), ("0F", 4), ("FF", 8)]):
        taskset = tuple[0]
        num_threads = tuple[1]
        for benchmark in benchmarks:
            results_array = [
                device_name,
                benchmark.model_name,
                benchmark.runtime,
                benchmark.driver,
                taskset,
                num_threads,
            ]
            benchmark.taskset = taskset
            benchmark.num_threads = num_threads
            results_array.extend(run_command(benchmark))
            write_benchmark_result(results_array, results_path)


def benchmark_mobile_gpu(
    device_name: str,
    command_factories: list[BenchmarkCommandFactory],
    results_path: str,
):
    benchmarks = []
    for factory in command_factories:
        benchmarks.extend(factory.generate_benchmark_commands("mobile", "gpu"))

    taskset = "80"
    num_threads = 1
    for benchmark in benchmarks:
        results_array = [
            device_name,
            benchmark.model_name,
            benchmark.runtime,
            benchmark.driver,
            taskset,
            num_threads,
        ]
        benchmark.taskset = taskset
        benchmark.num_threads = num_threads
        results_array.extend(run_command(benchmark))
        write_benchmark_result(results_array, results_path)


def main(args):
    # Create factories for all models to be benchmarked.
    command_factory = []
    command_factory.append(
        MobilebertFP32CommandFactory(args.base_dir, "mobilebert_float_384_gpu")
    )
    command_factory.append(MobilebertInt8CommandFactory(args.base_dir))
    command_factory.append(
        MobilebertFP32CommandFactory(args.base_dir, "albert_lite_base_squadv1_1")
    )
    command_factory.append(
        SimpleCommandFactory(args.base_dir, "mobilenet_v2_1.0_224", "1x224x224x3xf32")
    )
    command_factory.append(
        SimpleCommandFactory(
            args.base_dir, "mobilenet_v2_224_1.0_uint8", "1x224x224x3xui8"
        )
    )
    command_factory.append(
        SimpleCommandFactory(args.base_dir, "deeplabv3", "1x257x257x3xf32")
    )
    command_factory.append(
        SimpleCommandFactory(args.base_dir, "person_detect", "1x96x96x1xi8")
    )
    command_factory.append(
        SimpleCommandFactory(
            args.base_dir, "ssd_mobilenet_v2_static_1.0_int8", "1x320x320x3xi8"
        )
    )
    command_factory.append(
        SimpleCommandFactory(
            args.base_dir, "resnet_v2_101_1_default_1", "1x299x299x3xf32"
        )
    )
    command_factory.append(
        SimpleCommandFactory(
            args.base_dir, "ssd_mobilenet_v2_fpnlite_uint8", "1x320x320x3xui8"
        )
    )
    command_factory.append(
        SimpleCommandFactory(
            args.base_dir, "ssd_mobilenet_v2_fpnlite_fp32", "1x320x320x3xf32"
        )
    )
    command_factory.append(
        SimpleCommandFactory(
            args.base_dir, "efficientnet_lite0_int8_2", "1x224x224x3xui8"
        )
    )
    command_factory.append(
        SimpleCommandFactory(
            args.base_dir, "efficientnet_lite0_fp32_2", "1x224x224x3xf32"
        )
    )
    command_factory.append(
        SimpleCommandFactory(args.base_dir, "inception_v4_299_uint8", "1x299x299x3xui8")
    )
    command_factory.append(
        SimpleCommandFactory(args.base_dir, "inception_v4_299_fp32", "1x299x299x3xf32")
    )

    if args.mode == "desktop":
        results_path = os.path.join(args.output_dir, "results.csv")
        with open(results_path, "w") as f:
            f.write(
                "device,model,runtime,driver/delegate,threads,latency (ms),vmhwm (KB),vmrss (KB),rssfile (KB)\n"
            )

        if not args.disable_cpu:
            benchmark_desktop_cpu(args.device_name, command_factory, results_path)
        if not args.disable_gpu:
            benchmark_desktop_gpu(args.device_name, command_factory, results_path)
    else:
        assert args.mode == "mobile"
        results_path = os.path.join(args.output_dir, "results.csv")
        with open(results_path, "w") as f:
            f.write(
                "device,model,runtime,driver/delegate,taskset,threads,latency (ms),vmhwm (KB),vmrss (KB),rssfile (KB)\n"
            )
        if not args.disable_cpu:
            benchmark_mobile_cpu(args.device_name, command_factory, results_path)
        if not args.disable_gpu:
            benchmark_mobile_gpu(args.device_name, command_factory, results_path)


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--device_name",
        type=str,
        default=None,
        help="The name of the device the benchmark is running on e.g. Pixel 6",
    )
    parser.add_argument(
        "--base_dir",
        type=str,
        default=None,
        help="The directory where all benchmarking artifacts are located.",
    )
    parser.add_argument(
        "--output_dir",
        type=str,
        default=None,
        help="The directory to save output artifacts into.",
    )
    parser.add_argument(
        "--mode",
        type=str,
        choices=("desktop", "mobile"),
        default="desktop",
        help="The benchmarking mode to use. If mode is `mobile`, uses tasksets.",
    )
    parser.add_argument(
        "--disable_cpu", action="store_true", help="Disables running benchmarks on CPU."
    )
    parser.add_argument(
        "--disable_gpu", action="store_true", help="Disables running benchmarks on GPU."
    )
    return parser.parse_args()


if __name__ == "__main__":
    main(parse_args())