More benchmark android improvements (#8600)
* consistently sanitize paths and don't mutate variables
also, honor --verbose on adb forward commands
* make normal_benchmark_tool_dir optional (support capture-only flow)
* Require --capture_tarball when tracing
diff --git a/build_tools/benchmarks/run_benchmarks_on_android.py b/build_tools/benchmarks/run_benchmarks_on_android.py
index c7e12c5..a5a9209 100755
--- a/build_tools/benchmarks/run_benchmarks_on_android.py
+++ b/build_tools/benchmarks/run_benchmarks_on_android.py
@@ -228,7 +228,7 @@
benchmark_category_dir: str,
benchmark_case_dirs: Sequence[str],
tmp_dir: str,
- normal_benchmark_tool_dir: str,
+ normal_benchmark_tool_dir: Optional[str] = None,
traced_benchmark_tool_dir: Optional[str] = None,
trace_capture_tool: Optional[str] = None,
skip_benchmarks: Optional[Set[str]] = None,
@@ -294,9 +294,10 @@
# Read the file specifying which tool should be used for benchmarking
with open(os.path.join(benchmark_case_dir, MODEL_TOOLFILE_NAME)) as f:
tool = f.read().strip()
- adb_push_to_tmp_dir(os.path.join(normal_benchmark_tool_dir, tool),
- relative_dir=NORMAL_TOOL_REL_DIR,
- verbose=verbose)
+ if normal_benchmark_tool_dir:
+ adb_push_to_tmp_dir(os.path.join(normal_benchmark_tool_dir, tool),
+ relative_dir=NORMAL_TOOL_REL_DIR,
+ verbose=verbose)
if do_capture:
adb_push_to_tmp_dir(os.path.join(traced_benchmark_tool_dir, tool),
relative_dir=TRACED_TOOL_REL_DIR,
@@ -322,7 +323,7 @@
verbose=verbose)
benchmark_result_filename = None
- if benchmark_key not in skip_benchmarks:
+ if normal_benchmark_tool_dir and benchmark_key not in skip_benchmarks:
benchmark_results_basename = f"{benchmark_key}.json"
cmd = [
@@ -454,7 +455,7 @@
model_name_filter: Optional[str],
mode_filter: Optional[str],
tmp_dir: str,
- normal_benchmark_tool_dir: str,
+ normal_benchmark_tool_dir: Optional[str],
traced_benchmark_tool_dir: Optional[str],
trace_capture_tool: Optional[str],
skip_benchmarks: Optional[Set[str]],
@@ -511,8 +512,9 @@
for directory in sorted(os.listdir(root_benchmark_dir)):
benchmark_category_dir = os.path.join(root_benchmark_dir, directory)
- available_drivers = get_available_drivers(
- tool_dir=normal_benchmark_tool_dir, verbose=verbose)
+ any_tool_dir = normal_benchmark_tool_dir if normal_benchmark_tool_dir else traced_benchmark_tool_dir
+ available_drivers = get_available_drivers(tool_dir=any_tool_dir,
+ verbose=verbose)
matched_benchmarks = filter_benchmarks_for_category(
benchmark_category_dir=benchmark_category_dir,
cpu_target_arch_filter=cpu_target_arch,
@@ -592,11 +594,12 @@
metavar="<build-dir>",
type=check_dir_path,
help="Path to the build directory containing benchmark suites")
- parser.add_argument("--normal_benchmark_tool_dir",
- "--normal-benchmark-tool-dir",
- type=check_exe_path,
- required=True,
- help="Path to the normal iree tool directory")
+ parser.add_argument(
+ "--normal_benchmark_tool_dir",
+ "--normal-benchmark-tool-dir",
+ type=check_exe_path,
+ default=None,
+ help="Path to the normal (non-tracing) iree tool directory")
parser.add_argument("--traced_benchmark_tool_dir",
"--traced-benchmark-tool-dir",
type=check_exe_path,
@@ -687,11 +690,28 @@
return args
+def real_path_or_none(path: str) -> Optional[str]:
+ return os.path.realpath(path) if path else None
+
+
def main(args):
device_info = AndroidDeviceInfo.from_adb()
if args.verbose:
print(device_info)
+ if not args.normal_benchmark_tool_dir and not args.traced_benchmark_tool_dir:
+ raise ValueError(
+ "At least one of --normal_benchmark_tool_dir or --traced_benchmark_tool_dir should be specified."
+ )
+
+ do_capture = args.traced_benchmark_tool_dir is not None
+ if ((args.traced_benchmark_tool_dir is not None) != do_capture) or (
+ (args.trace_capture_tool is not None) != do_capture) or (
+ (args.capture_tarball is not None) != do_capture):
+ raise ValueError(
+ "The following 3 flags should be simultaneously all specified or all unspecified: --traced_benchmark_tool_dir, --trace_capture_tool, --capture_tarball"
+ )
+
if device_info.cpu_abi.lower() not in CPU_ABI_TO_TARGET_ARCH_MAP:
raise ValueError(f"Unrecognized CPU ABI: '{device_info.cpu_abi}'; "
"need to update the map")
@@ -709,9 +729,6 @@
previous_benchmarks = None
previous_captures = None
- do_capture = (args.traced_benchmark_tool_dir is not None and
- args.trace_capture_tool is not None)
-
# Collect names of previous benchmarks and captures that should be skipped and
# merged into the results.
if args.continue_from_directory is not None:
@@ -743,13 +760,11 @@
# Tracy client and server communicate over port 8086 by default. If we want
# to capture traces along the way, forward port via adb.
if do_capture:
- execute_cmd_and_get_output(["adb", "forward", "tcp:8086", "tcp:8086"])
+ execute_cmd_and_get_output(["adb", "forward", "tcp:8086", "tcp:8086"],
+ verbose=args.verbose)
atexit.register(execute_cmd_and_get_output,
- ["adb", "forward", "--remove", "tcp:8086"])
-
- args.traced_benchmark_tool_dir = os.path.realpath(
- args.traced_benchmark_tool_dir)
- args.trace_capture_tool = os.path.realpath(args.trace_capture_tool)
+ ["adb", "forward", "--remove", "tcp:8086"],
+ verbose=args.verbose)
results = BenchmarkResults()
commit = get_git_commit_hash("HEAD")
@@ -765,10 +780,11 @@
model_name_filter=args.model_name_regex,
mode_filter=args.mode_regex,
tmp_dir=args.tmp_dir,
- normal_benchmark_tool_dir=os.path.realpath(
+ normal_benchmark_tool_dir=real_path_or_none(
args.normal_benchmark_tool_dir),
- traced_benchmark_tool_dir=args.traced_benchmark_tool_dir,
- trace_capture_tool=args.trace_capture_tool,
+ traced_benchmark_tool_dir=real_path_or_none(
+ args.traced_benchmark_tool_dir),
+ trace_capture_tool=real_path_or_none(args.trace_capture_tool),
skip_benchmarks=previous_benchmarks,
skip_captures=previous_captures,
do_capture=do_capture,
