Avoid running long benchmarks as tests (#8176)
The same justification as
https://github.com/google/iree/commit/48896123d, but this time I went
to the extra trouble of creating a rule, so hopefully that will reduce
the overhead and people will use this for new benchmarks. I frequently
run all tests, so these really slow ones are painful.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e1a705e..a5244fa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -217,6 +217,7 @@
include(iree_check_test)
include(iree_trace_runner_test)
include(iree_native_test)
+include(iree_cc_binary_benchmark)
include(iree_benchmark_suite)
include(iree_hal_cts_test_suite)
diff --git a/build_tools/bazel/cc_binary_benchmark.bzl b/build_tools/bazel/cc_binary_benchmark.bzl
new file mode 100644
index 0000000..18fdd6c
--- /dev/null
+++ b/build_tools/bazel/cc_binary_benchmark.bzl
@@ -0,0 +1,57 @@
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Creates a binary and a test for a cc benchmark target.
+
+It's good to test that benchmarks run, but it's really annoying to run a billion
+iterations of them every time you try to run tests. So we create these as
+binaries and then invoke them as tests with `--benchmark_min_time=0`.
+"""
+
+load(":native_binary.bzl", "native_test")
+
+def cc_binary_benchmark(
+ name,
+ srcs = None,
+ data = None,
+ deps = None,
+ copts = None,
+ defines = None,
+ linkopts = None,
+ tags = None,
+ testonly = True,
+ size = "small",
+ timeout = None,
+ **kwargs):
+ """Creates a binary and a test for a cc benchmark target.
+
+ Arguments passed to the binary target:
+ name, srcs, data, deps, copts, defines, linkopts, tags, testonly, **kwargs
+ Arguments passed to the test target:
+ {name}_test, tags, size, timeout, **kwargs
+ """
+ native.cc_binary(
+ name = name,
+ srcs = srcs,
+ data = data,
+ deps = deps,
+ copts = copts,
+ defines = defines,
+ linkopts = linkopts,
+ tags = tags,
+ testonly = testonly,
+ **kwargs
+ )
+
+ native_test(
+ name = "{}_test".format(name),
+ src = name,
+ size = size,
+ tags = tags,
+ timeout = timeout,
+ args = ["--benchmark_min_time=0"],
+ **kwargs
+ )
diff --git a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
index c875d87..9553676 100644
--- a/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
+++ b/build_tools/bazel_to_cmake/bazel_to_cmake_converter.py
@@ -735,6 +735,43 @@
f"{labels_block}"
f")\n\n")
+ def cc_binary_benchmark(
+ self,
+ name,
+ srcs=None,
+ data=None,
+ deps=None,
+ copts=None,
+ defines=None,
+ linkopts=None,
+ tags=None,
+ testonly=True,
+ # unused
+ size="small",
+ timeout=None):
+
+ name_block = _convert_string_arg_block("NAME", name, quote=False)
+ srcs_block = _convert_srcs_block(srcs)
+ data_block = _convert_target_list_block("DATA", data)
+ deps_block = _convert_target_list_block("DEPS", deps)
+ copts_block = _convert_string_list_block("COPTS", copts, sort=False)
+ defines_block = _convert_string_list_block("DEFINES", defines)
+ defines_block = _convert_string_list_block("LINKOPTS", linkopts)
+ testonly_block = _convert_option_block("TESTONLY", testonly)
+ labels_block = _convert_string_list_block("LABELS", tags)
+
+ self.converter.body += (f"iree_cc_binary_benchmark(\n"
+ f"{name_block}"
+ f"{srcs_block}"
+ f"{data_block}"
+ f"{deps_block}"
+ f"{copts_block}"
+ f"{defines_block}"
+ f"{defines_block}"
+ f"{testonly_block}"
+ f"{labels_block}"
+ f")\n\n")
+
def iree_cmake_extra_content(self, content, inline=False):
if inline:
self.converter.body += (f"\n{content}\n")
diff --git a/build_tools/cmake/iree_cc_binary_benchmark.cmake b/build_tools/cmake/iree_cc_binary_benchmark.cmake
new file mode 100644
index 0000000..03eacda
--- /dev/null
+++ b/build_tools/cmake/iree_cc_binary_benchmark.cmake
@@ -0,0 +1,84 @@
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+include(CMakeParseArguments)
+
+# iree_cc_binary_benchmarks()
+#
+# Creates a binary and a test for a cc benchmark target.
+#
+# It's good to test that benchmarks run, but it's really annoying to run a
+# billion iterations of them every time you try to run tests. So we create these
+# as binaries and then invoke them as tests with `--benchmark_min_time=0`.
+#
+# Mirrors the bzl function of the same name. See iree_cc_binary and iree_cc_test
+# for more details on those rules
+#
+# Parameters:
+# NAME: name for the binary target. The test target will be "${NAME}_test"
+# SRCS: List of source files for the binary
+# DATA: List of other targets and files required for the binary
+# DEPS: List of other libraries to be linked in to the binary
+# COPTS: List of private compile options for the binary
+# DEFINES: List of public defines for the binary
+# LINKOPTS: List of link options for the binary
+# TESTONLY: whether the binary should only be compiled for tests
+# HOSTONLY: whether the binary should be compiled using host toolchain when
+# cross-compiling
+# LABELS: Additional labels to apply to the test. The package path is added
+# automatically.
+#
+# )
+
+function(iree_cc_binary_benchmark)
+
+ cmake_parse_arguments(
+ _RULE
+ "HOSTONLY;TESTONLY"
+ "NAME"
+ "SRCS;COPTS;DEFINES;LINKOPTS;DATA;DEPS;LABELS"
+ ${ARGN}
+ )
+
+ set(_MAYBE_TESTONLY "")
+ if (_RULE_TESTONLY)
+ set(_MAYBE_TESTONLY "TESTONLY")
+ endif()
+
+ set(_MAYBE_HOSTONLY "")
+ if (_RULE_HOSTONLY)
+ set(_MAYBE_HOSTONLY "HOSTONLY")
+ endif()
+
+ iree_cc_binary(
+ NAME
+ ${_RULE_NAME}
+ SRCS
+ ${_RULE_SRCS}
+ DEPS
+ ${_RULE_DEPS}
+ COPTS
+ ${_RULE_COPTS}
+ DEFINES
+ ${_RULE_DEFINES}
+ LINKOPTS
+ ${_RULE_LINKOPTS}
+ ${_MAYBE_TESTONLY}
+ ${_MAYBE_HOSTONLY}
+ )
+
+
+ iree_native_test(
+ NAME
+ ${_RULE_NAME}_test
+ ARGS
+ "--benchmark_min_time=0"
+ SRC
+ ::${_RULE_NAME}
+ LABELS
+ ${_RULE_LABELS}
+ )
+endfunction()
diff --git a/iree/base/internal/BUILD b/iree/base/internal/BUILD
index 30fb9e3..bf0ce3c 100644
--- a/iree/base/internal/BUILD
+++ b/iree/base/internal/BUILD
@@ -8,9 +8,9 @@
# These are not part of the IREE API. Though they may be used by external
# projects their API may change at any time.
-load("//build_tools/bazel:native_binary.bzl", "native_test")
load("//iree:build_defs.oss.bzl", "iree_cmake_extra_content")
load("//build_tools/bazel:iree_lit_test.bzl", "iree_lit_test_suite")
+load("//build_tools/bazel:cc_binary_benchmark.bzl", "cc_binary_benchmark")
package(
default_visibility = ["//visibility:public"],
@@ -209,7 +209,7 @@
],
)
-cc_test(
+cc_binary_benchmark(
name = "fpu_state_benchmark",
srcs = ["fpu_state_benchmark.cc"],
deps = [
@@ -283,7 +283,7 @@
],
)
-cc_binary(
+cc_binary_benchmark(
name = "synchronization_benchmark",
testonly = True,
srcs = ["synchronization_benchmark.cc"],
@@ -294,12 +294,6 @@
],
)
-native_test(
- name = "synchronization_benchmark_test",
- src = ":synchronization_benchmark",
- args = ["--benchmark_min_time=0"],
-)
-
cc_test(
name = "synchronization_test",
srcs = ["synchronization_test.cc"],
diff --git a/iree/base/internal/CMakeLists.txt b/iree/base/internal/CMakeLists.txt
index 65119e5..bb4a0a1 100644
--- a/iree/base/internal/CMakeLists.txt
+++ b/iree/base/internal/CMakeLists.txt
@@ -214,7 +214,7 @@
PUBLIC
)
-iree_cc_test(
+iree_cc_binary_benchmark(
NAME
fpu_state_benchmark
SRCS
@@ -224,6 +224,7 @@
benchmark
iree::base
iree::testing::benchmark_main
+ TESTONLY
)
iree_cc_test(
@@ -296,7 +297,7 @@
PUBLIC
)
-iree_cc_binary(
+iree_cc_binary_benchmark(
NAME
synchronization_benchmark
SRCS
@@ -308,15 +309,6 @@
TESTONLY
)
-iree_native_test(
- NAME
- "synchronization_benchmark_test"
- ARGS
- "--benchmark_min_time=0"
- SRC
- ::synchronization_benchmark
-)
-
iree_cc_test(
NAME
synchronization_test
diff --git a/iree/builtins/device/tools/BUILD b/iree/builtins/device/tools/BUILD
index bbb357c..67b708d 100644
--- a/iree/builtins/device/tools/BUILD
+++ b/iree/builtins/device/tools/BUILD
@@ -4,13 +4,15 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+load("//build_tools/bazel:cc_binary_benchmark.bzl", "cc_binary_benchmark")
+
package(
default_visibility = ["//visibility:public"],
features = ["layering_check"],
licenses = ["notice"], # Apache 2.0
)
-cc_test(
+cc_binary_benchmark(
name = "libdevice_benchmark",
srcs = ["libdevice_benchmark.c"],
deps = [
diff --git a/iree/builtins/device/tools/CMakeLists.txt b/iree/builtins/device/tools/CMakeLists.txt
index 1f3c2ac..e445a9b 100644
--- a/iree/builtins/device/tools/CMakeLists.txt
+++ b/iree/builtins/device/tools/CMakeLists.txt
@@ -10,7 +10,7 @@
iree_add_all_subdirs()
-iree_cc_test(
+iree_cc_binary_benchmark(
NAME
libdevice_benchmark
SRCS
@@ -20,6 +20,7 @@
iree::base::internal::flags
iree::builtins::device
iree::testing::benchmark
+ TESTONLY
)
iree_cc_test(
diff --git a/iree/hal/local/BUILD b/iree/hal/local/BUILD
index 7b06009..0dd87f4 100644
--- a/iree/hal/local/BUILD
+++ b/iree/hal/local/BUILD
@@ -8,6 +8,7 @@
# These are generally just wrappers around host heap memory and host threads.
load("//iree:build_defs.oss.bzl", "iree_cmake_extra_content")
+load("//build_tools/bazel:cc_binary_benchmark.bzl", "cc_binary_benchmark")
package(
default_visibility = ["//visibility:public"],
@@ -20,7 +21,7 @@
hdrs = ["executable_library.h"],
)
-cc_test(
+cc_binary_benchmark(
name = "executable_library_benchmark",
srcs = ["executable_library_benchmark.c"],
deps = [
diff --git a/iree/hal/local/CMakeLists.txt b/iree/hal/local/CMakeLists.txt
index 42fb8d2..3756a96 100644
--- a/iree/hal/local/CMakeLists.txt
+++ b/iree/hal/local/CMakeLists.txt
@@ -18,7 +18,7 @@
PUBLIC
)
-iree_cc_test(
+iree_cc_binary_benchmark(
NAME
executable_library_benchmark
SRCS
@@ -32,6 +32,7 @@
iree::hal
iree::hal::local::loaders::embedded_library_loader
iree::testing::benchmark
+ TESTONLY
)
iree_cc_test(
diff --git a/iree/hal/utils/BUILD b/iree/hal/utils/BUILD
index 42e00dc..f47f1ff 100644
--- a/iree/hal/utils/BUILD
+++ b/iree/hal/utils/BUILD
@@ -4,6 +4,8 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+load("//build_tools/bazel:cc_binary_benchmark.bzl", "cc_binary_benchmark")
+
package(
default_visibility = ["//visibility:public"],
features = ["layering_check"],
@@ -49,7 +51,7 @@
],
)
-cc_test(
+cc_binary_benchmark(
name = "resource_set_benchmark",
srcs = ["resource_set_benchmark.c"],
deps = [
diff --git a/iree/hal/utils/CMakeLists.txt b/iree/hal/utils/CMakeLists.txt
index cd18b36..3c60c69 100644
--- a/iree/hal/utils/CMakeLists.txt
+++ b/iree/hal/utils/CMakeLists.txt
@@ -55,7 +55,7 @@
PUBLIC
)
-iree_cc_test(
+iree_cc_binary_benchmark(
NAME
resource_set_benchmark
SRCS
@@ -66,6 +66,7 @@
iree::base::internal::prng
iree::hal
iree::testing::benchmark
+ TESTONLY
)
iree_cc_test(
diff --git a/iree/vm/BUILD b/iree/vm/BUILD
index d39c03a..2b13444 100644
--- a/iree/vm/BUILD
+++ b/iree/vm/BUILD
@@ -6,7 +6,7 @@
load("//iree:build_defs.oss.bzl", "iree_cmake_extra_content")
load("//build_tools/bazel:iree_bytecode_module.bzl", "iree_bytecode_module")
-load("//build_tools/bazel:native_binary.bzl", "native_test")
+load("//build_tools/bazel:cc_binary_benchmark.bzl", "cc_binary_benchmark")
# load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library")
package(
@@ -138,7 +138,7 @@
],
)
-cc_test(
+cc_binary_benchmark(
name = "native_module_benchmark",
srcs = ["native_module_benchmark.cc"],
deps = [
@@ -251,7 +251,7 @@
],
)
-cc_binary(
+cc_binary_benchmark(
name = "bytecode_module_benchmark",
testonly = True,
srcs = ["bytecode_module_benchmark.cc"],
@@ -266,12 +266,6 @@
],
)
-native_test(
- name = "bytecode_module_benchmark_test",
- src = ":bytecode_module_benchmark",
- args = ["--benchmark_min_time=0"],
-)
-
iree_bytecode_module(
name = "bytecode_module_benchmark_module",
testonly = True,
@@ -280,7 +274,7 @@
flags = ["-iree-vm-ir-to-bytecode-module"],
)
-cc_test(
+cc_binary_benchmark(
name = "bytecode_module_size_benchmark",
srcs = ["bytecode_module_size_benchmark.cc"],
deps = [
diff --git a/iree/vm/CMakeLists.txt b/iree/vm/CMakeLists.txt
index 857b8c9..f03dce3 100644
--- a/iree/vm/CMakeLists.txt
+++ b/iree/vm/CMakeLists.txt
@@ -126,7 +126,7 @@
PUBLIC
)
-iree_cc_test(
+iree_cc_binary_benchmark(
NAME
native_module_benchmark
SRCS
@@ -138,6 +138,7 @@
iree::base
iree::base::logging
iree::testing::benchmark_main
+ TESTONLY
)
iree_cc_test(
@@ -210,7 +211,7 @@
"notap"
)
-iree_cc_binary(
+iree_cc_binary_benchmark(
NAME
bytecode_module_benchmark
SRCS
@@ -226,15 +227,6 @@
TESTONLY
)
-iree_native_test(
- NAME
- "bytecode_module_benchmark_test"
- ARGS
- "--benchmark_min_time=0"
- SRC
- ::bytecode_module_benchmark
-)
-
iree_bytecode_module(
NAME
bytecode_module_benchmark_module
@@ -248,7 +240,7 @@
PUBLIC
)
-iree_cc_test(
+iree_cc_binary_benchmark(
NAME
bytecode_module_size_benchmark
SRCS
@@ -258,6 +250,7 @@
::bytecode_module_size_benchmark_module_c
::vm
iree::base
+ TESTONLY
)
iree_bytecode_module(
