diff --git a/build_tools/cmake/iree_benchmark_suite.cmake b/build_tools/cmake/iree_benchmark_suite.cmake
index 3795bd7..9c38441 100644
--- a/build_tools/cmake/iree_benchmark_suite.cmake
+++ b/build_tools/cmake/iree_benchmark_suite.cmake
@@ -185,6 +185,19 @@
     "BENCHMARK_MODES;BENCHMARK_TOOL;MODULES"
   )
 
+  # Try to check if the compiler supports the TARGET_BACKEND. If
+  # IREE_HOST_BINARY_ROOT is defined, we are using a compiler binary, in which
+  # case we can't check it's supported backend just by looking at this build
+  # dir's cmake variables --- we would have to implement a configure-check
+  # executing that compiler.
+  if (NOT DEFINED IREE_HOST_BINARY_ROOT)
+    string(TOUPPER ${_RULE_TARGET_BACKEND} _UPPERCASE_TARGET_BACKEND)
+    string(REPLACE "-" "_" _NORMALIZED_TARGET_BACKEND ${_UPPERCASE_TARGET_BACKEND})
+    if(NOT IREE_TARGET_BACKEND_${_NORMALIZED_TARGET_BACKEND})
+      return()
+    endif()
+  endif()
+
   iree_package_name(_PACKAGE_NAME)
 
   # Add the benchmark suite target.