diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d0cfe75..19d05a9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -594,6 +594,7 @@
         run: |
           ./build_tools/github_actions/docker_run.sh \
             --env "BUILD_PRESET=test" \
+            --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
             ${CONTAINER} \
             ./build_tools/cmake/build_runtime.sh \
             "${BUILD_DIR}"
diff --git a/build_tools/cmake/build_runtime.sh b/build_tools/cmake/build_runtime.sh
index 4fb41ff..8ed4c0e 100755
--- a/build_tools/cmake/build_runtime.sh
+++ b/build_tools/cmake/build_runtime.sh
@@ -17,6 +17,8 @@
 
 BUILD_DIR="${1:-${IREE_TARGET_BUILD_DIR:-build-runtime}}"
 BUILD_PRESET="${BUILD_PRESET:-test}"
+CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-RelWithDebInfo}"
+IREE_BUILD_PYTHON_BINDINGS="${IREE_BUILD_PYTHON_BINDINGS:-ON}"
 
 source build_tools/cmake/setup_build.sh
 # Note: not using ccache since the runtime build should be fast already.
@@ -25,12 +27,16 @@
 args=(
   "-G" "Ninja"
   "-B" "${BUILD_DIR}"
-  "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
-  "-DPYTHON_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
-  "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
+
+  "-DIREE_BUILD_COMPILER=OFF"
+  "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+
   "-DIREE_RUNTIME_OPTIMIZATION_PROFILE=lto"
   "-DIREE_FORCE_LTO_COMPAT_BINUTILS_ON_LINUX=ON"
-  "-DIREE_BUILD_COMPILER=OFF"
+
+  "-DIREE_BUILD_PYTHON_BINDINGS=${IREE_BUILD_PYTHON_BINDINGS}"
+  "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
+  "-DPYTHON_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"
 )
 
 case "${BUILD_PRESET}" in
diff --git a/compiler/bindings/python/test/tools/CMakeLists.txt b/compiler/bindings/python/test/tools/CMakeLists.txt
index fee8f34..ec96eea 100644
--- a/compiler/bindings/python/test/tools/CMakeLists.txt
+++ b/compiler/bindings/python/test/tools/CMakeLists.txt
@@ -7,13 +7,13 @@
 # These tests perform linking via the Compiler API, which is only supported
 # in bundled-LLVM builds at the moment (#14086).
 if(IREE_BUILD_BUNDLED_LLVM)
-iree_py_test(
-  NAME
-    compiler_core_test
-  SRCS
-    "compiler_core_test.py"
-)
-endif()  # IREE_BUILD_BUNDLED_LLVM
+  iree_py_test(
+    NAME
+      compiler_core_test
+    SRCS
+      "compiler_core_test.py"
+  )
+endif()
 
 if(IREE_INPUT_TORCH)
   iree_py_test(
diff --git a/runtime/bindings/python/CMakeLists.txt b/runtime/bindings/python/CMakeLists.txt
index 9f3151b..29a3e55 100644
--- a/runtime/bindings/python/CMakeLists.txt
+++ b/runtime/bindings/python/CMakeLists.txt
@@ -256,9 +256,11 @@
     "tests/system_setup_test.py"
 )
 
+# These tests use compiler APIs as well as runtime APIs.
+#
 # These tests perform linking via the Compiler API, which is only supported
 # in bundled-LLVM builds at the moment (#14086).
-if(IREE_BUILD_BUNDLED_LLVM)
+if(IREE_BUILD_COMPILER AND IREE_BUILD_BUNDLED_LLVM)
   iree_py_test(
     NAME
       benchmark_test
diff --git a/samples/py_custom_module/CMakeLists.txt b/samples/py_custom_module/CMakeLists.txt
index 5e1053d..85c82bc 100644
--- a/samples/py_custom_module/CMakeLists.txt
+++ b/samples/py_custom_module/CMakeLists.txt
@@ -8,9 +8,8 @@
   return()
 endif()
 
-# These tests perform linking via the Compiler API, which is only supported
-# in bundled-LLVM builds at the moment (#14086).
-if(NOT IREE_BUILD_BUNDLED_LLVM)
+# These tests use compiler APIs as well as runtime APIs.
+if(NOT IREE_BUILD_COMPILER)
   return()
 endif()
 
diff --git a/tests/e2e/collectives/CMakeLists.txt b/tests/e2e/collectives/CMakeLists.txt
index dfa3472..d33e393 100644
--- a/tests/e2e/collectives/CMakeLists.txt
+++ b/tests/e2e/collectives/CMakeLists.txt
@@ -4,9 +4,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# These tests perform linking via the Compiler API, which is only supported
-# in bundled-LLVM builds at the moment (#14086).
-if(NOT IREE_BUILD_BUNDLED_LLVM OR NOT IREE_ENABLE_COLLECTIVE_RUNTIME_TESTS)
+if(NOT IREE_BUILD_COMPILER OR NOT IREE_ENABLE_COLLECTIVE_RUNTIME_TESTS)
   return()
 endif()
 
diff --git a/tests/e2e/stablehlo_models/mnist_train_test/CMakeLists.txt b/tests/e2e/stablehlo_models/mnist_train_test/CMakeLists.txt
index a2ff8ef..7ae5613 100644
--- a/tests/e2e/stablehlo_models/mnist_train_test/CMakeLists.txt
+++ b/tests/e2e/stablehlo_models/mnist_train_test/CMakeLists.txt
@@ -4,9 +4,11 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+# These tests use compiler APIs as well as runtime APIs.
+#
 # These tests perform linking via the Compiler API, which is only supported
 # in bundled-LLVM builds at the moment (#14086).
-if(NOT IREE_BUILD_BUNDLED_LLVM)
+if(NOT IREE_BUILD_COMPILER OR NOT IREE_BUILD_BUNDLED_LLVM)
   return()
 endif()