build_tools/cmake/build_and_test_tsan.sh

#! /usr/bin/env bash
# Copyright 2022 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

# Build and test, using CMake/CTest, with ThreadSanitizer instrumentation.
#
# See https://clang.llvm.org/docs/ThreadSanitizer.html. Some tests are run many
# times to flush out non-deterministic failures.
#
# The desired build directory can be passed as the first argument. Otherwise, it
# uses the environment variable IREE_TSAN_BUILD_DIR, defaulting to "build-tsan".
# Designed for CI, but can be run manually. It reuses the build directory if it
# already exists. Expects to be run from the root of the IREE repository.

set -euo pipefail

BUILD_DIR="${1:-${IREE_TSAN_BUILD_DIR:-build-tsan}}"
# Enable CUDA and HIP/ROCM compiler and runtime by default if not on Darwin.
OFF_IF_DARWIN="$(uname | awk '{print ($1 == "Darwin") ? "OFF" : "ON"}')"
IREE_HAL_DRIVER_CUDA="${IREE_HAL_DRIVER_CUDA:-${OFF_IF_DARWIN}}"
IREE_HAL_DRIVER_HIP="${IREE_HAL_DRIVER_HIP:-${OFF_IF_DARWIN}}"
IREE_TARGET_BACKEND_CUDA="${IREE_TARGET_BACKEND_CUDA:-${OFF_IF_DARWIN}}"
IREE_TARGET_BACKEND_ROCM="${IREE_TARGET_BACKEND_ROCM:-${OFF_IF_DARWIN}}"
IREE_ROCM_TEST_TARGET_CHIP="${IREE_ROCM_TEST_TARGET_CHIP:-}"

source build_tools/cmake/setup_build.sh

CMAKE_ARGS=(
  "-G" "Ninja"
  "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}"

  # The debug information will help get more helpful TSan reports (stacks).
  "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
  "-DIREE_ENABLE_SPLIT_DWARF=ON"
  "-DIREE_ENABLE_THIN_ARCHIVES=ON"

  # Let's make linking fast. Also, there's the off chance that TSan might be
  # better supported with LLD.
  "-DIREE_ENABLE_LLD=ON"

  # Enable TSan in all C/C++ targets, including IREE runtime, compiler, tests.
  "-DIREE_ENABLE_TSAN=ON"


  "-DIREE_HAL_DRIVER_CUDA=${IREE_HAL_DRIVER_CUDA}"
  "-DIREE_HAL_DRIVER_HIP=${IREE_HAL_DRIVER_HIP}"
  "-DIREE_TARGET_BACKEND_CUDA=${IREE_TARGET_BACKEND_CUDA}"
  "-DIREE_TARGET_BACKEND_ROCM=${IREE_TARGET_BACKEND_ROCM}"
  "-DIREE_ROCM_TEST_TARGET_CHIP=${IREE_ROCM_TEST_TARGET_CHIP}"

  # Workaround for this weird issue:
  # https://github.com/google/benchmark/issues/773#issuecomment-616067912
  "-DRUN_HAVE_STD_REGEX=0"
  "-DRUN_HAVE_POSIX_REGEX=0"
  "-DCOMPILE_HAVE_GNU_POSIX_REGEX=0"

  "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
)

echo "::group::Configuring CMake"
"${CMAKE_BIN}" -B "${BUILD_DIR}" "${CMAKE_ARGS[@]?}"
echo "::endgroup::"

echo "::group::Building all"
"$CMAKE_BIN" --build "${BUILD_DIR}" -- -k 0
echo "::endgroup::"

echo "::group::Building test deps"
"$CMAKE_BIN" --build "${BUILD_DIR}" --target iree-test-deps -- -k 0
echo "::endgroup::"

# Disable actually running GPU tests. This tends to yield TSan reports that are
# specific to one's particular GPU driver and therefore hard to reproduce across
# machines and often non-actionable anyway.
# See, e.g., https://github.com/iree-org/iree/issues/9393.
export IREE_VULKAN_DISABLE=1
export IREE_METAL_DISABLE=1
export IREE_CUDA_DISABLE=1
export IREE_HIP_DISABLE=1

# Honor the "notsan" label on tests.
export IREE_EXTRA_COMMA_SEPARATED_CTEST_LABELS_TO_EXCLUDE=notsan

# Run all tests, once.
build_tools/cmake/ctest_all.sh "${BUILD_DIR}"

# Re-run many times certain tests that are cheap and prone to nondeterministic
# failure (typically, IREE runtime tests exercising multi-threading features).
# Benchmark wrapper tests run above as smoke tests, but their timing under TSan
# is not performance evidence and multiplying them here only adds load noise.
export IREE_CTEST_TESTS_REGEX="(^iree/base/|^iree/task/)"
export IREE_CTEST_REPEAT_UNTIL_FAIL_COUNT=32
export IREE_EXTRA_NEWLINE_SEPARATED_CTEST_TESTS_TO_EXCLUDE="${IREE_EXTRA_NEWLINE_SEPARATED_CTEST_TESTS_TO_EXCLUDE:-}
.*_benchmark_test"
build_tools/cmake/ctest_all.sh "${BUILD_DIR}"