Add cmake_configure_ci.py. This is a wrapper around a cmake configure invocation that auto-detects a number of things across different CI platforms.
diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 5f23416..44f4ada 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml
@@ -1,62 +1,125 @@ -name: Build and package +# Builds packages for native (non cross-compiled) targets on supported +# platforms. +# +# For these mainline distributions, we use cibuildwheel and drive the +# packaging through python, extracting native assets. While this may seem +# hopelessly round-about, it lets us leverage a lot of what cibuildwheel +# does for free and get python packages to boot. +name: Build Native Release Packages on: workflow_dispatch: + inputs: + package_suffix: + description: 'Suffix to append to package names' + required: true + default: '-cidev' + package_version: + description: 'Version of the package' + required: true + default: '0.1a1' + release_id: + description: 'Release id to upload artifacts to' + required: true + default: '' jobs: - mondo-build: - name: Performs a mondo-build of all optional components - runs-on: ubuntu-18.04 - env: - SRC_DIR: ${{ github.workspace }}/iree-src - BUILD_DIR: ${{ github.workspace }}/iree-build - INSTALL_DIR: ${{ github.workspace }}/iree-install + build_core: + name: "${{ matrix.os }} :: Build Core Packages" + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04] + experimental: [false] + # include: + # - os: macos-latest + # experimental: true + # - os: windows-2019 + # experimental: true + steps: - uses: actions/checkout@v2 with: - submodules: true - path: "iree-src" + path: 'main_checkout' + - uses: actions/setup-python@v2 + name: Install Python with: - python-version: '3.8' - - name: Report environment - shell: bash - run: | - echo "GITHUB_RUN_ID=${GITHUB_RUN_ID}" - echo "GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER}" - echo "GITHUB_WORKSPACE=${GITHUB_WORKSPACE}" - echo "PWD=${PWD}" - echo "PATH=${PATH}" - - name: Install python deps - shell: bash - run: | - # We aren't actually building anything that requires this, but the - # TensorFlow build hard fails without it, even though we are only - # building C++ (facepalm). - python -m pip install numpy - - name: Configure - shell: bash - run: | - cmake -B "${BUILD_DIR}" "${SRC_DIR}" \ - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ - -DIREE_BUILD_TENSORFLOW_COMPILER=ON \ - -DIREE_BUILD_XLA_COMPILER=ON \ - -DIREE_BUILD_TFLITE_COMPILER=ON \ - -DPython3_EXECUTABLE=$(which python) + python-version: '3.7' - - name: Build + - name: Write version info shell: bash run: | - cd "${BUILD_DIR}" - make -j 4 - make install - ls -lRh "${INSTALL_DIR}" + cat << EOF > ./main_checkout/version_info.json + { + "package-suffix": "${{ github.event.inputs.package_suffix }}", + "package-version": "${{ github.event.inputs.package_version }}", + "iree-revision": "$(cd ./main_checkout && git rev-parse HEAD)" + } + EOF + cat ./main_checkout/version_info.json - - name: Upload Install Directory - uses: actions/upload-artifact@v2 - with: - name: iree-install-ubuntu-18.04 - path: ${{env.INSTALL_DIR}}/ - retention-days: 5 + - name: Install cibuildwheel + shell: bash + run: | + python -m pip install cibuildwheel==1.7.2 + +# jobs: +# mondo-build: +# name: Performs a mondo-build of all optional components +# runs-on: ubuntu-18.04 +# env: +# SRC_DIR: ${{ github.workspace }}/iree-src +# BUILD_DIR: ${{ github.workspace }}/iree-build +# INSTALL_DIR: ${{ github.workspace }}/iree-install +# steps: +# - uses: actions/checkout@v2 +# with: +# submodules: true +# path: "iree-src" +# - uses: actions/setup-python@v2 +# with: +# python-version: '3.8' +# - name: Report environment +# shell: bash +# run: | +# echo "GITHUB_RUN_ID=${GITHUB_RUN_ID}" +# echo "GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER}" +# echo "GITHUB_WORKSPACE=${GITHUB_WORKSPACE}" +# echo "PWD=${PWD}" +# echo "PATH=${PATH}" +# - name: Install python deps +# shell: bash +# run: | +# # We aren't actually building anything that requires this, but the +# # TensorFlow build hard fails without it, even though we are only +# # building C++ (facepalm). +# python -m pip install numpy +# - name: Configure +# shell: bash +# run: | +# cmake -B "${BUILD_DIR}" "${SRC_DIR}" \ +# -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ +# -DIREE_BUILD_TENSORFLOW_COMPILER=ON \ +# -DIREE_BUILD_XLA_COMPILER=ON \ +# -DIREE_BUILD_TFLITE_COMPILER=ON \ +# -DPython3_EXECUTABLE=$(which python) + +# - name: Build +# shell: bash +# run: | +# cd "${BUILD_DIR}" +# make -j 4 +# make install +# ls -lRh "${INSTALL_DIR}" + +# - name: Upload Install Directory +# uses: actions/upload-artifact@v2 +# with: +# name: iree-install-ubuntu-18.04 +# path: ${{env.INSTALL_DIR}}/ +# retention-days: 5
diff --git a/bindings/python/build_requirements.txt b/bindings/python/build_requirements.txt new file mode 100644 index 0000000..e9967ef --- /dev/null +++ b/bindings/python/build_requirements.txt
@@ -0,0 +1,11 @@ +# Python package requirements for building. +# This is what the CI uses, but is supported for users as well. +# Usage: +# python -m pip install -r bindings/python/build_requirements.txt + +numpy>=1.19.4 +pybind11>=2.6.1 +# Ensure a recent cmake +cmake>=3.18.4 +ninja +wheel
diff --git a/build_tools/cmake/cmake_configure_ci.py b/build_tools/cmake/cmake_configure_ci.py new file mode 100644 index 0000000..992093e --- /dev/null +++ b/build_tools/cmake/cmake_configure_ci.py
@@ -0,0 +1,186 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This configure script wraps a normal CMake configure of the project for use +# in CI pipelines, doing some sanity checks, discovery and work-arounding. + +# This future is needed to print Python2 EOL message +from __future__ import print_function +import sys +if sys.version_info < (3,): + print("Python 2 has reached end-of-life and is no longer supported.") + sys.exit(-1) +if sys.platform == 'win32' and sys.maxsize.bit_length() == 31: + print( + "32-bit Windows Python runtime is not supported. Please switch to 64-bit Python." + ) + sys.exit(-1) + +import importlib +import os +import platform +import subprocess +import sysconfig +import tempfile + +is_windows = platform.system() == 'Windows' + + +def report(*args): + print('--', *args) + + +def get_setting(varname, default_value): + value = os.environ.get(varname) + if value is None: + return default_value + return value + + +def get_bool_setting(varname, default_value): + value = get_setting(varname, default_value) + if value is True or value is False: + return value + return value == '' or value == 'ON' or value == '1' + + +def which(thefile): + path = os.environ.get("PATH", os.defpath).split(os.pathsep) + for d in path: + fname = os.path.join(d, thefile) + fnames = [fname] + if sys.platform == 'win32': + exts = os.environ.get('PATHEXT', '').split(os.pathsep) + fnames += [fname + ext for ext in exts] + for name in fnames: + if os.access(name, os.F_OK | os.X_OK) and not os.path.isdir(name): + return name + return None + + +def use_tool_path(toolname, varname=None): + if not varname: varname = toolname.upper() + value = get_setting(f'USE_{varname}', 'ON') + if value.upper() == 'OFF': + return None + if value.upper() == 'ON' or value == '': + return which(toolname) + if os.access(value, os.F_OK | os.X_OK) and not os.path.isdir(value): + return value + + +repo_root = os.path.abspath( + get_setting('REPO_DIR', os.path.join(os.path.dirname(__file__), '..', + '..'))) + +################################################################################ +# Figure out where we are and where we are going. +################################################################################ +repo_root = os.path.abspath( + get_setting('REPO_DIR', os.path.join(os.path.dirname(__file__), '..', + '..'))) +report(f'Using REPO_DIR = {repo_root}') + +################################################################################ +# Build deps. +################################################################################ +requirements_file = os.path.join(repo_root, 'bindings', 'python', + 'build_requirements.txt') + +report('Installing python build requirements...') +subprocess.check_call( + [sys.executable, '-m', 'pip', 'install', '-r', requirements_file]) + +################################################################################ +# Load version_info.json +################################################################################ + + +def load_version_info(): + with open(os.path.join(repo_root, 'version_info.json'), 'rt') as f: + return json.load(f) + + +try: + version_info = load_version_info() +except FileNotFoundError: + report('version_info.json found') + version_info = {} + +################################################################################ +# CMake configure. +################################################################################ + +cmake_args = [ + f'-S{repo_root}', + f'-DPython3_EXECUTABLE:FILEPATH={sys.executable}', + f'-DPython3_INCLUDE_DIR:PATH={sysconfig.get_path("include")}', +] + +### HACK: Add a Python3_LIBRARY because cmake needs it, but it legitimately +### does not exist on manylinux (or any linux static python). +# Need to explicitly tell cmake about the python library. +python_libdir = sysconfig.get_config_var('LIBDIR') +python_library = sysconfig.get_config_var('LIBRARY') +if python_libdir and not os.path.isabs(python_library): + python_library = os.path.join(python_libdir, python_library) + +# On manylinux, python is a static build, which should be fine, but CMake +# disagrees. Fake it by letting it see a library that will never be needed. +if python_library and not os.path.exists(python_library): + python_libdir = os.path.join(tempfile.gettempdir(), 'fake_python', 'lib') + os.makedirs(python_libdir, exist_ok=True) + python_library = os.path.join(python_libdir, + sysconfig.get_config_var('LIBRARY')) + with open(python_library, 'wb') as f: + pass + +if python_library: + cmake_args.append(f'-DPython3_LIBRARY:PATH={python_library}') + +### Detect cmake. +use_cmake = use_tool_path('cmake') or 'cmake' + +### Detect generator. +if use_tool_path('ninja'): + report('Using ninja') + cmake_args.append('-GNinja') +elif is_windows: + cmake_args.extend(['-G', 'NMake Makefiles']) + +# Detect other build tools. +use_ccache = use_tool_path('ccache') +if use_ccache: + report(f'Using ccache {use_ccache}') + cmake_args.append(f'-DCMAKE_CXX_COMPILER_LAUNCHER={use_ccache}') + +# Clang +use_clang = use_tool_path('clang') +if use_clang: + report(f'Using clang {use_clang}') + cmake_args.append(f'-DCMAKE_C_COMPILER={use_clang}') +use_clangcpp = use_tool_path('clang++', 'CLANGCPP') +if use_clangcpp: + report(f'Using clang++ {use_clangcpp}') + cmake_args.append(f'-DCMAKE_CXX_COMPILER={use_clangcpp}') + +# LLD +use_lld = use_tool_path('lld') +if not is_windows and use_lld: + report(f'Using linker {use_lld}') + cmake_args.append('-DIREE_ENABLE_LLD=ON') + +cmake_args.extend(sys.argv[1:]) +report(f'Running cmake (generate): {" ".join(cmake_args)}') +subprocess.check_call([use_cmake] + cmake_args)