.github/workflows/build_package.yml

# Builds release packages on supported platforms:
#   * main-dist-linux (CMake 'install')
#   * py-compiler-pkg (`iree-compiler` Python package)
#     * Linux, macOS, Windows
#     * All supported Python versions (e.g. 3.9, 3.10, 3.11)
#   * py-runtime-pkg (`iree-runtime` Python package)
#     * Linux, macOS, Windows
#     * All supported Python versions (e.g. 3.9, 3.10, 3.11)
#   * py-tf-compiler-tools-pkg (`iree-tools-[tf, tflite]`, pure Python packages)

name: Build Release Packages

# Note: this workflow is typically invoked by schedule_candidate_release.yml.
on:
  workflow_dispatch:
    inputs:
      package_suffix:
        description: "Suffix to append to package names"
        required: false
        default: ""
      package_version:
        description: "Version of the package"
        required: true
        default: "0.1a1"
      release_id:
        description: "Release id to upload artifacts to"
        default: ""
      commit:
        description: "Commit to check out"
        default: ""

jobs:
  build_packages:
    name: "${{ matrix.build-family }} :: Build ${{ matrix.build-package }} Package"
    runs-on: ${{ matrix.runs-on }}
    continue-on-error: ${{ matrix.experimental }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # Ubuntu packages.
          - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
            build-family: linux-x86_64
            build-package: main-dist-linux
            experimental: false
          - runs-on: [self-hosted, arm64, os-family=Linux, runner-group=postsubmit]
            build-family: linux-aarch64
            build-package: main-dist-linux
            experimental: true
          - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
            build-family: linux-x86_64
            build-package: py-compiler-pkg
            experimental: false
          - runs-on: [self-hosted, arm64, os-family=Linux, runner-group=postsubmit]
            build-family: linux-aarch64
            build-package: py-compiler-pkg
            experimental: true
          - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
            build-family: linux-x86_64
            build-package: py-runtime-pkg
            experimental: false
          - runs-on: [self-hosted, arm64, os-family=Linux, runner-group=postsubmit]
            build-family: linux-aarch64
            build-package: py-runtime-pkg
            experimental: true
          - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
            build-family: linux-x86_64
            build-package: py-tf-compiler-tools-pkg
            experimental: false

          # MacOS packages.
          - runs-on:
              - ${{ github.repository == 'openxla/iree' && 'self-hosted' || 'macos-11' }}
              - os-family=macOS
              - runner-group=postsubmit
            build-family: macos
            build-package: py-compiler-pkg
            experimental: true
          - runs-on:
              - ${{ github.repository == 'openxla/iree' && 'self-hosted' || 'macos-11' }}
              - os-family=macOS
              - runner-group=postsubmit
            build-family: macos
            build-package: py-runtime-pkg
            experimental: true

          # Windows packages.
          - runs-on:
              - ${{ github.repository == 'openxla/iree' && 'windows-2022-64core' || 'windows-2022'}}
            build-family: windows
            build-package: py-compiler-pkg
            experimental: true
          - runs-on: windows-2022
            build-family: windows
            build-package: py-runtime-pkg
            experimental: true

    env:
      # These are also set in: build_tools/python_deploy/build_linux_packages.sh
      MANYLINUX_X86_64_IMAGE: ghcr.io/nod-ai/manylinux_x86_64:main
      MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64

    steps:
      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          path: "c"  # Windows can hit path length limits, so use a short path.
          submodules: true
          ref: ${{ github.event.inputs.commit }}

      ##########################################################################
      # OS specific setup
      ##########################################################################

      - name: Install dependencies (Windows)
        if: "matrix.build-family == 'windows'"
        shell: powershell
        run: ./c/build_tools/python_deploy/install_windows_deps.ps1
      - name: "Configure MSVC (Windows)"
        if: "matrix.build-family == 'windows'"
        uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d # v1.12.0

      ##########################################################################
      # Write version_info.json
      # Various tools will read this in order to embed release information.
      ##########################################################################
      - name: Write version info (release)
        if: "matrix.build-family != 'windows'"
        shell: bash
        run: |
          cat << EOF > ./c/version_info.json
          {
            "package-suffix": "${{ github.event.inputs.package_suffix }}",
            "package-version": "${{ github.event.inputs.package_version }}",
            "iree-revision": "$(cd ./c && git rev-parse HEAD)"
          }
          EOF
          cat ./c/version_info.json

      - name: Write version info Windows (release)
        if: "matrix.build-family == 'windows'"
        shell: powershell
        run: |
          cd c
          $verinfoprop = @{
            'package-suffix'= "${{ github.event.inputs.package_suffix }}"
            'package-version'= "${{ github.event.inputs.package_version }}"
            'iree-revision'= $(git rev-parse HEAD)
          }

          $info = New-Object -TypeName PSObject -Prop $verinfoprop
          $info = $info | ConvertTo-JSON
          $info | Out-File "version_info.json" -Encoding "ASCII"

      ##########################################################################
      # Build the main distribution tarball.
      # The main distribution consists of the project being built, installed
      # and archived. We have to split it per operating system, and Linux
      # is special because we build under a manylinux container which gives
      # broad compatibility. We use the Python based manylinux containers
      # since they come packaged with all dev tools we need. Note that the
      # manylinux containers have a default python 2.x with all supported
      # python versions under /opt/python (need to add one to the path). It
      # is not enough to just invoke it directly because then pip managed
      # console scripts (like cmake, ninja) will not be on the path.
      ##########################################################################
      - name: Main distribution (Linux for x86_64)
        if: "matrix.build-package == 'main-dist-linux' && matrix.build-family == 'linux-x86_64'"
        shell: bash
        run: |
          docker run --rm -w=/work \
            -v $PWD:/work \
            "${MANYLINUX_X86_64_IMAGE}" \
            bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./c/build_tools/github_actions/build_dist.py main-dist'

      - name: Main distribution (Linux for Arm64)
        if: "matrix.build-package == 'main-dist-linux' && matrix.build-family == 'linux-aarch64'"
        shell: bash
        run: |
          docker run --rm -w=/work \
            -v $PWD:/work \
            "${MANYLINUX_AARCH64_IMAGE}" \
            bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./c/build_tools/github_actions/build_dist.py main-dist'

      ##########################################################################
      # py-runtime-pkg
      # Builds the iree-runtime wheels.
      # One step per OS.
      ##########################################################################

      - name: Build runtime wheels (Linux, x86_64 and Arm64)
        if: "matrix.build-package == 'py-runtime-pkg' && contains(matrix.build-family, 'linux')"
        shell: bash
        env:
          package_suffix: ${{ github.event.inputs.package_suffix }}
          packages: "iree-runtime"
          output_dir: "${{ github.workspace }}/bindist"
        run: |
          [ -e ./bindist/* ] && rm ./bindist/*
          ./c/build_tools/python_deploy/build_linux_packages.sh

      - name: Build runtime wheels (MacOS)
        if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'macos'"
        shell: bash
        env:
          package_suffix: ${{ github.event.inputs.package_suffix }}
          packages: "iree-runtime"
          output_dir: "${{ github.workspace }}/bindist"
          override_python_versions: "3.11"
        run: |
          [ -e ./bindist/* ] && rm ./bindist/*
          ./c/build_tools/python_deploy/build_macos_packages.sh

      - name: Build runtime wheels (Windows)
        if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'windows'"
        shell: powershell
        env:
          package_suffix: ${{ github.event.inputs.package_suffix }}
          packages: "iree-runtime"
          output_dir: "${{ github.workspace }}/bindist"
          override_python_versions: "3.11"
        run: |
          if (Test-Path -Path "${{ github.workspace }}/bindist") {
            Remove-Item -Path "${{ github.workspace }}/bindist" -Recurse -Force
          }
          ./c/build_tools/python_deploy/build_windows_packages.ps1

      ##########################################################################
      # py-compiler-pkg
      # Builds the iree-compiler wheel.
      # One step per OS.
      ##########################################################################
      - name: Build compiler wheels (Linux, x86_64 and Arm64)
        if: "matrix.build-package == 'py-compiler-pkg' && contains(matrix.build-family, 'linux')"
        shell: bash
        env:
          package_suffix: ${{ github.event.inputs.package_suffix }}
          packages: "iree-compiler"
          output_dir: "${{ github.workspace }}/bindist"
        run: |
          [ -e ./bindist/* ] && rm ./bindist/*
          ./c/build_tools/python_deploy/build_linux_packages.sh

      - name: Build compiler wheels (MacOS)
        if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'macos'"
        shell: bash
        env:
          package_suffix: ${{ github.event.inputs.package_suffix }}
          packages: "iree-compiler"
          output_dir: "${{ github.workspace }}/bindist"
          override_python_versions: "3.11"
        run: |
          [ -e ./bindist/* ] && rm ./bindist/*
          ./c/build_tools/python_deploy/build_macos_packages.sh

      - name: Build compiler wheels (Windows)
        if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'windows'"
        shell: powershell
        env:
          package_suffix: ${{ github.event.inputs.package_suffix }}
          packages: "iree-compiler"
          output_dir: "${{ github.workspace }}/bindist"
          override_python_versions: "3.11"
        run: |
          if (Test-Path -Path "${{ github.workspace }}/bindist") {
            Remove-Item -Path "${{ github.workspace }}/bindist" -Recurse -Force
          }
          ./c/build_tools/python_deploy/build_windows_packages.ps1

      ##########################################################################
      # TF Compiler Tools
      # These wheels are not Python version specific and include only Python
      # code (no C/C++), so just build for one examplar python version on Linux.
      ##########################################################################

      - name: Build TF Compiler Tools wheels
        if: "matrix.build-package == 'py-tf-compiler-tools-pkg'"
        shell: bash
        run: |
          docker run --rm -w=/work \
            -v $PWD:/work \
            "${MANYLINUX_X86_64_IMAGE}" \
            bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./c/build_tools/github_actions/build_dist.py py-tf-compiler-tools-pkg'

      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
        with:
          # We upload all wheels (which includes deps so that subsequent
          # steps can run without further fetching).
          path: ./bindist/*
          retention-days: 5

      # TODO: Upload the tar.bz2 files too when ready
      - name: Upload Release Assets
        if: github.event.inputs.release_id != ''
        id: upload-release-assets
        uses: dwenegar/upload-release-assets@5bc3024cf83521df8ebfadf00ad0c4614fd59148 # v1
        env:
          GITHUB_TOKEN: ${{ secrets.WRITE_ACCESS_TOKEN }}
        with:
          release_id: ${{ github.event.inputs.release_id }}
          # Only upload iree artifacts.
          assets_path: ./bindist/iree*.*

  validate_and_publish:
    name: "Trigger validate and publish release"
    needs: build_packages
    if: github.event.inputs.release_id != ''
    runs-on: ubuntu-20.04
    steps:
      - name: "Invoke workflow :: Validate and Publish Release"
        uses: benc-uk/workflow-dispatch@798e70c97009500150087d30d9f11c5444830385 # v1.2.2
        with:
          workflow: Validate and Publish Release
          token: ${{ secrets.WRITE_ACCESS_TOKEN }}
          ref: "${{ env.tag_name }}"
          inputs: '{"release_id": "${{ github.event.inputs.release_id }}", "package_version": "${{ github.event.inputs.package_version }}", "build_run_id": "${{ github.run_id }}"}'