.github/workflows/build_package.yml - 3p/openxla/iree - Git at Google

 # Builds packages for native (non cross-compiled) targets on supported
 # platforms.
 #
 # For these mainline distributions, we use cibuildwheel and drive the
 # packaging through python, extracting native assets. While this may seem
 # hopelessly round-about, it lets us leverage a lot of what cibuildwheel
 # does for free and get python packages to boot.
 name: Build Native Release Packages

 on:
   workflow_dispatch:
     inputs:
       package_suffix:
         description: "Suffix to append to package names"
         required: false
         default: ""
       package_version:
         description: "Version of the package"
         required: true
         default: "0.1a1"
       release_id:
         description: "Release id to upload artifacts to"
         default: ""
       commit:
         description: "Commit to check out"
         default: ""

 jobs:
   build_core:
     name: "${{ matrix.build-family }} :: Build ${{ matrix.build-package }} Package"
     runs-on: ${{ matrix.runs-on }}
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
         include:
           # Ubuntu packages.
           - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
             build-family: linux
             build-package: main-dist-linux
             experimental: false
           - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
             build-family: linux
             build-package: py-compiler-pkg
             experimental: false
           - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
             build-family: linux
             build-package: py-runtime-pkg
             experimental: false
           - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
             build-family: linux
             build-package: py-tf-compiler-tools-pkg
             experimental: false

           # Macos packages.
           - runs-on: macos-11
             build-family: macos
             build-package: py-compiler-pkg
             experimental: true
           - runs-on: macos-11
             build-family: macos
             build-package: py-runtime-pkg
             experimental: true
     env:
       MANYLINUX_X86_64_IMAGE: gcr.io/iree-oss/manylinux2014_x86_64-release@sha256:b09c10868f846308bad2eab253a77d0a3f097816c40342bc289d8e62509bc5f9

     steps:
       - uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
         with:
           path: "main_checkout"
           submodules: true
           ref: ${{ github.event.inputs.commit }}

       ##########################################################################
       # OS specific setup
       ##########################################################################
       - name: Install MacOS Deps
         if: "matrix.build-family == 'macos'"
         shell: bash
         run: |
           sudo ./main_checkout/build_tools/python_deploy/install_macos_deps.sh

       ##########################################################################
       # Write version_info.json
       # Various tools will read this in order to embed release information.
       ##########################################################################
       - name: Write version info (release)
         shell: bash
         run: |
           cat << EOF > ./main_checkout/version_info.json
           {
             "package-suffix": "${{ github.event.inputs.package_suffix }}",
             "package-version": "${{ github.event.inputs.package_version }}",
             "iree-revision": "$(cd ./main_checkout && git rev-parse HEAD)"
           }
           EOF
           cat ./main_checkout/version_info.json

       ##########################################################################
       # Build the main distribution tarball.
       # The main distribution consists of the project being built, installed
       # and archived. We have to split it per operating system, and Linux
       # is special because we build under a manylinux container which gives
       # broad compatibility. We use the Python based manylinux containers
       # since they come packaged with all dev tools we need. Note that the
       # manylinux containers have a default python 2.x with all supported
       # python versions under /opt/python (need to add one to the path). It
       # is not enough to just invoke it directly because then pip managed
       # console scripts (like cmake, ninja) will not be on the path.
       ##########################################################################
       - name: Main distribution (Linux)
         if: "matrix.build-package == 'main-dist-linux'"
         shell: bash
         run: |
           docker run --rm -w=/work \
             -v $PWD:/work \
             "${MANYLINUX_X86_64_IMAGE}" \
             bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./main_checkout/build_tools/github_actions/build_dist.py main-dist'

       ##########################################################################
       # py-runtime-pkg
       # Builds the iree-runtime and iree-runtime-instrumented wheels.
       # One step per OS.
       ##########################################################################

       - name: Build runtime wheels (Linux)
         if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'linux'"
         shell: bash
         env:
           package_suffix: ${{ github.event.inputs.package_suffix }}
           packages: "iree-runtime iree-runtime-instrumented"
           output_dir: "${{ github.workspace }}/bindist"
         run: |
           ./main_checkout/build_tools/python_deploy/build_linux_packages.sh

       - name: Build runtime wheels (MacOS)
         if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'macos'"
         shell: bash
         env:
           package_suffix: ${{ github.event.inputs.package_suffix }}
           packages: "iree-runtime iree-runtime-instrumented"
           output_dir: "${{ github.workspace }}/bindist"
         run: |
           ./main_checkout/build_tools/python_deploy/build_macos_packages.sh

       ##########################################################################
       # py-compiler-pkg
       # Builds the iree-compielr wheel.
       # One step per OS.
       ##########################################################################
       - name: Build compiler wheels (Linux)
         if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'linux'"
         shell: bash
         env:
           package_suffix: ${{ github.event.inputs.package_suffix }}
           packages: "iree-compiler"
           output_dir: "${{ github.workspace }}/bindist"
         run: |
           ./main_checkout/build_tools/python_deploy/build_linux_packages.sh

       - name: Build compiler wheels (MacOS)
         if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'macos'"
         shell: bash
         env:
           package_suffix: ${{ github.event.inputs.package_suffix }}
           packages: "iree-compiler"
           output_dir: "${{ github.workspace }}/bindist"
         run: |
           # libzstd on GitHub Action bots is not compatible with MacOS universal.
           # https://github.com/iree-org/iree/issues/9955
           sudo rm -rf /usr/local/lib/libzstd.*.dylib
           sudo rm -rf /usr/local/lib/cmake/zstd/*

           ./main_checkout/build_tools/python_deploy/build_macos_packages.sh

       ##########################################################################
       # TF Compiler Tools
       # Compiler tools wheels are not python version specific, so just build
       # for one examplar python version.
       # TODO: This currently only builds on Linux and is hardcoded to do so.
       ##########################################################################

       - name: Build TF Compiler Tools wheels
         if: "matrix.build-package == 'py-tf-compiler-tools-pkg'"
         shell: bash
         run: |
           docker run --rm -w=/work \
             -v $PWD:/work \
             "${MANYLINUX_X86_64_IMAGE}" \
             bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./main_checkout/build_tools/github_actions/build_dist.py py-tf-compiler-tools-pkg'

       - uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2
         with:
           # We upload all wheels (which includes deps so that subsequent
           # steps can run without further fetching).
           path: ./bindist/*
           retention-days: 5

       # TODO: Upload the tar.bz2 files too when ready
       - name: Upload Release Assets
         if: github.event.inputs.release_id != ''
         id: upload-release-assets
         uses: dwenegar/upload-release-assets@5bc3024cf83521df8ebfadf00ad0c4614fd59148 # v1
         env:
           GITHUB_TOKEN: ${{ secrets.WRITE_ACCESS_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
           # Only upload iree artifacts.
           assets_path: ./bindist/iree*.*

   validate_and_publish:
     name: "Invoke workflow to validate and publish release"
     needs: build_core
     if: github.event.inputs.release_id != ''
     runs-on: ubuntu-20.04
     steps:
       - name: "Invoke workflow :: Validate and Publish Release"
         uses: benc-uk/workflow-dispatch@4c044c1613fabbe5250deadc65452d54c4ad4fc7 # v1
         with:
           workflow: Validate and Publish Release
           token: ${{ secrets.WRITE_ACCESS_TOKEN }}
           ref: "${{ env.tag_name }}"
           inputs: '{"release_id": "${{ github.event.inputs.release_id }}", "package_version": "${{ github.event.inputs.package_version }}", "build_run_id": "${{ github.run_id }}"}'
	# Builds packages for native (non cross-compiled) targets on supported
	# platforms.
	#
	# For these mainline distributions, we use cibuildwheel and drive the
	# packaging through python, extracting native assets. While this may seem
	# hopelessly round-about, it lets us leverage a lot of what cibuildwheel
	# does for free and get python packages to boot.
	name: Build Native Release Packages

	on:
	workflow_dispatch:
	inputs:
	package_suffix:
	description: "Suffix to append to package names"
	required: false
	default: ""
	package_version:
	description: "Version of the package"
	required: true
	default: "0.1a1"
	release_id:
	description: "Release id to upload artifacts to"
	default: ""
	commit:
	description: "Commit to check out"
	default: ""

	jobs:
	build_core:
	name: "${{ matrix.build-family }} :: Build ${{ matrix.build-package }} Package"
	runs-on: ${{ matrix.runs-on }}
	continue-on-error: ${{ matrix.experimental }}
	strategy:
	fail-fast: false
	matrix:
	include:
	# Ubuntu packages.
	- runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
	build-family: linux
	build-package: main-dist-linux
	experimental: false
	- runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
	build-family: linux
	build-package: py-compiler-pkg
	experimental: false
	- runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
	build-family: linux
	build-package: py-runtime-pkg
	experimental: false
	- runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
	build-family: linux
	build-package: py-tf-compiler-tools-pkg
	experimental: false

	# Macos packages.
	- runs-on: macos-11
	build-family: macos
	build-package: py-compiler-pkg
	experimental: true
	- runs-on: macos-11
	build-family: macos
	build-package: py-runtime-pkg
	experimental: true
	env:
	MANYLINUX_X86_64_IMAGE: gcr.io/iree-oss/manylinux2014_x86_64-release@sha256:b09c10868f846308bad2eab253a77d0a3f097816c40342bc289d8e62509bc5f9

	steps:
	- uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
	with:
	path: "main_checkout"
	submodules: true
	ref: ${{ github.event.inputs.commit }}

	##########################################################################
	# OS specific setup
	##########################################################################
	- name: Install MacOS Deps
	if: "matrix.build-family == 'macos'"
	shell: bash
	run: \|
	sudo ./main_checkout/build_tools/python_deploy/install_macos_deps.sh

	##########################################################################
	# Write version_info.json
	# Various tools will read this in order to embed release information.
	##########################################################################
	- name: Write version info (release)
	shell: bash
	run: \|
	cat << EOF > ./main_checkout/version_info.json
	{
	"package-suffix": "${{ github.event.inputs.package_suffix }}",
	"package-version": "${{ github.event.inputs.package_version }}",
	"iree-revision": "$(cd ./main_checkout && git rev-parse HEAD)"
	}
	EOF
	cat ./main_checkout/version_info.json

	##########################################################################
	# Build the main distribution tarball.
	# The main distribution consists of the project being built, installed
	# and archived. We have to split it per operating system, and Linux
	# is special because we build under a manylinux container which gives
	# broad compatibility. We use the Python based manylinux containers
	# since they come packaged with all dev tools we need. Note that the
	# manylinux containers have a default python 2.x with all supported
	# python versions under /opt/python (need to add one to the path). It
	# is not enough to just invoke it directly because then pip managed
	# console scripts (like cmake, ninja) will not be on the path.
	##########################################################################
	- name: Main distribution (Linux)
	if: "matrix.build-package == 'main-dist-linux'"
	shell: bash
	run: \|
	docker run --rm -w=/work \
	-v $PWD:/work \
	"${MANYLINUX_X86_64_IMAGE}" \
	bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./main_checkout/build_tools/github_actions/build_dist.py main-dist'

	##########################################################################
	# py-runtime-pkg
	# Builds the iree-runtime and iree-runtime-instrumented wheels.
	# One step per OS.
	##########################################################################

	- name: Build runtime wheels (Linux)
	if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'linux'"
	shell: bash
	env:
	package_suffix: ${{ github.event.inputs.package_suffix }}
	packages: "iree-runtime iree-runtime-instrumented"
	output_dir: "${{ github.workspace }}/bindist"
	run: \|
	./main_checkout/build_tools/python_deploy/build_linux_packages.sh

	- name: Build runtime wheels (MacOS)
	if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'macos'"
	shell: bash
	env:
	package_suffix: ${{ github.event.inputs.package_suffix }}
	packages: "iree-runtime iree-runtime-instrumented"
	output_dir: "${{ github.workspace }}/bindist"
	run: \|
	./main_checkout/build_tools/python_deploy/build_macos_packages.sh

	##########################################################################
	# py-compiler-pkg
	# Builds the iree-compielr wheel.
	# One step per OS.
	##########################################################################
	- name: Build compiler wheels (Linux)
	if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'linux'"
	shell: bash
	env:
	package_suffix: ${{ github.event.inputs.package_suffix }}
	packages: "iree-compiler"
	output_dir: "${{ github.workspace }}/bindist"
	run: \|
	./main_checkout/build_tools/python_deploy/build_linux_packages.sh

	- name: Build compiler wheels (MacOS)
	if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'macos'"
	shell: bash
	env:
	package_suffix: ${{ github.event.inputs.package_suffix }}
	packages: "iree-compiler"
	output_dir: "${{ github.workspace }}/bindist"
	run: \|
	# libzstd on GitHub Action bots is not compatible with MacOS universal.
	# https://github.com/iree-org/iree/issues/9955
	sudo rm -rf /usr/local/lib/libzstd.*.dylib
	sudo rm -rf /usr/local/lib/cmake/zstd/*

	./main_checkout/build_tools/python_deploy/build_macos_packages.sh

	##########################################################################
	# TF Compiler Tools
	# Compiler tools wheels are not python version specific, so just build
	# for one examplar python version.
	# TODO: This currently only builds on Linux and is hardcoded to do so.
	##########################################################################

	- name: Build TF Compiler Tools wheels
	if: "matrix.build-package == 'py-tf-compiler-tools-pkg'"
	shell: bash
	run: \|
	docker run --rm -w=/work \
	-v $PWD:/work \
	"${MANYLINUX_X86_64_IMAGE}" \
	bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./main_checkout/build_tools/github_actions/build_dist.py py-tf-compiler-tools-pkg'

	- uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2
	with:
	# We upload all wheels (which includes deps so that subsequent
	# steps can run without further fetching).
	path: ./bindist/*
	retention-days: 5

	# TODO: Upload the tar.bz2 files too when ready
	- name: Upload Release Assets
	if: github.event.inputs.release_id != ''
	id: upload-release-assets
	uses: dwenegar/upload-release-assets@5bc3024cf83521df8ebfadf00ad0c4614fd59148 # v1
	env:
	GITHUB_TOKEN: ${{ secrets.WRITE_ACCESS_TOKEN }}
	with:
	release_id: ${{ github.event.inputs.release_id }}
	# Only upload iree artifacts.
	assets_path: ./bindist/iree.

	validate_and_publish:
	name: "Invoke workflow to validate and publish release"
	needs: build_core
	if: github.event.inputs.release_id != ''
	runs-on: ubuntu-20.04
	steps:
	- name: "Invoke workflow :: Validate and Publish Release"
	uses: benc-uk/workflow-dispatch@4c044c1613fabbe5250deadc65452d54c4ad4fc7 # v1
	with:
	workflow: Validate and Publish Release
	token: ${{ secrets.WRITE_ACCESS_TOKEN }}
	ref: "${{ env.tag_name }}"
	inputs: '{"release_id": "${{ github.event.inputs.release_id }}", "package_version": "${{ github.event.inputs.package_version }}", "build_run_id": "${{ github.run_id }}"}'