Update the PIP build process to package from either bazel or cmake.
This lets us just build once (when primarily packaging for TF).
Closes https://github.com/google/iree/pull/2119
COPYBARA_INTEGRATE_REVIEW=https://github.com/google/iree/pull/2119 from stellaraccident:winpips1 f795fca455ab6c51b0fdf39c469a89f50b5bf89a
PiperOrigin-RevId: 314850314
diff --git a/integrations/tensorflow/bindings/python/packaging/BUILD.bazel b/bindings/python/packaging/BUILD.bazel
similarity index 82%
rename from integrations/tensorflow/bindings/python/packaging/BUILD.bazel
rename to bindings/python/packaging/BUILD.bazel
index bd82933..f95ba18 100644
--- a/integrations/tensorflow/bindings/python/packaging/BUILD.bazel
+++ b/bindings/python/packaging/BUILD.bazel
@@ -15,13 +15,19 @@
# This is a dummy binary that has the side-effect of building all of the TF
# python bindings. It is used to build wheel files.
py_binary(
- name = "all_tf_packages",
+ name = "all_pyiree_packages",
srcs = ["dummy_exclude_from_package.py"],
legacy_create_init = False,
main = "dummy_exclude_from_package.py",
python_version = "PY3",
+ tags = [
+ # Do not build with ... expansion
+ "manual",
+ ],
deps = [
"//bindings/python:pathsetup", # build_cleaner: keep
+ "//bindings/python/pyiree/compiler", # build_cleaner: keep
+ "//bindings/python/pyiree/rt", # build_cleaner: keep
"//integrations/tensorflow/bindings/python/pyiree/tf/compiler", # build_cleaner: keep
"//integrations/tensorflow/bindings/python/pyiree/tf/support", # build_cleaner: keep
],
diff --git a/bindings/python/packaging/README.md b/bindings/python/packaging/README.md
index 82b431a..8e9c7cf 100644
--- a/bindings/python/packaging/README.md
+++ b/bindings/python/packaging/README.md
@@ -5,15 +5,18 @@
will create `build` and `dist` directories where run. Wheels can be installed
with `pip3 install --user dist/*.whl`.
-## Building core wheels
+## Building core wheels with CMake
-Most of IREE is built/packaged with CMake. Canonical instructions follow:
+Most of IREE is built/packaged with CMake. For the parts that build with CMake,
+this is preferred.
+
+Canonical instructions follow:
### Linux
```shell
export LDFLAGS=-fuse-ld=/usr/bin/ld.lld-10
-export CMAKE_BUILD_ROOT=$HOME/build-iree-release
+export PYIREE_CMAKE_BUILD_ROOT=$HOME/build-iree-release
export IREE_SRC=$HOME/src/iree
rm -Rf $CMAKE_BUILD_ROOT; mkdir -p $CMAKE_BUILD_ROOT
cmake -GNinja -B$CMAKE_BUILD_ROOT -H$IREE_SRC \
@@ -30,16 +33,46 @@
## Building IREE/TensorFlow wheels
-TensorFlow integration must be built via Bazel. Canonical instructions follow:
+If building TensorFlow integration wheels, then this must be done via Bazel. In
+this case, it can be easiest to just package everything from a Bazel build to
+avoid multiple steps.
-### Linux
+Canonical instructions follow:
+
+### Env Setup
```shell
-export IREE_SRC=$HOME/src/iree
+IREE_SRC=$HOME/src/iree
+export PYIREE_BAZEL_BUILD_ROOT="$IREE_SRC/bazel-bin"
+if which cygpath; then
+ export PYIREE_BAZEL_BUILD_ROOT="$(cygpath -w "$PYIREE_BAZEL_BUILD_ROOT")"
+fi
+```
+
+### Building:
+
+```shell
cd $IREE_SRC
bazel build -c opt \
- //integrations/tensorflow/bindings/python/packaging:all_tf_packages
+ //bindings/python/packaging:all_pyiree_packages
+```
+
+# Packaging
+
+```shell
(cd $IREE_SRC/bindings/python/packaging && (
rm -Rf build;
python3 setup_tf.py bdist_wheel))
```
+
+```shell
+(cd $IREE_SRC/bindings/python/packaging && (
+rm -Rf build;
+python3 setup_compiler.py bdist_wheel))
+```
+
+```shell
+(cd $IREE_SRC/bindings/python/packaging && (
+rm -Rf build;
+python3 setup_rt.py bdist_wheel))
+```
diff --git a/bindings/python/packaging/common_setup.py b/bindings/python/packaging/common_setup.py
index 823b6a0..8fc0f4a 100644
--- a/bindings/python/packaging/common_setup.py
+++ b/bindings/python/packaging/common_setup.py
@@ -19,14 +19,48 @@
from datetime import date
-def get_package_dir():
- if "CMAKE_BUILD_ROOT" in os.environ:
- cmake_build_root = os.environ["CMAKE_BUILD_ROOT"]
+def get_exe_suffix():
+ if platform.system() == "Windows":
+ return ".exe"
+ else:
+ return ""
+
+
+def get_package_dir(prefix=("bindings", "python")):
+ cmake_build_root = os.environ.get("PYIREE_CMAKE_BUILD_ROOT")
+ bazel_build_root = os.environ.get("PYIREE_BAZEL_BUILD_ROOT")
+
+ if cmake_build_root and bazel_build_root:
+ print("ERROR: Both PYIREE_CMAKE_BUILD_ROOT and PYIREE_BAZEL_BUILD_ROOT"
+ "cannot be set at the same time")
+ sys.exit(1)
+
+ if cmake_build_root:
print("Using CMake build root:", cmake_build_root)
- pkg_dir = os.path.join(cmake_build_root, "bindings", "python")
+ pkg_dir = os.path.join(cmake_build_root, *prefix)
+ elif bazel_build_root:
+ print("Using Bazel build root:", bazel_build_root)
+ if not os.path.isdir(bazel_build_root):
+ print("ERROR: Could not find bazel-bin:", bazel_build_root)
+ sys.exit(1)
+ # Find the path to the runfiles of the built target:
+ # //bindings/python/packaging:all_pyiree_packages
+ runfiles_dir = os.path.join(
+ bazel_build_root, "bindings", "python", "packaging",
+ "all_pyiree_packages%s.runfiles" % (get_exe_suffix(),))
+ if not os.path.isdir(runfiles_dir):
+ print("ERROR: Could not find build target 'all_pyiree_packages':",
+ runfiles_dir)
+ print("Make sure to build target",
+ "//bindings/python/packaging:all_pyiree_packages")
+ sys.exit(1)
+ # And finally seek into the corresponding path in the runfiles dir.
+ # Aren't bazel paths fun???
+ # Note that the "iree_core" path segment corresponds to the workspace name.
+ pkg_dir = os.path.join(runfiles_dir, "iree_core", *prefix)
else:
print("ERROR: No build directory specified. Set one of these variables:")
- print(" CMAKE_BUILD_ROOT=/path/to/cmake/build")
+ print(" PYIREE_CMAKE_BUILD_ROOT=/path/to/cmake/build")
sys.exit(1)
if not os.path.exists(pkg_dir):
diff --git a/integrations/tensorflow/bindings/python/packaging/dummy_exclude_from_package.py b/bindings/python/packaging/dummy_exclude_from_package.py
similarity index 100%
rename from integrations/tensorflow/bindings/python/packaging/dummy_exclude_from_package.py
rename to bindings/python/packaging/dummy_exclude_from_package.py
diff --git a/bindings/python/packaging/setup_tf.py b/bindings/python/packaging/setup_tf.py
index 9ecec37..9b1432d 100644
--- a/bindings/python/packaging/setup_tf.py
+++ b/bindings/python/packaging/setup_tf.py
@@ -32,6 +32,7 @@
# dependencies.
import os
+import platform
import setuptools
import sys
@@ -40,36 +41,9 @@
import common_setup
-def find_bazel_runfiles_dir():
- bazel_bin = os.path.abspath(
- os.path.join(os.path.dirname(__file__), "..", "..", "..", "bazel-bin"))
- if not os.path.isdir(bazel_bin):
- print("ERROR: Could not find bazel-bin:", bazel_bin)
- sys.exit(1)
- # Find the path to the runfiles of the built target:
- # //integrations/tensorflow/bindings/python/packaging:all_tf_packages
- runfiles_dir = os.path.join(bazel_bin, "integrations", "tensorflow",
- "bindings", "python", "packaging",
- "all_tf_packages.runfiles")
- if not os.path.isdir(runfiles_dir):
- print("ERROR: Could not find build target 'all_tf_packages':", runfiles_dir)
- print(
- "Make sure to build target",
- "//integrations/tensorflow/bindings/python/packaging:all_tf_packages")
- sys.exit(1)
- # And finally seek into the corresponding path in the runfiles dir.
- # Aren't bazel paths fun???
- # Note that the "iree_core" path segment corresponds to the workspace name.
- package_path = os.path.join(runfiles_dir, "iree_core", "integrations",
- "tensorflow", "bindings", "python")
- if not os.path.isdir(package_path):
- print("ERROR: Could not find built python package:", package_path)
- sys.exit(1)
- return package_path
-
-
def run():
- package_dir = find_bazel_runfiles_dir()
+ package_dir = common_setup.get_package_dir(
+ prefix=("integrations", "tensorflow", "bindings", "python"))
packages = setuptools.find_namespace_packages(
package_dir,
include=[
