Add KWS Streaming dependency to Kokoro Docker images (#5986)
Adds a source dependency on the
[KWS Streaming library](https://github.com/google-research/google-research/tree/master/kws_streaming)
to run the tests on the OSS CI.
diff --git a/build_tools/docker/README.md b/build_tools/docker/README.md
index 07ce7d8..53421e6 100644
--- a/build_tools/docker/README.md
+++ b/build_tools/docker/README.md
@@ -77,6 +77,7 @@
sure to update `IMAGES_TO_DEPENDENCIES` in `manage_images.py` as well. If
you are adding new images, it is best add them via `git add` before
proceeding.
+
2. Build the image, push the image to GCR and update all references to the image
with the new GCR digest:
diff --git a/build_tools/docker/cmake-bazel-frontends-android/Dockerfile b/build_tools/docker/cmake-bazel-frontends-android/Dockerfile
index ff1416f..f310264 100644
--- a/build_tools/docker/cmake-bazel-frontends-android/Dockerfile
+++ b/build_tools/docker/cmake-bazel-frontends-android/Dockerfile
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM gcr.io/iree-oss/cmake-bazel-frontends@sha256:699f6470e618c151f36cb4769ccaf2cfbf334ddb2fc2b6e69eb60732fe69bb6b AS final
+FROM gcr.io/iree-oss/cmake-bazel-frontends@sha256:3c2945cf40699ac38d68ebd56b1ca9d72ed3e2fbe14f032a2a4fdc000b96b657 AS final
# Note that NDK_VERSION should be kept in sync with the one in cmake-android.
# That actually means this isn't really an ARG because we cannot build this
diff --git a/build_tools/docker/cmake-bazel-frontends-nvidia/Dockerfile b/build_tools/docker/cmake-bazel-frontends-nvidia/Dockerfile
index 3941ddc..87e0cef 100644
--- a/build_tools/docker/cmake-bazel-frontends-nvidia/Dockerfile
+++ b/build_tools/docker/cmake-bazel-frontends-nvidia/Dockerfile
@@ -41,7 +41,7 @@
# does not support Ubuntu 18.04.
# This allows to share configuration with base CMake, but it also means we need
# to MATCH the driver version between the host machine and the docker image.
-FROM gcr.io/iree-oss/cmake-bazel-frontends-vulkan@sha256:02c21884ec62a548992fc62031c41375d5cb7ebc33cb4ecbb8ac0ec2c83a2aa1 AS final
+FROM gcr.io/iree-oss/cmake-bazel-frontends-vulkan@sha256:fea4d99f19890a8e53adc2c0ca33a2ef845cdad5bcaae96224bc7d783ec83d19 AS final
ARG NVIDIA_COMMON_DEB
ARG NVIDIA_GL_DEB
ARG NVIDIA_COMPUTE_DEB
diff --git a/build_tools/docker/cmake-bazel-frontends-swiftshader/Dockerfile b/build_tools/docker/cmake-bazel-frontends-swiftshader/Dockerfile
index 2086d95..f1fabdb 100644
--- a/build_tools/docker/cmake-bazel-frontends-swiftshader/Dockerfile
+++ b/build_tools/docker/cmake-bazel-frontends-swiftshader/Dockerfile
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM gcr.io/iree-oss/cmake-bazel-frontends-vulkan@sha256:02c21884ec62a548992fc62031c41375d5cb7ebc33cb4ecbb8ac0ec2c83a2aa1 AS final
+FROM gcr.io/iree-oss/cmake-bazel-frontends-vulkan@sha256:fea4d99f19890a8e53adc2c0ca33a2ef845cdad5bcaae96224bc7d783ec83d19 AS final
COPY --from=gcr.io/iree-oss/swiftshader@sha256:ccae32c83c89a31e8fc5542e480c29f28bbf4a3b3b80198c06b687a92c6813f3 \
/swiftshader /swiftshader
diff --git a/build_tools/docker/cmake-bazel-frontends-vulkan/Dockerfile b/build_tools/docker/cmake-bazel-frontends-vulkan/Dockerfile
index c422cc3..bd2f109 100644
--- a/build_tools/docker/cmake-bazel-frontends-vulkan/Dockerfile
+++ b/build_tools/docker/cmake-bazel-frontends-vulkan/Dockerfile
@@ -15,7 +15,7 @@
# This image provides the Vulkan SDK. Requires a child image to provide
# a Vulkan ICD.
-FROM gcr.io/iree-oss/cmake-bazel-frontends@sha256:699f6470e618c151f36cb4769ccaf2cfbf334ddb2fc2b6e69eb60732fe69bb6b AS final
+FROM gcr.io/iree-oss/cmake-bazel-frontends@sha256:3c2945cf40699ac38d68ebd56b1ca9d72ed3e2fbe14f032a2a4fdc000b96b657 AS final
ARG VULKAN_SDK_VERSION=1.2.154.0
diff --git a/build_tools/docker/cmake-bazel-frontends/Dockerfile b/build_tools/docker/cmake-bazel-frontends/Dockerfile
index 3d54b25..b937596 100644
--- a/build_tools/docker/cmake-bazel-frontends/Dockerfile
+++ b/build_tools/docker/cmake-bazel-frontends/Dockerfile
@@ -12,6 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+FROM gcr.io/iree-oss/util@sha256:40846b4aea5886af3250399d6adfdb3e1195a8b0177706bb0375e812d62dc49c AS fetch-kws
+WORKDIR /fetch-kws
+ARG KWS_COMMIT=4bfbfe4ec19fe39cf9d1803e894d8bfb5fd56a45
+RUN wget "https://github.com/google-research/google-research/tarball/${KWS_COMMIT?}" \
+ -O google-research.tar.gz \
+ && tar --extract --gzip --file=google-research.tar.gz \
+ --wildcards */kws_streaming --strip-components=1
+
FROM gcr.io/iree-oss/cmake-python@sha256:c1f05139fa478107bdd5889d6b3ca9f8a4bcf19e77e3e8058e2084367a1aa23d AS final
# Install bazel.
@@ -24,3 +32,8 @@
# Install JAX.
RUN python3 -m pip install --upgrade jax jaxlib flax
+
+# Install KWS Streaming Library.
+RUN python3 -m pip install --upgrade tensorflow-model-optimization==0.5.1.dev0
+COPY --from=fetch-kws /fetch-kws/kws_streaming /local-python/kws_streaming
+ENV PYTHONPATH="/local-python:$PYTHONPATH"
diff --git a/build_tools/docker/prod_digests.txt b/build_tools/docker/prod_digests.txt
index c126126..3eb2e44 100644
--- a/build_tools/docker/prod_digests.txt
+++ b/build_tools/docker/prod_digests.txt
@@ -10,9 +10,9 @@
gcr.io/iree-oss/cmake-python-vulkan@sha256:6722f69c6300749f6bd4b141fc653244990381a6b0111f9c361061adcd65c07c
gcr.io/iree-oss/cmake-python-swiftshader@sha256:0be2b0c735a038365e7cad31f6b440805dd4e231e166a114ef22914a5469cbc8
gcr.io/iree-oss/cmake-python-nvidia@sha256:04b8257becb1b2916b1dcec952cf0380963e3ba04f0a7cdcbab1c13995879bb4
-gcr.io/iree-oss/cmake-bazel-frontends@sha256:699f6470e618c151f36cb4769ccaf2cfbf334ddb2fc2b6e69eb60732fe69bb6b
-gcr.io/iree-oss/cmake-bazel-frontends-vulkan@sha256:02c21884ec62a548992fc62031c41375d5cb7ebc33cb4ecbb8ac0ec2c83a2aa1
-gcr.io/iree-oss/cmake-bazel-frontends-nvidia@sha256:7fa938de1e5b2732f5c9818c5c8d3d5346a2772a2f935f85016710342e72e9f7
-gcr.io/iree-oss/cmake-bazel-frontends-swiftshader@sha256:012dd9b452da5c4f4248edf27a7a7faa0eeca6a7a2a1e21d00bc932b125246e1
+gcr.io/iree-oss/cmake-bazel-frontends@sha256:3c2945cf40699ac38d68ebd56b1ca9d72ed3e2fbe14f032a2a4fdc000b96b657
+gcr.io/iree-oss/cmake-bazel-frontends-vulkan@sha256:fea4d99f19890a8e53adc2c0ca33a2ef845cdad5bcaae96224bc7d783ec83d19
+gcr.io/iree-oss/cmake-bazel-frontends-nvidia@sha256:384fc9fd18bd12843ccd57d59b13782823fff9cc8192e276159a41669de50125
+gcr.io/iree-oss/cmake-bazel-frontends-swiftshader@sha256:d0561fe61107011284e3339f66d115301e2e9946cf8304f617e3d6ffba365489
gcr.io/iree-oss/cmake-riscv@sha256:a09ff1e6ab65a436822894acf58ef6c4cbc523581960e918a07ddf4a46c8af95
-gcr.io/iree-oss/cmake-bazel-frontends-android@sha256:dd11a49fa357ac3f96ef53c26b1faaf24ce834383300d1ed464e810bb33fafe7
+gcr.io/iree-oss/cmake-bazel-frontends-android@sha256:4175d0f29efa5643f576327ec61a05bedc6d4f741e4a0bbfd26fc3c887140061
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build_kokoro.sh
index d2d5ca4..812d697 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build_kokoro.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build_kokoro.sh
@@ -32,7 +32,7 @@
docker_setup
docker run "${DOCKER_RUN_ARGS[@]?}" \
- gcr.io/iree-oss/cmake-bazel-frontends-swiftshader@sha256:012dd9b452da5c4f4248edf27a7a7faa0eeca6a7a2a1e21d00bc932b125246e1 \
+ gcr.io/iree-oss/cmake-bazel-frontends-swiftshader@sha256:d0561fe61107011284e3339f66d115301e2e9946cf8304f617e3d6ffba365489 \
build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
# Kokoro will rsync this entire directory back to the executor orchestrating the
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build_kokoro.sh
index 25932a0..11749dd 100755
--- a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build_kokoro.sh
+++ b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build_kokoro.sh
@@ -38,7 +38,7 @@
docker run "${DOCKER_RUN_ARGS[@]?}" \
--gpus all \
- gcr.io/iree-oss/cmake-bazel-frontends-nvidia@sha256:7fa938de1e5b2732f5c9818c5c8d3d5346a2772a2f935f85016710342e72e9f7 \
+ gcr.io/iree-oss/cmake-bazel-frontends-nvidia@sha256:384fc9fd18bd12843ccd57d59b13782823fff9cc8192e276159a41669de50125 \
build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh
# Kokoro will rsync this entire directory back to the executor orchestrating the
diff --git a/integrations/tensorflow/e2e/keras/BUILD b/integrations/tensorflow/e2e/keras/BUILD
index a05e504..92895fd 100644
--- a/integrations/tensorflow/e2e/keras/BUILD
+++ b/integrations/tensorflow/e2e/keras/BUILD
@@ -123,10 +123,6 @@
"iree_vulkan",
],
},
- tags = [
- # Kokoro doesn't have kws_streaming installed
- "nokokoro",
- ],
deps = [
"//third_party/google_research/google_research/kws_streaming/models:models_lib",
"//third_party/google_research/google_research/kws_streaming/train:train_lib",
@@ -169,10 +165,6 @@
"iree_vulkan",
],
},
- tags = [
- # Kokoro doesn't have kws_streaming installed
- "nokokoro",
- ],
deps = [
"//third_party/google_research/google_research/kws_streaming/models:models_lib",
"//third_party/google_research/google_research/kws_streaming/train:train_lib",
@@ -215,10 +207,6 @@
"iree_vulkan",
],
},
- tags = [
- # Kokoro doesn't have kws_streaming installed
- "nokokoro",
- ],
deps = [
"//third_party/google_research/google_research/kws_streaming/models:models_lib",
"//third_party/google_research/google_research/kws_streaming/train:train_lib",
diff --git a/integrations/tensorflow/e2e/keras/CMakeLists.txt b/integrations/tensorflow/e2e/keras/CMakeLists.txt
index 77bf1d7..a8e29cd 100644
--- a/integrations/tensorflow/e2e/keras/CMakeLists.txt
+++ b/integrations/tensorflow/e2e/keras/CMakeLists.txt
@@ -39,8 +39,6 @@
",,,crnn,iree_llvmaot"
",,,gru,iree_llvmaot"
",,,lstm,iree_llvmaot"
- LABELS
- "nokokoro"
)
iree_e2e_cartesian_product_test_suite(
@@ -71,8 +69,6 @@
",,,tc_resnet,"
",,,xception,"
",,,crnn,iree_vulkan"
- LABELS
- "nokokoro"
)
iree_e2e_cartesian_product_test_suite(
@@ -105,8 +101,6 @@
",,,svdf_resnet,"
",,,tc_resnet,"
",,,xception,"
- LABELS
- "nokokoro"
)
### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
