pw_env_setup: Explicitly list CIPD platforms Explicitly list CIPD package platforms in JSON files. This makes it easier to follow which packages are loaded for each platform and lets us be clearer about which platforms are supported. This will also simplify roller configs because the rollers will no longer need to list platforms in builder properties. Instead, they can be processed from the JSON files. As a result there should be fewer issues where rollers are misconfigured. Roller recipe and config CLs will follow. Since recipe environment setup now uses pw_env_setup no changes need to be made to it to support this. Change-Id: I80db2d8926b259c880887765d110f287bc7a5ed3 Bug: 385 Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/46180 Reviewed-by: Wyatt Hepler <hepler@google.com> Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/pw_env_setup/docs.rst b/pw_env_setup/docs.rst index e6d9fb7..06e3295 100644 --- a/pw_env_setup/docs.rst +++ b/pw_env_setup/docs.rst
@@ -167,8 +167,23 @@ .. _sample project: https://pigweed.googlesource.com/pigweed/sample_project/+/master ``cipd_package_files`` - CIPD package file. JSON file consisting of a list of dictionaries with "path" - and "tags" keys, where "tags" is a list of strings. + CIPD package file. JSON file consisting of a list of dictionaries with "path", + "platforms", and "tags" keys. An example is below. + +.. code-block:: json + + { + "path": "infra/3pp/tools/go/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], + "tags": [ + "version:2@1.16.3" + ] + }, ``virtualenv.gn_targets`` Target for installing Python packages. Downstream projects will need to
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json b/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json index 240a571..ef62430 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json
@@ -1,42 +1,83 @@ [ { "path": "infra/tools/bb/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "latest" ] }, { "path": "infra/tools/luci-auth/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "latest" ] }, { "path": "infra/tools/luci/gerrit/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "latest" ] }, { "path": "infra/tools/luci/gitiles/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "latest" ] }, { "path": "infra/tools/luci/cas/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "latest" ] }, { "path": "infra/tools/luci/led/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "latest" ] }, { - "path": "infra/tools/luci/logdog/logdog/${os}-${arch=amd64}", + "path": "infra/tools/luci/logdog/logdog/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "latest" ]
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json index 2ca677c..0eb60f0 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
@@ -1,120 +1,207 @@ [ { - "path": "gn/gn/${os}-${arch=amd64,arm64}", + "path": "gn/gn/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "git_revision:39a87c0b36310bdf06b692c098f199a0d97fc810" ] }, { - "path": "infra/ninja/${os}-${arch=amd64}", + "path": "infra/ninja/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "version:1.9.0" ] }, { - "path": "infra/cmake/${os}-${arch=amd64}", + "path": "infra/cmake/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "version:3.16.1" ] }, { "_comment": "TODO(pwbug/93): Package Bazel for Windows.", - "path": "fuchsia/third_party/bazel/${os=linux,mac}-${arch=amd64}", + "path": "fuchsia/third_party/bazel/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64" + ], "tags": [ "version:4.0.0" ] }, { - "path": "pigweed/third_party/bloaty-embedded/${os=linux,mac}-${arch=amd64}", + "path": "pigweed/third_party/bloaty-embedded/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64" + ], "tags": [ "git_revision:2d87d204057b419f5290f8d38b61b9c2c5b4fb52-2" ] }, { - "path": "pigweed/third_party/gcc-arm-none-eabi/${os}-${arch=amd64}", + "path": "pigweed/third_party/gcc-arm-none-eabi/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "version:10-2020-q4-major" ] }, { - "path": "infra/3pp/tools/protoc/${os}-${arch}", + "path": "infra/3pp/tools/protoc/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "version:3.14.0" ] }, { "_comment": "TODO(pwbug/70) Put clang in cipd for Windows.", - "path": "fuchsia/third_party/clang/${os=linux,mac}-${arch}", + "path": "fuchsia/third_party/clang/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64" + ], "tags": [ "git_revision:3dc24bc31edbc01dea085b24a6a6b024d7ae531c" ] }, { "_comment": "When bumping the minor version (e.g., to 3.9.x) also update env_setup/virtualenv/init.py to check for the new version.", - "path": "infra/3pp/tools/cpython3/${os}-${arch}", + "path": "infra/3pp/tools/cpython3/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "version:3.8.2.chromium.10" ] }, { "path": "infra/3pp/tools/go/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "version:2@1.16.3" ] }, { - "path": "pigweed/third_party/protoc-gen-go/${os}-${arch=amd64}", + "path": "pigweed/third_party/protoc-gen-go/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "version:1.3.2" ] }, { "_comment": "TODO(pwbug/66) Put openocd in cipd for Windows.", - "path": "pigweed/third_party/openocd/${os=linux,mac}-${arch=amd64}", + "path": "pigweed/third_party/openocd/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64" + ], "tags": [ "git_revision:e41c0f4906e46d1076ce62a0da5518aa1ca280b8" ] }, { - "path": "pigweed/third_party/mingw64-x86_64-win32-seh/${os=windows}-${arch=amd64}", + "path": "pigweed/third_party/mingw64-x86_64-win32-seh/${platform}", + "platforms": [ + "windows-amd64" + ], "tags": [ "version:10.2.0-11" ] }, { - "path": "pigweed/host_tools/${os}-${arch=amd64}", + "path": "pigweed/host_tools/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], "tags": [ "git_revision:3f00a3eb4957306eab148fe3651844e46df5b379" ] }, { - "path": "infra/rbe/client/${os=linux,windows}-${arch=amd64}", + "path": "infra/rbe/client/${platform}", + "platforms": [ + "linux-amd64", + "windows-amd64" + ], "tags": [ "git_revision:8e834982e020c12b7c3c1f106961cead3d65dec1" ] }, { - "path": "fuchsia/third_party/qemu/${os=linux,mac}-${arch}", + "path": "fuchsia/third_party/qemu/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64" + ], "tags": [ "git_revision:841f14e74f5af7886cf49cfcd4fed264911ae58e" ] }, { "path": "fuchsia/third_party/kythe", + "platforms": [ + "linux-amd64" + ], "subdir": "kythe", "tags": [ "version:1.0.2" ] }, { - "path": "fuchsia/third_party/kythe-libs/${os=linux}-${arch=amd64}", + "path": "fuchsia/third_party/kythe-libs/${platform}", + "platforms": [ + "linux-amd64" + ], "subdir": "kythe", "tags": [ "version:2020-08-05" ] }, { - "path": "infra/3pp/tools/renode/${os=linux}-${arch=amd64}", + "path": "infra/3pp/tools/renode/${platform}", + "platforms": [ + "linux-amd64" + ], "tags": [ "version:2@renode-1.11.0+20210306gite7897c1" ]
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py index c4301ec..922d589 100755 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
@@ -24,6 +24,7 @@ import argparse import json import os +import platform import re import shutil import subprocess @@ -148,9 +149,28 @@ return True +def _platform(): + osname = { + 'darwin': 'mac', + 'linux': 'linux', + 'windows': 'windows', + }[platform.system().lower()] + + if platform.machine().startswith(('aarch64', 'armv8')): + arch = 'arm64' + elif platform.machine() == 'x86_64': + arch = 'amd64' + elif platform.machine() == 'i686': + arch = 'i386' + else: + arch = platform.machine() + + return '{}-{}'.format(osname, arch).lower() + + def write_ensure_file(package_file, ensure_file): with open(package_file, 'r') as ins: - data = json.load(ins) + packages = json.load(ins) # TODO(pwbug/103) Remove 30 days after bug fixed. if os.path.isdir(ensure_file): @@ -161,10 +181,14 @@ '$VerifiedPlatform mac-amd64\n' '$ParanoidMode CheckPresence\n') - for entry in data: - outs.write('@Subdir {}\n'.format(entry.get('subdir', ''))) - outs.write('{} {}\n'.format(entry['path'], - ' '.join(entry['tags']))) + for pkg in packages: + # If this is a new-style package manifest platform handling must + # be done here instead of by the cipd executable. + if 'platforms' in pkg and _platform() not in pkg['platforms']: + continue + + outs.write('@Subdir {}\n'.format(pkg.get('subdir', ''))) + outs.write('{} {}\n'.format(pkg['path'], ' '.join(pkg['tags']))) def update(