Delete old git_scripts directory Because of weirdness in our automation, we can't edit .github/workflow files in the same PR as other changes. These were moved to scripts/git in https://github.com/google/iree/pull/671 and the workflow files were redirected in https://github.com/google/iree/pull/672. PiperOrigin-RevId: 293220646
diff --git a/git_scripts/__init__.py b/git_scripts/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/git_scripts/__init__.py +++ /dev/null
@@ -1 +0,0 @@ -
diff --git a/git_scripts/submodule_versions.py b/git_scripts/submodule_versions.py deleted file mode 100755 index c65d051..0000000 --- a/git_scripts/submodule_versions.py +++ /dev/null
@@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# Lint as: python3 -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=missing-docstring -"""submodule_versions. - -Synchronizes the tracked SUBMODULE_VERSIONS file with the submodule state -in git. - -Typical usage: --------------- -Exporting current git submodule state to SUBMODULE_VERSIONS: - Syntax: ./git_scripts/submodule_versions.py export - -Importing versions in SUBMODULE_VERSIONS to git submodule state: - Syntax: ./git_scripts/submodule_versions.py import - -Checking whether SUBMODULE_VERSIONS and git state are in sync: - Syntax: ./git_scripts/submodule_versions.py check -""" - -import argparse -import os -import re -import sys - -import utils - -VERSIONS_FILE = "SUBMODULE_VERSIONS" - - -def get_submodule_versions(repo_dir): - raw_status = utils.execute(["git", "submodule", "status"], - cwd=repo_dir, - silent=True, - capture_output=True).decode("UTF-8") - status_lines = [] - for line in raw_status.splitlines(): - # Format is a status char followed by revision, space and path. - m = re.match(r"""^.([0-9a-z]+)\s+([^\s]+)""", line) - if m: - # Output as just the commit hash followed by space and path. - status_lines.append(m.group(1) + " " + m.group(2)) - return "\n".join(status_lines) + "\n" - - -def export_versions(repo_dir): - current_versions = get_submodule_versions(repo_dir) - versions_file_path = os.path.join(repo_dir, VERSIONS_FILE) - print("*** Exporting current submodule versions to:", versions_file_path) - with open(versions_file_path, "w", encoding="UTF-8") as f: - f.write(current_versions) - utils.execute(["git", "add", VERSIONS_FILE], cwd=repo_dir) - - -def parse_versions(versions_text): - versions = dict() - for line in versions_text.splitlines(): - comps = line.split(" ", maxsplit=2) - if len(comps) != 2: - continue - versions[comps[1]] = comps[0] - return versions - - -def get_diff_versions(repo_dir): - current_versions = parse_versions(get_submodule_versions(repo_dir)) - with open(os.path.join(repo_dir, VERSIONS_FILE), "r", encoding="UTF-8") as f: - written_versions = parse_versions(f.read()) - diff_versions = current_versions.items() ^ written_versions.items() - return { - k: (current_versions.get(k), written_versions.get(k)) - for k, _ in diff_versions - } - - -def sync_and_update_submodules(repo_dir): - print("*** Synchronizing/updating submodules") - utils.execute(["git", "submodule", "sync"], cwd=repo_dir) - utils.execute(["git", "submodule", "update"], cwd=repo_dir) - - -def import_versions(repo_dir): - print("*** Importing versions to git submodule state") - diff_versions = get_diff_versions(repo_dir) - if not diff_versions: - print("*** No submodule updates required") - return - for path, (current, written) in diff_versions.items(): - if current is None: - print(("Warning: Submodule %s does not exist but is " - "still in the version file") % (path,)) - continue - if written is None: - print("Warning: Submodule %s is not in the version file" % (current,)) - continue - # Directly update the submodule commit hash in the index. - # See: https://stackoverflow.com/questions/33514642 - command = ["git", "update-index", "--cacheinfo", "160000", written, path] - print("Updating", path, "to", written) - utils.execute(command, cwd=repo_dir) - - -def init_submodules(repo_dir): - print("*** Initializing submodules") - utils.execute(["git", "submodule", "init"], cwd=repo_dir) - - -def parallel_shallow_update_submodules(repo_dir): - print("*** Making shallow clone of submodules") - # TODO(gcmn) Figure out a way to quickly fetch submodules without relying on - # target SHA being within 10000 commits of HEAD. - magic_depth = 10000 - utils.execute([ - "git", "submodule", "update", "--jobs", "8", "--depth", - str(magic_depth) - ], - cwd=repo_dir) - - -def check_submodule_versions(repo_dir): - diff_versions = get_diff_versions(repo_dir) - if diff_versions: - print( - "Submodule state differs from SUBMODULE_VERSIONS file. Run (and commit) one of:" - ) - print( - " ./git_scripts/submodule_versions.py import # Use version in SUBMODULE_VERSIONS" - ) - print( - " ./git_scripts/submodule_versions.py export # Use version in git state" - ) - for k, (current, written) in diff_versions.items(): - print("%s : actual=%s written=%s" % (k, current, written)) - return False - return True - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--repo", help="Repository root directory") - parser.add_argument( - "command", help="Command to run (show|import|export|check|init)") - args = parser.parse_args() - - # Default repo path. - if args.repo is None: - args.repo = utils.find_git_toplevel() - return args - - -def main(args): - if args.command == "show": - print(get_submodule_versions(args.repo)) - elif args.command == "export": - sync_and_update_submodules(args.repo) - export_versions(args.repo) - elif args.command == "check": - if not check_submodule_versions(args.repo): - sys.exit(1) - elif args.command == "import": - import_versions(args.repo) - sync_and_update_submodules(args.repo) - elif args.command == "init": - init_submodules(args.repo) - # Redundant, since import_versions will only update if they differ, - # but good to only print output about the import if it's actually - # needed. - if not check_submodule_versions(args.repo): - print("Warning: git submodule state does not match SUBMODULE_VERSIONS. " - "Using state in SUBMODULE_VERSIONS") - import_versions(args.repo) - parallel_shallow_update_submodules(args.repo) - else: - print("Unrecognized command:", args.command) - sys.exit(1) - - -if __name__ == "__main__": - main(parse_arguments())
diff --git a/git_scripts/update_tf_llvm_submodules.py b/git_scripts/update_tf_llvm_submodules.py deleted file mode 100755 index 92a5550..0000000 --- a/git_scripts/update_tf_llvm_submodules.py +++ /dev/null
@@ -1,203 +0,0 @@ -#!/usr/bin/env python3 -# Lint as: python3 -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=missing-docstring -"""update_tf_llvm_submodules. - -Updates the third_party/tensorflow and third_party/llvm-project submodules -to new commits. We have special conditions around these submodules since -upstream will only accept an llvm-project version that is sync'd with the -corresponding version that tensorflow depends on. In addition, some BUILD -files must be sync'd for the new version. - -Typical usage: - Syntax: ./git_scripts/update_tf_llvm_modules.py - - By default, this will update the tensorflow submodule to remote HEAD and - update the llvm-project submodule to the corresponding version. It will - also sync BUILD file changes as needed and export the version metadata. -""" - -import argparse -import re -import os -import sys - -import submodule_versions -import utils - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--repo", help="Repository root directory") - parser.add_argument( - "--tensorflow", - help="Path to the tensorflow sources " - "(default to third_party/tensorflow)", - default=None) - parser.add_argument( - "--llvm", - help="Path to the LLVM sources " - "(defaults to third_party/llvm-project)", - default=None) - parser.add_argument( - "--tensorflow_commit", - help="Update TensorFlow to this commit (or 'KEEP', 'REMOTE')", - default="REMOTE") - parser.add_argument( - "--llvm_commit", - help="Update LLVM to this commit (or 'KEEP', 'REMOTE', 'TENSORFLOW')", - default="TENSORFLOW") - parser.add_argument( - "--update_build_files", - help="Updates the IREE LLVM build files from TensorFlow", - type=utils.str2bool, - nargs="?", - default=False) - args = parser.parse_args() - - # Default repo path. - if args.repo is None: - args.repo = utils.find_git_toplevel() - - # Set some defaults. - if not args.tensorflow: - args.tensorflow = os.path.join(args.repo, "third_party", "tensorflow") - if not args.llvm: - args.llvm = os.path.join(args.repo, "third_party", "llvm-project") - return args - - -def main(args): - print("IREE handy-dandy-LLVM-submodule-updater at your service...") - print(" IREE Path :", args.repo) - print(" LLVM Path :", args.llvm) - print(" TensorFlow Path :", args.tensorflow) - print(" Update Build files:", args.update_build_files) - current_llvm_commit = get_commit(args.llvm) - current_tensorflow_commit = get_commit(args.tensorflow) - - print("Current Commits: llvm =", current_llvm_commit, "tensorflow =", - current_tensorflow_commit) - - # Update TensorFlow - if args.tensorflow_commit == "KEEP": - print("Not updating TensorFlow (--tensorflow_commit == 'KEEP')") - else: - print("\n*** Updating TensorFlow to", args.tensorflow_commit, "***") - update_submodule(args.tensorflow, args.tensorflow_commit) - stage_path(args.repo, "third_party/tensorflow") - - # Update LLVM. - if args.llvm_commit == "TENSORFLOW": - args.llvm_commit = find_tensorflow_llvm_commit(args.tensorflow) - print("Found TensorFlow's LLVM commit:", args.llvm_commit) - if args.update_build_files is None: - print("Will update build files from TensorFlow", - "because --update_build_files not specified") - args.update_build_files = True - if args.llvm_commit == "KEEP": - print("Not updating LLVM (--llvm_commit == 'KEEP')") - else: - print("\n*** Updating LLVM to", args.llvm_commit, "***") - update_submodule(args.llvm, args.llvm_commit) - stage_path(args.repo, "third_party/llvm-project") - - # Update build files. - if not args.update_build_files: - print("Not updating build files (--update_build_files not specified)") - else: - print("\n*** Updating BUILD.bazel files ***") - update_build_files_from_tensorflow(args.repo, args.tensorflow) - - # Export SUBMODULE_VERSIONS. - print() # Add line break. - submodule_versions.export_versions(args.repo) - - -def get_commit(path, rev="HEAD"): - return utils.execute(["git", "rev-parse", rev], - cwd=path, - silent=True, - capture_output=True).decode("ISO-8859-1").strip() - - -def update_submodule(path, commit, tracking="origin/master"): - # Fetch. - utils.execute(["git", "fetch"], cwd=path) - # Determine commit. - if commit == "REMOTE": - commit = get_commit(path, rev=tracking) - print("Resolved remote commit:", commit) - - # Rebase to commit (will fail if not fast-forward). - utils.execute(["git", "checkout", commit], cwd=path) - - -def find_tensorflow_llvm_commit(tensorflow_path): - # TensorFlow keeps its commit in workspace.bzl on a line like: - # LLVM_COMMIT = "..." - # Yeah. This is how we do it. - workspace_path = os.path.join(tensorflow_path, "tensorflow", "workspace.bzl") - pattern_text = r"""\s*LLVM_COMMIT\s*=\s*"(.+)"\s*""" - pattern = re.compile(pattern_text, flags=re.MULTILINE) - for line in open(workspace_path, "r", encoding="UTF-8"): - m = re.match(pattern, line) - if m: - return m.group(1) - - print("ERROR: Could not find LLVM commit in %s." % workspace_path) - print("Request an explicit commit via --llvm_commit (and file a bug)") - print("Expected pattern match for:", pattern_text) - sys.exit(1) - - -def update_build_files_from_tensorflow(repo_path, tensorflow_path): - src_llvm_build = os.path.join(tensorflow_path, "third_party", "llvm", - "llvm.autogenerated.BUILD") - # NOTE(laurenzo): These will probably move upstream. - src_mlir_build = os.path.join(tensorflow_path, "third_party", "mlir", "BUILD") - src_mlir_test_build = os.path.join(tensorflow_path, "third_party", "mlir", - "test.BUILD") - overlay_path = os.path.join(repo_path, "build_tools", "bazel", - "third_party_import", "llvm-project", "overlay") - copy_text_file(repo_path, src_llvm_build, - os.path.join(overlay_path, "llvm", "BUILD.bazel")) - copy_text_file(repo_path, src_mlir_build, - os.path.join(overlay_path, "mlir", "BUILD.bazel")) - copy_text_file(repo_path, src_mlir_test_build, - os.path.join(overlay_path, "mlir", "test", "BUILD.bazel")) - - -def copy_text_file(repo_path, src_file, dst_file): - print("+ cp %s %s" % (src_file, dst_file)) - with open(src_file, "r", encoding="UTF-8") as f: - src_contents = f.read() - - if not os.path.exists(dst_file): - print("WARNING: Destination file does not exist:", dst_file) - with open(dst_file, "w", encoding="UTF-8") as f: - f.write(src_contents) - stage_path(repo_path, dst_file) - - -def stage_path(repo_path, to_stage): - # TODO(laurenzo): Move to utils.py. - utils.execute(["git", "add", to_stage], cwd=repo_path) - - -if __name__ == "__main__": - main(parse_arguments())
diff --git a/git_scripts/utils.py b/git_scripts/utils.py deleted file mode 100644 index b4328e7..0000000 --- a/git_scripts/utils.py +++ /dev/null
@@ -1,63 +0,0 @@ -# Lint as: python3 -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=missing-docstring - -import argparse -import os -import subprocess - - -def find_git_toplevel(): - """Finds the containing git top-level directory that contains this script.""" - return execute(["git", "rev-parse", "--show-toplevel"], - cwd=os.path.dirname(__file__), - capture_output=True, - silent=True).strip().decode("UTF-8") - - -def str2bool(v): - """Can be used as an argparse type to parse a bool.""" - if v is None: - return None - if isinstance(v, bool): - return v - if v.lower() in ("yes", "true", "t", "y", "1"): - return True - elif v.lower() in ("no", "false", "f", "n", "0"): - return False - else: - raise argparse.ArgumentTypeError("Boolean value expected.") - - -def execute(args, cwd, capture_output=False, silent=False, **kwargs): - """Executes a command. - - Args: - args: List of command line arguments. - cwd: Directory to execute in. - capture_output: Whether to capture the output. - silent: Whether to skip logging the invocation. - **kwargs: Extra arguments to pass to subprocess.exec - - Returns: - The output if capture_output, otherwise None. - """ - if not silent: - print("+", " ".join(args), " [from %s]" % cwd) - if capture_output: - return subprocess.check_output(args, cwd=cwd, **kwargs) - else: - return subprocess.check_call(args, cwd=cwd, **kwargs)