Download external Go dependencies in build
This change updates the Go GN integration to download external
dependencies for Go packages before running "go build". These
dependencies are listed in the pw_go_package template using build
metadata and collected to a "go get" invocation.
To support this, the pw_exec template is expanded to allow setting
positional arguments from a file.
Change-Id: If4f6c71f037b35bb041984da9982a1629d1d36b0
diff --git a/pw_build/BUILD.gn b/pw_build/BUILD.gn
index aaa8e2d..935c47c 100644
--- a/pw_build/BUILD.gn
+++ b/pw_build/BUILD.gn
@@ -57,3 +57,8 @@
"docs.rst",
]
}
+
+# Pool to limit a single thread to download external Go packages at a time.
+pool("go_download_pool") {
+ depth = 1
+}
diff --git a/pw_build/exec.gni b/pw_build/exec.gni
index afa2f48..7c8765d 100644
--- a/pw_build/exec.gni
+++ b/pw_build/exec.gni
@@ -22,15 +22,29 @@
# Args:
# program: The program to run. Can be a full path or just a name (in which case
# $PATH is searched).
+#
# args: Optional list of arguments to the program.
+#
# deps: Dependencies for this target.
+#
# inputs: Optional list of build inputs to the program.
+#
# outputs: Optional list of artifacts produced by the program's execution.
+#
# env: Optional list of key-value pairs defining environment variables for
# the program.
+#
# env_file: Optional path to a file containing a list of newline-separated
# key-value pairs defining environment variables for the program.
#
+# args_file: Optional path to a file containing additional positional arguments
+# to the program. Each line of the file is appended to the invocation. Useful
+# for specifying arguments from GN metadata.
+#
+# skip_empty_args: If args_file is provided, boolean indicating whether to skip
+# running the program if the file is empty. Used to avoid running commands
+# which error when called without arguments.
+#
# Example:
#
# pw_exec("hello_world") {
@@ -56,6 +70,17 @@
]
}
+ if (defined(invoker.args_file)) {
+ _script_args += [
+ "--args-file",
+ get_path_info(invoker.args_file, "abspath"),
+ ]
+
+ if (defined(invoker.skip_empty_args) && invoker.skip_empty_args) {
+ _script_args += [ "--skip-empty-args" ]
+ }
+ }
+
if (defined(invoker.env)) {
foreach(_env, invoker.env) {
_script_args += [
@@ -81,6 +106,7 @@
[
"deps",
"inputs",
+ "pool",
])
if (!defined(inputs)) {
diff --git a/pw_build/go.gni b/pw_build/go.gni
index be54b9e..86bc4d1 100644
--- a/pw_build/go.gni
+++ b/pw_build/go.gni
@@ -13,8 +13,7 @@
# the License.
# This file provides GN build integration for Go. These templates are limited,
-# only supporting legacy GOPATH-based builds. Third-party dependencies required
-# to build Go code must be installed separately.
+# supporting only legacy GOPATH-based builds.
import("exec.gni")
import("input_group.gni")
@@ -27,6 +26,7 @@
# Args:
# sources: List of Go source files.
# deps: Optional list of target dependencies.
+# external_deps: Optional list of Go package dependencies outside of Pigweed.
# gopath: Root of the GOPATH in which the package is located.
#
# Example:
@@ -34,7 +34,12 @@
# # In //my_module/go/src/example.com/foo/BUILD.gn
# pw_go_package("foo_package") {
# sources = [ "main.go" ]
-# deps = "//my_module:foo_proto_go"
+# deps = [
+# "//my_module:foo_proto_go"
+# ]
+# external_deps = [
+# "github.com/golang/glog"
+# ]
# gopath = "//my_module/go"
# }
#
@@ -57,6 +62,10 @@
}
}
metadata.gopath = [ "GOPATH+=${_gopath}" ]
+
+ if (defined(invoker.external_deps)) {
+ metadata.external_deps = invoker.external_deps
+ }
}
}
@@ -96,7 +105,38 @@
]
}
- _default_gopath = rebase_path("$dir_pigweed/env_setup/go")
+ # Collect all of the external dependencies of the executable and its packages.
+ _deps_metadata_target_name = "${target_name}_pw_go_deps"
+ _deps_metadata_file = "$target_gen_dir/${target_name}_pw_go_deps.txt"
+ generated_file(_deps_metadata_target_name) {
+ deps = invoker.deps
+ data_keys = [ "external_deps" ]
+ outputs = [
+ _deps_metadata_file,
+ ]
+ }
+
+ _default_gopath = rebase_path("$root_gen_dir/go")
+
+ # Create a target to download all external dependencies into the default
+ # GOPATH in the out directory. This is only run once; "go get" does not
+ # re-download existing packages.
+ _download_target_name = "${target_name}_pw_go_get"
+ pw_exec(_download_target_name) {
+ program = "go"
+ args = [ "get" ]
+ deps = [
+ ":$_deps_metadata_target_name",
+ ]
+ env = [ "GOPATH=$_default_gopath" ]
+ args_file = _deps_metadata_file
+
+ # If the args file is empty, don't run the "go get" command.
+ skip_empty_args = true
+
+ # Limit download parallelization to 1.
+ pool = "$dir_pw_build:go_download_pool"
+ }
# Run a "go build" command with the environment configured from metadata.
pw_exec(target_name) {
@@ -108,6 +148,7 @@
invoker.package,
]
deps = [
+ ":$_download_target_name",
":$_metadata_target_name",
]
env = [ "GOPATH+=$_default_gopath" ]
diff --git a/pw_build/py/exec.py b/pw_build/py/exec.py
index f57f142..823e275 100644
--- a/pw_build/py/exec.py
+++ b/pw_build/py/exec.py
@@ -29,18 +29,33 @@
if parser is None:
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument('-e',
- '--env',
- action='append',
- default=[],
- help='key=value environment pair for the process')
+ parser.add_argument(
+ '--args-file',
+ type=argparse.FileType('r'),
+ help='File containing extra positional arguments to the program',
+ )
+ parser.add_argument(
+ '-e',
+ '--env',
+ action='append',
+ default=[],
+ help='key=value environment pair for the process',
+ )
parser.add_argument(
'--env-file',
type=argparse.FileType('r'),
- help='File defining environment variables for the process')
- parser.add_argument('command',
- nargs=argparse.REMAINDER,
- help='Program to run with arguments')
+ help='File defining environment variables for the process',
+ )
+ parser.add_argument(
+ '--skip-empty-args',
+ action='store_true',
+ help='Don\'t run the program if --args-file is empty',
+ )
+ parser.add_argument(
+ 'command',
+ nargs=argparse.REMAINDER,
+ help='Program to run with arguments',
+ )
return parser
@@ -76,6 +91,18 @@
env = os.environ.copy()
+ # Command starts after the "--".
+ command = args.command[1:]
+
+ if args.args_file is not None:
+ empty = True
+ for line in args.args_file:
+ empty = False
+ command.append(line.strip())
+
+ if args.skip_empty_args and empty:
+ return 0
+
if args.env_file is not None:
for line in args.env_file:
apply_env_var(line, env)
@@ -84,7 +111,7 @@
for string in args.env:
apply_env_var(string, env)
- return subprocess.call(args.command[1:], env=env)
+ return subprocess.call(command, env=env)
if __name__ == '__main__':
diff --git a/pw_protobuf_compiler/proto.gni b/pw_protobuf_compiler/proto.gni
index c659a20..59637ee 100644
--- a/pw_protobuf_compiler/proto.gni
+++ b/pw_protobuf_compiler/proto.gni
@@ -81,6 +81,10 @@
pw_python_script(target_name) {
metadata = {
gopath = [ "GOPATH+=$_rebased_gopath" ]
+ external_deps = [
+ "github.com/golang/protobuf/proto",
+ "google.golang.org/grpc",
+ ]
}
script = _gen_script_path
args = [
diff --git a/pw_target_runner/go/src/pigweed.dev/pw_target_runner/BUILD.gn b/pw_target_runner/go/src/pigweed.dev/pw_target_runner/BUILD.gn
index 991445c..ab6a0aa 100644
--- a/pw_target_runner/go/src/pigweed.dev/pw_target_runner/BUILD.gn
+++ b/pw_target_runner/go/src/pigweed.dev/pw_target_runner/BUILD.gn
@@ -23,5 +23,6 @@
deps = [
"$dir_pw_target_runner:target_runner_proto_go",
]
+ external_deps = [ "google.golang.org/grpc" ]
gopath = "$dir_pw_target_runner/go"
}
diff --git a/pw_target_runner/go/src/pigweed.dev/pw_target_runner_server/BUILD.gn b/pw_target_runner/go/src/pigweed.dev/pw_target_runner_server/BUILD.gn
index e8ff14e..eda11cb 100644
--- a/pw_target_runner/go/src/pigweed.dev/pw_target_runner_server/BUILD.gn
+++ b/pw_target_runner/go/src/pigweed.dev/pw_target_runner_server/BUILD.gn
@@ -22,5 +22,6 @@
"$dir_pw_target_runner:exec_server_config_proto_go",
"$dir_pw_target_runner/go/src/pigweed.dev/pw_target_runner",
]
+ external_deps = [ "github.com/golang/protobuf/proto" ]
gopath = "$dir_pw_target_runner/go"
}
