Add pw_docgen module
This change adds a module which builds documentation using Sphinx.
Documentation files and assets are defined using a pw_doc_group GN
template which lists out all of the sources as metadata. Another GN
template, pw_doc_gen, collects the metadata from pw_doc_group targets
into a JSON file and defines an action that runs a script to build the
HTML docs.
A docgen script is provided by the pw_docgen module. This script reads
the JSON metadata file collected by a pw_doc_gen template, copies all of
the source and resource files into a documentation tree, and runs Sphinx
on the tree to render the docs as HTML.
Change-Id: I97890f05d850d5119cbd3bbf48138cd82f22d56c
diff --git a/modules.gni b/modules.gni
index ede125f..7fe69dc 100644
--- a/modules.gni
+++ b/modules.gni
@@ -23,6 +23,7 @@
}
dir_pw_build = "$dir_pigweed/pw_build"
+dir_pw_docgen = "$dir_pigweed/pw_docgen"
dir_pw_preprocessor = "$dir_pigweed/pw_preprocessor"
dir_pw_span = "$dir_pigweed/pw_span"
dir_pw_status = "$dir_pigweed/pw_status"
diff --git a/pw_build/python_script.gni b/pw_build/python_script.gni
index ced7a10..99b20c3 100644
--- a/pw_build/python_script.gni
+++ b/pw_build/python_script.gni
@@ -58,6 +58,8 @@
# $target_out_dir:foo ../out/obj/my_module/foo.exe (toolchain-dependent)
#
template("pw_python_script") {
+ assert(defined(invoker.script), "pw_python_script requires a script to run")
+
_script_args = [
# GN root directory relative to the build directory (in which the runner
# script is invoked).
@@ -69,6 +71,16 @@
root_out_dir,
]
+ if (defined(invoker.inputs)) {
+ _inputs = invoker.inputs
+ } else {
+ _inputs = []
+ }
+
+ # List the script to run as an input so that the action is re-run when it is
+ # modified.
+ _inputs += [ invoker.script ]
+
if (defined(invoker.outputs)) {
_outputs = invoker.outputs
} else {
@@ -99,12 +111,14 @@
_ignore_vars = [
"script",
"args",
+ "inputs",
"outputs",
]
forward_variables_from(invoker, "*", _ignore_vars)
script = "$dir_pw_build/py/python_runner.py"
args = _script_args
+ inputs = _inputs
outputs = _outputs
}
}
diff --git a/pw_docgen/docs.gni b/pw_docgen/docs.gni
new file mode 100644
index 0000000..1bd5991
--- /dev/null
+++ b/pw_docgen/docs.gni
@@ -0,0 +1,115 @@
+# Copyright 2019 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("$dir_pw_build/python_script.gni")
+
+# Defines a group of documentation files and assets.
+#
+# Args:
+# sources: Source files for the documentation (.rst or .md).
+# inputs: Additional resource files for the docs, such as images.
+# group_deps: Other pw_doc_group targets on which this group depends.
+# report_deps: Report card targets on which documentation depends.
+template("pw_doc_group") {
+ assert(defined(invoker.sources), "pw_doc_group requires a list of sources")
+
+ if (defined(invoker.inputs)) {
+ _inputs = invoker.inputs
+ } else {
+ _inputs = []
+ }
+
+ _all_deps = []
+ if (defined(invoker.group_deps)) {
+ _all_deps += invoker.group_deps
+ }
+ if (defined(invoker.report_deps)) {
+ _all_deps += invoker.report_deps
+ }
+
+ # Creates an action which depends on all of the source and input files so that
+ # docs are rebuilt on file modifications. The action does not do anything.
+ pw_python_script(target_name) {
+ metadata = {
+ pw_doc_sources = rebase_path(invoker.sources, root_build_dir)
+ pw_doc_inputs = rebase_path(_inputs, root_build_dir)
+ }
+ deps = _all_deps
+ script = "$dir_pw_build/py/nop.py"
+ inputs = invoker.sources + _inputs
+ stamp = true
+ }
+}
+
+# Creates a target to build HTML documentation from groups of sources.
+#
+# Args:
+# deps: List of pw_doc_group targets.
+# index: Top-level documentation index file.
+# conf: Configuration script (conf.py) for Sphinx.
+# output_directory: Path to directory to which HTML output is rendered.
+template("pw_doc_gen") {
+ assert(defined(invoker.deps),
+ "pw_doc_gen requires doc groups as dependencies")
+ assert(
+ defined(invoker.index),
+ "pw_doc_gen requires an 'index' argument pointing a top-level index.rst")
+ assert(defined(invoker.conf),
+ "pw_doc_gen requires a 'conf' argument pointing a top-level conf.py")
+ assert(defined(invoker.output_directory),
+ "pw_doc_gen requires an 'output_directory' argument")
+
+ # Collects all dependency metadata into a single JSON file.
+ _metadata_file_target = "${target_name}_metadata"
+ generated_file(_metadata_file_target) {
+ outputs = [
+ "$target_gen_dir/$target_name.json",
+ ]
+ data_keys = [
+ "pw_doc_sources",
+ "pw_doc_inputs",
+ ]
+ output_conversion = "json"
+ deps = invoker.deps
+ }
+
+ _script_args = [
+ "--gn-root",
+ "//",
+ "--sphinx-build-dir",
+ get_path_info("$target_gen_dir/pw_docgen_tree", "abspath"),
+ "--conf",
+ get_path_info(invoker.conf, "abspath"),
+ "--index",
+ get_path_info(invoker.index, "abspath"),
+ "--out-dir",
+ get_path_info(invoker.output_directory, "abspath"),
+ ]
+
+ # Metadata JSON file path.
+ _script_args +=
+ get_path_info(get_target_outputs(":$_metadata_file_target"), "abspath")
+
+ pw_python_script(target_name) {
+ script = "$dir_pw_docgen/py/docgen.py"
+ args = _script_args
+ deps = [
+ ":$_metadata_file_target",
+ ]
+ inputs = [
+ invoker.index,
+ ]
+ stamp = true
+ }
+}
diff --git a/pw_docgen/py/docgen.py b/pw_docgen/py/docgen.py
new file mode 100644
index 0000000..63f844f
--- /dev/null
+++ b/pw_docgen/py/docgen.py
@@ -0,0 +1,126 @@
+# Copyright 2019 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+"""Renders HTML documentation using Sphinx."""
+
+# TODO(frolv): Figure out a solution for installing all library dependencies
+# to run Sphinx and build RTD docs.
+
+import argparse
+import collections
+import json
+import os
+import shutil
+import subprocess
+import sys
+
+from typing import Dict, List, Tuple
+
+SCRIPT_HEADER: str = '''
+██████╗ ██╗ ██████╗ ██╗ ██╗███████╗███████╗██████╗ ██████╗ ██████╗ ██████╗███████╗
+██╔══██╗██║██╔════╝ ██║ ██║██╔════╝██╔════╝██╔══██╗ ██╔══██╗██╔═══██╗██╔════╝██╔════╝
+██████╔╝██║██║ ███╗██║ █╗ ██║█████╗ █████╗ ██║ ██║ ██║ ██║██║ ██║██║ ███████╗
+██╔═══╝ ██║██║ ██║██║███╗██║██╔══╝ ██╔══╝ ██║ ██║ ██║ ██║██║ ██║██║ ╚════██║
+██║ ██║╚██████╔╝╚███╔███╔╝███████╗███████╗██████╔╝ ██████╔╝╚██████╔╝╚██████╗███████║
+╚═╝ ╚═╝ ╚═════╝ ╚══╝╚══╝ ╚══════╝╚══════╝╚═════╝ ╚═════╝ ╚═════╝ ╚═════╝╚══════╝
+'''
+
+
+def parse_args() -> argparse.Namespace:
+ """Parses command-line arguments."""
+
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--sphinx-build-dir', type=str, required=True,
+ help='Directory in which to build docs')
+ parser.add_argument('--conf', type=str, required=True,
+ 'Path to conf.py file for Sphinx')
+ parser.add_argument('--gn-root', type=str, required=True,
+ 'Root of the GN build tree')
+ parser.add_argument('--index', type=str, required=True,
+ help='Path to root index.rst file')
+ parser.add_argument('--out-dir', type=str, required=True,
+ help='Output directory for rendered HTML docs')
+ parser.add_argument('metadata_file', type=argparse.FileType('r'))
+ return parser.parse_args()
+
+
+def build_docs(src_dir: str, dst_dir: str) -> int:
+ """Runs Sphinx to render HTML documentation from a doc tree."""
+
+ # TODO(frolv): Specify the Sphinx script from a prebuilts path instead of
+ # requiring it in the tree.
+ command = ['sphinx-build', '-b', 'html', '-d',
+ f'{dst_dir}/help', src_dir, f'{dst_dir}/html']
+ return subprocess.call(command)
+
+
+def mkdir(dirname: str, exist_ok: bool = False) -> None:
+ """Wrapper around os.makedirs that prints the operation."""
+ print(f'MKDIR {dirname}')
+ os.makedirs(dirname, exist_ok=exist_ok)
+
+
+def copy(src: str, dst: str) -> None:
+ """Wrapper around shutil.copy that prints the operation."""
+ print(f'COPY {src} -> {dst}')
+ shutil.copy(src, dst)
+
+
+def copy_doc_tree(args: argparse.Namespace) -> None:
+ """Copies doc source and input files into a build tree."""
+
+ def build_path(path):
+ """Converts a source path to a filename in the build directory."""
+ return f'{args.sphinx_build_dir}/{path[len(args.gn_root):]}'
+
+ source_files = json.load(args.metadata_file)
+ copy_paths = [build_path(f) for f in source_files]
+
+ mkdir(args.sphinx_build_dir)
+ copy(args.index, f'{args.sphinx_build_dir}/index.rst')
+ copy(args.conf, f'{args.sphinx_build_dir}/conf.py')
+
+ # Map of directory path to list of source and destination file paths.
+ dirs: Dict[str, List[Tuple[str, str]]] = collections.defaultdict(list)
+
+ for source_file, copy_path in zip(source_files, copy_paths):
+ dirname = os.path.dirname(copy_path)
+ dirs[dirname].append((source_file, copy_path))
+
+ for directory, file_pairs in dirs.items():
+ mkdir(directory, exist_ok=True)
+ for src, dst in file_pairs:
+ copy(src, dst)
+
+
+def main() -> int:
+ """Script entry point."""
+
+ args = parse_args()
+
+ # Clear out any existing docs for the target.
+ if os.path.exists(args.sphinx_build_dir):
+ shutil.rmtree(args.sphinx_build_dir)
+
+ print(SCRIPT_HEADER)
+ copy_doc_tree(args)
+
+ # Flush all script output before running Sphinx.
+ print('-' * 80, flush=True)
+
+ return build_docs(args.sphinx_build_dir, args.out_dir)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/pw_preprocessor/BUILD.gn b/pw_preprocessor/BUILD.gn
index 230d65b..112ab73 100644
--- a/pw_preprocessor/BUILD.gn
+++ b/pw_preprocessor/BUILD.gn
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations under
# the License.
+import("$dir_pw_docgen/docs.gni")
import("$dir_pw_unit_test/test.gni")
config("default_config") {
@@ -33,6 +34,12 @@
sources = public
}
+pw_doc_group("docs") {
+ sources = [
+ "docs.rst",
+ ]
+}
+
# All pw_preprocessor test binaries.
group("unit_tests") {
deps = [
