[doc] Completely replace docgen with hugo
This change completely replaces docgen and replaces or removes
docgen-specific markdown in documentation. It also does the following:
* Updates all local links to use hugo relative references so that a
broken link is a broken build.
* Uses upstream wavedrom, which breaks at least one page that depends
on local modifications.
* Renames most hw/ip/**/ip_name.doc and dv_plan documents for a more
aesthetic document tree layout.
* Moves some doc/ pages into their own page bundle.
* Updates util/build_docs.py to pre-generate registers, hwcfg, and
dashboard fragments and invoke hugo.
diff --git a/util/build_docs.py b/util/build_docs.py
index 00389b0..cea7709 100755
--- a/util/build_docs.py
+++ b/util/build_docs.py
@@ -3,130 +3,154 @@
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
#
-# pip3 install --user livereload
# Usage:
# run './build_docs.py' to generate the documentation and keep it updated
-# open 'http://localhost:5500/' to check live update (this opens the top
+# open 'http://localhost:1313/' to check live update (this opens the top
# level index page). you can also directly access a specific document by
-# accessing 'http://localhost:5500/path/to/doc.html',
-# e.g. http://localhost:5500/hw/ip/uart/doc/uart.html
+# accessing 'http://localhost:1313/path/to/doc',
+# e.g. http://localhost:1313/hw/ip/uart/doc
import argparse
+import io
import logging
import os
import shutil
+import subprocess
from pathlib import Path
-import livereload
+import hjson
-import docgen.generate
+import dashboard.gen_dashboard_entry as gen_dashboard_entry
+import reggen.gen_cfg_html as gen_cfg_html
+import reggen.gen_html as gen_html
+import reggen.validate as validate
+import testplanner.testplan_utils as testplan_utils
USAGE = """
build_docs [options]
"""
-MARKDOWN_EXTENSIONS = [
- '.md',
- '.mkd',
-]
-STATIC_ASSET_EXTENSIONS = [
- '.svg',
- '.png',
- '.jpg',
- '.css',
-]
-HJSON_EXTENSIONS = ['.hjson']
-
# Configurations
# TODO: Move to config.yaml
SRCTREE_TOP = Path(__file__).parent.joinpath('..').resolve()
config = {
# Toplevel source directory
- "topdir": SRCTREE_TOP,
+ "topdir":
+ SRCTREE_TOP,
- # A list of directories containing documentation within topdir. To ensure
- # the top-level sitemap doesn't have broken links, this should be kept
- # in-sync with the doctree tag in sitemap.md.
- "incdirs": ['./doc', './hw', './sw', './util'],
+ # Pre-generate register and hwcfg fragments from these files.
+ "hardware_definitions": [
+ "hw/ip/aes/data/aes.hjson",
+ "hw/ip/alert_handler/data/alert_handler.hjson",
+ "hw/ip/flash_ctrl/data/flash_ctrl.hjson",
+ "hw/ip/gpio/data/gpio.hjson",
+ "hw/ip/hmac/data/hmac.hjson",
+ "hw/ip/i2c/data/i2c.hjson",
+ "hw/ip/padctrl/data/padctrl.hjson",
+ "hw/ip/pinmux/data/pinmux.hjson",
+ "hw/ip/rv_plic/data/rv_plic.hjson",
+ "hw/ip/rv_timer/data/rv_timer.hjson",
+ "hw/ip/spi_device/data/spi_device.hjson",
+ "hw/ip/uart/data/uart.hjson",
+ "hw/ip/usbdev/data/usbdev.hjson",
+ "hw/ip/usbuart/data/usbuart.hjson",
+ ],
+
+ # Pre-generate dashboard fragments from these directories.
+ "dashboard_definitions": [
+ "hw/ip",
+ ],
+
+ # Pre-generate testplan fragments from these files.
+ "testplan_definitions": [
+ "hw/ip/gpio/data/gpio_testplan.hjson",
+ "hw/ip/hmac/data/hmac_testplan.hjson",
+ "hw/ip/i2c/data/i2c_testplan.hjson",
+ "hw/ip/rv_timer/data/rv_timer_testplan.hjson",
+ "hw/ip/uart/data/uart_testplan.hjson",
+ "util/testplanner/examples/foo_testplan.hjson",
+ ],
# Output directory for documents
- "outdir": SRCTREE_TOP.joinpath('opentitan-docs'),
- "verbose": False,
+ "outdir":
+ SRCTREE_TOP.joinpath('build', 'docs'),
+ "outdir-generated":
+ SRCTREE_TOP.joinpath('build', 'docs-generated'),
+ "verbose":
+ False,
}
-def get_doc_files(extensions=MARKDOWN_EXTENSIONS + STATIC_ASSET_EXTENSIONS):
- """Get the absolute path of files containing documentation
- """
- file_list = []
- # doc files on toplevel
- for ext in extensions:
- file_list += config["topdir"].glob('*' + ext)
- # doc files in include dirs
- for incdir in config['incdirs']:
- for ext in extensions:
- file_list += config["topdir"].joinpath(incdir).rglob('*' + ext)
- return file_list
+def generate_dashboards():
+ for dashboard in config["dashboard_definitions"]:
+ hjson_paths = []
+ hjson_paths.extend(
+ sorted(SRCTREE_TOP.joinpath(dashboard).rglob('*.prj.hjson')))
+
+ dashboard_path = config["outdir-generated"].joinpath(
+ dashboard, 'dashboard')
+ dashboard_html = open(dashboard_path, mode='w')
+ for hjson_path in hjson_paths:
+ gen_dashboard_entry.gen_dashboard_html(hjson_path, dashboard_html)
+ dashboard_html.close()
-def ensure_dest_dir(dest_pathname):
- os.makedirs(dest_pathname.parent, exist_ok=True)
+def generate_hardware_blocks():
+ for hardware in config["hardware_definitions"]:
+ hardware_file = open(SRCTREE_TOP.joinpath(hardware))
+ regs = hjson.load(hardware_file,
+ use_decimal=True,
+ object_pairs_hook=validate.checking_dict)
+ if validate.validate(regs) == 0:
+ logging.info("Parsed %s" % (hardware))
+ else:
+ logging.fatal("Failed to parse %s" % (hardware))
+
+ base_path = config["outdir-generated"].joinpath(hardware)
+ base_path.parent.mkdir(parents=True, exist_ok=True)
+
+ regs_html = open(base_path.parent.joinpath(base_path.name +
+ '.registers'),
+ mode='w')
+ gen_html.gen_html(regs, regs_html)
+ regs_html.close()
+
+ hwcfg_html = open(base_path.parent.joinpath(base_path.name + '.hwcfg'),
+ mode='w')
+ gen_cfg_html.gen_cfg_html(regs, hwcfg_html)
+ hwcfg_html.close()
-def path_src_to_dest(src_pathname, dest_filename_suffix=None):
- """Get the destination pathname from a source pathname
- """
- src_relpath = Path(src_pathname).relative_to(config["topdir"])
- dest_pathname = Path(config["outdir"]).joinpath(src_relpath)
- if dest_filename_suffix:
- dest_pathname = dest_pathname.with_suffix(dest_filename_suffix)
- return dest_pathname
+def generate_testplans():
+ for testplan in config["testplan_definitions"]:
+ plan = testplan_utils.parse_testplan(SRCTREE_TOP.joinpath(testplan))
+
+ plan_path = config["outdir-generated"].joinpath(testplan + '.testplan')
+ plan_path.parent.mkdir(parents=True, exist_ok=True)
+
+ testplan_html = open(plan_path, mode='w')
+ testplan_utils.gen_html_testplan_table(plan, testplan_html)
+ testplan_html.close()
-def process_file_markdown(src_pathname):
- """Process a markdown file and copy it to the destination
- """
- dest_pathname = path_src_to_dest(src_pathname, '.html')
-
- logging.info("Processing Markdown file: %s -> %s" %
- (str(src_pathname), str(dest_pathname)))
-
- ensure_dest_dir(dest_pathname)
-
- with open(dest_pathname, 'w', encoding='UTF-8') as f:
- outstr = docgen.generate.generate_doc(str(src_pathname),
- verbose=config['verbose'],
- inlinecss=True,
- inlinewave=True,
- asdiv=False)
- f.write(outstr)
-
- return dest_pathname
-
-
-def process_file_copytodest(src_pathname):
- """Copy a file to the destination directory with no further processing
- """
- dest_pathname = path_src_to_dest(src_pathname)
-
- logging.info("Copying %s -> %s" % (str(src_pathname), str(dest_pathname)))
-
- ensure_dest_dir(dest_pathname)
- shutil.copy(src_pathname, dest_pathname)
-
-
-def process_all_files():
- """Process all files
-
- The specific processing action depends on the file type.
- """
- src_files = get_doc_files()
-
- for src_pathname in src_files:
- if src_pathname.suffix in MARKDOWN_EXTENSIONS:
- process_file_markdown(src_pathname)
- elif src_pathname.suffix in STATIC_ASSET_EXTENSIONS:
- process_file_copytodest(src_pathname)
+def invoke_hugo(preview):
+ site_docs = SRCTREE_TOP.joinpath('site', 'docs')
+ config_file = str(site_docs.joinpath('config.toml'))
+ layout_dir = str(site_docs.joinpath('layouts'))
+ args = [
+ "hugo",
+ "--config",
+ config_file,
+ "--destination",
+ str(config["outdir"]),
+ "--contentDir",
+ str(SRCTREE_TOP),
+ "--layoutDir",
+ layout_dir,
+ ]
+ if preview:
+ args += ["server"]
+ subprocess.run(args, check=True, cwd=SRCTREE_TOP)
def main():
@@ -144,21 +168,14 @@
help="""starts a local server with live reload (updates triggered upon
changes in the documentation files). this feature is intended
to preview the documentation locally.""")
+ parser.add_argument('--hugo', help="""TODO""")
args = parser.parse_args()
- # Initial processing of all files
- process_all_files()
-
- if args.preview:
- # Setup livereload watcher
- server = livereload.Server()
- exts_to_watch = MARKDOWN_EXTENSIONS + \
- STATIC_ASSET_EXTENSIONS + \
- HJSON_EXTENSIONS
- for src_pathname in get_doc_files(exts_to_watch):
- server.watch(str(src_pathname), process_all_files)
- server.serve(root=config['topdir'].joinpath(config['outdir']))
+ generate_hardware_blocks()
+ generate_dashboards()
+ generate_testplans()
+ invoke_hugo(args.preview)
if __name__ == "__main__":
