[ipgen] Initial commit of IP generation tool Signed-off-by: Philipp Wagner <phw@lowrisc.org>
diff --git a/util/ipgen.py b/util/ipgen.py new file mode 100755 index 0000000..325e978 --- /dev/null +++ b/util/ipgen.py
@@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +r"""IP Generator: Produce IP blocks from IP templates +""" +import argparse +import logging +import sys +from pathlib import Path + +from ipgen import (IpBlockRenderer, IpConfig, IpTemplate, TemplateParseError, + TemplateRenderError) + + +def init_logging(verbose: bool) -> None: + """ Initialize the logging system """ + if verbose: + logging.basicConfig(format="%(levelname)s: %(message)s", + level=logging.DEBUG) + else: + logging.basicConfig(format="%(levelname)s: %(message)s") + + +def action_generate(ip_template: IpTemplate, args: argparse.Namespace) -> None: + """ Handle the 'generate' action/subcommand. """ + overwrite_output_dir = args.force + output_path = args.outdir + + # Read the IP configuration file. + config_fp = args.config_file + config_text = config_fp.read() + config_fp.close() + ip_config = IpConfig.from_text(config_text, "the file passed to --config") + + # Render the IP template into an IP block. + renderer = IpBlockRenderer(ip_template, ip_config) + renderer.render(output_path, overwrite_output_dir) + + print( + f"Wrote IP block {ip_config.instance_name!r} from template {ip_template.name!r} to '{output_path}'." + ) + + +def action_describe(ip_template: IpTemplate, args: argparse.Namespace) -> None: + """ Handle the 'describe' action/subcommand. """ + headline = f"IP template {ip_template.name!r}" + print(headline) + print('=' * len(headline)) + print() + print(f"The template is stored in '{ip_template.template_path}'.") + print() + print("Template parameters") + print("-------------------") + print() + for param in ip_template.params.values(): + print(f"{param.name}:") + print(f" Description: {param.desc}") + print(f" Type: {param.param_type}") + print(f" Default value: {param.default}") + print() + + +def main() -> int: + parser = argparse.ArgumentParser() + + # Shared arguments across all actions + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument( + "--verbose", + help="More info messages", + action="store_true", + ) + parent_parser.add_argument( + '-C', + '--template-dir', + type=Path, + required=True, + help='IP template directory', + ) + + subparsers = parser.add_subparsers( + metavar='ACTION', + title="actions", + description=("Use 'ipgen.py ACTION --help' to learn more about the " + "individual actions.")) + subparsers.required = True + + # 'describe' subparser + parser_generate = subparsers.add_parser( + "describe", + description="Show all information available for the IP template.", + help="Show details about an IP template", + parents=[parent_parser], + ) + parser_generate.set_defaults(func=action_describe) + + # 'generate' subparser + parser_generate = subparsers.add_parser( + "generate", + description="Generate an IP block from an IP template", + help="Generate an IP block from an IP template", + parents=[parent_parser], + ) + parser_generate.add_argument( + "-o", + "--outdir", + required=True, + type=Path, + help="output directory for the resulting IP block", + ) + parser_generate.add_argument( + "--force", + "-f", + required=False, + default=False, + action="store_true", + help="overwrite the output directory, if it exists", + ) + parser_generate.add_argument( + "--config-file", + "-c", + required=False, + type=argparse.FileType('r'), + help="path to a configuration file", + ) + parser_generate.set_defaults(func=action_generate) + + # Parse command line arguments, parse IP template, and invoke subparsers + args = parser.parse_args() + init_logging(args.verbose) + + try: + ip_template = IpTemplate.from_template_path(args.template_dir) + args.func(ip_template, args) + except (TemplateParseError, TemplateRenderError) as e: + if args.verbose: + # Show the full backtrace if operating in verbose mode. + logging.exception(e) + else: + # Otherwise just log the problem itself in a more user-friendly way. + logging.error(str(e)) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main())
diff --git a/util/ipgen/__init__.py b/util/ipgen/__init__.py new file mode 100644 index 0000000..7c72a5d --- /dev/null +++ b/util/ipgen/__init__.py
@@ -0,0 +1,7 @@ +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +from .lib import IpConfig, IpTemplate, TemplateParseError # noqa: F401 +from .renderer import (IpBlockRenderer, IpDescriptionOnlyRenderer, + TemplateRenderError) # noqa: F401
diff --git a/util/ipgen/lib.py b/util/ipgen/lib.py new file mode 100644 index 0000000..e1453e7 --- /dev/null +++ b/util/ipgen/lib.py
@@ -0,0 +1,209 @@ +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +from typing import Dict, Optional, Union, cast + +import hjson # type: ignore +from reggen.lib import check_int, check_keys, check_list, check_name, check_str +from reggen.params import BaseParam, Params + + +class TemplateParseError(Exception): + pass + + +class TemplateParameter(BaseParam): + """ A template parameter. """ + def __init__(self, name: str, desc: Optional[str], param_type: str, + default: str): + super().__init__(name, desc, param_type) + self.default = default + self.value = None + + def as_dict(self) -> Dict[str, object]: + rd = super().as_dict() + rd['default'] = self.default + return rd + + +def _parse_template_parameter(where: str, raw: object) -> TemplateParameter: + rd = check_keys(raw, where, ['name', 'desc', 'type'], ['default']) + + name = check_str(rd['name'], 'name field of ' + where) + + r_desc = rd.get('desc') + if r_desc is None: + desc = None + else: + desc = check_str(r_desc, 'desc field of ' + where) + + r_type = rd.get('type') + param_type = check_str(r_type, 'type field of ' + where) + if not param_type in ('string', 'int'): + raise ValueError('At {}, the {} param has an invalid type field {!r}. ' + 'Allowed values are: string, int.'.format( + where, name, param_type)) + + r_default = rd.get('default') + default = check_str(r_default, 'default field of ' + where) + if param_type[:3] == 'int': + check_int(default, + 'default field of {}, (an integer parameter)'.format(name)) + + return TemplateParameter(name, desc, param_type, default) + + +class TemplateParams(Params): + """ A group of template parameters. """ + @classmethod + def from_raw(cls, where: str, raw: object) -> 'TemplateParams': + """ Produce a TemplateParams instance from an object as it is in Hjson. + """ + ret = cls() + rl = check_list(raw, where) + for idx, r_param in enumerate(rl): + entry_where = 'entry {} in {}'.format(idx + 1, where) + param = _parse_template_parameter(entry_where, r_param) + if param.name in ret: + raise ValueError('At {}, found a duplicate parameter with ' + 'name {}.'.format(entry_where, param.name)) + ret.add(param) + return ret + + +class IpTemplate: + """ An IP template. + + An IP template is an IP block which needs to be parametrized before it + can be transformed into an actual IP block (which can then be instantiated + in a hardware design). + """ + + name: str + params: Params + template_path: Path + + def __init__(self, name: str, params: Params, template_path: Path): + self.name = name + self.params = params + self.template_path = template_path + + @classmethod + def from_template_path(cls, template_path: Path) -> 'IpTemplate': + """ Create an IpTemplate from a template directory. + + An IP template directory has a well-defined structure: + + - The IP template name (TEMPLATE_NAME) is equal to the directory name. + - It contains a file 'data/TEMPLATE_NAME.tpldesc.hjson' containing all + configuration information related to the template. + - It contains zero or more files ending in '.tpl'. These files are + Mako templates and rendered into an file in the same location without + the '.tpl' file extension. + """ + + # Check if the directory structure matches expectations. + if not template_path.is_dir(): + raise TemplateParseError( + "Template path {!r} is not a directory.".format( + str(template_path))) + if not (template_path / 'data').is_dir(): + raise TemplateParseError( + "Template path {!r} does not contain the required 'data' directory." + .format(str(template_path))) + + # The template name equals the name of the template directory. + template_name = template_path.stem + + # Find the template description file. + tpldesc_file = template_path / 'data/{}.tpldesc.hjson'.format( + template_name) + + # Read the template description from file. + try: + tpldesc_obj = hjson.load(open(tpldesc_file, 'r'), use_decimal=True) + except (OSError, FileNotFoundError) as e: + raise TemplateParseError( + "Unable to read template description file {!r}: {}".format( + str(tpldesc_file), str(e))) + + # Parse the template description file. + where = 'template description file {!r}'.format(str(tpldesc_file)) + if 'template_param_list' not in tpldesc_obj: + raise TemplateParseError( + f"Required key 'variables' not found in {where}") + + try: + params = TemplateParams.from_raw( + f"list of parameters in {where}", + tpldesc_obj['template_param_list']) + except ValueError as e: + raise TemplateParseError(e) from None + + return cls(template_name, params, template_path) + + +def check_param_dict(obj: object, what: str) -> Dict[str, Union[int, str]]: + """ Check that obj is a dict with str keys and int or str values. """ + if not isinstance(obj, dict): + raise ValueError( + "{} is expected to be a dict, but was actually a {}.".format( + what, + type(obj).__name__)) + + for key, value in obj.items(): + if not isinstance(key, str): + raise ValueError( + '{} has a key {!r}, which is not a string.'.format(what, key)) + + if not (isinstance(value, str) or isinstance(value, int)): + raise ValueError(f"key {key} of {what} has a value {value!r}, " + "which is neither a str, nor an int.") + + return cast(Dict[str, Union[int, str]], obj) + + +class IpConfig: + def __init__(self, + instance_name: str, + param_values: Dict[str, Union[str, int]] = {}): + self.instance_name = instance_name + self.param_values = param_values + + @classmethod + def from_raw(cls, raw: object, where: str) -> 'IpConfig': + """ Load an IpConfig from a raw object """ + + rd = check_keys(raw, 'configuration file ' + where, ['instance_name'], + ['param_values']) + instance_name = check_name(rd.get('instance_name'), + "the key 'instance_name' of " + where) + + param_values = check_param_dict(rd.get('param_values', []), + f"the key 'param_values' of {where}") + + return cls(instance_name, param_values) + + @classmethod + def from_text(cls, txt: str, where: str) -> 'IpConfig': + """Load an IpConfig from an Hjson description in txt""" + return cls.from_raw(hjson.loads(txt, use_decimal=True), where) + + def to_file(self, file_path: Path, header: Optional[str] = ""): + obj = {} + obj['instance_name'] = self.instance_name + obj['param_values'] = str(self.param_values) + + with open(file_path, 'w') as fp: + if header: + fp.write(header) + hjson.dump(obj, + fp, + ensure_ascii=False, + use_decimal=True, + for_json=True, + encoding='UTF-8', + indent=2) + fp.write("\n")
diff --git a/util/ipgen/renderer.py b/util/ipgen/renderer.py new file mode 100644 index 0000000..92d7c0c --- /dev/null +++ b/util/ipgen/renderer.py
@@ -0,0 +1,286 @@ +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import time +from pathlib import Path +from typing import Dict, Optional, Union + +import reggen.gen_rtl +from mako import exceptions as mako_exceptions # type: ignore +from mako.lookup import TemplateLookup as MakoTemplateLookup # type: ignore +from reggen.ip_block import IpBlock + +from .lib import IpConfig, IpTemplate + +_HJSON_LICENSE_HEADER = ("""// Copyright lowRISC contributors. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +""") + + +class TemplateRenderError(Exception): + pass + + +class IpTemplateRendererBase: + """ Render an IpTemplate into an IP block. """ + + ip_template: IpTemplate + ip_config: IpConfig + + _lookup: Optional[MakoTemplateLookup] = None + + def __init__(self, ip_template: IpTemplate, ip_config: IpConfig) -> None: + self.ip_template = ip_template + self.ip_config = ip_config + self._check_param_values() + + def _check_param_values(self) -> None: + """ Check if all parameters in IpConfig are defined in the template. """ + + for name in self.ip_config.param_values: + if name not in self.ip_template.params: + raise KeyError("No parameter named {!r} exists.".format(name)) + + def get_template_parameter_values(self) -> Dict[str, Union[str, int]]: + """ Get a typed mapping of all template parameters and their values. + """ + ret = {} + + for name, template_param in self.ip_template.params.items(): + if name in self.ip_config.param_values: + val = self.ip_config.param_values[name] + else: + val = template_param.default + + assert template_param.param_type in ('string', 'int') + try: + if template_param.param_type == 'string': + val_typed = str(val) # type: Union[int, str] + elif template_param.param_type == 'int': + if not isinstance(val, int): + val_typed = int(val, 0) + else: + val_typed = val + except (ValueError, TypeError): + raise TemplateRenderError( + "For parameter {} cannot convert value {!r} " + "to {}.".format(name, val, + template_param.param_type)) from None + + ret[name] = val_typed + + return ret + + def _get_mako_template_lookup(self) -> MakoTemplateLookup: + """ Get a Mako TemplateLookup object """ + + if self._lookup is None: + # Define the directory containing the IP template as "base" + # directory, allowing templates to include other templates within + # this directory using relative paths. + # Use strict_undefined to throw a NameError if undefined variables + # are used within a template. + self._lookup = MakoTemplateLookup( + directories=[str(self.ip_template.template_path)], + strict_undefined=True) + return self._lookup + + def _tplfunc_instance_vlnv(self, template_vlnv_str: str): + template_vlnv = template_vlnv_str.split(':', 3) + if len(template_vlnv) < 3: + raise TemplateRenderError( + f"{template_vlnv_str} isn't a valid FuseSoC VLNV. " + "Required format: 'vendor:library:name:version'") + template_core_name = template_vlnv[2] + + # Remove the template name from the start of the core name. + # For example, a core name `rv_plic_component` will result in an + # instance name 'my_instance_component' (instead of + # 'my_instance_rv_plic_component'). + if template_core_name.startswith(self.ip_template.name): + template_core_name = template_core_name[len(self.ip_template.name):] + + instance_core_name = self.ip_config.instance_name + template_core_name + instance_vlnv = 'lowrisc:opentitan:' + instance_core_name + return instance_vlnv + + def _render_mako_template_to_str(self, template_filepath: Path) -> str: + """ Render a template and return the rendered text. """ + + lookup = self._get_mako_template_lookup() + template_filepath_rel = template_filepath.relative_to( + self.ip_template.template_path) + template = lookup.get_template(str(template_filepath_rel)) + + helper_funcs = {} + helper_funcs['instance_vlnv'] = self._tplfunc_instance_vlnv + + # TODO: Introduce namespacing for the template parameters and other + # parameters, and/or pass the IpConfig instance to the template after + # we have converted more IP blocks to ipgen. + tpl_args = { + "instance_name": self.ip_config.instance_name, # type: ignore + **helper_funcs, # type: ignore + **self.get_template_parameter_values() # type: ignore + } + try: + return template.render(**tpl_args) + except: + raise TemplateRenderError( + "Unable to render template: " + + mako_exceptions.text_error_template().render()) from None + + def _render_mako_template_to_file(self, template_filepath: Path, + outdir_path: Path) -> None: + """ Render a file template into a file in outdir_path. + + The name of the output file matches the file name of the template file, + without the '.tpl' suffix. + """ + + assert outdir_path.is_dir() + + outfile_name = self._filename_without_tpl_suffix(template_filepath) + + with open(outdir_path / outfile_name, 'w') as f: + f.write(self._render_mako_template_to_str(template_filepath)) + + def _filename_without_tpl_suffix(self, filepath: Path) -> str: + """ Get the name of the file without a '.tpl' suffix. """ + assert filepath.suffix == '.tpl' + return filepath.stem + + +class IpDescriptionOnlyRenderer(IpTemplateRendererBase): + """ Generate the IP description only. + + The IP description is the content of what is typically stored + data/ip_name.hjson. + """ + def render(self) -> str: + template_path = self.ip_template.template_path + + # Look for a data/ip_name.hjson.tpl template file and render it. + hjson_tpl_path = template_path / 'data' / (self.ip_template.name + + '.hjson.tpl') + if hjson_tpl_path.is_file(): + return self._render_mako_template_to_str(hjson_tpl_path) + + # Otherwise, if a data/ip_name.hjson file exists, use that. + hjson_path = template_path / 'data' / (self.ip_template.name + + '.hjson') + try: + with open(hjson_path, 'r') as f: + return f.read() + except FileNotFoundError: + raise FileNotFoundError( + "Neither a IP description template at {}, " + "nor an IP description at {} exist!".format( + hjson_tpl_path, hjson_path)) + + +class IpBlockRenderer(IpTemplateRendererBase): + """ Produce a full IP block directory from a template. + + - Copy everything but Mako templates (files ending with '.tpl') into the + destination directory. + - Process all Mako templates and write the results to the output directory. + A template at <PATH>.tpl will write results to <PATH> in the output + directory. + - Run reggen to generate the register interface. + """ + def render(self, output_dir: Path, overwrite_output_dir: bool) -> None: + """ Render the IP template into output_dir. """ + + # Ensure that we operate on an absolute path for output_dir. + output_dir = output_dir.resolve() + + if not overwrite_output_dir and output_dir.exists(): + raise TemplateRenderError( + "Output directory '{}' exists and should not be overwritten.". + format(output_dir)) + + # Prepare the IP directory in a staging area to later atomically move it + # to the final destination. + output_dir_staging = output_dir.parent / f".~{output_dir.stem}.staging" + if output_dir_staging.is_dir(): + raise TemplateRenderError( + "Output staging directory '{}' already exists. Remove it and " + "try again.".format(output_dir_staging)) + + template_path = self.ip_template.template_path + + try: + # Copy everything but the templates. + shutil.copytree(template_path, + output_dir_staging, + ignore=shutil.ignore_patterns('*.tpl')) + + # Render templates. + for template_filepath in template_path.glob('**/*.tpl'): + template_filepath_rel = template_filepath.relative_to( + template_path) + + # Put the output file into the same relative directory as the + # template. The output file will also have the same name as the + # template, just without the '.tpl' suffix. + outdir_path = output_dir_staging / template_filepath_rel.parent + + self._render_mako_template_to_file(template_filepath, + outdir_path) + + # Generate register interface through reggen. + hjson_path = (output_dir_staging / 'data' / + (self.ip_template.name + '.hjson')) + rtl_path = output_dir_staging / 'rtl' + # TODO: Pass on template parameters to reggen? Or enable the user + # to set a different set of parameters in the renderer? + reggen.gen_rtl.gen_rtl(IpBlock.from_path(str(hjson_path), []), + str(rtl_path)) + + # Write IP configuration (to reproduce the generation process). + # TODO: Should the ipconfig file be written to the instance name, + # or the template name? + self.ip_config.to_file( + output_dir_staging / + 'data/{}.ipconfig.hjson'.format(self.ip_config.instance_name), + header=_HJSON_LICENSE_HEADER) + + # Safely overwrite the existing directory if necessary: + # + # - First move the existing directory out of the way. + # - Then move the staging directory with the new content in place. + # - Finally remove the old directory. + # + # If anything goes wrong in the meantime we are left with either + # the old or the new directory, and potentially some backups of + # outdated files. + do_overwrite = overwrite_output_dir and output_dir.exists() + output_dir_existing_bak = output_dir.with_suffix( + '.bak~' + str(int(time.time()))) + if do_overwrite: + os.rename(output_dir, output_dir_existing_bak) + + # Move the staging directory to the final destination. + os.rename(output_dir_staging, output_dir) + + # Remove the old/"overwritten" data. + if do_overwrite: + try: + shutil.rmtree(output_dir_existing_bak) + except Exception as e: + msg = ( + f'Unable to delete the backup directory ' + '{output_dir_existing_bak} of the overwritten data. ' + 'Please remove it manually.') + raise TemplateRenderError(msg).with_traceback( + e.__traceback__) + + finally: + # Ensure that the staging directory is removed at the end. Ignore + # errors as the directory should not exist at this point actually. + shutil.rmtree(output_dir_staging, ignore_errors=True)