[ipgen] Initial commit of IP generation tool
Signed-off-by: Philipp Wagner <phw@lowrisc.org>
diff --git a/util/ipgen/__init__.py b/util/ipgen/__init__.py
new file mode 100644
index 0000000..7c72a5d
--- /dev/null
+++ b/util/ipgen/__init__.py
@@ -0,0 +1,7 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+from .lib import IpConfig, IpTemplate, TemplateParseError # noqa: F401
+from .renderer import (IpBlockRenderer, IpDescriptionOnlyRenderer,
+ TemplateRenderError) # noqa: F401
diff --git a/util/ipgen/lib.py b/util/ipgen/lib.py
new file mode 100644
index 0000000..e1453e7
--- /dev/null
+++ b/util/ipgen/lib.py
@@ -0,0 +1,209 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+from pathlib import Path
+from typing import Dict, Optional, Union, cast
+
+import hjson # type: ignore
+from reggen.lib import check_int, check_keys, check_list, check_name, check_str
+from reggen.params import BaseParam, Params
+
+
+class TemplateParseError(Exception):
+ pass
+
+
+class TemplateParameter(BaseParam):
+ """ A template parameter. """
+ def __init__(self, name: str, desc: Optional[str], param_type: str,
+ default: str):
+ super().__init__(name, desc, param_type)
+ self.default = default
+ self.value = None
+
+ def as_dict(self) -> Dict[str, object]:
+ rd = super().as_dict()
+ rd['default'] = self.default
+ return rd
+
+
+def _parse_template_parameter(where: str, raw: object) -> TemplateParameter:
+ rd = check_keys(raw, where, ['name', 'desc', 'type'], ['default'])
+
+ name = check_str(rd['name'], 'name field of ' + where)
+
+ r_desc = rd.get('desc')
+ if r_desc is None:
+ desc = None
+ else:
+ desc = check_str(r_desc, 'desc field of ' + where)
+
+ r_type = rd.get('type')
+ param_type = check_str(r_type, 'type field of ' + where)
+ if not param_type in ('string', 'int'):
+ raise ValueError('At {}, the {} param has an invalid type field {!r}. '
+ 'Allowed values are: string, int.'.format(
+ where, name, param_type))
+
+ r_default = rd.get('default')
+ default = check_str(r_default, 'default field of ' + where)
+ if param_type[:3] == 'int':
+ check_int(default,
+ 'default field of {}, (an integer parameter)'.format(name))
+
+ return TemplateParameter(name, desc, param_type, default)
+
+
+class TemplateParams(Params):
+ """ A group of template parameters. """
+ @classmethod
+ def from_raw(cls, where: str, raw: object) -> 'TemplateParams':
+ """ Produce a TemplateParams instance from an object as it is in Hjson.
+ """
+ ret = cls()
+ rl = check_list(raw, where)
+ for idx, r_param in enumerate(rl):
+ entry_where = 'entry {} in {}'.format(idx + 1, where)
+ param = _parse_template_parameter(entry_where, r_param)
+ if param.name in ret:
+ raise ValueError('At {}, found a duplicate parameter with '
+ 'name {}.'.format(entry_where, param.name))
+ ret.add(param)
+ return ret
+
+
+class IpTemplate:
+ """ An IP template.
+
+ An IP template is an IP block which needs to be parametrized before it
+ can be transformed into an actual IP block (which can then be instantiated
+ in a hardware design).
+ """
+
+ name: str
+ params: Params
+ template_path: Path
+
+ def __init__(self, name: str, params: Params, template_path: Path):
+ self.name = name
+ self.params = params
+ self.template_path = template_path
+
+ @classmethod
+ def from_template_path(cls, template_path: Path) -> 'IpTemplate':
+ """ Create an IpTemplate from a template directory.
+
+ An IP template directory has a well-defined structure:
+
+ - The IP template name (TEMPLATE_NAME) is equal to the directory name.
+ - It contains a file 'data/TEMPLATE_NAME.tpldesc.hjson' containing all
+ configuration information related to the template.
+ - It contains zero or more files ending in '.tpl'. These files are
+ Mako templates and rendered into an file in the same location without
+ the '.tpl' file extension.
+ """
+
+ # Check if the directory structure matches expectations.
+ if not template_path.is_dir():
+ raise TemplateParseError(
+ "Template path {!r} is not a directory.".format(
+ str(template_path)))
+ if not (template_path / 'data').is_dir():
+ raise TemplateParseError(
+ "Template path {!r} does not contain the required 'data' directory."
+ .format(str(template_path)))
+
+ # The template name equals the name of the template directory.
+ template_name = template_path.stem
+
+ # Find the template description file.
+ tpldesc_file = template_path / 'data/{}.tpldesc.hjson'.format(
+ template_name)
+
+ # Read the template description from file.
+ try:
+ tpldesc_obj = hjson.load(open(tpldesc_file, 'r'), use_decimal=True)
+ except (OSError, FileNotFoundError) as e:
+ raise TemplateParseError(
+ "Unable to read template description file {!r}: {}".format(
+ str(tpldesc_file), str(e)))
+
+ # Parse the template description file.
+ where = 'template description file {!r}'.format(str(tpldesc_file))
+ if 'template_param_list' not in tpldesc_obj:
+ raise TemplateParseError(
+ f"Required key 'variables' not found in {where}")
+
+ try:
+ params = TemplateParams.from_raw(
+ f"list of parameters in {where}",
+ tpldesc_obj['template_param_list'])
+ except ValueError as e:
+ raise TemplateParseError(e) from None
+
+ return cls(template_name, params, template_path)
+
+
+def check_param_dict(obj: object, what: str) -> Dict[str, Union[int, str]]:
+ """ Check that obj is a dict with str keys and int or str values. """
+ if not isinstance(obj, dict):
+ raise ValueError(
+ "{} is expected to be a dict, but was actually a {}.".format(
+ what,
+ type(obj).__name__))
+
+ for key, value in obj.items():
+ if not isinstance(key, str):
+ raise ValueError(
+ '{} has a key {!r}, which is not a string.'.format(what, key))
+
+ if not (isinstance(value, str) or isinstance(value, int)):
+ raise ValueError(f"key {key} of {what} has a value {value!r}, "
+ "which is neither a str, nor an int.")
+
+ return cast(Dict[str, Union[int, str]], obj)
+
+
+class IpConfig:
+ def __init__(self,
+ instance_name: str,
+ param_values: Dict[str, Union[str, int]] = {}):
+ self.instance_name = instance_name
+ self.param_values = param_values
+
+ @classmethod
+ def from_raw(cls, raw: object, where: str) -> 'IpConfig':
+ """ Load an IpConfig from a raw object """
+
+ rd = check_keys(raw, 'configuration file ' + where, ['instance_name'],
+ ['param_values'])
+ instance_name = check_name(rd.get('instance_name'),
+ "the key 'instance_name' of " + where)
+
+ param_values = check_param_dict(rd.get('param_values', []),
+ f"the key 'param_values' of {where}")
+
+ return cls(instance_name, param_values)
+
+ @classmethod
+ def from_text(cls, txt: str, where: str) -> 'IpConfig':
+ """Load an IpConfig from an Hjson description in txt"""
+ return cls.from_raw(hjson.loads(txt, use_decimal=True), where)
+
+ def to_file(self, file_path: Path, header: Optional[str] = ""):
+ obj = {}
+ obj['instance_name'] = self.instance_name
+ obj['param_values'] = str(self.param_values)
+
+ with open(file_path, 'w') as fp:
+ if header:
+ fp.write(header)
+ hjson.dump(obj,
+ fp,
+ ensure_ascii=False,
+ use_decimal=True,
+ for_json=True,
+ encoding='UTF-8',
+ indent=2)
+ fp.write("\n")
diff --git a/util/ipgen/renderer.py b/util/ipgen/renderer.py
new file mode 100644
index 0000000..92d7c0c
--- /dev/null
+++ b/util/ipgen/renderer.py
@@ -0,0 +1,286 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Dict, Optional, Union
+
+import reggen.gen_rtl
+from mako import exceptions as mako_exceptions # type: ignore
+from mako.lookup import TemplateLookup as MakoTemplateLookup # type: ignore
+from reggen.ip_block import IpBlock
+
+from .lib import IpConfig, IpTemplate
+
+_HJSON_LICENSE_HEADER = ("""// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+""")
+
+
+class TemplateRenderError(Exception):
+ pass
+
+
+class IpTemplateRendererBase:
+ """ Render an IpTemplate into an IP block. """
+
+ ip_template: IpTemplate
+ ip_config: IpConfig
+
+ _lookup: Optional[MakoTemplateLookup] = None
+
+ def __init__(self, ip_template: IpTemplate, ip_config: IpConfig) -> None:
+ self.ip_template = ip_template
+ self.ip_config = ip_config
+ self._check_param_values()
+
+ def _check_param_values(self) -> None:
+ """ Check if all parameters in IpConfig are defined in the template. """
+
+ for name in self.ip_config.param_values:
+ if name not in self.ip_template.params:
+ raise KeyError("No parameter named {!r} exists.".format(name))
+
+ def get_template_parameter_values(self) -> Dict[str, Union[str, int]]:
+ """ Get a typed mapping of all template parameters and their values.
+ """
+ ret = {}
+
+ for name, template_param in self.ip_template.params.items():
+ if name in self.ip_config.param_values:
+ val = self.ip_config.param_values[name]
+ else:
+ val = template_param.default
+
+ assert template_param.param_type in ('string', 'int')
+ try:
+ if template_param.param_type == 'string':
+ val_typed = str(val) # type: Union[int, str]
+ elif template_param.param_type == 'int':
+ if not isinstance(val, int):
+ val_typed = int(val, 0)
+ else:
+ val_typed = val
+ except (ValueError, TypeError):
+ raise TemplateRenderError(
+ "For parameter {} cannot convert value {!r} "
+ "to {}.".format(name, val,
+ template_param.param_type)) from None
+
+ ret[name] = val_typed
+
+ return ret
+
+ def _get_mako_template_lookup(self) -> MakoTemplateLookup:
+ """ Get a Mako TemplateLookup object """
+
+ if self._lookup is None:
+ # Define the directory containing the IP template as "base"
+ # directory, allowing templates to include other templates within
+ # this directory using relative paths.
+ # Use strict_undefined to throw a NameError if undefined variables
+ # are used within a template.
+ self._lookup = MakoTemplateLookup(
+ directories=[str(self.ip_template.template_path)],
+ strict_undefined=True)
+ return self._lookup
+
+ def _tplfunc_instance_vlnv(self, template_vlnv_str: str):
+ template_vlnv = template_vlnv_str.split(':', 3)
+ if len(template_vlnv) < 3:
+ raise TemplateRenderError(
+ f"{template_vlnv_str} isn't a valid FuseSoC VLNV. "
+ "Required format: 'vendor:library:name:version'")
+ template_core_name = template_vlnv[2]
+
+ # Remove the template name from the start of the core name.
+ # For example, a core name `rv_plic_component` will result in an
+ # instance name 'my_instance_component' (instead of
+ # 'my_instance_rv_plic_component').
+ if template_core_name.startswith(self.ip_template.name):
+ template_core_name = template_core_name[len(self.ip_template.name):]
+
+ instance_core_name = self.ip_config.instance_name + template_core_name
+ instance_vlnv = 'lowrisc:opentitan:' + instance_core_name
+ return instance_vlnv
+
+ def _render_mako_template_to_str(self, template_filepath: Path) -> str:
+ """ Render a template and return the rendered text. """
+
+ lookup = self._get_mako_template_lookup()
+ template_filepath_rel = template_filepath.relative_to(
+ self.ip_template.template_path)
+ template = lookup.get_template(str(template_filepath_rel))
+
+ helper_funcs = {}
+ helper_funcs['instance_vlnv'] = self._tplfunc_instance_vlnv
+
+ # TODO: Introduce namespacing for the template parameters and other
+ # parameters, and/or pass the IpConfig instance to the template after
+ # we have converted more IP blocks to ipgen.
+ tpl_args = {
+ "instance_name": self.ip_config.instance_name, # type: ignore
+ **helper_funcs, # type: ignore
+ **self.get_template_parameter_values() # type: ignore
+ }
+ try:
+ return template.render(**tpl_args)
+ except:
+ raise TemplateRenderError(
+ "Unable to render template: " +
+ mako_exceptions.text_error_template().render()) from None
+
+ def _render_mako_template_to_file(self, template_filepath: Path,
+ outdir_path: Path) -> None:
+ """ Render a file template into a file in outdir_path.
+
+ The name of the output file matches the file name of the template file,
+ without the '.tpl' suffix.
+ """
+
+ assert outdir_path.is_dir()
+
+ outfile_name = self._filename_without_tpl_suffix(template_filepath)
+
+ with open(outdir_path / outfile_name, 'w') as f:
+ f.write(self._render_mako_template_to_str(template_filepath))
+
+ def _filename_without_tpl_suffix(self, filepath: Path) -> str:
+ """ Get the name of the file without a '.tpl' suffix. """
+ assert filepath.suffix == '.tpl'
+ return filepath.stem
+
+
+class IpDescriptionOnlyRenderer(IpTemplateRendererBase):
+ """ Generate the IP description only.
+
+ The IP description is the content of what is typically stored
+ data/ip_name.hjson.
+ """
+ def render(self) -> str:
+ template_path = self.ip_template.template_path
+
+ # Look for a data/ip_name.hjson.tpl template file and render it.
+ hjson_tpl_path = template_path / 'data' / (self.ip_template.name +
+ '.hjson.tpl')
+ if hjson_tpl_path.is_file():
+ return self._render_mako_template_to_str(hjson_tpl_path)
+
+ # Otherwise, if a data/ip_name.hjson file exists, use that.
+ hjson_path = template_path / 'data' / (self.ip_template.name +
+ '.hjson')
+ try:
+ with open(hjson_path, 'r') as f:
+ return f.read()
+ except FileNotFoundError:
+ raise FileNotFoundError(
+ "Neither a IP description template at {}, "
+ "nor an IP description at {} exist!".format(
+ hjson_tpl_path, hjson_path))
+
+
+class IpBlockRenderer(IpTemplateRendererBase):
+ """ Produce a full IP block directory from a template.
+
+ - Copy everything but Mako templates (files ending with '.tpl') into the
+ destination directory.
+ - Process all Mako templates and write the results to the output directory.
+ A template at <PATH>.tpl will write results to <PATH> in the output
+ directory.
+ - Run reggen to generate the register interface.
+ """
+ def render(self, output_dir: Path, overwrite_output_dir: bool) -> None:
+ """ Render the IP template into output_dir. """
+
+ # Ensure that we operate on an absolute path for output_dir.
+ output_dir = output_dir.resolve()
+
+ if not overwrite_output_dir and output_dir.exists():
+ raise TemplateRenderError(
+ "Output directory '{}' exists and should not be overwritten.".
+ format(output_dir))
+
+ # Prepare the IP directory in a staging area to later atomically move it
+ # to the final destination.
+ output_dir_staging = output_dir.parent / f".~{output_dir.stem}.staging"
+ if output_dir_staging.is_dir():
+ raise TemplateRenderError(
+ "Output staging directory '{}' already exists. Remove it and "
+ "try again.".format(output_dir_staging))
+
+ template_path = self.ip_template.template_path
+
+ try:
+ # Copy everything but the templates.
+ shutil.copytree(template_path,
+ output_dir_staging,
+ ignore=shutil.ignore_patterns('*.tpl'))
+
+ # Render templates.
+ for template_filepath in template_path.glob('**/*.tpl'):
+ template_filepath_rel = template_filepath.relative_to(
+ template_path)
+
+ # Put the output file into the same relative directory as the
+ # template. The output file will also have the same name as the
+ # template, just without the '.tpl' suffix.
+ outdir_path = output_dir_staging / template_filepath_rel.parent
+
+ self._render_mako_template_to_file(template_filepath,
+ outdir_path)
+
+ # Generate register interface through reggen.
+ hjson_path = (output_dir_staging / 'data' /
+ (self.ip_template.name + '.hjson'))
+ rtl_path = output_dir_staging / 'rtl'
+ # TODO: Pass on template parameters to reggen? Or enable the user
+ # to set a different set of parameters in the renderer?
+ reggen.gen_rtl.gen_rtl(IpBlock.from_path(str(hjson_path), []),
+ str(rtl_path))
+
+ # Write IP configuration (to reproduce the generation process).
+ # TODO: Should the ipconfig file be written to the instance name,
+ # or the template name?
+ self.ip_config.to_file(
+ output_dir_staging /
+ 'data/{}.ipconfig.hjson'.format(self.ip_config.instance_name),
+ header=_HJSON_LICENSE_HEADER)
+
+ # Safely overwrite the existing directory if necessary:
+ #
+ # - First move the existing directory out of the way.
+ # - Then move the staging directory with the new content in place.
+ # - Finally remove the old directory.
+ #
+ # If anything goes wrong in the meantime we are left with either
+ # the old or the new directory, and potentially some backups of
+ # outdated files.
+ do_overwrite = overwrite_output_dir and output_dir.exists()
+ output_dir_existing_bak = output_dir.with_suffix(
+ '.bak~' + str(int(time.time())))
+ if do_overwrite:
+ os.rename(output_dir, output_dir_existing_bak)
+
+ # Move the staging directory to the final destination.
+ os.rename(output_dir_staging, output_dir)
+
+ # Remove the old/"overwritten" data.
+ if do_overwrite:
+ try:
+ shutil.rmtree(output_dir_existing_bak)
+ except Exception as e:
+ msg = (
+ f'Unable to delete the backup directory '
+ '{output_dir_existing_bak} of the overwritten data. '
+ 'Please remove it manually.')
+ raise TemplateRenderError(msg).with_traceback(
+ e.__traceback__)
+
+ finally:
+ # Ensure that the staging directory is removed at the end. Ignore
+ # errors as the directory should not exist at this point actually.
+ shutil.rmtree(output_dir_staging, ignore_errors=True)
