[util] Move some generic parsing code to util
This is used in the OTBN code to convert dictionaries parsed from YAML
or hjson into proper objects. This patch moves the code to a new
util/serialize directory and teaches the OTBN code that was using it
how to find the new version.
The idea is that this might be useful for tightening up parsing in
e.g. reggen and topgen and will hopefully also be useful for future
tooling.
Signed-off-by: Rupert Swarbrick <rswarbrick@lowrisc.org>
diff --git a/hw/ip/otbn/dv/otbnsim/Makefile b/hw/ip/otbn/dv/otbnsim/Makefile
index 78b5104..bd7bd1a 100644
--- a/hw/ip/otbn/dv/otbnsim/Makefile
+++ b/hw/ip/otbn/dv/otbnsim/Makefile
@@ -20,7 +20,7 @@
lint-stamps := $(foreach scr,$(py-scripts),$(build-dir)/$(scr).stamp)
$(lint-stamps): $(build-dir)/%.stamp: % $(py-libs) | $(build-dir)
- env MYPYPATH="$$MYPYPATH:../../util" mypy --strict $< $(py-libs)
+ mypy --strict --config-file=mypy.ini $<
touch $@
.PHONY: lint
diff --git a/hw/ip/otbn/dv/otbnsim/mypy.ini b/hw/ip/otbn/dv/otbnsim/mypy.ini
new file mode 100644
index 0000000..fcc4244
--- /dev/null
+++ b/hw/ip/otbn/dv/otbnsim/mypy.ini
@@ -0,0 +1,3 @@
+[mypy]
+# Add OTBN and OpenTitan util dirs to MYPYPATH
+mypy_path = $MYPY_CONFIG_FILE_DIR/../../util, $MYPY_CONFIG_FILE_DIR/../../../../../util
diff --git a/hw/ip/otbn/dv/otbnsim/sim/ext_regs.py b/hw/ip/otbn/dv/otbnsim/sim/ext_regs.py
index 688cbe7..73b5469 100644
--- a/hw/ip/otbn/dv/otbnsim/sim/ext_regs.py
+++ b/hw/ip/otbn/dv/otbnsim/sim/ext_regs.py
@@ -4,7 +4,11 @@
from typing import Callable, Dict, List, Sequence
-from shared.otbn_reggen import Field, Register, RegBlock, load_registers
+from reggen.field import Field
+from reggen.register import Register
+from reggen.reg_block import RegBlock
+
+from shared.otbn_reggen import load_registers
from .trace import Trace
diff --git a/hw/ip/otbn/dv/rig/Makefile b/hw/ip/otbn/dv/rig/Makefile
index bc19684..da31c9e 100644
--- a/hw/ip/otbn/dv/rig/Makefile
+++ b/hw/ip/otbn/dv/rig/Makefile
@@ -20,8 +20,7 @@
lint-stamps := $(foreach s,$(pyscripts),$(lint-build-dir)/$(s).stamp)
$(lint-build-dir)/%.stamp: % $(pylibs) | $(lint-build-dir)
- env MYPYPATH=../../util:$$MYPYPATH \
- mypy --strict $<
+ mypy --strict --config-file=mypy.ini $<
touch $@
.PHONY: lint
diff --git a/hw/ip/otbn/dv/rig/mypy.ini b/hw/ip/otbn/dv/rig/mypy.ini
new file mode 100644
index 0000000..fcc4244
--- /dev/null
+++ b/hw/ip/otbn/dv/rig/mypy.ini
@@ -0,0 +1,3 @@
+[mypy]
+# Add OTBN and OpenTitan util dirs to MYPYPATH
+mypy_path = $MYPY_CONFIG_FILE_DIR/../../util, $MYPY_CONFIG_FILE_DIR/../../../../../util
diff --git a/hw/ip/otbn/dv/rig/otbn-rig b/hw/ip/otbn/dv/rig/otbn-rig
index 9241186..528f257 100755
--- a/hw/ip/otbn/dv/rig/otbn-rig
+++ b/hw/ip/otbn/dv/rig/otbn-rig
@@ -16,8 +16,8 @@
# can import modules like "shared.foo" and get the OTBN shared code.
_RIG_DIR = os.path.dirname(__file__)
_OTBN_DIR = os.path.normpath(os.path.join(_RIG_DIR, '../..'))
-_UTIL_DIR = os.path.join(_OTBN_DIR, 'util')
-sys.path.append(_UTIL_DIR)
+_OTBN_UTIL_DIR = os.path.join(_OTBN_DIR, 'util')
+sys.path.append(_OTBN_UTIL_DIR)
from shared.insn_yaml import InsnsFile, load_insns_yaml # noqa: E402
diff --git a/hw/ip/otbn/dv/rig/rig/config.py b/hw/ip/otbn/dv/rig/rig/config.py
index a543128..326c995 100644
--- a/hw/ip/otbn/dv/rig/rig/config.py
+++ b/hw/ip/otbn/dv/rig/rig/config.py
@@ -6,7 +6,7 @@
import random
from typing import Dict, List, Optional, Set, Tuple
-from shared.yaml_parse_helpers import check_str, check_keys, load_yaml
+from serialize.parse_helpers import check_str, check_keys, load_yaml
class Weights:
diff --git a/hw/ip/otbn/util/Makefile b/hw/ip/otbn/util/Makefile
index f2024a9..2a188c3 100644
--- a/hw/ip/otbn/util/Makefile
+++ b/hw/ip/otbn/util/Makefile
@@ -20,7 +20,7 @@
lint-stamps := $(foreach s,$(pyscripts),$(lint-build-dir)/$(s).stamp)
$(lint-build-dir)/%.stamp: % $(pylibs) | $(lint-build-dir)
- mypy --strict $< $(pylibs)
+ mypy --strict --config-file=mypy.ini $<
touch $@
.PHONY: lint
diff --git a/hw/ip/otbn/util/mypy.ini b/hw/ip/otbn/util/mypy.ini
new file mode 100644
index 0000000..7fe056b
--- /dev/null
+++ b/hw/ip/otbn/util/mypy.ini
@@ -0,0 +1,2 @@
+[mypy]
+mypy_path = $MYPY_CONFIG_FILE_DIR/../../../../util
diff --git a/hw/ip/otbn/util/shared/__init__.py b/hw/ip/otbn/util/shared/__init__.py
index e69de29..a798ae8 100644
--- a/hw/ip/otbn/util/shared/__init__.py
+++ b/hw/ip/otbn/util/shared/__init__.py
@@ -0,0 +1,18 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+
+# Ensure that the OpenTitan utils directory is on sys.path. This will allow us
+# (and anyone who depends on us) to import serialize.parse_helpers.
+#
+# This isn't massively clean: in particular, messing around with sys.path like
+# this in a library __init__ file can cause havoc with paths. *But* doing it
+# properly would either mean installing Python libraries or pasting the lines
+# below into every script that wanted to use the utility code.
+_OTBN_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '../..'))
+_OT_DIR = os.path.normpath(os.path.join(_OTBN_DIR, '../../..'))
+_OT_UTIL_DIR = os.path.join(_OT_DIR, 'util')
+sys.path.append(_OT_UTIL_DIR)
diff --git a/hw/ip/otbn/util/shared/encoding.py b/hw/ip/otbn/util/shared/encoding.py
index e697d7a..8df48b4 100644
--- a/hw/ip/otbn/util/shared/encoding.py
+++ b/hw/ip/otbn/util/shared/encoding.py
@@ -5,9 +5,10 @@
import re
from typing import Dict, Tuple, Union
+from serialize.parse_helpers import check_keys, check_str
+
from .bool_literal import BoolLiteral
from .encoding_scheme import EncSchemeField, EncSchemes
-from .yaml_parse_helpers import check_keys, check_str
class EncodingField:
diff --git a/hw/ip/otbn/util/shared/encoding_scheme.py b/hw/ip/otbn/util/shared/encoding_scheme.py
index 0858286..7b83c7d 100644
--- a/hw/ip/otbn/util/shared/encoding_scheme.py
+++ b/hw/ip/otbn/util/shared/encoding_scheme.py
@@ -7,9 +7,10 @@
import re
from typing import Dict, List, Optional, Set
+from serialize.parse_helpers import check_keys, check_str, check_list, index_list
+
from .bit_ranges import BitRanges
from .bool_literal import BoolLiteral
-from .yaml_parse_helpers import check_keys, check_str, check_list, index_list
class EncSchemeField:
diff --git a/hw/ip/otbn/util/shared/insn_yaml.py b/hw/ip/otbn/util/shared/insn_yaml.py
index e5a351c..cb808de 100644
--- a/hw/ip/otbn/util/shared/insn_yaml.py
+++ b/hw/ip/otbn/util/shared/insn_yaml.py
@@ -9,14 +9,15 @@
import re
from typing import Dict, List, Optional, Tuple, cast
+from serialize.parse_helpers import (check_keys, check_str, check_bool,
+ check_list, index_list, get_optional_str,
+ load_yaml)
+
from .encoding import Encoding
from .encoding_scheme import EncSchemes
from .lsu_desc import LSUDesc
from .operand import Operand
from .syntax import InsnSyntax
-from .yaml_parse_helpers import (check_keys, check_str, check_bool,
- check_list, index_list, get_optional_str,
- load_yaml)
class Insn:
diff --git a/hw/ip/otbn/util/shared/lsu_desc.py b/hw/ip/otbn/util/shared/lsu_desc.py
index 80c6ccd..c9bb7da 100644
--- a/hw/ip/otbn/util/shared/lsu_desc.py
+++ b/hw/ip/otbn/util/shared/lsu_desc.py
@@ -4,7 +4,7 @@
from typing import List
-from .yaml_parse_helpers import check_keys, check_str, check_int
+from serialize.parse_helpers import check_keys, check_str, check_int
class LSUDesc:
diff --git a/hw/ip/otbn/util/shared/mem_layout.py b/hw/ip/otbn/util/shared/mem_layout.py
index 6ff65b1..2d2a64c 100644
--- a/hw/ip/otbn/util/shared/mem_layout.py
+++ b/hw/ip/otbn/util/shared/mem_layout.py
@@ -20,7 +20,9 @@
from typing import Dict, Optional, Tuple
-from .otbn_reggen import load_registers, RegBlock
+from reggen.reg_block import RegBlock
+
+from .otbn_reggen import load_registers
# A window is represented as (offset, size)
_Window = Tuple[int, int]
diff --git a/hw/ip/otbn/util/shared/operand.py b/hw/ip/otbn/util/shared/operand.py
index d860041..1aa085c 100644
--- a/hw/ip/otbn/util/shared/operand.py
+++ b/hw/ip/otbn/util/shared/operand.py
@@ -5,10 +5,11 @@
import re
from typing import List, Optional, Tuple
+from serialize.parse_helpers import (check_keys, check_bool,
+ check_str, get_optional_str)
+
from .encoding import Encoding
from .encoding_scheme import EncSchemeField
-from .yaml_parse_helpers import (check_keys, check_bool,
- check_str, get_optional_str)
class OperandType:
diff --git a/hw/ip/otbn/util/shared/otbn_reggen.py b/hw/ip/otbn/util/shared/otbn_reggen.py
index a70110d..24425f1 100644
--- a/hw/ip/otbn/util/shared/otbn_reggen.py
+++ b/hw/ip/otbn/util/shared/otbn_reggen.py
@@ -5,33 +5,9 @@
'''A wrapper around reggen for otbn.hjson'''
import os
-import sys
from typing import Optional, Tuple
-
-# We use reggen to read the hjson file. Since that lives somewhere completely
-# different from this script (and there aren't __init__.py files scattered all
-# over the OpenTitan repository), we have to do sys.path hacks to find it.
-_OLD_SYS_PATH = sys.path
-try:
- _UTIL_PATH = os.path.join(os.path.dirname(__file__),
- '..', '..', '..', '..', '..', 'util')
- sys.path = [_UTIL_PATH] + _OLD_SYS_PATH
- import reggen.field # type: ignore
- import reggen.ip_block # type: ignore
- import reggen.reg_block # type: ignore
- import reggen.register # type: ignore
- import reggen.window # type: ignore
-finally:
- sys.path = _OLD_SYS_PATH
-
-# Re-export some reggen types so that code importing otbn_reggen can get them
-# transitively without having to mess around with sys.path.
-Register = reggen.register.Register
-Field = reggen.field.Field
-Window = reggen.window.Window
-RegBlock = reggen.reg_block.RegBlock
-IpBlock = reggen.ip_block.IpBlock
+from reggen import ip_block, reg_block
_LR_RETVAL = None # type: Optional[Tuple[int, object]]
@@ -51,7 +27,7 @@
'..', '..', 'data', 'otbn.hjson')
try:
- obj = IpBlock.from_path(path, [])
+ obj = ip_block.IpBlock.from_path(path, [])
except ValueError as err:
raise RuntimeError('Failed to parse {!r}: {}'.format(path, err))
@@ -60,6 +36,6 @@
reg_byte_width = (reg_bit_width + 7) // 8
registers = obj.reg_blocks[None]
- assert isinstance(registers, RegBlock)
+ assert isinstance(registers, reg_block.RegBlock)
_LR_RETVAL = (reg_byte_width, registers)
return _LR_RETVAL
diff --git a/util/serialize/__init__.py b/util/serialize/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/util/serialize/__init__.py
diff --git a/hw/ip/otbn/util/shared/yaml_parse_helpers.py b/util/serialize/parse_helpers.py
similarity index 97%
rename from hw/ip/otbn/util/shared/yaml_parse_helpers.py
rename to util/serialize/parse_helpers.py
index dfec345..db72c23 100644
--- a/hw/ip/otbn/util/shared/yaml_parse_helpers.py
+++ b/util/serialize/parse_helpers.py
@@ -2,7 +2,7 @@
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
-'''Code to help make typed objects out of parsed YAML'''
+'''Code to help make typed objects out of parsed YAML / json / hjson'''
from typing import Callable, Dict, List, Optional, Sequence, TypeVar
