[topgen] Use OrderedDict for top, xbar objects
This is related to PR #1552.
Problem:
In Azure Pipeline that uses Python 3.5, doesn't keep the order of
hjson keys. It results the generated hjson has a lot of change
randomly.
`dict` type in python is unordered dictionary. It doesn't maintain the
order of keys in the dictionary. `topgen` dumps a couple of hjson files,
such as generated tops (e.g.
`hw/top_earlgrey_data/autogen/top_earlgrey.gen.hjson`) and the crossbar
configurations (e.g.
`$TOP/ip/xbar_{name}/data/autogen/xbar_{name}.hjson`). This unordered
behavior creates random changes to the generated files, which increases
commit diffs.
From python3.7, the default dictionary type is changed to
`collections.OrderedDict`. So, if the `topgen` runs on the system >=
python3.7, it always maintain the order.
Resolution:
Use OrderedDict when create dictionary variables.
With OrderedDict (introduced in Python2.7), the generated hjson files
are remains in order.
Signed-off-by: Eunchan Kim <eunchan@opentitan.org>
diff --git a/util/topgen.py b/util/topgen.py
index 0efe46a..dab1da0 100755
--- a/util/topgen.py
+++ b/util/topgen.py
@@ -9,6 +9,7 @@
import sys
from io import StringIO
from pathlib import Path
+from collections import OrderedDict
import hjson
from mako.template import Template
@@ -86,6 +87,7 @@
# generate testbench for xbar
tlgen.generate_tb(xbar, dv_path)
+
def generate_alert_handler(top, out_path):
# default values
esc_cnt_dw = 32
@@ -231,7 +233,7 @@
# TODO: More secure way to gneerate RTL
hjson_obj = hjson.loads(out,
use_decimal=True,
- object_pairs_hook=validate.checking_dict)
+ object_pairs_hook=OrderedDict)
validate.validate(hjson_obj)
gen_rtl.gen_rtl(hjson_obj, str(rtl_path))
@@ -465,7 +467,9 @@
# load top configuration
try:
with open(args.topcfg, 'r') as ftop:
- topcfg = hjson.load(ftop, use_decimal=True)
+ topcfg = hjson.load(ftop,
+ use_decimal=True,
+ object_pairs_hook=OrderedDict)
except ValueError:
raise SystemExit(sys.exc_info()[1])
@@ -500,7 +504,7 @@
obj = hjson.load(x.open('r'),
use_decimal=True,
- object_pairs_hook=validate.checking_dict)
+ object_pairs_hook=OrderedDict)
if validate.validate(obj) != 0:
log.info("Parsing IP %s configuration failed. Skip" % x)
continue
@@ -541,7 +545,9 @@
# load top.complete configuration
try:
with open(args.topcfg, 'r') as ftop:
- completecfg = hjson.load(ftop, use_decimal=True)
+ completecfg = hjson.load(ftop,
+ use_decimal=True,
+ object_pairs_hook=OrderedDict)
except ValueError:
raise SystemExit(sys.exc_info()[1])
diff --git a/util/topgen/intermodule.py b/util/topgen/intermodule.py
index ad73c21..334b9db 100644
--- a/util/topgen/intermodule.py
+++ b/util/topgen/intermodule.py
@@ -4,6 +4,7 @@
import logging as log
from typing import Dict
+from collections import OrderedDict
from .lib import *
@@ -39,7 +40,7 @@
return result
-def elab_intermodule(topcfg):
+def elab_intermodule(topcfg: OrderedDict):
"""Check the connection of inter-module and categorize them
In the top template, it uses updated inter_module fields to create
@@ -51,7 +52,7 @@
list_of_intersignals = []
if "inter_signal" not in topcfg:
- topcfg["inter_signal"] = {}
+ topcfg["inter_signal"] = OrderedDict()
# Gather the inter_signal_list
instances = topcfg["module"] + topcfg["memory"]
@@ -105,12 +106,11 @@
assert "package" in rsp_struct, "Either req/rsp shall have 'package' field"
package = rsp_struct["package"]
- definitions.append({
- 'package': package,
- 'struct': req_struct["struct"],
- 'signame': sig_name,
- 'type': req_struct["type"]
- })
+ definitions.append(
+ OrderedDict([('package', package),
+ ('struct', req_struct["struct"]),
+ ('signame', sig_name),
+ ('type', req_struct["type"])]))
if rsp_len != 1:
log.warning("{req}[{i}] -> {rsp}".format(req=req, i=i,
diff --git a/util/topgen/lib.py b/util/topgen/lib.py
index e5ed163..598482e 100644
--- a/util/topgen/lib.py
+++ b/util/topgen/lib.py
@@ -5,6 +5,7 @@
import logging as log
from copy import deepcopy
from pathlib import Path
+from collections import OrderedDict
import hjson
import re
@@ -49,7 +50,11 @@
"""
p = xbar_path.glob('*.hjson')
try:
- xbar_objs = [hjson.load(x.open('r'), use_decimal=True) for x in p]
+ xbar_objs = [
+ hjson.load(x.open('r'),
+ use_decimal=True,
+ object_pairs_hook=OrderedDict) for x in p
+ ]
except ValueError:
raise Systemexit(sys.exc_info()[1])
@@ -160,7 +165,7 @@
width = first - last + 1
for p in range(first, last + 1):
- pads.append({"name": pad, "index": p})
+ pads.append(OrderedDict([("name", pad), ("index", p)]))
return pads
diff --git a/util/topgen/merge.py b/util/topgen/merge.py
index 62e96ec..0c3d01e 100644
--- a/util/topgen/merge.py
+++ b/util/topgen/merge.py
@@ -5,6 +5,7 @@
import logging as log
from copy import deepcopy
from functools import partial
+from collections import OrderedDict
from .lib import *
from .intermodule import elab_intermodule
@@ -245,10 +246,10 @@
"clock": xbar['clock'],
"reset": xbar['reset'],
"inst_type": predefined_modules["debug_mem"],
- "addr_range": [{
- "base_addr": top["debug_mem_base_addr"],
- "size_byte": "0x1000",
- }],
+ "addr_range": [OrderedDict([
+ ("base_addr", top["debug_mem_base_addr"]),
+ ("size_byte", "0x1000"),
+ ])],
"xbar": False,
"pipeline" : "true",
"pipeline_byp" : "true"
@@ -257,12 +258,10 @@
# Update if exists
node = nodeobj[0]
node["inst_type"] = predefined_modules["debug_mem"]
- node["addr_range"] = [{
- "base_addr":
- top["debug_mem_base_addr"],
- "size_byte":
- "0x1000"
- }]
+ node["addr_range"] = [
+ OrderedDict([("base_addr", top["debug_mem_base_addr"]),
+ ("size_byte", "0x1000")])
+ ]
node["xbar"] = False
process_pipeline_var(node)
else:
@@ -285,8 +284,8 @@
"clock" : deviceobj[0]["clock"],
"reset" : deviceobj[0]["reset"],
"inst_type" : deviceobj[0]["type"],
- "addr_range": [{"base_addr" : deviceobj[0]["base_addr"],
- "size_byte": deviceobj[0]["size"]}],
+ "addr_range": [OrderedDict([("base_addr", deviceobj[0]["base_addr"]),
+ ("size_byte", deviceobj[0]["size"])])],
"pipeline" : "true",
"pipeline_byp" : "true",
"xbar" : True if device in xbar_list else False
@@ -296,10 +295,10 @@
# found and exist in the nodes too
node = nodeobj[0]
node["inst_type"] = deviceobj[0]["type"]
- node["addr_range"] = [{
- "base_addr": deviceobj[0]["base_addr"],
- "size_byte": deviceobj[0]["size"]
- }]
+ node["addr_range"] = [
+ OrderedDict([("base_addr", deviceobj[0]["base_addr"]),
+ ("size_byte", deviceobj[0]["size"])])
+ ]
node["xbar"] = True if device in xbar_list else False
process_pipeline_var(node)
@@ -580,8 +579,9 @@
format(e))
-def merge_top(topcfg, ipobjs, xbarobjs):
- gencfg = deepcopy(topcfg)
+def merge_top(topcfg: OrderedDict, ipobjs: OrderedDict,
+ xbarobjs: OrderedDict) -> OrderedDict:
+ gencfg = topcfg
# Combine ip cfg into topcfg
for ip in ipobjs:
diff --git a/util/topgen/validate.py b/util/topgen/validate.py
index 2e1f447..b9947a0 100644
--- a/util/topgen/validate.py
+++ b/util/topgen/validate.py
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import logging as log
from enum import Enum
+from collections import OrderedDict
from reggen.validate import check_keys, val_types
@@ -50,7 +51,8 @@
top_optional = {
'interrupt_modules': ['l', 'list of the modules that connects to rv_plic'],
'interrupt': ['lnw', 'interrupts (generated)'],
- 'alert_modules': ['l', 'list of the modules that connects to alert_handler'],
+ 'alert_modules':
+ ['l', 'list of the modules that connects to alert_handler'],
'alert': ['lnw', 'alerts (generated)'],
'alert_async': ['l', 'async alerts (generated)'],
'pinmux': ['g', 'pinmux definition if doesn\'t exist, tool uses defaults'],
@@ -111,7 +113,7 @@
# If it does, return a dictionary of instance names to index in ip/xbarobjs
def check_target(top, objs, tgtobj):
error = 0
- idxs = {}
+ idxs = OrderedDict()
for i in range(len(objs)):
log.info("%d Order is %s" % (i, objs[i]['name'].lower()))
@@ -327,7 +329,7 @@
if not "pinmux" in top:
log.warning("Top {} has no 'pinmux' field. Please consider specifying \
pinmux and pads configuration")
- top["pinmux"] = {}
+ top["pinmux"] = OrderedDict()
# checking pinmux after pads as dio connects to PAD
error += check_pinmux(top, component)
