[otbn] Split up JSON dumping/loading in otbn-rig
This is a bit verbose, because we're properly parsing JSON data back
to ProgInsn objects, so have to do various sanity checks on the way to
make sure all the types match up.
The core of the patch is in the otbn-rig script, where we change the
command line to have two subcommands: 'gen' and 'asm'. Running 'gen'
will generate some JSON that describes the snippets in a random
instruction stream. Feeding that JSON to 'asm' generates assembly code
that can be assembled into a binary for testing.
This isn't quite so handy to use if you "just want to generate a darn
asm file!", but it's the shape we're going to use in the future and
running things in two stages like this will spot any silly mistakes in
the JSON parsing.
Here's an example of how to run it to generate a small example:
hw/ip/otbn/util/otbn-rig gen --size 10 -o tst.json
hw/ip/otbn/util/otbn-rig asm -o tst tst.json
This example can be assembled and linked with:
hw/ip/otbn/util/otbn-as tst.S -o tst.o
hw/ip/otbn/util/otbn-ld -o tst -T tst.ld tst.o
If you just want to see some assembly, the 'gen' phase will print its
JSON to stdout if not given an '-o' argument, the 'asm' phase will
read from stdin if not given a JSON file, and will print just the
assembly to stdout. So you can run:
hw/ip/otbn/util/otbn-rig gen --size 10 | \
hw/ip/otbn/util/otbn-rig asm
Of course, this doesn't generate a linker script, so this is only
useful for quick checks. Chaining like this does:
hw/ip/otbn/util/otbn-rig gen --size 10 | \
hw/ip/otbn/util/otbn-rig asm -o tst
(but you no longer get to keep the JSON intermediate file).
Signed-off-by: Rupert Swarbrick <rswarbrick@lowrisc.org>
diff --git a/hw/ip/otbn/util/otbn-rig b/hw/ip/otbn/util/otbn-rig
index 5c52d87..bd06bc5 100755
--- a/hw/ip/otbn/util/otbn-rig
+++ b/hw/ip/otbn/util/otbn-rig
@@ -10,42 +10,35 @@
import os
import random
import sys
+from typing import Optional, cast
-from shared.insn_yaml import load_file
+from shared.insn_yaml import InsnsFile, load_file
# Ensure that the OTBN utils directory is on sys.path. This means that RIG code
# can import modules like "shared.foo" and get the OTBN shared code.
sys.path.append(os.path.dirname(__file__))
-from rig.rig import gen_program # noqa: E402
+from rig.rig import gen_program, snippets_to_program # noqa: E402
+from rig.snippet import Snippet # noqa: E402
-def main() -> int:
- parser = argparse.ArgumentParser()
- parser.add_argument('--output', '-o',
- required=True,
- help='Path for JSON output of generated snippets')
- parser.add_argument('--asm-output',
- help='Optional path for generated program')
- parser.add_argument('--ld-output',
- help='Optional path for generated linker script')
- parser.add_argument('--seed', type=int, default=0,
- help='Random seed. Defaults to 0.')
- parser.add_argument('--size', type=int, default=100,
- help=('Max number of instructions in stream. '
- 'Defaults to 100.'))
- parser.add_argument('--start-addr', type=int, default=0,
- help='Reset address. Defaults to 0.')
-
- args = parser.parse_args()
- random.seed(args.seed)
-
+def get_insns_file() -> Optional[InsnsFile]:
+ '''Load up insns.yml'''
insns_yml = os.path.normpath(os.path.join(os.path.dirname(__file__),
'..', 'data', 'insns.yml'))
try:
- insns_file = load_file(insns_yml)
+ return load_file(insns_yml)
except RuntimeError as err:
- sys.stderr.write('{}\n'.format(err))
+ print(err, file=sys.stderr)
+ return None
+
+
+def gen_main(args: argparse.Namespace) -> int:
+ '''Entry point for the gen subcommand'''
+ random.seed(args.seed)
+
+ insns_file = get_insns_file()
+ if insns_file is None:
return 1
# Run the generator
@@ -54,35 +47,118 @@
# Write out the snippets to a JSON file
ser_snippets = [snippet.to_json() for snippet in snippets]
try:
- with open(args.output, 'w') as out_file:
- out_file.write(json.dumps(ser_snippets))
+ if args.output == '-':
+ json.dump(ser_snippets, sys.stdout)
+ # Add a newline at end of output: json.dump doesn't, and it makes a
+ # bit of a mess of some consoles.
+ sys.stdout.write('\n')
+ else:
+ with open(args.output, 'w') as out_file:
+ json.dump(ser_snippets, out_file)
except OSError as err:
- sys.stderr.write('Failed to open json output file {!r}: {}.'
- .format(args.output, err))
+ print('Failed to open json output file {!r}: {}.'
+ .format(args.output, err),
+ file=sys.stderr)
return 1
- # If assembly output was requested, dump that here.
- if args.asm_output is not None:
+ return 0
+
+
+def asm_main(args: argparse.Namespace) -> int:
+ '''Entry point for the asm subcommand'''
+
+ insns_file = get_insns_file()
+ if insns_file is None:
+ return 1
+
+ # Load snippets JSON
+ try:
+ snippets_json = json.load(args.snippets)
+ except json.JSONDecodeError as err:
+ print('Snippets file at {!r} is not valid JSON: {}.'
+ .format(args.snippets.name, err),
+ file=sys.stderr)
+ return 1
+
+ # Parse these to proper snippet objects and then make a program by
+ # combining them
+ try:
+ if not isinstance(snippets_json, list):
+ raise ValueError('Top-level structure should be a list.')
+ snippets = [Snippet.from_json(insns_file, idx, x)
+ for idx, x in enumerate(snippets_json)]
+ except ValueError as err:
+ print('Failed to parse snippets from {!r}: {}'
+ .format(args.snippets, err),
+ file=sys.stderr)
+ return 1
+
+ program = snippets_to_program(snippets)
+
+ # Dump the assembly output.
+ if args.output is None or args.output == '-':
+ program.dump_asm(sys.stdout)
+ else:
try:
- with open(args.asm_output, 'w') as out_file:
+ asm_path = args.output + '.S'
+ with open(asm_path, 'w') as out_file:
program.dump_asm(out_file)
except OSError as err:
- sys.stderr.write('Failed to open assembly output file {!r}: {}.'
- .format(args.asm_output, err))
+ print('Failed to open asm output file {!r}: {}.'
+ .format(args.output, err),
+ file=sys.stderr)
return 1
- # If a linker script was requested, dump that here
- if args.ld_output is not None:
try:
- with open(args.ld_output, 'w') as out_file:
+ ld_path = args.output + '.ld'
+ with open(ld_path, 'w') as out_file:
program.dump_linker_script(out_file)
except OSError as err:
- sys.stderr.write('Failed to open ld script output file {!r}: {}.'
- .format(args.ld_output, err))
+ print('Failed to open ld script output file {!r}: {}.'
+ .format(ld_path, err),
+ file=sys.stderr)
return 1
return 0
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers()
+
+ gen = subparsers.add_parser('gen', help='Generate a random program')
+ asm = subparsers.add_parser('asm', help='Convert snippets to assembly')
+
+ gen.add_argument('--output', '-o',
+ default='-',
+ help=("Path for JSON output of generated snippets. The "
+ "special path '-' (which is the default) means to "
+ "write to stdout"))
+ gen.add_argument('--seed', type=int, default=0,
+ help='Random seed. Defaults to 0.')
+ gen.add_argument('--size', type=int, default=100,
+ help=('Max number of instructions in stream. '
+ 'Defaults to 100.'))
+ gen.add_argument('--start-addr', type=int, default=0,
+ help='Reset address. Defaults to 0.')
+ gen.set_defaults(func=gen_main)
+
+ asm.add_argument('--output', '-o',
+ metavar='out',
+ help=('Base path for output filenames. Will generate '
+ 'out.S with an assembly listing and out.ld with '
+ 'a linker script. If this is not supplied, the '
+ 'assembly (but no linker script) will be dumped '
+ 'to stdout.'))
+ asm.add_argument('snippets', metavar='path.json',
+ type=argparse.FileType('r'), nargs='?', default=sys.stdin,
+ help=('A JSON file of snippets, as generated by '
+ 'otbn-rig gen.'))
+ asm.set_defaults(func=asm_main)
+
+ args = parser.parse_args()
+ return cast(int, args.func(args))
+
+
if __name__ == '__main__':
sys.exit(main())
diff --git a/hw/ip/otbn/util/rig/program.py b/hw/ip/otbn/util/rig/program.py
index 15db1b4..9994c85 100644
--- a/hw/ip/otbn/util/rig/program.py
+++ b/hw/ip/otbn/util/rig/program.py
@@ -5,7 +5,7 @@
import random
from typing import Dict, List, Optional, TextIO, Tuple
-from shared.insn_yaml import Insn
+from shared.insn_yaml import Insn, InsnsFile
class ProgInsn:
@@ -31,13 +31,124 @@
lsu_info: Optional[Tuple[str, int]]):
assert len(insn.operands) == len(operands)
assert (lsu_info is None) is (insn.lsu is None)
+ assert insn.syntax is not None
+
self.insn = insn
self.operands = operands
self.lsu_info = lsu_info
+ def to_asm(self) -> str:
+ '''Return an assembly representation of the instruction'''
+ insn = self.insn
+ # We should never try to generate an instruction without syntax
+ # (ensuring this is the job of the snippet generators)
+ assert insn.syntax is not None
+
+ # Build a dictionary from operand name to value from self.operands,
+ # which is a list of operand values in the same order as insn.operands.
+ op_vals = {}
+ assert len(insn.operands) == len(self.operands)
+ for operand, op_val in zip(insn.operands, self.operands):
+ op_vals[operand.name] = op_val
+
+ rendered_ops = insn.syntax.render_vals(op_vals, insn.name_to_operand)
+ if insn.glued_ops and rendered_ops:
+ mnem = insn.mnemonic + rendered_ops[0]
+ rendered_ops = rendered_ops[1:]
+ else:
+ mnem = insn.mnemonic
+
+ return '{:14}{}'.format(mnem, rendered_ops)
+
def to_json(self) -> object:
'''Serialize to an object that can be written as JSON'''
- return (self.insn.mnemonic, self.operands)
+ return (self.insn.mnemonic, self.operands, self.lsu_info)
+
+ @staticmethod
+ def from_json(insns_file: InsnsFile,
+ where: str,
+ json: object) -> 'ProgInsn':
+ '''The inverse of to_json.
+
+ where is a textual description of where we are in the file, used in
+ error messages.
+
+ '''
+ if not (isinstance(json, list) and len(json) == 3):
+ raise ValueError('{}, top-level data is not a triple.'
+ .format(where))
+
+ mnemonic, operands, json_lsu_info = json
+
+ if not isinstance(mnemonic, str):
+ raise ValueError('{}, mnemonic is {!r}, not a string.'
+ .format(where, mnemonic))
+
+ if not isinstance(operands, list):
+ raise ValueError('{}, operands are not represented by a list.'
+ .format(where))
+ op_vals = []
+ for op_idx, operand in enumerate(operands):
+ if not isinstance(operand, int):
+ raise ValueError('{}, operand {} is not an integer.'
+ .format(where, op_idx))
+ if operand < 0:
+ raise ValueError('{}, operand {} is {}, '
+ 'but should be non-negative.'
+ .format(where, op_idx, operand))
+ op_vals.append(operand)
+
+ lsu_info = None
+ if json_lsu_info is not None:
+ if not (isinstance(json_lsu_info, list) and
+ len(json_lsu_info) == 2):
+ raise ValueError('{}, non-None LSU info is not a pair.'
+ .format(where))
+ mem_type, addr = json_lsu_info
+
+ if not isinstance(mem_type, str):
+ raise ValueError('{}, LSU info mem_type is not a string.'
+ .format(where))
+ # These are the memory types in Model._known_regs, but we can't
+ # import that without a circular dependency. Rather than being
+ # clever, we'll just duplicate them for now.
+ if mem_type not in ['dmem', 'csr', 'wsr']:
+ raise ValueError('{}, invalid LSU mem_type: {!r}.'
+ .format(where, mem_type))
+
+ if not isinstance(addr, int):
+ raise ValueError('{}, LSU info target addr is not an integer.'
+ .format(where))
+ if addr < 0:
+ raise ValueError('{}, LSU info target addr is {}, '
+ 'but should be non-negative.'
+ .format(where, addr))
+
+ lsu_info = (mem_type, addr)
+
+ insn = insns_file.mnemonic_to_insn.get(mnemonic)
+ if insn is None:
+ raise ValueError('{}, unknown instruction {!r}.'
+ .format(where, mnemonic))
+
+ if (lsu_info is None) is not (insn.lsu is None):
+ raise ValueError('{}, LSU info is {}given, but the {} instruction '
+ '{} it.'
+ .format(where,
+ 'not ' if lsu_info is None else '',
+ mnemonic,
+ ("doesn't expect"
+ if insn.lsu is None else "expects")))
+ if len(insn.operands) != len(op_vals):
+ raise ValueError('{}, {} instruction has {} operands, but {} '
+ 'seen in JSON data.'
+ .format(where, mnemonic,
+ len(insn.operands), len(op_vals)))
+ if insn.syntax is None:
+ raise ValueError('{}, {} instruction has no syntax defined.'
+ .format(where, mnemonic))
+
+ return ProgInsn(insn, op_vals, lsu_info)
class OpenSection:
@@ -167,28 +278,7 @@
out_file.write('{}{}\n'.format('\n' if idx else '', comment))
out_file.write('.section .text.sec{:04}\n'.format(idx))
for pi in insns:
- insn = pi.insn
- # We should never try to generate an instruction without syntax
- # (ensuring this is the job of the snippet generators)
- assert insn.syntax is not None
-
- # Build a dictionary from operand name to value from
- # pi.operands, which is a list of operand values in the same
- # order as insn.operands.
- op_vals = {}
- assert len(pi.operands) == len(insn.operands)
- for operand, op_val in zip(insn.operands, pi.operands):
- op_vals[operand.name] = op_val
-
- rendered_ops = insn.syntax.render_vals(op_vals,
- insn.name_to_operand)
- if insn.glued_ops and rendered_ops:
- mnem = insn.mnemonic + rendered_ops[0]
- rendered_ops = rendered_ops[1:]
- else:
- mnem = insn.mnemonic
-
- out_file.write('{:14}{}\n'.format(mnem, rendered_ops))
+ out_file.write(pi.to_asm() + '\n')
def dump_linker_script(self, out_file: TextIO) -> None:
'''Write a linker script to link the program
diff --git a/hw/ip/otbn/util/rig/rig.py b/hw/ip/otbn/util/rig/rig.py
index aca1edf..5e94515 100644
--- a/hw/ip/otbn/util/rig/rig.py
+++ b/hw/ip/otbn/util/rig/rig.py
@@ -55,3 +55,18 @@
size = new_size
return snippets, program
+
+
+def snippets_to_program(snippets: List[Snippet]) -> Program:
+ '''Write a series of disjoint snippets to make a program'''
+ # Find the size of the memory that we can access. Both memories start
+ # at address 0: a strict Harvard architecture. (mems[x][0] is the LMA
+ # for memory x, not the VMA)
+ mems = get_memory_layout()
+ imem_lma, imem_size = mems['IMEM']
+ program = Program(imem_lma, imem_size)
+
+ for snippet in snippets:
+ snippet.insert_into_program(program)
+
+ return program
diff --git a/hw/ip/otbn/util/rig/snippet.py b/hw/ip/otbn/util/rig/snippet.py
index 258a6df..60f1c34 100644
--- a/hw/ip/otbn/util/rig/snippet.py
+++ b/hw/ip/otbn/util/rig/snippet.py
@@ -4,6 +4,8 @@
from typing import List, Tuple
+from shared.insn_yaml import InsnsFile
+
from .program import ProgInsn, Program
@@ -40,3 +42,57 @@
for addr, insns in self.parts:
lst.append((addr, [i.to_json() for i in insns]))
return lst
+
+ @staticmethod
+ def from_json(insns_file: InsnsFile,
+ idx: int,
+ json: object) -> 'Snippet':
+ '''The inverse of to_json.
+
+ idx is the 0-based number of the snippet in the file, just used for
+ error messages.
+
+ '''
+ if not isinstance(json, list):
+ raise ValueError('Object for snippet {} is not a list.'
+ .format(idx))
+
+ parts = []
+ for idx1, part in enumerate(json):
+ # Each element should be a pair: (addr, insns). This will have come
+ # out as a list (since tuples serialize as lists).
+ if not (isinstance(part, list) and len(part) == 2):
+ raise ValueError('Part {} for snippet {} is not a pair.'
+ .format(idx1, idx))
+
+ addr, insns_json = part
+
+ # The address should be an aligned non-negative integer and insns
+ # should itself be a list (of serialized Insn objects).
+ if not isinstance(addr, int):
+ raise ValueError('First coordinate of part {} for snippet {} '
+ 'is not an integer.'
+ .format(idx1, idx))
+ if addr < 0:
+ raise ValueError('Address of part {} for snippet {} is {}, '
+ 'but should be non-negative.'
+ .format(idx1, idx, addr))
+ if addr & 3:
+ raise ValueError('Address of part {} for snippet {} is {}, '
+ 'but should be 4-byte aligned.'
+ .format(idx1, idx, addr))
+
+ if not isinstance(insns_json, list):
+ raise ValueError('Second coordinate of part {} for snippet {} '
+ 'is not a list.'
+ .format(idx1, idx))
+
+ insns = []
+ for insn_idx, insn_json in enumerate(insns_json):
+ where = ('In snippet {}, part {}, instruction {}'
+ .format(idx, idx1, insn_idx))
+ insns.append(ProgInsn.from_json(insns_file, where, insn_json))
+
+ parts.append((addr, insns))
+
+ return Snippet(parts)
