[otbn] Allow concurrent runs of otbn_build.py on the same files
Because the Meson-based build system doesn't really allow multiple
toolchains, we use a hack to build OTBN code through "custom
commands". Doing this, in turn, means that we have to assemble and
link every assembly file that goes into an ELF in a single script.
If two different ELF files use the same assembly file, this means
we've got two different runs of the script that are both trying to
create the relevant object file at the same time, which causes
confusing errors.
This patch should fix the errors by assembling the objects to
temporary files and then (atomically) renaming them to put them in the
right place. Obviously, it would be nice to teach the build system to
do this properly, but that looks like it's a bigger job, so we're
applying the hacky fix for now.
Signed-off-by: Rupert Swarbrick <rswarbrick@lowrisc.org>
diff --git a/util/otbn_build.py b/util/otbn_build.py
index 0791aac..299acd8 100755
--- a/util/otbn_build.py
+++ b/util/otbn_build.py
@@ -40,6 +40,7 @@
import shlex
import subprocess
import sys
+import tempfile
from pathlib import Path
from typing import List, Optional
@@ -47,36 +48,78 @@
REPO_TOP = Path(__file__).parent.parent.resolve()
-def cmd_to_str(cmd):
+
+def cmd_to_str(cmd: List[str]) -> str:
return ' '.join([shlex.quote(str(a)) for a in cmd])
-def run_cmd(cmd, **kwargs):
- log.info(cmd_to_str(cmd))
- subprocess.run(cmd, **kwargs)
+def run_cmd(args, display_cmd=None):
+ '''Run the command in args.
+
+ If display_cmd is not None, it should be a string that is printed instead
+ of the actual arguments that ran (for hiding the details of temporary
+ files).
+
+ '''
+ str_args = [str(a) for a in args]
+ info_msg = cmd_to_str(str_args) if display_cmd is None else display_cmd
+ log.info(info_msg)
+
+ subprocess.run(str_args, check=True)
+
+
+def run_tool(tool: str, out_file: Path, args) -> None:
+ '''Run tool to produce out_file (using an '-o' argument)
+
+ This works by writing to a temporary file (in the same directory) and then
+ atomically replacing any existing destination file when done. This is
+ needed if we need to run multiple otbn_build processes that generate the
+ same files in parallel (a requirement because of our current Meson-based
+ infrastructure).
+
+ '''
+ out_dir, out_base = os.path.split(out_file)
+ tmpfile = tempfile.NamedTemporaryFile(prefix=out_base, dir=out_dir,
+ delete=False)
+ try:
+ run_cmd([tool, '-o', tmpfile.name] + args,
+ cmd_to_str([tool, '-o', out_file] + args))
+
+ # If we get here, the tool ran successfully, producing the output file.
+ # Use os.replace to rename appropriately.
+ os.replace(tmpfile.name, out_file)
+ finally:
+ # When we're done, or if something went wrong, close and try to delete
+ # the temporary file. The unlink should fail if the os.replace call
+ # above succeeded. That's fine.
+ tmpfile.close()
+ try:
+ os.unlink(tmpfile.name)
+ except FileNotFoundError:
+ pass
def call_otbn_as(src_file: Path, out_file: Path):
otbn_as_cmd = os.environ.get('OTBN_AS',
str(REPO_TOP / 'hw/ip/otbn/util/otbn-as'))
- run_cmd([otbn_as_cmd, '-o', out_file, src_file], check=True)
+ run_tool(otbn_as_cmd, out_file, [src_file])
def call_otbn_ld(src_files: List[Path], out_file: Path, linker_script: Optional[Path]):
otbn_ld_cmd = os.environ.get('OTBN_LD',
str(REPO_TOP / 'hw/ip/otbn/util/otbn-ld'))
- cmd = [otbn_ld_cmd, '-o', out_file]
+ args = []
if linker_script:
- cmd += ['-T', linker_script]
- cmd += src_files
- run_cmd(cmd, check=True)
+ args += ['-T', linker_script]
+ args += src_files
+ run_tool(otbn_ld_cmd, out_file, args)
def call_rv32_objcopy(args: List[str]):
rv32_tool_objcopy = os.environ.get('RV32_TOOL_OBJCOPY',
'riscv32-unknown-elf-objcopy')
- run_cmd([rv32_tool_objcopy] + args, check=True)
+ run_cmd([rv32_tool_objcopy] + args)
def main() -> int:
