[dvsim] Logic to copy repo to scratch area
This change adds logic to copy the repo to the scratch area using
`rsync` command. The copy is triggered on the presence of two things -
- DVSIM_REMOTE_DISPATCH env variable defined
- `scratch_root` not being tool-default
If these conditions are met, it is assumed that a different scratch
area is being used, which is accessible to the user's workspace /
workstation as well as to the remote dispatch system. It is also assumed
that the remote dispatch system cannot access the user's workspace,
which is why the copying is needed. If the repo is copied to the scratch
area, the `proj_root` is set to the destination location, so that in
HJson files, everything is referenced from the destination as opposed
to the actual project root.
The destination is chosen to be `{scratch_root}/{branch}/repo_top`. This
is common to all toolflows run from the same branch.
The other change is removal of `tool_srcs_dir` and `tool_srcs`, and
addition of `proj_root` as a mandatory variable set for the Makefile.
All sources that the EDA tools need to have access to can now be
referenced from `{proj_root}` in the HJson files.
Lastly, a `--local` switch is added to dvsim to prevent triggering the
repo copy. It is user's responsibility to ensure that with `--local`,
DVSim is invoked such that the EDA tools are invoked from user's
workstation.
In the `common_project_cfg.hjson`, the changes reflect the above
updates.
Signed-off-by: Srikrishna Iyer <sriyer@google.com>
[dvsim, dv] Removal of `tool_srcs*`
This changes removes `tool_srcs*` from HJson and other files and
references all dependent sources from `{proj_root}` instead. Commit
b7ed20676 is required to support this change.
Wherever applicable, `{dv_root}` is used in lieu of `{proj_root}` to
stay opentitan-independent.
Signed-off-by: Srikrishna Iyer <sriyer@google.com>
[dvsim, fpv] Removal of `tool_srcs*`
This changes removes `tool_srcs*` from HJson and other files and
references all dependent sources from `{proj_root}` instead. Commit
b7ed20676 is required to support this change.
The second change is addition of `fpv_root`, which is where things that
could be common across OT and other projects can be found. All
references to `{proj_root}/hw/formal` are replaced with `{fpv_root}`.
Signed-off-by: Srikrishna Iyer <sriyer@google.com>
[dvsim, lint] Removal of `tool_srcs*`
This changes removes `tool_srcs*` from HJson and other files and
references all dependent sources from `{proj_root}` instead. Commit
b7ed20676 is required to support this change.
The second change is addition of `{lint_root}`, which represents a
location where all lint sources common across projects may reside. All
references to `{proj_root}/hw/lint` are replaced with `{lint_root}`.
Signed-off-by: Srikrishna Iyer <sriyer@google.com>
[dvsim, syn] Removal of `tool_srcs*`
This changes removes `tool_srcs*` from HJson and other files and
references all dependent sources from `{proj_root}` instead. Commit
b7ed20676 is required to support this change.
The second change is the addion of `{syn_root}` where all common
synthesis sources common across projects may reside. All isntances of
`{proj_root}/hw/syn` are replaced with `{syn_root}` as applicable.
Signed-off-by: Srikrishna Iyer <sriyer@google.com>
diff --git a/util/dvsim/Deploy.py b/util/dvsim/Deploy.py
index 199d5a8..e43d854 100644
--- a/util/dvsim/Deploy.py
+++ b/util/dvsim/Deploy.py
@@ -654,8 +654,7 @@
self.mandatory_cmd_attrs.update({
# tool srcs
- "tool_srcs": False,
- "tool_srcs_dir": False,
+ "proj_root": False,
# Flist gen
"sv_flist_gen_cmd": False,
@@ -717,8 +716,7 @@
self.mandatory_cmd_attrs.update({
# tool srcs
- "tool_srcs": False,
- "tool_srcs_dir": False,
+ "proj_root": False,
# Flist gen
"sv_flist_gen_cmd": False,
@@ -727,8 +725,10 @@
# Build
"build_dir": False,
+ "pre_build_cmds": False,
"build_cmd": False,
"build_opts": False,
+ "post_build_cmds": False,
"build_log": False,
# Report processing
@@ -775,8 +775,6 @@
self.mandatory_cmd_attrs.update({
# tool srcs
- "tool_srcs": False,
- "tool_srcs_dir": False,
"proj_root": False,
"uvm_test": False,
"uvm_test_seq": False,
@@ -866,8 +864,7 @@
self.target = "cov_unr"
self.mandatory_cmd_attrs.update({
# tool srcs
- "tool_srcs": False,
- "tool_srcs_dir": False,
+ "proj_root": False,
# Need to generate filelist based on build mode
"sv_flist_gen_cmd": False,
@@ -1066,8 +1063,7 @@
self.mandatory_cmd_attrs.update({
# tool srcs
- "tool_srcs": False,
- "tool_srcs_dir": False,
+ "proj_root": False,
"cov_analyze_cmd": False,
"cov_analyze_opts": False
})
diff --git a/util/dvsim/dvsim.py b/util/dvsim/dvsim.py
index 801e427..99c7247 100755
--- a/util/dvsim/dvsim.py
+++ b/util/dvsim/dvsim.py
@@ -28,13 +28,16 @@
import textwrap
from signal import SIGINT, signal
-from CfgFactory import make_cfg
import Deploy
import utils
+from CfgFactory import make_cfg
# TODO: add dvsim_cfg.hjson to retrieve this info
version = 0.1
+# By default, all build and run artifacts go here.
+DEFAULT_SCRATCH_ROOT = os.getcwd() + "/scratch"
+
# The different categories that can be passed to the --list argument.
_LIST_CATEGORIES = ["build_modes", "run_modes", "tests", "regressions"]
@@ -48,7 +51,7 @@
scratch_root = os.environ.get('SCRATCH_ROOT')
if not arg_scratch_root:
if scratch_root is None:
- arg_scratch_root = os.getcwd() + "/scratch"
+ arg_scratch_root = DEFAULT_SCRATCH_ROOT
else:
# Scratch space could be mounted in a filesystem (such as NFS) on a network drive.
# If the network is down, it could cause the access access check to hang. So run a
@@ -60,7 +63,7 @@
if status == 0 and out != "":
arg_scratch_root = scratch_root
else:
- arg_scratch_root = os.getcwd() + "/scratch"
+ arg_scratch_root = DEFAULT_SCRATCH_ROOT
log.warning(
"Env variable $SCRATCH_ROOT=\"{}\" is not accessible.\n"
"Using \"{}\" instead.".format(scratch_root,
@@ -150,6 +153,29 @@
return (proj_root)
+def resolve_proj_root(args):
+ '''Update proj_root based on how DVSim is invoked.
+
+ If --remote env var is set, a location in the scratch area is chosen as the
+ new proj_root. The entire repo is copied over to this location. Else, the
+ proj_root is discovered using get_proj_root() method, unless the user
+ overrides it on the command line.
+
+ This function returns the updated proj_root path.
+ '''
+ proj_root = args.proj_root or get_proj_root()
+
+ # Check if jobs are dispatched to external compute machines. If yes,
+ # then the repo needs to be copied over to the scratch area
+ # accessible to those machines.
+ if args.remote:
+ dest = os.path.join(args.scratch_root, args.branch, "repo_top")
+ copy_repo(proj_root, dest, args.dry_run)
+ proj_root = dest
+
+ return proj_root
+
+
def sigint_handler(signal_received, frame):
# Kill processes and background jobs.
log.debug('SIGINT or CTRL-C detected. Exiting gracefully')
@@ -158,6 +184,44 @@
exit(1)
+def copy_repo(src, dest, dry_run):
+ '''Copy over the repo to a new location.
+
+ The repo is copied over from src to dest area. It tentatively uses the
+ rsync utility which provides the ability to specify a file containing some
+ exclude patterns to skip certain things from being copied over. With GitHub
+ repos, an existing `.gitignore` serves this purpose pretty well.
+ '''
+ rsync_cmd = "rsync --archive --update --executability --inplace "
+
+ # Supply `.gitignore` from the src area to skip temp files.
+ ignore_patterns_file = os.path.join(src, ".gitignore")
+ if os.path.exists(ignore_patterns_file):
+ # TODO: hack - include hw/foundry since it is excluded in .gitignore.
+ rsync_cmd += "--include=hw/foundry "
+ rsync_cmd += "--exclude-from={} ".format(ignore_patterns_file)
+ rsync_cmd += "--exclude=*.git* "
+
+ rsync_cmd += src + "/. " + dest
+
+ cmd = ["flock", "--timeout", "600", dest, "--command", rsync_cmd]
+
+ log.info("[copy_repo] [dest]: %s", dest)
+ log.log(utils.VERBOSE, "[copy_repo] [cmd]: \n%s", ' '.join(cmd))
+ if not dry_run:
+ # Make sure the dest exists first.
+ os.makedirs(dest, exist_ok=True)
+ try:
+ proc = subprocess.run(cmd,
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ except subprocess.CalledProcessError as e:
+ log.error("Failed to copy over %s to %s: %s", src, dest,
+ e.stderr.decode("utf-8").strip())
+ log.info("Done.")
+
+
def wrapped_docstring():
'''Return a text-wrapped version of the module docstring'''
paras = []
@@ -236,6 +300,10 @@
help=('Prepend this string when running each tool '
'command.'))
+ disg.add_argument("--remote",
+ action='store_true',
+ help=('Trigger copying of the repo to scratch area.'))
+
disg.add_argument("--max-parallel",
"-mp",
type=read_max_parallel,
@@ -530,6 +598,8 @@
args.scratch_root = resolve_scratch_root(args.scratch_root)
args.branch = resolve_branch(args.branch)
args.cfg = os.path.abspath(args.cfg)
+ proj_root = resolve_proj_root(args)
+ log.info("[proj_root]: %s", proj_root)
# Add timestamp to args that all downstream objects can use.
# Static variables - indicate timestamp.
@@ -557,15 +627,6 @@
# Build infrastructure from hjson file and create the list of items to
# be deployed.
-
- # Sets the project root directory: either specified from the command line
- # or set by automatically assuming we are in a GitHub repository and
- # automatically finding the root of this repository.
- if args.proj_root:
- proj_root = args.proj_root
- else:
- proj_root = get_proj_root()
-
global cfg
cfg = make_cfg(args.cfg, args, proj_root)
diff --git a/util/tlgen/xbar.sim_cfg.hjson.tpl b/util/tlgen/xbar.sim_cfg.hjson.tpl
index 9fdd341..68566ad 100644
--- a/util/tlgen/xbar.sim_cfg.hjson.tpl
+++ b/util/tlgen/xbar.sim_cfg.hjson.tpl
@@ -12,8 +12,8 @@
// Testplan hjson file.
testplan: "{proj_root}/${xbar.ip_path}/data/autogen/{dut}_testplan.hjson"
- // Add excl files to tool_srcs so that it gets copied over to the scratch area.
- tool_srcs: ["{proj_root}/${xbar.ip_path}/dv/cov/xbar_cov_excl.el"]
+ // Add xbar_${xbar.name} specific exclusion files.
+ vcs_cov_excl_files: ["{proj_root}/${xbar.ip_path}/dv/cov/xbar_cov_excl.el"]
// Import additional common sim cfg files.
import_cfgs: [// xbar common sim cfg file
