Add sw/host/vendor/google_verible_verilog_syntax_py
Signed-off-by: Mariusz Glebocki <mglebocki@antmicro.com>
diff --git a/python-requirements.txt b/python-requirements.txt
index 38dca7e..4dedce5 100644
--- a/python-requirements.txt
+++ b/python-requirements.txt
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
# Keep sorted.
+anytree # dependency of sw/host/vendor/google_verible_verilog_syntax_py
enlighten
flake8
gitpython
diff --git a/sw/host/vendor/google_verible_verilog_syntax_py.lock.hjson b/sw/host/vendor/google_verible_verilog_syntax_py.lock.hjson
new file mode 100644
index 0000000..dc47172
--- /dev/null
+++ b/sw/host/vendor/google_verible_verilog_syntax_py.lock.hjson
@@ -0,0 +1,15 @@
+// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+
+// This file is generated by the util/vendor script. Please do not modify it
+// manually.
+
+{
+ upstream:
+ {
+ url: https://github.com/google/verible.git
+ rev: ed1f3a9577047b801854fb36dd14ebe0aecbdddc
+ only_subdir: verilog/tools/syntax/export_json_examples
+ }
+}
diff --git a/sw/host/vendor/google_verible_verilog_syntax_py.vendor.hjson b/sw/host/vendor/google_verible_verilog_syntax_py.vendor.hjson
new file mode 100644
index 0000000..0aeafca
--- /dev/null
+++ b/sw/host/vendor/google_verible_verilog_syntax_py.vendor.hjson
@@ -0,0 +1,13 @@
+// Copyright lowRISC contributors.
+// Licensed under the Apache License, Version 2.0, see LICENSE for details.
+// SPDX-License-Identifier: Apache-2.0
+{
+ name: "google_verible_verilog_syntax_py",
+ target_dir: "google_verible_verilog_syntax_py",
+
+ upstream: {
+ url: "https://github.com/google/verible.git",
+ rev: "v0.0-943-ged1f3a9",
+ only_subdir: "verilog/tools/syntax/export_json_examples",
+ },
+}
diff --git a/sw/host/vendor/google_verible_verilog_syntax_py/BUILD b/sw/host/vendor/google_verible_verilog_syntax_py/BUILD
new file mode 100644
index 0000000..85da887
--- /dev/null
+++ b/sw/host/vendor/google_verible_verilog_syntax_py/BUILD
@@ -0,0 +1,55 @@
+load("@rules_python//python:defs.bzl", "py_test", "py_library", "py_binary")
+
+licenses(["notice"])
+
+py_library(
+ name = "verible_verilog_syntax_py",
+ srcs = ["verible_verilog_syntax.py"],
+ srcs_version = "PY3",
+ imports = ["."],
+ data = ["//verilog/tools/syntax:verible-verilog-syntax"],
+ deps = [
+ "@python_anytree//:anytree",
+ "//third_party/py/dataclasses",
+ ],
+)
+
+py_test(
+ name = "verible_verilog_syntax_py_test",
+ size = "small",
+ srcs = ["verible_verilog_syntax_test.py"],
+ srcs_version = "PY3",
+ python_version = "PY3",
+ main = "verible_verilog_syntax_test.py",
+ deps = [":verible_verilog_syntax_py"],
+ data = ["//verilog/tools/syntax:verible-verilog-syntax"],
+ args = ["$(location //verilog/tools/syntax:verible-verilog-syntax)"],
+)
+
+py_binary(
+ name = "print_modules",
+ srcs = ["print_modules.py"],
+ srcs_version = "PY3",
+ python_version = "PY3",
+ main = "print_modules.py",
+ deps = [
+ ":verible_verilog_syntax_py",
+ "@python_anytree//:anytree",
+ ],
+ data = ["//verilog/tools/syntax:verible-verilog-syntax"],
+ args = ["$(location //verilog/tools/syntax:verible-verilog-syntax)"],
+)
+
+py_binary(
+ name = "print_tree",
+ srcs = ["print_tree.py"],
+ srcs_version = "PY3",
+ python_version = "PY3",
+ main = "print_tree.py",
+ deps = [
+ ":verible_verilog_syntax_py",
+ "@python_anytree//:anytree",
+ ],
+ data = ["//verilog/tools/syntax:verible-verilog-syntax"],
+ args = ["$(location //verilog/tools/syntax:verible-verilog-syntax)"],
+)
diff --git a/sw/host/vendor/google_verible_verilog_syntax_py/print_modules.py b/sw/host/vendor/google_verible_verilog_syntax_py/print_modules.py
new file mode 100755
index 0000000..2974e66
--- /dev/null
+++ b/sw/host/vendor/google_verible_verilog_syntax_py/print_modules.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# Copyright 2017-2020 The Verible Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Print module name, ports, parameters and imports.
+
+Usage: print_modules.py PATH_TO_VERIBLE_VERILOG_SYNTAX \\
+ VERILOG_FILE [VERILOG_FILE [...]]
+
+This example shows how to use ``verible-verilog-syntax --export_json ...``
+output to extract information about module declarations found in System Verilog
+source files. Extracted information:
+
+* module name
+* module port names
+* module parameter names
+* module imports
+* module header code
+"""
+
+import sys
+
+import anytree
+
+import verible_verilog_syntax
+
+
+def process_file_data(path: str, data: verible_verilog_syntax.SyntaxData):
+ """Print information about modules found in SystemVerilog file.
+
+ This function uses verible_verilog_syntax.Node methods to find module
+ declarations and specific tokens containing following information:
+
+ * module name
+ * module port names
+ * module parameter names
+ * module imports
+ * module header code
+
+ Args:
+ path: Path to source file (used only for informational purposes)
+ data: Parsing results returned by one of VeribleVerilogSyntax' parse_*
+ methods.
+ """
+ if not data.tree:
+ return
+
+ modules_info = []
+
+ # Collect information about each module declaration in the file
+ for module in data.tree.iter_find_all({"tag": "kModuleDeclaration"}):
+ module_info = {
+ "header_text": "",
+ "name": "",
+ "ports": [],
+ "parameters": [],
+ "imports": [],
+ }
+
+ # Find module header
+ header = module.find({"tag": "kModuleHeader"})
+ if not header:
+ continue
+ module_info["header_text"] = header.text
+
+ # Find module name
+ name = header.find({"tag": ["SymbolIdentifier", "EscapedIdentifier"]},
+ iter_=anytree.PreOrderIter)
+ if not name:
+ continue
+ module_info["name"] = name.text
+
+ # Get the list of ports
+ for port in header.iter_find_all({"tag": ["kPortDeclaration", "kPort"]}):
+ port_id = port.find({"tag": ["SymbolIdentifier", "EscapedIdentifier"]})
+ module_info["ports"].append(port_id.text)
+
+ # Get the list of parameters
+ for param in header.iter_find_all({"tag": ["kParamDeclaration"]}):
+ param_id = param.find({"tag": ["SymbolIdentifier", "EscapedIdentifier"]})
+ module_info["parameters"].append(param_id.text)
+
+ # Get the list of imports
+ for pkg in module.iter_find_all({"tag": ["kPackageImportItem"]}):
+ module_info["imports"].append(pkg.text)
+
+ modules_info.append(module_info)
+
+ # Print results
+ if len(modules_info) > 0:
+ print(f"\033[1;97;7m{path} \033[0m\n")
+
+ def print_entry(key, values):
+ fmt_values = [f"\033[92m{v}\033[0m" for v in values]
+ value_part = (f"\n\033[33m// {' '*len(key)}".join(fmt_values)
+ or "\033[90m-\033[0m")
+ print(f"\033[33m// \033[93m{key}{value_part}")
+
+ for module_info in modules_info:
+ print_entry("name: ", [module_info["name"]])
+ print_entry("ports: ", module_info["ports"])
+ print_entry("parameters: ", module_info["parameters"])
+ print_entry("imports: ", module_info["imports"])
+ print(f"\033[97m{module_info['header_text']}\033[0m\n")
+
+
+def main():
+ if len(sys.argv) < 3:
+ print(f"Usage: {sys.argv[0]} PATH_TO_VERIBLE_VERILOG_SYNTAX " +
+ "VERILOG_FILE [VERILOG_FILE [...]]")
+ return 1
+
+ parser_path = sys.argv[1]
+ files = sys.argv[2:]
+
+ parser = verible_verilog_syntax.VeribleVerilogSyntax(executable=parser_path)
+ data = parser.parse_files(files)
+
+ for file_path, file_data in data.items():
+ process_file_data(file_path, file_data)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/sw/host/vendor/google_verible_verilog_syntax_py/print_tree.py b/sw/host/vendor/google_verible_verilog_syntax_py/print_tree.py
new file mode 100755
index 0000000..057c6da
--- /dev/null
+++ b/sw/host/vendor/google_verible_verilog_syntax_py/print_tree.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# Copyright 2017-2020 The Verible Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Pretty-print Verilog Concrete Syntax Tree
+
+Usage: print_tree.py PATH_TO_VERIBLE_VERILOG_SYNTAX \\
+ VERILOG_FILE [VERILOG_FILE [...]]
+
+Visualizes tree generated by ``verible-verilog-syntax --export_json ...``.
+Values enclosed in ``[]`` are node tags. ``@(S-E)`` marks token's start (S)
+and end (E) byte offsets in source code. When a token's text in source code
+is not the same as its tag, the text is printed in single quotes.
+"""
+
+import sys
+
+import anytree
+
+import verible_verilog_syntax
+
+
+def process_file_data(path: str, data: verible_verilog_syntax.SyntaxData):
+ """Print tree representation to the console.
+
+ The function uses anytree module (which is a base of a tree implementation
+ used in verible_verilog_syntax) method to print syntax tree representation
+ to the console.
+
+ Args:
+ path: Path to source file (used only for informational purposes)
+ data: Parsing results returned by one of VeribleVerilogSyntax' parse_*
+ methods.
+ """
+ print(f"\033[1;97;7m{path} \033[0m\n")
+ if data.tree:
+ for prefix, _, node in anytree.RenderTree(data.tree):
+ print(f"\033[90m{prefix}\033[0m{node.to_formatted_string()}")
+ print()
+
+
+def main():
+ if len(sys.argv) < 3:
+ print(f"Usage: {sys.argv[0]} PATH_TO_VERIBLE_VERILOG_SYNTAX " +
+ "VERILOG_FILE [VERILOG_FILE [...]]")
+ return 1
+
+ parser_path = sys.argv[1]
+ files = sys.argv[2:]
+
+ parser = verible_verilog_syntax.VeribleVerilogSyntax(executable=parser_path)
+ data = parser.parse_files(files, options={"gen_tree": True})
+
+ for file_path, file_data in data.items():
+ process_file_data(file_path, file_data)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/sw/host/vendor/google_verible_verilog_syntax_py/verible_verilog_syntax.py b/sw/host/vendor/google_verible_verilog_syntax_py/verible_verilog_syntax.py
new file mode 100644
index 0000000..f7cb4f0
--- /dev/null
+++ b/sw/host/vendor/google_verible_verilog_syntax_py/verible_verilog_syntax.py
@@ -0,0 +1,531 @@
+# Copyright 2017-2020 The Verible Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Wrapper for ``verible-verilog-syntax --export_json``"""
+
+import collections
+import json
+import re
+import subprocess
+from typing import Any, Callable, Dict, Iterable, List, Optional, Union
+
+import anytree
+import dataclasses
+
+_CSI_SEQUENCE = re.compile("\033\\[.*?m")
+
+
+def _colorize(formats: List[str], strings: List[str]) -> str:
+ result = ""
+ fi = 0
+ for s in strings:
+ result += f"\033[{formats[fi]}m{s}\033[0m"
+ fi = (fi+1) % len(formats)
+ return result
+
+
+# Type aliases
+
+CallableFilter = Callable[["Node"], bool]
+KeyValueFilter = Dict[str, Union[str, List[str]]]
+TreeIterator = Union["_TreeIteratorBase", anytree.iterators.AbstractIter]
+
+
+# Custom tree iterators with an option for reverse children iteration
+
+class _TreeIteratorBase:
+ def __init__(self, tree: "Node",
+ filter_: Optional[CallableFilter] = None,
+ reverse_children: bool = False):
+ self.tree = tree
+ self.reverse_children = reverse_children
+ self.filter_ = filter_ if filter_ else lambda n: True
+
+ def __iter__(self) -> Iterable["Node"]:
+ yield from self._iter_tree(self.tree)
+
+ def _iter_children(self, tree: Optional["Node"]) -> Iterable["Node"]:
+ if not tree or not hasattr(tree, "children"):
+ return []
+ return tree.children if not self.reverse_children \
+ else reversed(tree.children)
+
+ def _iter_tree(self, tree: Optional["Node"]) -> Iterable["Node"]:
+ raise NotImplementedError("Subclass must implement '_iter_tree' method")
+
+
+class PreOrderTreeIterator(_TreeIteratorBase):
+ def _iter_tree(self, tree: Optional["Node"]) -> Iterable["Node"]:
+ if self.filter_(tree):
+ yield tree
+ for child in self._iter_children(tree):
+ yield from self._iter_tree(child)
+
+
+class PostOrderTreeIterator(_TreeIteratorBase):
+ def _iter_tree(self, tree: Optional["Node"]) -> Iterable["Node"]:
+ for child in self._iter_children(tree):
+ yield from self._iter_tree(child)
+ if self.filter_(tree):
+ yield tree
+
+
+class LevelOrderTreeIterator(_TreeIteratorBase):
+ def _iter_tree(self, tree: Optional["Node"]) -> Iterable["Node"]:
+ queue = collections.deque([tree])
+ while len(queue) > 0:
+ n = queue.popleft()
+ if self.filter_(n):
+ yield n
+ queue.extend(self._iter_children(n))
+
+
+class Node(anytree.NodeMixin):
+ """Base VeribleVerilogSyntax syntax tree node.
+
+ Attributes:
+ parent (Optional[Node]): Parent node.
+ """
+ def __init__(self, parent: Optional["Node"] = None):
+ self.parent = parent
+
+ @property
+ def syntax_data(self) -> Optional["SyntaxData"]:
+ """Parent SyntaxData"""
+ return self.parent.syntax_data if self.parent else None
+
+ @property
+ def start(self) -> Optional[int]:
+ """Byte offset of node's first character in source text"""
+ raise NotImplementedError("Subclass must implement 'start' property")
+
+ @property
+ def end(self) -> Optional[int]:
+ """Byte offset of a character just past the node in source text."""
+ raise NotImplementedError("Subclass must implement 'end' property")
+
+ @property
+ def text(self) -> str:
+ """Source code fragment spanning all tokens in a node."""
+ start = self.start
+ end = self.end
+ sd = self.syntax_data
+ if ((start is not None) and (end is not None) and sd and sd.source_code
+ and end <= len(sd.source_code)):
+ return sd.source_code[start:end].decode("utf-8")
+ return ""
+
+ def __repr__(self) -> str:
+ return _CSI_SEQUENCE.sub("", self.to_formatted_string())
+
+ def to_formatted_string(self) -> str:
+ """Print node representation formatted for printing in terminal."""
+ return super().__repr__()
+
+
+class BranchNode(Node):
+ """Syntax tree branch node
+
+ Attributes:
+ tag (str): Node tag.
+ children (Optional[Node]): Child nodes.
+ """
+ def __init__(self, tag: str, parent: Optional[Node] = None,
+ children: Optional[List[Node]] = None):
+ super().__init__(parent)
+ self.tag = tag
+ self.children = children if children is not None else []
+
+ @property
+ def start(self) -> Optional[int]:
+ first_token = self.find(lambda n: isinstance(n, TokenNode),
+ iter_=PostOrderTreeIterator)
+ return first_token.start if first_token else None
+
+ @property
+ def end(self) -> Optional[int]:
+ last_token = self.find(lambda n: isinstance(n, TokenNode),
+ iter_=PostOrderTreeIterator, reverse_children=True)
+ return last_token.end if last_token else None
+
+ def iter_find_all(self, filter_: Union[CallableFilter, KeyValueFilter, None],
+ max_count: int = 0,
+ iter_: TreeIterator = LevelOrderTreeIterator,
+ **kwargs) -> Iterable[Node]:
+ """Iterate all nodes matching specified filter.
+
+ Args:
+ filter_: Describes what to search for. Might be:
+ * Callable taking Node as an argument and returning True for accepted
+ nodes.
+ * Dict mapping Node attribute names to searched value or list of
+ searched values.
+ max_count: Stop searching after finding that many matching nodes.
+ iter_: Tree iterator. Decides in what order nodes are visited.
+
+ Yields:
+ Nodes matching specified filter.
+ """
+ def as_list(v):
+ return v if isinstance(v, list) else [v]
+
+ if filter_ and not callable(filter_):
+ filters = filter_
+ def f(node):
+ for attr,value in filters.items():
+ if not hasattr(node, attr):
+ return False
+ if getattr(node, attr) not in as_list(value):
+ return False
+ return True
+ filter_ = f
+
+ for node in iter_(self, filter_, **kwargs):
+ yield node
+ max_count -= 1
+ if max_count == 0:
+ break
+
+ def find(self, filter_: Union[CallableFilter, KeyValueFilter, None],
+ iter_: TreeIterator = LevelOrderTreeIterator, **kwargs) \
+ -> Optional[Node]:
+ """Find node matching specified filter.
+
+ Args:
+ filter_: Describes what to search for. Might be:
+ * Callable taking Node as an argument and returning True for accepted
+ node.
+ * Dict mapping Node attribute names to searched value or list of
+ searched values.
+ iter_: Tree iterator. Decides in what order nodes are visited.
+
+ Returns:
+ First Node matching filter.
+ """
+ return next(self.iter_find_all(filter_, max_count=1, iter_=iter_,
+ **kwargs), None)
+
+ def find_all(self, filter_: Union[CallableFilter, KeyValueFilter, None],
+ max_count: int = 0, iter_: TreeIterator = LevelOrderTreeIterator,
+ **kwargs) -> List[Node]:
+ """Find all nodes matching specified filter.
+
+ Args:
+ filter_: Describes what to search for. Might be:
+ * Callable taking Node as an argument and returning True for accepted
+ nodes.
+ * Dict mapping Node attribute names to searched value or list of
+ searched values.
+ max_count: Stop searching after finding that many matching nodes.
+ iter_: Tree iterator. Decides in what order nodes are visited.
+
+ Returns:
+ List of nodes matching specified filter.
+ """
+ return list(self.iter_find_all(filter_, max_count=max_count, iter_=iter_,
+ **kwargs))
+
+ def to_formatted_string(self) -> str:
+ tag = self.tag if self.tag == repr(self.tag)[1:-1] else repr(self.tag)
+ return _colorize(["37", "1;97"], ["[", tag, "]"])
+
+
+class RootNode(BranchNode):
+ """Syntax tree root node."""
+ def __init__(self, tag: str, syntax_data: Optional["SyntaxData"] = None,
+ children: Optional[List[Node]] = None):
+ super().__init__(tag, None, children)
+ self._syntax_data = syntax_data
+
+ @property
+ def syntax_data(self) -> Optional["SyntaxData"]:
+ return self._syntax_data
+
+
+class LeafNode(Node):
+ """Syntax tree leaf node.
+
+ This specific class is used for null nodes.
+ """
+ @property
+ def start(self) -> None:
+ """Byte offset of token's first character in source text"""
+ return None
+
+ @property
+ def end(self) -> None:
+ """Byte offset of a character just past the token in source text."""
+ return None
+
+ def to_formatted_string(self) -> str:
+ return _colorize(["90"], ["null"])
+
+
+class TokenNode(LeafNode):
+ """Tree node with token data
+
+ Represents single token in a syntax tree.
+
+ Attributes:
+ tag (str): Token tag.
+ """
+
+ def __init__(self, tag: str, start: int, end: int,
+ parent: Optional[Node] = None):
+ super().__init__(parent)
+ self.tag = tag
+ self._start = start
+ self._end = end
+
+ @property
+ def start(self) -> int:
+ return self._start
+
+ @property
+ def end(self) -> int:
+ return self._end
+
+ def to_formatted_string(self) -> str:
+ tag = self.tag if self.tag == repr(self.tag)[1:-1] else repr(self.tag)
+ parts = [
+ _colorize(["37", "1;97"], ["[", tag, "]"]),
+ _colorize(["33", "93"], ["@(", self.start, "-", self.end, ")"]),
+ ]
+ text = self.text
+ if self.tag != text:
+ parts.append(_colorize(["32", "92"], ["'", repr(text)[1:-1], "'"]))
+ return " ".join(parts)
+
+
+class Token:
+ """Token data
+
+ Represents single token in tokens and rawtokens lists.
+
+ Attributes:
+ tag (str): Token tag.
+ start (int): Byte offset of token's first character in source text.
+ end (int): Byte offset of a character just past the token in source text.
+ syntax_data (Optional["SyntaxData"]): Parent SyntaxData.
+ """
+
+ def __init__(self, tag: str, start: int, end: int,
+ syntax_data: Optional["SyntaxData"] = None):
+ self.tag = tag
+ self.start = start
+ self.end = end
+ self.syntax_data = syntax_data
+
+ @property
+ def text(self) -> str:
+ """Token text in source code."""
+ sd = self.syntax_data
+ if sd and sd.source_code and self.end <= len(sd.source_code):
+ return sd.source_code[self.start:self.end].decode("utf-8")
+ return ""
+
+ def __repr__(self) -> str:
+ return _CSI_SEQUENCE.sub("", self.to_formatted_string())
+
+ def to_formatted_string(self) -> str:
+ tag = self.tag if self.tag == repr(self.tag)[1:-1] else repr(self.tag)
+ parts = [
+ _colorize(["37", "1;97"], ["[", tag, "]"]),
+ _colorize(["33", "93"], ["@(", self.start, "-", self.end, ")"]),
+ _colorize(["32", "92"], ["'", repr(self.text)[1:-1], "'"]),
+ ]
+ return " ".join(parts)
+
+
+@dataclasses.dataclass
+class Error:
+ line: int
+ column: int
+ phase: str
+ message: str = ""
+
+
+@dataclasses.dataclass
+class SyntaxData:
+ source_code: Optional[str] = None
+ tree: Optional[RootNode] = None
+ tokens: Optional[List[Token]] = None
+ rawtokens: Optional[List[Token]] = None
+ errors: Optional[List[Error]] = None
+
+
+class VeribleVerilogSyntax:
+ """``verible-verilog-syntax`` wrapper.
+
+ This class provides methods for running ``verible-verilog-syntax`` and
+ transforming its output into Python data structures.
+
+ Args:
+ executable: path to ``verible-verilog-syntax`` binary.
+ """
+
+ def __init__(self, executable: str = "verible-verilog-syntax"):
+ self.executable = executable
+
+ @staticmethod
+ def _transform_tree(tree, data: SyntaxData, skip_null: bool) -> RootNode:
+ def transform(tree):
+ if tree is None:
+ return None
+ if "children" in tree:
+ children = [
+ transform(child) or LeafNode()
+ for child in tree["children"]
+ if not (skip_null and child is None)
+ ]
+ tag = tree["tag"]
+ return BranchNode(tag, children=children)
+ tag = tree["tag"]
+ start = tree["start"]
+ end = tree["end"]
+ return TokenNode(tag, start, end)
+
+ if "children" not in tree:
+ return None
+
+ children = [
+ transform(child) or LeafNode()
+ for child in tree["children"]
+ if not (skip_null and child is None)
+ ]
+ tag = tree["tag"]
+ return RootNode(tag, syntax_data=data, children=children)
+
+
+ @staticmethod
+ def _transform_tokens(tokens, data: SyntaxData) -> List[Token]:
+ return [Token(t["tag"], t["start"], t["end"], data) for t in tokens]
+
+
+ @staticmethod
+ def _transform_errors(tokens) -> List[Error]:
+ return [Error(t["line"], t["column"], t["phase"], t.get("message", None))
+ for t in tokens]
+
+ def _parse(self, paths: List[str], input_: str = None,
+ options: Dict[str, Any] = None) -> Dict[str, SyntaxData]:
+ """Common implementation of parse_* methods"""
+ options = {
+ "gen_tree": True,
+ "skip_null": False,
+ "gen_tokens": False,
+ "gen_rawtokens": False,
+ **(options or {}),
+ }
+
+ args = ["-export_json"]
+ if options["gen_tree"]:
+ args.append("-printtree")
+ if options["gen_tokens"]:
+ args.append("-printtokens")
+ if options["gen_rawtokens"]:
+ args.append("-printrawtokens")
+
+ proc = subprocess.run([self.executable, *args , *paths],
+ stdout=subprocess.PIPE,
+ input=input_,
+ encoding="utf-8",
+ check=False)
+
+ json_data = json.loads(proc.stdout)
+ data = {}
+ for file_path, file_json in json_data.items():
+ file_data = SyntaxData()
+
+ if file_path == "-":
+ file_data.source_code = input_.encode("utf-8")
+ else:
+ with open(file_path, "rb") as f:
+ file_data.source_code = f.read()
+
+ if "tree" in file_json:
+ file_data.tree = VeribleVerilogSyntax._transform_tree(
+ file_json["tree"], file_data, options["skip_null"])
+
+ if "tokens" in file_json:
+ file_data.tokens = VeribleVerilogSyntax._transform_tokens(
+ file_json["tokens"], file_data)
+
+ if "rawtokens" in file_json:
+ file_data.rawtokens = VeribleVerilogSyntax._transform_tokens(
+ file_json["rawtokens"], file_data)
+
+ if "errors" in file_json:
+ file_data.errors = VeribleVerilogSyntax._transform_errors(
+ file_json["errors"])
+
+ data[file_path] = file_data
+
+ return data
+
+ def parse_files(self, paths: List[str], options: Dict[str, Any] = None) \
+ -> Dict[str, SyntaxData]:
+ """Parse multiple SystemVerilog files.
+
+ Args:
+ paths: list of paths to files to parse.
+ options: dict with parsing options.
+ Available options:
+ gen_tree (boolean): whether to generate syntax tree.
+ skip_null (boolean): null nodes won't be stored in a tree if True.
+ gen_tokens (boolean): whether to generate tokens list.
+ gen_rawtokens (boolean): whether to generate raw token list.
+ By default only ``gen_tree`` is True.
+
+ Returns:
+ A dict that maps file names to their parsing results in SyntaxData object.
+ """
+ return self._parse(paths, options = options)
+
+ def parse_file(self, path: str, options: Dict[str, Any] = None) \
+ -> Optional[SyntaxData]:
+ """Parse single SystemVerilog file.
+
+ Args:
+ path: path to a file to parse.
+ options: dict with parsing options.
+ Available options:
+ gen_tree (boolean): whether to generate syntax tree.
+ skip_null (boolean): null nodes won't be stored in a tree if True.
+ gen_tokens (boolean): whether to generate tokens list.
+ gen_rawtokens (boolean): whether to generate raw token list.
+ By default only ``gen_tree`` is True.
+
+ Returns:
+ Parsing results in SyntaxData object.
+ """
+ return self._parse([path], options = options).get(path, None)
+
+ def parse_string(self, string: str, options: Dict[str, Any] = None) \
+ -> Optional[SyntaxData]:
+ """Parse a string with SystemVerilog code.
+
+ Args:
+ string: SystemVerilog code to parse.
+ options: dict with parsing options.
+ Available options:
+ gen_tree (boolean): whether to generate syntax tree.
+ skip_null (boolean): null nodes won't be stored in a tree if True.
+ gen_tokens (boolean): whether to generate tokens list.
+ gen_rawtokens (boolean): whether to generate raw token list.
+ By default only ``gen_tree`` is True.
+
+ Returns:
+ Parsing results in SyntaxData object.
+ """
+ return self._parse(["-"], input_=string, options=options).get("-", None)
diff --git a/sw/host/vendor/google_verible_verilog_syntax_py/verible_verilog_syntax_test.py b/sw/host/vendor/google_verible_verilog_syntax_py/verible_verilog_syntax_test.py
new file mode 100755
index 0000000..c183bee
--- /dev/null
+++ b/sw/host/vendor/google_verible_verilog_syntax_py/verible_verilog_syntax_test.py
@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+# Copyright 2017-2020 The Verible Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""VeribleVerilogSyntax test"""
+
+
+import sys
+import tempfile
+import unittest
+
+import verible_verilog_syntax
+
+
+class VeribleVerilogSyntaxTest(unittest.TestCase):
+ def setUp(self):
+ if len(sys.argv) > 1:
+ self.parser = verible_verilog_syntax.VeribleVerilogSyntax(
+ executable=sys.argv[1])
+ else:
+ self.parser = verible_verilog_syntax.VeribleVerilogSyntax()
+
+ self.assertIsNotNone(self.parser)
+
+ def create_temp_file(self, content):
+ tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".sv")
+ tmp.write(content)
+ tmp.flush()
+ return tmp
+
+
+class TestParseMethods(VeribleVerilogSyntaxTest):
+ def test_parse_string(self):
+ data = self.parser.parse_string("module X(); endmodule;")
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.tree)
+ self.assertIsNone(data.errors)
+
+ def test_parse_string_with_all_options(self):
+ data = self.parser.parse_string("module X(); endmodule;", options={
+ "gen_tree": True,
+ "gen_tokens": True,
+ "gen_rawtokens": True,
+ })
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.tree)
+ self.assertIsNotNone(data.tokens)
+ self.assertIsNotNone(data.rawtokens)
+ self.assertIsNone(data.errors)
+
+ def test_parse_string_error(self):
+ data = self.parser.parse_string("endmodule X(); module;")
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.errors)
+
+ def test_parse_file(self):
+ file_ = self.create_temp_file("module X(); endmodule;")
+ data = self.parser.parse_file(file_.name)
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.tree)
+ self.assertIsNone(data.errors)
+
+ def test_parse_file_with_all_options(self):
+ file_ = self.create_temp_file("module X(); endmodule;")
+ data = self.parser.parse_file(file_.name, options={
+ "gen_tree": True,
+ "gen_tokens": True,
+ "gen_rawtokens": True,
+ })
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.tree)
+ self.assertIsNotNone(data.tokens)
+ self.assertIsNotNone(data.rawtokens)
+ self.assertIsNone(data.errors)
+
+ def test_parse_file_error(self):
+ file_ = self.create_temp_file("endmodule X(); module;")
+ data = self.parser.parse_file(file_.name)
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.errors)
+
+ def test_parse_files(self):
+ files = [
+ self.create_temp_file("module X(); endmodule;"),
+ self.create_temp_file("module Y(); endmodule;"),
+ ]
+ data = self.parser.parse_files([f.name for f in files])
+ self.assertIsNotNone(data)
+ for f in files:
+ self.assertIsNotNone(data[f.name])
+ self.assertIsNotNone(data[f.name].tree)
+ self.assertIsNone(data[f.name].errors)
+
+ def test_parse_files_with_all_options(self):
+ files = [
+ self.create_temp_file("module X(); endmodule;"),
+ self.create_temp_file("module Y(); endmodule;"),
+ ]
+ data = self.parser.parse_files([f.name for f in files], options={
+ "gen_tree": True,
+ "gen_tokens": True,
+ "gen_rawtokens": True,
+ })
+ self.assertIsNotNone(data)
+ for f in files:
+ self.assertIsNotNone(data[f.name])
+ self.assertIsNotNone(data[f.name].tree)
+ self.assertIsNotNone(data[f.name].tokens)
+ self.assertIsNotNone(data[f.name].rawtokens)
+ self.assertIsNone(data[f.name].errors)
+
+ def test_parse_files_error(self):
+ # One file with, and one without errors
+ files = [
+ self.create_temp_file("endmodule X(); module;"),
+ self.create_temp_file("module Y(); endmodule;"),
+ ]
+ data = self.parser.parse_files([f.name for f in files])
+ self.assertIsNotNone(data)
+ for f in files:
+ self.assertIsNotNone(data[f.name])
+
+ self.assertIsNotNone(data[files[0].name].errors)
+ self.assertIsNone(data[files[1].name].errors)
+
+
+class TestTree(VeribleVerilogSyntaxTest):
+ def setUp(self):
+ super().setUp()
+ data = self.parser.parse_string(
+ "module ModuleName#(parameter PARAM_NAME=42)\n" +
+ " (input portIn, output portOut);\n" +
+ " import import_pkg_name::*;\n" +
+ " wire wireName;\n" +
+ "endmodule;\n" +
+ "module OtherModule; endmodule;\n")
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.tree)
+ self.tree = data.tree
+
+ def test_find(self):
+ header = self.tree.find({"tag": "kModuleHeader"})
+ self.assertIsNotNone(header)
+ module_name = header.find({"tag": "SymbolIdentifier"})
+ self.assertIsNotNone(module_name)
+ self.assertEqual(module_name.text, "ModuleName")
+ nonexistent = header.find({"tag": "SomeUndefinedTag"})
+ self.assertIsNone(nonexistent)
+
+ def test_find_all(self):
+ headers = self.tree.find_all({"tag": "kModuleHeader"})
+ self.assertEqual(len(headers), 2)
+
+ identifiers = self.tree.find_all({"tag": "SymbolIdentifier"})
+ self.assertEqual(len(identifiers), 7)
+
+ some_identifiers = self.tree.find_all({"tag": "SymbolIdentifier"},
+ max_count=4)
+ self.assertEqual(len(some_identifiers), 4)
+
+ def test_iter_find_all(self):
+ identifiers = [n.text
+ for n
+ in self.tree.iter_find_all({"tag": "SymbolIdentifier"})]
+ self.assertIn("ModuleName", identifiers)
+ self.assertIn("PARAM_NAME", identifiers)
+ self.assertIn("OtherModule", identifiers)
+
+ def test_custom_filter(self):
+ def tokens_past_135_byte(node):
+ return (isinstance(node, verible_verilog_syntax.TokenNode)
+ and node.start > 135)
+
+ other_module_tokens = self.tree.find_all(tokens_past_135_byte)
+ self.assertGreaterEqual(len(other_module_tokens), 5)
+ for token in other_module_tokens:
+ self.assertGreater(token.start, 135)
+
+ def test_search_order(self):
+ level_order = self.tree.find_all({"tag": "SymbolIdentifier"})
+ depth_order = self.tree.find_all({"tag": "SymbolIdentifier"},
+ iter_=verible_verilog_syntax.PreOrderTreeIterator)
+
+ def check_items_order(iterable, items_to_check):
+ index = 0
+ for item in iterable:
+ if items_to_check[index] == item:
+ index += 1
+ if index == len(items_to_check):
+ return True
+ return False
+
+ self.assertTrue(check_items_order([n.text for n in level_order],
+ ["ModuleName", "OtherModule", "portIn"]))
+ self.assertTrue(check_items_order([n.text for n in depth_order],
+ ["ModuleName", "portIn", "OtherModule"]))
+
+ def test_node_properties(self):
+ header = self.tree.find({"tag": "kModuleHeader"})
+ self.assertIsNotNone(header)
+
+ module_kw = header.find({"tag": "module"})
+ self.assertEqual(module_kw.text, "module")
+ self.assertEqual(module_kw.start, 0)
+ self.assertEqual(module_kw.end, 6)
+
+ semicolon = header.find({"tag": ";"})
+ self.assertEqual(semicolon.text, ";")
+ self.assertEqual(semicolon.start, 78)
+ self.assertEqual(semicolon.end, 79)
+
+ self.assertEqual(header.start, module_kw.start)
+ self.assertEqual(header.end, semicolon.end)
+ self.assertTrue(header.text.startswith("module"))
+ self.assertTrue(header.text.endswith(");"))
+
+
+class TestTokens(VeribleVerilogSyntaxTest):
+ def test_tokens(self):
+ data = self.parser.parse_string(
+ "module X(input portIn, output portOut); endmodule;", options={
+ "gen_tree": False,
+ "gen_tokens": True,
+ })
+
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.tokens)
+
+ identifiers = [t for t in data.tokens if t.tag == "SymbolIdentifier"]
+
+ module_name = identifiers[0]
+ self.assertEqual(module_name.text, "X")
+ self.assertEqual(module_name.start, 7)
+ self.assertEqual(module_name.end, 8)
+
+ texts = [t.text for t in identifiers]
+ self.assertSequenceEqual(texts, ["X", "portIn", "portOut"])
+
+
+ def test_rawtokens(self):
+ data = self.parser.parse_string(
+ "module X(input portIn, output portOut); endmodule;", options={
+ "gen_tree": False,
+ "gen_rawtokens": True,
+ })
+
+ self.assertIsNotNone(data)
+ self.assertIsNotNone(data.rawtokens)
+
+ identifiers = [t for t in data.rawtokens if t.tag == "SymbolIdentifier"]
+
+ module_name = identifiers[0]
+ self.assertEqual(module_name.text, "X")
+ self.assertEqual(module_name.start, 7)
+ self.assertEqual(module_name.end, 8)
+
+ texts = [t.text for t in identifiers]
+ self.assertSequenceEqual(texts, ["X", "portIn", "portOut"])
+
+
+if __name__ == "__main__":
+ unittest.main(argv=sys.argv[0:1], verbosity=2)
