experimental/web/testing/parse_test_list.py

#!/usr/bin/env python3

# Copyright 2022 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Parses a list of tests from ctest into a format for the web test runner.

Example usage:
  ctest --test-dir build-emscripten --show-only=json-v1 > /tmp/ctest.json
  python3 parse_test_list.py \
      --ctest_dump=/tmp/ctest.json \
      --build_dir=build-emscripten \
      --output_format=html \
      -o /tmp/parsed.html
"""

import argparse
import json
import os


def parse_arguments():
  """Parses command line arguments."""
  parser = argparse.ArgumentParser()
  parser.add_argument("--ctest_dump",
                      type=str,
                      required=True,
                      help="Path to the output of `ctest --show-only=json-v1`")
  parser.add_argument(
      "--build_dir",
      type=str,
      required=True,
      help="Path to the CMake build directory (absolute or relative)")
  parser.add_argument(
      "--output_format",
      type=str,
      choices=("html", "json"),
      default="html",
      help=
      "Output format, either 'html' for the test runner or 'json' for a list of JSON objects",
  )
  parser.add_argument("-o",
                      "--output",
                      type=str,
                      required=True,
                      help="Output file path")
  return parser.parse_args()


def get_normalized_relative_path(absolute_path, root_directory):
  # Strip the root directory prefix and get a relative path.
  relative_path = os.path.relpath(absolute_path, root_directory)
  # Replace the path separator (such as '\' on Windows) with web-style '/'.
  normalized_path = relative_path.replace(os.sep, '/')
  return normalized_path


def parse_ctest_dump(ctest_dump_path, build_dir):
  parsed_tests = []

  # Open the ctest dump JSON file and parse each test.
  # https://cmake.org/cmake/help/latest/manual/ctest.1.html#show-as-json-object-model
  with open(ctest_dump_path, "rt") as f:
    data = json.load(f)
    for test in data["tests"]:
      parsed_test = {
          "testName": test["name"],
          "requiredFiles": [],
          "args": [],
      }

      # Parse the 'command' list into the source file and its arguments.
      #   /path/to/test_runner.js  # such as iree-check-module.js or test.js
      #   arg 1                    # such as --device=local-task
      #   arg 2                    # such as check_vmvx_op.mlir_module.vmfb
      test_source_absolute_path = test["command"][0]
      parsed_test["sourceFile"] = get_normalized_relative_path(
          test_source_absolute_path, build_dir)

      parsed_test["args"] = test["command"][1:]

      # Parse the test "properties".
      # Note: required file paths are relative to the working directory.
      for property in test["properties"]:
        if property["name"] == "REQUIRED_FILES":
          parsed_test["requiredFiles"] = property["value"]
        elif property["name"] == "WORKING_DIRECTORY":
          working_directory_absolute_path = property["value"]
          parsed_test["workingDirectory"] = get_normalized_relative_path(
              working_directory_absolute_path, build_dir)

      parsed_tests.append(parsed_test)

  print("Parsed {} tests from '{}'".format(len(parsed_tests), ctest_dump_path))
  return parsed_tests


def print_parsed_tests(parsed_tests, output_path, output_format):
  with open(output_path, "wt") as f:
    if output_format == "html":
      print("Outputting parsed tests as HTML to '" + output_path + "'")
      for test in parsed_tests:
        f.write(
            "<li><a href=\"test-runner.html?testName={testName}&sourceFile={sourceFile}&workingDirectory={workingDirectory}&requiredFiles={requiredFiles}&args={args}\" target=testRunner>{testName}</a></li>\n"
            .format(testName=test["testName"],
                    sourceFile=test["sourceFile"],
                    workingDirectory=test["workingDirectory"],
                    requiredFiles="[" + ",".join(test["requiredFiles"]) + "]",
                    args="[" + ",".join(test["args"]) + "]"))
    elif output_format == "json":
      print("Outputting parsed tests as JSON to '" + output_path + "'")
      f.write(json.dumps(parsed_tests, indent=2))
    else:
      raise Exception("Unknown output format: '" + output_format + "'")


def main(args):
  # Refine the provided build directory path to a normalized, absolute path.
  build_dir = args.build_dir
  if not os.path.isabs(build_dir):
    build_dir = os.path.join(os.getcwd(), build_dir)
  build_dir = os.path.normpath(build_dir)

  # Create the output directory as needed (relative paths are fine here).
  output_dir = os.path.dirname(args.output)
  if output_dir and not os.path.isdir(output_dir):
    os.makedirs(output_dir)

  parsed_tests = parse_ctest_dump(args.ctest_dump, build_dir)
  parsed_tests.sort(key=lambda test: test["testName"])

  print_parsed_tests(parsed_tests, args.output, args.output_format)


if __name__ == "__main__":
  main(parse_arguments())