)]}'
{
  "commit": "03aaf320a4b519469668dead50d12ef1c136fb3b",
  "tree": "3fe241e9efcd55cc0904d10422a318e2240a50f5",
  "parents": [
    "bb0b2768312ec4219805ef952ab85dae77348611"
  ],
  "author": {
    "name": "Philipp Wagner",
    "email": "phw@lowrisc.org",
    "time": "Wed Sep 09 19:11:44 2020 +0100"
  },
  "committer": {
    "name": "Philipp Wagner",
    "email": "mail@philipp-wagner.com",
    "time": "Tue Oct 06 14:56:47 2020 +0100"
  },
  "message": "[test] Refactor system tests to use pytest as runner\n\nSystem tests, living in `test/systemtest`, test the whole end-to-end\ndesign of OpenTitan: they initialize the target (e.g. a simulation, an\nFPGA, or later an ASIC), load test software as needed, and check for\noutputs or other expected behavior. To do so, multiple components need\nto work together. For example, a simulation process, components for data\ntransfer (such as OpenOCD), components which interact with (simulated or\nreal) external interfaces, etc. Pytest provides a very convenient way to\nhandle such situations through their fixtures. Our current system tests\nmake use of this functionality.\n\nWhat we haven\u0027t been using in the past is the ability of pytest to serve\nas test runner: select tests, collect their output and results, etc.\n(Instead, a shell script was used.)\n\nThis PR refactors the system tests to make use of pytest as test runner,\nwith one main consequence: Tests are now \"opinionated\", they don\u0027t take\nconfiguration parameters through command line arguments any more. Tests\nexpect build outputs in the distribution directory (`build-bin`) to be\nin the well-known layout for this directory.\n\nNotes:\n* This commit lays the groundwork to add tests running on an FPGA target\n  in a similar way as the verilated tests, which should explain some of\n  the structuring of the code.\n* The ability to run the debug \"compliance test\" built into OpenOCD,\n  which was present in the `test/systemtest/openocd_verilator_test.py`\n  file, has been removed temporarily. This test was unmaintained, and\n  will be brought back in revised form at a later time.\n* Equally, the \"FPGA test runner\" available in\n  `test/systemtest/functional_fpga_test.py` has been rarely used (as a\n  poll on Slack showed) and is also removed.\n  It will be brought back in modified form in a future commit.\n\nSigned-off-by: Philipp Wagner \u003cphw@lowrisc.org\u003e\n",
  "tree_diff": [
    {
      "type": "modify",
      "old_id": "3c611d32a23d8b0ce2fcc4d05c0f9c1dd577e8fc",
      "old_mode": 33188,
      "old_path": "azure-pipelines.yml",
      "new_id": "79233ce3d3520eb95a80606ba0117025312162c4",
      "new_mode": 33188,
      "new_path": "azure-pipelines.yml"
    },
    {
      "type": "delete",
      "old_id": "4ca02cc3496022aace84ade16f358fc0448d2bd2",
      "old_mode": 33261,
      "old_path": "ci/run_verilator_pytest.sh",
      "new_id": "0000000000000000000000000000000000000000",
      "new_mode": 0,
      "new_path": "/dev/null"
    },
    {
      "type": "modify",
      "old_id": "99a97923d608cd0aeed9f703e9efb1a6806ff949",
      "old_mode": 33188,
      "old_path": "python-requirements.txt",
      "new_id": "4e029495e62674038330f8176db0b3d1a652aa63",
      "new_mode": 33188,
      "new_path": "python-requirements.txt"
    },
    {
      "type": "add",
      "old_id": "0000000000000000000000000000000000000000",
      "old_mode": 0,
      "old_path": "/dev/null",
      "new_id": "71971ce71dca67aa48e507d61f929d2f59dfa864",
      "new_mode": 33188,
      "new_path": "test/.gitignore"
    },
    {
      "type": "delete",
      "old_id": "16d3c4dbbfec555a7690fe6a0f9f812de215fab2",
      "old_mode": 33188,
      "old_path": "test/systemtest/.gitignore",
      "new_id": "0000000000000000000000000000000000000000",
      "new_mode": 0,
      "new_path": "/dev/null"
    },
    {
      "type": "add",
      "old_id": "0000000000000000000000000000000000000000",
      "old_mode": 0,
      "old_path": "/dev/null",
      "new_id": "c3aaf165139a8404e8b1542a867ef4491bfb9afe",
      "new_mode": 33188,
      "new_path": "test/systemtest/README.md"
    },
    {
      "type": "add",
      "old_id": "0000000000000000000000000000000000000000",
      "old_mode": 0,
      "old_path": "/dev/null",
      "new_id": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
      "new_mode": 33188,
      "new_path": "test/systemtest/__init__.py"
    },
    {
      "type": "add",
      "old_id": "0000000000000000000000000000000000000000",
      "old_mode": 0,
      "old_path": "/dev/null",
      "new_id": "8e24218b5df9bba535487bb81d862413e799ec2a",
      "new_mode": 33188,
      "new_path": "test/systemtest/config.py"
    },
    {
      "type": "modify",
      "old_id": "117e3ed2894c11f902f3699a443d690b8947f31e",
      "old_mode": 33188,
      "old_path": "test/systemtest/conftest.py",
      "new_id": "2b6f2feab9a4e1ae88644ab9a7652b6e588994cc",
      "new_mode": 33188,
      "new_path": "test/systemtest/conftest.py"
    },
    {
      "type": "add",
      "old_id": "0000000000000000000000000000000000000000",
      "old_mode": 0,
      "old_path": "/dev/null",
      "new_id": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
      "new_mode": 33188,
      "new_path": "test/systemtest/earlgrey/__init__.py"
    },
    {
      "type": "add",
      "old_id": "0000000000000000000000000000000000000000",
      "old_mode": 0,
      "old_path": "/dev/null",
      "new_id": "f357e9a17ebe489d5f3a744110b4b695c4defaf9",
      "new_mode": 33188,
      "new_path": "test/systemtest/earlgrey/test_sim_verilator.py"
    },
    {
      "type": "delete",
      "old_id": "4de7377e6cc149ccea62cba02b1b9711e867e827",
      "old_mode": 33188,
      "old_path": "test/systemtest/functional_fpga_test.py",
      "new_id": "0000000000000000000000000000000000000000",
      "new_mode": 0,
      "new_path": "/dev/null"
    },
    {
      "type": "delete",
      "old_id": "4ae54d53f13068792ec15415368bf30f226df7bd",
      "old_mode": 33188,
      "old_path": "test/systemtest/functional_verilator_test.py",
      "new_id": "0000000000000000000000000000000000000000",
      "new_mode": 0,
      "new_path": "/dev/null"
    },
    {
      "type": "delete",
      "old_id": "a4e89fe2cbed014ef3d932b64143a52cf459e604",
      "old_mode": 33188,
      "old_path": "test/systemtest/openocd_verilator_test.py",
      "new_id": "0000000000000000000000000000000000000000",
      "new_mode": 0,
      "new_path": "/dev/null"
    },
    {
      "type": "delete",
      "old_id": "2868619be90fc393b70b6fa870264fa67ec5c8ac",
      "old_mode": 33188,
      "old_path": "test/systemtest/test_utils.py",
      "new_id": "0000000000000000000000000000000000000000",
      "new_mode": 0,
      "new_path": "/dev/null"
    },
    {
      "type": "add",
      "old_id": "0000000000000000000000000000000000000000",
      "old_mode": 0,
      "old_path": "/dev/null",
      "new_id": "cf15c431b406fa2a0e6d3e52f6a498806693a77a",
      "new_mode": 33188,
      "new_path": "test/systemtest/utils.py"
    }
  ]
}