)]}' { "commit": "e98a03bc2c48b23334c52cbbaf4c46ceb691e138", "tree": "40dfd0d407211cde1c4c21d2f1a356b205dd6c8f", "parents": [ "03535d8f82d043e486e2088becc1682b2aa21da0" ], "author": { "name": "Ben Vanik", "email": "ben.vanik@gmail.com", "time": "Fri Sep 23 14:17:05 2022 -0700" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Sep 23 21:17:05 2022 +0000" }, "message": "Adding `--expected_output\u003d` flag to iree-run-module. (#10433)\n\nThis allows for super basic blackbox testing of compiled modules by way\r\nof iree-run-module that does not require any input changes like\r\niree-check-module does. The flag accepts the same values as\r\n`--function_input` (including npy files) in addition to `(ignored)` for\r\nwhen a value is not of interest.\r\n\r\nWhen the flag is provided the normal output printing is disabled and\r\ninstead a comparison runs and prints detailed information about any\r\ndiffs, for example:\r\n```\r\n[FAILED] result[0]: element at index 49 (0) does not match the expected (123); expected that the view is equal to contents of a view of 5x1x10xf32\r\n expected:\r\n5x1x10xf32\u003d[[0.743973 ... 0 0 123]]\r\n actual:\r\n5x1x10xf32\u003d[[0.743973 ... 0 0 0]]\r\n```\r\n\r\nThe matching utilities are intended to be wrapped in gtest C++ matchers\r\nso that they can be used in iree-check-module and other tools but that\r\nis left for future work. This separation is required so that\r\niree-run-module and other tools using the comparison utilities don\u0027t\r\nneed to depend on gtest. It also means that users authoring their own\r\ntest tools for bare metal/etc can easily use the comparison features.\r\n\r\nThere are three flags like `--expected_f32_threshold\u003d` (+f16/f64) that\r\nallow for basic threshold overrides. Full customization of the equality\r\ncomparisons that do smarter things (ULP, integer ranges, etc) are also\r\nleft for future work and what\u0027s used here matches iree-check-module\u0027s\r\ncurrent behavior.\r\n\r\nFixes #10376.", "tree_diff": [ { "type": "modify", "old_id": "0b53eb835b6fa0fd6bdccfbabe410254c3237880", "old_mode": 33188, "old_path": "build_tools/bazel_to_cmake/bazel_to_cmake_targets.py", "new_id": "2e60681649ef14c95e4966fb05660d6aaec6440e", "new_mode": 33188, "new_path": "build_tools/bazel_to_cmake/bazel_to_cmake_targets.py" }, { "type": "modify", "old_id": "590e1d54c7c5f8818d0a32746dfa60323ff80a6c", "old_mode": 33188, "old_path": "runtime/src/iree/base/string_builder.c", "new_id": "2aa6994cc9bce941a2434994820feb5fea9fb783", "new_mode": 33188, "new_path": "runtime/src/iree/base/string_builder.c" }, { "type": "modify", "old_id": "ff6eeba6709a1c00575280db4c839106dca3b898", "old_mode": 33188, "old_path": "runtime/src/iree/base/string_builder.h", "new_id": "007f87cb1d61d74e69e66230b97f08d95b87a0e7", "new_mode": 33188, "new_path": "runtime/src/iree/base/string_builder.h" }, { "type": "modify", "old_id": "fad7034ab83b58f0be24bf1e56a74c734142e700", "old_mode": 33188, "old_path": "runtime/src/iree/base/string_builder_test.cc", "new_id": "35987fe1048a64b3b134ae8a68813616249285e7", "new_mode": 33188, "new_path": "runtime/src/iree/base/string_builder_test.cc" }, { "type": "modify", "old_id": "b6e046dd1239391c5873e1ce1056c1e54c48bb78", "old_mode": 33188, "old_path": "runtime/src/iree/hal/string_util.c", "new_id": "160ed73d7e135124f595cc953497bf08a35b0fd0", "new_mode": 33188, "new_path": "runtime/src/iree/hal/string_util.c" }, { "type": "modify", "old_id": "3a531165218d9326ab47e5eaede9f01afc8edd88", "old_mode": 33188, "old_path": "runtime/src/iree/hal/string_util.h", "new_id": "2c14e2f2d9fbff0947e483cf656fa6455041a731", "new_mode": 33188, "new_path": "runtime/src/iree/hal/string_util.h" }, { "type": "modify", "old_id": "e793590da9bebdb85f4272dd0d730af191e25352", "old_mode": 33188, "old_path": "runtime/src/iree/tooling/BUILD", "new_id": "b4d71c6b886037b37eda88f72c85c5bc63d39bcf", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/BUILD" }, { "type": "modify", "old_id": "22a901b140d342fdcd3d38d736a3d5aec1ae9f14", "old_mode": 33188, "old_path": "runtime/src/iree/tooling/CMakeLists.txt", "new_id": "c20fade9fe72308dd89e94bb6bc059836d3fc74a", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/CMakeLists.txt" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "9112ec2d612d93d9e05bfce17b85a86c734d2adb", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/buffer_view_matchers.c" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "f6a6b078957b9f269f97973851ddc85e2ab4d72a", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/buffer_view_matchers.h" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "a4857b78742322f77964029f3302d938c5695726", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/buffer_view_matchers_test.cc" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "d6d477a91af0391306a2ada0ac5fb8562073e2f8", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/comparison.cc" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "747d087a8da4a6204b8ed62d8455bc1e6b14873d", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/comparison.h" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "b60e96e406a9dd68b6b44304653088f87bb639da", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/comparison_test.cc" }, { "type": "modify", "old_id": "c6474d7adeeb7c44c7302f37c17b87d757abb731", "old_mode": 33188, "old_path": "runtime/src/iree/tooling/vm_util.cc", "new_id": "319176a17322c2cea6b1402e7d72241e52101773", "new_mode": 33188, "new_path": "runtime/src/iree/tooling/vm_util.cc" }, { "type": "modify", "old_id": "fdaa9456220cf229d04282e987057ec7c4b216ce", "old_mode": 33188, "old_path": "tools/BUILD", "new_id": "725cffc2a5563c22e122f4ed4420e2b246282388", "new_mode": 33188, "new_path": "tools/BUILD" }, { "type": "modify", "old_id": "ef37edb9a594626d2d8caf9d8b08e04932d7109a", "old_mode": 33188, "old_path": "tools/CMakeLists.txt", "new_id": "0a99dbf4315e65a8086ae9e3a2e0e8fef5873769", "new_mode": 33188, "new_path": "tools/CMakeLists.txt" }, { "type": "modify", "old_id": "b4eda1db5c6e1476ee25118e9a0a03cd66121d0d", "old_mode": 33188, "old_path": "tools/iree-run-module-main.cc", "new_id": "ffe7837cfa0ace183ccfeca739fc78a45dd426f9", "new_mode": 33188, "new_path": "tools/iree-run-module-main.cc" }, { "type": "modify", "old_id": "f6d88a82b792de413366ea868518182e969fc94f", "old_mode": 33188, "old_path": "tools/test/BUILD", "new_id": "c54e2b125253cabe8a9eafa95312bae58b952af9", "new_mode": 33188, "new_path": "tools/test/BUILD" }, { "type": "modify", "old_id": "e4075f289ed67e0f6bd431e8e44704023aff335b", "old_mode": 33188, "old_path": "tools/test/CMakeLists.txt", "new_id": "3a1e4184aae136aaf349998c468456081dc638f8", "new_mode": 33188, "new_path": "tools/test/CMakeLists.txt" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "f7647c8bb64f2af120dd4de707c18c0047616943", "new_mode": 33188, "new_path": "tools/test/iree-run-module-expected.mlir" } ] }