Add Mobilebert Quant to Presubmits (#8796)
diff --git a/integrations/tensorflow/test/iree_tfl_tests/llvmaot_mobilebert_tf2_quant.run b/integrations/tensorflow/test/iree_tfl_tests/llvmaot_mobilebert_tf2_quant.run new file mode 100644 index 0000000..73ba64b --- /dev/null +++ b/integrations/tensorflow/test/iree_tfl_tests/llvmaot_mobilebert_tf2_quant.run
@@ -0,0 +1,2 @@ +# REQUIRES: llvmaot +# RUN: %PYTHON -m iree_tfl_tests.mobilebert_tf2_quant_test --target_backend=llvmaot -artifacts_dir=%t
diff --git a/integrations/tensorflow/test/iree_tfl_tests/vulkan_mobilebert_tf2_quant.run b/integrations/tensorflow/test/iree_tfl_tests/vulkan_mobilebert_tf2_quant.run new file mode 100644 index 0000000..b7bc694 --- /dev/null +++ b/integrations/tensorflow/test/iree_tfl_tests/vulkan_mobilebert_tf2_quant.run
@@ -0,0 +1,3 @@ +# REQUIRES: vulkan +# RUN: %PYTHON -m iree_tfl_tests.mobilebert_tf2_quant_test --target_backend=vulkan -artifacts_dir=%t +# XFAIL: * \ No newline at end of file
diff --git a/integrations/tensorflow/test/lit.cfg.py b/integrations/tensorflow/test/lit.cfg.py index 44cbca8..772f7ba 100644 --- a/integrations/tensorflow/test/lit.cfg.py +++ b/integrations/tensorflow/test/lit.cfg.py
@@ -40,6 +40,7 @@ 'lit.site.cfg.py', 'test_util.py', 'manual_test.py', + 'squad_test_data.py', ] config.substitutions.extend([ @@ -64,9 +65,9 @@ disable_features_param = lit_config.params.get('DISABLE_FEATURES') disable_features = [] if disable_features_param: - disable_features = disable_features_param.split(',') + disable_features = disable_features_param.split(',') if 'llvmaot' not in disable_features: - config.available_features.add('llvmaot') + config.available_features.add('llvmaot') features_param = lit_config.params.get('FEATURES') if features_param: config.available_features.update(features_param.split(','))
diff --git a/integrations/tensorflow/test/python/iree_tfl_tests/mobilebert_tf2_quant_test.py b/integrations/tensorflow/test/python/iree_tfl_tests/mobilebert_tf2_quant_test.py new file mode 100644 index 0000000..fa95b23 --- /dev/null +++ b/integrations/tensorflow/test/python/iree_tfl_tests/mobilebert_tf2_quant_test.py
@@ -0,0 +1,49 @@ +# RUN: %PYTHON %s + +import absl.testing +import numpy as np +from . import squad_test_data +from . import test_util + +# Source https://tfhub.dev/iree/lite-model/mobilebert/int8/1 +model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilebert-baseline-tf2-quant.tflite" + + +class MobileBertTest(test_util.TFLiteModelTest): + + def __init__(self, *args, **kwargs): + super(MobileBertTest, self).__init__(model_path, *args, **kwargs) + + # Inputs modified to be useful mobilebert inputs. + def generate_inputs(self, input_details): + for input in input_details: + absl.logging.info("\t%s, %s", str(input["shape"]), + input["dtype"].__name__) + + input_0 = np.asarray(squad_test_data._INPUT_WORD_ID, + dtype=input_details[0]["dtype"]) + input_1 = np.asarray(squad_test_data._INPUT_TYPE_ID, + dtype=input_details[1]["dtype"]) + input_2 = np.asarray(squad_test_data._INPUT_MASK, + dtype=input_details[2]["dtype"]) + return [ + input_0.reshape(input_details[0]["shape"]), + input_1.reshape(input_details[1]["shape"]), + input_2.reshape(input_details[2]["shape"]) + ] + + def compare_results(self, iree_results, tflite_results, details): + super(MobileBertTest, self).compare_results(iree_results, tflite_results, + details) + # We have confirmed in large scale accuracy tests that differences this large is acceptable. + self.assertTrue( + np.isclose(iree_results[0], tflite_results[0], atol=5.0).all()) + self.assertTrue( + np.isclose(iree_results[1], tflite_results[1], atol=5.0).all()) + + def test_compile_tflite(self): + self.compile_and_execute() + + +if __name__ == '__main__': + absl.testing.absltest.main()
diff --git a/integrations/tensorflow/test/python/iree_tfl_tests/squad_test_data.py b/integrations/tensorflow/test/python/iree_tfl_tests/squad_test_data.py new file mode 100644 index 0000000..20a73f3 --- /dev/null +++ b/integrations/tensorflow/test/python/iree_tfl_tests/squad_test_data.py
@@ -0,0 +1,58 @@ +# An example input combination from the Squad 1.1 dataset. +_INPUT_WORD_ID = [ + 101, 2129, 2116, 19576, 2015, 2106, 3854, 4679, 2486, 1029, 102, 1996, + 14169, 2165, 2019, 2220, 2599, 1999, 3565, 4605, 2753, 1998, 2196, 11145, + 1012, 8446, 2001, 3132, 2011, 7573, 1005, 1055, 3639, 1010, 2029, 14159, + 2032, 2698, 2335, 1998, 3140, 2032, 2046, 2093, 20991, 2015, 1010, 2164, + 1037, 19576, 2029, 2027, 6757, 2005, 1037, 7921, 1012, 7573, 15674, 3854, + 4679, 2001, 2315, 3565, 4605, 12041, 1010, 3405, 2274, 3948, 10455, 1010, + 1016, 13714, 14918, 1010, 1998, 2048, 3140, 19576, 2015, 1012, 102, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +] +_INPUT_TYPE_ID = [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 +] +_INPUT_MASK = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 +]