Add full int8 ViT benchmark
diff --git a/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py b/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py
index 8fdae05..ff488d9 100644
--- a/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py
+++ b/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py
@@ -23,7 +23,10 @@
tf_models.GPT2_117M_1x4_FP32_TF,
tf_models.GPT2_117M_1x1_FP32_TF,
]
- QUANT_MODELS = [tflite_models.MOBILEBERT_INT8]
+ QUANT_MODELS = [
+ tflite_models.MOBILEBERT_INT8,
+ tflite_models.VIT_INT8_TFL,
+ ]
ARMV8_A_CPU_TARGET = iree_definitions.CompileTarget(
target_architecture=common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC,
diff --git a/build_tools/python/e2e_test_framework/models/model_groups.py b/build_tools/python/e2e_test_framework/models/model_groups.py
index fb30ca9..ddbef8d 100644
--- a/build_tools/python/e2e_test_framework/models/model_groups.py
+++ b/build_tools/python/e2e_test_framework/models/model_groups.py
@@ -138,6 +138,9 @@
common_definitions.CpuBenchmarkConfig(
model=torch_models.FALCON7B_1X100XI64_GPTQ_TORCH, threads=[8]
),
+ common_definitions.CpuBenchmarkConfig(
+ model=tflite_models.VIT_INT8_TFL, threads=[8]
+ ),
]
X86_64_BENCHMARK_CONFIG_LONG = [
diff --git a/build_tools/python/e2e_test_framework/models/tflite_models.py b/build_tools/python/e2e_test_framework/models/tflite_models.py
index 6fdcd79..77a9f2f 100644
--- a/build_tools/python/e2e_test_framework/models/tflite_models.py
+++ b/build_tools/python/e2e_test_framework/models/tflite_models.py
@@ -140,3 +140,13 @@
entry_function="main",
input_types=["1x224x224x3xui8"],
)
+
+VIT_INT8_TFL = common_definitions.Model(
+ name="Vit_int8",
+ id=unique_ids.MODEL_VIT_INT8_TFL,
+ tags=["int8", "vision-transformer"],
+ source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE,
+ source_url="https://storage.googleapis.com/iree-model-artifacts/tflite/tflite_models_1698315913/VIT_CLASSIFICATION_INT8_TFLITE_3X224X224XINT8/model_int8.tflite",
+ entry_function="main",
+ input_types=["1x3x224x224xi8"],
+)
diff --git a/build_tools/python/e2e_test_framework/unique_ids.py b/build_tools/python/e2e_test_framework/unique_ids.py
index bf09665..0a9b6fe 100644
--- a/build_tools/python/e2e_test_framework/unique_ids.py
+++ b/build_tools/python/e2e_test_framework/unique_ids.py
@@ -73,6 +73,7 @@
MODEL_MOBILENET_V3SMALL = "58855e40-eba9-4a71-b878-6b35e3460244"
MODEL_PERSON_DETECT_INT8 = "bc1338be-e3df-44fd-82e4-40ba9560a073"
MODEL_EFFICIENTNET_INT8 = "4a6f545e-1b4e-41a5-9236-792aa578184b"
+MODEL_VIT_INT8_TFL = "227cf5aa-5e5c-411b-8c7d-ecc83e6128dd"
# Tensorflow.
MODEL_MINILM_L12_H384_UNCASED_INT32_SEQLEN128 = "ecf5c970-ee97-49f0-a4ed-df1f34e9d493"
diff --git a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake b/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake
index 126b294..8025fa1 100644
--- a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake
+++ b/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake
@@ -132,6 +132,13 @@
)
iree_fetch_artifact(
+ NAME "model-Vit_int8"
+ SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/tflite/tflite_models_1698315913/VIT_CLASSIFICATION_INT8_TFLITE_3X224X224XINT8/model_int8.tflite"
+ OUTPUT "${ROOT_ARTIFACTS_DIR}/model_Vit_int8.tflite"
+ UNPACK
+)
+
+iree_fetch_artifact(
NAME "model-BertLargeTFBatch1"
SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/tensorflow/tf_models_2.15.0.dev20230817_1692333975/BERT_LARGE_FP32_TF_384XI32_BATCH1/stablehlo.mlirbc"
OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargeTFBatch1.mlirbc"
diff --git a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake
index fb90137..b6adad4 100644
--- a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake
+++ b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake
@@ -70,6 +70,12 @@
OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir"
)
+iree_import_tflite_model(
+ TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-Vit_int8_tflite_"
+ SOURCE "${ROOT_ARTIFACTS_DIR}/model_Vit_int8.tflite"
+ OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir"
+)
+
iree_bytecode_module(
NAME "iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_"
SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir"
@@ -558,6 +564,21 @@
)
iree_bytecode_module(
+ NAME "iree-module-Vit_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_"
+ SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir"
+ MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_/module.vmfb"
+ FLAGS
+ "--iree-hal-target-backends=llvm-cpu"
+ "--iree-input-type=tosa"
+ "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu"
+ "--iree-llvmcpu-target-cpu=cascadelake"
+ "--iree-opt-data-tiling"
+ "--iree-llvmcpu-enable-microkernels"
+ FRIENDLY_NAME "Vit_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,data-tiling,ukernel]"
+ PUBLIC
+)
+
+iree_bytecode_module(
NAME "iree-module-BertLargeTFBatch1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_"
SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTFBatch1.mlirbc"
MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTFBatch1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_/module.vmfb"
@@ -1109,6 +1130,18 @@
)
iree_bytecode_module(
+ NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_"
+ SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir"
+ MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_/module.vmfb"
+ FLAGS
+ "--iree-hal-target-backends=llvm-cpu"
+ "--iree-input-type=tosa"
+ "--iree-llvmcpu-target-triple=aarch64-none-linux-android29"
+ FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags]"
+ PUBLIC
+)
+
+iree_bytecode_module(
NAME "iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_"
SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir"
MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_/module.vmfb"
@@ -1180,6 +1213,21 @@
)
iree_bytecode_module(
+ NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_"
+ SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir"
+ MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_/module.vmfb"
+ FLAGS
+ "--iree-hal-target-backends=llvm-cpu"
+ "--iree-input-type=tosa"
+ "--iree-llvmcpu-target-triple=aarch64-none-linux-android29"
+ "--iree-opt-data-tiling"
+ "--iree-llvmcpu-enable-microkernels"
+ "--iree-llvmcpu-target-cpu-features=+dotprod"
+ FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,data-tiling,ukernel,dotprod]"
+ PUBLIC
+)
+
+iree_bytecode_module(
NAME "iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_"
SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir"
MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_/module.vmfb"
@@ -1980,6 +2028,25 @@
)
iree_bytecode_module(
+ NAME "iree-module-Vit_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_compile-stats_"
+ SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir"
+ MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_compile-stats_/module.vmfb"
+ FLAGS
+ "--iree-hal-target-backends=llvm-cpu"
+ "--iree-input-type=tosa"
+ "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu"
+ "--iree-llvmcpu-target-cpu=cascadelake"
+ "--iree-opt-data-tiling"
+ "--iree-llvmcpu-enable-microkernels"
+ "--iree-vm-emit-polyglot-zip=true"
+ "--iree-llvmcpu-debug-symbols=false"
+ "--iree-scheduling-dump-statistics-format=json"
+ "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_compile-stats_/scheduling_stats.json"
+ FRIENDLY_NAME "Vit_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,data-tiling,ukernel,compile-stats]"
+ PUBLIC
+)
+
+iree_bytecode_module(
NAME "iree-module-BertLargeTFBatch1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_compile-stats_"
SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTFBatch1.mlirbc"
MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTFBatch1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb"
@@ -2691,6 +2758,22 @@
)
iree_bytecode_module(
+ NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_compile-stats_"
+ SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir"
+ MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_compile-stats_/module.vmfb"
+ FLAGS
+ "--iree-hal-target-backends=llvm-cpu"
+ "--iree-input-type=tosa"
+ "--iree-llvmcpu-target-triple=aarch64-none-linux-android29"
+ "--iree-vm-emit-polyglot-zip=true"
+ "--iree-llvmcpu-debug-symbols=false"
+ "--iree-scheduling-dump-statistics-format=json"
+ "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json"
+ FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,compile-stats]"
+ PUBLIC
+)
+
+iree_bytecode_module(
NAME "iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_compile-stats_"
SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir"
MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_compile-stats_/module.vmfb"
@@ -2782,6 +2865,25 @@
)
iree_bytecode_module(
+ NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_compile-stats_"
+ SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir"
+ MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_compile-stats_/module.vmfb"
+ FLAGS
+ "--iree-hal-target-backends=llvm-cpu"
+ "--iree-input-type=tosa"
+ "--iree-llvmcpu-target-triple=aarch64-none-linux-android29"
+ "--iree-opt-data-tiling"
+ "--iree-llvmcpu-enable-microkernels"
+ "--iree-llvmcpu-target-cpu-features=+dotprod"
+ "--iree-vm-emit-polyglot-zip=true"
+ "--iree-llvmcpu-debug-symbols=false"
+ "--iree-scheduling-dump-statistics-format=json"
+ "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_compile-stats_/scheduling_stats.json"
+ FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,data-tiling,ukernel,dotprod,compile-stats]"
+ PUBLIC
+)
+
+iree_bytecode_module(
NAME "iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_compile-stats_"
SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir"
MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_compile-stats_/module.vmfb"
@@ -3019,6 +3121,7 @@
${PACKAGE_NAME}_iree-imported-model-MobileSSD_fp32_tflite_
${PACKAGE_NAME}_iree-imported-model-PersonDetect_int8_tflite_
${PACKAGE_NAME}_iree-imported-model-PoseNet_fp32_tflite_
+ ${PACKAGE_NAME}_iree-imported-model-Vit_int8_tflite_
${PACKAGE_NAME}_model-BertForMaskedLMTF
${PACKAGE_NAME}_model-BertLargeTF
${PACKAGE_NAME}_model-EfficientNetV2STF
@@ -3063,6 +3166,8 @@
${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_
${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_
${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_
)
add_dependencies(iree-benchmark-suites-android-gpu
@@ -3154,6 +3259,9 @@
${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_compile-stats_
${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_compile-stats_
${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_compile-stats_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_compile-stats_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_compile-stats_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_compile-stats_
${PACKAGE_NAME}_iree-module-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_
${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_
${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_
@@ -3270,6 +3378,9 @@
${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_
${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_
${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_data-tiling_ukernel_dotprod_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_
${PACKAGE_NAME}_iree-module-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_
${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_
${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_
@@ -3345,6 +3456,7 @@
${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_
${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_
${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_
+ ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_data-tiling_ukernel_
)
add_dependencies(iree-benchmark-suites-x86_64-large
