Update ResNet50 PyTorch batch sizes (#12891)
Updates ResNet50 PyTorch batch sizes to match the MLPerf configs in
https://github.com/mlcommons/inference_results_v2.1/tree/master/closed/NVIDIA/configs/resnet50
x86 and CUDA benchmark runtimes are unchanged.
diff --git a/build_tools/python/e2e_test_framework/models/model_groups.py b/build_tools/python/e2e_test_framework/models/model_groups.py
index 31068f8..8149ae4 100644
--- a/build_tools/python/e2e_test_framework/models/model_groups.py
+++ b/build_tools/python/e2e_test_framework/models/model_groups.py
@@ -56,13 +56,10 @@
torch_models.RESNET50_1X3X224X224_FP32_TORCH,
torch_models.RESNET50_8X3X224X224_FP32_TORCH,
# Disabled due to https://github.com/openxla/iree/issues/12774.
- #torch_models.RESNET50_16X3X224X224_FP32_TORCH,
- #torch_models.RESNET50_32X3X224X224_FP32_TORCH,
#torch_models.RESNET50_64X3X224X224_FP32_TORCH,
#torch_models.RESNET50_128X3X224X224_FP32_TORCH,
#torch_models.RESNET50_256X3X224X224_FP32_TORCH,
- #torch_models.RESNET50_512X3X224X224_FP32_TORCH,
- #torch_models.RESNET50_1024X3X224X224_FP32_TORCH,
+ #torch_models.RESNET50_2048X3X224X224_FP32_TORCH,
]
ALL = SMALL + LARGE + BERT_LARGE_TORCH_BATCHES + RESNET50_TORCH_BATCHES
diff --git a/build_tools/python/e2e_test_framework/models/torch_models.py b/build_tools/python/e2e_test_framework/models/torch_models.py
index 8b5106b..7c210d7 100644
--- a/build_tools/python/e2e_test_framework/models/torch_models.py
+++ b/build_tools/python/e2e_test_framework/models/torch_models.py
@@ -173,7 +173,7 @@
tags=["fp32", "cnn", "batch-1"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_1/linalg.mlir",
+ "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/RESNET50/batch_1/linalg.mlir",
entry_function="forward",
input_types=["1x3x224x224xf32"])
@@ -183,37 +183,17 @@
tags=["fp32", "cnn", "batch-8"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_8/linalg.mlir",
+ "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/RESNET50/batch_8/linalg.mlir",
entry_function="forward",
input_types=["8x3x224x224xf32"])
-RESNET50_16X3X224X224_FP32_TORCH = common_definitions.Model(
- id=unique_ids.MODEL_RESNET50_16X3X224X224_FP32_TORCH,
- name="Resnet50PTBatch16",
- tags=["fp32", "cnn", "batch-16"],
- source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
- source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_16/linalg.mlir",
- entry_function="forward",
- input_types=["16x3x224x224xf32"])
-
-RESNET50_32X3X224X224_FP32_TORCH = common_definitions.Model(
- id=unique_ids.MODEL_RESNET50_32X3X224X224_FP32_TORCH,
- name="Resnet50PTBatch32",
- tags=["fp32", "cnn", "batch-32"],
- source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
- source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_32/linalg.mlir",
- entry_function="forward",
- input_types=["32x3x224x224xf32"])
-
RESNET50_64X3X224X224_FP32_TORCH = common_definitions.Model(
id=unique_ids.MODEL_RESNET50_64X3X224X224_FP32_TORCH,
name="Resnet50PTBatch64",
tags=["fp32", "cnn", "batch-64"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_64/linalg.mlir",
+ "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/RESNET50/batch_64/linalg.mlir",
entry_function="forward",
input_types=["64x3x224x224xf32"])
@@ -223,7 +203,7 @@
tags=["fp32", "cnn", "batch-128"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_128/linalg.mlir",
+ "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/RESNET50/batch_128/linalg.mlir",
entry_function="forward",
input_types=["128x3x224x224xf32"])
@@ -233,26 +213,16 @@
tags=["fp32", "cnn", "batch-256"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_256/linalg.mlir",
+ "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/RESNET50/batch_256/linalg.mlir",
entry_function="forward",
input_types=["256x3x224x224xf32"])
-RESNET50_512X3X224X224_FP32_TORCH = common_definitions.Model(
- id=unique_ids.MODEL_RESNET50_512X3X224X224_FP32_TORCH,
- name="Resnet50PTBatch512",
- tags=["fp32", "cnn", "batch-512"],
+RESNET50_2048X3X224X224_FP32_TORCH = common_definitions.Model(
+ id=unique_ids.MODEL_RESNET50_2048X3X224X224_FP32_TORCH,
+ name="Resnet50PTBatch2048",
+ tags=["fp32", "cnn", "batch-2048"],
source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_512/linalg.mlir",
+ "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/RESNET50/batch_2048/linalg.mlir",
entry_function="forward",
- input_types=["512x3x224x224xf32"])
-
-RESNET50_1024X3X224X224_FP32_TORCH = common_definitions.Model(
- id=unique_ids.MODEL_RESNET50_1024X3X224X224_FP32_TORCH,
- name="Resnet50PTBatch1024",
- tags=["fp32", "cnn", "batch-1024"],
- source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
- source_url=
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_1024/linalg.mlir",
- entry_function="forward",
- input_types=["1024x3x224x224xf32"])
+ input_types=["2048x3x224x224xf32"])
diff --git a/build_tools/python/e2e_test_framework/unique_ids.py b/build_tools/python/e2e_test_framework/unique_ids.py
index 773a343..2dc5cce 100644
--- a/build_tools/python/e2e_test_framework/unique_ids.py
+++ b/build_tools/python/e2e_test_framework/unique_ids.py
@@ -100,13 +100,10 @@
MODEL_RESNET50_1X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-1"
MODEL_RESNET50_8X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-8"
-MODEL_RESNET50_16X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-16"
-MODEL_RESNET50_32X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-32"
MODEL_RESNET50_64X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-64"
MODEL_RESNET50_128X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-128"
MODEL_RESNET50_256X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-256"
-MODEL_RESNET50_512X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-512"
-MODEL_RESNET50_1024X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-1024"
+MODEL_RESNET50_2048X3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-2048"
# Model input data
MODEL_INPUT_DATA_ZEROS = "8d4a034e-944d-4725-8402-d6f6e61be93c"
diff --git a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake b/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake
index bf19b8b..4b91285 100644
--- a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake
+++ b/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake
@@ -262,7 +262,7 @@
NAME
"model-fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-1"
SOURCE_URL
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_1/linalg.mlir"
+ "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/RESNET50/batch_1/linalg.mlir"
OUTPUT
"${ROOT_ARTIFACTS_DIR}/model_fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-1_Resnet50PTBatch1.mlir"
UNPACK
@@ -272,7 +272,7 @@
NAME
"model-fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-8"
SOURCE_URL
- "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/RESNET50/batch_8/linalg.mlir"
+ "https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230401.795_1680469670/RESNET50/batch_8/linalg.mlir"
OUTPUT
"${ROOT_ARTIFACTS_DIR}/model_fd05da43-5e37-4fa0-88f8-3ceec1682345-batch-8_Resnet50PTBatch8.mlir"
UNPACK
