diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 985ed44..226e09d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -198,39 +198,14 @@
 
   ##############################################################################
 
-  # Depends on all the other jobs to provide a single anchor that indicates the
-  # final status. Status reporting will become more sophisticated in the future
-  # and we can hopefully avoid the need to explicitly list every single job...
-  summary:
-    # Even if you have an explicit if condition, you still need to override
-    # GitHub's default behavior of not running if any dependencies failed.
+  # Aggregate job status and alerting on failures.
+  ci_summary:
     if: always()
-    runs-on: ubuntu-20.04
     needs:
       - setup
       - runtime
       - runtime_small
       - runtime_tracing
-    steps:
-      - name: "Checking out repository"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Getting failed jobs
-        id: failed_jobs
-        run: |
-          echo '${{ toJson(needs) }}'
-          FAILED_JOBS="$(echo '${{ toJson(needs) }}' \
-            | jq --raw-output \
-            'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \
-          )"
-          echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT
-          if [[ "${FAILED_JOBS}" != "" ]]; then
-            echo "The following jobs failed: ${FAILED_JOBS}"
-            exit 1
-          fi
-      - name: Posting to Discord
-        uses: sarisia/actions-status-discord@ce8cc68e4e626000136b3c702d049a154243e490 # v1.14.7
-        if: failure() && github.ref_name == 'main'
-        with:
-          webhook: ${{ secrets.DISCORD_WEBHOOK }}
-          description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}"
-          url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
+    uses: ./.github/workflows/workflow_summary.yml
+    with:
+      jobs-json: ${{ toJson(needs) }}
diff --git a/.github/workflows/workflow_summary.yml b/.github/workflows/workflow_summary.yml
new file mode 100644
index 0000000..de19634
--- /dev/null
+++ b/.github/workflows/workflow_summary.yml
@@ -0,0 +1,62 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Checks the result status of each job provided by 'jobs-json' and sends an
+# alert if at least one job failed.
+#
+# Usage:
+#   ```yml
+#   jobs:
+#      job_1:
+#          ...
+#      job_2:
+#          ...
+#      my_summary:
+#        if: always()
+#        needs:
+#          - job_1
+#          - job_2
+#        uses: ./.github/workflows/workflow_summary.yml
+#        with:
+#          jobs-json: ${{ toJson(needs) }}
+#   ```
+
+name: Workflow Summary
+
+on:
+  workflow_call:
+    inputs:
+        jobs-json:
+          type: string
+          description: The output of `toJson(needs)`
+
+permissions:
+  contents: read
+
+jobs:
+  summary:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Getting failed jobs
+        id: failed_jobs
+        run: |
+          echo '${{ inputs.jobs-json }}'
+          FAILED_JOBS="$(echo '${{ inputs.jobs-json }}' \
+            | jq --raw-output \
+            'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \
+          )"
+          echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT
+          if [[ "${FAILED_JOBS}" != "" ]]; then
+            echo "The following jobs failed: ${FAILED_JOBS}"
+            exit 1
+          fi
+      - name: Posting to Discord
+        uses: sarisia/actions-status-discord@ce8cc68e4e626000136b3c702d049a154243e490 # v1.14.7
+        if: failure() && github.ref_name == 'main' && github.repository_owner == 'iree-org'
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}"
+          url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
diff --git a/build_tools/github_actions/configure_ci.py b/build_tools/github_actions/configure_ci.py
index 55247e9..8144aa8 100755
--- a/build_tools/github_actions/configure_ci.py
+++ b/build_tools/github_actions/configure_ci.py
@@ -39,8 +39,8 @@
 import fnmatch
 import json
 import os
-import re
 import pathlib
+import re
 import string
 import subprocess
 import sys
@@ -111,7 +111,12 @@
 RUNNER_ENV_DEFAULT = "prod"
 RUNNER_ENV_OPTIONS = [RUNNER_ENV_DEFAULT, "testing"]
 
-CONTROL_JOBS = frozenset(["setup", "summary"])
+CONTROL_JOB_REGEXES = frozenset(
+    [
+        re.compile("setup"),
+        re.compile(".*summary"),
+    ]
+)
 
 # Jobs to run only on postsubmit by default.
 # They may also run on presubmit only under certain conditions.
@@ -380,7 +385,8 @@
 
     workflow = yaml.load(workflow_file.read_text(), Loader=yaml.SafeLoader)
     all_jobs = set(workflow["jobs"].keys())
-    all_jobs -= CONTROL_JOBS
+    for regex in CONTROL_JOB_REGEXES:
+        all_jobs = {j for j in all_jobs if not regex.match(j)}
 
     if ALL_KEY in all_jobs:
         raise ValueError(f"Workflow has job with reserved name '{ALL_KEY}'")