Adding HAL semaphore support for statuses-as-failure-payloads. (#18912)
This allows an implementation to have a single atomic value for a
semaphore that encodes the user payload or an error payload that
optionally references an iree_status_t object. Implementations not using
the status feature can ignore it but must perform a
greater-than-or-equal check on `IREE_HAL_SEMAPHORE_FAILURE_VALUE`
instead of equality.
diff --git a/runtime/src/iree/hal/cts/semaphore_submission_test.h b/runtime/src/iree/hal/cts/semaphore_submission_test.h
index a158082..b745761 100644
--- a/runtime/src/iree/hal/cts/semaphore_submission_test.h
+++ b/runtime/src/iree/hal/cts/semaphore_submission_test.h
@@ -882,7 +882,7 @@
EXPECT_EQ(iree_status_code(wait_status), IREE_STATUS_ABORTED);
uint64_t value = 1234;
iree_status_t query_status = iree_hal_semaphore_query(semaphore2, &value);
- EXPECT_EQ(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
+ EXPECT_GE(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
CheckStatusContains(query_status, status);
signal_thread.join();
diff --git a/runtime/src/iree/hal/cts/semaphore_test.h b/runtime/src/iree/hal/cts/semaphore_test.h
index 54e907e..7d0592f 100644
--- a/runtime/src/iree/hal/cts/semaphore_test.h
+++ b/runtime/src/iree/hal/cts/semaphore_test.h
@@ -406,7 +406,7 @@
EXPECT_EQ(iree_status_code(wait_status), IREE_STATUS_ABORTED);
uint64_t value = 1234;
iree_status_t query_status = iree_hal_semaphore_query(semaphore, &value);
- EXPECT_EQ(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
+ EXPECT_GE(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
CheckStatusContains(query_status, status);
iree_hal_semaphore_release(semaphore);
@@ -431,7 +431,7 @@
EXPECT_EQ(iree_status_code(wait_status), IREE_STATUS_ABORTED);
uint64_t value = 1234;
iree_status_t query_status = iree_hal_semaphore_query(semaphore, &value);
- EXPECT_EQ(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
+ EXPECT_GE(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
CheckStatusContains(query_status, status);
signal_thread.join();
@@ -467,7 +467,7 @@
uint64_t value = 1234;
iree_status_t semaphore1_query_status =
iree_hal_semaphore_query(semaphore1, &value);
- EXPECT_EQ(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
+ EXPECT_GE(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
CheckStatusContains(semaphore1_query_status, status);
// semaphore2 must not have changed.
@@ -511,7 +511,7 @@
uint64_t value = 1234;
iree_status_t semaphore1_query_status =
iree_hal_semaphore_query(semaphore1, &value);
- EXPECT_EQ(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
+ EXPECT_GE(value, IREE_HAL_SEMAPHORE_FAILURE_VALUE);
CheckStatusContains(semaphore1_query_status, status);
// semaphore2 must not have changed.
diff --git a/runtime/src/iree/hal/drivers/cuda/event_semaphore.c b/runtime/src/iree/hal/drivers/cuda/event_semaphore.c
index fb86efe..0c0cf41 100644
--- a/runtime/src/iree/hal/drivers/cuda/event_semaphore.c
+++ b/runtime/src/iree/hal/drivers/cuda/event_semaphore.c
@@ -325,7 +325,7 @@
}
iree_slim_mutex_lock(&semaphore->mutex);
- if (semaphore->current_value == IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
+ if (semaphore->current_value >= IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
iree_slim_mutex_unlock(&semaphore->mutex);
IREE_TRACE_ZONE_END(z0);
return iree_make_status(IREE_STATUS_ABORTED);
@@ -350,7 +350,7 @@
}
iree_slim_mutex_lock(&semaphore->mutex);
- if (semaphore->current_value == IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
+ if (semaphore->current_value >= IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
status = iree_make_status(IREE_STATUS_ABORTED);
}
iree_slim_mutex_unlock(&semaphore->mutex);
@@ -444,7 +444,7 @@
iree_hal_cuda_semaphore_t* semaphore =
iree_hal_cuda_semaphore_cast(semaphore_list.semaphores[i]);
iree_slim_mutex_lock(&semaphore->mutex);
- if (semaphore->current_value == IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
+ if (semaphore->current_value >= IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
iree_slim_mutex_unlock(&semaphore->mutex);
status = iree_make_status(IREE_STATUS_ABORTED);
break;
diff --git a/runtime/src/iree/hal/drivers/hip/event_semaphore.c b/runtime/src/iree/hal/drivers/hip/event_semaphore.c
index 926eb54..de10b09 100644
--- a/runtime/src/iree/hal/drivers/hip/event_semaphore.c
+++ b/runtime/src/iree/hal/drivers/hip/event_semaphore.c
@@ -323,7 +323,7 @@
}
iree_slim_mutex_lock(&semaphore->mutex);
- if (semaphore->current_value == IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
+ if (semaphore->current_value >= IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
iree_slim_mutex_unlock(&semaphore->mutex);
IREE_TRACE_ZONE_END(z0);
return iree_make_status(IREE_STATUS_ABORTED);
@@ -346,7 +346,7 @@
}
iree_slim_mutex_lock(&semaphore->mutex);
- if (semaphore->current_value == IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
+ if (semaphore->current_value >= IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
status = iree_make_status(IREE_STATUS_ABORTED);
}
iree_slim_mutex_unlock(&semaphore->mutex);
@@ -440,7 +440,7 @@
iree_hal_hip_semaphore_t* semaphore =
iree_hal_hip_semaphore_cast(semaphore_list.semaphores[i]);
iree_slim_mutex_lock(&semaphore->mutex);
- if (semaphore->current_value == IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
+ if (semaphore->current_value >= IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
iree_slim_mutex_unlock(&semaphore->mutex);
status = iree_make_status(IREE_STATUS_ABORTED);
break;
diff --git a/runtime/src/iree/hal/semaphore.h b/runtime/src/iree/hal/semaphore.h
index 8cc073b..52571ed 100644
--- a/runtime/src/iree/hal/semaphore.h
+++ b/runtime/src/iree/hal/semaphore.h
@@ -30,10 +30,6 @@
};
typedef uint32_t iree_hal_semaphore_flags_t;
-//===----------------------------------------------------------------------===//
-// iree_hal_semaphore_t
-//===----------------------------------------------------------------------===//
-
// The maximum valid payload value of an iree_hal_semaphore_t.
// Payload values larger than this indicate that the semaphore has failed.
//
@@ -56,8 +52,66 @@
// https://vulkan.gpuinfo.org/displayextensionproperty.php?name=maxTimelineSemaphoreValueDifference
#define IREE_HAL_SEMAPHORE_MAX_VALUE (2147483647ull - 1)
+// The minimum value for a semaphore that indicates failure. Any value
+// greater-than-or-equal-to (>=) this indicates the semaphore has failed.
+//
+// If the upper bit 63 is set then the value represents an iree_status_t.
+// Use iree_hal_semaphore_failure_as_status to convert a payload value to a
+// status. Not all implementations do (or can) support encoding statuses and may
+// only ever be able to set a failing semaphore to this value.
#define IREE_HAL_SEMAPHORE_FAILURE_VALUE (IREE_HAL_SEMAPHORE_MAX_VALUE + 1)
+// Bit indicating that a failing semaphore value can be interpreted as an
+// iree_status_t.
+#define IREE_HAL_SEMAPHORE_FAILURE_VALUE_STATUS_BIT 0x8000000000000000ull
+
+// Returns a semaphore payload value that encodes the given |status|.
+// Ownership of the status is transferred to the semaphore and it must be
+// freed by a consumer. Not all implementations can support failure status
+// payloads and this should only be used by those implementations that can.
+static inline uint64_t iree_hal_status_as_semaphore_failure(
+ iree_status_t status) {
+ return IREE_HAL_SEMAPHORE_FAILURE_VALUE_STATUS_BIT |
+ (((uint64_t)status) >> 1);
+}
+
+// Returns OK if the |value| does not indicate an error.
+// Returns an error status if the semaphore payload value represents a failure.
+// If the payload contains an encoded iree_status_t it will be cloned and the
+// new copy will be returned to the caller.
+static inline iree_status_t iree_hal_semaphore_failure_as_status(
+ uint64_t value) {
+ if (value >= IREE_HAL_SEMAPHORE_FAILURE_VALUE) {
+ if (value & IREE_HAL_SEMAPHORE_FAILURE_VALUE_STATUS_BIT) {
+ // The top bits of a pointer are sign-extended from bit 47 so we can
+ // restore the top bit by left-shifting the upper bits and then
+ // right-shifting with sign extension. We only use a single bit today and
+ // so bit 62 should still be the original value of the pointer.
+ // Note that if the status is just a code (no allocated pointer) this
+ // clone is a no-op and the code will be returned without an allocation.
+ //
+ // See:
+ // https://en.wikipedia.org/wiki/X86-64#Canonical_form_addresses
+ return iree_status_clone((iree_status_t)(((int64_t)value << 1) >> 1));
+ } else {
+ return iree_status_from_code(IREE_STATUS_INTERNAL);
+ }
+ } else {
+ return iree_ok_status();
+ }
+}
+
+// Frees an iree_status_t encoded in a semaphore |value|, if any.
+static inline void iree_hal_semaphore_failure_free(uint64_t value) {
+ if (value & IREE_HAL_SEMAPHORE_FAILURE_VALUE_STATUS_BIT) {
+ iree_status_free((iree_status_t)(((int64_t)value << 1) >> 1));
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// iree_hal_semaphore_t
+//===----------------------------------------------------------------------===//
+
// Synchronization mechanism for host->device, device->host, host->host,
// and device->device notification. Semaphores behave like Vulkan timeline
// semaphores (or D3D12 fences) and contain a monotonically increasing
