Add possibly useful likely intrinsics

dsharlet · Jan 13, 2025 · 9f95b4a · 9f95b4a
1 parent cd03198
commit 9f95b4a
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 5 deletions.
diff --git a/base/util.h b/base/util.h
@@ -40,6 +40,14 @@ namespace slinky {
 #define SLINKY_NO_STACK_PROTECTOR /* nothing */
 #endif
 
+#if defined(__GNUC__)
+#define SLINKY_LIKELY(condition) (__builtin_expect(!!(condition), 1))
+#define SLINKY_UNLIKELY(condition) (__builtin_expect(!!(condition), 0))
+#else
+#define SLINKY_LIKELY(condition) (!!(condition))
+#define SLINKY_UNLIKELY(condition) (!!(condition))
+#endif
+
 class unreachable {
 public:
   unreachable() = default;

diff --git a/runtime/buffer.cc b/runtime/buffer.cc
@@ -526,16 +526,16 @@ SLINKY_NO_INLINE index_t make_for_each_loops_impl(
     }
 
     // Align the bases for dimensions we will access via linear pointer arithmetic.
-    if (bases[0]) {
+    if (SLINKY_LIKELY(bases[0])) {
       // This function is expected to adjust all bases to point to the min of `buf_dim`. For non-folded dimensions, that
       // is true by construction, but not for folded dimensions.
       index_t offset = buf_dim.flat_offset_bytes(buf_dim.min());
       bases[0] = offset_bytes_non_null(bases[0], offset);
     }
     for (std::size_t n = 1; n < bufs_size; n++) {
-      if (bases[n] && d < static_cast<index_t>(bufs[n]->rank)) {
+      if (SLINKY_LIKELY(bases[n] && d < static_cast<index_t>(bufs[n]->rank))) {
         const dim& buf_n_dim = bufs[n]->dim(d);
-        if (buf_n_dim.contains(buf_dim)) {
+        if (SLINKY_LIKELY(buf_n_dim.contains(buf_dim))) {
           index_t offset = buf_n_dim.flat_offset_bytes(buf_dim.min());
           bases[n] = offset_bytes_non_null(bases[n], offset);
         } else {

diff --git a/runtime/buffer.h b/runtime/buffer.h
@@ -801,7 +801,7 @@ void for_each_impl_linear(const std::array<void*, NumBufs>& bases, const for_eac
     } else {
       for_each_impl(bases_i, loop, f);
     }
-    if (--extent <= 0) break;
+    if (SLINKY_UNLIKELY(--extent <= 0)) break;
     bases_i[0] = offset_bytes_non_null(bases_i[0], strides[0]);
     // This is a critical loop, and it seems we can't trust the compiler to unroll it. These ifs are constexpr.
     if (1 < NumBufs) bases_i[1] = offset_bytes(bases_i[1], strides[1]);
@@ -836,7 +836,7 @@ void for_each_impl_folded(const std::array<void*, NumBufs>& bases, const for_eac
 template <typename F, std::size_t NumBufs>
 SLINKY_ALWAYS_INLINE inline void for_each_impl(
     const std::array<void*, NumBufs>& bases, const for_each_loop<NumBufs>* loop, const F& f) {
-  if (loop->impl == for_each_loop<>::innermost) {
+  if (SLINKY_LIKELY(loop->impl == for_each_loop<>::innermost)) {
     for_each_impl_linear<true>(bases, loop, f);
   } else if (loop->impl == 0) {
     for_each_impl_linear<false>(bases, loop, f);