From 2632ee0c196225b762eab75380c5990f2ff61d53 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sat, 1 Feb 2025 13:18:38 +0100 Subject: [PATCH 1/9] use gcc --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3a986a0..a7cf514 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,9 @@ args = ["-L"] build_args = ["-j"] install_components = ["python_modules"] +[tool.py-build-cmake.cmake.options] +CMAKE_C_COMPILER="gcc-14" +CMAKE_CXX_COMPILER="g++-14" [tool.black] target-version = ["py313"] From 8bd99b3ebff314ee40702a625b3892c52860f215 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sat, 1 Feb 2025 13:43:50 +0100 Subject: [PATCH 2/9] Revert "use gcc" This reverts commit 58e9a5f41d3df44c522e09f2f8ce7658f15a0847. --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a7cf514..3a986a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,9 +64,6 @@ args = ["-L"] build_args = ["-j"] install_components = ["python_modules"] -[tool.py-build-cmake.cmake.options] -CMAKE_C_COMPILER="gcc-14" -CMAKE_CXX_COMPILER="g++-14" [tool.black] target-version = ["py313"] From 65d17cf21839d12289e98b59bfeb22ea8e6d8d23 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sat, 1 Feb 2025 16:06:54 +0100 Subject: [PATCH 3/9] _Atomic --- src/cereggii/atomic_ops.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cereggii/atomic_ops.c b/src/cereggii/atomic_ops.c index b5d2b3c..9e2a200 100644 --- a/src/cereggii/atomic_ops.c +++ b/src/cereggii/atomic_ops.c @@ -97,7 +97,7 @@ CereggiiAtomic_CompareExchangeInt64(int64_t *obj, int64_t expected, int64_t desi inline int CereggiiAtomic_CompareExchangeInt128(__int128_t *obj, __int128_t expected, __int128_t desired) { - return __sync_bool_compare_and_swap_16(obj, expected, desired); + return __sync_bool_compare_and_swap_16((_Atomic(__int128_t) *) obj, expected, desired); } inline int @@ -139,7 +139,7 @@ CereggiiAtomic_CompareExchangeUInt64(uint64_t *obj, uint64_t expected, uint64_t inline int CereggiiAtomic_CompareExchangeUInt128(__uint128_t *obj, __uint128_t expected, __uint128_t desired) { - return __sync_bool_compare_and_swap_16(obj, expected, desired); + return __sync_bool_compare_and_swap_16((_Atomic(__uint128_t) *) obj, expected, desired); } inline int From 7d95e9cd6717acb357397a27afcd11fc58eb43e0 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sat, 1 Feb 2025 16:07:03 +0100 Subject: [PATCH 4/9] fix --- src/cereggii/atomic_dict/node_ops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cereggii/atomic_dict/node_ops.c b/src/cereggii/atomic_dict/node_ops.c index 62e539c..f6eee32 100644 --- a/src/cereggii/atomic_dict/node_ops.c +++ b/src/cereggii/atomic_dict/node_ops.c @@ -49,7 +49,7 @@ AtomicDict_ZoneOf(uint64_t ix, AtomicDict_Meta *meta) #define UPPER_SEED 12923598712359872066ull #define LOWER_SEED 7467732452331123588ull #ifdef __aarch64__ -#define REHASH(x) (uint64_t) (__builtin_arm_crc32d((x), LOWER_SEED) | (((uint64_t) __builtin_arm_crc32d((x), UPPER_SEED)) << 32)) +#define REHASH(x) (uint64_t) (__builtin_aarch64_crc32cx((x), LOWER_SEED) | (__builtin_aarch64_crc32cx((x), UPPER_SEED) << 32)) #else #define REHASH(x) (uint64_t) (__builtin_ia32_crc32di((x), LOWER_SEED) | (__builtin_ia32_crc32di((x), UPPER_SEED) << 32)) #endif // __aarch64__ From d75846d45e1d5623c24fe75a70053708d2c60bc1 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sat, 1 Feb 2025 16:33:30 +0100 Subject: [PATCH 5/9] fix double word cas --- src/cereggii/atomic_ops.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cereggii/atomic_ops.c b/src/cereggii/atomic_ops.c index 9e2a200..b5d2b3c 100644 --- a/src/cereggii/atomic_ops.c +++ b/src/cereggii/atomic_ops.c @@ -97,7 +97,7 @@ CereggiiAtomic_CompareExchangeInt64(int64_t *obj, int64_t expected, int64_t desi inline int CereggiiAtomic_CompareExchangeInt128(__int128_t *obj, __int128_t expected, __int128_t desired) { - return __sync_bool_compare_and_swap_16((_Atomic(__int128_t) *) obj, expected, desired); + return __sync_bool_compare_and_swap_16(obj, expected, desired); } inline int @@ -139,7 +139,7 @@ CereggiiAtomic_CompareExchangeUInt64(uint64_t *obj, uint64_t expected, uint64_t inline int CereggiiAtomic_CompareExchangeUInt128(__uint128_t *obj, __uint128_t expected, __uint128_t desired) { - return __sync_bool_compare_and_swap_16((_Atomic(__uint128_t) *) obj, expected, desired); + return __sync_bool_compare_and_swap_16(obj, expected, desired); } inline int From df99f2ebda01d1cf1551a6b68de0319a1b657942 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sat, 1 Feb 2025 16:45:10 +0100 Subject: [PATCH 6/9] compilation fixes --- src/cereggii/atomic_dict/node_ops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cereggii/atomic_dict/node_ops.c b/src/cereggii/atomic_dict/node_ops.c index f6eee32..62e539c 100644 --- a/src/cereggii/atomic_dict/node_ops.c +++ b/src/cereggii/atomic_dict/node_ops.c @@ -49,7 +49,7 @@ AtomicDict_ZoneOf(uint64_t ix, AtomicDict_Meta *meta) #define UPPER_SEED 12923598712359872066ull #define LOWER_SEED 7467732452331123588ull #ifdef __aarch64__ -#define REHASH(x) (uint64_t) (__builtin_aarch64_crc32cx((x), LOWER_SEED) | (__builtin_aarch64_crc32cx((x), UPPER_SEED) << 32)) +#define REHASH(x) (uint64_t) (__builtin_arm_crc32d((x), LOWER_SEED) | (((uint64_t) __builtin_arm_crc32d((x), UPPER_SEED)) << 32)) #else #define REHASH(x) (uint64_t) (__builtin_ia32_crc32di((x), LOWER_SEED) | (__builtin_ia32_crc32di((x), UPPER_SEED) << 32)) #endif // __aarch64__ From 3d4ec2a50f4479acab5f6ac912356c82b843c16a Mon Sep 17 00:00:00 2001 From: dpdani Date: Tue, 4 Feb 2025 18:11:57 +0100 Subject: [PATCH 7/9] `AtomicDict` now uses regular linear probing instead of Robin Hood hashing --- src/CMakeLists.txt | 2 - src/cereggii/_cereggii.pyi | 1 - src/cereggii/atomic_dict/atomic_dict.c | 36 +- src/cereggii/atomic_dict/blocks.c | 14 +- src/cereggii/atomic_dict/delete.c | 138 +------ src/cereggii/atomic_dict/insert.c | 150 +------- src/cereggii/atomic_dict/lookup.c | 121 ++---- src/cereggii/atomic_dict/meta.c | 63 +-- src/cereggii/atomic_dict/migrate.c | 148 ++------ src/cereggii/atomic_dict/node_ops.c | 401 +------------------- src/cereggii/atomic_dict/node_sizes_table.c | 66 ---- src/cereggii/atomic_dict/robin_hood.c | 145 ------- src/cereggii/cereggii.c | 1 - src/include/atomic_dict.h | 4 - src/include/atomic_dict_internal.h | 175 +-------- tests/test_atomic_dict.py | 113 ------ 16 files changed, 133 insertions(+), 1445 deletions(-) delete mode 100644 src/cereggii/atomic_dict/node_sizes_table.c delete mode 100644 src/cereggii/atomic_dict/robin_hood.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d79bcd4..4f22d0d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -32,8 +32,6 @@ Python3_add_library(_cereggii MODULE "cereggii/atomic_dict/meta.c" "cereggii/atomic_dict/migrate.c" "cereggii/atomic_dict/node_ops.c" - "cereggii/atomic_dict/node_sizes_table.c" - "cereggii/atomic_dict/robin_hood.c" "cereggii/atomic_int/atomic_int.c" "cereggii/atomic_int/handle.c" "cereggii/atomic_event.c" diff --git a/src/cereggii/_cereggii.pyi b/src/cereggii/_cereggii.pyi index ecab38f..e5bf44d 100644 --- a/src/cereggii/_cereggii.pyi +++ b/src/cereggii/_cereggii.pyi @@ -434,7 +434,6 @@ class AtomicDict: `AtomicDict`. This can greatly reduce contention when the keys in the input are repeated. """ - def compact(self) -> None: ... def _debug(self) -> dict: """ Provide some debugging information. diff --git a/src/cereggii/atomic_dict/atomic_dict.c b/src/cereggii/atomic_dict/atomic_dict.c index 4ac4923..58fb3bd 100644 --- a/src/cereggii/atomic_dict/atomic_dict.c +++ b/src/cereggii/atomic_dict/atomic_dict.c @@ -14,7 +14,7 @@ PyObject * AtomicDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) { AtomicDict *self = NULL; - self = PyObject_GC_New(AtomicDict, &AtomicDict_Type); + self = PyObject_GC_New(AtomicDict, type); if (self != NULL) { self->metadata = NULL; self->metadata = (AtomicRef *) AtomicRef_new(&AtomicRef_Type, NULL, NULL); @@ -327,27 +327,18 @@ AtomicDict_UnsafeInsert(AtomicDict_Meta *meta, Py_hash_t hash, uint64_t pos) .index = pos, .tag = hash, }; - uint64_t ix = AtomicDict_Distance0Of(hash, meta); + const uint64_t d0 = AtomicDict_Distance0Of(hash, meta); - for (int probe = 0; probe < meta->max_distance; probe++) { - AtomicDict_ReadNodeAt((ix + probe) % meta->size, &temp, meta); + for (uint64_t distance = 0; distance < SIZE_OF(meta); distance++) { + AtomicDict_ReadNodeAt((d0 + distance) & (SIZE_OF(meta) - 1), &temp, meta); if (temp.node == 0) { - node.distance = probe; - AtomicDict_WriteNodeAt((ix + probe) % meta->size, &node, meta); + AtomicDict_WriteNodeAt((d0 + distance) & (SIZE_OF(meta) - 1), &node, meta); goto done; } - - if (temp.distance < probe) { - // non-atomic robin hood - node.distance = probe; - AtomicDict_WriteNodeAt((ix + probe) % meta->size, &node, meta); - ix = ix + probe - temp.distance; - probe = temp.distance; - node = temp; - } } - // probes exhausted + + // full return -1; done: return 0; @@ -548,18 +539,9 @@ AtomicDict_Debug(AtomicDict *self) PyObject *block_info = NULL; meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); - metadata = Py_BuildValue("{sOsOsOsOsOsOsOsOsOsOsOsOsOsOsO}", + metadata = Py_BuildValue("{sOsOsOsOsOsO}", "log_size\0", Py_BuildValue("B", meta->log_size), "generation\0", Py_BuildValue("n", (Py_ssize_t) meta->generation), - "node_size\0", Py_BuildValue("B", meta->node_size), - "distance_size\0", Py_BuildValue("B", meta->distance_size), - "tag_size\0", Py_BuildValue("B", meta->tag_size), - "node_mask\0", Py_BuildValue("k", meta->node_mask), - "index_mask\0", Py_BuildValue("k", meta->index_mask), - "distance_mask\0", Py_BuildValue("k", meta->distance_mask), - "tag_mask\0", Py_BuildValue("k", meta->tag_mask), - "tombstone\0", Py_BuildValue("k", meta->tombstone.node), - "is_compact\0", Py_BuildValue("B", meta->is_compact), "inserting_block\0", Py_BuildValue("l", meta->inserting_block), "greatest_allocated_block\0", Py_BuildValue("l", meta->greatest_allocated_block), "greatest_deleted_block\0", Py_BuildValue("l", meta->greatest_deleted_block), @@ -572,7 +554,7 @@ AtomicDict_Debug(AtomicDict *self) goto fail; AtomicDict_Node node; - for (uint64_t i = 0; i < meta->size; i++) { + for (uint64_t i = 0; i < SIZE_OF(meta); i++) { AtomicDict_ReadNodeAt(i, &node, meta); PyObject *n = Py_BuildValue("k", node.node); if (n == NULL) diff --git a/src/cereggii/atomic_dict/blocks.c b/src/cereggii/atomic_dict/blocks.c index 3cc38fe..04e50b6 100644 --- a/src/cereggii/atomic_dict/blocks.c +++ b/src/cereggii/atomic_dict/blocks.c @@ -33,7 +33,7 @@ AtomicDictBlock_traverse(AtomicDict_Block *self, visitproc visit, void *arg) for (int i = 0; i < ATOMIC_DICT_ENTRIES_IN_BLOCK; ++i) { entry = self->entries[i]; - if (entry.value == NULL || entry.flags & ENTRY_FLAGS_TOMBSTONE || entry.flags & ENTRY_FLAGS_SWAPPED) + if (entry.value == NULL) continue; Py_VISIT(entry.key); @@ -50,7 +50,7 @@ AtomicDictBlock_clear(AtomicDict_Block *self) for (int i = 0; i < ATOMIC_DICT_ENTRIES_IN_BLOCK; ++i) { entry = self->entries[i]; - if (entry.flags & ENTRY_FLAGS_TOMBSTONE || entry.flags & ENTRY_FLAGS_SWAPPED) + if (entry.value == NULL) continue; self->entries[i].key = NULL; @@ -108,10 +108,10 @@ AtomicDict_GetEmptyEntry(AtomicDict *self, AtomicDict_Meta *meta, AtomicDict_Res CereggiiAtomic_CompareExchangeInt64(&meta->inserting_block, inserting_block, inserting_block + 1); goto reserve_in_inserting_block; // even if the above CAS fails } - if (greatest_allocated_block + 1 >= meta->size >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK) { + if (greatest_allocated_block + 1 >= SIZE_OF(meta) >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK) { return 0; // must grow } - assert(greatest_allocated_block + 1 <= meta->size >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK); + assert(greatest_allocated_block + 1 <= SIZE_OF(meta) >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK); AtomicDict_Block *block = NULL; block = AtomicDictBlock_New(meta); @@ -121,7 +121,7 @@ AtomicDict_GetEmptyEntry(AtomicDict *self, AtomicDict_Meta *meta, AtomicDict_Res block->entries[0].flags = ENTRY_FLAGS_RESERVED; if (CereggiiAtomic_CompareExchangePtr((void **) &meta->blocks[greatest_allocated_block + 1], NULL, block)) { - if (greatest_allocated_block + 2 < meta->size >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK) { + if (greatest_allocated_block + 2 < SIZE_OF(meta) >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK) { CereggiiAtomic_StorePtr((void **) &meta->blocks[greatest_allocated_block + 2], NULL); } CereggiiAtomic_CompareExchangeInt64(&meta->greatest_allocated_block, @@ -144,7 +144,7 @@ AtomicDict_GetEmptyEntry(AtomicDict *self, AtomicDict_Meta *meta, AtomicDict_Res done: assert(entry_loc->entry != NULL); assert(entry_loc->entry->key == NULL); - assert(entry_loc->location < meta->size); + assert(entry_loc->location < SIZE_OF(meta)); return 1; fail: entry_loc->entry = NULL; @@ -178,7 +178,7 @@ AtomicDict_ReadEntry(AtomicDict_Entry *entry_p, AtomicDict_Entry *entry) { entry->flags = entry_p->flags; entry->value = entry_p->value; - if (entry->value == NULL || entry->flags & ENTRY_FLAGS_TOMBSTONE || entry->flags & ENTRY_FLAGS_SWAPPED) { + if (entry->value == NULL) { entry->key = NULL; entry->value = NULL; entry->hash = -1; diff --git a/src/cereggii/atomic_dict/delete.c b/src/cereggii/atomic_dict/delete.c index f8f43fb..72d613a 100644 --- a/src/cereggii/atomic_dict/delete.c +++ b/src/cereggii/atomic_dict/delete.c @@ -6,24 +6,11 @@ #include "atomic_ops.h" -inline int -AtomicDict_IncrementGreatestDeletedBlock(AtomicDict_Meta *meta, int64_t gab, int64_t gdb) -{ - CereggiiAtomic_CompareExchangeInt64(&meta->greatest_deleted_block, gdb, gdb + 1); - - if ((gab - gdb + meta->greatest_refilled_block) * ATOMIC_DICT_ENTRIES_IN_BLOCK <= meta->size * 1 / 3) { - return 1; - } - - return 0; -} - int AtomicDict_Delete(AtomicDict_Meta *meta, PyObject *key, Py_hash_t hash) { AtomicDict_SearchResult result; AtomicDict_Lookup(meta, key, hash, &result); - int should_shrink = 0; if (result.error) goto fail; @@ -44,120 +31,13 @@ AtomicDict_Delete(AtomicDict_Meta *meta, PyObject *key, Py_hash_t hash) Py_DECREF(result.entry.value); result.entry.value = NULL; -// do { -// if (CereggiiAtomic_CompareExchangeUInt8( -// &result.entry_p->flags, -// result.entry.flags, -// result.entry.flags | ENTRY_FLAGS_TOMBSTONE -// )) { -// result.entry.flags |= ENTRY_FLAGS_TOMBSTONE; -// } else { -// // what if swapped? -// AtomicDict_ReadEntry(result.entry_p, &result.entry); -// } -// } while (!(result.entry.flags & ENTRY_FLAGS_TOMBSTONE)); -// -// uint64_t entry_ix = result.node.index; -// AtomicDict_BufferedNodeReader reader; -// AtomicDict_Node temp[16]; -// int begin_write, end_write; -// -// do { -// AtomicDict_LookupEntry(meta, entry_ix, hash, &result); -// assert(!result.error); -// assert(result.found); -// reader.zone = -1; -// AtomicDict_ReadNodesFromZoneStartIntoBuffer(result.position, &reader, meta); -// AtomicDict_CopyNodeBuffers(reader.buffer, temp); -// AtomicDict_RobinHoodDelete(meta, temp, reader.idx_in_buffer); -// AtomicDict_ComputeBeginEndWrite(meta, reader.buffer, temp, &begin_write, &end_write); -// } while (!AtomicDict_AtomicWriteNodesAt(result.position - reader.idx_in_buffer + begin_write, -// end_write - begin_write, -// &reader.buffer[begin_write], &temp[begin_write], meta)); -// - uint64_t block_num; - int64_t gab, gdb; - AtomicDict_EntryLoc swap_loc; - AtomicDict_Entry swap; -// -// recycle_entry: - block_num = AtomicDict_BlockOf(result.node.index); - gab = meta->greatest_allocated_block; - gdb = meta->greatest_deleted_block; -// -// if (gdb > gab) -// goto recycle_entry; - - if (block_num == gdb + 1) { - int all_deleted = 1; + AtomicDict_Node tombstone = { + .index = 0, + .tag = TOMBSTONE(meta), + }; - for (int i = 0; i < ATOMIC_DICT_ENTRIES_IN_BLOCK; ++i) { - swap_loc.location = ((gdb + 1) << ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK) + i; - swap_loc.entry = AtomicDict_GetEntryAt(swap_loc.location, meta); - AtomicDict_ReadEntry(swap_loc.entry, &swap); - - if (block_num == 0 && i == 0) - continue; - - if (!(swap.flags & ENTRY_FLAGS_TOMBSTONE || swap.flags & ENTRY_FLAGS_SWAPPED)) { - all_deleted = 0; - break; - } - } - - if (all_deleted) { - should_shrink = AtomicDict_IncrementGreatestDeletedBlock(meta, gab, gdb); - } - } - - if (block_num > gdb + 1) { -// for (int i = 0; i < ATOMIC_DICT_ENTRIES_IN_BLOCK; ++i) { -// swap_loc.location = ((gdb + 1) << ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK) + -// ((i + hash) % ATOMIC_DICT_ENTRIES_IN_BLOCK); -// swap_loc.entry = AtomicDict_GetEntryAt(swap_loc.location, meta); -// AtomicDict_ReadEntry(swap_loc.entry, &swap); -// -// if (!(swap.value == NULL || swap.flags & ENTRY_FLAGS_TOMBSTONE || swap.flags & ENTRY_FLAGS_SWAPPED)) -// goto swap_found; -// } - -// should_shrink = AtomicDict_IncrementGreatestDeletedBlock(meta, gab, gdb); -// goto recycle_entry; // don't handle failure - -// swap_found: -// result.entry_p->key = swap.key; -// result.entry_p->value = swap.value; // todo: what if value was updated? => use AtomicRef -// result.entry_p->hash = swap.hash; -// if (!CereggiiAtomic_CompareExchangeUInt8( -// &swap_loc.entry->flags, -// swap.flags, -// swap.flags | ENTRY_FLAGS_SWAPPED -// )) { -// AtomicDict_ReadEntry(swap_loc.entry, &swap); -// if (swap.value == NULL || swap.flags & ENTRY_FLAGS_TOMBSTONE || swap.flags & ENTRY_FLAGS_SWAPPED) -// goto recycle_entry; -// } -// -// CereggiiAtomic_StoreUInt8(&result.entry_p->flags, result.entry.flags & ~ENTRY_FLAGS_TOMBSTONE); -// -// AtomicDict_SearchResult swap_search; -// do_swap: -// AtomicDict_LookupEntry(meta, swap_loc.location, swap.hash, &swap_search); -// AtomicDict_Node swapped = { -// .tag = swap_search.node.tag, -// .distance = swap_search.node.distance, -// .index = entry_ix, -// }; -// -// if (!AtomicDict_AtomicWriteNodesAt(swap_search.position, 1, &swap_search.node, &swapped, meta)) { -// goto do_swap; -// } -// swap_loc.entry->key = NULL; -// swap_loc.entry->value = NULL; - } - - if (should_shrink) - return 2; + int ok = AtomicDict_AtomicWriteNodeAt(result.position, &result.node, &tombstone, meta); + assert(ok); return 1; @@ -215,12 +95,6 @@ AtomicDict_DelItem(AtomicDict *self, PyObject *key) goto fail; } - if (deleted == 2) { // should shrink - int success = AtomicDict_Shrink(self); - if (success < 0) - goto fail; - } - Py_DECREF(meta); return 0; diff --git a/src/cereggii/atomic_dict/insert.c b/src/cereggii/atomic_dict/insert.c index da0c09b..6edf999 100644 --- a/src/cereggii/atomic_dict/insert.c +++ b/src/cereggii/atomic_dict/insert.c @@ -61,7 +61,7 @@ AtomicDict_ExpectedUpdateEntry(AtomicDict_Meta *meta, uint64_t entry_ix, if (!*done) { AtomicDict_ReadEntry(entry_p, &entry); - if (*current == NULL || entry.flags & ENTRY_FLAGS_TOMBSTONE || entry.flags & ENTRY_FLAGS_SWAPPED) + if (*current == NULL) return 0; } } while (!*done); @@ -80,85 +80,6 @@ AtomicDict_ExpectedUpdateEntry(AtomicDict_Meta *meta, uint64_t entry_ix, return -1; } -int -AtomicDict_ExpectedInsertOrUpdateCloseToDistance0(AtomicDict_Meta *meta, PyObject *key, Py_hash_t hash, - PyObject *expected, PyObject *desired, PyObject **current, - AtomicDict_EntryLoc *entry_loc, int *must_grow, int *done, - int *expectation, AtomicDict_Node *to_insert, uint64_t distance_0, - int skip_entry_check) -{ - assert(entry_loc != NULL); - AtomicDict_BufferedNodeReader reader; - AtomicDict_Node temp[16]; - int begin_write, end_write; - - beginning: - reader.zone = -1; - AtomicDict_ReadNodesFromZoneStartIntoBuffer(distance_0, &reader, meta); - - for (int i = (int) (distance_0 % meta->nodes_in_zone); i < meta->nodes_in_zone; i++) { - if (reader.buffer[i].node == 0) - goto empty_slot; - - if (!skip_entry_check) { - if (reader.buffer[i].tag != (hash & meta->tag_mask)) { - continue; - } - - int updated = AtomicDict_ExpectedUpdateEntry(meta, reader.buffer[i].index, key, hash, expected, desired, - current, done, expectation); - if (updated < 0) - goto fail; - - if (updated) - return 1; - } - } - return 0; - - empty_slot: - if (expected != NOT_FOUND && expected != ANY) { - *done = 1; - *expectation = 0; - return 0; - } - - AtomicDict_CopyNodeBuffers(reader.buffer, temp); - - to_insert->index = entry_loc->location; - to_insert->distance = 0; - to_insert->tag = hash; - - AtomicDict_RobinHoodResult rhr = - AtomicDict_RobinHoodInsert(meta, temp, to_insert, (int) (distance_0 % meta->nodes_in_zone)); - - if (rhr == grow) { - if (skip_entry_check) { - return 0; - } - - *must_grow = 1; - goto fail; - } - assert(rhr == ok); - - AtomicDict_ComputeBeginEndWrite(meta, reader.buffer, temp, &begin_write, &end_write); - - *done = AtomicDict_AtomicWriteNodesAt( - distance_0 - (distance_0 % meta->nodes_in_zone) + begin_write, end_write - begin_write, - &reader.buffer[begin_write], &temp[begin_write], meta - ); - - if (!*done) { - reader.zone = -1; - goto beginning; - } - - return 1; - fail: - return -1; -} - PyObject * AtomicDict_ExpectedInsertOrUpdate(AtomicDict_Meta *meta, PyObject *key, Py_hash_t hash, @@ -187,36 +108,14 @@ AtomicDict_ExpectedInsertOrUpdate(AtomicDict_Meta *meta, PyObject *key, Py_hash_ uint64_t distance_0 = AtomicDict_Distance0Of(hash, meta); AtomicDict_Node node; - if (expected == NOT_FOUND || expected == ANY) { - // insert fast-path - if (AtomicDict_ReadRawNodeAt(distance_0, meta) == 0) { - assert(entry_loc != NULL); - - node.index = entry_loc->location; - node.distance = 0; - node.tag = hash; - - if (AtomicDict_AtomicWriteNodesAt(distance_0, 1, &meta->zero, &node, meta)) { - return NOT_FOUND; - } - } - } - - beginning: done = 0; expectation = 1; uint64_t distance = 0; PyObject *current = NULL; - uint8_t is_compact = meta->is_compact; AtomicDict_Node to_insert; - if (AtomicDict_ExpectedInsertOrUpdateCloseToDistance0(meta, key, hash, expected, desired, ¤t, entry_loc, - must_grow, &done, &expectation, &to_insert, - distance_0, skip_entry_check) < 0) - goto fail; - while (!done) { - uint64_t ix = (distance_0 + distance) & (meta->size - 1); + uint64_t ix = (distance_0 + distance) & ((1 << meta->log_size) - 1); AtomicDict_ReadNodeAt(ix, &node, meta); if (node.node == 0) { @@ -226,29 +125,16 @@ AtomicDict_ExpectedInsertOrUpdate(AtomicDict_Meta *meta, PyObject *key, Py_hash_ } assert(entry_loc != NULL); - // non-compact insert - if (is_compact) { - CereggiiAtomic_StoreUInt8(&meta->is_compact, 0); - } - to_insert.index = entry_loc->location; - to_insert.distance = meta->max_distance; to_insert.tag = hash; - done = AtomicDict_AtomicWriteNodesAt(ix, 1, &node, &to_insert, meta); + done = AtomicDict_AtomicWriteNodeAt(ix, &node, &to_insert, meta); if (!done) continue; // don't increase distance - } else if (node.node == meta->tombstone.node) { + } else if (node.node == TOMBSTONE(meta)) { // pass - } else if (is_compact && !AtomicDict_NodeIsReservation(&node, meta) && ( - (ix - node.distance > distance_0) - )) { - if (expected != NOT_FOUND && expected != ANY) { - expectation = 0; - break; - } - } else if (node.tag != (hash & meta->tag_mask)) { + } else if (node.tag != (hash & TAG_MASK(meta))) { // pass } else if (!skip_entry_check) { int updated = AtomicDict_ExpectedUpdateEntry(meta, node.index, key, hash, expected, desired, @@ -262,7 +148,7 @@ AtomicDict_ExpectedInsertOrUpdate(AtomicDict_Meta *meta, PyObject *key, Py_hash_ distance++; - if (distance >= meta->size) { + if (distance >= (1 << meta->log_size)) { // traversed the entire dictionary without finding an empty slot *must_grow = 1; goto fail; @@ -272,28 +158,25 @@ AtomicDict_ExpectedInsertOrUpdate(AtomicDict_Meta *meta, PyObject *key, Py_hash_ // assert(expected == ANY => expectation == 1); assert(expected != ANY || expectation == 1); - if (expected != NOT_FOUND && expectation == 0 && meta->is_compact != is_compact) - goto beginning; - if (expectation && expected == NOT_FOUND) { return NOT_FOUND; - } else if (expectation && expected == ANY) { + } + if (expectation && expected == ANY) { if (current == NULL) { return NOT_FOUND; - } else { - return current; } - } else if (expectation) { + return current; + } + if (expectation) { assert(current != NULL); // no need to incref: // - should incref because it's being returned // - should decref because it has just been removed from the dict return current; - } else { - return EXPECTATION_FAILED; } + return EXPECTATION_FAILED; - fail: +fail: return NULL; } @@ -417,7 +300,7 @@ AtomicDict_CompareAndSet(AtomicDict *self, PyObject *key, PyObject *expected, Py goto fail; if (must_grow || (meta->greatest_allocated_block - meta->greatest_deleted_block + meta->greatest_refilled_block) * - ATOMIC_DICT_ENTRIES_IN_BLOCK >= meta->size * 2 / 3) { + ATOMIC_DICT_ENTRIES_IN_BLOCK >= SIZE_OF(meta) * 2 / 3) { migrated = AtomicDict_Grow(self); if (migrated < 0) @@ -557,10 +440,7 @@ reduce_flush(AtomicDict *self, PyObject *local_buffer, PyObject *aggregate) if (hash == -1) goto fail; - __builtin_prefetch(AtomicDict_IndexAddressOf( - AtomicDict_Distance0Of(hash, meta), - meta - )); + __builtin_prefetch(&meta->index[AtomicDict_Distance0Of(hash, meta)]); } for (int i = 0; i < num_items; ++i) { diff --git a/src/cereggii/atomic_dict/lookup.c b/src/cereggii/atomic_dict/lookup.c index 1c275cd..7729ed6 100644 --- a/src/cereggii/atomic_dict/lookup.c +++ b/src/cereggii/atomic_dict/lookup.c @@ -14,44 +14,24 @@ AtomicDict_Lookup(AtomicDict_Meta *meta, PyObject *key, Py_hash_t hash, AtomicDict_SearchResult *result) { // caller must ensure PyObject_Hash(.) didn't raise an error - uint64_t ix = AtomicDict_Distance0Of(hash, meta); - uint8_t is_compact; - uint64_t probe, reservations; - AtomicDict_Node node; - - beginning: - is_compact = meta->is_compact; - reservations = 0; - - for (probe = 0; probe < meta->size; probe++) { - AtomicDict_ReadNodeAt(ix + probe + reservations, &node, meta); - - if (AtomicDict_NodeIsReservation(&node, meta)) { - probe--; - reservations++; - result->is_reservation = 1; - goto check_entry; - } - result->is_reservation = 0; + const uint64_t d0 = AtomicDict_Distance0Of(hash, meta); + uint64_t distance = 0; - if (node.node == 0) { - goto not_found; - } + for (; distance < (1 << meta->log_size); distance++) { + AtomicDict_ReadNodeAt(d0 + distance, &result->node, meta); - if ( - is_compact && ( - (ix + probe + reservations - node.distance > ix) - || (probe >= meta->max_distance) - )) { + if (result->node.node == 0) { goto not_found; } - check_entry: - if (node.tag == (hash & meta->tag_mask)) { - result->entry_p = AtomicDict_GetEntryAt(node.index, meta); + if (result->node.node == TOMBSTONE(meta)) + continue; + + if (result->node.tag == (hash & TAG_MASK(meta))) { + result->entry_p = AtomicDict_GetEntryAt(result->node.index, meta); AtomicDict_ReadEntry(result->entry_p, &result->entry); - if (result->entry.value == NULL || result->entry.flags & ENTRY_FLAGS_TOMBSTONE) { + if (result->entry.value == NULL) { continue; } if (result->entry.key == key) { @@ -60,21 +40,22 @@ AtomicDict_Lookup(AtomicDict_Meta *meta, PyObject *key, Py_hash_t hash, if (result->entry.hash != hash) { continue; } + int cmp = PyObject_RichCompareBool(result->entry.key, key, Py_EQ); if (cmp < 0) { // exception thrown during compare goto error; - } else if (cmp == 0) { + } + if (cmp == 0) { continue; } goto found; } - } // probes exhausted + } + + // have looped over the entire index without finding the key => not found not_found: - if (is_compact != meta->is_compact) { - goto beginning; - } result->error = 0; result->found = 0; result->entry_p = NULL; @@ -85,8 +66,7 @@ AtomicDict_Lookup(AtomicDict_Meta *meta, PyObject *key, Py_hash_t hash, found: result->error = 0; result->found = 1; - result->position = (ix + probe + reservations) & (meta->size - 1); - result->node = node; + result->position = (d0 + distance) & ((1 << meta->log_size) - 1); } void @@ -95,62 +75,30 @@ AtomicDict_LookupEntry(AtomicDict_Meta *meta, uint64_t entry_ix, Py_hash_t hash, { // index-only search - uint64_t ix = AtomicDict_Distance0Of(hash, meta); - uint8_t is_compact; - uint64_t probe, reservations; - AtomicDict_Node node; - - beginning: - is_compact = meta->is_compact; - reservations = 0; + const uint64_t d0 = AtomicDict_Distance0Of(hash, meta); + uint64_t distance = 0; - for (probe = 0; probe < meta->size; probe++) { - AtomicDict_ReadNodeAt(ix + probe + reservations, &node, meta); + for (; distance < (1 << meta->log_size); distance++) { + AtomicDict_ReadNodeAt(d0 + distance, &result->node, meta); - if (AtomicDict_NodeIsTombstone(&node, meta)) { - probe--; - reservations++; - continue; - } - - if (AtomicDict_NodeIsReservation(&node, meta)) { - probe--; - reservations++; - result->is_reservation = 1; - goto check_entry; - } - result->is_reservation = 0; - - if (node.node == 0) { - goto not_found; - } - - if ( - is_compact && ( - (ix + probe + reservations - node.distance > ix) - || (probe >= meta->max_distance) - )) { + if (result->node.node == 0) { goto not_found; } check_entry: - if (node.index == entry_ix) { + if (result->node.index == entry_ix) { goto found; } } // probes exhausted not_found: - if (is_compact != meta->is_compact) { - goto beginning; - } result->error = 0; result->found = 0; return; found: result->error = 0; result->found = 1; - result->position = (ix + probe + reservations) & (meta->size - 1); - result->node = node; + result->position = (d0 + distance) & ((1 << meta->log_size) - 1); } @@ -275,7 +223,7 @@ AtomicDict_BatchGetItem(AtomicDict *self, PyObject *args, PyObject *kwargs) hashes[(chunk_end - 1) % chunk_size] = hash; keys[(chunk_end - 1) % chunk_size] = key; - __builtin_prefetch(AtomicDict_IndexAddressOf(AtomicDict_Distance0Of(hash, meta), meta)); + __builtin_prefetch(&meta->index[AtomicDict_Distance0Of(hash, meta)]); if (chunk_end % chunk_size == 0) break; @@ -288,19 +236,16 @@ AtomicDict_BatchGetItem(AtomicDict *self, PyObject *args, PyObject *kwargs) uint64_t d0 = AtomicDict_Distance0Of(hash, meta); AtomicDict_Node node; - for (uint64_t j = d0; j < d0 + meta->max_distance; ++j) { - AtomicDict_ReadNodeAt(j, &node, meta); + AtomicDict_ReadNodeAt(d0, &node, meta); - if (node.node == 0) - break; + if (node.node == 0) + continue; - if (AtomicDict_NodeIsTombstone(&node, meta)) - continue; + if (node.node == TOMBSTONE(meta)) + continue; - if (node.tag == (hash & meta->tag_mask)) { - __builtin_prefetch(AtomicDict_GetEntryAt(node.index, meta)); - break; - } + if (node.tag == (hash & TAG_MASK(meta))) { + __builtin_prefetch(AtomicDict_GetEntryAt(node.index, meta)); } } diff --git a/src/cereggii/atomic_dict/meta.c b/src/cereggii/atomic_dict/meta.c index bbdb176..e89d9aa 100644 --- a/src/cereggii/atomic_dict/meta.c +++ b/src/cereggii/atomic_dict/meta.c @@ -16,15 +16,11 @@ AtomicDictMeta_New(uint8_t log_size) uint64_t *index = NULL; AtomicDict_Meta *meta = NULL; - AtomicDict_NodeSizeInfo node_sizes = AtomicDict_NodeSizesTable[log_size]; - generation = PyMem_RawMalloc(1); if (generation == NULL) goto fail; - uint64_t index_size = node_sizes.node_size / 8; - index_size *= (uint64_t) 1 << log_size; - index = PyMem_RawMalloc(index_size); + index = PyMem_RawMalloc(sizeof(uint64_t) * (1 << log_size)); if (index == NULL) goto fail; @@ -39,62 +35,13 @@ AtomicDictMeta_New(uint8_t log_size) meta->inserting_block = -1; meta->log_size = log_size; - meta->size = 1UL << log_size; meta->generation = generation; meta->index = index; - meta->is_compact = 1; - meta->node_size = node_sizes.node_size; - meta->distance_size = node_sizes.distance_size; - meta->max_distance = (1 << meta->distance_size) - 1; - meta->tag_size = node_sizes.tag_size; - meta->nodes_in_region = 8 / (meta->node_size / 8); //#if defined(__aarch64__) && !defined(__APPLE__) // meta->nodes_in_zone = 8 / (meta->node_size / 8); //#else // meta->nodes_in_zone = 16 / (meta->node_size / 8); //#endif - meta->nodes_in_zone = 16 / (meta->node_size / 8); - if (meta->node_size == 64) { - meta->node_mask = ULONG_MAX; - } else { - meta->node_mask = (1UL << node_sizes.node_size) - 1; - } - meta->index_mask = ((1UL << log_size) - 1) << (node_sizes.node_size - log_size); - meta->distance_mask = ((1UL << node_sizes.distance_size) - 1) << node_sizes.tag_size; - meta->tag_mask = (Py_hash_t) (1UL << node_sizes.tag_size) - 1; - meta->d0_shift = SIZEOF_PY_HASH_T * CHAR_BIT - meta->log_size; - switch (node_sizes.node_size) { - case 8: - meta->shift_mask = 8 - 1; - meta->read_nodes_in_region = AtomicDict_Read8NodesAt; - meta->read_nodes_in_zone = AtomicDict_Read16NodesAt; - break; - case 16: - meta->shift_mask = 4 - 1; - meta->read_nodes_in_region = AtomicDict_Read4NodesAt; - meta->read_nodes_in_zone = AtomicDict_Read8NodesAt; - break; - case 32: - meta->shift_mask = 2 - 1; - meta->read_nodes_in_region = AtomicDict_Read2NodesAt; - meta->read_nodes_in_zone = AtomicDict_Read4NodesAt; - break; - case 64: - meta->shift_mask = 1 - 1; - meta->read_nodes_in_region = AtomicDict_Read1NodeAt; - meta->read_nodes_in_zone = AtomicDict_Read2NodesAt; - break; - } - - meta->tombstone.distance = 0; - meta->tombstone.index = 0; - meta->tombstone.tag = 0; - AtomicDict_ComputeRawNode(&meta->tombstone, meta); - - meta->zero.distance = meta->max_distance; - meta->zero.index = 0; - meta->zero.tag = 0; - AtomicDict_ComputeRawNode(&meta->zero, meta); meta->new_gen_metadata = NULL; meta->migration_leader = 0; @@ -126,7 +73,7 @@ AtomicDictMeta_New(uint8_t log_size) void AtomicDictMeta_ClearIndex(AtomicDict_Meta *meta) { - memset(meta->index, 0, meta->node_size / 8 * meta->size); + memset(meta->index, 0, sizeof(uint64_t) * SIZE_OF(meta)); } int @@ -135,7 +82,7 @@ AtomicDictMeta_InitBlocks(AtomicDict_Meta *meta) AtomicDict_Block **blocks = NULL; // here we're abusing virtual memory: // the entire array will not necessarily be allocated to physical memory. - blocks = PyMem_RawMalloc(sizeof(AtomicDict_Block *) * (meta->size >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK)); + blocks = PyMem_RawMalloc(sizeof(AtomicDict_Block *) * (SIZE_OF(meta) >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK)); if (blocks == NULL) goto fail; @@ -157,7 +104,7 @@ AtomicDictMeta_CopyBlocks(AtomicDict_Meta *from_meta, AtomicDict_Meta *to_meta) { assert(from_meta != NULL); assert(to_meta != NULL); - assert(from_meta->size <= to_meta->size); + assert(from_meta->log_size <= to_meta->log_size); AtomicDict_Block **previous_blocks = from_meta->blocks; int64_t inserting_block = from_meta->inserting_block; @@ -169,7 +116,7 @@ AtomicDictMeta_CopyBlocks(AtomicDict_Meta *from_meta, AtomicDict_Meta *to_meta) // here we're abusing virtual memory: // the entire array will not necessarily be allocated to physical memory. AtomicDict_Block **blocks = PyMem_RawMalloc(sizeof(AtomicDict_Block *) * - (to_meta->size >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK)); + (SIZE_OF(to_meta) >> ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK)); if (blocks == NULL) goto fail; diff --git a/src/cereggii/atomic_dict/migrate.c b/src/cereggii/atomic_dict/migrate.c index 69d16c6..80bc94a 100644 --- a/src/cereggii/atomic_dict/migrate.c +++ b/src/cereggii/atomic_dict/migrate.c @@ -32,81 +32,6 @@ AtomicDict_Grow(AtomicDict *self) return -1; } -int -AtomicDict_Shrink(AtomicDict *self) -{ - AtomicDict_Meta *meta = NULL; - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); - if (meta == NULL) - goto fail; - - if (meta->log_size == self->min_log_size) { - Py_DECREF(meta); - return 0; - } - - int migrate = AtomicDict_Migrate(self, meta, meta->log_size, meta->log_size - 1); - if (migrate < 0) - goto fail; - - Py_DECREF(meta); - return migrate; - - fail: - Py_XDECREF(meta); - return -1; -} - -int -AtomicDict_Compact(AtomicDict *self) -{ - int migrate, is_compact = 1; - AtomicDict_Meta *meta = NULL; - - do { - /** - * do one migration with equal log_sizes first to reduce the blocks. - * then, if the resulting meta is not compact, increase the log_size - * and migrate, until meta is compact. - */ - - meta = NULL; - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); - if (meta == NULL) - goto fail; - - migrate = AtomicDict_Migrate(self, meta, meta->log_size, meta->log_size + !is_compact); - if (migrate < 0) - goto fail; - - Py_DECREF(meta); - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); - is_compact = meta->is_compact; - Py_DECREF(meta); - - } while (!is_compact); - - return migrate; - - fail: - Py_XDECREF(meta); - return -1; -} - -PyObject * -AtomicDict_Compact_callable(AtomicDict *self) -{ - int migrate = AtomicDict_Compact(self); - - if (migrate < 0) { - PyErr_SetString(PyExc_RuntimeError, "error during compaction."); - return NULL; - } - - Py_RETURN_NONE; -} - - int AtomicDict_MaybeHelpMigrate(AtomicDict_Meta *current_meta, PyMutex *self_mutex) { @@ -318,8 +243,7 @@ AtomicDict_MigrateReInsertAll(AtomicDict_Meta *current_meta, AtomicDict_Meta *ne entry_loc.location = (lock << ATOMIC_DICT_LOG_ENTRIES_IN_BLOCK) + i; entry_loc.entry = AtomicDict_GetEntryAt(entry_loc.location, new_meta); - if (entry_loc.entry->key == NULL || entry_loc.entry->value == NULL || - entry_loc.entry->flags & ENTRY_FLAGS_TOMBSTONE || entry_loc.entry->flags & ENTRY_FLAGS_SWAPPED) + if (entry_loc.entry->key == NULL || entry_loc.entry->value == NULL) continue; int must_grow; @@ -354,7 +278,7 @@ AtomicDict_IndexNotFound(uint64_t index, AtomicDict_Meta *meta) { AtomicDict_Node node; - for (uint64_t i = 0; i < meta->size; ++i) { + for (uint64_t i = 0; i < SIZE_OF(meta); ++i) { AtomicDict_ReadNodeAt(i, &node, meta); if (node.index == index) { @@ -371,10 +295,10 @@ AtomicDict_DebugLen(AtomicDict_Meta *meta) uint64_t len = 0; AtomicDict_Node node; - for (uint64_t i = 0; i < meta->size; ++i) { + for (uint64_t i = 0; i < SIZE_OF(meta); ++i) { AtomicDict_ReadNodeAt(i, &node, meta); - if (node.node != 0 && !AtomicDict_NodeIsTombstone(&node, meta)) + if (node.node != 0 && node.node != TOMBSTONE(meta)) len++; } @@ -382,53 +306,30 @@ AtomicDict_DebugLen(AtomicDict_Meta *meta) } inline void -AtomicDict_MigrateNode(AtomicDict_Node *node, uint64_t *distance_0, uint64_t *distance, AtomicDict_Meta *new_meta, - uint64_t size_mask) +AtomicDict_MigrateNode(AtomicDict_Node *node, AtomicDict_Meta *new_meta) { // assert(AtomicDict_IndexNotFound(node->index, new_meta)); Py_hash_t hash = AtomicDict_GetEntryAt(node->index, new_meta)->hash; - uint64_t ix; uint64_t d0 = AtomicDict_Distance0Of(hash, new_meta); - - if (d0 != *distance_0) { - *distance_0 = d0; - *distance = 0; - } - node->tag = hash; + uint64_t position; - if (!new_meta->is_compact) { - do { - ix = (d0 + *distance) & size_mask; - (*distance)++; - } while (AtomicDict_ReadRawNodeAt(ix, new_meta) != 0); - - node->distance = new_meta->max_distance; - - assert(AtomicDict_ReadRawNodeAt(ix, new_meta) == 0); - AtomicDict_WriteNodeAt(ix, node, new_meta); - - return; - } - - AtomicDict_RobinHoodResult inserted = AtomicDict_RobinHoodInsertRaw(new_meta, node, (int64_t) d0); - - assert(inserted != failed); + for (uint64_t distance = 0; distance < SIZE_OF(new_meta); distance++) { + position = (d0 + distance) & (SIZE_OF(new_meta) - 1); - if (inserted == grow) { - new_meta->is_compact = 0; + if (new_meta->index[position] == 0) { + AtomicDict_WriteNodeAt(position, node, new_meta); + break; + } } - - (*distance)++; } #define ATOMIC_DICT_BLOCKWISE_MIGRATE_SIZE 4096 -//#define ATOMIC_DICT_BLOCKWISE_MIGRATE_SIZE 64 void AtomicDict_BlockWiseMigrate(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_meta, int64_t start_of_block) { - uint64_t current_size = current_meta->size; + uint64_t current_size = SIZE_OF(current_meta); uint64_t current_size_mask = current_size - 1; uint64_t i = start_of_block; @@ -442,7 +343,7 @@ AtomicDict_BlockWiseMigrate(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_ // find first empty slot while (i < end_of_block) { - if (AtomicDict_ReadRawNodeAt(i, current_meta) == 0) + if (current_meta->index[i] == 0) break; i++; @@ -452,32 +353,27 @@ AtomicDict_BlockWiseMigrate(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_ return; // initialize slots in range [i, end_of_block] - memset(AtomicDict_IndexAddressOf(i * 2, new_meta), 0, (end_of_block - i) * 2 * new_meta->node_size / 8); + memset(&new_meta->index[i * 2], 0, (end_of_block - i) * 2 * sizeof(uint64_t)); - uint64_t new_size_mask = new_meta->size - 1; - uint64_t distance = 0; - uint64_t distance_0 = ULLONG_MAX; + uint64_t new_size_mask = SIZE_OF(new_meta) - 1; for (; i < end_of_block; i++) { AtomicDict_ReadNodeAt(i, &node, current_meta); - if (node.node == 0 || AtomicDict_NodeIsTombstone(&node, current_meta)) + if (node.node == 0 || node.node == TOMBSTONE(new_meta)) continue; - AtomicDict_MigrateNode(&node, &distance_0, &distance, new_meta, new_size_mask); + AtomicDict_MigrateNode(&node, new_meta); } assert(i == end_of_block); while (node.node != 0 || i == end_of_block) { - memset( - AtomicDict_IndexAddressOf((i * 2) & new_size_mask, new_meta), 0, - ((i + 1) * 2 - (i * 2)) * new_meta->node_size / 8 - ); + memset(&new_meta->index[(i * 2) & new_size_mask], 0, ((i + 1) * 2 - i * 2) * sizeof(uint64_t)); AtomicDict_ReadNodeAt(i & current_size_mask, &node, current_meta); - if (node.node != 0 && !AtomicDict_NodeIsTombstone(&node, current_meta)) { - AtomicDict_MigrateNode(&node, &distance_0, &distance, new_meta, new_size_mask); + if (node.node != 0 && node.node != TOMBSTONE(current_meta)) { + AtomicDict_MigrateNode(&node, new_meta); } i++; @@ -487,7 +383,7 @@ AtomicDict_BlockWiseMigrate(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_ int AtomicDict_MigrateNodes(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_meta) { - uint64_t current_size = current_meta->size; + uint64_t current_size = SIZE_OF(current_meta); int64_t node_to_migrate = CereggiiAtomic_AddInt64(¤t_meta->node_to_migrate, ATOMIC_DICT_BLOCKWISE_MIGRATE_SIZE); diff --git a/src/cereggii/atomic_dict/node_ops.c b/src/cereggii/atomic_dict/node_ops.c index 62e539c..5a4d9bf 100644 --- a/src/cereggii/atomic_dict/node_ops.c +++ b/src/cereggii/atomic_dict/node_ops.c @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +#include #include "atomic_dict_internal.h" #include "atomic_ops.h" @@ -13,37 +14,10 @@ inline void AtomicDict_ComputeRawNode(AtomicDict_Node *node, AtomicDict_Meta *meta) { - assert(node->index < meta->size); - assert(node->distance <= meta->max_distance); + assert(node->index < (1 << meta->log_size)); node->node = - (node->index << (meta->node_size - meta->log_size)) - | (meta->distance_mask - ((uint64_t) node->distance << (uint64_t) meta->tag_size)) - | (node->tag & meta->tag_mask); -} - -inline int -AtomicDict_NodeIsReservation(AtomicDict_Node *node, AtomicDict_Meta *meta) -{ - return node->node != 0 && node->distance == (1 << meta->distance_size) - 1; -} - -inline int -AtomicDict_NodeIsTombstone(AtomicDict_Node *node, AtomicDict_Meta *meta) -{ - return node->node == meta->tombstone.node; -} - -inline uint64_t -AtomicDict_RegionOf(uint64_t ix, AtomicDict_Meta *meta) -{ - ix = ix & (meta->size - 1); - return (ix & ~meta->shift_mask) / meta->nodes_in_region; -} - -inline uint64_t -AtomicDict_ZoneOf(uint64_t ix, AtomicDict_Meta *meta) -{ - return AtomicDict_RegionOf(ix, meta) & (ULONG_MAX - 1UL); + (node->index << (NODE_SIZE - meta->log_size)) + | (node->tag & TAG_MASK(meta)); } #define UPPER_SEED 12923598712359872066ull @@ -64,27 +38,7 @@ AtomicDict_ReHash(AtomicDict *Py_UNUSED(self), PyObject *ob) inline uint64_t AtomicDict_Distance0Of(Py_hash_t hash, AtomicDict_Meta *meta) { - return REHASH(hash) >> meta->d0_shift; -} - -inline uint64_t -AtomicDict_ShiftInRegionOf(uint64_t ix, AtomicDict_Meta *meta) -{ - return ix & meta->shift_mask; -} - -inline uint8_t * -AtomicDict_IndexAddressOf(uint64_t ix, AtomicDict_Meta *meta) -{ - uint64_t shift = AtomicDict_ShiftInRegionOf(ix, meta); - uint64_t region = AtomicDict_RegionOf(ix, meta); - return (uint8_t *) &meta->index[region] + shift * (meta->node_size / 8); -} - -inline int -AtomicDict_IndexAddressIsAligned(uint64_t ix, int alignment, AtomicDict_Meta *meta) -{ - return (int64_t) AtomicDict_IndexAddressOf(ix, meta) % alignment == 0; + return REHASH(hash) >> (SIZEOF_PY_HASH_T * CHAR_BIT - meta->log_size); } inline void @@ -92,256 +46,17 @@ AtomicDict_ParseNodeFromRaw(uint64_t node_raw, AtomicDict_Node *node, AtomicDict_Meta *meta) { node->node = node_raw; - node->index = (node_raw & meta->index_mask) >> (meta->node_size - meta->log_size); - node->distance = (~(node_raw & meta->distance_mask) & meta->distance_mask) >> meta->tag_size; - node->tag = node_raw & meta->tag_mask; -} - - -inline void -AtomicDict_ParseNodeFromRegion(uint64_t ix, uint64_t region, AtomicDict_Node *node, - AtomicDict_Meta *meta) -{ - uint64_t shift = AtomicDict_ShiftInRegionOf(ix, meta); - uint64_t node_raw = - (region & (meta->node_mask << (shift * meta->node_size))) >> (shift * meta->node_size); - AtomicDict_ParseNodeFromRaw(node_raw, node, meta); -} - -inline uint64_t -AtomicDict_ParseRawNodeFromRegion(uint64_t ix, uint64_t region, AtomicDict_Meta *meta) -{ - uint64_t shift = AtomicDict_ShiftInRegionOf(ix, meta); - uint64_t node_raw = - (region & (meta->node_mask << (shift * meta->node_size))) >> (shift * meta->node_size); - return node_raw; -} - -inline void -AtomicDict_CopyNodeBuffers(AtomicDict_Node *from_buffer, AtomicDict_Node *to_buffer) -{ - for (int i = 0; i < 16; ++i) { - to_buffer[i] = from_buffer[i]; - } -} - -inline void -AtomicDict_ComputeBeginEndWrite(AtomicDict_Meta *meta, AtomicDict_Node *read_buffer, AtomicDict_Node *temp, - int *begin_write, int *end_write) -{ - int j; - *begin_write = -1; - for (j = 0; j < meta->nodes_in_zone; ++j) { - AtomicDict_ComputeRawNode(&temp[j], meta); - if (temp[j].node != read_buffer[j].node) { - *begin_write = j; - break; - } - } - assert(*begin_write != -1); - *end_write = -1; - for (j = meta->nodes_in_zone - 1; j > *begin_write; --j) { - AtomicDict_ComputeRawNode(&temp[j], meta); - if (temp[j].node != read_buffer[j].node) { - *end_write = j + 1; - break; - } - } - if (*end_write == -1) { - *end_write = meta->nodes_in_zone; - } - assert(*end_write > *begin_write); - - int n = *end_write - *begin_write; - int must_write = AtomicDict_MustWriteBytes(n, meta); - int must_write_nodes = must_write / (meta->node_size / 8); - - if (n != must_write_nodes || !AtomicDict_IndexAddressIsAligned(*begin_write, n, meta)) { - // no need to check for alignment from the beginning of the index, since - // we already assume that begin_write = 0 is already 16-bytes aligned. - - if (must_write_nodes == meta->nodes_in_zone) { - *begin_write = 0; - *end_write = meta->nodes_in_zone; - } else { - if (*begin_write <= meta->nodes_in_region && *end_write <= meta->nodes_in_region) { - *begin_write = 0; - *end_write = meta->nodes_in_region; - } else if (*begin_write >= meta->nodes_in_region && *end_write >= meta->nodes_in_region) { - *begin_write = meta->nodes_in_region; - *end_write = meta->nodes_in_zone; - } else { - // fallback to entire zone - *begin_write = 0; - *end_write = meta->nodes_in_zone; - } - } - } + node->index = node_raw >> (NODE_SIZE - meta->log_size); + node->tag = node_raw & TAG_MASK(meta); } inline void AtomicDict_ReadNodeAt(uint64_t ix, AtomicDict_Node *node, AtomicDict_Meta *meta) { - uint64_t node_region = meta->index[AtomicDict_RegionOf(ix, meta)]; - AtomicDict_ParseNodeFromRegion(ix, node_region, node, meta); -} - -inline int64_t -AtomicDict_ReadRawNodeAt(uint64_t ix, AtomicDict_Meta *meta) -{ - uint64_t node_region = meta->index[AtomicDict_RegionOf(ix, meta)]; - return (int64_t) AtomicDict_ParseRawNodeFromRegion(ix, node_region, meta); -} - -/** - * the following functions read more than one node at a time. - * NB: they expect all the nodes to be in two successive words of memory! - * this means that in order to call e.g. AtomicDict_Read8NodesAt, - * meta->node_size must be at most 16. - **/ - -void -AtomicDict_Read1NodeAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta) -{ - uint64_t node_region = meta->index[AtomicDict_RegionOf(ix, meta)]; - AtomicDict_ParseNodeFromRegion(ix, node_region, &nodes[0], meta); -} - -void -AtomicDict_Read2NodesAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta) -{ - uint64_t little = AtomicDict_RegionOf(ix, meta); - uint64_t big = AtomicDict_RegionOf(ix + 1, meta); - uint64_t node_region_little = meta->index[little]; - uint64_t node_region_big = node_region_little; - if (big != little) { - node_region_big = meta->index[big]; - } - AtomicDict_ParseNodeFromRegion(ix, node_region_little, &nodes[0], meta); - AtomicDict_ParseNodeFromRegion(ix + 1, node_region_big, &nodes[1], meta); -} - -void -AtomicDict_Read4NodesAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta) -{ - assert(meta->node_size <= 32); - uint64_t little = AtomicDict_RegionOf(ix, meta); - uint64_t middle = AtomicDict_RegionOf(ix + 2, meta); - uint64_t big = AtomicDict_RegionOf(ix + 3, meta); - uint64_t node_region_little = meta->index[little]; - uint64_t node_region_middle = node_region_little; - uint64_t node_region_big = node_region_little; - if (middle != little) { - node_region_middle = meta->index[middle]; - } - if (big != little) { - node_region_big = meta->index[big]; - } - - uint64_t region; - for (int i = 0; i < 4; ++i) { - uint64_t r = AtomicDict_RegionOf(ix + i, meta); - if (r == little) { - region = node_region_little; - } else if (r == middle) { - region = node_region_middle; - } else { - region = node_region_big; - } - AtomicDict_ParseNodeFromRegion(ix + i, region, &nodes[i], meta); - } -} - -void -AtomicDict_Read8NodesAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta) -{ - assert(meta->node_size <= 16); - uint64_t little = AtomicDict_RegionOf(ix, meta); - uint64_t middle = AtomicDict_RegionOf(ix + 4, meta); - uint64_t big = AtomicDict_RegionOf(ix + 7, meta); - uint64_t node_region_little = meta->index[little]; - uint64_t node_region_middle = node_region_little; - uint64_t node_region_big = node_region_little; - if (middle != little) { - node_region_middle = meta->index[middle]; - } - if (big != little) { - node_region_big = meta->index[big]; - } - - uint64_t region; - for (int i = 0; i < 8; ++i) { - uint64_t r = AtomicDict_RegionOf(ix + i, meta); - if (r == little) { - region = node_region_little; - } else if (r == middle) { - region = node_region_middle; - } else { - region = node_region_big; - } - AtomicDict_ParseNodeFromRegion(ix + i, region, &nodes[i], meta); - } -} - -void -AtomicDict_Read16NodesAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta) -{ - assert(meta->node_size == 8); - uint64_t little = AtomicDict_RegionOf(ix, meta); - uint64_t middle = AtomicDict_RegionOf(ix + 8, meta); - uint64_t big = AtomicDict_RegionOf(ix + 15, meta); - uint64_t node_region_little = meta->index[little]; - uint64_t node_region_middle = node_region_little; - uint64_t node_region_big = node_region_little; - if (middle != little) { - node_region_middle = meta->index[middle]; - } - if (big != little && big != middle) { - node_region_big = meta->index[big]; - } - - uint64_t region; - for (int i = 0; i < 16; ++i) { - uint64_t r = AtomicDict_RegionOf(ix + i, meta); - if (r == little) { - region = node_region_little; - } else if (r == middle) { - region = node_region_middle; - } else { - region = node_region_big; - } - AtomicDict_ParseNodeFromRegion(ix + i, region, &nodes[i], meta); - } -} - -inline void -AtomicDict_ReadNodesFromZoneIntoBuffer(uint64_t idx, AtomicDict_BufferedNodeReader *reader, AtomicDict_Meta *meta) -{ - idx &= (int64_t) (meta->size - 1); - uint64_t zone = AtomicDict_ZoneOf(idx, meta); - - if (reader->zone != zone) { - meta->read_nodes_in_zone(idx, reader->buffer, meta); - reader->zone = (int64_t) zone; - reader->nodes_offset = (int) -(idx % meta->nodes_in_zone); - } - - reader->idx_in_buffer = (int) (idx % meta->nodes_in_zone + reader->nodes_offset); - assert(reader->idx_in_buffer < meta->nodes_in_zone); - reader->node = reader->buffer[reader->idx_in_buffer]; -} - -inline void -AtomicDict_ReadNodesFromZoneStartIntoBuffer(uint64_t idx, AtomicDict_BufferedNodeReader *reader, - AtomicDict_Meta *meta) -{ - AtomicDict_ReadNodesFromZoneIntoBuffer(idx / meta->nodes_in_zone * meta->nodes_in_zone, - reader, meta); - reader->idx_in_buffer = (int) (idx % meta->nodes_in_zone + reader->nodes_offset); - reader->node = reader->buffer[reader->idx_in_buffer]; + const uint64_t raw = meta->index[ix & ((1 << meta->log_size) - 1)]; + AtomicDict_ParseNodeFromRaw(raw, node, meta); } - inline void AtomicDict_WriteNodeAt(uint64_t ix, AtomicDict_Node *node, AtomicDict_Meta *meta) { @@ -353,102 +68,16 @@ inline void AtomicDict_WriteRawNodeAt(uint64_t ix, uint64_t raw_node, AtomicDict_Meta *meta) { assert(ix >= 0); - assert(ix < meta->size); + assert(ix < (1 << meta->log_size)); - uint64_t shift = AtomicDict_ShiftInRegionOf(ix, meta); - assert(shift < meta->nodes_in_region); - uint64_t region = AtomicDict_RegionOf(ix, meta); - assert(region < meta->size / meta->nodes_in_region); - uint64_t *region_address = &meta->index[region]; - - switch (meta->node_size) { - case 8: - *((uint8_t *) region_address + shift) = raw_node; - break; - case 16: - *((uint16_t *) region_address + shift) = raw_node; - break; - case 32: - *((uint32_t *) region_address + shift) = raw_node; - break; - case 64: - *((uint64_t *) region_address + shift) = raw_node; - break; - default: - assert(0); - } + meta->index[ix] = raw_node; } inline int -AtomicDict_MustWriteBytes(int n, AtomicDict_Meta *meta) +AtomicDict_AtomicWriteNodeAt(uint64_t ix, AtomicDict_Node *expected, AtomicDict_Node *desired, AtomicDict_Meta *meta) { - int n_bytes = (meta->node_size >> 3) * n; - assert(n_bytes <= 16); - if (n_bytes == 1) { - return 1; - } - if (n_bytes <= 2) { - return 2; - } - if (n_bytes <= 4) { - return 4; - } - if (n_bytes <= 8) { - return 8; - } - return 16; -} - -inline int -AtomicDict_AtomicWriteNodesAt(uint64_t ix, int n, AtomicDict_Node *expected, AtomicDict_Node *desired, - AtomicDict_Meta *meta) -{ - assert(ix >= 0); - assert(ix < meta->size); - assert(ix <= meta->size - n); // XXX implement index circular behavior - assert(n > 0); - assert(n <= meta->nodes_in_zone); - - uint64_t little = AtomicDict_RegionOf(ix, meta); - uint64_t middle = AtomicDict_RegionOf(ix + n / 2, meta); - uint64_t big = AtomicDict_RegionOf(ix + n - 1, meta); - assert(little <= middle <= little + 1); // XXX implement index circular behavior - assert(middle <= big <= middle + 1); // XXX implement index circular behavior - __uint128_t expected_raw = 0, desired_raw = 0, node; // il bello sta nelle piccole cose - int i; - for (i = 0; i < n; ++i) { - AtomicDict_ComputeRawNode(&expected[i], meta); - AtomicDict_ComputeRawNode(&desired[i], meta); - } - for (i = 0; i < n; ++i) { - node = expected[i].node; - node <<= meta->node_size * i; - expected_raw |= node; - node = desired[i].node; - node <<= meta->node_size * i; - desired_raw |= node; - } - - int must_write = AtomicDict_MustWriteBytes(n, meta); - int must_write_nodes = must_write / (meta->node_size / 8); - assert(i == must_write_nodes); + AtomicDict_ComputeRawNode(expected, meta); + AtomicDict_ComputeRawNode(desired, meta); - uint8_t *index_address = AtomicDict_IndexAddressOf(ix, meta); - switch (must_write) { - case 1: - return CereggiiAtomic_CompareExchangeUInt8(index_address, expected_raw, desired_raw); - case 2: - return CereggiiAtomic_CompareExchangeUInt16((uint16_t *) index_address, expected_raw, desired_raw); - case 4: - return CereggiiAtomic_CompareExchangeUInt32((uint32_t *) index_address, expected_raw, desired_raw); - case 8: - return CereggiiAtomic_CompareExchangeUInt64((uint64_t *) index_address, expected_raw, desired_raw); - case 16: - // assert memory access is aligned - // this is not required for <=8 bytes CAS on x86_64 - assert(AtomicDict_IndexAddressIsAligned(ix, 16, meta)); - return CereggiiAtomic_CompareExchangeUInt128((__uint128_t *) index_address, expected_raw, desired_raw); - default: - assert(0); - } + return CereggiiAtomic_CompareExchangeUInt64(&meta->index[ix], expected->node, desired->node); } diff --git a/src/cereggii/atomic_dict/node_sizes_table.c b/src/cereggii/atomic_dict/node_sizes_table.c deleted file mode 100644 index 8f027de..0000000 --- a/src/cereggii/atomic_dict/node_sizes_table.c +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: 2023-present dpdani -// -// SPDX-License-Identifier: Apache-2.0 - -#include "atomic_dict_internal.h" - - -const AtomicDict_NodeSizeInfo AtomicDict_NodeSizesTable[] = { - {0, 0, 0}, - {0, 0, 0}, - {0, 0, 0}, - {0, 0, 0}, - {0, 0, 0}, - {0, 0, 0}, - {8, 2, 0}, // this is the minimum log_size - {16, 3, 6}, - {16, 3, 5}, - {16, 4, 3}, - {16, 4, 2}, - {32, 4, 17}, - {32, 4, 16}, - {32, 4, 15}, - {32, 4, 14}, - {32, 4, 13}, - {32, 4, 12}, - {32, 5, 10}, - {32, 5, 9}, - {32, 5, 8}, - {32, 5, 7}, - {32, 5, 6}, - {32, 5, 5}, - {32, 5, 4}, - {32, 5, 3}, - {32, 5, 2}, - {64, 5, 33}, - {64, 5, 32}, - {64, 5, 31}, - {64, 5, 30}, - {64, 5, 29}, - {64, 5, 28}, - {64, 5, 27}, - {64, 6, 25}, - {64, 6, 24}, - {64, 6, 23}, - {64, 6, 22}, - {64, 6, 21}, - {64, 6, 20}, - {64, 6, 19}, - {64, 6, 18}, - {64, 6, 17}, - {64, 6, 16}, - {64, 6, 15}, - {64, 6, 14}, - {64, 6, 13}, - {64, 6, 12}, - {64, 6, 11}, - {64, 6, 10}, - {64, 6, 9}, - {64, 6, 8}, - {64, 6, 7}, - {64, 6, 6}, - {64, 6, 5}, - {64, 6, 4}, - {64, 6, 3}, - {64, 6, 2}, -}; diff --git a/src/cereggii/atomic_dict/robin_hood.c b/src/cereggii/atomic_dict/robin_hood.c deleted file mode 100644 index 03d909c..0000000 --- a/src/cereggii/atomic_dict/robin_hood.c +++ /dev/null @@ -1,145 +0,0 @@ -// SPDX-FileCopyrightText: 2023-present dpdani -// -// SPDX-License-Identifier: Apache-2.0 - -#define PY_SSIZE_T_CLEAN - -#include "atomic_dict_internal.h" - - -/** - * Lythe and listin, gentilmen, - * That be of frebore blode; - * I shall you tel of a gode yeman, - * His name was Robyn Hode. - * */ - - -AtomicDict_RobinHoodResult -AtomicDict_RobinHoodInsert(AtomicDict_Meta *meta, AtomicDict_Node *nodes, AtomicDict_Node *to_insert, - int distance_0_ix) -{ - /* - * assumptions: - * 1. there are meta->nodes_in_two_regions valid nodes in `nodes` - * 2. there is at least 1 empty node - * */ - assert(distance_0_ix >= 0); - assert(distance_0_ix < meta->nodes_in_zone); - - AtomicDict_Node current = *to_insert; - AtomicDict_Node temp; - int probe = 0; - int cursor; - - beginning: - for (; probe < meta->nodes_in_zone; probe++) { - if (probe >= meta->max_distance) { - return grow; - } - cursor = distance_0_ix + probe; - if (nodes[cursor].node == 0) { - if (!AtomicDict_NodeIsReservation(¤t, meta)) { - current.distance = probe; - } - nodes[cursor] = current; - return ok; - } - - if (nodes[cursor].distance < probe) { - if (!AtomicDict_NodeIsReservation(¤t, meta)) { - current.distance = probe; - AtomicDict_ComputeRawNode(¤t, meta); - } - temp = nodes[cursor]; - nodes[cursor] = current; - current = temp; - distance_0_ix = cursor - temp.distance; - probe = temp.distance; - goto beginning; - } - } - return failed; -} - -AtomicDict_RobinHoodResult -AtomicDict_RobinHoodInsertRaw(AtomicDict_Meta *meta, AtomicDict_Node *to_insert, int64_t distance_0_ix) -{ - AtomicDict_Node current = *to_insert; - current.distance = 0; - AtomicDict_Node temp; - int64_t probe = 0; - int64_t cursor; - - beginning: - for (; probe < meta->size; probe++) { - cursor = (distance_0_ix + probe) & ((int64_t) meta->size - 1); - - if (probe >= meta->max_distance) { - while (AtomicDict_ReadRawNodeAt(cursor, meta) != 0) { - probe++; - cursor = (distance_0_ix + probe) & ((int64_t) meta->size - 1); - } - - current.distance = meta->max_distance; - assert(AtomicDict_ReadRawNodeAt(cursor, meta) == 0); - AtomicDict_WriteNodeAt(cursor, ¤t, meta); - - return grow; - } - - if (AtomicDict_ReadRawNodeAt(cursor, meta) == 0) { - assert(!AtomicDict_NodeIsReservation(¤t, meta)); - current.distance = probe; - - assert(AtomicDict_ReadRawNodeAt(cursor, meta) == 0); - AtomicDict_WriteNodeAt(cursor, ¤t, meta); - return ok; - } - - AtomicDict_ReadNodeAt(cursor, &temp, meta); - - if (temp.distance < probe) { - current.distance = probe; - assert(!AtomicDict_NodeIsReservation(¤t, meta)); - - AtomicDict_WriteNodeAt(cursor, ¤t, meta); - - current = temp; - distance_0_ix = cursor - temp.distance; - probe = temp.distance; - goto beginning; - } - } - return failed; -} - -AtomicDict_RobinHoodResult -AtomicDict_RobinHoodDelete(AtomicDict_Meta *meta, AtomicDict_Node *nodes, int to_delete) -{ - assert(to_delete >= 0); - assert(to_delete < meta->nodes_in_zone); - - AtomicDict_Node temp; - - nodes[to_delete] = meta->tombstone; - int probe; - - for (probe = to_delete; probe + 1 < meta->nodes_in_zone && - nodes[probe + 1].node != 0 && - !AtomicDict_NodeIsReservation(&nodes[probe + 1], meta) && - nodes[probe + 1].distance > 0; probe++) { - temp = nodes[probe]; - nodes[probe] = nodes[probe + 1]; - nodes[probe + 1] = temp; - nodes[probe].distance--; - } - -// if (probe + 1 < meta->nodes_in_zone && ( -// nodes[probe + 1].node == 0 || nodes[probe + 1].distance == 0 -// )) { -// AtomicDict_ParseNodeFromRaw(0, &nodes[probe], meta); -// } - - return ok; -} diff --git a/src/cereggii/cereggii.c b/src/cereggii/cereggii.c index 306bfe6..25fa9af 100644 --- a/src/cereggii/cereggii.c +++ b/src/cereggii/cereggii.c @@ -216,7 +216,6 @@ PyTypeObject AtomicRef_Type = { static PyMethodDef AtomicDict_methods[] = { {"_debug", (PyCFunction) AtomicDict_Debug, METH_NOARGS, NULL}, {"_rehash", (PyCFunction) AtomicDict_ReHash, METH_O, NULL}, - {"compact", (PyCFunction) AtomicDict_Compact_callable, METH_NOARGS, NULL}, {"get", (PyCFunction) AtomicDict_GetItemOrDefaultVarargs, METH_VARARGS | METH_KEYWORDS, NULL}, {"len_bounds", (PyCFunction) AtomicDict_LenBounds, METH_NOARGS, NULL}, {"approx_len", (PyCFunction) AtomicDict_ApproxLen, METH_NOARGS, NULL}, diff --git a/src/include/atomic_dict.h b/src/include/atomic_dict.h index 5301b13..946b85c 100644 --- a/src/include/atomic_dict.h +++ b/src/include/atomic_dict.h @@ -54,10 +54,6 @@ int AtomicDict_Reduce(AtomicDict *self, PyObject *iterable, PyObject *aggregate, PyObject *AtomicDict_Reduce_callable(AtomicDict *self, PyObject *args, PyObject *kwargs); -int AtomicDict_Compact(AtomicDict *self); - -PyObject *AtomicDict_Compact_callable(AtomicDict *self); - PyObject *AtomicDict_LenBounds(AtomicDict *self); PyObject *AtomicDict_ApproxLen(AtomicDict *self); diff --git a/src/include/atomic_dict_internal.h b/src/include/atomic_dict_internal.h index c30ee09..55ba075 100644 --- a/src/include/atomic_dict_internal.h +++ b/src/include/atomic_dict_internal.h @@ -18,24 +18,33 @@ typedef struct AtomicDict_Entry { } AtomicDict_Entry; #define ENTRY_FLAGS_RESERVED 128 -#define ENTRY_FLAGS_TOMBSTONE 64 -#define ENTRY_FLAGS_SWAPPED 32 -#define ENTRY_FLAGS_COMPACTED 16 -#define ENTRY_FLAGS_LOCKED 8 -// #define ENTRY_FLAGS_? 4 -// #define ENTRY_FLAGS_? 2 -// #define ENTRY_FLAGS_? 1 +// #define ENTRY_FLAGS_? 64 +// #define ENTRY_FLAGS_? 32 +// #define ENTRY_FLAGS_? 16 +// #define ENTRY_FLAGS_? 8 +// #define ENTRY_FLAGS_? 4 +// #define ENTRY_FLAGS_? 2 +// #define ENTRY_FLAGS_? 1 typedef struct AtomicDict_EntryLoc { AtomicDict_Entry *entry; uint64_t location; } AtomicDict_EntryLoc; +/* + * Node layout in memory + * + * +--------+--------+ + * | index | tag | + * +--------+--------+ + * */ +#define NODE_SIZE 64 +#define TAG_MASK(meta) ((1ULL << (NODE_SIZE - (meta)->log_size)) - 1) +#define TOMBSTONE(meta) (TAG_MASK(meta)) typedef struct AtomicDict_Node { uint64_t node; uint64_t index; - uint8_t distance; uint64_t tag; } AtomicDict_Node; @@ -70,7 +79,6 @@ struct AtomicDict_Meta { PyObject_HEAD uint8_t log_size; // = node index_size - uint64_t size; void *generation; @@ -82,29 +90,6 @@ struct AtomicDict_Meta { int64_t greatest_deleted_block; int64_t greatest_refilled_block; - uint8_t is_compact; - - uint8_t node_size; - uint8_t distance_size; - uint8_t max_distance; - uint8_t tag_size; - uint8_t nodes_in_region; - uint8_t nodes_in_zone; - - uint64_t node_mask; - uint64_t index_mask; - uint64_t distance_mask; - Py_hash_t tag_mask; - uint64_t shift_mask; - uint64_t d0_shift; - - AtomicDict_Node tombstone; - AtomicDict_Node zero; - - void (*read_nodes_in_region)(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta); - - void (*read_nodes_in_zone)(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta); - // migration AtomicDict_Meta *new_gen_metadata; uintptr_t migration_leader; @@ -116,6 +101,8 @@ struct AtomicDict_Meta { AtomicEvent *migration_done; }; +#define SIZE_OF(meta) (1ull << (meta)->log_size) + int AtomicDictMeta_traverse(AtomicDict_Meta *self, visitproc visit, void *arg); int AtomicDictMeta_clear(AtomicDict_Meta *self); @@ -153,94 +140,20 @@ void AtomicDict_ComputeRawNode(AtomicDict_Node *node, AtomicDict_Meta *meta); void AtomicDict_ParseNodeFromRaw(uint64_t node_raw, AtomicDict_Node *node, AtomicDict_Meta *meta); -void AtomicDict_ParseNodeFromRegion(uint64_t ix, uint64_t region, AtomicDict_Node *node, - AtomicDict_Meta *meta); - -uint64_t AtomicDict_ParseRawNodeFromRegion(uint64_t ix, uint64_t region, AtomicDict_Meta *meta); - -uint64_t AtomicDict_RegionOf(uint64_t ix, AtomicDict_Meta *meta); - -uint64_t AtomicDict_ZoneOf(uint64_t ix, AtomicDict_Meta *meta); - uint64_t AtomicDict_Distance0Of(Py_hash_t hash, AtomicDict_Meta *meta); -uint64_t AtomicDict_ShiftInRegionOf(uint64_t ix, AtomicDict_Meta *meta); - -uint8_t *AtomicDict_IndexAddressOf(uint64_t ix, AtomicDict_Meta *meta); - -int AtomicDict_IndexAddressIsAligned(uint64_t ix, int alignment, AtomicDict_Meta *meta); - void AtomicDict_ReadNodeAt(uint64_t ix, AtomicDict_Node *node, AtomicDict_Meta *meta); -int64_t AtomicDict_ReadRawNodeAt(uint64_t ix, AtomicDict_Meta *meta); - -void AtomicDict_Read1NodeAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta); - -void AtomicDict_Read2NodesAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta); - -void AtomicDict_Read4NodesAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta); - -void AtomicDict_Read8NodesAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta); - -void AtomicDict_Read16NodesAt(uint64_t ix, AtomicDict_Node *nodes, AtomicDict_Meta *meta); - -void AtomicDict_CopyNodeBuffers(AtomicDict_Node *from_buffer, AtomicDict_Node *to_buffer); - -void AtomicDict_ComputeBeginEndWrite(AtomicDict_Meta *meta, AtomicDict_Node *read_buffer, AtomicDict_Node *temp, - int *begin_write, int *end_write); - -typedef struct AtomicDict_BufferedNodeReader { - int64_t zone; - AtomicDict_Node buffer[16]; - AtomicDict_Node node; - int idx_in_buffer; - int nodes_offset; -} AtomicDict_BufferedNodeReader; - -void AtomicDict_ReadNodesFromZoneIntoBuffer(uint64_t idx, AtomicDict_BufferedNodeReader *reader, - AtomicDict_Meta *meta); - -void AtomicDict_ReadNodesFromZoneStartIntoBuffer(uint64_t idx, AtomicDict_BufferedNodeReader *reader, - AtomicDict_Meta *meta); - void AtomicDict_WriteNodeAt(uint64_t ix, AtomicDict_Node *node, AtomicDict_Meta *meta); void AtomicDict_WriteRawNodeAt(uint64_t ix, uint64_t raw_node, AtomicDict_Meta *meta); -int AtomicDict_NodeIsReservation(AtomicDict_Node *node, AtomicDict_Meta *meta); - -int AtomicDict_NodeIsTombstone(AtomicDict_Node *node, AtomicDict_Meta *meta); - -int AtomicDict_MustWriteBytes(int n, AtomicDict_Meta *meta); - -int AtomicDict_AtomicWriteNodesAt(uint64_t ix, int n, AtomicDict_Node *expected, AtomicDict_Node *desired, - AtomicDict_Meta *meta); - - -/// delete -int AtomicDict_IncrementGreatestDeletedBlock(AtomicDict_Meta *meta, int64_t gab, int64_t gdb); - - -/// insert -typedef enum AtomicDict_InsertedOrUpdated { - error, - inserted, - updated, - nop, - retry, - must_grow, -} AtomicDict_InsertedOrUpdated; - -AtomicDict_InsertedOrUpdated -AtomicDict_CheckNodeEntryAndMaybeUpdate(uint64_t distance_0, uint64_t i, AtomicDict_Node *node, - AtomicDict_Meta *meta, Py_hash_t hash, PyObject *key, PyObject *value); +int AtomicDict_AtomicWriteNodeAt(uint64_t ix, AtomicDict_Node *expected, AtomicDict_Node *desired, AtomicDict_Meta *meta); /// migrations int AtomicDict_Grow(AtomicDict *self); -int AtomicDict_Shrink(AtomicDict *self); - int AtomicDict_MaybeHelpMigrate(AtomicDict_Meta *meta, PyMutex *self_mutex); int AtomicDict_Migrate(AtomicDict *self, AtomicDict_Meta *current_meta, uint8_t from_log_size, uint8_t to_log_size); @@ -258,19 +171,11 @@ void AtomicDict_SlowMigrate(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_ int AtomicDict_MigrateReInsertAll(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_meta); -int AtomicDict_PrepareHashArray(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_meta); - void -AtomicDict_MigrateNode(AtomicDict_Node *node, uint64_t *distance_0, uint64_t *distance, AtomicDict_Meta *new_meta, - uint64_t size_mask); +AtomicDict_MigrateNode(AtomicDict_Node *node, AtomicDict_Meta *new_meta); int AtomicDict_MigrateNodes(AtomicDict_Meta *current_meta, AtomicDict_Meta *new_meta); -void -AtomicDict_SeekToProbeStart(AtomicDict_Meta *meta, uint64_t *pos, uint64_t displacement, uint64_t current_size_mask); - -void AtomicDict_SeekToProbeEnd(AtomicDict_Meta *meta, uint64_t *pos, uint64_t displacement, uint64_t current_size_mask); - int AtomicDict_NodesMigrationDone(const AtomicDict_Meta *current_meta); @@ -319,23 +224,6 @@ void AtomicDict_EndSynchronousOperation(AtomicDict *self); void AtomicDict_AccessorStorage_dealloc(AtomicDict_AccessorStorage *self); -/// robin hood hashing -typedef enum AtomicDict_RobinHoodResult { - ok, - failed, - grow, -} AtomicDict_RobinHoodResult; - -AtomicDict_RobinHoodResult AtomicDict_RobinHoodInsert(AtomicDict_Meta *meta, AtomicDict_Node *nodes, - AtomicDict_Node *to_insert, int distance_0_ix); - -AtomicDict_RobinHoodResult AtomicDict_RobinHoodInsertRaw(AtomicDict_Meta *meta, AtomicDict_Node *to_insert, - int64_t distance_0_ix); - -AtomicDict_RobinHoodResult AtomicDict_RobinHoodDelete(AtomicDict_Meta *meta, AtomicDict_Node *nodes, - int to_delete); - - /// iter struct AtomicDict_FastIterator { PyObject_HEAD @@ -364,7 +252,6 @@ typedef struct AtomicDict_SearchResult { AtomicDict_Node node; AtomicDict_Entry *entry_p; AtomicDict_Entry entry; - int is_reservation; } AtomicDict_SearchResult; void AtomicDict_Lookup(AtomicDict_Meta *meta, PyObject *key, Py_hash_t hash, @@ -382,27 +269,7 @@ PyObject *AtomicDict_ExpectedInsertOrUpdate(AtomicDict_Meta *meta, PyObject *key AtomicDict_EntryLoc *entry_loc, int *must_grow, int skip_entry_check); -/// node sizes table -/* - * Node layout in memory - * - * +--------+--------+--------+ - * | index | inv(d) | tag | - * +--------+--------+--------+ - * - * inv(d) := max_distance - distance - * max_distance := (1 << distance_size) - * */ - -typedef struct { - const uint8_t node_size; - const uint8_t distance_size; - const uint8_t tag_size; -} AtomicDict_NodeSizeInfo; - #define ATOMIC_DICT_MIN_LOG_SIZE 6 #define ATOMIC_DICT_MAX_LOG_SIZE 56 -extern const AtomicDict_NodeSizeInfo AtomicDict_NodeSizesTable[]; - #endif //CEREGGII_DEV_ATOMIC_DICT_INTERNAL_H diff --git a/tests/test_atomic_dict.py b/tests/test_atomic_dict.py index b5a9fb9..3863c82 100644 --- a/tests/test_atomic_dict.py +++ b/tests/test_atomic_dict.py @@ -128,42 +128,6 @@ def test_setitem_updates_an_inserted_value(): assert d[0] == 2 -def test_setitem_distance_1_insert(): - pos_0 = keys_for_hash_for_log_size[6][0][0] - pos_1 = keys_for_hash_for_log_size[6][1][0] - pos_0_again = keys_for_hash_for_log_size[6][0][1] - - d = AtomicDict({pos_0: 1}) - assert d[pos_0] == 1 - d[pos_0_again] = 42 - assert d[pos_0] == 1 - assert d[pos_0_again] == 42 - # debug = d._debug() - # assert debug["index"][1] == 10 - d = AtomicDict() - d[pos_0] = 1 - assert d[pos_0] == 1 - d[pos_1] = 2 - assert d[pos_0] == 1 - assert d[pos_1] == 2 - d[pos_0_again] = 3 - assert d[pos_0] == 1 - assert d[pos_1] == 2 - assert d[pos_0_again] == 3 - # debug = d._debug() - # assert debug["index"][0] == 7 - # assert debug["index"][1] == 14 - # assert debug["index"][2] == 10 - - -def test_insert_non_compact(): - d = AtomicDict({k: None for k in range(60)}) - d[64] = 1 - for k in range(60): - assert d[k] is None - assert d[64] == 1 - - def test_full_dict(): d = AtomicDict({k: None for k in range(63)}) assert len(d._debug()["index"]) == 128 @@ -441,83 +405,6 @@ def test_grow(): assert d[_] is None -def test_compact(): - keys = keys_for_hash_for_log_size[6] - d = AtomicDict( - { - keys[0][0]: None, - keys[1][0]: None, - keys[0][1]: None, - keys[1][1]: None, - } - ) - for _ in range(20): - d[keys[0][_]] = None - assert len(list(filter(lambda _: _ != 0, Counter(d._debug()["index"]).keys()))) == len( - {keys[0][0], keys[1][0], keys[0][1], keys[1][1], *[keys[0][i] for i in range(_ + 1)]} - ), _ - for _ in [keys[0][0], keys[1][0], keys[0][1], keys[1][1]]: - assert d[_] is None - for _ in range(20): - assert d[keys[0][_]] is None - debug_before = d._debug() - assert len(list(filter(lambda _: _ != 0, Counter(debug_before["index"]).keys()))) == len( - {keys[0][0], keys[1][0], keys[0][1], keys[1][1], *[keys[0][_] for _ in range(20)]} - ) - assert not debug_before["meta"]["is_compact"] - # 8192 == 2 ** 13 will be in a reservation - entry_of_8192 = list(filter(lambda _: _[3] == keys[0][13], debug_before["blocks"][0]["entries"])) - assert len(entry_of_8192) == 1 - entry_of_8192 = entry_of_8192[0] - non_empty_nodes = len(list(filter(lambda _: _ != 0, debug_before["index"]))) - node_of_8192 = list( - filter( - lambda _: (_ & debug_before["meta"]["index_mask"]) - >> (debug_before["meta"]["node_size"] - debug_before["meta"]["log_size"]) - == entry_of_8192[0], - debug_before["index"], - ) - ) - assert len(node_of_8192) == 1 - node_of_8192 = node_of_8192[0] - assert node_of_8192 & debug_before["meta"]["distance_mask"] == 0 - d.compact() - debug_after = d._debug() - assert len(list(filter(lambda _: _ != 0, debug_after["index"]))) == non_empty_nodes - assert len(list(filter(lambda _: _ != 0, Counter(debug_before["index"]).keys()))) == len( - {keys[0][0], keys[1][0], keys[0][1], keys[1][1], *[keys[0][_] for _ in range(20)]} - ) - assert debug_after["meta"]["is_compact"] - entry_of_8192 = list(filter(lambda _: _[3] == keys[0][13], debug_after["blocks"][0]["entries"])) - assert len(entry_of_8192) == 1 - entry_of_8192 = entry_of_8192[0] - node_of_8192 = list( - filter( - lambda _: (_ & debug_after["meta"]["index_mask"]) - >> (debug_after["meta"]["node_size"] - debug_after["meta"]["log_size"]) - == entry_of_8192[0], - debug_after["index"], - ) - ) - assert len(node_of_8192) == 1 - node_of_8192 = node_of_8192[0] - assert node_of_8192 & debug_after["meta"]["distance_mask"] != 0 - - d = AtomicDict({keys[0][0]: None, keys[1][0]: None}) - for _ in range(20): - d[keys[0][_]] = None - assert d._debug()["meta"]["log_size"] == 7 - d.compact() - for _ in range(20): - assert d[keys[0][_]] is None - # assert d._debug()["meta"]["log_size"] == 9 - - d = AtomicDict({}, min_size=2**16) - assert len(d._debug()["index"]) == 2**16 - d.compact() - assert len(d._debug()["index"]) == 2**16 - - def test_grow_then_shrink(): d = AtomicDict() assert d._debug()["meta"]["log_size"] == 6 From d5ef22b56b56fdff3ac550cf8abb781e29336b8e Mon Sep 17 00:00:00 2001 From: dpdani Date: Tue, 4 Feb 2025 20:26:52 +0100 Subject: [PATCH 8/9] remove a scaling bottleneck --- src/cereggii/atomic_dict/accessor_storage.c | 14 +++++++++++ src/cereggii/atomic_dict/atomic_dict.c | 8 +++--- src/cereggii/atomic_dict/delete.c | 16 +++++------- src/cereggii/atomic_dict/insert.c | 18 ++++++-------- src/cereggii/atomic_dict/lookup.c | 27 +++++++++++---------- src/cereggii/atomic_dict/migrate.c | 9 ++++--- src/include/atomic_dict_internal.h | 6 ++--- 7 files changed, 54 insertions(+), 44 deletions(-) diff --git a/src/cereggii/atomic_dict/accessor_storage.c b/src/cereggii/atomic_dict/accessor_storage.c index cc758b3..b886f88 100644 --- a/src/cereggii/atomic_dict/accessor_storage.c +++ b/src/cereggii/atomic_dict/accessor_storage.c @@ -24,6 +24,7 @@ AtomicDict_GetOrCreateAccessorStorage(AtomicDict *self) storage->reservation_buffer.tail = 0; storage->reservation_buffer.count = 0; memset(storage->reservation_buffer.reservations, 0, sizeof(AtomicDict_EntryLoc) * RESERVATION_BUFFER_SIZE); + storage->meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); int set = PyThread_tss_set(self->accessor_key, storage); if (set != 0) @@ -55,6 +56,18 @@ AtomicDict_GetAccessorStorage(Py_tss_t *accessor_key) return storage; } +inline AtomicDict_Meta * +AtomicDict_GetMeta(AtomicDict *self, AtomicDict_AccessorStorage *storage) +{ + if (self->metadata->reference == (PyObject *) storage->meta) + return storage->meta; + + Py_CLEAR(storage->meta); + storage->meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); + assert(storage->meta != NULL); + return storage->meta; +} + void AtomicDict_BeginSynchronousOperation(AtomicDict *self) { @@ -86,6 +99,7 @@ AtomicDict_EndSynchronousOperation(AtomicDict *self) void AtomicDict_AccessorStorage_dealloc(AtomicDict_AccessorStorage *self) { + Py_CLEAR(self->meta); Py_TYPE(self)->tp_free((PyObject *) self); } diff --git a/src/cereggii/atomic_dict/atomic_dict.c b/src/cereggii/atomic_dict/atomic_dict.c index 58fb3bd..55205dd 100644 --- a/src/cereggii/atomic_dict/atomic_dict.c +++ b/src/cereggii/atomic_dict/atomic_dict.c @@ -372,7 +372,11 @@ PyObject * AtomicDict_LenBounds(AtomicDict *self) { AtomicDict_Meta *meta = NULL; - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); + AtomicDict_AccessorStorage *storage = AtomicDict_GetOrCreateAccessorStorage(self); + if (storage == NULL) + goto fail; + + meta = AtomicDict_GetMeta(self, storage); if (meta == NULL) goto fail; @@ -398,7 +402,6 @@ AtomicDict_LenBounds(AtomicDict *self) // visit the grb found += AtomicDict_CountKeysInBlock(grb, meta); } - Py_DECREF(meta); meta = NULL; if (supposedly_full_blocks < 0) { @@ -429,7 +432,6 @@ AtomicDict_LenBounds(AtomicDict *self) return Py_BuildValue("(ll)", lower + found, upper + found); fail: - Py_XDECREF(meta); return NULL; } diff --git a/src/cereggii/atomic_dict/delete.c b/src/cereggii/atomic_dict/delete.c index 72d613a..1b63c35 100644 --- a/src/cereggii/atomic_dict/delete.c +++ b/src/cereggii/atomic_dict/delete.c @@ -54,8 +54,13 @@ AtomicDict_DelItem(AtomicDict *self, PyObject *key) assert(key != NULL); AtomicDict_Meta *meta = NULL; + AtomicDict_AccessorStorage *storage = NULL; + storage = AtomicDict_GetOrCreateAccessorStorage(self); + if (storage == NULL) + goto fail; + beginning: - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); + meta = AtomicDict_GetMeta(self, storage); if (meta == NULL) goto fail; @@ -63,17 +68,10 @@ AtomicDict_DelItem(AtomicDict *self, PyObject *key) if (hash == -1) goto fail; - AtomicDict_AccessorStorage *storage = NULL; - storage = AtomicDict_GetOrCreateAccessorStorage(self); - - if (storage == NULL) - goto fail; - PyMutex_Lock(&storage->self_mutex); int migrated = AtomicDict_MaybeHelpMigrate(meta, &storage->self_mutex); if (migrated) { // self_mutex was unlocked during the operation - Py_DECREF(meta); meta = NULL; goto beginning; } @@ -95,10 +93,8 @@ AtomicDict_DelItem(AtomicDict *self, PyObject *key) goto fail; } - Py_DECREF(meta); return 0; fail: - Py_XDECREF(meta); return -1; } diff --git a/src/cereggii/atomic_dict/insert.c b/src/cereggii/atomic_dict/insert.c index 6edf999..a155949 100644 --- a/src/cereggii/atomic_dict/insert.c +++ b/src/cereggii/atomic_dict/insert.c @@ -238,7 +238,7 @@ AtomicDict_CompareAndSet(AtomicDict *self, PyObject *key, PyObject *expected, Py goto fail; beginning: - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); + meta = AtomicDict_GetMeta(self, storage); if (meta == NULL) goto fail; @@ -246,8 +246,6 @@ AtomicDict_CompareAndSet(AtomicDict *self, PyObject *key, PyObject *expected, Py int migrated = AtomicDict_MaybeHelpMigrate(meta, &storage->self_mutex); if (migrated) { // self_mutex was unlocked during the operation - Py_DECREF(meta); - meta = NULL; goto beginning; } @@ -269,8 +267,6 @@ AtomicDict_CompareAndSet(AtomicDict *self, PyObject *key, PyObject *expected, Py if (migrated < 0) goto fail; - Py_DECREF(meta); - meta = NULL; goto beginning; } @@ -307,15 +303,12 @@ AtomicDict_CompareAndSet(AtomicDict *self, PyObject *key, PyObject *expected, Py goto fail; if (must_grow) { // insertion didn't happen - Py_DECREF(meta); goto beginning; } } - Py_DECREF(meta); return result; fail: - Py_XDECREF(meta); Py_DECREF(key); Py_DECREF(desired); return NULL; @@ -409,8 +402,13 @@ reduce_flush(AtomicDict *self, PyObject *local_buffer, PyObject *aggregate) Py_ssize_t pos = 0; + AtomicDict_AccessorStorage *storage = NULL; + storage = AtomicDict_GetOrCreateAccessorStorage(self); + if (storage == NULL) + goto fail; + get_meta: - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); + meta = AtomicDict_GetMeta(self, storage); if (meta == NULL) goto fail; @@ -478,7 +476,6 @@ reduce_flush(AtomicDict *self, PyObject *local_buffer, PyObject *aggregate) PyMem_RawFree(keys); PyMem_RawFree(expected); PyMem_RawFree(desired); - Py_DECREF(meta); return 0; fail: @@ -491,7 +488,6 @@ reduce_flush(AtomicDict *self, PyObject *local_buffer, PyObject *aggregate) if (desired != NULL) { PyMem_RawFree(desired); } - Py_XDECREF(meta); return -1; } diff --git a/src/cereggii/atomic_dict/lookup.c b/src/cereggii/atomic_dict/lookup.c index 7729ed6..48dd8aa 100644 --- a/src/cereggii/atomic_dict/lookup.c +++ b/src/cereggii/atomic_dict/lookup.c @@ -111,28 +111,28 @@ AtomicDict_GetItemOrDefault(AtomicDict *self, PyObject *key, PyObject *default_v goto fail; AtomicDict_SearchResult result; + AtomicDict_AccessorStorage *storage = NULL; + storage = AtomicDict_GetOrCreateAccessorStorage(self); + if (storage == NULL) + goto fail; + retry: - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); + meta = AtomicDict_GetMeta(self, storage); result.entry.value = NULL; AtomicDict_Lookup(meta, key, hash, &result); if (result.error) goto fail; - if (AtomicRef_Get(self->metadata) != (PyObject *) meta) { - Py_DECREF(meta); + if (AtomicDict_GetMeta(self, storage) != meta) goto retry; - } - Py_DECREF(meta); // for AtomicRef_Get (if condition) if (result.entry_p == NULL) { result.entry.value = default_value; } - Py_DECREF(meta); return result.entry.value; fail: - Py_XDECREF(meta); return NULL; } @@ -202,8 +202,13 @@ AtomicDict_BatchGetItem(AtomicDict *self, PyObject *args, PyObject *kwargs) goto fail; AtomicDict_SearchResult result; + AtomicDict_AccessorStorage *storage = NULL; + storage = AtomicDict_GetOrCreateAccessorStorage(self); + if (storage == NULL) + goto fail; + retry: - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); + meta = AtomicDict_GetMeta(self, storage); if (meta == NULL) goto fail; @@ -275,18 +280,14 @@ AtomicDict_BatchGetItem(AtomicDict *self, PyObject *args, PyObject *kwargs) Py_END_CRITICAL_SECTION(); - if (self->metadata->reference != (PyObject *) meta) { - Py_DECREF(meta); + if (AtomicDict_GetMeta(self, storage) != meta) goto retry; - } - Py_DECREF(meta); PyMem_RawFree(hashes); PyMem_RawFree(keys); Py_INCREF(batch); return batch; fail: - Py_XDECREF(meta); if (hashes != NULL) { PyMem_RawFree(hashes); } diff --git a/src/cereggii/atomic_dict/migrate.c b/src/cereggii/atomic_dict/migrate.c index 80bc94a..9ffcc84 100644 --- a/src/cereggii/atomic_dict/migrate.c +++ b/src/cereggii/atomic_dict/migrate.c @@ -16,19 +16,20 @@ int AtomicDict_Grow(AtomicDict *self) { AtomicDict_Meta *meta = NULL; - meta = (AtomicDict_Meta *) AtomicRef_Get(self->metadata); - if (meta == NULL) + AtomicDict_AccessorStorage *storage = NULL; + storage = AtomicDict_GetOrCreateAccessorStorage(self); + if (storage == NULL) goto fail; + meta = AtomicDict_GetMeta(self, storage); + int migrate = AtomicDict_Migrate(self, meta, meta->log_size, meta->log_size + 1); if (migrate < 0) goto fail; - Py_DECREF(meta); return migrate; fail: - Py_XDECREF(meta); return -1; } diff --git a/src/include/atomic_dict_internal.h b/src/include/atomic_dict_internal.h index 55ba075..7889a23 100644 --- a/src/include/atomic_dict_internal.h +++ b/src/include/atomic_dict_internal.h @@ -203,12 +203,10 @@ typedef struct AtomicDict_AccessorStorage { PyObject_HEAD PyMutex self_mutex; - int32_t local_len; - int participant_in_migration; - AtomicDict_ReservationBuffer reservation_buffer; + AtomicDict_Meta *meta; } AtomicDict_AccessorStorage; extern PyTypeObject AtomicDictAccessorStorage_Type; @@ -217,6 +215,8 @@ AtomicDict_AccessorStorage *AtomicDict_GetOrCreateAccessorStorage(AtomicDict *se AtomicDict_AccessorStorage *AtomicDict_GetAccessorStorage(Py_tss_t *accessor_key); +AtomicDict_Meta *AtomicDict_GetMeta(AtomicDict *self, AtomicDict_AccessorStorage *storage); + void AtomicDict_BeginSynchronousOperation(AtomicDict *self); void AtomicDict_EndSynchronousOperation(AtomicDict *self); From 28bb1087111828dfb6c5ea4ff24cb90b0fc5f86e Mon Sep 17 00:00:00 2001 From: dpdani Date: Tue, 4 Feb 2025 21:06:04 +0100 Subject: [PATCH 9/9] fixes --- src/cereggii/atomic_dict/atomic_dict.c | 1 - src/include/_internal_py_core.h | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cereggii/atomic_dict/atomic_dict.c b/src/cereggii/atomic_dict/atomic_dict.c index 55205dd..75e3181 100644 --- a/src/cereggii/atomic_dict/atomic_dict.c +++ b/src/cereggii/atomic_dict/atomic_dict.c @@ -267,7 +267,6 @@ AtomicDict_init(AtomicDict *self, PyObject *args, PyObject *kwargs) } Py_DECREF(meta); // so that the only meta's refcount depends only on AtomicRef - assert(Py_REFCNT(meta) == 1); return 0; fail: Py_XDECREF(meta); diff --git a/src/include/_internal_py_core.h b/src/include/_internal_py_core.h index 55fc0cb..51134b4 100644 --- a/src/include/_internal_py_core.h +++ b/src/include/_internal_py_core.h @@ -1,3 +1,6 @@ +#ifndef CEREGGII_PY_CORE_H +#define CEREGGII_PY_CORE_H + #include "Python.h" #ifdef Py_GIL_DISABLED @@ -190,3 +193,5 @@ _Py_TryIncref(PyObject *op) return 0; #endif } + +#endif // CEREGGII_PY_CORE_H