From 2a94de75b3299681cf2379c2718c4ebc939069a9 Mon Sep 17 00:00:00 2001 From: Yaodong Sheng Date: Fri, 15 Sep 2023 11:07:24 -0400 Subject: [PATCH] first commit --- Artifact.md | 151 ++ Docker/Dockerfile | 36 + LICENSE | 21 + Makefile | 5 + artifact/Makefile | 8 + artifact/ds/README.md | 14 + artifact/ds/STMCAS/dlist_carumap.h | 646 +++++++++ artifact/ds/STMCAS/dlist_omap.h | 351 +++++ artifact/ds/STMCAS/ibst_omap.h | 397 ++++++ artifact/ds/STMCAS/rbtree_omap.h | 948 +++++++++++++ artifact/ds/STMCAS/skiplist_cached_opt_omap.h | 567 ++++++++ artifact/ds/STMCAS/slist_omap.h | 328 +++++ artifact/ds/baseline/ext_ticket_bst/plaf.h | 41 + .../ds/baseline/ext_ticket_bst/ticket_impl.h | 379 +++++ .../ds/baseline/int_bst_pathcas/casword.h | 70 + .../int_bst_pathcas/internal_kcas_avl.h | 1251 +++++++++++++++++ .../int_bst_pathcas/internal_kcas_bst.h | 767 ++++++++++ artifact/ds/baseline/int_bst_pathcas/kcas.h | 104 ++ .../int_bst_pathcas/kcas_reuse_htm_impl.h | 575 ++++++++ artifact/ds/baseline/lazylist_omap.h | 158 +++ artifact/ds/baseline/lfskiplist_omap.h | 561 ++++++++ artifact/ds/handSTM/dlist_carumap.h | 499 +++++++ artifact/ds/handSTM/dlist_omap.h | 172 +++ artifact/ds/handSTM/ibst_omap.h | 267 ++++ artifact/ds/handSTM/iht_carumap.h | 286 ++++ artifact/ds/handSTM/rbtree_omap.h | 365 +++++ artifact/ds/handSTM/rbtree_omap_opt.h | 385 +++++ artifact/ds/handSTM/rbtree_tl2_omap.h | 437 ++++++ artifact/ds/handSTM/skiplist_omap_bigtx.h | 334 +++++ artifact/ds/handSTM/slist_omap.h | 154 ++ artifact/ds/hybrid/dlist_carumap.h | 696 +++++++++ artifact/ds/hybrid/rbtree_omap_drop.h | 599 ++++++++ artifact/ds/include/ca_umap_list_adapter.h | 86 ++ artifact/ds/xSTM/dlist_omap.h | 174 +++ artifact/ds/xSTM/ibst_omap.h | 260 ++++ artifact/ds/xSTM/rbtree_omap.h | 370 +++++ artifact/ds/xSTM/rbtree_tl2_omap.h | 428 ++++++ artifact/policies/README.md | 78 + artifact/policies/STMCAS/include/base.h | 98 ++ artifact/policies/STMCAS/include/field.h | 50 + artifact/policies/STMCAS/include/raii.h | 100 ++ artifact/policies/STMCAS/stmcas.h | 38 + artifact/policies/baseline/thread.h | 82 ++ artifact/policies/exoTM/exotm.h | 302 ++++ artifact/policies/handSTM/eager_c1.h | 51 + artifact/policies/handSTM/eager_c2.h | 51 + artifact/policies/handSTM/include/field.h | 651 +++++++++ artifact/policies/handSTM/include/raii.h | 84 ++ artifact/policies/handSTM/include/redo_base.h | 146 ++ artifact/policies/handSTM/include/undo_base.h | 146 ++ artifact/policies/handSTM/lazy.h | 52 + artifact/policies/handSTM/wb_c1.h | 53 + artifact/policies/handSTM/wb_c2.h | 50 + artifact/policies/hybrid/include/base.h | 179 +++ artifact/policies/hybrid/include/field.h | 458 ++++++ artifact/policies/hybrid/include/raii.h | 196 +++ artifact/policies/hybrid/lazy.h | 53 + artifact/policies/hybrid/wb_c1.h | 56 + artifact/policies/hybrid/wb_c2.h | 53 + artifact/policies/include/hash.h | 18 + artifact/policies/include/minivector.h | 102 ++ artifact/policies/include/orec_policies.h | 76 + artifact/policies/include/rdtsc_rand.h | 17 + artifact/policies/include/redolog_nocast.h | 332 +++++ artifact/policies/include/timestamp_smr.h | 141 ++ artifact/policies/include/undolog.h | 206 +++ artifact/policies/xSTM/Makefile | 7 + artifact/policies/xSTM/common/tm_api.h | 223 +++ artifact/policies/xSTM/common/tm_defines.h | 184 +++ artifact/policies/xSTM/common/xSTM.mk | 3 + artifact/policies/xSTM/libs/.gitignore | 1 + artifact/policies/xSTM/libs/Makefile | 45 + artifact/policies/xSTM/libs/README.md | 33 + artifact/policies/xSTM/libs/include/cm.h | 66 + .../policies/xSTM/libs/include/constants.h | 28 + artifact/policies/xSTM/libs/include/epochs.h | 286 ++++ artifact/policies/xSTM/libs/include/orec_t.h | 74 + .../policies/xSTM/libs/include/pad_word.h | 32 + .../policies/xSTM/libs/include/platform.h | 82 ++ .../policies/xSTM/libs/include/timesource.h | 55 + .../xSTM/libs/stm_algs/exo_eager_c1.h | 304 ++++ .../xSTM/libs/stm_algs/exo_eager_c2.h | 307 ++++ .../policies/xSTM/libs/stm_algs/exo_lazy_c1.h | 304 ++++ .../policies/xSTM/libs/stm_algs/exo_lazy_c2.h | 305 ++++ .../xSTM/libs/stm_algs/include/alloc.h | 94 ++ .../xSTM/libs/stm_algs/include/deferred.h | 35 + .../xSTM/libs/stm_algs/include/redolog.h | 370 +++++ .../xSTM/libs/stm_algs/include/stackframe.h | 70 + .../xSTM/libs/stm_algs/orec_eager_c1.h | 315 +++++ .../xSTM/libs/stm_algs/orec_eager_c2.h | 325 +++++ .../xSTM/libs/stm_algs/orec_lazy_c1.h | 321 +++++ .../xSTM/libs/stm_algs/orec_lazy_c2.h | 326 +++++ .../xSTM/libs/stm_instances/exo_eager_c1_q.cc | 34 + .../xSTM/libs/stm_instances/exo_eager_c2_q.cc | 34 + .../xSTM/libs/stm_instances/exo_lazy_c1_q.cc | 34 + .../xSTM/libs/stm_instances/exo_lazy_c2_q.cc | 34 + .../xSTM/libs/stm_instances/include/clone.h | 78 + .../xSTM/libs/stm_instances/include/execute.h | 125 ++ .../xSTM/libs/stm_instances/include/frame.h | 28 + .../libs/stm_instances/include/loadstore.h | 40 + .../xSTM/libs/stm_instances/include/mem.h | 37 + .../libs/stm_instances/include/memfuncs.h | 135 ++ .../xSTM/libs/stm_instances/include/stats.h | 12 + .../libs/stm_instances/orec_gv1_eager_c1_q.cc | 39 + .../libs/stm_instances/orec_gv1_eager_c2_q.cc | 39 + .../libs/stm_instances/orec_gv1_lazy_c1_q.cc | 39 + .../libs/stm_instances/orec_gv1_lazy_c2_q.cc | 39 + .../libs/stm_instances/orec_tsc_eager_c1_q.cc | 39 + .../libs/stm_instances/orec_tsc_eager_c2_q.cc | 39 + .../libs/stm_instances/orec_tsc_lazy_c1_q.cc | 32 + .../libs/stm_instances/orec_tsc_lazy_c2_q.cc | 39 + artifact/policies/xSTM/libs/tm_names.mk | 10 + artifact/policies/xSTM/plugin/Makefile | 5 + .../policies/xSTM/plugin/plugin/.gitignore | 3 + .../xSTM/plugin/plugin/CMakeLists.txt | 61 + artifact/policies/xSTM/plugin/plugin/Makefile | 10 + .../policies/xSTM/plugin/plugin/README.md | 34 + .../xSTM/plugin/plugin/boundary_transform.cc | 108 ++ .../policies/xSTM/plugin/plugin/discovery.cc | 312 ++++ .../xSTM/plugin/plugin/function_transform.cc | 518 +++++++ .../xSTM/plugin/plugin/local_config.h | 15 + .../policies/xSTM/plugin/plugin/mappings.cc | 56 + .../xSTM/plugin/plugin/optimizations.cc | 45 + .../policies/xSTM/plugin/plugin/raii_lite.cc | 252 ++++ .../policies/xSTM/plugin/plugin/signatures.cc | 148 ++ .../policies/xSTM/plugin/plugin/signatures.h | 94 ++ .../policies/xSTM/plugin/plugin/tm_plugin.cc | 71 + .../policies/xSTM/plugin/plugin/tm_plugin.h | 174 +++ artifact/policies/xSTM/plugin/plugin/types.h | 18 + artifact/scripts/.gitignore | 1 + artifact/scripts/ChartCfg.py | 17 + artifact/scripts/ExpCfg.py | 100 ++ artifact/scripts/GetData.py | 49 + artifact/scripts/MakeChart.py | 128 ++ artifact/scripts/Makefile | 7 + artifact/scripts/README.md | 44 + artifact/scripts/Runner.py | 14 + artifact/scripts/Targets.py | 167 +++ artifact/scripts/Types.py | 119 ++ artifact/scripts/Util.py | 31 + artifact/ubench/Makefile | 15 + artifact/ubench/README.md | 50 + artifact/ubench/STMCAS/Makefile | 44 + artifact/ubench/STMCAS/build.mk | 16 + artifact/ubench/STMCAS/common.mk | 22 + artifact/ubench/STMCAS/dlist_carumap.cc | 10 + artifact/ubench/STMCAS/dlist_caumap.cc | 12 + artifact/ubench/STMCAS/dlist_omap.cc | 10 + artifact/ubench/STMCAS/dlist_opt_caumap.cc | 12 + artifact/ubench/STMCAS/dlist_opt_omap.cc | 10 + artifact/ubench/STMCAS/ibst_omap.cc | 10 + artifact/ubench/STMCAS/rbtree_omap.cc | 10 + .../ubench/STMCAS/skiplist_cached_opt_omap.cc | 10 + artifact/ubench/STMCAS/slist_omap.cc | 10 + artifact/ubench/STMCAS/slist_opt_caumap.cc | 12 + artifact/ubench/baseline/Makefile | 37 + artifact/ubench/baseline/ebst_ticket_omap.cc | 11 + artifact/ubench/baseline/iavl_pathcas_omap.cc | 14 + artifact/ubench/baseline/ibst_pathcas_omap.cc | 14 + artifact/ubench/baseline/lazylist_caumap.cc | 13 + artifact/ubench/baseline/lazylist_omap.cc | 11 + artifact/ubench/baseline/lfskiplist_omap.cc | 11 + artifact/ubench/config.mk | 12 + artifact/ubench/handSTM/Makefile | 44 + artifact/ubench/handSTM/build.mk | 16 + artifact/ubench/handSTM/common.mk | 15 + artifact/ubench/handSTM/dlist_carumap.cc | 10 + artifact/ubench/handSTM/dlist_caumap.cc | 12 + artifact/ubench/handSTM/ibst_omap.cc | 10 + artifact/ubench/handSTM/rbtree_omap.cc | 10 + .../ubench/handSTM/skiplist_omap_bigtx.cc | 10 + artifact/ubench/handSTM/slist_omap.cc | 10 + artifact/ubench/hybrid/Makefile | 44 + artifact/ubench/hybrid/build.mk | 16 + artifact/ubench/hybrid/common.mk | 14 + artifact/ubench/hybrid/dlist_carumap.cc | 10 + artifact/ubench/hybrid/rbtree_omap_drop.cc | 10 + .../ubench/include/bench_thread_context.h | 32 + artifact/ubench/include/config.h | 138 ++ artifact/ubench/include/experiment.h | 173 +++ artifact/ubench/include/experiment_pathcas.h | 173 +++ artifact/ubench/include/launch.h | 17 + artifact/ubench/include/launch_multi.h | 18 + artifact/ubench/include/manager.h | 146 ++ artifact/ubench/xSTM/Makefile | 40 + artifact/ubench/xSTM/build.mk | 26 + artifact/ubench/xSTM/common.mk | 24 + artifact/ubench/xSTM/ibst_omap.cc | 11 + 188 files changed, 25916 insertions(+) create mode 100644 Artifact.md create mode 100644 Docker/Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 artifact/Makefile create mode 100644 artifact/ds/README.md create mode 100644 artifact/ds/STMCAS/dlist_carumap.h create mode 100644 artifact/ds/STMCAS/dlist_omap.h create mode 100644 artifact/ds/STMCAS/ibst_omap.h create mode 100644 artifact/ds/STMCAS/rbtree_omap.h create mode 100644 artifact/ds/STMCAS/skiplist_cached_opt_omap.h create mode 100644 artifact/ds/STMCAS/slist_omap.h create mode 100644 artifact/ds/baseline/ext_ticket_bst/plaf.h create mode 100644 artifact/ds/baseline/ext_ticket_bst/ticket_impl.h create mode 100644 artifact/ds/baseline/int_bst_pathcas/casword.h create mode 100644 artifact/ds/baseline/int_bst_pathcas/internal_kcas_avl.h create mode 100644 artifact/ds/baseline/int_bst_pathcas/internal_kcas_bst.h create mode 100644 artifact/ds/baseline/int_bst_pathcas/kcas.h create mode 100644 artifact/ds/baseline/int_bst_pathcas/kcas_reuse_htm_impl.h create mode 100644 artifact/ds/baseline/lazylist_omap.h create mode 100644 artifact/ds/baseline/lfskiplist_omap.h create mode 100644 artifact/ds/handSTM/dlist_carumap.h create mode 100644 artifact/ds/handSTM/dlist_omap.h create mode 100644 artifact/ds/handSTM/ibst_omap.h create mode 100644 artifact/ds/handSTM/iht_carumap.h create mode 100644 artifact/ds/handSTM/rbtree_omap.h create mode 100644 artifact/ds/handSTM/rbtree_omap_opt.h create mode 100644 artifact/ds/handSTM/rbtree_tl2_omap.h create mode 100644 artifact/ds/handSTM/skiplist_omap_bigtx.h create mode 100644 artifact/ds/handSTM/slist_omap.h create mode 100644 artifact/ds/hybrid/dlist_carumap.h create mode 100644 artifact/ds/hybrid/rbtree_omap_drop.h create mode 100644 artifact/ds/include/ca_umap_list_adapter.h create mode 100644 artifact/ds/xSTM/dlist_omap.h create mode 100644 artifact/ds/xSTM/ibst_omap.h create mode 100644 artifact/ds/xSTM/rbtree_omap.h create mode 100644 artifact/ds/xSTM/rbtree_tl2_omap.h create mode 100644 artifact/policies/README.md create mode 100644 artifact/policies/STMCAS/include/base.h create mode 100644 artifact/policies/STMCAS/include/field.h create mode 100644 artifact/policies/STMCAS/include/raii.h create mode 100644 artifact/policies/STMCAS/stmcas.h create mode 100644 artifact/policies/baseline/thread.h create mode 100644 artifact/policies/exoTM/exotm.h create mode 100644 artifact/policies/handSTM/eager_c1.h create mode 100644 artifact/policies/handSTM/eager_c2.h create mode 100644 artifact/policies/handSTM/include/field.h create mode 100644 artifact/policies/handSTM/include/raii.h create mode 100644 artifact/policies/handSTM/include/redo_base.h create mode 100644 artifact/policies/handSTM/include/undo_base.h create mode 100644 artifact/policies/handSTM/lazy.h create mode 100644 artifact/policies/handSTM/wb_c1.h create mode 100644 artifact/policies/handSTM/wb_c2.h create mode 100644 artifact/policies/hybrid/include/base.h create mode 100644 artifact/policies/hybrid/include/field.h create mode 100644 artifact/policies/hybrid/include/raii.h create mode 100644 artifact/policies/hybrid/lazy.h create mode 100644 artifact/policies/hybrid/wb_c1.h create mode 100644 artifact/policies/hybrid/wb_c2.h create mode 100644 artifact/policies/include/hash.h create mode 100644 artifact/policies/include/minivector.h create mode 100644 artifact/policies/include/orec_policies.h create mode 100644 artifact/policies/include/rdtsc_rand.h create mode 100644 artifact/policies/include/redolog_nocast.h create mode 100644 artifact/policies/include/timestamp_smr.h create mode 100644 artifact/policies/include/undolog.h create mode 100644 artifact/policies/xSTM/Makefile create mode 100644 artifact/policies/xSTM/common/tm_api.h create mode 100644 artifact/policies/xSTM/common/tm_defines.h create mode 100644 artifact/policies/xSTM/common/xSTM.mk create mode 100644 artifact/policies/xSTM/libs/.gitignore create mode 100644 artifact/policies/xSTM/libs/Makefile create mode 100644 artifact/policies/xSTM/libs/README.md create mode 100644 artifact/policies/xSTM/libs/include/cm.h create mode 100644 artifact/policies/xSTM/libs/include/constants.h create mode 100644 artifact/policies/xSTM/libs/include/epochs.h create mode 100644 artifact/policies/xSTM/libs/include/orec_t.h create mode 100644 artifact/policies/xSTM/libs/include/pad_word.h create mode 100644 artifact/policies/xSTM/libs/include/platform.h create mode 100644 artifact/policies/xSTM/libs/include/timesource.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/exo_eager_c1.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/exo_eager_c2.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/exo_lazy_c1.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/exo_lazy_c2.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/include/alloc.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/include/deferred.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/include/redolog.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/include/stackframe.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/orec_eager_c1.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/orec_eager_c2.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/orec_lazy_c1.h create mode 100644 artifact/policies/xSTM/libs/stm_algs/orec_lazy_c2.h create mode 100644 artifact/policies/xSTM/libs/stm_instances/exo_eager_c1_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/exo_eager_c2_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/exo_lazy_c1_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/exo_lazy_c2_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/include/clone.h create mode 100644 artifact/policies/xSTM/libs/stm_instances/include/execute.h create mode 100644 artifact/policies/xSTM/libs/stm_instances/include/frame.h create mode 100644 artifact/policies/xSTM/libs/stm_instances/include/loadstore.h create mode 100644 artifact/policies/xSTM/libs/stm_instances/include/mem.h create mode 100644 artifact/policies/xSTM/libs/stm_instances/include/memfuncs.h create mode 100644 artifact/policies/xSTM/libs/stm_instances/include/stats.h create mode 100644 artifact/policies/xSTM/libs/stm_instances/orec_gv1_eager_c1_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/orec_gv1_eager_c2_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/orec_gv1_lazy_c1_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/orec_gv1_lazy_c2_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/orec_tsc_eager_c1_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/orec_tsc_eager_c2_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/orec_tsc_lazy_c1_q.cc create mode 100644 artifact/policies/xSTM/libs/stm_instances/orec_tsc_lazy_c2_q.cc create mode 100644 artifact/policies/xSTM/libs/tm_names.mk create mode 100644 artifact/policies/xSTM/plugin/Makefile create mode 100644 artifact/policies/xSTM/plugin/plugin/.gitignore create mode 100644 artifact/policies/xSTM/plugin/plugin/CMakeLists.txt create mode 100644 artifact/policies/xSTM/plugin/plugin/Makefile create mode 100644 artifact/policies/xSTM/plugin/plugin/README.md create mode 100644 artifact/policies/xSTM/plugin/plugin/boundary_transform.cc create mode 100644 artifact/policies/xSTM/plugin/plugin/discovery.cc create mode 100644 artifact/policies/xSTM/plugin/plugin/function_transform.cc create mode 100644 artifact/policies/xSTM/plugin/plugin/local_config.h create mode 100644 artifact/policies/xSTM/plugin/plugin/mappings.cc create mode 100644 artifact/policies/xSTM/plugin/plugin/optimizations.cc create mode 100644 artifact/policies/xSTM/plugin/plugin/raii_lite.cc create mode 100644 artifact/policies/xSTM/plugin/plugin/signatures.cc create mode 100644 artifact/policies/xSTM/plugin/plugin/signatures.h create mode 100644 artifact/policies/xSTM/plugin/plugin/tm_plugin.cc create mode 100644 artifact/policies/xSTM/plugin/plugin/tm_plugin.h create mode 100644 artifact/policies/xSTM/plugin/plugin/types.h create mode 100644 artifact/scripts/.gitignore create mode 100644 artifact/scripts/ChartCfg.py create mode 100644 artifact/scripts/ExpCfg.py create mode 100644 artifact/scripts/GetData.py create mode 100644 artifact/scripts/MakeChart.py create mode 100644 artifact/scripts/Makefile create mode 100644 artifact/scripts/README.md create mode 100644 artifact/scripts/Runner.py create mode 100644 artifact/scripts/Targets.py create mode 100644 artifact/scripts/Types.py create mode 100644 artifact/scripts/Util.py create mode 100644 artifact/ubench/Makefile create mode 100644 artifact/ubench/README.md create mode 100644 artifact/ubench/STMCAS/Makefile create mode 100644 artifact/ubench/STMCAS/build.mk create mode 100644 artifact/ubench/STMCAS/common.mk create mode 100644 artifact/ubench/STMCAS/dlist_carumap.cc create mode 100644 artifact/ubench/STMCAS/dlist_caumap.cc create mode 100644 artifact/ubench/STMCAS/dlist_omap.cc create mode 100644 artifact/ubench/STMCAS/dlist_opt_caumap.cc create mode 100644 artifact/ubench/STMCAS/dlist_opt_omap.cc create mode 100644 artifact/ubench/STMCAS/ibst_omap.cc create mode 100644 artifact/ubench/STMCAS/rbtree_omap.cc create mode 100644 artifact/ubench/STMCAS/skiplist_cached_opt_omap.cc create mode 100644 artifact/ubench/STMCAS/slist_omap.cc create mode 100644 artifact/ubench/STMCAS/slist_opt_caumap.cc create mode 100644 artifact/ubench/baseline/Makefile create mode 100644 artifact/ubench/baseline/ebst_ticket_omap.cc create mode 100644 artifact/ubench/baseline/iavl_pathcas_omap.cc create mode 100644 artifact/ubench/baseline/ibst_pathcas_omap.cc create mode 100644 artifact/ubench/baseline/lazylist_caumap.cc create mode 100644 artifact/ubench/baseline/lazylist_omap.cc create mode 100644 artifact/ubench/baseline/lfskiplist_omap.cc create mode 100644 artifact/ubench/config.mk create mode 100644 artifact/ubench/handSTM/Makefile create mode 100644 artifact/ubench/handSTM/build.mk create mode 100644 artifact/ubench/handSTM/common.mk create mode 100644 artifact/ubench/handSTM/dlist_carumap.cc create mode 100644 artifact/ubench/handSTM/dlist_caumap.cc create mode 100644 artifact/ubench/handSTM/ibst_omap.cc create mode 100644 artifact/ubench/handSTM/rbtree_omap.cc create mode 100644 artifact/ubench/handSTM/skiplist_omap_bigtx.cc create mode 100644 artifact/ubench/handSTM/slist_omap.cc create mode 100644 artifact/ubench/hybrid/Makefile create mode 100644 artifact/ubench/hybrid/build.mk create mode 100644 artifact/ubench/hybrid/common.mk create mode 100644 artifact/ubench/hybrid/dlist_carumap.cc create mode 100644 artifact/ubench/hybrid/rbtree_omap_drop.cc create mode 100644 artifact/ubench/include/bench_thread_context.h create mode 100644 artifact/ubench/include/config.h create mode 100644 artifact/ubench/include/experiment.h create mode 100644 artifact/ubench/include/experiment_pathcas.h create mode 100644 artifact/ubench/include/launch.h create mode 100644 artifact/ubench/include/launch_multi.h create mode 100644 artifact/ubench/include/manager.h create mode 100644 artifact/ubench/xSTM/Makefile create mode 100644 artifact/ubench/xSTM/build.mk create mode 100644 artifact/ubench/xSTM/common.mk create mode 100644 artifact/ubench/xSTM/ibst_omap.cc diff --git a/Artifact.md b/Artifact.md new file mode 100644 index 0000000..38a2e98 --- /dev/null +++ b/Artifact.md @@ -0,0 +1,151 @@ +# Artifact: exoTM/STMCAS Mechanisms, Policies, and Data Structures + +## Abstract + +This artifact consists of synchronization libraries and data structures for +evaluating the performance of the exoTM synchronization mechanism and STMCAS +synchronization policy. It consists of synchronization libraries, data +structure implementations, and microbenchmarks for stress-testing those data +structures. The code requires an Intel CPU with support for the `rdtscp` +instruction, which has been available on most Intel CPUs for more than 10 years. +For the most meaningful evaluation, a system with a large number of cores is +recommended. The provided Dockerfile handles all of the necessary software +dependencies. + +## Description + +This repository consists of the following components: + +* Synchronization Policies (`artifact/policies`) +* Data Structures (`artifact/ds`) +* Microbenchmarks (`artifact/ubench`) +* Evaluation Scripts (`artifact/scripts`) +* Build Environment (`Docker`) + +### Synchronization Policies + +This artifact considers five synchronization policies + +* Compiler-based STM (xSTM) +* Hand-instrumented STM (handSTM) +* Software Transactional Multiword Compare and Swap (STMCAS) +* handSTM+STMCAS (hybrid) +* Traditional blocking/nonblocking approaches (baseline) + +Each synchronization policy can be found in a subfolder of `artifact/policies`. +Most policies are "header-only" C++ files, which do not require special +compilation. The exception is xSTM, for which we provide a version of the +llvm-transmem TM plugin for C++. + +### Data Structures + +This artifact includes several data structures implemented with STMCAS +(doubly-linked list, skip list, singly-linked list, closed addressing resizable +unordered map, binary search tree, red/black tree). As appropriate, these data +structures are also provided for other synchronization policies. The `ds` +folder holds all data structures. The subfolders of `ds` correspond to the +different synchronization policies. + +### Microbechmarks + +The artifact's microbenchmark harness runs a stress test microbenchmark. The +microbenchmark has a variety of configuration options, some related to the data +structure's configuration (e.g., initial size of the unordered map), others +related to the experiment's configuration (e.g., operation mix, number of +threads). + +### Build Environment + +The easiest way to set up an appropriate build environment is to build a Docker +container. The included `Dockerfile` has instructions for building an +appropriate container. The dependencies are relatively minimal: + +* Ubuntu 22.04 +* Clang++ 15 +* CMake (only needed for xSTM) +* Standard Linux build tools +* Standard Python3 charting tools + +## Hardware Dependencies + +This artifact has been tested on a system with 192 GB of RAM and two Intel Xeon +Platinum 8160 CPUs (48 threads / 96 cores), running Ubuntu 22.04. In general, +any modern x86 CPU should work. The exoTM/STMCAS codes do not require many +advanced x86 features. The most noteworthy is the `rdtscp` instruction, which +has been available in most Intel processors for over a decade. + +Please note that the baseline data structures based on the PathCAS +synchronization methodology require support for Intel TSX. If you do not have a +machine with TSX support, you will need to comment out lines 112/113 and 138/139 +in `artifact/scripts/Targets.py`. Otherwise the automated testing/charting +scripts will fail. + +## Software Dependencies + +This artifact was developed and tested on Linux systems, running a variety of +kernel versions. The xSTM policy that we compare against requires Clang 15, so +we have opted to use Clang throughout the artifact. Our build configuration +uses the `-std=c++20` flag, but we do not require any particularly advanced +features (e.g., no concepts or coroutines). For exoTM/STMCAS, any modern C++ +compiler should be satisfactory. + +## Data Sets + +The artifact does not require any special data sets. + +## Instructions for Repeating the Experiments in the Paper + +If you wish to repeat the experiments from our paper, follow these instructions: + +1. Check out this repository (`git clone git@github.com:exotm/pact23.git`) +2. Build the Docker image (`cd Docker && sudo docker build -t exotm_ae . && cd ..`) +3. Launch a container (`sudo docker run --privileged --rm -v $(pwd):/root -it exotm_ae`) +4. Build and run (`make`) + +Please note that the Docker image will require roughly 1.7 GB of disk space. To +check out and build the source code will require another 60 MB. + +Also note that you will probably want to run a parallel make command in step 4 +(e.g., `make -j 16`). + +### Experiment Workflow + +The top-level Makefile first builds all necessary executable files. Please see +the README.md files in subfolders for more details. In general, each data +structure will produce its own executable. + +Once all executables are built, the Makefile will invoke `scripts/Runner.py` to +collect data and plot charts. For the charts in the paper, this script took +about 6 hours to run, and required about 1GB of space to store the charts and +data files. + +When the script completes, the `scripts/data` folder will hold all results. The +charts can be found in the `scripts/charts` folder. A second set of charts, +with error bars, can be found in `scripts/variance`. + +Note that typing `make clean` will remove all build artifacts and also all +experiment results and charts. + +## Instructions for Reusing the Artifact (Adding New Data Structures) + +Below we discuss the process one can use to add new data structures. + +1. Create a new `.h` file with the implementation of the data structure. This + should go in the appropriate sub-folder of `artifact/ds`, based on the + synchronization policy used by the data structure. +2. Create a new `.cc` file in the appropriate sub-folder of `artifact/ubench`, + depending on the synchronization policy used by the data structure. Note + that these files are typically quite small (~7 lines), as they only include + other files, define some types, and invoke a policy's initializer. +3. In the same folder as the `.cc` file, add the `.cc` file's name (without an + extension) to the `DS` variable in the `common.mk` file. Typing `make` + should now build a version of the microbenchmark for testing the new data + structure. Under rare circumstances, the Makefile might issue a warning + about duplicate rules in the generated `rules.mk` file. Should this happen, + type `make clean` and then `make` (or `make -j 16`, for a parallel build). +4. To integrate the new data structure into the test scripts for an existing + chart, first add it to the `exeNames` listing in + `artifact/scripts/ExpCfg.py`. Then locate the chart(s) to augment in + `artifact/scripts/Targets.py` and add a new `Curve` with a matching + `exeName`. + diff --git a/Docker/Dockerfile b/Docker/Dockerfile new file mode 100644 index 0000000..3e3db3c --- /dev/null +++ b/Docker/Dockerfile @@ -0,0 +1,36 @@ +# Dockerfile to build llvm-15 developer image +FROM ubuntu:jammy + +# Apply all updates +RUN apt-get update -y +RUN apt-get upgrade -y + +# Install basic C++ and Python development tools +RUN DEBIAN_FRONTEND=noninteractive apt install -y build-essential cmake g++-multilib pip + +# Install LLVM 15 +RUN DEBIAN_FRONTEND=noninteractive apt install -y wget gnupg gnupg2 gnupg1 lsb-release software-properties-common +RUN DEBIAN_FRONTEND=noninteractive wget https://apt.llvm.org/llvm.sh +RUN DEBIAN_FRONTEND=noninteractive chmod +x llvm.sh +RUN DEBIAN_FRONTEND=noninteractive ./llvm.sh 15 +RUN DEBIAN_FRONTEND=noninteractive rm ./llvm.sh + +# Install Python charting tools +RUN pip3 install --no-cache-dir numpy matplotlib + +# Set the working directory +WORKDIR /root + +# To use this Dockerfile +# 1 - Make an image named exotm_ae +# - Go to the folder where this Dockerfile exists +# - sudo docker build -t exotm_ae . +# - Note: you don't need the 'sudo' part on Windows +# - The resulting image size will be about 1.64 GB +# 2 - Go to top level folder +# 3 - Launch an interactive container, and mount your working folder +# - sudo docker run --privileged --rm -v $(pwd):/root -it exotm_ae +# 4 - When your terminal starts up: +# - You will be logged in as root, and in the `/root` folder +# - You should see your exotm folder's contents in there +# - Type 'make' to run all experiments and build all charts diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d6162e6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Yaodong Sheng, Ahmed Hassan, Michael Spear + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b6de3d5 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +all: + $(MAKE) -C artifact + +clean: + $(MAKE) -C artifact clean diff --git a/artifact/Makefile b/artifact/Makefile new file mode 100644 index 0000000..ae0a7dc --- /dev/null +++ b/artifact/Makefile @@ -0,0 +1,8 @@ +all: + $(MAKE) -C policies/xSTM + $(MAKE) -C ubench + $(MAKE) -C scripts +clean: + $(MAKE) -C policies/xSTM clean + $(MAKE) -C ubench clean + $(MAKE) -C scripts clean diff --git a/artifact/ds/README.md b/artifact/ds/README.md new file mode 100644 index 0000000..ae0e992 --- /dev/null +++ b/artifact/ds/README.md @@ -0,0 +1,14 @@ +# Data Structures + +This folder stores the data structures that we use in our evaluation. They are +organized according to the synchronization policy they employ. + +The data structures in the `baseline` folder are taken from the open-source +repositories that correspond to those works. We have modified them in the +following ways: + +- We have converted code, as necessary, to move the data structure + implementation entirely to headers. +- We have modified the data structures to use the facilities in the + `policies/baseline` policy, so that there is an apples-to-apples comparison + with regard to hashing, random numbers, and safe memory reclamation. diff --git a/artifact/ds/STMCAS/dlist_carumap.h b/artifact/ds/STMCAS/dlist_carumap.h new file mode 100644 index 0000000..1994a50 --- /dev/null +++ b/artifact/ds/STMCAS/dlist_carumap.h @@ -0,0 +1,646 @@ +#pragma once + +#include +#include +#include + +/// An unordered map, implemented as a resizable array of lists (closed +/// addressing, resizable). This map supports get(), insert() and remove() +/// operations. +/// +/// This implementation is based loosely on Liu's nonblocking resizable hash +/// table from PODC 2014. At the current time, we do not support the heuristic +/// for contracting the list, but we do support expanding the list. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param STMCAS The STMCAS implementation (PO or PS) +template class dlist_carumap { + using WSTEP = typename STMCAS::WSTEP; + using RSTEP = typename STMCAS::RSTEP; + using snapshot_t = typename STMCAS::snapshot_t; + using ownable_t = typename STMCAS::ownable_t; + template using FIELD = typename STMCAS::template sField; + + /// A list node. It has prev and next pointers, but no key or value. It's + /// useful for sentinels, so that K and V don't have to be default + /// constructable. + /// + /// NB: we do not need a `valid` bit, because any operation that would clear + /// it would also acquire this node's orec, and thus any node that would + /// encounter a cleared valid bit would also detect an orec inconsistency. + struct node_t : ownable_t { + FIELD prev; // Pointer to predecessor + FIELD next; // Pointer to successor + + /// Construct a node + node_t() : ownable_t(), prev(nullptr), next(nullptr) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~node_t() {} + }; + + /// We need to know if buckets have been rehashed to a new table. We do this + /// by making the head of each bucket a `sentinel_t`, and adding a `closed` + /// bool. Note that the tail of each bucket's list is just a node_t. + struct sentinel_t : node_t { + /// Track if this sentinel is for a bucket that has been rehashed + /// + /// NB: Could we use `prev` to indicated `closed` + FIELD closed; // Has it been rehashed? + + /// Construct a sentinel_t + sentinel_t() : node_t(), closed(false) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~sentinel_t() {} + }; + + /// A list node that also has a key and value. Note that keys are const, and + /// values are only accessed while the node is locked, so neither is a + /// tm_field. + struct data_t : node_t { + const K key; // The key of this key/value pair + V val; // The value of this key/value pair + + /// Construct a data_t + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(const K &_key, const V &_val) : node_t(), key(_key), val(_val) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~data_t() {} + }; + + /// An array of lists, along with its size + /// + /// NB: to avoid indirection, the array is in-lined into the tbl_t. To make + /// this compatible with SMR, tbl_t must be ownable. + class tbl_t : public ownable_t { + using bucket_t = FIELD; + + /// Construct a table + /// + /// @param `_size` The desired size of the table + tbl_t(uint64_t _size) : size(_size) {} + + public: + const uint64_t size; // The size of the table + bucket_t tbl[]; // The buckets of the table + + /// Allocate a tbl_t of size `size` + /// + /// @param size The desired size + /// @param tx The calling operation's descriptor + /// + /// @return A table, all of whose buckets are set to null + static tbl_t *make(uint64_t size, WSTEP &tx) { + tbl_t *tbl = (tbl_t *)malloc(sizeof(tbl_t) + size * sizeof(bucket_t)); + auto ret = new (tbl) tbl_t(size); + for (size_t i = 0; i < size; ++i) + ret->tbl[i].set(nullptr, tx); + return ret; + } + }; + + ownable_t *tbl_orec; // An orec for protecting `active` and `frozen` + FIELD active; // The active table + FIELD frozen; // The frozen table + std::hash _pre_hash; // A weak hash function for converting keys to ints + const uint64_t RESIZE_THRESHOLD; // Max bucket size before resizing + + /// A pair consisting of a pointer and an orec version. + struct node_ver_t { + node_t *_obj = nullptr; // The start of a bucket + uint64_t _ver = 0; // NB: _ver may not be related to _obj + }; + + /// Result of trying to resize a bucket + enum resize_result_t { + CANNOT_ACQUIRE, // Couldn't get orec... retry + ALREADY_RESIZED, // Already resized by another thread + RESIZE_OK // Bucket successfully resized + }; + + /// Given a key, determine the bucket into which it should go. As in the Liu + /// hash, we do not change the hash function when we resize, we just change + /// the number of bits to use + /// + /// @param key The key to hash + /// @param size The size of the table into which this should be hashed + /// + /// @return An integer between 0 and size + uint64_t table_hash(STMCAS *me, const K &key, const uint64_t size) const { + return me->hash(_pre_hash(key)) % size; + } + +public: + /// Default construct a map as having a valid active table. + /// + /// NB: This constructor calls std::terminate if the provided size is not a + /// power of 2. + /// + /// @param me The operation that is creating this umap + /// @param cfg A config object with `buckets` and `resize_threshold` + dlist_carumap(STMCAS *me, auto *cfg) + : tbl_orec(new ownable_t()), RESIZE_THRESHOLD(cfg->resize_threshold) { + // Enforce power-of-2 initial size + if (std::popcount(cfg->buckets) != 1) + throw("cfg->buckets should be power of 2"); + + // Create an initial active table in which all of the buckets are + // initialized but empty (null <- head <-> tail -> null). + WSTEP tx(me); + active.set(tbl_t::make(cfg->buckets, tx), tx); + for (size_t i = 0; i < cfg->buckets; ++i) + active.get(tx)->tbl[i].set(create_list(tx), tx); + // NB: since all buckets are initialized, nobody will ever go to the + // frozen table, so we can leave it as null + frozen.set(nullptr, tx); + } + +private: + /// Create a dlist with head and tail sentinels + /// + /// @param tx A writing TM context. Even though this code can't fail, we need + /// the context in order to use tm_field correctly. + /// + /// @return A pointer to the head sentinel of the list + sentinel_t *create_list(WSTEP &tx) { + // NB: By default, a node's prev and next will be nullptr, which is what we + // want for head->prev and tail->next. + auto head = new sentinel_t(); + auto tail = new node_t(); + head->next.set(tail, tx); + tail->prev.set(head, tx); + return head; + } + + /// `resize()` is an internal method for changing the size of the active + /// table. Strictly speaking, it should be called `expand`, because for now we + /// only support expansion, not contraction. When `insert()` discovers that + /// it has made a bucket "too big", it will continue to do its insertion and + /// then, after linearizing, it will call `resize()`. `remove()` does not + /// currently call `resize()`. + /// + /// At a high level, `resize()` is supposed to be open-nested and not to incur + /// any blocking, except due to orec conflicts. We accomplish this through + /// laziness and stealing. resize() finishes the /last/ resize, moves the + /// `active` table to `frozen`, and installs a new `active` table. Subsequent + /// operations will do most of the migrating. + /// + /// @param me The calling thread's descriptor + /// @param a_ver The version of `active` when the resize was triggered + void resize(STMCAS *me, uint64_t a_ver) { + // Get the current active and frozen tables, and the frozen table size + tbl_t *ft = nullptr, *at = nullptr; + { + RSTEP tx(me); + ft = frozen.get(tx); + at = active.get(tx); + if (!tx.check_continuation(tbl_orec, a_ver)) + return; // someone else must be starting a resize, so we can quit + } + + // If ft is null, then there's no frozen table, so things will be easy + if (ft == nullptr) { + WSTEP tx(me); + + // Make and initialize the table *before* acquiring orecs, to minimize the + // critical section. The table is 2x as big. + auto new_tbl = tbl_t::make(at->size * 2, tx); + + // Lock the table, move it from `active` to `frozen`, then install the new + // table. + if (!tx.acquire_continuation(tbl_orec, a_ver)) { + // NB: new_tbl is private. We don't need SMR + delete new_tbl; + return; // Someone else is resizing, and that's good enough for `me` + } + frozen.set(at, tx); + active.set(new_tbl, tx); + return; + } + + // Migrate everything out of frozen, remove the frozen table, and retry + // + // NB: prepare_resize removes the frozen table. That will change a_ver, so + // we need to capture the new a_ver value so that our next attempt won't + // fail erroneously. + a_ver = prepare_resize(me, a_ver, ft, at); + if (a_ver == 0) + return; // Someone else finished resizing for `me` + + resize(me, a_ver); // Try again now that it's clean + } + + /// Finish one lazy resize, so that another may begin. + /// + /// This really just boils down to migrating everything from `frozen` to + /// `active` and then nulling `frozen` and reclaiming it. + /// + /// NB: This code takes the "frozen" and "active" tables as arguments. + /// Consequently, we don't care about arbitrary delays. If a thread calls + /// this, rehashes half the table, and then suspends, another thread can + /// rehash everything else and install a new active table. When the first + /// thread wakes, it'll find a bunch of empty buckets, and it'll be safe. + /// + /// @param me The calling thread's descriptor + /// @param a_ver The active table version when this was called + /// @param f_tbl The "frozen table", really the "source" table + /// @param a_tbl The "active table", really the "destination" table + /// + /// @return {0} if another thread stole the job of nulling `frozen`. + /// When this happens, there must be a concurrent resize, + /// and since both are trying to do the same thing (expand), + /// the one who receives {0} can just get out of the other's + /// way + /// {integer} the new orec version of `active` + uint64_t prepare_resize(STMCAS *me, uint64_t a_ver, tbl_t *f_tbl, + tbl_t *a_tbl) { + // NB: Right now, next_index == completed. If we randomized the start + // point, concurrent calls to prepare_resize() would contend less + uint64_t next_index = 0; // Next bucket to migrate + uint64_t completed = 0; // Number of buckets migrated + + // Migrate all data from `frozen` to `active` + while (completed != f_tbl->size) { + WSTEP tx(me); + + // Try to rehash the next bucket + auto bucket = f_tbl->tbl[next_index].get(tx); + auto res = + rehash_expand_bucket(me, bucket, next_index, f_tbl->size, a_tbl, tx); + // If we can't acquire all nodes in this bucket, try again, because it + // might just mean someone else was doing an operation in the bucket. + if (res == CANNOT_ACQUIRE) { + tx.unwind(); + continue; + } + // If this bucket is already rehashed by others, there is a chance that + // the current resize phase is finished, so check + if (res == ALREADY_RESIZED) { + // check if the active table version changed since resize() was + // called, if so, we know resize is finished, return + if (!tx.check_continuation(tbl_orec, a_ver)) { + tx.unwind(); + return 0; + } + } + + // Move to the next bucket + ++next_index; + ++completed; + } + + // Uninstall the `frozen` table, since it has been emptied. Save the commit + // time, so we can validate tbl_orec later. + tbl_t *old; + { + WSTEP tx(me); + if (tx.acquire_continuation(tbl_orec, a_ver)) { + old = f_tbl; + frozen.set(nullptr, tx); + } else + return 0; + } + auto last_commit_time = me->get_last_wo_end_time(); + + // Reclaim `old`'s buckets, then `old` itself + { + WSTEP tx(me); + for (size_t i = 0; i < f_tbl->size; i++) { + // use singleton_reclaim to reclaim head and tail of each bucket + auto head = old->tbl[i].get(tx); + auto tail = head->next.get(tx); + tx.reclaim(head); + tx.reclaim(tail); + } + tx.reclaim(old); + } + return last_commit_time; + } + + /// Get a pointer to the bucket in the active table that holds `key`. This + /// may cause some rehashing to happen. + /// + /// NB: The pattern here is unconventional. get_bucket() is the first step in + /// WSTEP transactions. If it doesn't rehash, then the caller WSTEP + /// continues its operation. If it does rehash, then the caller WSTEP + /// commits and restarts, which is a poor-man's open-nested transaction. + /// If it encounters an inconsistency, the caller WSTEP should "abort" by + /// unwinding and restarting. In the third case, this returns *while + /// holding an orec* + /// + /// @param me The calling thread's descriptor + /// @param key The key whose bucket is sought + /// @param tx An active WSTEP transaction + /// + /// @return On success, a pointer to the head of a bucket, along with + /// `tbl_orec`'s value. {nullptr, 0} on any rehash or inconsistency + node_ver_t get_bucket(STMCAS *me, const K &key, WSTEP &tx) { + // Get the head of the appropriate bucket in the active table + // + // NB: Validate or else a_tbl[a_idx] could be out of bounds + auto a_tbl = active.get(tx); + uint64_t a_ver = tx.check_orec(tbl_orec); + if (a_ver == STMCAS::END_OF_TIME) + return {nullptr, a_ver}; + auto a_idx = table_hash(me, key, a_tbl->size); + auto a_bucket = a_tbl->tbl[a_idx].get(tx); // NB: caller will validate + if (a_bucket) + return {a_bucket, a_ver}; // not null --> no resize needed + + // Find the bucket in the frozen table that needs rehashing + auto f_tbl = frozen.get(tx); + if (tx.check_orec(tbl_orec) == STMCAS::END_OF_TIME) + return {nullptr, 0}; // this op delayed, rehash finished by someone else! + auto f_idx = table_hash(me, key, f_tbl->size); + auto f_bucket = f_tbl->tbl[f_idx].get(tx); + if (!tx.acquire_consistent(f_bucket)) + return {nullptr, 0}; // someone else is using the old bucket + + // Rehash it, tell caller to commit so the rehash appears to be open nested + // + // NB: if the rehash fails, it's due to someone else rehashing, which is OK + rehash_expand_bucket(me, f_bucket, f_idx, f_tbl->size, a_tbl, tx); + return {nullptr, 0}; + } + + /// Re-hash one list in the frozen table into two lists in the active table + /// + /// @param me The calling thread's descriptor + /// @param f_list A pointer to an (acquired!) list head in the frozen table + /// @param f_idx The index of flist in the frozen table + /// @param f_size The size of the frozen table + /// @param a_tbl A reference to the active table + /// @param tx An active WSTEP transaction + /// + /// @return RESIZE_OK - The frozen bucket was rehashed into `a_tbl` + /// ALREADY_RESIZED - The frozen bucket was empty + /// CANNOT_ACQUIRE - The operation could not acquire all orecs + resize_result_t rehash_expand_bucket(STMCAS *me, sentinel_t *f_list, + uint64_t f_idx, uint64_t f_size, + tbl_t *a_tbl, WSTEP &tx) { + // Stop if this bucket is already rehashed + if (f_list->closed.get(tx)) // true is effectively const, skip validation + return ALREADY_RESIZED; + // Fail if we cannot acquire all nodes in f_list + if (!list_acquire_all(f_list, tx)) + return CANNOT_ACQUIRE; + + // Shuffle nodes from f_list into two new lists that will go into `a_tbl` + auto l1 = create_list(tx), l2 = create_list(tx); + auto curr = f_list->next.get(tx); + while (curr->next.get(tx) != nullptr) { + auto next = curr->next.get(tx); + auto data = static_cast(curr); + auto dest = table_hash(me, data->key, a_tbl->size) == f_idx ? l1 : l2; + auto succ = dest->next.get(tx); + dest->next.set(data, tx); + data->next.set(succ, tx); + data->prev.set(dest, tx); + succ->prev.set(data, tx); + curr = next; + } + // curr is tail, set head->tail + f_list->next.set(curr, tx); + // put the lists into the active table, close the frozen bucket + a_tbl->tbl[f_idx].set(l1, tx); + a_tbl->tbl[f_idx + f_size].set(l2, tx); + f_list->closed.set(true, tx); + return RESIZE_OK; + } + + /// Acquire all of the nodes in the list starting at `head`, including the + /// head and tail sentinels + /// + /// @param head The head of the list whose nodes should be acquired + /// @param tail The calling WSTEP transaction + /// + /// @return true if all nodes are acquired, false otherwise + bool list_acquire_all(node_t *head, WSTEP &tx) { + node_t *curr = head; + while (curr) { + if (!tx.acquire_consistent(curr)) + return false; + curr = curr->next.get(tx); + } + return true; + } + + /// Given the head sentinel of a list, search through the list to find the + /// node with key `key`, if such a node exists in the list. If it doesn't, + /// then return the head pointer, along with a count of non-sentinel nodes in + /// the list + /// + /// @param key The key for which we are searching + /// @param head The start of the list to search + /// @param tx An active WSTEP transaction + /// + /// @return {nullptr, 0} if the transaction discovered an inconsistency + /// {head, count} if the key was not found + /// {node, 0} if the key was found at `node` + std::pair list_get_or_head(const K &key, sentinel_t *head, + WSTEP &tx) { + // Get the head's successor; on any inconsistency, return. + auto curr = head->next.get(tx); + uint64_t head_orec = tx.check_orec(head); + if (head_orec == STMCAS::END_OF_TIME) + return {nullptr, 0}; + + uint64_t count = 0; // Number of nodes encountered during the loop + + while (true) { + // if we reached the tail, return the head + if (curr->next.get(tx) == nullptr) + return {head, count}; // No validation: tail's next is effectively const + + // return curr if it has a matching key + if (static_cast(curr)->key == key) + return {curr, 0}; + + // read `next` consistently + // + // NB: We could skip this, and just validate before `return {curr, 0}` + auto next = curr->next.get(tx); + if (tx.check_orec(curr) == STMCAS::END_OF_TIME) + return {nullptr, 0}; + curr = next; + ++count; + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(STMCAS *me, const K &key, V &val) { + while (true) { + WSTEP tx(me); + // Get the bucket in `active` where `key` should be. "Abort" and retry on + // any inconsistency; commit and retry if `get_bucket` resized + auto [bucket, _] = get_bucket(me, key, tx); + if (!bucket) + continue; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + auto [node, __] = + list_get_or_head(key, static_cast(bucket), tx); + + // If we got back null, there was an inconsistency, so retry + if (!node) { + tx.unwind(); + continue; + } + + // If we got back the head, return false + if (node == bucket) { + tx.unwind(); // because we didn't update shared memory + return false; + } + + if (std::is_scalar::value) { + data_t *dn = static_cast(node); + V val_copy = reinterpret_cast *>(&dn->val)->load( + std::memory_order_acquire); + if (tx.check_orec(node) == STMCAS::END_OF_TIME) { + tx.unwind(); + continue; + } + val = val_copy; + return true; + } else { + // Acquire, read, unwind (because no writes!) + if (!tx.acquire_consistent(node)) { + tx.unwind(); + continue; + } + val = static_cast(node)->val; + tx.unwind(); + return true; + } + } + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(STMCAS *me, const K &key, V &val) { + // If we discover that a bucket becomes too full, we'll insert, linearize, + // and then resize in a new transaction before returning. Tracking + // `active`'s version prevents double-resizing under concurrency. + uint64_t a_ver = 0; + while (true) { + WSTEP tx(me); + auto [bucket, a_version] = get_bucket(me, key, tx); + if (!bucket) + continue; + a_ver = a_version; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + auto [node, count] = + list_get_or_head(key, static_cast(bucket), tx); + + // If we got back null, there was an inconsistency, so retry + if (!node) { + tx.unwind(); + continue; + } + + // If we didn't get the head, the key already exists, so return false + if (node != bucket) { + tx.unwind(); + return false; + } + + // Lock the node and its successor + if (!tx.acquire_consistent(node)) { + tx.unwind(); + continue; + } + auto next = node->next.get(tx); + if (!tx.acquire_aggressive(next)) { + tx.unwind(); + continue; + } + + // Stitch in a new node + data_t *new_dn = new data_t(key, val); + new_dn->next.set(next, tx); + new_dn->prev.set(node, tx); + node->next.set(new_dn, tx); + next->prev.set(new_dn, tx); + if (count >= RESIZE_THRESHOLD) + break; // need to resize! + return true; + } + + resize(me, a_ver); + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(STMCAS *me, const K &key) { + while (true) { + WSTEP tx(me); + // Get the bucket in `active` where `key` should be. Abort and retry on + // any inconsistency; commit and retry if `get_bucket` resized + auto [bucket, _] = get_bucket(me, key, tx); + if (!bucket) + continue; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + // + // NB: While `bucket` has not been reclaimed, `active.tbl` may have + // changed. Fortunately, list_get_or_head will validate it. + auto [node, __] = + list_get_or_head(key, static_cast(bucket), tx); + + // If we got back the head, return false + if (node == bucket) { + tx.unwind(); // because we didn't update shared memory + return false; + } + + // If the `node` is null, list_get_or_head failed and we need to retry + // Otherwise, it's unowned and the keys match, so lock `node` and its + // neighbors, else retry + if (!node || !tx.acquire_consistent(node) || + !tx.acquire_aggressive(node->prev.get(tx)) || + !tx.acquire_aggressive(node->next.get(tx))) { + tx.unwind(); + continue; + } + + // unstitch it + auto pred = node->prev.get(tx), succ = node->next.get(tx); + pred->next.set(succ, tx); + succ->prev.set(pred, tx); + tx.reclaim(node); + return true; + } + } +}; diff --git a/artifact/ds/STMCAS/dlist_omap.h b/artifact/ds/STMCAS/dlist_omap.h new file mode 100644 index 0000000..3cdd5b7 --- /dev/null +++ b/artifact/ds/STMCAS/dlist_omap.h @@ -0,0 +1,351 @@ +#pragma once + +#include +#include + +/// An ordered map, implemented as a doubly-linked list. This map supports +/// get(), insert(), and remove() operations. +/// +/// Note that the AVOID_OREC_CHECKS flag can be used to create an "optimized" +/// version of this data structure, where list traversal (get_leq) avoids +/// checking orecs in most cases. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param STMCAS The STMCAS implementation (PO or PS) +/// @param AVOID_OREC_CHECKS A flag to enable an optimization that avoids +/// checking orecs when get_leq is doing its read-only +/// traversal +template +class dlist_omap { + using WSTEP = typename STMCAS::WSTEP; + using RSTEP = typename STMCAS::RSTEP; + using snapshot_t = typename STMCAS::snapshot_t; + using ownable_t = typename STMCAS::ownable_t; + template using FIELD = typename STMCAS::template sField; + + /// A list node. It has prev and next pointers, but no key or value. It's + /// useful for sentinels, so that K and V don't have to be default + /// constructable. + /// + /// NB: we do not need a `valid` bit, because any operation that would clear + /// it would also acquire this node's orec, and thus any node that would + /// encounter a cleared valid bit would also detect an orec inconsistency. + struct node_t : ownable_t { + FIELD prev; // Pointer to predecessor + FIELD next; // Pointer to successor + + /// Construct a node + node_t() : ownable_t(), prev(nullptr), next(nullptr) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~node_t() {} + }; + + /// A list node that also has a key and value. Note that keys are const, and + /// values are only accessed while the node is locked, so neither is a + /// tm_field. + struct data_t : public node_t { + const K key; // The key of this key/value pair + V val; // The value of this key/value pair + + /// Construct a data_t + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(const K &_key, const V &_val) : node_t(), key(_key), val(_val) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~data_t() {} + }; + + /// The pair returned by predecessor queries: a node and it's observed version + struct leq_t { + node_t *_obj = nullptr; // The object + uint64_t _ver = 0; // The observed version of the object + }; + + node_t *const head; // The list head pointer + node_t *const tail; // The list tail pointer + + /// During get_leq, we have a way to periodically capture snapshots, so that a + /// failed search can resume from an intermediate point. This specifies how + /// frequently to take a snapshot (higher is less frequent, i.e., once per + /// SNAPSHOT_FREQUENCY nodes). + const int SNAPSHOT_FREQUENCY; + +public: + /// Default construct a list by constructing and connecting two sentinel nodes + /// + /// @param me The operation that is constructing the list + /// @param cfg A configuration object that has a `snapshot_freq` field + dlist_omap(STMCAS *me, auto *cfg) + : head(new node_t()), tail(new node_t()), + SNAPSHOT_FREQUENCY(cfg->snapshot_freq) { + // NB: Even though this code can't abort and doesn't acquire orecs, we still + // need to use a transaction (WSTEP), because we can't set fields of a + // node_t without a legal WSTEP context. We can cheat, though, and not + // bother to acquire orecs, because we know nothing is shared. + WSTEP tx(me); + head->next.set(tail, tx); + tail->prev.set(head, tx); + } + +private: + /// Convert a snapshot_t into a leq_t + leq_t leq(const snapshot_t &s) { return leq_t{(node_t *)s._obj, s._ver}; } + + /// Convert a leq_t into a snapshot_t + snapshot_t snapshot(const leq_t &l) { return snapshot_t{l._obj, l._ver}; } + + /// get_leq is an inclusive predecessor query that returns the largest node + /// whose key is <= the provided key. It can return the head sentinel, but + /// not the tail sentinel. + /// + /// There is no atomicity between get_leq and its caller. It returns the node + /// it found, along with the value of the orec for that node at the time it + /// was accessed. The caller needs to validate the orec before using the + /// returned node. + /// + /// @param me The calling thread's descriptor + /// @param key The key for which we are doing a predecessor query. + /// + /// @return The node that was found, and its orec value + leq_t get_leq(STMCAS *me, const K key) { + // Start a transactional traversal from the head node, or from the latest + // valid snapshot, if we have one. If a transaction encounters an + // inconsistency, it will come back to here to start a new traversal. + while (true) { + RSTEP tx(me); + + // Figure out where to start this traversal: initially we start at head, + // but on a retry, we might have a snapshot. + // + // NB: snapshots are always < key + leq_t curr = + (me->snapshots.empty()) ? leq_t({head, 0}) : leq(me->snapshots.top()); + + // We need to validate the start point. A clever trick is that we know + // that it hasn't been reclaimed. Thus we can read its next pointer + // before validating. This helps with a pair of performance goals for the + // upcoming `while` loop: (1) read all fields before validating an object, + // and (2) avoid re-validating in subsequent loop iterations. + auto *next = curr._obj->next.get(tx); + + // Validate the start point + if (curr._obj == head) { + // For head, be sure to save curr._ver in case we end up returning head + if ((curr._ver = tx.check_orec(curr._obj)) == STMCAS::END_OF_TIME) + continue; + } else { + // Validate snapshot as a continuation. Drop the snapshot on failure. + if (!tx.check_continuation(curr._obj, curr._ver)) { + me->snapshots.drop(); + continue; + } + } + + // Prepare a countdown timer for snapshots + int nodes_until_snapshot = SNAPSHOT_FREQUENCY; + + // Starting at `next`, search for key. Breaking out of this will take us + // back to the top of the function. + while (true) { + // Case 1: `next` is tail --> stop the search at curr + if (next == tail) { + if (AVOID_OREC_CHECKS) { + if ((curr._ver = tx.check_orec(curr._obj)) == STMCAS::END_OF_TIME) + break; + } else { + // it's already validated + } + return curr; + } + + // read next's `next` and `key`, then validate + // + // NB: key is const, doesn't require validation, but it's free here :) + auto next_next = next->next.get(tx); + auto nkey = static_cast(next)->key; + uint64_t next_ver = 0; + if (!AVOID_OREC_CHECKS) { + if ((next_ver = tx.check_orec(next)) == STMCAS::END_OF_TIME) + break; // validation failure... goto top, get a new snapshot + } + + // Case 2: `next` is a data node: stop if next->key >= key + if (nkey > key) { + if (AVOID_OREC_CHECKS && + (curr._ver = tx.check_orec(curr._obj)) == STMCAS::END_OF_TIME) + break; + return curr; + } + if (nkey == key) { + if (AVOID_OREC_CHECKS) { + next_ver = tx.check_orec(next); + if (next_ver == STMCAS::END_OF_TIME) + break; // retry + } + return {next, next_ver}; + } + + // Case 3: keep traversing to `next`. Maybe take a snapshot first + if (--nodes_until_snapshot == 0) { + if (AVOID_OREC_CHECKS) { + if ((curr._ver = tx.check_orec(curr._obj)) != STMCAS::END_OF_TIME) + me->snapshots.push_back(snapshot(curr)); + } else + me->snapshots.push_back(snapshot(curr)); + nodes_until_snapshot = SNAPSHOT_FREQUENCY; + } + // NB: the way we pre-read things means only one check_orec per + // iteration + curr._obj = next; + if (!AVOID_OREC_CHECKS) + curr._ver = next_ver; + next = next_next; + } + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(STMCAS *me, const K &key, V &val) { + // If we can't use the result of get_leq, we'll loop back, and the next + // get_leq will start from a snapshot + me->snapshots.clear(); + while (true) { + // get_leq will use a read-only transaction to find the largest node with + // a key <= `key`. + // + // Postconditions of get_leq: n != null, n != tail, we have a valid + // node/version pair, and n.key <= `key` + auto n = get_leq(me, key); + + // Since we have EBR, we can read n.key without validating and fast-fail + // on key-not-found + if (n._obj == head || static_cast(n._obj)->key != key) + return false; + + // Use a hand-over-hand TM pattern to finish the get(). If the value is + // scalar, we can cast it to atomic, read it, and validate. Otherwise we + // need to lock the node. + if (std::is_scalar::value) { + RSTEP tx(me); + + // NB: given EBR, we don't need to worry about n._obj being deleted, so + // we don't need to validate before looking at the value + data_t *dn = static_cast(n._obj); + V val_copy = reinterpret_cast *>(&dn->val)->load( + std::memory_order_acquire); + if (!tx.check_continuation(n._obj, n._ver)) + continue; + val = val_copy; + return true; + } else { + WSTEP tx(me); + + // If this acquire_continuation succeeds, it's not deleted, it's a data + // node, and it's valid. If it fails, we need to restart + if (!tx.acquire_continuation(n._obj, n._ver)) { + tx.unwind(); // not strictly needed, but a good habit :) + continue; + } + + // NB: we aren't changing val, so we can unwind when we're done with it + val = static_cast(n._obj)->val; + tx.unwind(); + return true; + } + } + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(STMCAS *me, const K &key, V &val) { + // NB: The pattern here is similar to `get` + me->snapshots.clear(); + while (true) { + auto n = get_leq(me, key); + + // Since we have EBR, we can look at n._obj->key without validation. If + // it matches `key`, return false. + if (n._obj != head && static_cast(n._obj)->key == key) + return false; + + // Either n._obj is `head`, or it's a key that's too small. Let's insert! + WSTEP tx(me); + // lock n and n's successor, fail if we can't lock both + if (!tx.acquire_continuation(n._obj, n._ver)) { + tx.unwind(); + continue; + } + auto next = n._obj->next.get(tx); + if (!tx.acquire_aggressive(next)) { // NB: don't need consistency here + tx.unwind(); + continue; + } + + // stitch in a new node + data_t *new_dn = new data_t(key, val); + new_dn->next.set(next, tx); + new_dn->prev.set(n._obj, tx); + n._obj->next.set(new_dn, tx); + next->prev.set(new_dn, tx); + return true; + } + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(STMCAS *me, const K &key) { + // NB: The pattern here is similar to `get` + me->snapshots.clear(); + while (true) { + auto n = get_leq(me, key); + + if (n._obj == head || static_cast(n._obj)->key != key) + return false; + + WSTEP tx(me); + // lock n, then its neighbors + // + // NB: Locking `n` is the secret sauce for getting this all to work + // without mark bits. The orec change means that others will discover + // that they can't use `n`, which is exactly what we want + if (!tx.acquire_continuation(n._obj, n._ver) || + !tx.acquire_aggressive(n._obj->prev.get(tx)) || + !tx.acquire_aggressive(n._obj->next.get(tx))) { + tx.unwind(); + continue; + } + + // unstitch it + auto pred = n._obj->prev.get(tx), succ = n._obj->next.get(tx); + pred->next.set(succ, tx); + succ->prev.set(pred, tx); + tx.reclaim(n._obj); + return true; + } + } +}; diff --git a/artifact/ds/STMCAS/ibst_omap.h b/artifact/ds/STMCAS/ibst_omap.h new file mode 100644 index 0000000..000ac55 --- /dev/null +++ b/artifact/ds/STMCAS/ibst_omap.h @@ -0,0 +1,397 @@ +#pragma once + +#include +#include +#include + +/// An ordered map, implemented as an unbalanced, internal binary search tree. +/// This map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param STMCAS The STMCAS implementation (PO or PS) +template class ibst_omap { + using WSTEP = typename STMCAS::WSTEP; + using RSTEP = typename STMCAS::RSTEP; + using snapshot_t = typename STMCAS::snapshot_t; + using ownable_t = typename STMCAS::ownable_t; + template using FIELD = typename STMCAS::template sField; + + /// An easy-to-remember way of indicating the left and right children + enum DIRS { LEFT = 0, RIGHT = 1 }; + + /// node_t is the base type for all tree nodes. It doesn't have key/value + /// fields. + struct node_t : public ownable_t { + /// The node's children. Be sure to use LEFT and RIGHT to index it + FIELD children[2]; + + /// Construct a node_t. This should only be called from a writer + /// transaction + /// + /// @param tx A writing transactional context + /// @param _left The left child of this node + /// @param _right The right child of this node + node_t(WSTEP &tx, node_t *_left = nullptr, node_t *_right = nullptr) + : ownable_t() { + children[LEFT].set(_left, tx); + children[RIGHT].set(_right, tx); + } + }; + + /// A pair with ownable and orec value; equivalent to the type in snapshots + struct leq_t { + node_t *_obj = nullptr; // The object + uint64_t _ver = 0; // The observed version of the object + }; + + /// A pair holding a child node and its parent, with orec validation info + struct ret_pair_t { + leq_t child; // The child + leq_t parent; // The parent of that child + }; + + /// Our tree uses a sentinel root node, so that we always have a valid node + /// for which to compute an orec. The sentinel's *LEFT* child is the true + /// root of the tree. That is, logically sentinel has the value "TOP". + node_t *sentinel; + + /// data_t is the type for all internal and leaf nodes in the data structure. + /// It extends the base type with a key and value. + /// + /// NB: keys are *not* const, because we want to overwrite nodes instead of + /// swapping them + struct data_t : public node_t { + FIELD key; // The key stored in this node + V val; // The value stored in this node + + /// Construct a node + /// + /// @param tx a WSTEP_TM reference + /// @param _left left child of the node + /// @param _right right child of the node + /// @param _key the key of the node + /// @param _val the value of the node + data_t(WSTEP &tx, node_t *_left, node_t *_right, const K &_key, V &_val) + : node_t(tx, _left, _right), key(_key), val(_val) {} + }; + +public: + /// Default construct an empty tree + /// + /// @param _op The operation that is constructing the list + /// @param cfg A configuration object + ibst_omap(STMCAS *me, auto *cfg) { + // NB: Even though the constructor is operating on private data, it needs a + // TM context in order to use tm_fields + WSTEP tx(me); + sentinel = new node_t(tx); + } + +private: + /// Search for a `key` in the tree, and return the node holding it, as well + /// as the node's parent. If the key is not found, return null, and the + /// node that ought to be parent of the (not found) `key`. + /// + /// NB: The caller is responsible for clearing the checkpoint stack before + /// calling get_node(). + /// + /// @param me The calling thread's descriptor + /// @param key The key to search for + /// + /// @return {{found, orec}, {parent, orec}} if `key` is in the tree + /// {{nullptr, 0}, {parent, orec}} if `key` is not in the tree + ret_pair_t get_node(STMCAS *me, const K &key) { + // This loop delineates the search transaction. It commences from the end + // of the longest consistent prefix in the checkpoint stack + while (true) { + // Open a RSTEP transaction to traverse downward to the target node: + leq_t parent = {nullptr, 0}, child = {nullptr, 0}; + RSTEP tx(me); + + // Validate the checkpoints to find a starting point. When this is done, + // there must be at least one entry in the checkpoints (the sentinel), and + // it must be valid. + // + // NB: When this step is done, the curr->child relationship is validated, + // but we haven't read any of child's fields, or checked child's orec. + // Every checkpointed node must be valid at the time of checkpointing. + // + + // If the stack is empty or only holds the sentinel, start from {sentinel, + // root} + if (me->snapshots.size() <= 1) { + parent._obj = sentinel; + child._obj = parent._obj->children[LEFT].get(tx); + parent._ver = tx.check_orec(parent._obj); + if (parent._ver == STMCAS::END_OF_TIME) + continue; // retry + me->snapshots.clear(); + me->snapshots.push_back({parent._obj, parent._ver}); + } + // If the stack is larger, we can find the longest valid prefix + else { + // Trim the stack to a set of consistent checkpoints + for (auto cp = me->snapshots.begin(); cp != me->snapshots.end(); ++cp) { + if (!tx.check_continuation(cp->_obj, cp->_ver)) { + me->snapshots.reset(cp - me->snapshots.begin()); + break; // the rest of the checkpoints aren't valid + } + } + // If we don't have more than a sentinel, restart + if (me->snapshots.size() <= 1) + continue; + // Use the key to choose a child of the last good checkpoint + auto top = me->snapshots.top(); + parent._obj = static_cast(top._obj); + parent._ver = top._ver; + auto parent_key = static_cast(parent._obj)->key.get(tx); + child._obj = parent._obj->children[(key < parent_key) ? 0 : 1].get(tx); + // Validate that the read was valid + if (!tx.check_continuation(parent._obj, parent._ver)) + continue; + } + + // Traverse downward from the parent until we find null child or `key` + while (true) { + // nullptr == not found, so stop. We know parent was valid, so we can + // just return it + if (!child._obj) + return {{nullptr, 0}, parent}; + + // It's time to move downward. Read fields of child, then validate it. + // + // NB: we may not use grandchild, but it's better to read it here + auto child_key = static_cast(child._obj)->key.get(tx); + auto grandchild = + child._obj->children[(key < child_key) ? LEFT : RIGHT].get(tx); + child._ver = tx.check_orec(child._obj); + if (child._ver == STMCAS::END_OF_TIME) + break; // retry + + // If the child key matches, return {child, parent}. We know both are + // valid (parent came from stack; we just checked child) + // + // NB: the snapshotting code requires that no node with matching key + // goes into `snapshots` + if (child_key == key) + return {child, parent}; + + // Otherwise add the child to the checkpoint stack and traverse downward + me->snapshots.push_back({child._obj, child._ver}); + parent = child; + child = {grandchild, 0}; + } + } + } + + /// Given a node and its orec value, find the tree node that holds the key + /// that logically succeeds it (i.e., the leftmost descendent of the right + /// child) + /// + /// NB: The caller must ensure that `node` has a valid right child before + /// calling this method + /// + /// @param me The calling thread's descriptor + /// @param node An object and orec value to use as the starting point + /// + /// @return {{found, orec}, {parent, orec}} if no inconsistency occurs + /// {{nullptr, 0}, {nullptr, 0}} on any consistency violation + ret_pair_t get_succ_pair(STMCAS *me, leq_t &node) { + // NB: We expect the successor to be relatively close to the node, so we + // don't bother with checkpoints. However, we are willing to retry, + // since it's unlikely that `node` itself will change. + while (true) { + RSTEP tx(me); + // Ensure `node` is not deleted before reading its fields + if (!tx.check_continuation(node._obj, node._ver)) + return {{nullptr, 0}, {nullptr, 0}}; + + // Read the right child, ensure consistency + leq_t parent = node, child = {node._obj->children[RIGHT].get(tx), 0}; + if (!tx.check_continuation(node._obj, node._ver)) + return {{nullptr, 0}, {nullptr, 0}}; + + // Find the leftmost non-null node in the tree rooted at child + while (true) { + auto next = child._obj->children[LEFT].get(tx); + child._ver = tx.check_orec(child._obj); + if (child._ver == STMCAS::END_OF_TIME) + break; // retry + // If next is null, `child` is the successor. Otherwise keep traversing + if (!next) + return {child, parent}; + parent = child; + child = {next, 0}; + } + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(STMCAS *me, const K &key, V &val) { + me->snapshots.clear(); + while (true) { + // Get the node that holds `key`, if it is present, and also its parent. + // If it isn't present, we'll get a null pointer. That corresponds to a + // consistent read of the parent, which means we already linearized and + // we're done + auto [curr, _] = get_node(me, key); + if (curr._obj == nullptr) + return false; + + // Use an optimistic read if V can be read atomically + if (std::is_scalar::value) { + RSTEP tx(me); + auto *dn = static_cast(curr._obj); + V val_copy = reinterpret_cast *>(&dn->val)->load( + std::memory_order_acquire); + if (!tx.check_continuation(curr._obj, curr._ver)) + continue; + val = val_copy; + return true; + } else { + WSTEP tx(me); + if (!tx.acquire_continuation(curr._obj, curr._ver)) { + tx.unwind(); + continue; + } + auto dn = static_cast(curr._obj); + val = dn->val; + tx.unwind(); // because this WSTEP_TM didn't write anything + return true; + } + } + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(STMCAS *me, const K &key, V &val) { + me->snapshots.clear(); + while (true) { + auto [child, parent] = get_node(me, key); + if (child._obj) + return false; + WSTEP tx(me); + if (tx.acquire_continuation(parent._obj, parent._ver)) { + // We must have a null child and a valid parent. If it's sentinel, we + // must insert as LEFT. Otherwise, compute which child to set. + auto cID = (parent._obj == sentinel ? LEFT : RIGHT) & + (key > static_cast(parent._obj)->key.get(tx)); + auto child = new data_t(tx, nullptr, nullptr, key, val); + parent._obj->children[cID].set(child, tx); + return true; + } + } + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(STMCAS *me, const K &key) { + me->snapshots.clear(); + while (true) { + auto [target, parent] = get_node(me, key); + if (target._obj == nullptr) + return false; + + // Consistently read the target node's children + // + // NB: a concurrent thread could delete `target`, or could move `target` + // as part of some other `remove`. The call to `check_continuation()` + // will detect these cases and restart. + data_t *t_child[2]; + { + RSTEP tx(me); + t_child[RIGHT] = + static_cast(target._obj->children[RIGHT].get(tx)); + t_child[LEFT] = + static_cast(target._obj->children[LEFT].get(tx)); + if (!tx.check_continuation(target._obj, target._ver)) + continue; + } + + // If either child is null, and if the parent is still valid, then we can + // unstitch the target, link the parent to a grandchild and we're done. + if (!t_child[LEFT] || !t_child[RIGHT]) { + // Acquire the (possibly null) grandchild to link to the parent + auto gID = t_child[LEFT] ? LEFT : RIGHT; + WSTEP tx(me); + if (!tx.acquire_continuation(target._obj, target._ver) || + !tx.acquire_continuation(parent._obj, parent._ver)) { + tx.unwind(); + continue; + } + + // Which child of the parent is target? + auto cID = + parent._obj->children[LEFT].get(tx) == target._obj ? LEFT : RIGHT; + + // Unstitch and reclaim + parent._obj->children[cID].set(t_child[gID], tx); + tx.reclaim(target._obj); + return true; + } + + // `target` has two children. WLOG, the leftmost descendent of the right + // child is `target`'s successor, and must have at most one child. We + // want to put that node's key and value into `target`, and then remove + // that node by setting its parent's LEFT to its RIGHT (which might be + // null). + auto [succ, s_parent] = get_succ_pair(me, target); + if (!succ._obj) + continue; + + // If target's successor is target's right child, then target._ver must + // equal s_parent._ver. As long as we lock target._obj before we try + // to lock s_parent._obj, we'll get the check for free. + { + WSTEP tx(me); + + if (!tx.acquire_continuation(target._obj, target._ver) || + !tx.acquire_continuation(succ._obj, succ._ver) || + !tx.acquire_continuation(s_parent._obj, s_parent._ver)) { + tx.unwind(); + continue; + } // Postcondition of acquisition: target, succ, and s_parent are valid + + // Copy `succ`'s key/value into `target` + static_cast(target._obj) + ->key.set(static_cast(succ._obj)->key.get(tx), tx); + static_cast(target._obj)->val = + static_cast(succ._obj)->val; + + // Unstitch `succ` by setting its parent's left to its right + // Case 1: there are intermediate nodes between target and successor + if (s_parent._obj != target._obj) + s_parent._obj->children[LEFT].set(succ._obj->children[RIGHT].get(tx), + tx); + // Case 2: target is successor's parent + else + s_parent._obj->children[RIGHT].set(succ._obj->children[RIGHT].get(tx), + tx); + tx.reclaim(succ._obj); + return true; + } + } + } +}; diff --git a/artifact/ds/STMCAS/rbtree_omap.h b/artifact/ds/STMCAS/rbtree_omap.h new file mode 100644 index 0000000..3fceba3 --- /dev/null +++ b/artifact/ds/STMCAS/rbtree_omap.h @@ -0,0 +1,948 @@ +#pragma once + +#include +#include +#include +#include +#include + +/// An ordered map, implemented as an unbalanced, internal binary search tree. +/// This map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param STMCAS The STMCAS implementation (PO or PS) +template class rbtree_omap { + using WSTEP = typename STMCAS::WSTEP; + using RSTEP = typename STMCAS::RSTEP; + using snapshot_t = typename STMCAS::snapshot_t; + using ownable_t = typename STMCAS::ownable_t; + template using FIELD = typename STMCAS::template sField; + + /// An easy-to-remember way of indicating the left and right children + enum DIRS { LEFT = 0, RIGHT = 1 }; + + /// the color of a node + enum COLOR { RED = 0, BLACK = 1 }; + + /// node_t is the base type for all tree nodes. It doesn't have key/value + /// fields. + struct node_t : public ownable_t { + FIELD children[2]; // The node's children; index with LEFT/RIGHT + FIELD color; // The node's color + + /// Construct a node_t. This should only be called from a writer + /// transaction + /// + /// @param tx A writing transactional context + /// @param _color The color for this node + /// @param _left The left child of this node + /// @param _right The right child of this node + node_t(WSTEP &tx, COLOR _color, node_t *_left = nullptr, + node_t *_right = nullptr) + : ownable_t() { + color.set(_color, tx); + children[LEFT].set(_left, tx); + children[RIGHT].set(_right, tx); + } + }; + + /// The pair returned by get_leq; equivalent to the type in snapshots + struct leq_t { + node_t *_obj = nullptr; // The object + uint64_t _ver = 0; // The observed version of the object + }; + + /// Our tree uses a sentinel root node, so that we always have a valid node + /// for which to compute an orec. The sentinel's *LEFT* child is the true + /// root of the tree. That is, logically sentinel has the value "TOP". + node_t *sentinel; + + /// data_t is the type for all internal and leaf nodes in the data structure. + /// It extends the base type with a key and value. + /// + /// NB: keys are *not* const, because we want to overwrite nodes instead of + /// swapping them + struct data_t : public node_t { + FIELD key; // The key stored in this node + V val; // The value stored in this node + FIELD parent; // The node's parent + + /// Construct a node + /// + /// @param tx A writing transaction context + /// @param _parent The node's parent + /// @param _left The node's left child + /// @param _right The node's right child + /// @param _key The node's key + /// @param _val The node's value + /// @param _color The color of this node + data_t(WSTEP &tx, node_t *_parent, node_t *_left, node_t *_right, + const K &_key, V &_val, COLOR _color) + : node_t(tx, _color, _left, _right), key(_key), val(_val), + parent(_parent) {} + }; + +public: + /// Default construct an empty tree + /// + /// @param me The operation that is constructing the tree + /// @param cfg An unused configuration object + rbtree_omap(STMCAS *me, auto *cfg) { + // NB: Even though the constructor is operating on private data, it needs a + // TM context for the constructor + WSTEP tx(me); + sentinel = new node_t(tx, BLACK); + } + +private: + /// Search for a `key` in the tree, and return the node holding it. If the + /// key is not found, return the node that ought to be parent of the (not + /// found) `key`. + /// + /// NB: The caller is responsible for clearing the checkpoint stack before + /// calling get_node(). + /// + /// @param me The calling thread's descriptor + /// @param key The key to search for + /// + /// @return {found, orec} if `key` is in the tree; + /// {parent, orec} if `key` is not in the tree + leq_t get_node(STMCAS *me, const K &key) const { + // This loop delineates the search transaction. It commences from the end + // of the longest consistent prefix in the checkpoint stack + while (true) { + // Open a RSTEP transaction to traverse downward to the target node: + leq_t parent = {nullptr, 0}, child = {nullptr, 0}; + RSTEP tx(me); + + // Validate the checkpoints to find a starting point. When this is done, + // there must be at least one entry in the checkpoints (the sentinel), and + // it must be valid. + // + // NB: When this step is done, the curr->child relationship is validated, + // but we haven't read any of child's fields, or checked child's orec. + // Every checkpointed node must be valid at the time of checkpointing. + + // If stack is empty or only holds sentinel, start from {sentinel, root} + if (me->snapshots.size() <= 1) { + parent._obj = sentinel; + child._obj = parent._obj->children[LEFT].get(tx); + parent._ver = tx.check_orec(parent._obj); + if (parent._ver == STMCAS::END_OF_TIME) + continue; // retry + me->snapshots.clear(); + me->snapshots.push_back({parent._obj, parent._ver}); + } + // If the stack is larger, we can find the longest valid prefix + else { + // Trim the stack to a set of consistent checkpoints + for (auto cp = me->snapshots.begin(); cp != me->snapshots.end(); ++cp) { + if (!tx.check_continuation(cp->_obj, cp->_ver)) { + me->snapshots.reset(cp - me->snapshots.begin()); + break; // the rest of the checkpoints aren't valid + } + } + // If we don't have more than a sentinel, restart + if (me->snapshots.size() <= 1) + continue; + // Use the key to choose a child of the last good checkpoint + // + // NB: top.key != key, because we never put a matching key into + // snapshots, and if a remove caused a key to change, we'll fail to + // validate that node. + auto top = me->snapshots.top(); + parent = {static_cast(top._obj), top._ver}; + auto parent_key = static_cast(parent._obj)->key.get(tx); + child._obj = parent._obj->children[(key < parent_key) ? 0 : 1].get(tx); + // Validate that the reads of parent were valid + if (!tx.check_continuation(parent._obj, parent._ver)) + continue; + } + + // Traverse downward from the parent until we find null child or `key` + while (true) { + // nullptr == not found, so stop. Parent was valid, so return it + if (!child._obj) + return parent; + + // It's time to move downward. Read fields of child, then validate it. + // + // NB: we may not use grandchild, but it's better to read it here + auto child_key = static_cast(child._obj)->key.get(tx); + auto grandchild = + child._obj->children[(key < child_key) ? LEFT : RIGHT].get(tx); + child._ver = tx.check_orec(child._obj); + if (child._ver == STMCAS::END_OF_TIME) + break; // retry + + // If the child key matches, return {child, parent}. We know both are + // valid (parent came from stack; we just checked child) + // + // NB: the snapshot code requires that no node with matching key goes + // into `snapshots` + if (child_key == key) + return child; + + // Otherwise add the child to the checkpoint stack and traverse downward + me->snapshots.push_back({child._obj, child._ver}); + parent = child; + child = {grandchild, 0}; + } + } + } + + /// Given a node and its orec value, find the tree node that holds the key + /// that logically succeeds it (i.e., the leftmost descendent of the right + /// child) + /// + /// NB: The caller must ensure that `node` has a valid right child before + /// calling this method + /// + /// @param me The calling thread's descriptor + /// @param node An object and orec value to use as the starting point + /// + /// @return {{found, orec}, {parent, orec}} if no inconsistency occurs + /// {{nullptr, 0}, {nullptr, 0}} on any consistency violation + leq_t get_succ(STMCAS *me, leq_t &node) { + // NB: We expect the successor to be relatively close to the node, so we + // don't bother with checkpoints. However, we are willing to retry, + // since it's unlikely that `node` itself will change. + while (true) { + RSTEP tx(me); + // Read the right child, ensure consistency + // + // NB: Since we have smr, we can read `node` even if it is deleted. The + // subsequent validation will suffice. + leq_t child = {node._obj->children[RIGHT].get(tx), 0}; + if (!tx.check_continuation(node._obj, node._ver)) + return {nullptr, 0}; + + // Find the leftmost non-null node in the tree rooted at child + while (true) { + auto next = child._obj->children[LEFT].get(tx); + child._ver = tx.check_orec(child._obj); + if (child._ver == STMCAS::END_OF_TIME) + break; // retry + // If next is null, `child` is the successor. Otherwise keep traversing + if (!next) + return child; + child = {next, 0}; + } + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(STMCAS *me, const K &key, V &val) const { + me->snapshots.clear(); + while (true) { + // Get the node that holds `key`, if it is present. If it isn't present, + // we'll get the parent of where it would be. Whatever we get is + // validated, so if it's the sentinel, we're done. + auto curr = get_node(me, key); + if (curr._obj == sentinel) + return false; + + // Use an optimistic read if V can be read atomically + if (std::is_scalar::value) { + RSTEP tx(me); + auto *dn = static_cast(curr._obj); + auto dn_key = dn->key.get(tx); + V val_copy = reinterpret_cast *>(&dn->val)->load( + std::memory_order_acquire); + if (!tx.check_continuation(curr._obj, curr._ver)) + continue; + if (dn_key != key) + return false; + val = val_copy; + return true; + } else { + WSTEP tx(me); + if (!tx.acquire_continuation(curr._obj, curr._ver)) { + tx.unwind(); + continue; + } + auto dn = static_cast(curr._obj); + if (dn->key.get(tx) != key) { + tx.unwind(); + return false; + } + val = dn->val; + tx.unwind(); // because this WSTEP_TM didn't write anything + return true; + } + } + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(STMCAS *me, const K &key, V &val) { + me->snapshots.clear(); + while (true) { + // Get the node that holds `key`, if it is present. If it isn't present, + // we'll get the parent of where it would be. Whatever we get is + // validated, so if it matches, we're done. + auto leq = get_node(me, key); + + // We're going to assume that we'll insert, so open a WSTEP transaction. + // If we can't lock the node, restart + WSTEP tx(me); + if (!tx.acquire_continuation(leq._obj, leq._ver)) + continue; + + // If the key matches, the insertion attempt fails + if (leq._obj != sentinel && + static_cast(leq._obj)->key.get(tx) == key) { + tx.unwind(); + return false; + } + + // We must have a null child and a valid parent. If it's sentinel, we + // must insert as LEFT. Otherwise, compute which child to set. + node_t *parent = leq._obj; + auto cID = (leq._obj == sentinel ? LEFT : RIGHT) & + (key > static_cast(parent)->key.get(tx)); + + // We are strict 2PL here: first we must acquire everything that will be + // written. `fix_root` tracks if the root will need special cleanup. + bool fix_root = false; + if (!insert_acquire_aggressive_all(cID, static_cast(parent), tx, + fix_root)) { + tx.unwind(); + continue; + } + + // Now we can link the child to the parent + auto child = new data_t(tx, parent, nullptr, nullptr, key, val, RED); + tx.acquire_aggressive(child); + parent->children[cID].set(child, tx); + + // Rebalance in response to this insertion, then we're done + insert_fixup(child, tx, fix_root); + return true; + } + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(STMCAS *me, const K &key) { + me->snapshots.clear(); + while (true) { + // Get the node that holds `key`, if it is present. If it isn't present, + // we'll get the parent of where it would be. Whatever we get is + // validated, so if it's sentinel, we're done. + auto target = get_node(me, key); + if (target._obj == sentinel) + return false; + + // Read the node's key and its children; detect if the key doesn't match + // + // NB: we can't open a WSTEP yet, because an upcoming call to get_succ + // does + // is going to use an RSTEP + data_t *t_child[2]; + { + RSTEP tx(me); + auto dn_key = static_cast(target._obj)->key.get(tx); + t_child[RIGHT] = + static_cast(target._obj->children[RIGHT].get(tx)); + t_child[LEFT] = + static_cast(target._obj->children[LEFT].get(tx)); + if (!tx.check_continuation(target._obj, target._ver)) + continue; + if (dn_key != key) + return false; + } + + // If target has <=1 child, then we will un-stitch by pointing its parent + // to that child. Otherwise, we'll un-stitch by swapping target with its + // successor and then removing the successor by pointing successor's + // parent to successor's child. Here's where we get the child and + // successor + leq_t succ = {target._obj, target._ver}; // succ is target if only 1 child + data_t *child = nullptr; // The child who gets swapped up + if (!t_child[LEFT]) { + child = t_child[RIGHT]; + } else if (!t_child[RIGHT]) { + child = t_child[LEFT]; + } else { + succ = get_succ(me, target); + if (!succ._obj) + continue; + RSTEP tx(me); + child = static_cast(succ._obj->children[RIGHT].get(tx)); + if (!tx.check_continuation(succ._obj, succ._ver)) + continue; + } + + // We're going to assume that we'll remove, so open a WSTEP transaction + // and acquire succ and child. + // + // NB: acquire continuation on succ is necessary regardless of whether + // it's target or not, but child can be aggressive. + WSTEP tx(me); + if (!tx.acquire_continuation(succ._obj, succ._ver) || + (child && !tx.acquire_aggressive(child))) { + tx.unwind(); + continue; + } + + // Now acquire child's children, if child is not null + { + auto x_l = child ? child->children[LEFT].get(tx) : nullptr; + auto x_r = child ? child->children[RIGHT].get(tx) : nullptr; + if ((x_l && !tx.acquire_aggressive(x_l)) || + (x_r && !tx.acquire_aggressive(x_r))) { + tx.unwind(); + continue; + } + } + + // We are strict 2PL here: first we must acquire everything that will be + // written. remove_acquire_aggressive_all does most of the job: + if (!remove_acquire_aggressive_all( + child, static_cast(succ._obj), tx)) { + tx.unwind(); + continue; + } + + // Lastly, we need to acquire the target and the successor's parent + if ((!tx.acquire_continuation(target._obj, target._ver)) || + (!tx.acquire_aggressive( + static_cast(succ._obj)->parent.get(tx)))) { + tx.unwind(); + continue; + } + + // Now we can start moving keys and values, unstitching, and cleaning up + + // We need the successor's original color to know if we need to call fixup + auto original_succ_color = succ._obj->color.get(tx); + // If we call fixup, we need the child's CID and parent + DIRS cID_c; + node_t *c_parent; + + // If either child is null, and if the parent is still valid, then we can + // un-stitch target, then link target's parent to target's grandchild + if (!t_child[LEFT] || !t_child[RIGHT]) { + // get target's parent, figure out which of its children target is: + c_parent = static_cast(target._obj)->parent.get(tx); + cID_c = c_parent->children[LEFT].get(tx) == target._obj ? LEFT : RIGHT; + // Unstitch, reclaim + c_parent->children[cID_c].set(child, tx); + if (child) + child->parent.set(c_parent, tx); + tx.reclaim(target._obj); + } + // When both children of target are not null, we have to swap, then + // unstitch + else { + // Get the successor's parent, then copy succ's k/v into target + auto s_p = static_cast(succ._obj)->parent.get(tx); + auto dn = static_cast(target._obj); + dn->key.set(static_cast(succ._obj)->key.get(tx), tx); + dn->val = static_cast(succ._obj)->val; + + // Unstitch `succ` by setting its parent's left to its right (i.e., + // child) + + // Case 1: there are intermediate nodes between target and successor + if (s_p != target._obj) { + s_p->children[LEFT].set(child, tx); + cID_c = LEFT; + } + // Case 2: target is successor's parent + else { + s_p->children[RIGHT].set(child, tx); + cID_c = RIGHT; + } + // don't forget the back-link from child to parent + if (child) + child->parent.set(s_p, tx); + + tx.reclaim(succ._obj); + c_parent = s_p; + } + + // Rebalance and recolor in response to this removal, then we're done + if (original_succ_color == BLACK) + remove_fixup(child, static_cast(c_parent), cID_c, tx); + return true; + } + } + +private: + /// Acquire all of the nodes that will need to change if `z_p` is to receive a + /// new child in position `CID_z`. + /// + /// @param cID_z The index of the child being added to z_p + /// @param z_p The (acquired) node who will be receiving a new child + /// @param tx A writing transaction context + /// @param fix_root A reference parameter indicating if the root was reached + /// + /// @return True if all nodes were acquired, false otherwise + bool insert_acquire_aggressive_all(int cID_z, data_t *z_p, WSTEP &tx, + bool &fix_root) { + // If we're giving the sentinel a child, then we immediately stop + // traversing upward... the sentinel is already locked. + if (z_p == sentinel) { + fix_root = true; + return true; + } + + // z is the child of z_p. In the first round, it's the to-insert node, + // which we haven't created yet, so we let it be null and pretend it's RED + data_t *z = nullptr; + + // Invariant: z_p is already on each iteration + while (z_p->color.get(tx) == RED) { + // Acquire the grandparent + data_t *z_p_p = static_cast(z_p->parent.get(tx)); + if (!tx.acquire_aggressive(z_p_p)) + return false; + + // Now acquire z's aunt (z_a) (z_p's sibling) if it exists + auto cID_z_p = z_p == z_p_p->children[LEFT].get(tx) ? LEFT : RIGHT; + auto cID_z_a = cID_z_p == LEFT ? RIGHT : LEFT; + data_t *z_a = static_cast(z_p_p->children[cID_z_a].get(tx)); + if (z_a && !tx.acquire_aggressive(z_a)) + return false; + + // case 1: z_a is RED --> colors will propagate + // z_p_p + // / \. + // z_p z_a + // | + // z + if (z_a && static_cast(z_a)->color.get(tx) == RED) { + // NB: {z, z_p, z_a, z_p_p} are acquired, but we're going to jump to the + // great grandparent. The loop invariant requires us to acquire it + // now. + z = static_cast(z_p_p); + z_p = static_cast(z->parent.get(tx)); + if (!tx.acquire_aggressive(z_p)) + return false; + if (z_p == sentinel) // we painted z which is root, we need to fix that + fix_root = true; + cID_z = z == z_p->children[LEFT].get(tx) ? LEFT : RIGHT; + continue; + } + + // Invariant: z_a is black or nullptr + + // case 2: cID_z != cID_z_p --> do a /cID_z_p/ rotation on z_p + // z_p_p + // / \. + // z_a z_p + // / + // z + // / \. + // z_e z_c + if (cID_z != cID_z_p) { + // NB: z == nullptr should only be true in the first iteration + // + // NB: {z, z_p, z_p_p} are acquired. one of z's children will get a new + // parent, so acquire it + data_t *z_c = + z ? static_cast(z->children[cID_z_p].get(tx)) : nullptr; + if (z_c && !tx.acquire_aggressive(z_c)) + return false; + + // Now we roll right into case 3 and finish the rebalance: + // case 2->3: perform a /cID_z_a/_rotation on z_p_p + // z_p_p_p + // | + // z_p_p + // / \. + // z_a z (==>new z_p) + // / \. + // z_e z_p (==> new z) + // / + // z_c + + // {z, z_p_p} are already acquired. We need to acquire z_e and z_p_p_p + data_t *z_e = + z ? static_cast(z->children[cID_z_a].get(tx)) : nullptr; + if (z_e && !tx.acquire_aggressive(z_e)) + return false; + + auto z_p_p_p = z_p_p->parent.get(tx); + if (!tx.acquire_aggressive(z_p_p_p)) + return false; + if (z_p_p_p == sentinel) + fix_root = true; + return true; + } + + // case 3: perform a /cID_y/_rotation on z_p_p + // z_p_p_p + // | + // z_p_p + // / \. + // z_a z_p + // / \. + // w z + + // {z_p, z_p_p} are already acquired. We need to acquire w and z_p_p_p + auto z_p_p_p = z_p_p->parent.get(tx); + if (!tx.acquire_aggressive(z_p_p_p)) + return false; + if (z_p_p_p == sentinel) + fix_root = true; + + auto w = z_p->children[cID_z_a].get(tx); + if (w && !tx.acquire_aggressive(w)) + return false; + return true; + } + + // At last, we've acquired everything we need, and can stop + return true; + } + + /// Do all of the rotations and color changes that correspond to z being + /// inserted into the tree. This should only be called after + /// insert_acquire_aggressive_all has acquired everything that this method + /// will modify. Consequently, this code is identical to the sequential code. + /// + /// @param z The new child being added + /// @param tx A writing transaction context + /// @param fix_root Is the root acquired? + void insert_fixup(data_t *z, WSTEP &tx, bool fix_root) { + auto z_p = z->parent.get(tx); + // Normal case: z is not the root + while (z_p->color.get(tx) == RED) { + auto cID_z = z == z_p->children[LEFT].get(tx) ? LEFT : RIGHT; + node_t *z_p_p = static_cast(z_p)->parent.get(tx); + auto cID_z_p = z_p == z_p_p->children[LEFT].get(tx) ? LEFT : RIGHT; + auto cID_z_a = cID_z_p == LEFT ? RIGHT : LEFT; + data_t *z_a = static_cast(z_p_p->children[cID_z_a].get(tx)); + // case 1: + if (z_a && z_a->color.get(tx) == RED) { + z_p->color.set(BLACK, tx); + z_a->color.set(BLACK, tx); + z_p_p->color.set(RED, tx); + z = static_cast(z_p_p); + z_p = z->parent.get(tx); + continue; + } + + // case 2 + if (cID_z == cID_z_a) { + z = static_cast(z_p); + if (cID_z == RIGHT) + left_rotate(z, tx); + else + right_rotate(z, tx); + z_p = z->parent.get(tx); + z_p_p = static_cast(z_p)->parent.get(tx); + } + + // case 3 (includes fallthrough from 2->3) + z_p->color.set(BLACK, tx); + z_p_p->color.set(RED, tx); + if (cID_z_a == RIGHT) + right_rotate(static_cast(z_p_p), tx); + else + left_rotate(static_cast(z_p_p), tx); + } + + // Clean up the root if necessary + if (fix_root) { + auto r = sentinel->children[LEFT].get(tx); + static_cast(r)->color.set(BLACK, tx); + } + } + + /// Acquire all of the nodes that will need to change if `y` is to be removed + /// and `x` is to move into its place + /// + /// @param x The node that moves upward + /// @param y The node that will be removed + /// @param tx A writing transaction context + /// + /// @return True if all nodes were acquired, false otherwise + bool remove_acquire_aggressive_all(data_t *x, data_t *y, WSTEP &tx) { + // If `y` isn't black, we won't have to do any rebalancing + if (y->color.get(tx) != BLACK) + return true; + + // When x swaps into y's place, it gets y's parent but keeps its color + auto x_color = x ? x->color.get(tx) : BLACK; + x = y; + + // loop invariants: x != nullptr, x.color == black, x is acquired + while (x->parent.get(tx) != sentinel && x_color == BLACK) { + data_t *x_p = static_cast(x->parent.get(tx)); + if (!tx.acquire_aggressive(x_p)) + return false; + + // We need to know if x is left or right, and we need its sibling (w) + DIRS cID_x = LEFT, cID_w = RIGHT; + if (x != x_p->children[LEFT].get(tx)) { + cID_x = RIGHT; + cID_w = LEFT; + } + auto w = x_p->children[cID_w].get(tx); + if (!tx.acquire_aggressive(w)) + return false; + + // case 1: Do a cID_x rotation to push x down + // x_p_p + // | + // x_p + // \. + // w + // / + // w_c + if (static_cast(w)->color.get(tx) == RED) { + // {x_p, w} are acquired. Need to acquire cID_x'th child of w and x_p_p + data_t *w_c = static_cast(w->children[cID_x].get(tx)); + auto x_p_p = x_p->parent.get(tx); + if ((w_c && !tx.acquire_aggressive(w_c)) || + (!tx.acquire_aggressive(x_p_p))) + return false; + // w_c's children's colors determine if we need to propagate. + // Since we're going to read both children's colors, we need to acquire + // them (as w_c_c and w_c_e) + data_t *w_c_c = + w_c ? static_cast(w_c->children[cID_x].get(tx)) : nullptr; + data_t *w_c_e = + w_c ? static_cast(w_c->children[cID_w].get(tx)) : nullptr; + + if ((w_c_c && !tx.acquire_aggressive(w_c_c)) || + (w_c_e && !tx.acquire_aggressive(w_c_e))) + return false; + + // If case 1 becomes case 3, we do a cID_w-rotation on w_c + // x_p + // \. + // w_c + // / + // w_c_c + // \. + // w_c_c_e + // If case 1 becomes case 3 becomes case 4, we need to do a + // /cID_x/_rotation on x_p + // w + // / + // x_p + // \. + // w_c_c + // / + // w_c_c_c + if (((w_c_e && w_c_e->color.get(tx) == BLACK) || !w_c_e) && w_c_c && + w_c_c->color.get(tx) == RED) { + // {w, w_c_c, x_p} are already acquired. Need to acquire w_c_c's + // children + data_t *w_c_c_c = + w_c_c ? static_cast(w_c_c->children[cID_x].get(tx)) + : nullptr; + data_t *w_c_c_e = + w_c_c ? static_cast(w_c_c->children[cID_w].get(tx)) + : nullptr; + if ((w_c_c_e && !tx.acquire_aggressive(w_c_c_e)) || + (w_c_c_c && !tx.acquire_aggressive(w_c_c_c))) + return false; + return true; + } + + // If case 1 becomes case 4, we need to do a /cID_x/_rotation on x_p + // w + // / + // x_p + // \. + // w_c + // / + // w_c_c + if (w_c_e && w_c_e->color.get(tx) == RED) { + // {w, w_c, w_c_c, x_p} are already acquired, so we're done + return true; + } + // otherwise : case1 becomes case2, propagate, see below + } + // W's children's colors determine if we need to propagate, so acquire + // both + else { + data_t *w_c = static_cast(w->children[cID_x].get(tx)); + data_t *w_e = static_cast(w->children[cID_w].get(tx)); + if ((w_c && !tx.acquire_aggressive(w_c)) || + (w_e && !tx.acquire_aggressive(w_e))) + return false; + // case 3: we need to do a /cID_w/_rotation on w + // x_p + // \. + // w + // / + // w_c + // \. + // w_c_e + // {w, x_p, w_c} are already acquired. Acquire w_c's cID_w child + // case 3 -> case 4, we need to perform a /cID_x/ rotation on x_p + // x_p_p + // | + // x_p + // \. + // w_c + // / + // w_c_c + // {w, x_p} are already acquired: acquire w_c's cID_x child, and x_p_p + data_t *x_p_p = static_cast(x_p->parent.get(tx)); + if (x_p_p && !tx.acquire_aggressive(x_p_p)) + return false; + + if (((w_e && w_e->color.get(tx) == BLACK) || !w_e) && w_c && + w_c->color.get(tx) == RED) { + data_t *w_c_c = + w_c ? static_cast(w_c->children[cID_x].get(tx)) + : nullptr; + data_t *w_c_e = + w_c ? static_cast(w_c->children[cID_w].get(tx)) + : nullptr; + if ((w_c_e && !tx.acquire_aggressive(w_c_e)) || + (w_c_c && !tx.acquire_aggressive(w_c_c))) + return false; + return true; + } + + // case 4: w_c and/or w_e is red, so cID_x rotate x_p + // {x_p, x_p_p, w, w_c} already acquired, nothing needed + if ((w_c && w_c->color.get(tx) == RED) || + (w_e && w_e->color.get(tx) == RED)) + return true; + } + // case 2 : propagate the fixup to x's parent. + x = static_cast(x_p); + x_color = x->color.get(tx); + } + return true; + } + + /// Do all of the color changes and rotations that correspond to x's parent + /// being deleted, resulting in x becoming the child of x_p. This should only + /// be called after remove_acquire_aggressive_all has acquired everything that + /// this method will modify. Consequently, this code is identical to the + /// sequential code. + /// + /// @param x The node that moved up + /// @param x_p The new parent of `x` + /// @param cID_x Which child of `x_p` is `x`? + /// @param tx A writing transaction context + void remove_fixup(data_t *x, data_t *x_p, DIRS cID_x, WSTEP &tx) { + auto x_color = x ? x->color.get(tx) : BLACK; + while (x_p != sentinel && x_color == BLACK) { + // `w` is the sibling of `x` + auto cID_w = cID_x == LEFT ? RIGHT : LEFT; + data_t *w = static_cast(x_p->children[cID_w].get(tx)); + if (w && static_cast(w)->color.get(tx) == RED) { + static_cast(w)->color.set(BLACK, tx); + static_cast(x_p)->color.set(RED, tx); + if (cID_x == LEFT) + left_rotate(static_cast(x_p), tx); + else + right_rotate(static_cast(x_p), tx); + w = static_cast(x_p->children[cID_w].get(tx)); + } + // check both children's colors to decide about propagating: + data_t *w_c = + w ? static_cast(w->children[cID_x].get(tx)) : nullptr; + data_t *w_e = + w ? static_cast(w->children[cID_w].get(tx)) : nullptr; + if ((w_c && w_c->color.get(tx) == RED) || + (w_e && w_e->color.get(tx) == RED)) { + if (!w_e || w_e->color.get(tx) == BLACK) { + w_c->color.set(BLACK, tx); + w->color.set(RED, tx); + if (cID_x == LEFT) + right_rotate(w, tx); + else + left_rotate(w, tx); + w = static_cast(x_p->children[cID_w].get(tx)); + } + w->color.set(x_p->color.get(tx), tx); + x_p->color.set(BLACK, tx); + auto w_e = w->children[cID_w].get(tx); + static_cast(w_e)->color.set(BLACK, tx); + if (cID_x == LEFT) + left_rotate(x_p, tx); + else + right_rotate(x_p, tx); + break; + } else { + w->color.set(RED, tx); + x = x_p; + x_p = static_cast(x->parent.get(tx)); + x_color = x->color.get(tx); + cID_x = x == x_p->children[LEFT].get(tx) ? LEFT : RIGHT; + } + } + if (x) + x->color.set(BLACK, tx); + } + + /// Perform a left rotation on `x`, pushing it downward + /// + /// @param x The node to rotate downward + /// @param tx A writing transaction context + void left_rotate(data_t *x, WSTEP &tx) { + auto y = static_cast(x->children[RIGHT].get(tx)); + auto y_l = y->children[LEFT].get(tx); + x->children[RIGHT].set(y_l, tx); + if (y_l) + static_cast(y_l)->parent.set(x, tx); + + auto x_p = x->parent.get(tx); + y->parent.set(x_p, tx); + if (x_p == sentinel) + sentinel->children[LEFT].set(y, tx); + else + x_p->children[x == x_p->children[LEFT].get(tx) ? LEFT : RIGHT].set(y, tx); + + y->children[LEFT].set(x, tx); + x->parent.set(y, tx); + } + + /// Perform a right rotation on `y`, pushing it downward + /// + /// @param y The node to rotate downward + /// @param tx A writing transaction context + void right_rotate(data_t *y, WSTEP &tx) { + auto x = static_cast(y->children[LEFT].get(tx)); + auto x_r = x->children[RIGHT].get(tx); + y->children[LEFT].set(x_r, tx); + if (x_r) + static_cast(x_r)->parent.set(y, tx); + + auto y_p = y->parent.get(tx); + x->parent.set(y_p, tx); + if (y_p == sentinel) + sentinel->children[LEFT].set(x, tx); + else + y_p->children[y == y_p->children[RIGHT].get(tx) ? RIGHT : LEFT].set(x, + tx); + + x->children[RIGHT].set(y, tx); + y->parent.set(x, tx); + } +}; diff --git a/artifact/ds/STMCAS/skiplist_cached_opt_omap.h b/artifact/ds/STMCAS/skiplist_cached_opt_omap.h new file mode 100644 index 0000000..49b7a2f --- /dev/null +++ b/artifact/ds/STMCAS/skiplist_cached_opt_omap.h @@ -0,0 +1,567 @@ +#pragma once + +#include +#include +#include +#include +#include + +/// An ordered map, implemented as a doubly-linked skip list. This map supports +/// get(), insert(), and remove() operations. +/// +/// This version of the skiplist is heavily optimized to use the biggest STMCAS +/// operations it can, while still avoiding aborts. This means, for example, +/// trying to stitch as many layers as possible (and to do so via recording old +/// values). +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param STMCAS The STMCAS implementation (PO or PS) +/// @param dummy_key A fake key, to use in sentinel nodes +/// @param dummy_val A fake value, to use in sentinel nodes +template +class skiplist_cached_opt_omap { + using WSTEP = typename STMCAS::WSTEP; + using RSTEP = typename STMCAS::RSTEP; + using STEP = typename STMCAS::STEP; + using ownable_t = typename STMCAS::ownable_t; + template using FIELD = typename STMCAS::template sField; + + /// data_t is a node in the skip list. It has a key, a value, an owner, and a + /// "tower" of predecessor and successor pointers + /// + /// NB: Height isn't always the size of tower... it tracks how many levels are + /// fully and correctly stitched, so it changes during insertion and + /// removal. + struct data_t : public ownable_t { + /// A pair of data pointers, for the successor and predecessor at a level of + /// the tower + struct level_t { + FIELD key; // Key of the successor + FIELD next; // Succ at this level + }; + + const K key; // The key stored in this node + std::atomic val; // The value stored in this node + const uint8_t height; // # valid tower nodes + level_t tower[]; // Tower of pointers to pred/succ + + private: + /// Construct a data node. This is private to force the use of our make_* + /// methods, which handle allocating enough space for the tower. + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(K _key, V _val, uint8_t _height) + : ownable_t(), key(_key), val(_val), height(_height) {} + + public: + /// Construct a sentinel (head or tail) node. Note that the sentinels can't + /// easily be of a node type that lacks key and value fields, or else the + /// variable-length array would preclude inheriting from it. + /// + /// @param iHeight The max number of index layers this node will have + static data_t *make_sentinel(uint8_t iHeight) { + int node_size = sizeof(data_t) + (iHeight + 1) * sizeof(level_t); + void *region = calloc(1, node_size); + return new (region) data_t(dummy_key, dummy_val, iHeight); + } + + /// Construct a data node + /// + /// @param iHeight The max number of index layers this node will have + /// @param key The key to store in this node + /// @param val The value to store in this node + static data_t *make_data(uint64_t iHeight, K key, V val) { + int node_size = sizeof(data_t) + (iHeight + 1) * sizeof(level_t); + void *region = calloc(1, node_size); + return new (region) data_t(key, val, iHeight); + } + }; + + const int NUM_INDEX_LAYERS; // # of index layers. Doesn't count data layer + data_t *const head; // The head sentinel + data_t *const tail; // The tail sentinel + +public: + /// Default construct a skip list by stitching a head sentinel to a tail + /// sentinel at each level + /// + /// @param _op The operation that is constructing the list + /// @param cfg A configuration object that has a `snapshot_freq` field + skiplist_cached_opt_omap(STMCAS *_op, auto *cfg) + : NUM_INDEX_LAYERS(cfg->max_levels), + head(data_t::make_sentinel(NUM_INDEX_LAYERS)), + tail(data_t::make_sentinel(NUM_INDEX_LAYERS)) { + // NB: Even though the constructor is operating on private data, it needs a + // TM context in order to set the head and tail's towers to each other + WSTEP tx(_op); + for (auto i = 0; i <= NUM_INDEX_LAYERS; i++) { + head->tower[i].key.set(dummy_key, tx); + head->tower[i].next.set(tail, tx); + } + } + + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(STMCAS *me, const K &key, V &val) { + while (true) { + RSTEP tx(me); + // Do a leq... if head, we fail. n will never be null or tail + auto n = get_leq(tx, key); + if (n == nullptr) + continue; + + if (n == head || n->key != key) + return false; + + // since we have EBR, `val` can be atomic, making this code quite simple + + // NB: get() doesn't care if the node is owned, just that it's still in + // the skiplist + V val_copy = n->val.load(std::memory_order_acquire); + // Check after reading value + if (tx.check_orec(n) == STMCAS::END_OF_TIME) + continue; + val = val_copy; + return true; + } + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(STMCAS *me, const K &key, V &val) { + data_t *preds[NUM_INDEX_LAYERS]; + int target_height = randomLevel(me); // The target index height of new_dn + + while (true) { + WSTEP tx(me); + // Get the insertion point, lock it or retry + auto n = get_leq(tx, key, preds, target_height); + + if (n == nullptr) + continue; + + // Since we have EBR, we can look at n->key without validation. If + // it matches `key`, return false. + if (n != head && n->key == key) + return false; + + // Acquire the pred of the to-be-inserted node + if (!tx.acquire_consistent(n)) { + tx.unwind(); + continue; + } + auto next = n->tower[0].next.get(tx); + + // If this is a "short" insert, we can finish quickly + if (target_height == 0) { + auto new_dn = data_t::make_data(target_height, key, val); + new_dn->tower[0].key.set(next->key, tx); + new_dn->tower[0].next.set(next, tx); + // NB: we don't need to acquire new_dn in this case, because anyone who + // finds their way to it will find it fully stitched in. + n->tower[0].key.set(key, tx); + n->tower[0].next.set(new_dn, tx); + return true; + } + + // Slow path for when the node is tall, and we have a lot of acquiring to + // do + if (index_stitch(tx, me, n, next, preds, key, val, target_height)) + return true; + tx.unwind(); + } + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(STMCAS *me, const K &key) { + data_t *preds[NUM_INDEX_LAYERS]; + + while (true) { + WSTEP tx(me); + // Get predecessor, find its next, if != key return false + data_t *n = get_le(tx, key, preds); + if (n == nullptr) + continue; + auto found = n->tower[0].next.get(tx); + if (found == nullptr) { + tx.unwind(); + continue; + } + if (found == tail || found->key != key) + return false; + + // Acquire the target, make sure it's not owned + if (!tx.acquire_consistent(found)) { + tx.unwind(); + continue; + } + // Acquire the predecessor so we can edit its next pointer + if (!tx.acquire_consistent(n)) { + tx.unwind(); + continue; + } + + // Fast-path unstitch when it has height 0 + if (found->height == 0) { + auto nxt = found->tower[0].next.get(tx); + n->tower[0].next.set(nxt, tx); + n->tower[0].key.set(nxt->key, tx); + // NB: don't forget to set `node`'s pointers to null! + found->tower[0].next.set(nullptr, tx); + tx.reclaim(found); + return true; + } + + // Slow-path unstitch when it's tall + if (index_unstitch(tx, me, found, n, preds)) + return true; + tx.unwind(); + } + } + +private: + /// get_leq uses the towers to skip from the head sentinel to the node + /// with the largest key <= the search key. It can return the head data + /// sentinel, but not the tail sentinel. + /// + /// There is no atomicity between get_leq and its caller. It returns the + /// node it found, along with the value of the orec for that node at the time + /// it was accessed. The caller needs to validate the orec before using the + /// returned node. + /// + /// get_leq *can* return an OWNED node. + /// + /// @param me The calling thread's descriptor + /// @param key The key for which we are doing a predecessor query. + /// + /// @return The data node that was found, and its orec's value + __attribute__((noinline)) data_t *get_leq(STEP &tx, const K &key) { + // We always start at the head sentinel. Scan its tower to find the + // highest non-tail level + data_t *curr = head; + int current_level = 0; + for (int i = NUM_INDEX_LAYERS; i > 0; --i) { + if (head->tower[i].next.get(tx) != tail) { + current_level = i; + break; + } + } + + // Traverse over and down through the index layers + while (current_level > 0) { + // Advance curr by moving forward in this index layer + curr = index_leq(tx, key, curr, current_level); + if (curr == nullptr) + return nullptr; + // On a key match, we can exit immediately + if (curr->key == key) + return curr; + --current_level; // Move down a level + } + + // Search in the data layer. Only return if result valid + return data_leq(tx, key, curr); + } + + /// A version of get_leq that is specialized for insert, where we need to get + /// the predecessors at all levels + __attribute__((noinline)) data_t *get_leq(WSTEP &tx, const K &key, + data_t **preds, int target_height) { + // We always start at the head sentinel. Scan its tower to find the + // highest non-tail level + data_t *curr = head; + int current_level = 0; + for (int i = NUM_INDEX_LAYERS; i > 0; --i) { + if (head->tower[i].next.get(tx) != tail) { + current_level = i; + break; + } + if (current_level <= target_height) + preds[i - 1] = head; + } + + // Traverse over and down through the index layers + while (current_level > 0) { + // Advance curr by moving forward in this index layer + curr = index_leq(tx, key, curr, current_level); + if (curr == nullptr) + return nullptr; + // On a key match, we can exit immediately + if (curr->key == key) + return curr; + // we need to save current node to preds + if (current_level <= target_height) + preds[current_level - 1] = curr; + --current_level; // Move down a level + } + + // Search in the data layer. Only return if result valid + return data_leq(tx, key, curr); + } + + /// A version of get_le that is specialized for remove, where we need to get + /// the predecessors at all levels + __attribute__((noinline)) data_t *get_le(WSTEP &tx, const K &key, + data_t **preds) { + // We always start at the head sentinel. Scan its tower to find the + // highest non-tail level + data_t *curr = head; + int current_level = 0; + for (int i = NUM_INDEX_LAYERS; i > 0; --i) { + if (head->tower[i].next.get(tx) != tail) { + current_level = i; + break; + } + preds[i - 1] = head; + } + + // Traverse over and down through the index layers + while (current_level > 0) { + // Advance curr by moving forward in this index layer + curr = index_le(tx, key, curr, current_level); + // Deal with index_le failing by returning null + if (curr == nullptr) + return nullptr; + // we need to save current node to preds + preds[current_level - 1] = curr; + --current_level; // Move down a level + } + + // Search in the data layer. Only return if result valid + return data_le(tx, key, curr); + } + + /// Traverse forward from `start`, considering only tower level `level`, + /// stopping at the largest key <= `key` + /// + /// This can return nodes that are OWNED. The caller must check. + /// + /// @param tx The enclosing RSTEP_TM operation's descriptor + /// @param key The key for which we are doing a predecessor query. + /// @param start The start position of this traversal. + /// @param level The tower level to consider + /// + /// @return The node that was found (possibly `start`). The caller must + /// validate the node + data_t *index_leq(STEP &tx, K key, data_t *start, uint64_t level) { + // NB: The consistency argument here is nuanced: keys are immutable. Next + // pointers are never modified during an unstitch. Thus we can race + // forward, and let the caller validate whatever we find. + auto curr = start; + while (true) { + data_t *next = curr->tower[level].next.get(tx); + auto next_key = curr->tower[level].key.get(tx); + if (tx.check_orec(curr) == STMCAS::END_OF_TIME) + return nullptr; + if (next == nullptr) + return nullptr; + if (next == tail) + return curr; + if (next_key == key) + return next; + if (next_key > key) + return curr; + curr = next; + } + } + + /// Traverse forward from `start`, considering only tower level `level`, + /// stopping at the largest key <= `key` + /// + /// This can return nodes that are OWNED. The caller must check. + /// + /// @param tx The enclosing RSTEP_TM operation's descriptor + /// @param key The key for which we are doing a predecessor query. + /// @param start The start position of this traversal. + /// @param level The tower level to consider + /// + /// @return The node that was found (possibly `start`). The caller must + /// validate the node + data_t *index_le(STEP &tx, K key, data_t *start, uint64_t level) { + // NB: The consistency argument here is nuanced: keys are immutable. Next + // pointers are never modified during an unstitch. Thus we can race + // forward, and let the caller validate whatever we find. + auto curr = start; + while (true) { + data_t *next = curr->tower[level].next.get(tx); + auto next_key = curr->tower[level].key.get(tx); + if (tx.check_orec(curr) == STMCAS::END_OF_TIME) + return nullptr; + if (next == nullptr) + return nullptr; + if (next == tail || next_key >= key) + return curr; + curr = next; + } + } + + /// Traverse in the data layer to find the largest node with key <= `key`. + /// + /// This can return an OWNED node + /// + /// @param tx The enclosing RSTEP_TM operation's descriptor + /// @param key The key for which we are doing a predecessor query. + /// @param start The start position of this traversal. This may be the head, + /// or an intermediate point in the list + /// + /// @return The node that was found (possibly `start`), and its orec value. + /// {nullptr, 0} can be returned on inconsistency + data_t *data_leq(STEP &tx, K key, data_t *start) { + // Set up the start point for our traversal, then start iterating + data_t *curr = start; + data_t *next = curr->tower[0].next.get(tx); + while (true) { + // Case 0: `next` is nullptr: restart + if (next == nullptr) + return nullptr; + // Case 1: `next` is tail --> stop the search at curr + if (next == tail) + return curr; + // Case 2: `next` is a data node: stop if next->key >= key + auto nkey = next->key; + if (nkey > key) + return curr; + if (nkey == key) + return next; + // Case 3: Keep traversing + curr = next; + next = next->tower[0].next.get(tx); + } + } + + /// Traverse in the data layer to find the largest node with key <= `key`. + /// + /// @param tx The enclosing RSTEP_TM operation's descriptor + /// @param key The key for which we are doing a predecessor query. + /// @param start The start position of this traversal. This may be the head, + /// or an intermediate point in the list + /// + /// @return The node that was found (possibly `start`), and its orec value. + /// {nullptr, 0} can be returned on inconsistency + data_t *data_le(STEP &tx, K key, data_t *start) { + // Set up the start point for our traversal, then start iterating + data_t *curr = start; + data_t *next = curr->tower[0].next.get(tx); + while (true) { + if (next == nullptr) + return nullptr; + if (next == tail) + return curr; + auto nkey = next->key; + if (nkey >= key) + return curr; + curr = next; + next = next->tower[0].next.get(tx); + } + } + + /// Generate a random level for a new node + /// + /// NB: This code has been verified to produce a nice geometric distribution + /// in constant time per call + /// + /// @param me The caller's STMCAS operation + /// + /// @return a random number between 0 and NUM_INDEX_LAYERS, inclusive + int randomLevel(STMCAS *me) { + // Get a random int between 0 and 0xFFFFFFFF + int rr = me->rand(); + // Add 1 to it, then find the lowest nonzero bit. This way, we never return + // a zero for small integers, and the distribution is correct. + int res = __builtin_ffs(rr + 1); + // Now take one off of that, so that we return a zero-based integer + res -= 1; + // But if rr was 0xFFFFFFFF, we've got a problem, so coerce it back + // Also, drop it down to within NUM_INDEX_LAYERS + return (res < 0 || res > NUM_INDEX_LAYERS) ? NUM_INDEX_LAYERS : res; + } + + /// index_stitch is a small atomic operation that stitches a node in at a + /// given index level. + /// + /// @param me The currently active STMCAS operation + /// @param node The node that was just inserted and stitched into `level` + /// @param level The level below where we're stitching + /// @param release Should `node` be marked UNOWNED before returning? + bool index_stitch(WSTEP &tx, STMCAS *me, data_t *n, data_t *s, data_t **preds, + const K &key, V &val, int target_height) { + // acquire all the levels or fail. n and s are already acquired + for (int level = 0; level < target_height; ++level) { + // preds[level] is actually a /level + 1/ height node + auto pred = preds[level]; + if (!tx.acquire_consistent(pred)) + return false; + } + + // `n` is the predecessor to the node we're making, `s` is the successor + // Fully initialize new_dn before we make it visible at any level. This + // suffices to avoid acquiring the new node. + data_t *new_dn = data_t::make_data(target_height, key, val); + for (int level = 0; level < new_dn->height; ++level) { + auto succ = preds[level]->tower[level + 1].next.get(tx); + new_dn->tower[level + 1].key.set(succ->key, tx); + new_dn->tower[level + 1].next.set(succ, tx); + } + new_dn->tower[0].key.set(s->key, tx); + new_dn->tower[0].next.set(s, tx); + + // Make it visible in the data level, then in index levels from bottom up + n->tower[0].next.set(new_dn, tx); + n->tower[0].key.set(new_dn->key, tx); + for (int level = 0; level < new_dn->height; ++level) { + preds[level]->tower[level + 1].next.set(new_dn, tx); + preds[level]->tower[level + 1].key.set(new_dn->key, tx); + } + return true; + } + + /// Unstitch `node`, starting at its topmost index layer. Reclaim once it's + /// fully unstitched. + /// + /// @param me The currently active STMCAS operation + /// @param node The node that we are unstitching + bool index_unstitch(WSTEP &tx, STMCAS *me, data_t *node, data_t *prev, + data_t **preds) { + // Acquire everything, from bottom to top + for (int level = 0; level < node->height; ++level) + if (!tx.acquire_consistent(preds[level])) + return false; + + // Now update all the pointers, from top to bottom + for (int level = node->height; level >= 0; --level) { + auto pre = (level > 0) ? preds[level - 1] : prev; + auto nxt = node->tower[level].next.get(tx); + pre->tower[level].key.set(nxt->key, tx); + pre->tower[level].next.set(nxt, tx); + } + + // NB: don't forget to set `node`'s pointers to null! + for (int level = node->height; level >= 0; --level) + node->tower[level].next.set(nullptr, tx); + // Reclaim it and we're done + tx.reclaim(node); + return true; + } +}; diff --git a/artifact/ds/STMCAS/slist_omap.h b/artifact/ds/STMCAS/slist_omap.h new file mode 100644 index 0000000..e5fc922 --- /dev/null +++ b/artifact/ds/STMCAS/slist_omap.h @@ -0,0 +1,328 @@ +#pragma once + +#include +#include + +/// An ordered map, implemented as a singly-linked list. This map supports +/// get(), insert(), and remove() operations. +/// +/// Note that the AVOID_OREC_CHECKS flag can be used to create an "optimized" +/// version of this data structure, where list traversal (get_leq) avoids +/// checking orecs in most cases. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param STMCAS The STMCAS implementation (PO or PS) +/// @param AVOID_OREC_CHECKS A flag to enable an optimization that avoids +/// checking orecs when get_leq is doing its read-only +/// traversal +template +class slist_omap { + using WSTEP = typename STMCAS::WSTEP; + using RSTEP = typename STMCAS::RSTEP; + using snapshot_t = typename STMCAS::snapshot_t; + using ownable_t = typename STMCAS::ownable_t; + template using FIELD = typename STMCAS::template sField; + + /// A list node. It has a next pointer, but no key or value. It's useful for + /// sentinels, so that K and V don't have to be default constructable. + struct node_t : ownable_t { + FIELD next; // Pointer to successor + + /// Construct a node + node_t() : ownable_t(), next(nullptr) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~node_t() {} + }; + + /// A list node that also has a key and value. Note that keys are const, and + /// values are only accessed while the node is locked, so neither is a + /// tm_field. + struct data_t : public node_t { + const K key; // The key of this key/value pair + V val; // The value of this key/value pair + + /// Construct a data_t + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(const K &_key, const V &_val) : node_t(), key(_key), val(_val) {} + }; + + /// The pair returned by predecessor queries: a node and it's observed version + struct leq_t { + node_t *_obj = nullptr; // The object + uint64_t _ver = 0; // The observed version of the object + }; + + node_t *const head; // The list head pointer + node_t *const tail; // The list tail pointer + + /// During get_leq, we have a way to periodically capture snapshots, so that a + /// failed search can resume from an intermediate point. This specifies how + /// frequently to take a snapshot (higher is less frequent, i.e., once per + /// SNAPSHOT_FREQUENCY nodes). + const int SNAPSHOT_FREQUENCY; + +public: + /// Default construct a list by constructing and connecting two sentinel nodes + /// + /// @param me The operation that is constructing the list + /// @param cfg A configuration object that has a `snapshot_freq` field + slist_omap(STMCAS *me, auto *cfg) + : head(new node_t()), tail(new node_t()), + SNAPSHOT_FREQUENCY(cfg->snapshot_freq) { + // NB: Even though this code can't abort and doesn't acquire orecs, we still + // need to use a transaction (WSTEP), because we can't set fields of a + // node_t without a legal WSTEP context. We can cheat, though, and not + // bother to acquire orecs, because we know nothing is shared. + WSTEP tx(me); + head->next.set(tail, tx); + } + +private: + /// Convert a snapshot_t into a leq_t + leq_t leq(const snapshot_t &s) { return leq_t{(node_t *)s._obj, s._ver}; } + + /// Convert a leq_t into a snapshot_t + snapshot_t snapshot(const leq_t &l) { return snapshot_t{l._obj, l._ver}; } + + /// get_leq is an inclusive predecessor query that returns the largest node + /// whose key is <= the provided key. It can return the head sentinel, but + /// not the tail sentinel. + /// + /// There is no atomicity between get_leq and its caller. It returns the node + /// it found, along with the value of the orec for that node at the time it + /// was accessed. The caller needs to validate the orec before using the + /// returned node. + /// + /// @param me The calling thread's descriptor + /// @param key The key for which we are doing a predecessor query. + /// @param lt_mode When `true`, this behaves as `get_lt`. When `false`, it + /// behaves as `get_leq`. + /// + /// @return The node that was found, and its orec value + leq_t get_leq(STMCAS *me, const K key, bool lt_mode = false) { + // Start a transactional traversal from the head node, or from the latest + // valid snapshot, if we have one. If a transaction encounters an + // inconsistency, it will come back to here to start a new traversal. + while (true) { + RSTEP tx(me); + + // Figure out where to start this traversal: initially we start at head, + // but on a retry, we might have a snapshot. + // + // NB: snapshots are always < key + leq_t curr = + (me->snapshots.empty()) ? leq_t{head, 0} : leq(me->snapshots.top()); + + // Validate the start point + if (curr._obj == head) { + // For head, be sure to save curr._ver in case we end up returning head + if ((curr._ver = tx.check_orec(curr._obj)) == STMCAS::END_OF_TIME) + continue; + } else { + // Validate snapshot as a continuation. Drop the snapshot on failure. + if (!tx.check_continuation(curr._obj, curr._ver)) { + me->snapshots.drop(); + continue; + } + } + + // Prepare a countdown timer for snapshots + int nodes_until_snapshot = SNAPSHOT_FREQUENCY; + + // Starting at `next`, search for key. Breaking out of this will take us + // back to the top of the function. + while (true) { + // Read the next node, fail if we can't do it consistently + auto *next = curr._obj->next.get(tx); + uint64_t next_ver = 0; + if (!AVOID_OREC_CHECKS) { + next_ver = tx.check_orec(next); + if (next_ver == STMCAS::END_OF_TIME) + break; + } + + // Stop if next's key is too big or next is tail + if (next == tail) { + if (AVOID_OREC_CHECKS && + (curr._ver = tx.check_orec(curr._obj)) == STMCAS::END_OF_TIME) + break; + return curr; + } + data_t *dn = static_cast(next); + if (lt_mode ? dn->key >= key : dn->key > key) { + if (AVOID_OREC_CHECKS && + (curr._ver = tx.check_orec(curr._obj)) == STMCAS::END_OF_TIME) + break; + return curr; + } + + // Stop if `next` is the match we were hoping for + if (dn->key == key) { + if (AVOID_OREC_CHECKS) { + next_ver = tx.check_orec(next); + if (next_ver == STMCAS::END_OF_TIME) + break; + } + return {next, next_ver}; + } + + // Keep traversing to `next`. Maybe take a snapshot first + if (--nodes_until_snapshot == 0) { + // if (AVOID_OREC_CHECKS && + // (curr._ver = tx.check_orec(curr._obj)) == STMCAS::END_OF_TIME) + // break; + // me->snapshots.push_back(snapshot(curr)); + if (AVOID_OREC_CHECKS) { + if ((curr._ver = tx.check_orec(curr._obj)) != STMCAS::END_OF_TIME) + me->snapshots.push_back(snapshot(curr)); + } else + me->snapshots.push_back(snapshot(curr)); + nodes_until_snapshot = SNAPSHOT_FREQUENCY; + } + curr._obj = next; + if (!AVOID_OREC_CHECKS) + curr._ver = next_ver; + } + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(STMCAS *me, const K &key, V &val) { + // If we can't use the result of get_leq, we'll loop back, and the next + // get_leq will start from a snapshot + me->snapshots.clear(); + while (true) { + // get_leq will use a read-only transaction to find the largest node with + // a key <= `key`. + // + // Postconditions of get_leq: n != null, n != tail, we have a valid + // node/version pair, and n.key <= `key` + auto n = get_leq(me, key); + + // Since we have EBR, we can read n.key without validating and fast-fail + // on key-not-found + if (n._obj == head || static_cast(n._obj)->key != key) + return false; + + // Use a hand-over-hand TM pattern to finish the get(). If the value is + // scalar, we can cast it to atomic, read it, and validate. Otherwise we + // need to lock the node. + if (std::is_scalar::value) { + RSTEP tx(me); + + // NB: given EBR, we don't need to worry about n._obj being deleted, so + // we don't need to validate before looking at the value + data_t *dn = static_cast(n._obj); + V val_copy = reinterpret_cast *>(&dn->val)->load( + std::memory_order_acquire); + if (!tx.check_continuation(n._obj, n._ver)) + continue; + val = val_copy; + return true; + } else { + WSTEP tx(me); + + // If this acquire continuation succeeds, it's not deleted, it's a data + // node, and it's valid. If it fails, we need to restart + if (!tx.acquire_continuation(n._obj, n._ver)) { + tx.unwind(); // not strictly needed, but a good habit :) + continue; + } + + // NB: we aren't changing val, so we can unwind when we're done with it + val = static_cast(n._obj)->val; + tx.unwind(); + return true; + } + } + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(STMCAS *me, const K &key, V &val) { + // NB: The pattern here is similar to `get` + me->snapshots.clear(); + while (true) { + auto n = get_leq(me, key); + + // Since we have EBR, we can look at n._obj->key without validation. If + // it matches `key`, return false. + if (n._obj != head && static_cast(n._obj)->key == key) + return false; + + // either n._obj is `head`, or it's a key that's too small. Let's insert! + WSTEP tx(me); + // lock n, fail if we can't get it + if (!tx.acquire_continuation(n._obj, n._ver)) { + tx.unwind(); + continue; + } + + // stitch in a new node + data_t *new_dn = new data_t(key, val); + new_dn->next.set(n._obj->next.get(tx), tx); + n._obj->next.set(new_dn, tx); + return true; + } + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(STMCAS *me, const K &key) { + // NB: The pattern here is similar to `get` + me->snapshots.clear(); + while (true) { + // NB: this will be a lt query, not a leq query + auto prev = get_leq(me, key, true); + + WSTEP tx(me); + // lock the predecessor, read its next + if (!tx.acquire_continuation(prev._obj, prev._ver)) { + tx.unwind(); + continue; + } + auto curr = prev._obj->next.get(tx); + + // if curr doesn't have a matching key, fail + if (curr == tail || static_cast(curr)->key != key) { + tx.unwind(); + return false; + } + + // lock the node to remove, then unstitch it + if (!tx.acquire_aggressive(curr)) { + tx.unwind(); + continue; + } + auto next = curr->next.get(tx); + prev._obj->next.set(next, tx); + tx.reclaim(curr); + return true; + } + } +}; diff --git a/artifact/ds/baseline/ext_ticket_bst/plaf.h b/artifact/ds/baseline/ext_ticket_bst/plaf.h new file mode 100644 index 0000000..c0a6744 --- /dev/null +++ b/artifact/ds/baseline/ext_ticket_bst/plaf.h @@ -0,0 +1,41 @@ +/** + * C++ record manager implementation (PODC 2015) by Trevor Brown. + * + * Copyright (C) 2015 Trevor Brown + * + */ + +#ifndef MACHINECONSTANTS_H +#define MACHINECONSTANTS_H + +#ifndef MAX_THREADS_POW2 +#define MAX_THREADS_POW2 \ + 128 // MUST BE A POWER OF TWO, since this is used for some bitwise operations +#endif +#ifndef LOGICAL_PROCESSORS +#define LOGICAL_PROCESSORS MAX_THREADS_POW2 +#endif + +#ifndef SOFTWARE_BARRIER +#define SOFTWARE_BARRIER asm volatile("" : : : "memory") +#endif + +// the following definition is only used to pad data to avoid false sharing. +// although the number of words per cache line is actually 8, we inflate this +// figure to counteract the effects of prefetching multiple adjacent cache +// lines. +#define PREFETCH_SIZE_WORDS 16 +#define PREFETCH_SIZE_BYTES 128 +#define BYTES_IN_CACHE_LINE 64 + +#define CAT2(x, y) x##y +#define CAT(x, y) CAT2(x, y) + +#define PAD64 volatile char CAT(___padding, __COUNTER__)[64] +#define PAD volatile char CAT(___padding, __COUNTER__)[128] + +#define CASB __sync_bool_compare_and_swap +#define CASV __sync_val_compare_and_swap +#define FAA __sync_fetch_and_add + +#endif /* MACHINECONSTANTS_H */ diff --git a/artifact/ds/baseline/ext_ticket_bst/ticket_impl.h b/artifact/ds/baseline/ext_ticket_bst/ticket_impl.h new file mode 100644 index 0000000..44fb5e9 --- /dev/null +++ b/artifact/ds/baseline/ext_ticket_bst/ticket_impl.h @@ -0,0 +1,379 @@ +/* + * File: bst_tk.c + * Author: Vasileios Trigonakis + * Description: Asynchronized Concurrency: The Secret to Scaling Concurrent + * Search Data Structures, Tudor David, Rachid Guerraoui, Vasileios + *Trigonakis, ASPLOS '15 bst_tk.c is part of ASCYLIB + * + * Copyright (c) 2014 Vasileios Trigonakis , + * Tudor David + * Distributed Programming Lab (LPD), EPFL + * + * ASCYLIB is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +/* + * File: ticket.h + * Author: Trevor Brown + * + * Substantial improvements to interface, memory reclamation and bug fixing. + * + * Created on June 7, 2017, 1:38 PM + */ + +#pragma once + +#include "plaf.h" +#include +#include +#include +#define likely(x) __builtin_expect((x), 1) +#define unlikely(x) __builtin_expect((x), 0) + +#if !defined(COMPILER_BARRIER) +#define COMPILER_BARRIER asm volatile("" ::: "memory") +#endif + +typedef union tl32 { + struct { + volatile uint16_t version; + volatile uint16_t ticket; + }; + volatile uint32_t to_uint32; +} tl32_t; + +typedef union tl { + tl32_t lr[2]; + uint64_t to_uint64; +} tl_t; + +static inline int tl_trylock_version(volatile tl_t *tl, volatile tl_t *tl_old, + int right) { + uint16_t version = tl_old->lr[right].version; + // uint16_t one = (uint16_t) 1; + if (unlikely(version != tl_old->lr[right].ticket)) { + return 0; + } + +#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 6 + tl32_t tlo = { + {.version = version, + .ticket = version}}; //{ .version = version, .ticket = version}; + tl32_t tln = { + {.version = version, + .ticket = + (uint16_t)(version + + one)}}; //{.version = version, .ticket = (version + 1)}; + return CASV(&tl->lr[right].to_uint32, tlo.to_uint32, tln.to_uint32) == + tlo.to_uint32; +#else + tl32_t tlo = {{version, version}}; + tl32_t tln = {{version, (uint16_t)(version + 1)}}; +#endif + return CASV(&tl->lr[right].to_uint32, tlo.to_uint32, tln.to_uint32) == + tlo.to_uint32; +} + +#define TLN_REMOVED 0x0000FFFF0000FFFF0000LL + +static inline int tl_trylock_version_both(volatile tl_t *tl, + volatile tl_t *tl_old) { + uint16_t v0 = tl_old->lr[0].version; + uint16_t v1 = tl_old->lr[1].version; + if (unlikely(v0 != tl_old->lr[0].ticket || v1 != tl_old->lr[1].ticket)) { + return 0; + } + +#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 6 + tl_t tlo = {.to_uint64 = tl_old->to_uint64}; + return CASV(&tl->to_uint64, tlo.to_uint64, TLN_REMOVED) == tlo.to_uint64; +#else + /* tl_t tlo; */ + /* tlo.uint64_t = tl_old->to_uint64; */ + uint64_t tlo = *(uint64_t *)tl_old; + + return CASV((uint64_t *)tl, tlo, TLN_REMOVED) == tlo; +#endif +} + +static inline void tl_unlock(volatile tl_t *tl, int right) { + /* PREFETCHW(tl); */ + COMPILER_BARRIER; + tl->lr[right].version = tl->lr[right].version + 1; + COMPILER_BARRIER; +} + +static inline void tl_revert(volatile tl_t *tl, int right) { + /* PREFETCHW(tl); */ + COMPILER_BARRIER; + tl->lr[right].ticket = tl->lr[right].ticket - 1; + COMPILER_BARRIER; +} + +template +struct node_t : DESCRIPTOR::reclaimable_t { + skey_t key; + sval_t val; + struct node_t *volatile left; + struct node_t *volatile right; + volatile tl_t lock; +#ifdef USE_PADDING + char pad[PAD_SIZE]; +#endif +}; + +template +class ticket { +private: + PAD; + node_t *root; + PAD; + const skey_t KEY_MIN; + const skey_t KEY_MAX; + const sval_t NO_VALUE; + PAD; + int init[MAX_THREADS_POW2] = { + 0, + }; + PAD; + + node_t * + new_node(skey_t key, sval_t val, node_t *l, + node_t *r); + node_t *new_node_no_init(); + +public: + ticket(DESCRIPTOR *me, auto *cfg) + : KEY_MIN(_KEY_MIN), KEY_MAX(_KEY_MAX), NO_VALUE(_VAL_RESERVED) { + node_t *_min = + new_node(KEY_MIN, NO_VALUE, NULL, NULL); + node_t *_max = + new_node(KEY_MAX, NO_VALUE, NULL, NULL); + root = new_node(KEY_MAX, NO_VALUE, _min, _max); + } + + ~ticket() {} + + node_t *getRoot() { return root; } + + sval_t get_internal(DESCRIPTOR *me, skey_t &key, sval_t &val); + sval_t insert_internal(DESCRIPTOR *me, skey_t &key, sval_t &val); + sval_t remove_internal(DESCRIPTOR *me, skey_t &key); + + node_t *get_root() { return root; } + + bool get(DESCRIPTOR *me, const skey_t &key, sval_t &val) { + skey_t k = key; + sval_t v = get_internal(me, k, val); + if (v == NO_VALUE) + return false; + val = v; + return true; + } + + bool insert(DESCRIPTOR *me, const skey_t &key, sval_t &val) { + sval_t v = val; + skey_t k = key; + return NO_VALUE == insert_internal(me, k, v); + } + + bool remove(DESCRIPTOR *me, const skey_t &key) { + sval_t k = key; + return NO_VALUE != remove_internal(me, k); + } +}; + +template +node_t *ticket::new_node( + skey_t key, sval_t val, node_t *l, + node_t *r) { + auto node = new_node_no_init(); + node->val = val; + node->key = key; + node->left = l; + node->right = r; + return node; +} + +template +node_t * +ticket::new_node_no_init() { + auto node = new node_t(); + if (unlikely(node == NULL)) { + // perror("malloc @ new_node"); + // exit(1); + } + node->lock.to_uint64 = 0; + node->val = NO_VALUE; + return node; +} + +template +sval_t ticket::get_internal(D *me, skey_t &key, + sval_t &val) { + node_t *curr = root; + + while (likely(curr->left != NULL)) { + if (key < curr->key) { + curr = curr->left; + } else { + curr = curr->right; + } + } + + if (curr->key == key) { + return curr->val; + } + + return NO_VALUE; +} + +template +sval_t ticket::insert_internal(D *me, + skey_t &key, + sval_t &val) { + node_t *curr; + node_t *pred = NULL; + volatile uint64_t curr_ver = 0; + uint64_t pred_ver = 0, right = 0; + +retry : { // reclamation guarded section + curr = root; + do { + curr_ver = curr->lock.to_uint64; + + pred = curr; + pred_ver = curr_ver; + + if (key < curr->key) { + right = 0; + curr = curr->left; + } else { + right = 1; + curr = curr->right; + } + } while (likely(curr->left != NULL)); + + if (curr->key == key) { + // insert if absent + return curr->val; + } + + // node_t* nn_leaked = new_node(tid, key, val, NULL, + // NULL); + node_t *nn = new_node(key, val, NULL, NULL); + node_t *nr = new_node_no_init(); + + if ((!tl_trylock_version(&pred->lock, (volatile tl_t *)&pred_ver, right))) { + // recmgr->deallocate(tid, nn); + // recmgr->deallocate(tid, nr); + goto retry; + } + + if (key < curr->key) { + nr->key = curr->key; + nr->left = nn; + nr->right = curr; + } else { + nr->key = key; + nr->left = curr; + nr->right = nn; + } + + if (right) { + pred->right = nr; + } else { + pred->left = nr; + } + + tl_unlock(&pred->lock, right); + + return NO_VALUE; +} +} + +template +sval_t ticket::remove_internal(D *me, + skey_t &key) { + node_t *curr; + node_t *pred = NULL; + node_t *ppred = NULL; + volatile uint64_t curr_ver = 0; + uint64_t pred_ver = 0, ppred_ver = 0, right = 0, pright = 0; + +retry : + +{ // reclamation guarded section + curr = root; + + do { + curr_ver = curr->lock.to_uint64; + + ppred = pred; + ppred_ver = pred_ver; + pright = right; + + pred = curr; + pred_ver = curr_ver; + + if (key < curr->key) { + right = 0; + curr = curr->left; + } else { + right = 1; + curr = curr->right; + } + } while (likely(curr->left != NULL)); + + if (curr->key != key) { + return NO_VALUE; + } + + if ((!tl_trylock_version(&ppred->lock, (volatile tl_t *)&ppred_ver, + pright))) { + goto retry; + } + + if ((!tl_trylock_version_both(&pred->lock, (volatile tl_t *)&pred_ver))) { + tl_revert(&ppred->lock, pright); + goto retry; + } + + if (pright) { + if (right) { + ppred->right = pred->left; + } else { + ppred->right = pred->right; + } + + } else { + if (right) { + ppred->left = pred->left; + } else { + ppred->left = pred->right; + } + } + + tl_unlock(&ppred->lock, pright); + + me->reclaim(curr); // recmgr->retire(tid, curr); + me->reclaim(pred); // recmgr->retire(tid, pred); + + return curr->val; +} +} diff --git a/artifact/ds/baseline/int_bst_pathcas/casword.h b/artifact/ds/baseline/int_bst_pathcas/casword.h new file mode 100644 index 0000000..bdac6dd --- /dev/null +++ b/artifact/ds/baseline/int_bst_pathcas/casword.h @@ -0,0 +1,70 @@ +#pragma once + +#include "kcas.h" +#include +#include +#include +#include +using namespace std; + +// #define casword_t uintptr_t +#define SHIFT_BITS 2 +#define CASWORD_CAST(x) ((CASWORD_BITS_TYPE)(x)) + +template casword::casword() { + T a = 0; + bits = CASWORD_CAST(a); +} + +template inline T casword::setInitVal(T other) { + if (is_pointer::value) { + bits = CASWORD_CAST(other); + } else { + bits = CASWORD_CAST(other); + assert((bits & 0xE000000000000000) == 0); + bits = bits << SHIFT_BITS; + } + + return other; +} + +template inline casword::operator T() { + if (is_pointer::value) { + return (T)kcas::instance.readPtr(&bits); + } else { + return (T)kcas::instance.readVal(&bits); + } +} + +template inline T casword::operator->() { + assert(is_pointer::value); + return *this; +} + +template inline T casword::getValue() { + if (is_pointer::value) { + return (T)kcas::instance.readPtr(&bits); + } else { + return (T)kcas::instance.readVal(&bits); + } +} + +template inline casword_t casword::getValueUnsafe(bool &isPtr) { + isPtr = is_pointer::value; + return bits; +} + +template +inline void casword::addToDescriptor(T oldVal, T newVal) { + auto descriptor = kcas::instance.getDescriptor(); + auto c_oldVal = (casword_t)oldVal; + auto c_newVal = (casword_t)newVal; + assert(((c_oldVal & 0xE000000000000000) == 0) && + ((c_newVal & 0xE000000000000000) == 0)); + + if (is_pointer::value) { + descriptor->addPtrAddr(&bits, c_oldVal, c_newVal); + } else { + descriptor->addValAddr(&bits, c_oldVal, c_newVal); + } +} diff --git a/artifact/ds/baseline/int_bst_pathcas/internal_kcas_avl.h b/artifact/ds/baseline/int_bst_pathcas/internal_kcas_avl.h new file mode 100644 index 0000000..0cece1c --- /dev/null +++ b/artifact/ds/baseline/int_bst_pathcas/internal_kcas_avl.h @@ -0,0 +1,1251 @@ +#pragma once + +#include + +#ifndef KCAS_TYPE +#define KCAS_HTM +#define KCAS_TYPE "KCAS_HTM" +#endif + +#define MAX_KCAS 21 +#include "kcas.h" + +#include +#include +#include +#include +#include +#include + +using namespace std; + +#define MAX_THREADS 200 +#define MAX_PATH_SIZE 16384 +#define PADDING_BYTES 128 + +#define IS_MARKED(word) (word & 0x1) + +template +struct Node : RecordManager::reclaimable_t { + casword key; + casword vNumMark; + casword *> left; + casword *> right; + casword *> parent; + casword height; + casword value; +}; + +enum RetCode : int { + RETRY = 0, + UNNECCESSARY = 0, + FAILURE = -1, + SUCCESS = 1, + SUCCESS_WITH_HEIGHT_UPDATE = 2 +}; + +template +class InternalKCAS { +private: + /* + * ObservedNode acts as a Node-VersionNumber pair to track an "observed + * version number" of a given node. We can then be sure that a version number + * does not change after we have read it by comparing the current version + * number to this saved value NOTE: This is a thread-private structure, no + * fields need to be volatile + */ + struct ObservedNode { + ObservedNode() {} + Node *node = NULL; + casword_t oVNumMark = -1; + }; + + struct PathContainer { + ObservedNode path[MAX_PATH_SIZE]; + volatile char padding[PADDING_BYTES]; + }; + + volatile char padding0[PADDING_BYTES]; + // Debugging, used to validate that no thread's parent can't be NULL, save for + // the root + bool init = false; + const int numThreads; + const int minKey; + const long long maxKey; + volatile char padding4[PADDING_BYTES]; + Node *root; + volatile char padding5[PADDING_BYTES]; + RecordManager *const recmgr; + volatile char padding7[PADDING_BYTES]; + PathContainer paths[MAX_THREADS]; + volatile char padding8[PADDING_BYTES]; + +public: + InternalKCAS(RecordManager *me, auto *cfg); + + ~InternalKCAS(); + + bool contains(const int tid, const K &key); + + V insertIfAbsent(const int tid, const K &key, const V &value); + + V erase(const int tid, const K &key); + + bool validate(); + + void printDebuggingDetails(); + + Node *getRoot(); + + void initThread(const int tid); + + void deinitThread(const int tid); + + int getHeight(Node *node); + + RecordManager *const debugGetRecMgr() { return recmgr; } + + // These are the functions that the benchmark will call + bool get(RecordManager *me, const K &key, V &val) { + return contains(me->tid, key); + } + + bool insert(RecordManager *me, const K &key, V &val) { + return 0 == insertIfAbsent(me->tid, key, val); + } + + bool remove(RecordManager *me, const K &key) { + return 0 != erase(me->tid, key); + } + +private: + Node * + createNode(const int tid, Node *parent, K key, V value); + + void freeSubtree(const int tid, Node *node); + + long validateSubtree(Node *node, long smaller, + long larger, std::unordered_set &keys, + ofstream &graph, ofstream &log, bool &errorFound); + + int internalErase(const int tid, ObservedNode &parentObserved, + ObservedNode &nodeObserved, const K &key); + + int internalInsert(const int tid, ObservedNode &parentObserved, const K &key, + const V &value); + + int countChildren(const int tid, Node *node); + + int getSuccessor(const int tid, Node *node, + ObservedNode &succObserved, const K &key); + + bool validatePath(const int tid, const int &size, const K &key, + ObservedNode path[]); + + int search(const int tid, ObservedNode &parentObserved, + ObservedNode &nodeObserved, const K &key); + + int rotateRight(const int tid, ObservedNode &parentObserved, + ObservedNode &nodeObserved, ObservedNode &leftChildObserved); + + int rotateLeft(const int tid, ObservedNode &parentObserved, + ObservedNode &nodeObserved, ObservedNode &rightChildObserved); + + int rotateLeftRight(const int tid, ObservedNode &parentObserved, + ObservedNode &nodeObserved, + ObservedNode &leftChildObserved, + ObservedNode &leftRigthChildObserved); + + int rotateRightLeft(const int tid, ObservedNode &parentObserved, + ObservedNode &nodeObserved, + ObservedNode &rightChildObserved, + ObservedNode &rightLeftChildObserved); + + void fixHeightAndRebalance(const int tid, Node *node); + + int fixHeight(const int tid, ObservedNode &observedNode); +}; + +template +Node * +InternalKCAS::createNode( + const int tid, Node *parent, K key, V value) { + Node *node = new Node(); + // No node, save for root, should have a NULL parent + // assert(!init || parent->key < maxKey); + node->key.setInitVal(key); + node->value.setInitVal(value); + node->parent.setInitVal(parent); + node->vNumMark.setInitVal(0); + node->left.setInitVal(NULL); + node->right.setInitVal(NULL); + node->height.setInitVal(1); + return node; +} + +template +InternalKCAS::InternalKCAS( + RecordManager *_recmgr, auto *cfg) + : numThreads(_numThreads), minKey(_minKey), maxKey(_maxKey), + recmgr(_recmgr) { + assert(_numThreads < MAX_THREADS); + int tid = 0; + initThread(tid); + root = createNode(0, NULL, maxKey, 0); + init = true; +} + +template +InternalKCAS::~InternalKCAS() { + int tid = 0; + initThread(tid); + freeSubtree(tid, root); + deinitThread(tid); + // delete recmgr; +} + +template +inline Node * +InternalKCAS::getRoot() { + return root->left; +} + +template +void InternalKCAS::initThread(const int tid) { + // recmgr->initThread(tid); +} + +template +void InternalKCAS::deinitThread(const int tid) { + // recmgr->deinitThread(tid); +} + +template +int InternalKCAS::getHeight( + Node *node) { + return node == NULL ? 0 : node->height; +} + +/* getSuccessor(const int tid, Node * node, ObservedNode &succObserved, int key) + * ### Gets the successor of a given node in it's subtree ### + * returns the successor of a given node stored within an ObservedNode with the + * observed version number. + * Returns an integer, 1 indicating the process was successful, 0 indicating a + * retry + */ +template +inline int +InternalKCAS::getSuccessor( + const int tid, Node *node, ObservedNode &oSucc, + const K &key) { + auto &path = paths[tid].path; + + while (true) { + Node *succ = node->right; + path[0].node = node; + path[0].oVNumMark = node->vNumMark; + int currSize = 1; + + while (succ != NULL) { + assert(currSize < MAX_PATH_SIZE - 1); + path[currSize].node = succ; + path[currSize].oVNumMark = succ->vNumMark; + currSize++; + succ = succ->left; + } + + if (validatePath(tid, currSize, key, path) && currSize > 1) { + oSucc = path[currSize - 1]; + return RetCode::SUCCESS; + } else { + return RetCode::RETRY; + } + } +} + +template +inline bool +InternalKCAS::contains( + const int tid, const K &key) { + assert(key <= maxKey); + int result; + ObservedNode oNode; + ObservedNode oParent; + // auto guard = recmgr->getGuard(tid); + + while ((result = search(tid, oParent, oNode, key)) == RetCode::RETRY) { + /* keep trying until we get a result */ + } + return result == RetCode::SUCCESS; +} + +/* search(const int tid, ObservedNode &predObserved, ObservedNode + * &parentObserved, ObservedNode &nodeObserved, const int &key) A proposed + * successor-predecessor pair is generated by searching for a given key, if the + * key is not found, the path is then validated to ensure it was not missed. + * Where appropriate, the predecessor (predObserved), parent (parentObserved) + * and node (nodeObserved) are provided to the caller. + */ +template +int InternalKCAS::search( + const int tid, ObservedNode &oParent, ObservedNode &oNode, const K &key) { + assert(key <= maxKey); + + K currKey; + casword_t nodeVNumMark; + + ObservedNode *path = paths[tid].path; + path[0].node = root; + path[0].oVNumMark = root->vNumMark; + + Node *node = root->left; + + int currSize = 1; + + while (true) { + assert(currSize < MAX_PATH_SIZE - 1); + // We have hit a terminal node without finding our key, must validate + if (node == NULL) { + if (validatePath(tid, currSize, key, path)) { + oParent = path[currSize - 1]; + return RetCode::FAILURE; + } else { + return RetCode::RETRY; + } + } + + nodeVNumMark = node->vNumMark; + + currKey = node->key; + + path[currSize].node = node; + path[currSize].oVNumMark = nodeVNumMark; + currSize++; + + if (key > currKey) { + node = node->right; + } else if (key < currKey) { + node = node->left; + } // no validation required on finding a key + else { + oParent = path[currSize - 2]; + oNode = path[currSize - 1]; + return RetCode::SUCCESS; + } + } +} + +/* validatePath(const int tid, const int size, const int key, ObservedNode + * path[MAX_PATH_SIZE]) + * ### Validates all nodes in a path such that they are not marked and their + * version numbers have not changed ### validated a given path, ensuring that + * all version numbers of observed nodes still match the version numbers stored + * locally within nodes within the tree. This provides the caller with certainty + * that there was a time that this path existed in the tree Returns true for a + * valid path Returns false for an invalid path (some node version number + * changed) + */ +template +inline bool +InternalKCAS::validatePath( + const int tid, const int &size, const K &key, ObservedNode path[]) { + assert(size > 0 && size < MAX_PATH_SIZE); + + for (int i = 0; i < size; i++) { + ObservedNode oNode = path[i]; + if (oNode.node->vNumMark != oNode.oVNumMark || IS_MARKED(oNode.oVNumMark)) { + return false; + } + } + return true; +} + +template +inline V InternalKCAS::insertIfAbsent(const int tid, const K &key, + const V &value) { + ObservedNode oParent; + ObservedNode oNode; + + while (true) { + // auto guard = recmgr->getGuard(tid); + + int res; + while ((res = (search(tid, oParent, oNode, key))) == RetCode::RETRY) { + /* keep trying until we get a result */ + } + + if (res == RetCode::SUCCESS) { + return (V)oNode.node->value; + } + + assert(res == RetCode::FAILURE); + if (internalInsert(tid, oParent, key, value)) { + return 0; + } + } +} + +template +int InternalKCAS::internalInsert(const int tid, ObservedNode &oParent, + const K &key, const V &value) { + /* INSERT KCAS (K = 2-3) + * predecessor's version number*: vNumber -> vNumber + 1 + * parent's version number: vNumber -> vNumber + 1 + * parent's child pointer: NULL -> newNode + */ + + kcas::start(); + Node *parent = oParent.node; + + Node *newNode = createNode(tid, parent, key, value); + + if (key > parent->key) { + kcas::add(&parent->right, (Node *)NULL, newNode); + } else if (key < parent->key) { + kcas::add(&parent->left, (Node *)NULL, newNode); + } else { + // recmgr->reclaim(newNode); + return RetCode::RETRY; + } + + kcas::add(&parent->vNumMark, oParent.oVNumMark, oParent.oVNumMark + 2); + + if (kcas::execute()) { + fixHeightAndRebalance(tid, parent); + return RetCode::SUCCESS; + } + + // recmgr->reclaim(newNode); + + return RetCode::RETRY; +} + +template +inline V +InternalKCAS::erase( + const int tid, const K &key) { + ObservedNode oParent; + ObservedNode oNode; + + while (true) { + // auto guard = recmgr->getGuard(tid); + + int res = 0; + while ((res = (search(tid, oParent, oNode, key))) == RetCode::RETRY) { + /* keep trying until we get a result */ + } + + if (res == RetCode::FAILURE) { + return 0; + } + + assert(res == RetCode::SUCCESS); + if ((res = internalErase(tid, oParent, oNode, key))) { + return (V)oNode.node->value; + } + } +} + +template +int InternalKCAS::internalErase(const int tid, ObservedNode &oParent, + ObservedNode &oNode, const K &key) { + Node *parent = oParent.node; + Node *node = oNode.node; + + int numChildren = countChildren(tid, node); + + kcas::start(); + + if (IS_MARKED(oParent.oVNumMark) || IS_MARKED(oNode.oVNumMark)) { + return RetCode::RETRY; + } + + if (numChildren == 0) { + /* No-Child Delete + * Unlink node + */ + + if (key > parent->key) { + kcas::add(&parent->right, node, (Node *)NULL); + } else if (key < parent->key) { + kcas::add(&parent->left, node, (Node *)NULL); + } else { + return RetCode::RETRY; + } + + kcas::add(&parent->vNumMark, oParent.oVNumMark, oParent.oVNumMark + 2, + &node->vNumMark, oNode.oVNumMark, oNode.oVNumMark + 3); + + if (kcas::execute()) { + assert(IS_MARKED(node->vNumMark)); + // recmgr->reclaim(node); + fixHeightAndRebalance(tid, parent); + + return RetCode::SUCCESS; + } + + return RetCode::RETRY; + } else if (numChildren == 1) { + /* One-Child Delete + * Reroute parent pointer around removed node + */ + + Node *left = node->left; + Node *right = node->right; + Node *reroute; + + // determine which child will be the replacement + if (left != NULL) { + reroute = left; + } else if (right != NULL) { + reroute = right; + } else { + return RetCode::RETRY; + } + + casword_t rerouteVNum = reroute->vNumMark; + + if (IS_MARKED(rerouteVNum)) { + return RetCode::RETRY; + } + + if (key > parent->key) { + kcas::add(&parent->right, node, reroute); + } else if (key < parent->key) { + kcas::add(&parent->left, node, reroute); + } else { + return RetCode::RETRY; + } + + kcas::add(&reroute->parent, node, parent, &reroute->vNumMark, rerouteVNum, + rerouteVNum + 2, &node->vNumMark, oNode.oVNumMark, + oNode.oVNumMark + 3, &parent->vNumMark, oParent.oVNumMark, + oParent.oVNumMark + 2); + + if (kcas::execute()) { + assert(IS_MARKED(node->vNumMark)); + // recmgr->reclaim(node); + fixHeightAndRebalance(tid, parent); + + return RetCode::SUCCESS; + } + + return RetCode::RETRY; + } else if (numChildren == 2) { + /* Two-Child Delete + * Promotion of descendant successor to this node by replacing the key/value + * pair at the node + */ + + ObservedNode oSucc; + + // the (decendant) successor's key will be promoted + if (getSuccessor(tid, node, oSucc, key) == RetCode::RETRY) { + return RetCode::RETRY; + } + + if (oSucc.node == NULL) { + return RetCode::RETRY; + } + + Node *succ = oSucc.node; + Node *succParent = succ->parent; + + ObservedNode oSuccParent; + oSuccParent.node = succParent; + oSuccParent.oVNumMark = succParent->vNumMark; + + if (oSuccParent.node == NULL) { + return RetCode::RETRY; + } + + K succKey = succ->key; + + assert(succKey <= maxKey); + + if (IS_MARKED(oSuccParent.oVNumMark)) { + return RetCode::RETRY; + } + + Node *succRight = succ->right; + + if (succRight != NULL) { + casword_t succRightVNum = succRight->vNumMark; + + if (IS_MARKED(succRightVNum)) { + return RetCode::RETRY; + } + + kcas::add(&succRight->parent, succ, succParent, &succRight->vNumMark, + succRightVNum, succRightVNum + 2); + } + + if (succParent->right == succ) { + kcas::add(&succParent->right, succ, succRight); + } else if (succParent->left == succ) { + kcas::add(&succParent->left, succ, succRight); + } else { + return RetCode::RETRY; + } + + V nodeVal = node->value; + V succVal = succ->value; + + kcas::add(&node->value, nodeVal, succVal, &node->key, key, succKey, + &succ->vNumMark, oSucc.oVNumMark, oSucc.oVNumMark + 3, + &succParent->vNumMark, oSuccParent.oVNumMark, + oSuccParent.oVNumMark + 2); + + if (succParent != node) { + kcas::add(&node->vNumMark, oNode.oVNumMark, oNode.oVNumMark + 2); + } + + if (kcas::execute()) { + assert(IS_MARKED(succ->vNumMark)); + // recmgr->reclaim(succ); + // successor's parent is the only node that's height will have been + // impacted + fixHeightAndRebalance(tid, succParent); + return RetCode::SUCCESS; + } + + return RetCode::RETRY; + } + assert(false); + return RetCode::RETRY; +} + +template +void InternalKCAS:: + fixHeightAndRebalance(const int tid, Node *node) { + ObservedNode oNode; + ObservedNode oParent; + + int propRes; + + while (node != root) { + ObservedNode oRight; + ObservedNode oLeft; + ObservedNode oRightLeft; + ObservedNode oLeftRight; + ObservedNode oRightRight; + ObservedNode oLeftLeft; + + oNode.node = node; + oNode.oVNumMark = node->vNumMark; + + oParent.node = node->parent; + oParent.oVNumMark = oParent.node->vNumMark; + + if (IS_MARKED(oNode.oVNumMark)) { + return; + } + + Node *left = node->left; + if (left != NULL) { + oLeft.node = left; + oLeft.oVNumMark = left->vNumMark; + } + + Node *right = node->right; + if (right != NULL) { + oRight.node = right; + oRight.oVNumMark = right->vNumMark; + } + + int localBalance = getHeight(left) - getHeight(right); + + if (localBalance >= 2) { + if (left == NULL || IS_MARKED(oLeft.oVNumMark)) { + continue; + } + + Node *leftRight = left->right; + Node *leftLeft = left->left; + + if (leftRight != NULL) { + oLeftRight.node = leftRight; + oLeftRight.oVNumMark = leftRight->vNumMark; + } + + if (leftLeft != NULL) { + oLeftLeft.node = leftLeft; + oLeftLeft.oVNumMark = leftLeft->vNumMark; + } + + int leftBalance = getHeight(leftLeft) - getHeight(leftRight); + + if (leftBalance < 0) { + if (leftRight == NULL) { + continue; + } + if (rotateLeftRight(tid, oParent, oNode, oLeft, oLeftRight)) { + // node is now the lowest on the tree, so it must be rebalanced first + // cannot simply loop... + fixHeightAndRebalance(tid, node); + fixHeightAndRebalance(tid, left); + fixHeightAndRebalance(tid, leftRight); + node = oParent.node; + } + } else { + if (rotateRight(tid, oParent, oNode, oLeft) == RetCode::SUCCESS) { + fixHeightAndRebalance(tid, node); + fixHeightAndRebalance(tid, left); + node = oParent.node; + } + } + } else if (localBalance <= -2) { + if (right == NULL || IS_MARKED(oRight.oVNumMark)) { + continue; + } + + Node *rightLeft = right->left; + Node *rightRight = right->right; + + if (rightLeft != NULL) { + oRightLeft.node = rightLeft; + oRightLeft.oVNumMark = rightLeft->vNumMark; + } + + if (rightRight != NULL) { + oRightRight.node = rightRight; + oRightRight.oVNumMark = rightRight->vNumMark; + } + + int rightBalance = getHeight(rightLeft) - getHeight(rightRight); + + if (rightBalance > 0) { + if (rightLeft == NULL) { + continue; + } + + if (rotateRightLeft(tid, oParent, oNode, oRight, oRightLeft)) { + fixHeightAndRebalance(tid, node); + fixHeightAndRebalance(tid, right); + fixHeightAndRebalance(tid, rightLeft); + node = oParent.node; + } + } else { + if (rotateLeft(tid, oParent, oNode, oRight) == RetCode::SUCCESS) { + fixHeightAndRebalance(tid, node); + fixHeightAndRebalance(tid, right); + node = oParent.node; + } + } + } else { + // no rebalance occurred? check if the height is still ok + if ((propRes = fixHeight(tid, oNode)) == RetCode::FAILURE) { + continue; + } else if (propRes == RetCode::SUCCESS_WITH_HEIGHT_UPDATE) { + node = node->parent; + } else { + return; + } + } + } + return; +} + +template +int InternalKCAS::fixHeight( + const int tid, ObservedNode &oNode) { + + Node *node = oNode.node; + Node *left = node->left; + Node *right = node->right; + + casword_t leftOVNumMark; + casword_t rightOVNumMark; + + kcas::start(); + + if (left != NULL) { + leftOVNumMark = left->vNumMark; + kcas::add(&left->vNumMark, leftOVNumMark, leftOVNumMark); + } + + if (right != NULL) { + rightOVNumMark = right->vNumMark; + kcas::add(&right->vNumMark, rightOVNumMark, rightOVNumMark); + } + + int oldHeight = node->height; + + int newHeight = 1 + max(getHeight(left), getHeight(right)); + + // Check if rebalance is actually necessary + if (oldHeight == newHeight) { + if (node->vNumMark == oNode.oVNumMark && + (left == NULL || left->vNumMark == leftOVNumMark) && + (right == NULL || right->vNumMark == rightOVNumMark)) { + return RetCode::UNNECCESSARY; + } else { + return RetCode::FAILURE; + } + } + + kcas::add(&node->height, oldHeight, newHeight, &node->vNumMark, + oNode.oVNumMark, oNode.oVNumMark + 2); + + if (kcas::execute()) { + return RetCode::SUCCESS_WITH_HEIGHT_UPDATE; + } + + return RetCode::FAILURE; +} + +template +int InternalKCAS::rotateRight(const int tid, ObservedNode &oParent, + ObservedNode &oNode, + ObservedNode &oLeft) { + Node *parent = oParent.node; + Node *node = oNode.node; + Node *left = oLeft.node; + + kcas::start(); + + /***Pointers to Parents and Children***/ + // could fail fast here, should consider + if (parent->right == node) { + kcas::add(&parent->right, node, left); + } else if (parent->left == node) { + kcas::add(&parent->left, node, left); + } else { + return RetCode::FAILURE; + } + + Node *leftRight = left->right; + if (leftRight != NULL) { + casword_t leftRightOVNumMark = leftRight->vNumMark; + kcas::add(&leftRight->parent, left, node, &leftRight->vNumMark, + leftRightOVNumMark, leftRightOVNumMark + 2); + } + + Node *leftLeft = left->left; + if (leftLeft != NULL) { + casword_t leftLeftOVNumMark = leftLeft->vNumMark; + kcas::add(&leftLeft->vNumMark, leftLeftOVNumMark, leftLeftOVNumMark); + } + + Node *right = node->right; + if (right != NULL) { + casword_t rightOVNumMark = right->vNumMark; + kcas::add(&right->vNumMark, rightOVNumMark, rightOVNumMark); + } + + int oldNodeHeight = node->height; + int oldLeftHeight = left->height; + + int newNodeHeight = 1 + max(getHeight(leftRight), getHeight(right)); + int newLeftHeight = 1 + max(getHeight(leftLeft), newNodeHeight); + + kcas::add(&left->parent, node, parent, &node->left, left, leftRight, + &left->right, leftRight, node, &node->parent, parent, left, + &node->height, oldNodeHeight, newNodeHeight, &left->height, + oldLeftHeight, newLeftHeight, &parent->vNumMark, oParent.oVNumMark, + oParent.oVNumMark + 2, &node->vNumMark, oNode.oVNumMark, + oNode.oVNumMark + 2, &left->vNumMark, oLeft.oVNumMark, + oLeft.oVNumMark + 2); + + if (kcas::execute()) + return RetCode::SUCCESS; + return RetCode::FAILURE; +} + +template +int InternalKCAS::rotateLeft(const int tid, ObservedNode &oParent, + ObservedNode &oNode, + ObservedNode &oRight) { + Node *parent = oParent.node; + Node *node = oNode.node; + Node *right = oRight.node; + + kcas::start(); + + /***Pointers to Parents and Children***/ + // could fail fast here, should consider + if (parent->right == node) { + kcas::add(&parent->right, node, right); + } else if (parent->left == node) { + kcas::add(&parent->left, node, right); + } else { + return RetCode::FAILURE; + } + + Node *rightLeft = right->left; + if (rightLeft != NULL) { + casword_t rightLeftOVNumMark = rightLeft->vNumMark; + kcas::add(&rightLeft->parent, right, node, &rightLeft->vNumMark, + rightLeftOVNumMark, rightLeftOVNumMark + 2); + } + + Node *rightRight = right->right; + if (rightRight != NULL) { + casword_t rightRightOVNumMark = rightRight->vNumMark; + kcas::add(&rightRight->vNumMark, rightRightOVNumMark, rightRightOVNumMark); + } + + Node *left = node->left; + if (left != NULL) { + casword_t leftOVNumMark = left->vNumMark; + kcas::add(&left->vNumMark, leftOVNumMark, leftOVNumMark); + } + + int oldNodeHeight = node->height; + int oldRightHeight = right->height; + + int newNodeHeight = 1 + max(getHeight(left), getHeight(rightLeft)); + int newRightHeight = 1 + max(newNodeHeight, getHeight(rightRight)); + + kcas::add(&right->parent, node, parent, &node->right, right, rightLeft, + &right->left, rightLeft, node, &node->parent, parent, right, + &node->height, oldNodeHeight, newNodeHeight, &right->height, + oldRightHeight, newRightHeight, &parent->vNumMark, + oParent.oVNumMark, oParent.oVNumMark + 2, &node->vNumMark, + oNode.oVNumMark, oNode.oVNumMark + 2, &right->vNumMark, + oRight.oVNumMark, oRight.oVNumMark + 2); + + if (kcas::execute()) + return RetCode::SUCCESS; + return RetCode::FAILURE; +} + +template +int InternalKCAS::rotateLeftRight(const int tid, ObservedNode &oParent, + ObservedNode &oNode, + ObservedNode &oLeft, + ObservedNode &oLeftRight) { + Node *parent = oParent.node; + Node *node = oNode.node; + Node *left = oLeft.node; + Node *leftRight = oLeftRight.node; + + kcas::start(); + + /***Pointers to Parents and Children***/ + // could fail fast here, should consider + if (parent->right == node) { + kcas::add(&parent->right, node, leftRight); + } else if (parent->left == node) { + kcas::add(&parent->left, node, leftRight); + } else { + return RetCode::FAILURE; + } + + Node *leftRightLeft = leftRight->left; + if (leftRightLeft != NULL) { + casword_t leftRightLeftOVNumMark = leftRightLeft->vNumMark; + kcas::add(&leftRightLeft->parent, leftRight, left, &leftRightLeft->vNumMark, + leftRightLeftOVNumMark, leftRightLeftOVNumMark + 2); + } + + Node *leftRightRight = leftRight->right; + if (leftRightRight != NULL) { + casword_t leftRightRightOVNumMark = leftRightRight->vNumMark; + kcas::add(&leftRightRight->parent, leftRight, node, + &leftRightRight->vNumMark, leftRightRightOVNumMark, + leftRightRightOVNumMark + 2); + } + + Node *right = node->right; + if (right != NULL) { + casword_t rightOVNumMark = right->vNumMark; + kcas::add(&right->vNumMark, rightOVNumMark, rightOVNumMark); + } + + Node *leftLeft = left->left; + if (leftLeft != NULL) { + casword_t leftLeftOVNumMark = leftLeft->vNumMark; + kcas::add(&leftLeft->vNumMark, leftLeftOVNumMark, leftLeftOVNumMark); + } + + int oldNodeHeight = node->height; + int oldLeftHeight = left->height; + int oldLeftRightHeight = leftRight->height; + + int newNodeHeight = 1 + max(getHeight(leftRightRight), getHeight(right)); + int newLeftHeight = 1 + max(getHeight(leftLeft), getHeight(leftRightLeft)); + int newLeftRightHeight = 1 + max(newNodeHeight, newLeftHeight); + + kcas::add(&leftRight->parent, left, parent, &leftRight->left, leftRightLeft, + left, &left->parent, node, leftRight, &leftRight->right, + leftRightRight, node, &node->parent, parent, leftRight, + &left->right, leftRight, leftRightLeft, &node->left, left, + leftRightRight, &node->height, oldNodeHeight, newNodeHeight, + &left->height, oldLeftHeight, newLeftHeight, &leftRight->height, + oldLeftRightHeight, newLeftRightHeight, &leftRight->vNumMark, + oLeftRight.oVNumMark, oLeftRight.oVNumMark + 2, &parent->vNumMark, + oParent.oVNumMark, oParent.oVNumMark + 2, &node->vNumMark, + oNode.oVNumMark, oNode.oVNumMark + 2, &left->vNumMark, + oLeft.oVNumMark, oLeft.oVNumMark + 2); + + if (kcas::execute()) + return RetCode::SUCCESS; + return RetCode::FAILURE; +} + +template +int InternalKCAS::rotateRightLeft(const int tid, ObservedNode &oParent, + ObservedNode &oNode, + ObservedNode &oRight, + ObservedNode &oRightLeft) { + Node *parent = oParent.node; + Node *node = oNode.node; + Node *right = oRight.node; + Node *rightLeft = oRightLeft.node; + + kcas::start(); + + if (parent->right == node) { + kcas::add(&parent->right, node, rightLeft); + } else if (parent->left == node) { + kcas::add(&parent->left, node, rightLeft); + } else { + return RetCode::FAILURE; + } + + Node *rightLeftRight = rightLeft->right; + if (rightLeftRight != NULL) { + casword_t rightLeftRightOVNumMark = rightLeftRight->vNumMark; + + if (IS_MARKED(rightLeftRightOVNumMark)) + return RetCode::FAILURE; + + kcas::add(&rightLeftRight->parent, rightLeft, right, + &rightLeftRight->vNumMark, rightLeftRightOVNumMark, + rightLeftRightOVNumMark + 2); + } + + Node *rightLeftLeft = rightLeft->left; + if (rightLeftLeft != NULL) { + casword_t rightLeftLeftOVNumMark = rightLeftLeft->vNumMark; + + if (IS_MARKED(rightLeftLeftOVNumMark)) + return RetCode::FAILURE; + + kcas::add(&rightLeftLeft->parent, rightLeft, node, &rightLeftLeft->vNumMark, + rightLeftLeftOVNumMark, rightLeftLeftOVNumMark + 2); + } + + Node *left = node->left; + if (left != NULL) { + casword_t leftOVNumMark = left->vNumMark; + kcas::add(&left->vNumMark, leftOVNumMark, leftOVNumMark); + } + + Node *rightRight = right->right; + if (rightRight != NULL) { + casword_t rightRightOVNumMark = rightRight->vNumMark; + kcas::add(&rightRight->vNumMark, rightRightOVNumMark, rightRightOVNumMark); + } + + int oldNodeHeight = node->height; + int oldRightHeight = right->height; + int oldRightLeftHeight = rightLeft->height; + + int newNodeHeight = 1 + max(getHeight(rightLeftLeft), getHeight(left)); + int newRightHeight = + 1 + max(getHeight(rightRight), getHeight(rightLeftRight)); + int newRightLeftHeight = 1 + max(newNodeHeight, newRightHeight); + + kcas::add(&rightLeft->parent, right, parent, &rightLeft->right, + rightLeftRight, right, &right->parent, node, rightLeft, + &rightLeft->left, rightLeftLeft, node, &node->parent, parent, + rightLeft, &right->left, rightLeft, rightLeftRight, &node->right, + right, rightLeftLeft, &node->height, oldNodeHeight, newNodeHeight, + &right->height, oldRightHeight, newRightHeight, &rightLeft->height, + oldRightLeftHeight, newRightLeftHeight, &rightLeft->vNumMark, + oRightLeft.oVNumMark, oRightLeft.oVNumMark + 2, &parent->vNumMark, + oParent.oVNumMark, oParent.oVNumMark + 2, &node->vNumMark, + oNode.oVNumMark, oNode.oVNumMark + 2, &right->vNumMark, + oRight.oVNumMark, oRight.oVNumMark + 2); + + if (kcas::execute()) + return RetCode::SUCCESS; + return RetCode::FAILURE; +} + +template +inline int +InternalKCAS::countChildren( + const int tid, Node *node) { + return (node->left == NULL ? 0 : 1) + (node->right == NULL ? 0 : 1); +} + +template +long InternalKCAS::validateSubtree(Node *node, + long smaller, long larger, + std::unordered_set &keys, + ofstream &graph, ofstream &log, + bool &errorFound) { + + if (node == NULL) + return 0; + graph << "\"" << node << "\"" + << "[label=\"K: " << node->key << " - H: " << node->height << "\"];\n"; + + if (IS_MARKED(node->vNumMark)) { + log << "MARKED NODE! " << node->key << "\n"; + errorFound = true; + } + Node *nodeLeft = node->left; + Node *nodeRight = node->right; + + if (nodeLeft != NULL) { + graph << "\"" << node << "\" -> \"" << nodeLeft << "\""; + if (node->key < nodeLeft->key) { + assert(false); + graph << "[color=red]"; + } else { + graph << "[color=blue]"; + } + + graph << ";\n"; + } + + if (nodeRight != NULL) { + graph << "\"" << node << "\" -> \"" << nodeRight << "\""; + if (node->key > nodeRight->key) { + assert(false); + graph << "[color=red]"; + } else { + graph << "[color=green]"; + } + graph << ";\n"; + } + + Node *parent = node->parent; + graph << "\"" << node << "\" -> \"" << parent + << "\"" + "[color=grey];\n"; + // casword_t height = node->height; + + if (!(keys.count(node->key) == 0)) { + log << "DUPLICATE KEY! " << node->key << "\n"; + errorFound = true; + } + + if (!((nodeLeft == NULL || nodeLeft->parent == node) && + (nodeRight == NULL || nodeRight->parent == node))) { + log << "IMPROPER PARENT! " << node->key << "\n"; + errorFound = true; + } + + if ((node->key < smaller) || (node->key > larger)) { + log << "IMPROPER LOCAL TREE! " << node->key << "\n"; + errorFound = true; + } + + if (nodeLeft == NULL && nodeRight == NULL && getHeight(node) > 1) { + log << "Leaf with height > 1! " << node->key << "\n"; + errorFound = true; + } + + keys.insert(node->key); + + long lHeight = validateSubtree(node->left, smaller, node->key, keys, graph, + log, errorFound); + long rHeight = validateSubtree(node->right, node->key, larger, keys, graph, + log, errorFound); + + long ret = 1 + max(lHeight, rHeight); + + if (node->height != ret) { + log << "Node " << node->key << " with height " << ret + << " thinks it has height " << node->height << "\n"; + errorFound = true; + } + + if (abs(lHeight - rHeight) > 1) { + log << "Imbalanced Node! " << node->key << "(" << lHeight << ", " << rHeight + << ") - " << node->height << "\n"; + errorFound = true; + } + + return ret; +} + +template +bool InternalKCAS::validate() { + std::unordered_set keys = {}; + bool errorFound; + + rename("graph.dot", "graph_before.dot"); + ofstream graph; + graph.open("graph.dot"); + graph << "digraph G {\n"; + + ofstream log; + log.open("log.txt", std::ofstream::out | std::ofstream::app); + + auto t = std::time(nullptr); + auto tm = *std::localtime(&t); + log << "Run at: " << std::put_time(&tm, "%d-%m-%Y %H-%M-%S") << "\n"; + + // long ret = validateSubtree(root->left, minKey, maxKey, keys, graph, log, + // errorFound); + graph << "}"; + graph.close(); + + if (!errorFound) { + log << "Validated Successfully!\n"; + } + + log.close(); + + return !errorFound; +} + +template +void InternalKCAS::printDebuggingDetails() {} + +template +void InternalKCAS::freeSubtree(const int tid, + Node *node) { + if (node == NULL) + return; + freeSubtree(tid, node->left); + freeSubtree(tid, node->right); + // recmgr->reclaim(node); +} diff --git a/artifact/ds/baseline/int_bst_pathcas/internal_kcas_bst.h b/artifact/ds/baseline/int_bst_pathcas/internal_kcas_bst.h new file mode 100644 index 0000000..8ef7e20 --- /dev/null +++ b/artifact/ds/baseline/int_bst_pathcas/internal_kcas_bst.h @@ -0,0 +1,767 @@ +#pragma once + +#ifndef KCAS_TYPE +#define KCAS_HTM +#define KCAS_TYPE "KCAS_HTM" +#endif + +#include +#define MAX_KCAS 16 +#include "kcas.h" + +#include +#include +#include +#include +#include + +using namespace std; + +#define MAX_THREADS 200 +#define MAX_PATH_SIZE 16384 +#define PADDING_BYTES 128 +#define KCAS_MAX_K 16 + +#define IS_MARKED(word) (word & 0x1) + +template +struct Node : RecordManager::reclaimable_t { + casword key; + casword vNumMark; + casword *> left; + casword *> right; + casword value; +}; + +enum RetCode : int { + RETRY = 0, + UNNECCESSARY = 0, + FAILURE = -1, + SUCCESS = 1, + SUCCESS_WITH_HEIGHT_UPDATE = 2 +}; + +template +class InternalKCAS { +private: + /* + * ObservedNode acts as a Node-VersionNumber pair to track an "observed + * version number" of a given node. We can then be sure that a version number + * does not change after we have read it by comparing the current version + * number to this saved value NOTE: This is a thread-private structure, no + * fields need to be volatile + */ + struct ObservedNode { + ObservedNode() {} + + Node *node = NULL; + casword_t oVNumMark = -1; + }; + + struct PathContainer { + ObservedNode path[MAX_PATH_SIZE]; + volatile char padding[PADDING_BYTES]; + }; + + volatile char padding0[PADDING_BYTES]; + // Debugging, used to validate that no thread's parent can't be NULL, save for + // the root + bool init = false; + const int numThreads; + const int minKey; + const long long maxKey; + volatile char padding4[PADDING_BYTES]; + Node *root; + volatile char padding5[PADDING_BYTES]; + RecordManager *const recmgr; + volatile char padding7[PADDING_BYTES]; + PathContainer paths[MAX_THREADS]; + volatile char padding8[PADDING_BYTES]; + +public: + InternalKCAS(RecordManager *me, auto *cfg); + + ~InternalKCAS(); + + bool contains(const int tid, const K &key); + + V insertIfAbsent(const int tid, const K &key, const V &value); + + V erase(const int tid, const K &key); + + bool validate(); + + void printDebuggingDetails(); + + Node *getRoot(); + + void initThread(const int tid); + + void deinitThread(const int tid); + + RecordManager *const debugGetRecMgr() { return recmgr; } + // These are the functions that the benchmark will call + bool get(RecordManager *me, const K &key, V &val) { + return contains(me->tid, key); + } + + bool insert(RecordManager *me, const K &key, V &val) { + return 0 == insertIfAbsent(me->tid, key, val); + } + + bool remove(RecordManager *me, const K &key) { + return 0 != erase(me->tid, key); + } + +private: + Node *createNode(const int tid, K key, V value); + + void freeSubtree(const int tid, Node *node); + + long validateSubtree(Node *node, long smaller, + long larger, std::unordered_set &keys, + ofstream &graph, ofstream &log, bool &errorFound); + + int internalErase(const int tid, ObservedNode &parentObserved, + ObservedNode &nodeObserved, const K &key); + + int internalInsert(const int tid, ObservedNode &predecessorObserved, + ObservedNode &parentObserved, const K &key, + const V &value); + + int countChildren(const int tid, Node *node); + + int getSuccessor(const int tid, Node *node, + ObservedNode &succObserved, ObservedNode &oSuccParent, + const K &key); + + bool validatePath(const int tid, const int &size, const K &key, + ObservedNode *path); + + int search(const int tid, ObservedNode &predObserved, + ObservedNode &parentObserved, ObservedNode &nodeObserved, + const K &key); +}; + +template +Node * +InternalKCAS::createNode( + const int tid, K key, V value) { + Node *node = + new Node(); // recmgr->template allocate >(tid); + // No node, save for root, should have a NULL parent + node->key.setInitVal(key); + node->value.setInitVal(value); + node->vNumMark.setInitVal(0); + node->left.setInitVal(NULL); + node->right.setInitVal(NULL); + return node; +} + +template +InternalKCAS::InternalKCAS( + RecordManager *_recmgr, auto *cfg) + : numThreads(_numThreads), minKey(_minKey), maxKey(_maxKey), + recmgr(_recmgr) { + + // root = createNode(0, (maxKey + 1 & 0x00FFFFFFFFFFFFFF), NULL); + root = createNode(0, maxKey, 0); + init = true; +} + +template +InternalKCAS::~InternalKCAS() { + // auto guard = recmgr->getGuard(tid); + freeSubtree(0, root); + // delete recmgr; +} + +template +inline Node * +InternalKCAS::getRoot() { + return root->left; +} + +template +void InternalKCAS::initThread(const int tid) { + // recmgr->initThread(tid); +} + +template +void InternalKCAS::deinitThread(const int tid) { + // recmgr->deinitThread(tid); +} + +/* getSuccessor(const int tid, Node * node, ObservedNode &succObserved, int key) + * ### Gets the successor of a given node in it's subtree ### + * returns the successor of a given node stored within an ObservedNode with the + * observed version number. + * Returns an integer, 1 indicating the process was successful, 0 indicating a + * retry + */ +template +inline int +InternalKCAS::getSuccessor( + const int tid, Node *node, ObservedNode &oSucc, + ObservedNode &oSuccParent, const K &key) { + ObservedNode path[MAX_PATH_SIZE]; + + while (true) { + Node *succ = node->right; + path[0].node = node; + path[0].oVNumMark = node->vNumMark; + int currSize = 1; + + while (succ != NULL) { + assert(currSize < MAX_PATH_SIZE - 1); + path[currSize].node = succ; + path[currSize].oVNumMark = succ->vNumMark; + currSize++; + succ = succ->left; + } + + if (currSize < 2) { + return RetCode::RETRY; + } + + oSuccParent = path[currSize - 2]; + oSucc = path[currSize - 1]; + if (IS_MARKED(oSuccParent.oVNumMark) || IS_MARKED(oSucc.oVNumMark)) { + return RetCode::RETRY; + } else { + return RetCode::SUCCESS; + } + } +} + +template +inline bool +InternalKCAS::contains( + const int tid, const K &key) { + assert(key <= maxKey); + int result; + ObservedNode oNode; + ObservedNode oParent; + ObservedNode oPred; + + while ((result = search(tid, oPred, oParent, oNode, key)) == + RetCode::RETRY) { /* keep trying until we get a result */ + } + return result == RetCode::SUCCESS; +} + +/* search(const int tid, ObservedNode &predObserved, ObservedNode + * &parentObserved, ObservedNode &nodeObserved, const int &key) A proposed + * successor-predecessor pair is generated by searching for a given key, if the + * key is not found, the path is then validated to ensure it was not missed. + * Where appropriate, the predecessor (predObserved), parent (parentObserved) + * and node (nodeObserved) are provided to the caller. + */ +template +int InternalKCAS::search( + const int tid, ObservedNode &oPred, ObservedNode &oParent, + ObservedNode &oNode, const K &key) { + assert(key <= maxKey); + + K currKey; + casword_t nodeVNumMark; + + ObservedNode *path = paths[tid].path; + path[0].node = root; + path[0].oVNumMark = root->vNumMark; + + Node *node = root->left; + + ObservedNode *oPredPtr = NULL; + ObservedNode *oSuccPtr = &path[0]; + + int currSize = 1; + + while (true) { + assert(currSize < MAX_PATH_SIZE - 1); + // We have hit a terminal node without finding our key, must validate + if (node == NULL) { + if (oPredPtr != NULL) { + oPred = *oPredPtr; + + // The path could be valid, but we could be in the wrong sub-tree + if (key <= oPredPtr->node->key || key >= oSuccPtr->node->key) { + return RetCode::RETRY; + } + // The path could be valid, but we could be in the wrong sub-tree + } else if (key >= oSuccPtr->node->key) { + return RetCode::RETRY; + } + + if (validatePath(tid, currSize, key, path)) { + oParent = path[currSize - 1]; + return RetCode::FAILURE; + } else { + return RetCode::RETRY; + } + } + + nodeVNumMark = node->vNumMark; + + currKey = node->key; + + path[currSize].node = node; + path[currSize].oVNumMark = nodeVNumMark; + currSize++; + + if (key > currKey) { + node = node->right; + oPredPtr = &path[currSize - 1]; + } else if (key < currKey) { + node = node->left; + oSuccPtr = &path[currSize - 1]; + } + // no validation required on finding a key + else { + if (oPredPtr != NULL) { + oPred = *oPredPtr; + } + + oParent = path[currSize - 2]; + oNode = path[currSize - 1]; + return RetCode::SUCCESS; + } + } +} + +/* validatePath(const int tid, const int size, const int key, ObservedNode + * path[MAX_PATH_SIZE]) + * ### Validates all nodes in a path such that they are not marked and their + * version numbers have not changed ### validated a given path, ensuring that + * all version numbers of observed nodes still match the version numbers stored + * locally within nodes within the tree. This provides the caller with certainty + * that there was a time that this path existed in the tree Returns true for a + * valid path Returns false for an invalid path (some node version number + * changed) + */ +template +inline bool +InternalKCAS::validatePath( + const int tid, const int &size, const K &key, ObservedNode *path) { + assert(size > 0); + + for (int i = 0; i < size; i++) { + ObservedNode oNode = path[i]; + if (oNode.node->vNumMark != oNode.oVNumMark || IS_MARKED(oNode.oVNumMark)) { + return false; + } + } + return true; +} + +template +inline V InternalKCAS::insertIfAbsent(const int tid, const K &key, + const V &value) { + ObservedNode oParent; + ObservedNode oNode; + ObservedNode oPred; + // auto guard = recmgr->getGuard(tid); + + while (true) { + + int res; + while ((res = (search(tid, oPred, oParent, oNode, key))) == + RetCode::RETRY) { /* keep trying until we get a result */ + } + + if (res == RetCode::SUCCESS) { + return (V)oNode.node->value; + } + + assert(res == RetCode::FAILURE); + if (internalInsert(tid, oPred, oParent, key, value)) { + return 0; + } + } +} + +template +int InternalKCAS::internalInsert(const int tid, ObservedNode &oPred, + ObservedNode &oParent, const K &key, + const V &value) { + /* INSERT KCAS (K = 2-3) + * predecessor's version number*: vNumber -> vNumber + 1 + * parent's version number: vNumber -> vNumber + 1 + * parent's child pointer: NULL -> newNode + */ + + kcas::start(); + Node *parent = oParent.node; + + Node *pred = oPred.node; + + if (pred != NULL) { + if (pred->key == key) { + return RetCode::RETRY; + } else if (pred != parent) { + kcas::add(&pred->vNumMark, oPred.oVNumMark, oPred.oVNumMark); + } + } + + Node *newNode = createNode(tid, key, value); + + if (key > parent->key) { + kcas::add(&parent->right, (Node *)NULL, newNode); + } else if (key < parent->key) { + kcas::add(&parent->left, (Node *)NULL, newNode); + } else { + return RetCode::RETRY; + } + + kcas::add(&parent->vNumMark, oParent.oVNumMark, oParent.oVNumMark + 2); + + if (kcas::execute()) { + return RetCode::SUCCESS; + } + + // recmgr->reclaim(newNode); + + return RetCode::RETRY; +} + +template +inline V +InternalKCAS::erase( + const int tid, const K &key) { + ObservedNode oPred; + ObservedNode oParent; + ObservedNode oNode; + // auto guard = recmgr->getGuard(tid); + + while (true) { + int res = 0; + while ((res = (search(tid, oPred, oParent, oNode, key))) == + RetCode::RETRY) { /* keep trying until we get a result */ + } + + if (res == RetCode::FAILURE) { + return 0; + } + + assert(res == RetCode::SUCCESS); + if ((res = internalErase(tid, oParent, oNode, key))) { + return (V)oNode.node->value; + } + } +} + +template +int InternalKCAS::internalErase(const int tid, ObservedNode &oParent, + ObservedNode &oNode, const K &key) { + Node *parent = oParent.node; + Node *node = oNode.node; + + int numChildren = countChildren(tid, node); + + kcas::start(); + + if (IS_MARKED(oParent.oVNumMark) || IS_MARKED(oNode.oVNumMark)) { + return RetCode::RETRY; + } + + if (numChildren == 0) { + /* ERASE KCAS - NO CHILDREN (K = 4) + * node's mark: false -> true + * parent's child pointer: node -> NULL + * parent's version number: vNumber -> vNumber + 1 + * node's version number: vNumber -> vNumber + 1 + */ + + if (key > parent->key) { + kcas::add(&parent->right, node, (Node *)NULL); + } else if (key < parent->key) { + kcas::add(&parent->left, node, (Node *)NULL); + } else { + return RetCode::RETRY; + } + + kcas::add(&parent->vNumMark, oParent.oVNumMark, oParent.oVNumMark + 2, + &node->vNumMark, oNode.oVNumMark, oNode.oVNumMark + 3); + + if (kcas::execute()) { + assert(IS_MARKED(node->vNumMark)); + // recmgr->reclaim(node); + + return RetCode::SUCCESS; + } + + return RetCode::RETRY; + } else if (numChildren == 1) { + /* ERASE KCAS - 1 CHILD (K = 6) + * reroute child's version number: vNumber -> vNumber + 1 + * reroute child's parent: node -> parent + * node's mark: false -> true + * parent's child pointer: node -> reroute child + * parent's version number: vNumber -> vNumber + 1 + * node's version number: vNumber -> vNumber + 1 + */ + Node *left = node->left; + Node *right = node->right; + Node *reroute; + + // determine which child will be the replacement + if (left != NULL) { + reroute = left; + } else if (right != NULL) { + reroute = right; + } else { + return RetCode::RETRY; + } + + casword_t rerouteVNum = reroute->vNumMark; + + if (IS_MARKED(rerouteVNum)) { + return RetCode::RETRY; + } + + if (key > parent->key) { + kcas::add(&parent->right, node, reroute); + } else if (key < parent->key) { + kcas::add(&parent->left, node, reroute); + } else { + return RetCode::RETRY; + } + + kcas::add(&reroute->vNumMark, rerouteVNum, rerouteVNum + 2, &node->vNumMark, + oNode.oVNumMark, oNode.oVNumMark + 3, &parent->vNumMark, + oParent.oVNumMark, oParent.oVNumMark + 2); + + if (kcas::execute()) { + assert(IS_MARKED(node->vNumMark)); + // recmgr->reclaim(node); + + return RetCode::SUCCESS; + } + + return RetCode::RETRY; + } else if (numChildren == 2) { + /* ERASE KCAS - 2 CHILD (K = 6 or 8) + * successor's right child version number*: vNumber -> vNumber + 1 + * successor's right child parent*: node -> parent + * node's key key -> successor's key + * successor's parent's child pointer: successor -> successor's + * right child successor's mark: false -> true + * successor's version number: vNumber -> vNumber + 1 + * successor's parent's version number: vNumber -> vNumber + 1 + * node's version number: vNumber -> vNumber + 1 + * node's value: value -> succ's value + */ + + ObservedNode oSucc; + ObservedNode oSuccParent; + + // the (decendant) successor's key will be promoted + if (getSuccessor(tid, node, oSucc, oSuccParent, key) == RetCode::RETRY) { + return RetCode::RETRY; + } + + if (oSucc.node == NULL) { + return RetCode::RETRY; + } + + Node *succ = oSucc.node; + Node *succParent = oSuccParent.node; + + if (oSuccParent.node == NULL) { + return RetCode::RETRY; + } + + K succKey = succ->key; + + assert(succKey <= maxKey); + + if (IS_MARKED(oSuccParent.oVNumMark)) { + return RetCode::RETRY; + } + + Node *succRight = succ->right; + + if (succRight != NULL) { + casword_t succRightVNum = succRight->vNumMark; + + if (IS_MARKED(succRightVNum)) { + return RetCode::RETRY; + } + + kcas::add(&succRight->vNumMark, succRightVNum, succRightVNum + 2); + } + + if (succParent->right == succ) { + kcas::add(&succParent->right, succ, succRight); + } else if (succParent->left == succ) { + kcas::add(&succParent->left, succ, succRight); + } else { + return RetCode::RETRY; + } + + V nodeVal = node->value; + V succVal = succ->value; + + kcas::add(&node->value, nodeVal, succVal, &node->key, key, succKey, + &succ->vNumMark, oSucc.oVNumMark, oSucc.oVNumMark + 3, + &succParent->vNumMark, oSuccParent.oVNumMark, + oSuccParent.oVNumMark + 2); + + if (succParent != node) { + kcas::add(&node->vNumMark, oNode.oVNumMark, oNode.oVNumMark + 2); + } + + if (kcas::execute()) { + assert(IS_MARKED(succ->vNumMark)); + // recmgr->reclaim(succ); + return RetCode::SUCCESS; + } + + return RetCode::RETRY; + } + assert(false); + return RetCode::RETRY; +} + +template +inline int +InternalKCAS::countChildren( + const int tid, Node *node) { + return (node->left == NULL ? 0 : 1) + (node->right == NULL ? 0 : 1); +} + +template +long InternalKCAS::validateSubtree(Node *node, + long smaller, long larger, + std::unordered_set &keys, + ofstream &graph, ofstream &log, + bool &errorFound) { + + if (node == NULL) + return 0; + graph << "\"" << node << "\"" + << "[label=\"K: " << node->key << "\"];\n"; + + if (IS_MARKED(node->vNumMark)) { + log << "MARKED NODE! " << node->key << "\n"; + errorFound = true; + } + Node *nodeLeft = node->left; + Node *nodeRight = node->right; + + if (nodeLeft != NULL) { + graph << "\"" << node << "\" -> \"" << nodeLeft << "\""; + if (node->key < nodeLeft->key) { + assert(false); + graph << "[color=red]"; + } else { + graph << "[color=blue]"; + } + + graph << ";\n"; + } + + if (nodeRight != NULL) { + graph << "\"" << node << "\" -> \"" << nodeRight << "\""; + if (node->key > nodeRight->key) { + assert(false); + graph << "[color=red]"; + } else { + graph << "[color=green]"; + } + graph << ";\n"; + } + + if (!(keys.count(node->key) == 0)) { + log << "DUPLICATE KEY! " << node->key << "\n"; + errorFound = true; + } + + if ((node->key < smaller) || (node->key > larger)) { + log << "IMPROPER LOCAL TREE! " << node->key << "\n"; + errorFound = true; + } + + keys.insert(node->key); + long ret = 1 + max(validateSubtree(node->left, smaller, node->key, keys, + graph, log, errorFound), + validateSubtree(node->right, node->key, larger, keys, + graph, log, errorFound)); + + return ret; +} + +template +bool InternalKCAS::validate() { + std::unordered_set keys = {}; + bool errorFound = false; + + rename("graph.dot", "graph_before.dot"); + ofstream graph; + graph.open("graph.dot"); + graph << "digraph G {\n"; + + ofstream log; + log.open("log.txt", std::ofstream::out | std::ofstream::app); + + auto t = std::time(nullptr); + auto tm = *std::localtime(&t); + log << "Run at: " << std::put_time(&tm, "%d-%m-%Y %H-%M-%S") << "\n"; + + // long ret = validateSubtree(root->left, minKey, maxKey, keys, graph, log, + // errorFound); + graph << "}"; + graph.close(); + + if (!errorFound) { + log << "Validated Successfully!\n"; + } + + log.close(); + + return !errorFound; +} + +template +void InternalKCAS::printDebuggingDetails() {} + +template +void InternalKCAS::freeSubtree(const int tid, + Node *node) { + if (node == NULL) + return; + freeSubtree(tid, node->left); + freeSubtree(tid, node->right); + // recmgr->reclaim(tid,node); +} diff --git a/artifact/ds/baseline/int_bst_pathcas/kcas.h b/artifact/ds/baseline/int_bst_pathcas/kcas.h new file mode 100644 index 0000000..81d19d2 --- /dev/null +++ b/artifact/ds/baseline/int_bst_pathcas/kcas.h @@ -0,0 +1,104 @@ +#pragma once +#include // Include this to use uintptr_t +#define casword_t uintptr_t // Move the definition of casword_t here +#define CASWORD_BITS_TYPE casword_t + +#ifndef likely +#define likely(x) __builtin_expect((x), 1) +#endif +#ifndef unlikely +#define unlikely(x) __builtin_expect((x), 0) +#endif + +template struct casword { +public: + CASWORD_BITS_TYPE volatile bits; + +public: + casword(); + + T setInitVal(T other); + + operator T(); + + T operator->(); + + T getValue(); + + casword_t getValueUnsafe(bool &isPtr); + + void addToDescriptor(T oldVal, T newVal); +}; + +#if defined KCAS_LOCKFREE +#include "kcas_reuse_impl.h" +#elif defined KCAS_HTM +#include "kcas_reuse_htm_impl.h" +#elif defined KCAS_HTM_FULL +#include "kcas_reuse_htm_full_impl.h" +#elif defined KCAS_VALIDATE +#include "kcas_validate.h" +#elif defined KCAS_VALIDATE_HTM +#include "kcas_validate_htm.h" +#else +#error must define one of KCAS_LOCKFREE KCAS_HTM KCAS_HTM_FULL +#endif + +namespace kcas { +#if defined KCAS_LOCKFREE +KCASLockFree instance; +#elif defined KCAS_HTM +KCASHTM instance; +#elif defined KCAS_HTM_FULL +KCASHTM_FULL instance; +#elif defined KCAS_VALIDATE +KCASValidate instance; +#elif defined KCAS_VALIDATE_HTM +KCASValidateHTM instance; +#endif + +void writeInitPtr(casword_t volatile *addr, casword_t const newval) { + return instance.writeInitPtr(addr, newval); +} + +void writeInitVal(casword_t volatile *addr, casword_t const newval) { + return instance.writeInitVal(addr, newval); +} + +casword_t readPtr(casword_t volatile *addr) { return instance.readPtr(addr); } + +casword_t readVal(casword_t volatile *addr) { return instance.readVal(addr); } + +bool execute() { return instance.execute(); } + +inline kcasptr_t getDescriptor() { return instance.getDescriptor(); } + +bool start() { + instance.start(); + return true; +} + +template void add(casword *caswordptr, T oldVal, T newVal) { + return instance.add(caswordptr, oldVal, newVal); +} + +template +void add(casword *caswordptr, T oldVal, T newVal, Args... args) { + instance.add(caswordptr, oldVal, newVal, args...); +} + +#if defined KCAS_VALIDATE || defined KCAS_VALIDATE_HTM + +inline bool validate() { return instance.validate(); } + +inline bool validateAndExecute() { return instance.validateAndExecute(); } + +template inline casword_t visit(NodePtrType node) { + assert(node != NULL); + return instance.visit(node); +} +#endif + +}; // namespace kcas + +#include "casword.h" diff --git a/artifact/ds/baseline/int_bst_pathcas/kcas_reuse_htm_impl.h b/artifact/ds/baseline/int_bst_pathcas/kcas_reuse_htm_impl.h new file mode 100644 index 0000000..2b8bf10 --- /dev/null +++ b/artifact/ds/baseline/int_bst_pathcas/kcas_reuse_htm_impl.h @@ -0,0 +1,575 @@ +#pragma once + +#include +#include +#include +#include +#include + +using namespace std; + +/** + * Note: this algorithm supports a limited number of threads (print LAST_TID to + * see how many). It should be several thousand, at least. The alg can be + * tweaked to support more. + */ + +#define BOOL_CAS __sync_bool_compare_and_swap +#define VAL_CAS __sync_val_compare_and_swap + +/** + * + * Descriptor reuse macros + * + */ + +/** + * seqbits_t corresponds to the seqBits field of the descriptor. + * it contains the mutable fields of the descriptor and a sequence number. + * the width, offset and mask for the sequence number is defined below. + * this sequence number width, offset and mask are also shared by tagptr_t. + * + * in particular, for any tagptr_t x and seqbits_t y, the sequence numbers + * in x and y are equal iff x&MASK_SEQ == y&MASK_SEQ (despite differing types). + * + * tagptr_t consists of a triple . + * these three fields are defined by the TAGPTR_ macros below. + */ + +typedef intptr_t tagptr_t; +typedef intptr_t seqbits_t; +#include + +#ifndef WIDTH_SEQ +#define WIDTH_SEQ 48 +#endif +#define OFFSET_SEQ 14 +#define MASK_SEQ \ + ((uintptr_t)((1LL << WIDTH_SEQ) - 1) \ + << OFFSET_SEQ) /* cast to avoid signed bit shifting */ +#define UNPACK_SEQ(tagptrOrSeqbits) \ + (((uintptr_t)(tagptrOrSeqbits)) >> OFFSET_SEQ) + +#define TAGPTR_OFFSET_USER 0 +#define TAGPTR_OFFSET_TID 3 +#define TAGPTR_MASK_USER \ + ((1 << TAGPTR_OFFSET_TID) - 1) /* assumes TID is next field after USER */ +#define TAGPTR_MASK_TID (((1 << OFFSET_SEQ) - 1) & (~(TAGPTR_MASK_USER))) +#define TAGPTR_UNPACK_TID(tagptr) \ + ((int)((((tagptr_t)(tagptr)) & TAGPTR_MASK_TID) >> TAGPTR_OFFSET_TID)) +#define TAGPTR_UNPACK_PTR(descArray, tagptr) \ + (&(descArray)[TAGPTR_UNPACK_TID((tagptr))]) +#define TAGPTR_NEW(tid, seqBits, userBits) \ + ((tagptr_t)(((UNPACK_SEQ(seqBits)) << OFFSET_SEQ) | \ + ((tid) << TAGPTR_OFFSET_TID) | \ + (tagptr_t)(userBits) << TAGPTR_OFFSET_USER)) +// assert: there is no thread with tid DUMMY_TID that ever calls TAGPTR_NEW +#define LAST_TID (TAGPTR_MASK_TID >> TAGPTR_OFFSET_TID) +#define TAGPTR_STATIC_DESC(id) ((tagptr_t)TAGPTR_NEW(LAST_TID - 1 - id, 0)) +#define TAGPTR_DUMMY_DESC(id) ((tagptr_t)TAGPTR_NEW(LAST_TID, id << OFFSET_SEQ)) + +#define comma , + +#define SEQBITS_UNPACK_FIELD(seqBits, mask, offset) \ + ((((seqbits_t)(seqBits)) & (mask)) >> (offset)) +// TODO: make more efficient version "SEQBITS_CAS_BIT" +// TODO: change sequence # unpacking to masking for quick comparison +// note: if there is only one subfield besides seq#, then the third if-block is +// redundant, and you should just return false if the cas fails, since the only +// way the cas fails and the field being cas'd contains still old is if the +// sequence number has changed. +#define SEQBITS_CAS_FIELD(successBit, fldSeqBits, snapSeqBits, oldval, val, \ + mask, offset) \ + { \ + seqbits_t __v = (fldSeqBits); \ + while (1) { \ + if (UNPACK_SEQ(__v) != UNPACK_SEQ((snapSeqBits))) { \ + (successBit) = false; \ + break; \ + } \ + if (((successBit) = __sync_bool_compare_and_swap( \ + &(fldSeqBits), (__v & ~(mask)) | (oldval), \ + (__v & ~(mask)) | ((val) << (offset))))) { \ + break; \ + } \ + __v = (fldSeqBits); \ + if (SEQBITS_UNPACK_FIELD(__v, (mask), (offset)) != (oldval)) { \ + (successBit) = false; \ + break; \ + } \ + } \ + } +// TODO: change sequence # unpacking to masking for quick comparison +// note: SEQBITS_FAA_FIELD would be very similar to SEQBITS_CAS_FIELD; i think +// one would simply delete the last if block and change the new val from +// (val)<seqBits before (at least the reading \ + portion of) the memcpy */ \ + (UNPACK_SEQ(__src->seqBits) == UNPACK_SEQ((tagptr))); \ + }) +#define DESC_READ_FIELD(successBit, fldSeqBits, tagptr, mask, offset) \ + ({ \ + seqbits_t __seqBits = (fldSeqBits); \ + successBit = (__seqBits & MASK_SEQ) == ((tagptr)&MASK_SEQ); \ + SEQBITS_UNPACK_FIELD(__seqBits, (mask), (offset)); \ + }) +#define DESC_NEW(descArray, macro_seqBitsNew, tid) \ + &(descArray)[(tid)]; \ + { /* note: only the process invoking this following macro can change the \ + sequence# */ \ + seqbits_t __v = (descArray)[(tid)].seqBits; \ + (descArray)[(tid)].seqBits = macro_seqBitsNew(__v); \ + /*__sync_synchronize();*/ \ + } +#define DESC_INITIALIZED(descArray, tid) \ + (descArray)[(tid)].seqBits = (descArray)[(tid)].seqBits + (1 << OFFSET_SEQ); + +#define DESC_INIT_ALL(descArray, macro_seqBitsNew) \ + { \ + for (int i = 0; i < (LAST_TID - 1); ++i) { \ + (descArray)[i].seqBits = macro_seqBitsNew(0); \ + } \ + } + +/** + * + * KCAS implementation + * + */ + +#define kcastagptr_t uintptr_t +#define rdcsstagptr_t uintptr_t +#define rdcssptr_t rdcssdesc_t * +#define kcasptr_t kcasdesc_t * +#define RDCSS_TAGBIT 0x1 +#define KCAS_TAGBIT 0x2 + +#define KCAS_STATE_UNDECIDED 0 +#define KCAS_STATE_SUCCEEDED 4 +#define KCAS_STATE_FAILED 8 + +#define KCAS_LEFTSHIFT 2 +#define HTM_READ_DESCRIPTOR 20 +#define HTM_BAD_OLD_VAL 30 +#define MAX_RETRIES 5 + +#define KCAS_MAX_THREADS 500 + +void *volatile thread_ids[KCAS_MAX_THREADS] = {}; + +class TIDGenerator { +public: + int myslot = -1; + + TIDGenerator() { + int i; + while (true) { + i = 0; + while (thread_ids[i]) { + ++i; + } + + assert(i < KCAS_MAX_THREADS); + if (__sync_bool_compare_and_swap(&thread_ids[i], 0, this)) { + myslot = i; + break; + } + } + } + + ~TIDGenerator() { thread_ids[myslot] = 0; } + + operator int() { return myslot; } + + int getId() { return myslot; } + + void explicitRelease() { thread_ids[myslot] = 0; } +}; + +thread_local TIDGenerator kcas_tid; + +struct rdcssdesc_t { + volatile seqbits_t seqBits; + casword_t volatile *addr1; + casword_t old1; + casword_t volatile *addr2; + casword_t old2; + casword_t new2; + const static int size = sizeof(seqBits) + sizeof(addr1) + sizeof(old1) + + sizeof(addr2) + sizeof(old2) + sizeof(new2); + volatile char padding[128 + ((64 - size % 64) % + 64)]; // add padding to prevent false sharing +}; + +struct kcasentry_t { // just part of kcasdesc_t, not a standalone descriptor + casword_t volatile *addr; + casword_t oldval; + casword_t newval; +}; + +template class kcasdesc_t { +public: + volatile seqbits_t seqBits; + casword_t numEntries; + kcasentry_t entries[MAX_K]; + const static int size = + sizeof(seqBits) + sizeof(numEntries) + sizeof(entries); + volatile char padding[128 + ((64 - size % 64) % + 64)]; // add padding to prevent false sharing + + inline void addValAddr(casword_t volatile *addr, casword_t oldval, + casword_t newval) { + entries[numEntries].addr = addr; + entries[numEntries].oldval = oldval << KCAS_LEFTSHIFT; + entries[numEntries].newval = newval << KCAS_LEFTSHIFT; + ++numEntries; + assert(numEntries <= MAX_K); + } + + inline void addPtrAddr(casword_t volatile *addr, casword_t oldval, + casword_t newval) { + entries[numEntries].addr = addr; + entries[numEntries].oldval = oldval; + entries[numEntries].newval = newval; + ++numEntries; + assert(numEntries <= MAX_K); + } +}; + +inline static bool isRdcss(casword_t val) { return (val & RDCSS_TAGBIT); } + +inline static bool isKcas(casword_t val) { return (val & KCAS_TAGBIT); } + +template class KCASHTM { +public: + /** + * Data definitions + */ +private: + // descriptor reduction algorithm +#define KCAS_SEQBITS_OFFSET_STATE 0 +#define KCAS_SEQBITS_MASK_STATE 0xf +#define KCAS_SEQBITS_NEW(seqBits) \ + ((((seqBits)&MASK_SEQ) + (1 << OFFSET_SEQ)) | \ + (KCAS_STATE_UNDECIDED << KCAS_SEQBITS_OFFSET_STATE)) +#define RDCSS_SEQBITS_NEW(seqBits) (((seqBits)&MASK_SEQ) + (1 << OFFSET_SEQ)) + volatile char __padding_desc[128]; + kcasdesc_t kcasDescriptors[LAST_TID + 1] __attribute__((aligned(64))); + rdcssdesc_t rdcssDescriptors[LAST_TID + 1] __attribute__((aligned(64))); + volatile char __padding_desc3[128]; + + /** + * Function declarations + */ +public: + KCASHTM(); + void writeInitPtr(casword_t volatile *addr, casword_t const newval); + void writeInitVal(casword_t volatile *addr, casword_t const newval); + casword_t readPtr(casword_t volatile *addr); + casword_t readVal(casword_t volatile *addr); + bool execute(); + + kcasptr_t getDescriptor(); + bool start(); + casword_t rdcssRead(casword_t volatile *addr); + void helpOther(kcastagptr_t tagptr); + void deinitThread(); + template void add(casword *caswordptr, T oldVal, T newVal); + template + void add(casword *caswordptr, T oldVal, T newVal, Args... args); + +private: + casword_t rdcss(rdcssptr_t ptr, rdcsstagptr_t tagptr); + bool help(kcastagptr_t tagptr, kcasptr_t ptr, bool helpingOther); + void rdcssHelp(rdcsstagptr_t tagptr, rdcssptr_t snapshot, bool helpingOther); + void rdcssHelpOther(rdcsstagptr_t tagptr); +}; + +template +void KCASHTM::rdcssHelp(rdcsstagptr_t tagptr, rdcssptr_t snapshot, + bool helpingOther) { + bool readSuccess; + casword_t v = + DESC_READ_FIELD(readSuccess, *snapshot->addr1, snapshot->old1, + KCAS_SEQBITS_MASK_STATE, KCAS_SEQBITS_OFFSET_STATE); + if (!readSuccess) + v = KCAS_STATE_SUCCEEDED; // return; + + if (v == KCAS_STATE_UNDECIDED) { + BOOL_CAS(snapshot->addr2, (casword_t)tagptr, snapshot->new2); + } else { + // the "fuck it i'm done" action (the same action you'd take if the kcas + // descriptor hung around indefinitely) + BOOL_CAS(snapshot->addr2, (casword_t)tagptr, snapshot->old2); + } +} + +template void KCASHTM::rdcssHelpOther(rdcsstagptr_t tagptr) { + rdcssdesc_t newSnapshot; + const int sz = rdcssdesc_t::size; + if (DESC_SNAPSHOT(rdcssdesc_t, rdcssDescriptors, &newSnapshot, tagptr, sz)) { + rdcssHelp(tagptr, &newSnapshot, true); + } +} + +template +casword_t KCASHTM::rdcss(rdcssptr_t ptr, rdcsstagptr_t tagptr) { + casword_t r; + do { + r = VAL_CAS(ptr->addr2, ptr->old2, (casword_t)tagptr); + if (unlikely(isRdcss(r))) { + rdcssHelpOther((rdcsstagptr_t)r); + } else + break; + } while (true); + if (r == ptr->old2) + rdcssHelp(tagptr, ptr, false); // finish our own operation + return r; +} + +template +casword_t KCASHTM::rdcssRead(casword_t volatile *addr) { + casword_t r; + do { + r = *addr; + if (unlikely(isRdcss(r))) { + rdcssHelpOther((rdcsstagptr_t)r); + } else + break; + } while (true); + return r; +} + +template KCASHTM::KCASHTM() { + DESC_INIT_ALL(kcasDescriptors, KCAS_SEQBITS_NEW); + DESC_INIT_ALL(rdcssDescriptors, RDCSS_SEQBITS_NEW); +} + +template void KCASHTM::helpOther(kcastagptr_t tagptr) { + kcasdesc_t newSnapshot; + const int sz = kcasdesc_t::size; + // cout<<"size of kcas descriptor is "<)<<" and + // sz="<, kcasDescriptors, &newSnapshot, tagptr, + sz)) { + help(tagptr, &newSnapshot, true); + } +} + +template +bool KCASHTM::help(kcastagptr_t tagptr, kcasptr_t snapshot, + bool helpingOther) { + // phase 1: "locking" addresses for this kcas + int newstate; + + // read state field + kcasptr_t ptr = TAGPTR_UNPACK_PTR(kcasDescriptors, tagptr); + bool successBit; + int state = + DESC_READ_FIELD(successBit, ptr->seqBits, tagptr, KCAS_SEQBITS_MASK_STATE, + KCAS_SEQBITS_OFFSET_STATE); + if (!successBit) { + assert(helpingOther); + return false; + } + + if (state == KCAS_STATE_UNDECIDED) { + newstate = KCAS_STATE_SUCCEEDED; + for (int i = helpingOther; i < snapshot->numEntries; i++) { + retry_entry: + // prepare rdcss descriptor and run rdcss + rdcssdesc_t *rdcssptr = + DESC_NEW(rdcssDescriptors, RDCSS_SEQBITS_NEW, kcas_tid.getId()); + rdcssptr->addr1 = (casword_t *)&ptr->seqBits; + rdcssptr->old1 = tagptr; // pass the sequence number (as part of tagptr) + rdcssptr->old2 = snapshot->entries[i].oldval; + rdcssptr->addr2 = snapshot->entries[i].addr; // p stopped here (step 2) + rdcssptr->new2 = (casword_t)tagptr; + DESC_INITIALIZED(rdcssDescriptors, kcas_tid.getId()); + + casword_t val; + val = rdcss(rdcssptr, TAGPTR_NEW(kcas_tid.getId(), rdcssptr->seqBits, + RDCSS_TAGBIT)); + + // check for failure of rdcss and handle it + if (isKcas(val)) { + // if rdcss failed because of a /different/ kcas, we help it + if (val != (casword_t)tagptr) { + helpOther((kcastagptr_t)val); + goto retry_entry; + } + } else { + if (val != snapshot->entries[i].oldval) { + newstate = KCAS_STATE_FAILED; + break; + } + } + } + SEQBITS_CAS_FIELD(successBit, ptr->seqBits, snapshot->seqBits, + KCAS_STATE_UNDECIDED, newstate, KCAS_SEQBITS_MASK_STATE, + KCAS_SEQBITS_OFFSET_STATE); + } + // phase 2 (all addresses are now "locked" for this kcas) + state = DESC_READ_FIELD(successBit, ptr->seqBits, tagptr, + KCAS_SEQBITS_MASK_STATE, KCAS_SEQBITS_OFFSET_STATE); + if (!successBit) + return false; + + bool succeeded = (state == KCAS_STATE_SUCCEEDED); + + for (int i = 0; i < snapshot->numEntries; i++) { + casword_t newval = + succeeded ? snapshot->entries[i].newval : snapshot->entries[i].oldval; + BOOL_CAS(snapshot->entries[i].addr, (casword_t)tagptr, newval); + } + + return succeeded; +} + +// TODO: replace crappy bubblesort with something fast for large MAX_K (maybe +// even use insertion sort for small MAX_K) +template static void kcasdesc_sort(kcasptr_t ptr) { + kcasentry_t temp; + bool swapped = false; + for (int i = 0; i < ptr->numEntries; i++) { + for (int j = 0; j < ptr->numEntries - i - 1; j++) { + if (ptr->entries[j].addr > ptr->entries[j + 1].addr) { + temp = ptr->entries[j]; + ptr->entries[j] = ptr->entries[j + 1]; + ptr->entries[j + 1] = temp; + swapped = true; + } + } + if (!swapped) + break; + } +} + +template bool KCASHTM::execute() { + assert(kcas_tid.getId() != -1); + + auto desc = &kcasDescriptors[kcas_tid.getId()]; + // sort entries in the kcas descriptor to guarantee progress + + DESC_INITIALIZED(kcasDescriptors, kcas_tid.getId()); + kcastagptr_t tagptr = + TAGPTR_NEW(kcas_tid.getId(), desc->seqBits, KCAS_TAGBIT); + + for (int i = 0; i < MAX_RETRIES; i++) { + int status; + if ((status = _xbegin()) == _XBEGIN_STARTED) { + for (int j = 0; j < desc->numEntries; j++) { + casword_t val = *desc->entries[j].addr; + if (val != desc->entries[j].oldval) { + if (isKcas(val)) + _xabort(HTM_READ_DESCRIPTOR); + _xabort(HTM_BAD_OLD_VAL); + } + } + for (int j = 0; j < desc->numEntries; j++) { + *desc->entries[j].addr = desc->entries[j].newval; + } + _xend(); + return true; + } else { + if (_XABORT_EXPLICIT & status) { + if (_XABORT_CODE(status) == HTM_READ_DESCRIPTOR) { + break; + } else if (_XABORT_CODE(status) == HTM_BAD_OLD_VAL) { + return false; + } + } + } + } + + kcasdesc_sort(desc); + return help(tagptr, desc, false); +} + +template +inline casword_t KCASHTM::readPtr(casword_t volatile *addr) { + casword_t r; + do { + r = rdcssRead(addr); + if (unlikely(isKcas(r))) { + helpOther((kcastagptr_t)r); + } else + break; + } while (true); + return r; +} + +template +inline casword_t KCASHTM::readVal(casword_t volatile *addr) { + return ((casword_t)readPtr(addr)) >> KCAS_LEFTSHIFT; +} + +template +inline void KCASHTM::writeInitPtr(casword_t volatile *addr, + casword_t const newval) { + *addr = newval; +} + +template +inline void KCASHTM::writeInitVal(casword_t volatile *addr, + casword_t const newval) { + writeInitPtr(addr, newval << KCAS_LEFTSHIFT); +} + +template bool KCASHTM::start() { + // allocate a new kcas descriptor + kcasptr_t ptr = DESC_NEW(kcasDescriptors, KCAS_SEQBITS_NEW, kcas_tid.getId()); + ptr->numEntries = 0; + return true; +} + +template inline kcasptr_t KCASHTM::getDescriptor() { + return &kcasDescriptors[kcas_tid.getId()]; +} + +template void KCASHTM::deinitThread() { + kcas_tid.explicitRelease(); +} + +template +template +inline void KCASHTM::add(casword *caswordptr, T oldVal, T newVal) { + caswordptr->addToDescriptor(oldVal, newVal); +} + +template +template +void KCASHTM::add(casword *caswordptr, T oldVal, T newVal, + Args... args) { + caswordptr->addToDescriptor(oldVal, newVal); + add(args...); +} diff --git a/artifact/ds/baseline/lazylist_omap.h b/artifact/ds/baseline/lazylist_omap.h new file mode 100644 index 0000000..371b17c --- /dev/null +++ b/artifact/ds/baseline/lazylist_omap.h @@ -0,0 +1,158 @@ +#pragma once + +#include + +/// An ordered map, implemented as a singly-linked list, using the lazy list +/// algorithm. This map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param DESCRIPTOR A thread descriptor type, for safe memory reclamation +template +class lazylist_omap { + /// A simple test-and-test-and-set lock + /// + /// NB: The original code used a pthread_spinlock_t, which was causing + /// excessive latency. This implementation is more faster, because it + /// uses XCHG without a function call. + struct lock_t { + std::atomic lock_ = {false}; // An atomic bool to serve as the lock + + /// Acquire the lock + void acquire() { + while (true) { + if (!lock_.exchange(true)) + break; + while (lock_.load()) { + } + } + } + + /// Release the lock + void release() { lock_.store(false); } + }; + + struct node_t : DESCRIPTOR::reclaimable_t { + const K key; + V val; + std::atomic next; + lock_t lock; + node_t(K _key, V _val) : key(_key), val(_val), next(nullptr) {} + ~node_t() {} + }; + +public: + node_t *head; // A pointer to the list head sentinel + node_t *tail; // A pointer to the list tail sentinel + + /// Default construct a list by constructing and connecting two sentinel nodes + lazylist_omap(DESCRIPTOR *me, auto *cfg) { + head = new node_t(DUMMY_KEY, DUMMY_VAL); + tail = new node_t(DUMMY_KEY, DUMMY_VAL); + head->next = tail; + } + +private: + bool is_marked_ref(uintptr_t ptr) { return ptr & 1; } + + uintptr_t unset_mark(uintptr_t ptr) { return ptr & (UINTPTR_MAX - 1); } + + uintptr_t set_mark(uintptr_t ptr) { return ptr | 1; } + + inline node_t *get_unmarked_ref(node_t *ptr) { + return (node_t *)unset_mark((uintptr_t)ptr); + } + + inline node_t *get_marked_ref(node_t *ptr) { + return (node_t *)set_mark((uintptr_t)ptr); + } + + /* + * Checking that both curr and pred are both unmarked and that pred's next + * pointer points to curr to verify that the entries are adjacent and present + * in the list. + */ + inline int parse_validate(node_t *pred, node_t *curr) { + return (!is_marked_ref((uintptr_t)pred->next.load()) && + !is_marked_ref((uintptr_t)curr->next.load()) && + (pred->next == curr)); + } + +public: + bool get(DESCRIPTOR *me, const K &key, V &val) { + node_t *curr = head; + while (curr->key < key && curr != tail) + curr = get_unmarked_ref(curr->next); + V v = curr->val; + auto res = ((curr != head) && (curr->key == key) && + !is_marked_ref((uintptr_t)curr->next.load())); + if (res) + val = v; + return res; + } + + // a failed validate is counted as an abort + bool insert(DESCRIPTOR *me, const K &key, const V &val) { + node_t *curr, *pred; + int result, validated, notVal; + while (true) { + pred = head; + curr = get_unmarked_ref(pred->next); + while (curr->key < key && curr != tail) { + pred = curr; + curr = get_unmarked_ref(curr->next); + } + pred->lock.acquire(); + curr->lock.acquire(); + validated = parse_validate(pred, curr); + notVal = (curr->key != key || curr == tail); + result = (validated && notVal); + if (result) { + node_t *newnode = new node_t(key, val); + newnode->next = curr; + pred->next = newnode; + } + curr->lock.release(); + pred->lock.release(); + if (validated) + return result; + } + } + + /* + * Logically remove an element by setting a mark bit to 1 + * before removing it physically. + * + * NB. it is not safe to free the element after physical deletion as a + * pre-empted find operation may currently be parsing the element. + * TODO: must implement a stop-the-world garbage collector to correctly + * free the memory. + */ + // a failed validate is counted as an abort + bool remove(DESCRIPTOR *me, const K &key) { + node_t *pred, *curr; + int result, validated, isVal; + while (1) { + pred = head; + curr = get_unmarked_ref(pred->next); + while (curr->key < key && curr != tail) { + pred = curr; + curr = get_unmarked_ref(curr->next); + } + pred->lock.acquire(); + curr->lock.acquire(); + validated = parse_validate(pred, curr); + isVal = key == curr->key && curr != tail; + result = validated && isVal; + if (result) { + curr->next = get_marked_ref(curr->next); + pred->next = get_unmarked_ref(curr->next); + me->reclaim(curr); + } + curr->lock.release(); + pred->lock.release(); + if (validated) + return result; + } + } +}; diff --git a/artifact/ds/baseline/lfskiplist_omap.h b/artifact/ds/baseline/lfskiplist_omap.h new file mode 100644 index 0000000..862232a --- /dev/null +++ b/artifact/ds/baseline/lfskiplist_omap.h @@ -0,0 +1,561 @@ +/****************************************************************************** + * Skip lists, allowing concurrent update by use of CAS primitives. + * + * Copyright (c) 2001-2003, K A Fraser + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * A really simple random-number generator. Crappy linear congruential + * taken from glibc, but has at least a 2^32 period. + */ + +#define rand_next(_ptst) ((_ptst)->rand = ((_ptst)->rand * 1103515245) + 12345) + +struct ptst_t { + unsigned long rand; +}; + +extern pthread_key_t ptst_key; + +typedef unsigned long long tick_t; +#define RDTICK() \ + ({ \ + tick_t __t; \ + __asm__ __volatile__("rdtsc" : "=A"(__t)); \ + __t; \ + }) + +/* + * POINTER MARKING + */ + +#define get_marked_ref(_p) ((void *)(((unsigned long)(_p)) | 1)) +#define get_unmarked_ref(_p) ((void *)(((unsigned long)(_p)) & ~1)) +#define is_marked_ref(_p) (((unsigned long)(_p)) & 1) + +/* Number of unique block sizes we can deal with. */ +#define MAX_SIZES 20 + +typedef unsigned long setkey_t; +typedef void *setval_t; + +/************************************* + * INTERNAL DEFINITIONS + */ + +/* Fine for 2^NUM_LEVELS nodes. */ +#define NUM_LEVELS 20 + +/* Internal key values with special meanings. */ +#define SENTINEL_KEYMIN (1UL) /* Key value of first dummy node. */ +#define SENTINEL_KEYMAX (~0UL) /* Key value of last dummy node. */ + +/* + * Used internally by set access functions, so that callers can use + * key values 0 and 1, without knowing these have special meanings. + */ +#define CALLER_TO_INTERNAL_KEY(_k) ((_k) + 2) + +pthread_key_t ptst_key; + +/* + * Enter/leave a critical region. A thread gets a state handle for + * use during critical regions. + */ +void critical_exit(ptst_t *) {} + +ptst_t *critical_enter(void) { + ptst_t *ptst; + + ptst = (ptst_t *)pthread_getspecific(ptst_key); + if (ptst == NULL) { + + ptst = (ptst_t *)malloc(sizeof(*ptst)); + if (ptst == NULL) + exit(1); + memset(ptst, 0, sizeof(*ptst)); + // ptst->gc = gc_init(); + ptst->rand = RDTICK(); + + pthread_setspecific(ptst_key, ptst); + } + + // gc_enter(ptst); + return (ptst); +} + +static void ptst_destructor(ptst_t *ptst) {} + +void _init_ptst_subsystem(void) { + if (pthread_key_create(&ptst_key, (void (*)(void *))ptst_destructor)) { + exit(1); + } +} + +/* + * SKIP LIST + */ + +struct node_t { + std::atomic level; +#define LEVEL_MASK 0x0ff +#define READY_FOR_FREE 0x100 + setkey_t k; + std::atomic v; + std::atomic next[1]; +}; + +struct set_t { + node_t head; +}; + +/* + * PRIVATE FUNCTIONS + */ + +/* + * Random level generator. Drop-off rate is 0.5 per level. + * Returns value 1 <= level <= NUM_LEVELS. + */ +static int get_level(ptst_t *ptst) { + unsigned long r = rand_next(ptst); + int l = 1; + r = (r >> 4) & ((1 << (NUM_LEVELS - 1)) - 1); + while ((r & 1)) { + l++; + r >>= 1; + } + return (l); +} + +/* + * Allocate a new node, and initialise its @level field. + * NB. Initialisation will eventually be pushed into garbage collector, + * because of dependent read reordering. + */ +static node_t *alloc_node(ptst_t *ptst) { + int l; + node_t *n; + l = get_level(ptst); + // NB: Removed call to gc_alloc, because everything should use the same + // allocator + // n = (node_t *)gc_alloc(ptst, gc_id[l - 1]); + n = (node_t *)malloc(sizeof(*n) + l * sizeof(node_t *)); + n->level.store(l, std::memory_order_relaxed); + return (n); +} + +/* Free a node to the garbage collector. */ +static void free_node(ptst_t *ptst, node_t *n) { + // NB: For now, just leak it... + // gc_free(ptst, (void *)n, gc_id[(n->level & LEVEL_MASK) - 1]); +} + +/* + * Search for first non-deleted node, N, with key >= @k at each level in @l. + * RETURN VALUES: + * Array @pa: @pa[i] is non-deleted predecessor of N at level i + * Array @na: @na[i] is N itself, which should be pointed at by @pa[i] + * MAIN RETURN VALUE: same as @na[0]. + */ +static node_t *strong_search_predecessors(set_t *l, setkey_t k, node_t **pa, + node_t **na) { + node_t *x, *x_next, *old_x_next, *y, *y_next; + setkey_t y_k; + int i; + +retry: + // RMB(); // NB: This fence was unnecessary + + x = &l->head; + for (i = NUM_LEVELS - 1; i >= 0; i--) { + /* We start our search at previous level's unmarked predecessor. */ + x_next = x->next[i].load(); + /* If this pointer's marked, so is @pa[i+1]. May as well retry. */ + if (is_marked_ref(x_next)) + goto retry; + + for (y = x_next;; y = y_next) { + /* Shift over a sequence of marked nodes. */ + for (;;) { + y_next = y->next[i].load(); + if (!is_marked_ref(y_next)) + break; + y = (node_t *)get_unmarked_ref(y_next); + } + + y_k = y->k; + if (y_k >= k) + break; + + /* Update estimate of predecessor at this level. */ + x = y; + x_next = y_next; + } + + /* Swing forward pointer over any marked nodes. */ + if (x_next != y) { + old_x_next = x_next; + x->next[i].compare_exchange_strong(old_x_next, y); + if (old_x_next != x_next) + goto retry; + } + + if (pa) + pa[i] = x; + if (na) + na[i] = y; + } + + return (y); +} + +/* This function does not remove marked nodes. Use it optimistically. */ +node_t *weak_search_predecessors(set_t *l, setkey_t k, node_t **pa, + node_t **na) { + node_t *x, *x_next; + setkey_t x_next_k; + int i; + + x = &l->head; + for (i = NUM_LEVELS - 1; i >= 0; i--) { + for (;;) { + x_next = x->next[i].load(); + x_next = (node_t *)get_unmarked_ref(x_next); + + x_next_k = x_next->k; + if (x_next_k >= k) + break; + + x = x_next; + } + + if (pa) + pa[i] = x; + if (na) + na[i] = x_next; + } + + return (x_next); +} + +/* + * Mark @x deleted at every level in its list from @level down to level 1. + * When all forward pointers are marked, node is effectively deleted. + * Future searches will properly remove node by swinging predecessors' + * forward pointers. + */ +static void mark_deleted(node_t *x, int level) { + node_t *x_next; + + while (--level >= 0) { + x_next = x->next[level].load(); + while (!is_marked_ref(x_next)) { + x->next[level].compare_exchange_strong(x_next, + (node_t *)get_marked_ref(x_next)); + } + } +} + +static int check_for_full_delete(node_t *x) { + int level = x->level.load(); + return ((level & READY_FOR_FREE) || + !x->level.compare_exchange_strong(level, level | READY_FOR_FREE)); +} + +static void do_full_delete(ptst_t *ptst, set_t *l, node_t *x, int level) { + int k = x->k; + (void)strong_search_predecessors(l, k, NULL, NULL); + free_node(ptst, x); +} + +/* + * PUBLIC FUNCTIONS + */ + +set_t *set_alloc(void) { + set_t *l; + node_t *n; + int i; + + n = (node_t *)malloc(sizeof(*n) + (NUM_LEVELS - 1) * sizeof(node_t *)); + n->k = SENTINEL_KEYMAX; + + /* + * Set the forward pointers of final node to other than NULL, + * otherwise READ_FIELD() will continually execute costly barriers. + * Note use of 0xfe -- that doesn't look like a marked value! + */ + for (i = 0; i < NUM_LEVELS; i++) { + n->next[i].store((node_t *)0xfe); + } + + l = (set_t *)malloc(sizeof(*l) + (NUM_LEVELS - 1) * sizeof(node_t *)); + l->head.k = SENTINEL_KEYMIN; + l->head.level.store(NUM_LEVELS, std::memory_order_relaxed); + for (i = 0; i < NUM_LEVELS; i++) { + l->head.next[i].store(n); + } + + return (l); +} + +bool set_update(set_t *l, setkey_t k, setval_t &v, int overwrite) { + setval_t ov, new_ov; + ptst_t *ptst; + node_t *preds[NUM_LEVELS]; + node_t *succs[NUM_LEVELS]; + node_t *pred, *succ, *_new = NULL, *new_next, *old_next; + int i, level; + bool result = false; + + k = CALLER_TO_INTERNAL_KEY(k); + + ptst = critical_enter(); + + succ = weak_search_predecessors(l, k, preds, succs); + +retry: + ov = NULL; + result = false; + + if (succ->k == k) { + /* Already a @k node in the list: update its mapping. */ + new_ov = succ->v; + // NB: Removed overwrite ability, for compatibility with harness + // do { + if ((ov = new_ov) == NULL) { + /* Finish deleting the node, then retry. */ + level = succ->level; + mark_deleted(succ, level & LEVEL_MASK); + succ = strong_search_predecessors(l, k, preds, succs); + goto retry; + } + // } while (overwrite && ((new_ov = CASPO(&succ->v, ov, v)) != ov)); + + if (_new != NULL) + free_node(ptst, _new); + goto out; + } + + result = true; + + /* Not in the list, so initialise a new node for insertion. */ + if (_new == NULL) { + _new = alloc_node(ptst); + _new->k = k; + _new->v.store(v, std::memory_order_relaxed); + } + level = _new->level; + + /* If successors don't change, this saves us some CAS operations. */ + for (i = 0; i < level; i++) { + _new->next[i].store(succs[i], std::memory_order_relaxed); + } + + /* We've committed when we've inserted at level 1. */ + // WMB_NEAR_CAS(); /* make sure node fully initialised before inserting */ + old_next = succ; + preds[0]->next[0].compare_exchange_strong(old_next, _new); + if (old_next != succ) { + succ = strong_search_predecessors(l, k, preds, succs); + goto retry; + } + + /* Insert at each of the other levels in turn. */ + i = 1; + while (i < level) { + pred = preds[i]; + succ = succs[i]; + + /* Someone *can* delete @_new under our feet! */ + new_next = _new->next[i].load(); + if (is_marked_ref(new_next)) + goto success; + + /* Ensure forward pointer of new node is up to date. */ + if (new_next != succ) { + old_next = new_next; + _new->next[i].compare_exchange_strong(old_next, succ); + if (is_marked_ref(old_next)) + goto success; + assert(old_next == new_next); + } + + /* Ensure we have unique key values at every level. */ + if (succ->k == k) + goto new_world_view; + assert((pred->k < k) && (succ->k > k)); + + /* Replumb predecessor's forward pointer. */ + old_next = succ; + pred->next[i].compare_exchange_strong(old_next, _new); + if (old_next != succ) { + new_world_view: + // RMB(); /* get up-to-date view of the world. */ + (void)strong_search_predecessors(l, k, preds, succs); + continue; + } + + /* Succeeded at this level. */ + i++; + } + +success: + /* Ensure node is visible at all levels before punting deletion. */ + // WEAK_DEP_ORDER_WMB(); + if (check_for_full_delete(_new)) { + // MB(); /* make sure we see all marks in @new. */ + do_full_delete(ptst, l, _new, level - 1); + } +out: + critical_exit(ptst); + return (result); +} + +bool set_remove(set_t *l, setkey_t k, setval_t &v) { + setval_t new_v; + ptst_t *ptst; + node_t *preds[NUM_LEVELS], *x; + int level, i; + bool result = false; + + k = CALLER_TO_INTERNAL_KEY(k); + v = NULL; + + ptst = critical_enter(); + + x = weak_search_predecessors(l, k, preds, NULL); + + if (x->k > k) + goto out; + level = x->level; + level = level & LEVEL_MASK; + + /* Once we've marked the value field, the node is effectively deleted. */ + new_v = x->v; + for (;;) { + v = new_v; + if (v == NULL) + goto out; + + x->v.compare_exchange_strong(new_v, NULL); + if (new_v == v) + break; + } + + result = true; + + /* Committed to @x: mark lower-level forward pointers. */ + // WEAK_DEP_ORDER_WMB(); /* enforce above as linearisation point */ + mark_deleted(x, level); + + /* + * We must swing predecessors' pointers, or we can end up with + * an unbounded number of marked but not fully deleted nodes. + * Doing this creates a bound equal to number of threads in the system. + * Furthermore, we can't legitimately call 'free_node' until all shared + * references are gone. + */ + for (i = level - 1; i >= 0; i--) { + node_t *tmp = x; + preds[i]->next[i].compare_exchange_strong( + tmp, (node_t *)get_unmarked_ref(x->next[i].load())); + if (tmp != x) { + if ((i != (level - 1)) || check_for_full_delete(x)) { + // MB(); /* make sure we see node at all levels. */ + do_full_delete(ptst, l, x, i); + } + goto out; + } + } + + free_node(ptst, x); + +out: + critical_exit(ptst); + return (result); +} + +bool set_lookup(set_t *l, setkey_t k, setval_t &v) { + ptst_t *ptst; + node_t *x; + + k = CALLER_TO_INTERNAL_KEY(k); + + ptst = critical_enter(); + bool res = false; + + x = weak_search_predecessors(l, k, NULL, NULL); + if (x->k == k) { + v = x->v; + res = true; + } + critical_exit(ptst); + + return res; +} + +template class fraser_skiplist { + set_t *sl; + +public: + fraser_skiplist(DESCRIPTOR *me, auto *cfg) { + _init_ptst_subsystem(); + sl = set_alloc(); + } + + ~fraser_skiplist() {} + + bool get(DESCRIPTOR *me, const K &k, V &v) { return set_lookup(sl, k, v); } + + bool insert(DESCRIPTOR *me, const K &key, V &v) { + K k = key; + return set_update(sl, k, v, 0); + } + + bool remove(DESCRIPTOR *me, const K &k) { + V v; + return set_remove(sl, k, v); + } +}; diff --git a/artifact/ds/handSTM/dlist_carumap.h b/artifact/ds/handSTM/dlist_carumap.h new file mode 100644 index 0000000..280c49f --- /dev/null +++ b/artifact/ds/handSTM/dlist_carumap.h @@ -0,0 +1,499 @@ +#pragma once + +#include +#include +#include + +/// An unordered map, implemented as a resizable array of lists (closed +/// addressing, resizable). This map supports get(), insert() and remove() +/// operations. +/// +/// This implementation is based loosely on Liu's nonblocking resizable hash +/// table from PODC 2014. At the current time, we do not support the heuristic +/// for contracting the list, but we do support expanding the list. +/// +/// @tparam K The type of the keys stored in this map +/// @tparam V The type of the values stored in this map +/// @tparam HANDSTM The thread's descriptor type, for interacting with STM +template class dlist_carumap { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + /// A list node. It has prev and next pointers, but no key or value. It's + /// useful for sentinels, so that K and V don't have to be default + /// constructable. + struct node_t : ownable_t { + FIELD prev; // Pointer to predecessor + FIELD next; // Pointer to successor + + /// Construct a node + node_t() : ownable_t(), prev(nullptr), next(nullptr) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~node_t() {} + }; + + /// We need to know if buckets have been rehashed to a new table. We do this + /// by making the head of each bucket a `sentinel_t`, and adding a `closed` + /// bool. Note that the tail of each bucket's list is just a node_t. + struct sentinel_t : node_t { + /// Track if this sentinel is for a bucket that has been rehashed + FIELD closed; + + /// Construct a sentinel_t + sentinel_t() : node_t(), closed(false) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~sentinel_t() {} + }; + + /// A list node that also has a key and value + struct data_t : node_t { + const K key; // The key of this key/value pair + FIELD val; // The value of this key/value pair + + /// Construct a data_t + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(const K &_key, const V &_val) : node_t(), key(_key), val(_val) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~data_t() {} + }; + + /// An array of lists, along with its size + /// + /// NB: to avoid indirection, the array is inlined into the tbl_t. To make + /// this compatible with SMR, tbl_t must be ownable. + class tbl_t : public ownable_t { + using bucket_t = FIELD; + + /// Construct a table + /// + /// @param _size The desired size of the table + tbl_t(uint64_t _size) : size(_size) {} + + public: + const uint64_t size; // The size of the table + bucket_t tbl[]; // The buckets of the table + + /// Allocate a tbl_t of size `size` + /// + /// @param size The desired size + /// @param tx The calling operation's descriptor + /// + /// @return A table, all of whose buckets are set to null + static tbl_t *make(uint64_t size, WOSTM &tx) { + tbl_t *tbl = + tx.LOG_NEW((tbl_t *)malloc(sizeof(tbl_t) + size * sizeof(bucket_t))); + auto ret = new (tbl) tbl_t(size); + for (size_t i = 0; i < size; ++i) + ret->tbl[i].set(tx, ret, nullptr); + return ret; + } + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~tbl_t() {} + }; + + ownable_t *tbl_orec; // An orec for protecting `active` and `frozen` + FIELD active; // The active table + FIELD frozen; // The frozen table + std::hash _pre_hash; // A weak hash function for converting keys to ints + const uint64_t RESIZE_THRESHOLD; // Max bucket size before resizing + + /// Given a key, determine the bucket into which it should go. As in the Liu + /// hash, we do not change the hash function when we resize, we just change + /// the number of bits to use + /// + /// @param key The key to hash + /// @param size The size of the table into which this should be hashed + /// + /// @return An integer between 0 and size + uint64_t table_hash(HANDSTM *me, const K &key, const uint64_t size) const { + return me->hash(_pre_hash(key)) % size; + } + +public: + /// Default construct a map as having a valid active table. + /// + /// NB: Calls std::terminate if the provided size is not a power of 2. + /// + /// @param me The operation that is creating this umap + /// @param cfg A config object with `buckets` and `resize_threshold` + dlist_carumap(HANDSTM *me, auto *cfg) + : tbl_orec(new ownable_t()), RESIZE_THRESHOLD(cfg->resize_threshold) { + // Enforce power-of-2 initial size + if (std::popcount(cfg->buckets) != 1) + throw("cfg->buckets should be power of 2"); + + // Create an initial active table in which all of the buckets are + // initialized but empty (null <- head <-> tail -> null). + BEGIN_WO(me); + auto n = tbl_t::make(cfg->buckets, wo); + for (size_t i = 0; i < cfg->buckets; ++i) + n->tbl[i].set(wo, n, create_list(wo)); + // NB: since all buckets are initialized, nobody will ever go to the + // frozen table, so we can leave it as null + active.set(wo, tbl_orec, n); + frozen.set(wo, tbl_orec, nullptr); + } + +private: + /// Create a dlist with head and tail sentinels + /// + /// @param tx A writing TM context. Even though this code can't fail, we need + /// the context in order to use tm_field correctly. + /// + /// @return A pointer to the head sentinel of the list + sentinel_t *create_list(WOSTM &tx) { + // NB: By default, a node's prev and next will be nullptr, which is what we + // want for head->prev and tail->next. + auto head = tx.LOG_NEW(new sentinel_t()); + auto tail = tx.LOG_NEW(new node_t()); + head->next.set(tx, head, tail); + tail->prev.set(tx, tail, head); + return head; + } + + /// `resize()` is an internal method for changing the size of the active + /// table. Strictly speaking, it should be called `expand`, because for now we + /// only support expansion, not contraction. When `insert()` discovers that + /// it has made a bucket "too big", it will linearize its insertion, then call + /// resize(). + /// + /// At a high level, `resize()` is supposed to be open-nested and not to incur + /// any blocking, except due to orec conflicts. We accomplish this through + /// laziness and stealing. resize() finishes the /last/ resize, moves the + /// `active` table to `frozen`, and installs a new `active` table. Subsequent + /// operations will do most of the migrating. Note that resize() returns once + /// *anyone* resizes the table. + /// + /// @param me The calling thread's descriptor + /// @param a_tbl The active table, to resize + void resize(HANDSTM *me, tbl_t *a_tbl) { + tbl_t *ft = nullptr; // The frozen table + while (true) { + // If ft is null, then there's no frozen table, so just install a new + // active table and all is good. + { + BEGIN_WO(me); + // If someone else initiated a resize, then this attempt can end + // immediately + auto new_at = active.get(wo, tbl_orec); + if (new_at != a_tbl) + return; + + // If the frozen table is clean, just do a swap and we're done + ft = frozen.get(wo, tbl_orec); + if (ft == nullptr) { + // Make and initialize a table that is twice as big, move active to + // frozen, and make the new table active. + auto new_tbl = tbl_t::make(a_tbl->size * 2, wo); + frozen.set(wo, tbl_orec, a_tbl); + active.set(wo, tbl_orec, new_tbl); + return; + } + } + + // There is still an incomplete migration from frozen to active. Migrate + // everything out of frozen, remove the frozen table, and retry + prepare_resize(me, ft, a_tbl); + } + } + + /// Finish one lazy resize, so that another may begin. + /// + /// This really just boils down to migrating everything from `frozen` to + /// `active` and then nulling `frozen` and reclaiming it. + /// + /// NB: This code takes the "frozen" and "active" tables as arguments. + /// Consequently, we don't care about arbitrary delays. If a thread calls + /// this, rehashes half the table, and then suspends, another thread can + /// rehash everything else and install a new active table. When the first + /// thread wakes, it'll find a bunch of empty buckets, and it'll be safe. + /// + /// @param me The calling thread's descriptor + /// @param f_tbl The "frozen table", really the "source" table + /// @param a_tbl The "active table", really the "destination" table + void prepare_resize(HANDSTM *me, tbl_t *f_tbl, tbl_t *a_tbl) { + // NB: Right now, next_index == completed. If we randomized the start + // point, concurrent calls to prepare_resize() would contend less + uint64_t next_index = 0; // Next bucket to migrate + uint64_t completed = 0; // Number of buckets migrated + + // Migrate all data from `frozen` to `active` + while (completed != f_tbl->size) { + BEGIN_WO(me); + + // Try to rehash the next bucket. If it was already rehashed, there's a + // chance that the current resize phase is finished, so check + auto bkt = f_tbl->tbl[next_index].get(wo, f_tbl); + if (!rehash_expand_bucket(me, bkt, next_index, f_tbl->size, a_tbl, wo)) { + // NB: A "finished" concurrent resize will change `active`, but if + // another thread just got past this loop and uninstalled `frozen`, + // that's also cause for early return. + if (active.get(wo, tbl_orec) != a_tbl || !frozen.get(wo, tbl_orec)) + return; + } + + // Move to the next bucket + ++next_index; + ++completed; + } + + // Try to uninstall the `frozen` table, since it has been emptied. + { + BEGIN_WO(me); + if (frozen.get(wo, tbl_orec) != f_tbl) + return; + frozen.set(wo, tbl_orec, nullptr); + } + + // Reclaim `old`'s buckets, then `old` itself + // + // NB: This needs to be a transaction because of the API for reclamation, + // but there shouldn't be conflicts. + { + BEGIN_WO(me); + for (size_t i = 0; i < f_tbl->size; i++) { + // reclaim head and tail of each bucket + auto head = f_tbl->tbl[i].get(wo, f_tbl); + auto tail = head->next.get(wo, head); + wo.reclaim(head); + wo.reclaim(tail); + } + wo.reclaim(f_tbl); + } + } + + /// Get a pointer to the bucket in the active table that holds `key`. This + /// may cause some rehashing to happen. + /// + /// NB: The pattern here is unconventional. get_bucket() is the first step in + /// WSTEP transactions. If it doesn't rehash, then the caller WSTEP + /// continues its operation. If it does rehash, then the caller WSTEP + /// commits and restarts, which is a poor-man's open-nested transaction. + /// If it encounters an inconsistency, the caller WSTEP will be aborted. + /// + /// @param me The calling thread's descriptor + /// @param key The key whose bucket is sought + /// @param tx An active WSTEP transaction + /// + /// @return On success, a pointer to the head of a bucket, along with + /// `tbl_orec`'s value. nullptr on any rehash. + node_t *get_bucket(HANDSTM *me, const K &key, WOSTM &tx) { + // Get the head of the appropriate bucket in the active table + // + // NB: Validate or else a_tbl[a_idx] could be out of bounds + auto a_tbl = active.get(tx, tbl_orec); + auto a_idx = table_hash(me, key, a_tbl->size); + if (auto a_bucket = a_tbl->tbl[a_idx].get(tx, a_tbl)) + return a_bucket; // not null --> no resize needed + + // Find the bucket in the frozen table that needs rehashing + auto f_tbl = frozen.get(tx, tbl_orec); + auto f_idx = table_hash(me, key, f_tbl->size); + auto f_bucket = f_tbl->tbl[f_idx].get(tx, f_tbl); + + // Rehash it, tell caller to commit so the rehash appears to be open nested + // + // NB: if the rehash fails, it's due to someone else rehashing, which is OK + rehash_expand_bucket(me, f_bucket, f_idx, f_tbl->size, a_tbl, tx); + return nullptr; + } + + /// Re-hash one list in the frozen table into two lists in the active table + /// + /// @param me The calling thread's descriptor + /// @param f_list A pointer to an (acquired!) list head in the frozen table + /// @param f_idx The index of flist in the frozen table + /// @param f_size The size of the frozen table + /// @param a_tbl A reference to the active table + /// @param tx An active WSTEP transaction + /// + /// @return RESIZE_OK - The frozen bucket was rehashed into `a_tbl` + /// ALREADY_RESIZED - The frozen bucket was empty + bool rehash_expand_bucket(HANDSTM *me, sentinel_t *f_list, uint64_t f_idx, + uint64_t f_size, tbl_t *a_tbl, WOSTM &tx) { + // Stop if this bucket is already rehashed + if (f_list->closed.get(tx, f_list)) + return false; + + // Shuffle nodes from f_list into two new lists that will go into `a_tbl` + auto l1 = create_list(tx), l2 = create_list(tx); + auto curr = f_list->next.get(tx, f_list); + while (curr->next.get(tx, curr) != nullptr) { + auto next = curr->next.get(tx, curr); + auto data = static_cast(curr); + auto dest = table_hash(me, data->key, a_tbl->size) == f_idx ? l1 : l2; + auto succ = dest->next.get(tx, dest); + dest->next.set(tx, dest, data); + data->next.set(tx, data, succ); + data->prev.set(tx, data, dest); + succ->prev.set(tx, succ, data); + curr = next; + } + // curr is tail, set head->tail + f_list->next.set(tx, f_list, curr); + // put the lists into the active table, close the frozen bucket + a_tbl->tbl[f_idx].set(tx, a_tbl, l1); + a_tbl->tbl[f_idx + f_size].set(tx, a_tbl, l2); + f_list->closed.set(tx, f_list, true); + return true; + } + + /// Given the head sentinel of a list, search through the list to find the + /// node with key `key`, if such a node exists in the list. If it doesn't, + /// then return the head pointer, along with a count of non-sentinel nodes in + /// the list + /// + /// @param key The key for which we are searching + /// @param head The start of the list to search + /// @param tx An active WSTEP transaction + /// + /// @return {head, count} if the key was not found + /// {node, 0} if the key was found at `node` + std::pair list_get_or_head(const K &key, sentinel_t *head, + WOSTM &tx) { + // Get the head's successor; on any inconsistency, it'll abort + auto curr = head->next.get(tx, head); + uint64_t count = 0; // Number of nodes encountered during the loop + while (true) { + // if we reached the tail, return the head + if (curr->next.get(tx, curr) == nullptr) + return {head, count}; + + // return curr if it has a matching key + if (static_cast(curr)->key == key) + return {curr, 0}; + + // read `next` consistently + auto next = curr->next.get(tx, curr); + curr = next; + ++count; + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(HANDSTM *me, const K &key, V &val) { + while (true) { + BEGIN_WO(me); + // Get the bucket in `active` where `key` should be. Returns `nullptr` if + // it did some resizing, in which case we should commit the resize, then + // try again. + auto bucket = get_bucket(me, key, wo); + if (!bucket) + continue; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + auto [node, _] = + list_get_or_head(key, static_cast(bucket), wo); + + // If we got back the head, return false, otherwise read out the data + if (node == bucket) + return false; + data_t *dn = static_cast(node); + val = dn->val.get(wo, dn); + return true; + } + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(HANDSTM *me, const K &key, V &val) { + // If we discover that a bucket becomes too full, we'll insert, linearize, + // and then resize in a new transaction before returning. + tbl_t *a_tbl = nullptr; + while (true) { + BEGIN_WO(me); + auto bucket = get_bucket(me, key, wo); + if (!bucket) + continue; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + auto [node, count] = + list_get_or_head(key, static_cast(bucket), wo); + + // If we didn't get the head, the key already exists, so return false + if (node != bucket) + return false; + + auto next = node->next.get(wo, node); + + // Stitch in a new node + data_t *new_dn = wo.LOG_NEW(new data_t(key, val)); + new_dn->next.set(wo, new_dn, next); + new_dn->prev.set(wo, new_dn, node); + node->next.set(wo, node, new_dn); + next->prev.set(wo, next, new_dn); + if (count >= RESIZE_THRESHOLD) { + a_tbl = active.get(wo, tbl_orec); + break; // need to resize! + } + return true; + } + + resize(me, a_tbl); + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(HANDSTM *me, const K &key) { + while (true) { + BEGIN_WO(me); + // Get the bucket in `active` where `key` should be. Returns `nullptr` if + // it did some resizing, in which case we should commit the resize, then + // try again. + auto bucket = get_bucket(me, key, wo); + if (!bucket) + continue; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + // + // NB: This is a big transaction, so the active table can't have changed + auto [node, __] = + list_get_or_head(key, static_cast(bucket), wo); + + // If we got back the head, return false + if (node == bucket) + return false; + + // unstitch it + auto pred = node->prev.get(wo, node), succ = node->next.get(wo, node); + pred->next.set(wo, pred, succ); + succ->prev.set(wo, succ, pred); + wo.reclaim(node); + return true; + } + } +}; diff --git a/artifact/ds/handSTM/dlist_omap.h b/artifact/ds/handSTM/dlist_omap.h new file mode 100644 index 0000000..03cafe3 --- /dev/null +++ b/artifact/ds/handSTM/dlist_omap.h @@ -0,0 +1,172 @@ +#pragma once + +/// An ordered map, implemented as a doubly-linked list. This map supports +/// get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param HANDSTM A thread descriptor type, for safe memory reclamation +template class dlist_omap { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + /// A list node. It has prev and next pointers, but no key or value. It's + /// useful for sentinels, so that K and V don't have to be default + /// constructable. + struct node_t : ownable_t { + FIELD prev; // Pointer to predecessor + FIELD next; // Pointer to successor + + /// Construct a node + node_t() : prev(nullptr), next(nullptr) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~node_t() {} + }; + + /// A list node that also has a key and value. Note that keys are const. + struct data_t : public node_t { + const K key; // The key of this pair + FIELD val; // The value of this pair + + /// Construct a data_t + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(const K &_key, const V &_val) : node_t(), key(_key), val(_val) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~data_t() {} + }; + + node_t *const head; // The list head pointer + node_t *const tail; // The list tail pointer + +public: + /// Default construct a list by constructing and connecting two sentinel nodes + /// + /// @param me The operation that is constructing the list + /// @param cfg A configuration object + dlist_omap(HANDSTM *me, auto *cfg) : head(new node_t()), tail(new node_t()) { + BEGIN_WO(me); + head->next.set(wo, head, tail); + tail->prev.set(wo, tail, head); + } + +private: + /// get_leq is an inclusive predecessor query that returns the largest node + /// whose key is <= the provided key. It can return the head sentinel, but + /// not the tail sentinel. + /// + /// @param tx The active transaction + /// @param key The key for which we are doing a predecessor query. + /// + /// @return The node that was found + node_t *get_leq(STM &tx, const K key) { + // Start at the head; read the next now, to avoid reading it in multiple + // iterations of the loop + node_t *curr = head; + auto *next = curr->next.get(tx, curr); + + // Starting at `next`, search for key. + while (true) { + // Case 1: `next` is tail --> stop the search at curr + if (next == tail) + return curr; + + // read next's `next` and `key` + auto next_next = next->next.get(tx, next); + auto nkey = static_cast(next)->key; + + // Case 2: `next` is a data node: stop if next->key >= key + if (nkey > key) + return curr; + if (nkey == key) + return next; + + // Case 3: keep traversing to `next` + curr = next; + next = next_next; + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The thread context + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(HANDSTM *me, const K &key, V &val) { + BEGIN_RO(me); // RO tx(me); + // get_leq will use a read-only transaction to find the largest node with + // a key <= `key`. + auto n = get_leq(ro, key); + + // Since we have EBR, we can read n.key without validating and fast-fail + // on key-not-found + if (n == head || static_cast(n)->key != key) + return false; + + // NB: given EBR, we don't need to worry about n._obj being deleted, so + // we don't need to validate before looking at the value + data_t *dn = static_cast(n); + val = dn->val.get(ro, dn); + return true; + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The thread context + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(HANDSTM *me, const K &key, V &val) { + BEGIN_WO(me); // WO tx(me); + + auto n = get_leq(wo, key); + if (n != head && static_cast(n)->key == key) + return false; + + auto next = n->next.get(wo, n); + + // stitch in a new node + data_t *new_dn = wo.LOG_NEW(new data_t(key, val)); + new_dn->next.set(wo, new_dn, next); + new_dn->prev.set(wo, new_dn, n); + n->next.set(wo, n, new_dn); + next->prev.set(wo, next, new_dn); + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The thread context + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(HANDSTM *me, const K &key) { + BEGIN_WO(me); // WO tx(me); + + auto n = get_leq(wo, key); + if (n == head || static_cast(n)->key != key) + return false; + + // unstitch it + auto pred = n->prev.get(wo, n), succ = n->next.get(wo, n); + pred->next.set(wo, pred, succ); + succ->prev.set(wo, succ, pred); + wo.reclaim(n); + return true; + } +}; diff --git a/artifact/ds/handSTM/ibst_omap.h b/artifact/ds/handSTM/ibst_omap.h new file mode 100644 index 0000000..7a6f63a --- /dev/null +++ b/artifact/ds/handSTM/ibst_omap.h @@ -0,0 +1,267 @@ +#pragma once + +/// An ordered map, implemented as an unbalanced, internal binary search tree. +/// This map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param HANDSTM A thread HANDSTM type, for safe memory reclamation +template class ibst_omap { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + /// An easy-to-remember way of indicating the left and right children + enum DIRS { LEFT = 0, RIGHT = 1 }; + + /// node_t is the base type for all tree nodes. It doesn't have key/value + /// fields. + struct node_t : ownable_t { + /// The node's children. Be sure to use LEFT and RIGHT to index it + FIELD children[2]; + + /// Construct a node_t. This should only be called from a writer + /// transaction + /// + /// @param _left The left child of this node + /// @param _right The right child of this node + node_t(WOSTM &wo, node_t *_left = nullptr, node_t *_right = nullptr) { + children[LEFT].set(wo, this, _left); + children[RIGHT].set(wo, this, _right); + } + }; + + /// A pair holding a child node and its parent + struct ret_pair_t { + node_t *child; // The child + node_t *parent; // The parent of that child + }; + + /// Our tree uses a sentinel root node, so that we always have a valid node + /// for which to compute an orec. The sentinel's *LEFT* child is the true + /// root of the tree. That is, logically sentinel has the value "TOP". + node_t *sentinel; + + /// data_t is the type for all internal and leaf nodes in the data structure. + /// It extends the base type with a key and value. + /// + /// NB: keys are *not* const, because we want to overwrite nodes instead of + /// swapping them + struct data_t : public node_t { + FIELD key; // The key stored in this node + FIELD val; // The value stored in this node + + /// Construct a node + /// + /// @param _left left child of the node + /// @param _right right child of the node + /// @param _key the key of the node + /// @param _val the value of the node + data_t(WOSTM &wo, node_t *_left, node_t *_right, const K &_key, V &_val) + : node_t(wo, _left, _right) { + key.set(wo, this, _key); + val.set(wo, this, _val); + } + }; + +public: + /// Default construct an empty tree + /// + /// @param me The operation that is constructing the list + /// @param cfg A configuration object + ibst_omap(HANDSTM *me, auto *cfg) { + // NB: Even though the constructor is operating on private data, it needs a + // TM context in order to use tm_fields + BEGIN_WO(me); + sentinel = new node_t(wo); + } + +private: + /// Search for a `key` in the tree, and return the node holding it, as well + /// as the node's parent. If the key is not found, return null, and the + /// node that ought to be parent of the (not found) `key`. + /// + /// NB: The caller is responsible for clearing the checkpoint stack before + /// calling get_node(). + /// + /// @param key The key to search for + /// + /// @return {found, parent} if `key` is in the tree + /// {nullptr, parent} if `key` is not in the tree + ret_pair_t get_node(STM &tx, const K &key) { + // Traverse downward to the target node: + node_t *parent = sentinel; + node_t *child = parent->children[LEFT].get(tx, parent); + + // Traverse downward from the parent until we find null child or `key` + while (true) { + // nullptr == not found, so stop. We know parent was valid, so we can + // just return it + if (!child) + return {nullptr, parent}; + + // It's time to move downward. Read fields of child and grandchild + // + // NB: we may not use grandchild, but it's better to read it here + auto child_key = static_cast(child)->key.get(tx, child); + auto grandchild = + child->children[(key < child_key) ? LEFT : RIGHT].get(tx, child); + + // If the child key matches, return {child, parent}. We know both are + // valid (parent came from stack; we just checked child) + // + // NB: the snapshotting code requires that no node with matching key + // goes into `snapshots` + if (child_key == key) + return {child, parent}; + + // Otherwise traverse downward + parent = child; + child = grandchild; + } + } + + /// Given a node and its orec value, find the tree node that holds the key + /// that logically succeeds it (i.e., the leftmost descendent of the right + /// child) + /// + /// NB: The caller must ensure that `node` has a valid right child before + /// calling this method + /// + /// @param me The active CCDS operation + /// @param node An object and orec value to use as the starting point + /// + /// @return {{found, orec}, {parent, orec}} if no inconsistency occurs + /// {{nullptr, 0}, {nullptr, 0}} on any consistency violation + ret_pair_t get_succ_pair(STM &tx, node_t *node) { + // Read the right child + node_t *parent = node, *child = node->children[RIGHT].get(tx, node); + + // Find the leftmost non-null node in the tree rooted at child + while (true) { + auto next = child->children[LEFT].get(tx, child); + // If next is null, `child` is the successor. Otherwise keep traversing + if (!next) + return {child, parent}; + parent = child; + child = next; + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's HANDSTM + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(HANDSTM *me, const K &key, V &val) { + BEGIN_RO(me); + // Get the node that holds `key`, if it is present, and also its parent. + // If it isn't present, we'll get a null pointer. That corresponds to a + // consistent read of the parent, which means we already linearized and + // we're done + auto [curr, _] = get_node(ro, key); + if (curr == nullptr) + return false; + + // read the value + auto dn = static_cast(curr); + val = dn->val.get(ro, dn); + return true; + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's HANDSTM + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(HANDSTM *me, const K &key, V &val) { + BEGIN_WO(me); + auto [child, parent] = get_node(wo, key); + if (child) + return false; + // We must have a null child and a valid parent. If it's sentinel, we + // must insert as LEFT. Otherwise, compute which child to set. + auto cID = (parent == sentinel ? LEFT : RIGHT) & + (key > static_cast(parent)->key.get(wo, parent)); + auto new_child = new data_t(wo, nullptr, nullptr, key, val); + parent->children[cID].set(wo, parent, new_child); + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's HANDSTM + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(HANDSTM *me, const K &key) { + BEGIN_WO(me); + auto [target, parent] = get_node(wo, key); + if (target == nullptr) + return false; + + // Read the target node's children + data_t *t_child[2]; + t_child[RIGHT] = + static_cast(target->children[RIGHT].get(wo, target)); + t_child[LEFT] = + static_cast(target->children[LEFT].get(wo, target)); + + // If either child is null, and if the parent is still valid, then we can + // unstitch the target, link the parent to a grandchild and we're done. + if (!t_child[LEFT] || !t_child[RIGHT]) { + // Acquire the (possibly null) grandchild to link to the parent + auto gID = t_child[LEFT] ? LEFT : RIGHT; + + // Which child of the parent is target? + auto cID = + parent->children[LEFT].get(wo, parent) == target ? LEFT : RIGHT; + + // Unstitch and reclaim + parent->children[cID].set(wo, parent, t_child[gID]); + wo.reclaim(target); + return true; + } + + // `target` has two children. WLOG, the leftmost descendent of the right + // child is `target`'s successor, and must have at most one child. We + // want to put that node's key and value into `target`, and then remove + // that node by setting its parent's LEFT to its RIGHT (which might be + // null). + auto [succ, s_parent] = get_succ_pair(wo, target); + + // If target's successor is target's right child, then target._ver must + // equal s_parent._ver. As long as we lock target before we try + // to lock s_parent, we'll get the check for free. + + // Copy `succ`'s key/value into `target` + static_cast(target)->key.set( + wo, target, static_cast(succ)->key.get(wo, succ)); + static_cast(target)->val.set( + wo, target, static_cast(succ)->val.get(wo, succ)); + + // Unstitch `succ` by setting its parent's left to its right + // Case 1: there are intermediate nodes between target and successor + if (s_parent != target) + s_parent->children[LEFT].set(wo, s_parent, + succ->children[RIGHT].get(wo, succ)); + // Case 2: target is successor's parent + else + s_parent->children[RIGHT].set(wo, s_parent, + succ->children[RIGHT].get(wo, succ)); + wo.reclaim(succ); + return true; + } +}; diff --git a/artifact/ds/handSTM/iht_carumap.h b/artifact/ds/handSTM/iht_carumap.h new file mode 100644 index 0000000..fcab57c --- /dev/null +++ b/artifact/ds/handSTM/iht_carumap.h @@ -0,0 +1,286 @@ +#pragma once + +/// iht_umap is an HANDSTM implementation of the interlocked hash table. There +/// is one significant simplification: +/// +/// - It *does not* employ the max-depth trick for ensuring constant time +/// access. The worst-case asymptotic complexity is thus O(log(log(N))). +/// That's probably small enough that nobody will ever care. +/// +/// - Note that TM makes it very easy to use the trick where the type of a node +/// is embedded in the type, rather than in the pointer to the type. This is +/// more like the original IHT, less like our baseline version. +template class iht_carumap { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + /// Common parent for EList and PList types. It uses a bool as a proxy for + /// RTTI for distinguishing between PLists and ELists + /// + /// NB: In golang, the lock would go in Base. + struct Base : ownable_t { + const bool isEList; // Is this an EList (true) or a PList (false) + + // Construct the base type by setting its `isElist` field + Base(bool _isEList) : isEList(_isEList) {} + }; + + /// EList (ElementList) stores a bunch of K/V pairs + /// + /// NB: We construct with a factory, so the pairs can be a C-style variable + /// length array field. + struct EList : Base { + /// The key/value pair. We don't structure split, so that we can have the + /// array as a field. + struct pair_t { + FIELD key; // A key + FIELD val; // A value + }; + + FIELD count; // # live elements + pair_t pairs[]; // The K/V pairs stored in this EList + + private: + /// Force construction via the make_elist factory + EList() : Base(true), count(0) {} + + public: + /// Construct a EList that can hold up to `size` elements + static EList *make(WOSTM &wo, size_t size) { + EList *e = new (wo.LOG_NEW( + (ownable_t *)malloc(sizeof(EList) + size * sizeof(pair_t)))) EList(); + return e; + } + + /// Insert into an EList, without checking if there is enough room + void unchecked_insert(WOSTM &wo, const K &key, const V &val) { + auto c = count.get(wo, this); + pairs[c].key.set(wo, this, key); + pairs[c].val.set(wo, this, val); + count.set(wo, this, c + 1); + } + }; + + /// PList (PointerList) stores a bunch of pointers and their associated locks + /// + /// NB: We construct with a factory, so the pairs can be a C-style variable + /// length array field. This means that `depth` and `count` can't be + /// const, but that's OK. + struct PList : Base { + /// A wrapper around a pointer to a Base object + struct bucket_t { + FIELD base; // pointer to P/E List + }; + + bucket_t buckets[]; // The pointers stored in this PList + + private: + /// Force construction via the make_plist factory + PList() : Base(false) {} + + public: + /// Construct a PList at depth `depth` that can hold up to `size` elements + static PList *make(WOSTM &wo, size_t size) { + PList *p = new (wo.LOG_NEW((ownable_t *)malloc( + sizeof(PList) + size * sizeof(bucket_t)))) PList(); + for (size_t i = 0; i < size; ++i) + p->buckets[i].base.set(wo, p, nullptr); + return p; + } + }; + + const size_t elist_size; // The size of all ELists + const size_t plist_size; // The size of the root PList + PList *root; // The root PList + std::hash pre_hash; // A low-quality hash function from K to size_t + + /// For the time being, we re-hash a key at each level, xor-ing in the level + /// so that keys are unlikely to collide repeatedly. + /// + /// TODO: This shouldn't be too expensive, but we can probably do better. + uint64_t level_hash(HANDSTM *me, const K &key, size_t level) { + return me->hash(level ^ pre_hash(key)); + } + + /// Given a PList where the `index`th bucket is a full EList, create a new + /// PList that is twice the size of `parent` and hash the full EList's + /// elements into it. This only takes O(1) time. + /// + /// @param parent The PList whose bucket needs rehashing + /// @param pcount The number of elements in `parent` + /// @param pdepth The depth of `parent` + /// @param pidx The index in `parent` of the bucket to rehash + PList *rehash(HANDSTM *me, WOSTM &wo, PList *parent, size_t pcount, + size_t pdepth, size_t pidx) { + // Make a new PList that is twice as big, with all locks set to E_UNLOCKED + auto p = PList::make(wo, pcount * 2); + + // hash everything from the full EList into it + auto source = + static_cast(parent->buckets[pidx].base.get(wo, parent)); + auto c = source->count.get(wo, source); + for (size_t i = 0; i < c; ++i) { + auto k = source->pairs[i].key.get(wo, source); + auto b = level_hash(me, k, pdepth + 1) % pcount; + auto base = p->buckets[b].base.get(wo, p); + if (base == nullptr) { + base = EList::make(wo, elist_size); + p->buckets[b].base.set(wo, p, base); + } + EList *dest = static_cast(base); + dest->unchecked_insert(wo, k, source->pairs[i].val.get(wo, source)); + } + + // The caller locked the pointer to the EList, so we can reclaim the EList + wo.reclaim(source); + return p; + } + +public: + /// Construct an IHT by configuring the constants and building the root PList + /// + /// @param me Unused thread descriptor + /// @param cfg A configuration object with `chunksize` and `buckets` fields, + /// for setting the EList size and root PList size. + iht_carumap(HANDSTM *me, auto *cfg) + : elist_size(cfg->chunksize), plist_size(cfg->buckets) { + BEGIN_WO(me); + root = PList::make(wo, plist_size); + } + + /// Search for a key in the map. If found, return `true` and set the ref + /// parameter `val` to the associated value. Otherwise return `false`. + /// + /// @param me Thread context + /// @param key The key to search for + /// @param val The value (pass-by-ref) that was found + bool get(HANDSTM *me, const K &key, V &val) { + BEGIN_RO(me); + auto curr = root; // Start at the root PList + size_t depth = 1, count = plist_size; + while (true) { + auto bucket = level_hash(me, key, depth) % count; + // If it's null, fail + auto b = curr->buckets[bucket].base.get(ro, curr); + if (b == nullptr) + return false; + + // If it's a PList, keep traversing + if (!b->isEList) { + curr = static_cast(b); + ++depth; + count *= 2; + continue; + } + + // If it's not null, do a linear search of the keys + auto e = static_cast(b); + auto c = e->count.get(ro, e); + for (size_t i = 0; i < c; ++i) { + if (e->pairs[i].key.get(ro, e) == key) { + val = e->pairs[i].val.get(ro, e); + return true; + } + } + // Not found + return false; + } + } + + /// Search for a key in the map. If found, remove it and its associated value + /// and return `true`. Otherwise return `false`. + /// + /// @param me Thread context + /// @param key The key to search for + bool remove(HANDSTM *me, const K &key) { + BEGIN_WO(me); + auto curr = root; // Start at the root PList + size_t depth = 1, count = plist_size; + while (true) { + auto bucket = level_hash(me, key, depth) % count; + // If it's null, fail + auto b = curr->buckets[bucket].base.get(wo, curr); + if (b == nullptr) + return false; + + // If it's a PList, keep traversing + if (!b->isEList) { + curr = static_cast(b); + ++depth; + count *= 2; + continue; + } + + // If it's not null, do a linear search of the keys + auto e = static_cast(b); + auto c = e->count.get(wo, e); + for (size_t i = 0; i < c; ++i) { + if (e->pairs[i].key.get(wo, e) == key) { + // remove the K/V pair by overwriting, but only if there's >1 key + if (c > 1) { + e->pairs[i].key.set(wo, e, e->pairs[c - 1].key.get(wo, e)); + e->pairs[i].val.set(wo, e, e->pairs[c - 1].val.get(wo, e)); + } + e->count.set(wo, e, c - 1); + return true; + } + } + + // Not found + return false; + } + } + + /// Insert a new key/value pair into the map, but only if the key is not + /// already present. Return `true` if a mapping was added, `false` otherwise. + /// + /// @param me Thread context + /// @param key The key to try to insert + /// @param val The value to try to insert + bool insert(HANDSTM *me, const K key, const V val) { + BEGIN_WO(me); + auto curr = root; // Start at the root PList + size_t depth = 1, count = plist_size; + while (true) { + auto bucket = level_hash(me, key, depth) % count; + // If it's null, make a new EList, insert, and we're done + auto b = curr->buckets[bucket].base.get(wo, curr); + if (b == nullptr) { + auto e = EList::make(wo, elist_size); + e->unchecked_insert(wo, key, val); + curr->buckets[bucket].base.set(wo, curr, e); + return true; + } + + // If it's a PList, keep traversing + if (!b->isEList) { + curr = static_cast(b); + ++depth; + count *= 2; + continue; + } + + // If It's not null, do a linear search of the keys, return false if found + auto e = static_cast(b); + auto c = e->count.get(wo, e); + for (size_t i = 0; i < c; ++i) { + if (e->pairs[i].key.get(wo, e) == key) + return false; + } + + // Not found: insert if room + if (c < elist_size) { + e->unchecked_insert(wo, key, val); + return true; + } + + // Otherwise expand and keep traversing, because pathological hash + // collisions are always possible. + curr->buckets[bucket].base.set( + wo, curr, rehash(me, wo, curr, count, depth, bucket)); + } + } +}; diff --git a/artifact/ds/handSTM/rbtree_omap.h b/artifact/ds/handSTM/rbtree_omap.h new file mode 100644 index 0000000..835cd95 --- /dev/null +++ b/artifact/ds/handSTM/rbtree_omap.h @@ -0,0 +1,365 @@ +#pragma once + +/// An ordered map, implemented as a balanced, internal binary search tree. This +/// map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param HANDSTM A thread descriptor type, for safe memory reclamation +/// @param dummy_key A default key to use +/// @param dummy_val A default value to use +template +class rbtree_omap { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + static const int RED = 0; // Enum for red + static const int BLACK = 1; // Enum for black + + /// nodes in a red/black tree + struct node_t : ownable_t { + FIELD key; // Key stored at this node + FIELD val; // Value stored at this node + FIELD color; // color (RED or BLACK) + FIELD parent; // pointer to parent + FIELD ID; // 0/1 for left/right child + FIELD child[2]; // L/R children + + /// basic constructor + node_t(WOSTM &wo, int color, K key, V val, node_t *parent, long ID, + node_t *child0, node_t *child1) + : key(key), val(val), color(color), parent(parent), ID(ID) { + child[0].set(wo, this, child0); + child[1].set(wo, this, child1); + } + }; + + node_t *sentinel; // The (sentinel) root node of the tree + +public: + /// Construct a list by creating a sentinel node at the head + rbtree_omap(HANDSTM *me, auto *) { + BEGIN_WO(me); + sentinel = new node_t(wo, BLACK, dummy_key, dummy_val, nullptr, 0, nullptr, + nullptr); + } + + // binary search for the node that has v as its value + bool get(HANDSTM *me, const K &key, V &val) const { + BEGIN_RO(me); + node_t *curr = sentinel->child[0].get(ro, sentinel); + while (curr != nullptr && curr->key.get(ro, curr) != key) + curr = curr->child[(key < curr->key.get(ro, curr)) ? 0 : 1].get(ro, curr); + bool res = (curr != nullptr) && (curr->key.get(ro, curr) == key); + if (res) + val = curr->val.get(ro, curr); + return res; + } + + // insert a node with k/v as its pair if no such key exists in the tree + bool insert(HANDSTM *me, const K &key, V &val) { + bool res = false; + { + BEGIN_WO(me); + // find insertion point + node_t *curr = sentinel; + int cID = 0; + node_t *child = curr->child[cID].get(wo, curr); + while (child != nullptr) { + long ckey = child->key.get(wo, child); + if (ckey == key) + return false; + cID = key < ckey ? 0 : 1; + curr = child; + child = curr->child[cID].get(wo, curr); + } + + // make a red node and connect it to `curr` + res = true; + child = new node_t(wo, RED, key, val, curr, cID, nullptr, nullptr); + curr->child[cID].set(wo, curr, child); + + // balance the tree + while (true) { + // Get the parent, grandparent, and their relationship + node_t *parent = child->parent.get(wo, child); + int pID = parent->ID.get(wo, parent); + node_t *gparent = parent->parent.get(wo, parent); + + // Easy exit condition: no more propagation needed + if ((gparent == sentinel) || (BLACK == parent->color.get(wo, parent))) + break; + + // If parent's sibling is also red, we push red up to grandparent + node_t *psib = gparent->child[1 - pID].get(wo, gparent); + if ((psib != nullptr) && (RED == psib->color.get(wo, psib))) { + parent->color.set(wo, parent, BLACK); + psib->color.set(wo, psib, BLACK); + gparent->color.set(wo, gparent, RED); + child = gparent; + continue; // restart loop at gparent level + } + + int cID = child->ID.get(wo, child); + if (cID != pID) { + // set child's child to parent's cID'th child + node_t *baby = child->child[1 - cID].get(wo, child); + parent->child[cID].set(wo, parent, baby); + if (baby != nullptr) { + baby->parent.set(wo, baby, parent); + baby->ID.set(wo, baby, cID); + } + // move parent into baby's position as a child of child + child->child[1 - cID].set(wo, child, parent); + parent->parent.set(wo, parent, child); + parent->ID.set(wo, parent, 1 - cID); + // move child into parent's spot as pID'th child of gparent + gparent->child[pID].set(wo, gparent, child); + child->parent.set(wo, child, gparent); + child->ID.set(wo, child, pID); + // now swap child with curr and fall through + node_t *temp = child; + child = parent; + parent = temp; + } + + parent->color.set(wo, parent, BLACK); + gparent->color.set(wo, gparent, RED); + // promote parent + node_t *ggparent = gparent->parent.get(wo, gparent); + int gID = gparent->ID.get(wo, gparent); + node_t *ochild = parent->child[1 - pID].get(wo, parent); + // make gparent's pIDth child ochild + gparent->child[pID].set(wo, gparent, ochild); + if (ochild != nullptr) { + ochild->parent.set(wo, ochild, gparent); + ochild->ID.set(wo, ochild, pID); + } + // make gparent the 1-pID'th child of parent + parent->child[1 - pID].set(wo, parent, gparent); + gparent->parent.set(wo, gparent, parent); + gparent->ID.set(wo, gparent, 1 - pID); + // make parent the gIDth child of ggparent + ggparent->child[gID].set(wo, ggparent, parent); + parent->parent.set(wo, parent, ggparent); + parent->ID.set(wo, parent, gID); + } + + // now just set the root to black + node_t *root = sentinel->child[0].get(wo, sentinel); + if (root->color.get(wo, root) != BLACK) + root->color.set(wo, root, BLACK); + } + + return res; + } + + // remove the node with k as its key if it exists in the tree + bool remove(HANDSTM *me, const K &key) { + BEGIN_WO(me); + // find key + node_t *curr = sentinel->child[0].get(wo, sentinel); + + while (curr != nullptr) { + int ckey = curr->key.get(wo, curr); + if (ckey == key) + break; + curr = curr->child[key < ckey ? 0 : 1].get(wo, curr); + } + + // if we didn't find v, we're done + if (curr == nullptr) + return false; + + // If `curr` has two children, we need to swap it with its successor + if ((curr->child[1].get(wo, curr) != nullptr) && + ((curr->child[0].get(wo, curr)) != nullptr)) { + node_t *leftmost = curr->child[1].get(wo, curr); + while (leftmost->child[0].get(wo, leftmost) != nullptr) + leftmost = leftmost->child[0].get(wo, leftmost); + curr->key.set(wo, curr, leftmost->key.get(wo, leftmost)); + curr->val.set(wo, curr, leftmost->val.get(wo, leftmost)); + curr = leftmost; + } + + // extract x from the tree and prep it for deletion + node_t *parent = curr->parent.get(wo, curr); + node_t *child = + curr->child[(curr->child[0].get(wo, curr) != nullptr) ? 0 : 1].get( + wo, curr); + int xID = curr->ID.get(wo, curr); + parent->child[xID].set(wo, parent, child); + if (child != nullptr) { + child->parent.set(wo, child, parent); + child->ID.set(wo, child, xID); + } + + // fix black height violations + if ((BLACK == curr->color.get(wo, curr)) && (child != nullptr)) { + if (RED == child->color.get(wo, child)) { + curr->color.set(wo, curr, RED); + child->color.set(wo, child, BLACK); + } + } + + // rebalance... be sure to save the deletion target! + node_t *to_delete = curr; + while (true) { + parent = curr->parent.get(wo, curr); + if ((parent == sentinel) || (RED == curr->color.get(wo, curr))) + break; + int cID = curr->ID.get(wo, curr); + node_t *sibling = parent->child[1 - cID].get(wo, parent); + + // we'd like y's sibling s to be black + // if it's not, promote it and recolor + if (RED == sibling->color.get(wo, sibling)) { + /* + Bp Bs + / \ / \ + By Rs => Rp B2 + / \ / \ + B1 B2 By B1 + */ + parent->color.set(wo, parent, RED); + sibling->color.set(wo, sibling, BLACK); + // promote sibling + node_t *gparent = parent->parent.get(wo, parent); + int pID = parent->ID.get(wo, parent); + node_t *nephew = sibling->child[cID].get(wo, sibling); + // set nephew as 1-cID child of parent + parent->child[1 - cID].set(wo, parent, nephew); + nephew->parent.set(wo, nephew, parent); + nephew->ID.set(wo, nephew, 1 - cID); + // make parent the cID child of the sibling + sibling->child[cID].set(wo, sibling, parent); + parent->parent.set(wo, parent, sibling); + parent->ID.set(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].set(wo, gparent, sibling); + sibling->parent.set(wo, sibling, gparent); + sibling->ID.set(wo, sibling, pID); + // reset sibling + sibling = nephew; + } + + // Handle when the far nephew is red + node_t *n = sibling->child[1 - cID].get(wo, sibling); + if ((n != nullptr) && (RED == (n->color.get(wo, n)))) { + /* + ?p ?s + / \ / \ + By Bs => Bp Bn + / \ / \ + ?1 Rn By ?1 + */ + sibling->color.set(wo, sibling, parent->color.get(wo, parent)); + parent->color.set(wo, parent, BLACK); + n->color.set(wo, n, BLACK); + // promote sibling + node_t *gparent = parent->parent.get(wo, parent); + int pID = parent->ID.get(wo, parent); + node_t *nephew = sibling->child[cID].get(wo, sibling); + // make nephew the 1-cID child of parent + parent->child[1 - cID].set(wo, parent, nephew); + if (nephew != nullptr) { + nephew->parent.set(wo, nephew, parent); + nephew->ID.set(wo, nephew, 1 - cID); + } + // make parent the cID child of the sibling + sibling->child[cID].set(wo, sibling, parent); + parent->parent.set(wo, parent, sibling); + parent->ID.set(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].set(wo, gparent, sibling); + sibling->parent.set(wo, sibling, gparent); + sibling->ID.set(wo, sibling, pID); + break; // problem solved + } + + n = sibling->child[cID].get(wo, sibling); + if ((n != nullptr) && (RED == (n->color.get(wo, n)))) { + /* + ?p ?p + / \ / \ + By Bs => By Bn + / \ \ + Rn B1 Rs + \ + B1 + */ + sibling->color.set(wo, sibling, RED); + n->color.set(wo, n, BLACK); + // promote n + node_t *gneph = n->child[1 - cID].get(wo, n); + // make gneph the cID child of sibling + sibling->child[cID].set(wo, sibling, gneph); + if (gneph != nullptr) { + gneph->parent.set(wo, gneph, sibling); + gneph->ID.set(wo, gneph, cID); + } + // make sibling the 1-cID child of n + n->child[1 - cID].set(wo, n, sibling); + sibling->parent.set(wo, sibling, n); + sibling->ID.set(wo, sibling, 1 - cID); + // make n the 1-cID child of parent + parent->child[1 - cID].set(wo, parent, n); + n->parent.set(wo, n, parent); + n->ID.set(wo, n, 1 - cID); + // swap sibling and `n` + node_t *temp = sibling; + sibling = n; + n = temp; + + // now the far nephew is red... copy of code from above + sibling->color.set(wo, sibling, parent->color.get(wo, parent)); + parent->color.set(wo, parent, BLACK); + n->color.set(wo, n, BLACK); + // promote sibling + node_t *gparent = parent->parent.get(wo, parent); + int pID = parent->ID.get(wo, parent); + node_t *nephew = sibling->child[cID].get(wo, sibling); + // make nephew the 1-cID child of parent + parent->child[1 - cID].set(wo, parent, nephew); + if (nephew != nullptr) { + nephew->parent.set(wo, nephew, parent); + nephew->ID.set(wo, nephew, 1 - cID); + } + // make parent the cID child of the sibling + sibling->child[cID].set(wo, sibling, parent); + parent->parent.set(wo, parent, sibling); + parent->ID.set(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].set(wo, gparent, sibling); + sibling->parent.set(wo, sibling, gparent); + sibling->ID.set(wo, sibling, pID); + + break; // problem solved + } + + /* + ?p ?p + / \ / \ + Bx Bs => Bp Rs + / \ / \ + B1 B2 B1 B2 + */ + + sibling->color.set(wo, sibling, RED); // propagate upwards + + // advance to parent and balance again + curr = parent; + } + + // if curr was red, this fixes the balance + curr->color.set(wo, curr, BLACK); + + // free the node and return + wo.reclaim(to_delete); + + return true; + } +}; diff --git a/artifact/ds/handSTM/rbtree_omap_opt.h b/artifact/ds/handSTM/rbtree_omap_opt.h new file mode 100644 index 0000000..13d6633 --- /dev/null +++ b/artifact/ds/handSTM/rbtree_omap_opt.h @@ -0,0 +1,385 @@ +#pragma once + +/// An ordered map, implemented as a balanced, internal binary search tree. This +/// map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param HANDSTM A thread descriptor type, for safe memory reclamation +/// @param dummy_key A default key to use +/// @param dummy_val A default value to use +template +class rbtree_omap_opt { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + static const int RED = 0; // Enum for red + static const int BLACK = 1; // Enum for black + + /// nodes in a red/black tree + struct node_t : ownable_t { + FIELD key; // Key stored at this node + FIELD val; // Value stored at this node + FIELD color; // color (RED or BLACK) + FIELD parent; // pointer to parent + FIELD ID; // 0/1 for left/right child + FIELD child[2]; // L/R children + + /// basic constructor + node_t(WOSTM &wo, int color, K key, V val, node_t *parent, long ID, + node_t *child0, node_t *child1) + : key(key), val(val), color(color), parent(parent), ID(ID) { + child[0].set_cap(wo, this, child0); + child[1].set_cap(wo, this, child1); + } + }; + + node_t *sentinel; // The (sentinel) root node of the tree + +public: + /// Construct a list by creating a sentinel node at the head + rbtree_omap_opt(HANDSTM *me, auto *) { + BEGIN_WO(me); + sentinel = new node_t(wo, BLACK, dummy_key, dummy_val, nullptr, 0, nullptr, + nullptr); + } + + // binary search for the node that has v as its value + bool get(HANDSTM *me, const K &key, V &val) const { + BEGIN_RO(me); + node_t *curr = sentinel->child[0].get(ro, sentinel); + K k = key; + while (true) { + if (curr == nullptr) + break; + k = curr->key.get(ro, curr); + if (k == key) + break; + curr = curr->child[key < k ? 0 : 1].re_get(ro, curr); + } + bool res = (curr != nullptr) && (k == key); + if (res) + val = curr->val.re_get(ro, curr); + return res; + } + + // insert a node with k/v as its pair if no such key exists in the tree + bool insert(HANDSTM *me, const K &key, V &val) { + bool res = false; + { + BEGIN_WO(me); + // find insertion point + node_t *curr = sentinel; + int cID = 0; + node_t *child = curr->child[cID].get(wo, curr); + while (child != nullptr) { + auto ckey = child->key.get(wo, child); + if (ckey == key) + return false; + cID = key < ckey ? 0 : 1; + curr = child; + child = curr->child[cID].re_get(wo, curr); + } + + // make a red node and connect it to `curr` + res = true; + child = new node_t(wo, RED, key, val, curr, cID, nullptr, nullptr); + curr->child[cID].set(wo, curr, child); + + // balance the tree + while (true) { + // Get the parent, grandparent, and their relationship + // NB: child is captured or owned + node_t *parent = child->parent.get_mine(wo, child); + int pID = parent->ID.get(wo, parent); + auto parentcolor = parent->color.get_in_seq(wo, parent); + node_t *gparent = parent->parent.re_get(wo, parent); + + // Easy exit condition: no more propagation needed + if ((gparent == sentinel) || (BLACK == parentcolor)) + break; + + // If parent's sibling is also red, we push red up to grandparent + node_t *psib = gparent->child[1 - pID].get(wo, gparent); + if ((psib != nullptr) && (RED == psib->color.get(wo, psib))) { + parent->color.set(wo, parent, BLACK); + psib->color.set(wo, psib, BLACK); + gparent->color.set(wo, gparent, RED); + child = gparent; + continue; // restart loop at gparent level + } + + int cID = child->ID.get_mine(wo, child); + if (cID != pID) { + // set child's child to parent's cID'th child + node_t *baby = child->child[1 - cID].get_mine(wo, child); + parent->child[cID].set(wo, parent, baby); + if (baby != nullptr) { + baby->parent.set(wo, baby, parent); + baby->ID.set_mine(wo, baby, cID); + } + // move parent into baby's position as a child of child + child->child[1 - cID].set_mine(wo, child, parent); + parent->parent.set_mine(wo, parent, child); + parent->ID.set_mine(wo, parent, 1 - cID); + // move child into parent's spot as pID'th child of gparent + gparent->child[pID].set(wo, gparent, child); + child->parent.set_mine(wo, child, gparent); + child->ID.set_mine(wo, child, pID); + // now swap child with parent and fall through + node_t *temp = child; + child = parent; + parent = temp; + } + + parent->color.set(wo, parent, BLACK); + gparent->color.set(wo, gparent, RED); + // promote parent + node_t *ggparent = gparent->parent.get_mine(wo, gparent); + int gID = gparent->ID.get_mine(wo, gparent); + node_t *ochild = parent->child[1 - pID].get_mine(wo, parent); + // make gparent's pIDth child ochild + gparent->child[pID].set_mine(wo, gparent, ochild); + if (ochild != nullptr) { + ochild->parent.set(wo, ochild, gparent); + ochild->ID.set_mine(wo, ochild, pID); + } + // make gparent the 1-pID'th child of parent + parent->child[1 - pID].set_mine(wo, parent, gparent); + gparent->parent.set_mine(wo, gparent, parent); + gparent->ID.set_mine(wo, gparent, 1 - pID); + // make parent the gIDth child of ggparent + ggparent->child[gID].set(wo, ggparent, parent); + parent->parent.set_mine(wo, parent, ggparent); + parent->ID.set_mine(wo, parent, gID); + } + + // now just set the root to black + node_t *root = sentinel->child[0].get(wo, sentinel); + if (root->color.get(wo, root) != BLACK) + root->color.set(wo, root, BLACK); + } + + return res; + } + + // remove the node with k as its key if it exists in the tree + bool remove(HANDSTM *me, const K &key) { + BEGIN_WO(me); + // find key + node_t *curr = sentinel->child[0].get(wo, sentinel); + + while (curr != nullptr) { + auto ckey = curr->key.get(wo, curr); + if (ckey == key) + break; + curr = curr->child[key < ckey ? 0 : 1].re_get(wo, curr); + } + + // if we didn't find v, we're done + if (curr == nullptr) + return false; + + // If `curr` has two children, we need to swap it with its successor + if ((curr->child[1].get(wo, curr) != nullptr) && + ((curr->child[0].get(wo, curr)) != nullptr)) { + auto lchild = curr->child[0].get_in_seq(wo, curr); + auto rchild = curr->child[1].re_get(wo, curr); + if ((lchild != nullptr) && (rchild != nullptr)) { + node_t *leftmost = rchild; + while (true) { + auto next = leftmost->child[0].get(wo, leftmost); + if (next == nullptr) + break; + leftmost = next; + } + auto lk = leftmost->key.get_in_seq(wo, leftmost); + auto lv = leftmost->val.re_get(wo, leftmost); + curr->key.set(wo, curr, lk); + curr->val.set_mine(wo, curr, lv); + curr = leftmost; + } + } + + // extract x from the tree and prep it for deletion + node_t *parent = curr->parent.get(wo, curr); + node_t *lchild = curr->child[0].get_in_seq(wo, curr); + node_t *rchild = curr->child[1].get_in_seq(wo, curr); + node_t *child = (lchild != nullptr) ? lchild : rchild; + int xID = curr->ID.re_get(wo, curr); + parent->child[xID].set(wo, parent, child); + if (child != nullptr) { + child->parent.set(wo, child, parent); + child->ID.set_mine(wo, child, xID); + } + + // fix black height violations + if ((BLACK == curr->color.re_get(wo, curr)) && (child != nullptr)) { + if (RED == child->color.get_mine(wo, child)) { + curr->color.set(wo, curr, RED); + child->color.set_mine(wo, child, BLACK); + } + } + + // rebalance... be sure to save the deletion target! + node_t *to_delete = curr; + while (true) { + parent = curr->parent.get(wo, curr); + if ((parent == sentinel) || (RED == curr->color.re_get(wo, curr))) + break; + int cID = curr->ID.re_get(wo, curr); + node_t *sibling = parent->child[1 - cID].get(wo, parent); + + // we'd like y's sibling s to be black + // if it's not, promote it and recolor + if (RED == sibling->color.get(wo, sibling)) { + /* + Bp Bs + / \ / \ + By Rs => Rp B2 + / \ / \ + B1 B2 By B1 + */ + parent->color.set(wo, parent, RED); + sibling->color.set(wo, sibling, BLACK); + // promote sibling + node_t *gparent = parent->parent.get_mine(wo, parent); + int pID = parent->ID.get_mine(wo, parent); + node_t *nephew = sibling->child[cID].get_mine(wo, sibling); + // set nephew as 1-cID child of parent + parent->child[1 - cID].set_mine(wo, parent, nephew); + nephew->parent.set(wo, nephew, parent); + nephew->ID.set_mine(wo, nephew, 1 - cID); + // make parent the cID child of the sibling + sibling->child[cID].set_mine(wo, sibling, parent); + parent->parent.set_mine(wo, parent, sibling); + parent->ID.set_mine(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].set(wo, gparent, sibling); + sibling->parent.set_mine(wo, sibling, gparent); + sibling->ID.set_mine(wo, sibling, pID); + // reset sibling + sibling = nephew; + } + + // Handle when the far nephew is red + node_t *n = sibling->child[1 - cID].re_get(wo, sibling); + if ((n != nullptr) && (RED == (n->color.get(wo, n)))) { + /* + ?p ?s + / \ / \ + By Bs => Bp Bn + / \ / \ + ?1 Rn By ?1 + */ + sibling->color.set(wo, sibling, parent->color.re_get(wo, parent)); + parent->color.set(wo, parent, BLACK); + n->color.set(wo, n, BLACK); + // promote sibling + node_t *gparent = parent->parent.get_mine(wo, parent); + int pID = parent->ID.get_mine(wo, parent); + node_t *nephew = sibling->child[cID].get_mine(wo, sibling); + // make nephew the 1-cID child of parent + parent->child[1 - cID].set_mine(wo, parent, nephew); + if (nephew != nullptr) { + nephew->parent.set(wo, nephew, parent); + nephew->ID.set_mine(wo, nephew, 1 - cID); + } + // make parent the cID child of the sibling + sibling->child[cID].set_mine(wo, sibling, parent); + parent->parent.set_mine(wo, parent, sibling); + parent->ID.set_mine(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].set(wo, gparent, sibling); + sibling->parent.set_mine(wo, sibling, gparent); + sibling->ID.set_mine(wo, sibling, pID); + break; // problem solved + } + + n = sibling->child[cID].re_get(wo, sibling); + if ((n != nullptr) && (RED == (n->color.get(wo, n)))) { + /* + ?p ?p + / \ / \ + By Bs => By Bn + / \ \ + Rn B1 Rs + \ + B1 + */ + sibling->color.set(wo, sibling, RED); + n->color.set(wo, n, BLACK); + // promote n + node_t *gneph = n->child[1 - cID].get_mine(wo, n); + // make gneph the cID child of sibling + sibling->child[cID].set_mine(wo, sibling, gneph); + if (gneph != nullptr) { + gneph->parent.set(wo, gneph, sibling); + gneph->ID.set_mine(wo, gneph, cID); + } + // make sibling the 1-cID child of n + n->child[1 - cID].set_mine(wo, n, sibling); + sibling->parent.set_mine(wo, sibling, n); + sibling->ID.set_mine(wo, sibling, 1 - cID); + // make n the 1-cID child of parent + parent->child[1 - cID].set(wo, parent, n); + n->parent.set_mine(wo, n, parent); + n->ID.set_mine(wo, n, 1 - cID); + // swap sibling and `n` + node_t *temp = sibling; + sibling = n; + n = temp; + + // now the far nephew is red... copy of code from above + sibling->color.set_mine(wo, sibling, + parent->color.get_mine(wo, parent)); + parent->color.set_mine(wo, parent, BLACK); + n->color.set_mine(wo, n, BLACK); + // promote sibling + node_t *gparent = parent->parent.get_mine(wo, parent); + int pID = parent->ID.get_mine(wo, parent); + node_t *nephew = sibling->child[cID].get_mine(wo, sibling); + // make nephew the 1-cID child of parent + parent->child[1 - cID].set_mine(wo, parent, nephew); + if (nephew != nullptr) { + nephew->parent.set(wo, nephew, parent); + nephew->ID.set_mine(wo, nephew, 1 - cID); + } + // make parent the cID child of the sibling + sibling->child[cID].set_mine(wo, sibling, parent); + parent->parent.set_mine(wo, parent, sibling); + parent->ID.set_mine(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].set(wo, gparent, sibling); + sibling->parent.set_mine(wo, sibling, gparent); + sibling->ID.set_mine(wo, sibling, pID); + + break; // problem solved + } + + /* + ?p ?p + / \ / \ + Bx Bs => Bp Rs + / \ / \ + B1 B2 B1 B2 + */ + + sibling->color.set(wo, sibling, RED); // propagate upwards + + // advance to parent and balance again + curr = parent; + } + + // if curr was red, this fixes the balance + curr->color.set(wo, curr, BLACK); + + // free the node and return + wo.reclaim(to_delete); + + return true; + } +}; diff --git a/artifact/ds/handSTM/rbtree_tl2_omap.h b/artifact/ds/handSTM/rbtree_tl2_omap.h new file mode 100644 index 0000000..b95d4cc --- /dev/null +++ b/artifact/ds/handSTM/rbtree_tl2_omap.h @@ -0,0 +1,437 @@ +/* ============================================================================= + * + * rbtree.h + * -- Red-black balanced binary search tree + * + * ============================================================================= + * + * Copyright (C) Sun Microsystems Inc., 2006. All Rights Reserved. + * Authors: Dave Dice, Nir Shavit, Ori Shalev. + * + * STM: Transactional Locking for Disjoint Access Parallelism + * + * Transactional Locking II, + * Dave Dice, Ori Shalev, Nir Shavit + * DISC 2006, Sept 2006, Stockholm, Sweden. + */ + +#pragma once + +// NB: The tl2 rbtree does not have a sentinel that points to the root. +// Consequently, we will make this an ownable_t, so it can have an orec to +// protect the root pointer. +template +class rbtree_tl2_omap { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + static const int RED = 0; // Enum for red + static const int BLACK = 1; // Enum for black + + struct node_t : ownable_t { + FIELD key; + FIELD val; + FIELD p; + FIELD l; + FIELD r; + FIELD c; + char dummy[64]; + + node_t(WOSTM &wo, const K &_key, V &_val) { + key.set(wo, this, _key); + val.set(wo, this, _val); + p.set(wo, this, nullptr); + l.set(wo, this, nullptr); + r.set(wo, this, nullptr); + c.set(wo, this, RED); + } + }; + + ownable_t root_orec; + FIELD root; + + char dummy[64]; + + node_t *lookup(STM &tx, K k) { + node_t *p = this->root.get(tx, &root_orec); + while (p != nullptr) { + if (k == p->key.get(tx, p)) { + return p; + } + p = (k < p->key.get(tx, p)) ? leftOf(tx, p) : rightOf(tx, p); + } + return nullptr; + } + + void rotateLeft(WOSTM &wo, node_t *x) { + node_t *r = rightOf(wo, x); + node_t *rl = leftOf(wo, r); + setRight(wo, x, rl); + if (rl != nullptr) { + setParent(wo, rl, x); + } + + node_t *xp = parentOf(wo, x); + setParent(wo, r, xp); + if (xp == nullptr) { + this->root.set(wo, &root_orec, r); + } else if (leftOf(wo, xp) == x) { + setLeft(wo, xp, r); + } else { + setRight(wo, xp, r); + } + setLeft(wo, r, x); + setParent(wo, x, r); + } + + void rotateRight(WOSTM &wo, node_t *x) { + node_t *l = leftOf(wo, x); + node_t *lr = rightOf(wo, l); + setLeft(wo, x, lr); + if (lr != nullptr) { + setParent(wo, lr, x); + } + node_t *xp = parentOf(wo, x); + setParent(wo, l, xp); + if (xp == nullptr) { + this->root.set(wo, &root_orec, l); + } else if (rightOf(wo, xp) == x) { + setRight(wo, xp, l); + } else { + setLeft(wo, xp, l); + } + setRight(wo, l, x); + setParent(wo, x, l); + } + + node_t *parentOf(WOSTM &wo, node_t *n) { + return (n ? n->p.get(wo, n) : nullptr); + } + + node_t *leftOf(STM &tx, node_t *n) { return (n ? n->l.get(tx, n) : nullptr); } + + node_t *rightOf(STM &tx, node_t *n) { + return (n ? n->r.get(tx, n) : nullptr); + } + + int colorOf(WOSTM &wo, node_t *n) { return (n ? n->c.get(wo, n) : BLACK); } + + void setColor(WOSTM &wo, node_t *n, int c) { + if (n != nullptr) { + n->c.set(wo, n, c); + } + } + + void setParent(WOSTM &wo, node_t *n, node_t *p) { + if (n != nullptr) { + n->p.set(wo, n, p); + } + } + + void setLeft(WOSTM &wo, node_t *n, node_t *l) { + if (n != nullptr) { + n->l.set(wo, n, l); + } + } + + void setRight(WOSTM &wo, node_t *n, node_t *r) { + if (n != nullptr) { + n->r.set(wo, n, r); + } + } + + void fixAfterInsertion(WOSTM &wo, node_t *x) { + setColor(wo, x, RED); + while (x != nullptr && x != this->root.get(wo, &root_orec)) { + node_t *xp = parentOf(wo, x); + if (colorOf(wo, xp) != RED) { + break; + } + + if (parentOf(wo, x) == leftOf(wo, parentOf(wo, parentOf(wo, x)))) { + node_t *y = rightOf(wo, parentOf(wo, parentOf(wo, x))); + if (colorOf(wo, y) == RED) { + setColor(wo, parentOf(wo, x), BLACK); + setColor(wo, y, BLACK); + setColor(wo, parentOf(wo, parentOf(wo, x)), RED); + x = parentOf(wo, parentOf(wo, x)); + } else { + if (x == rightOf(wo, parentOf(wo, x))) { + x = parentOf(wo, x); + rotateLeft(wo, x); + } + setColor(wo, parentOf(wo, x), BLACK); + setColor(wo, parentOf(wo, parentOf(wo, x)), RED); + if (parentOf(wo, parentOf(wo, x)) != nullptr) { + rotateRight(wo, parentOf(wo, parentOf(wo, x))); + } + } + } else { + node_t *y = leftOf(wo, parentOf(wo, parentOf(wo, x))); + if (colorOf(wo, y) == RED) { + setColor(wo, parentOf(wo, x), BLACK); + setColor(wo, y, BLACK); + setColor(wo, parentOf(wo, parentOf(wo, x)), RED); + x = parentOf(wo, parentOf(wo, x)); + } else { + if (x == leftOf(wo, parentOf(wo, x))) { + x = parentOf(wo, x); + rotateRight(wo, x); + } + setColor(wo, parentOf(wo, x), BLACK); + setColor(wo, parentOf(wo, parentOf(wo, x)), RED); + if (parentOf(wo, parentOf(wo, x)) != nullptr) { + rotateLeft(wo, parentOf(wo, parentOf(wo, x))); + } + } + } + } + node_t *rt = this->root.get(wo, &root_orec); + if (colorOf(wo, rt) != BLACK) { + setColor(wo, rt, BLACK); + } + } + + node_t *insertOrGet(WOSTM &wo, K k, V v, node_t *n) { + node_t *t = this->root.get(wo, &root_orec); + if (t == nullptr) { + if (n == nullptr) { + return nullptr; + } + setColor(wo, n, BLACK); + this->root.set(wo, &root_orec, n); + return nullptr; + } + + for (;;) { + if (k == t->key.get(wo, t)) { + return t; + } else if (k < t->key.get(wo, t)) { + node_t *tl = leftOf(wo, t); + if (tl != nullptr) { + t = tl; + } else { + setParent(wo, n, t); + setLeft(wo, t, n); + fixAfterInsertion(wo, n); + return nullptr; + } + } else { + node_t *tr = rightOf(wo, t); + if (tr != nullptr) { + t = tr; + } else { + setParent(wo, n, t); + setRight(wo, t, n); + fixAfterInsertion(wo, n); + return nullptr; + } + } + } + } + + node_t *successor(WOSTM &wo, node_t *t) { + if (t == nullptr) { + return nullptr; + } else if (rightOf(wo, t) != nullptr) { + node_t *p = rightOf(wo, t); + while (leftOf(wo, p) != nullptr) { + p = leftOf(wo, p); + } + return p; + } else { + node_t *p = parentOf(wo, t); + node_t *ch = t; + while (p != nullptr && ch == rightOf(wo, p)) { + ch = p; + p = parentOf(wo, p); + } + return p; + } + } + + void fixAfterDeletion(WOSTM &wo, node_t *x) { + while (x != this->root.get(wo, &root_orec) && colorOf(wo, x) == BLACK) { + if (x == leftOf(wo, parentOf(wo, x))) { + node_t *sib = rightOf(wo, parentOf(wo, x)); + if (colorOf(wo, sib) == RED) { + setColor(wo, sib, BLACK); + setColor(wo, parentOf(wo, x), RED); + rotateLeft(wo, parentOf(wo, x)); + sib = rightOf(wo, parentOf(wo, x)); + } + if (colorOf(wo, leftOf(wo, sib)) == BLACK && + colorOf(wo, rightOf(wo, sib)) == BLACK) { + setColor(wo, sib, RED); + x = parentOf(wo, x); + } else { + if (colorOf(wo, rightOf(wo, sib)) == BLACK) { + setColor(wo, leftOf(wo, sib), BLACK); + setColor(wo, sib, RED); + rotateRight(wo, sib); + sib = rightOf(wo, parentOf(wo, x)); + } + setColor(wo, sib, colorOf(wo, parentOf(wo, x))); + setColor(wo, parentOf(wo, x), BLACK); + setColor(wo, rightOf(wo, sib), BLACK); + rotateLeft(wo, parentOf(wo, x)); + + x = this->root.get(wo, &root_orec); + } + } else { + node_t *sib = leftOf(wo, parentOf(wo, x)); + if (colorOf(wo, sib) == RED) { + setColor(wo, sib, BLACK); + setColor(wo, parentOf(wo, x), RED); + rotateRight(wo, parentOf(wo, x)); + sib = leftOf(wo, parentOf(wo, x)); + } + if (colorOf(wo, rightOf(wo, sib)) == BLACK && + colorOf(wo, leftOf(wo, sib)) == BLACK) { + setColor(wo, sib, RED); + x = parentOf(wo, x); + } else { + if (colorOf(wo, leftOf(wo, sib)) == BLACK) { + setColor(wo, rightOf(wo, sib), BLACK); + setColor(wo, sib, RED); + rotateLeft(wo, sib); + sib = leftOf(wo, parentOf(wo, x)); + } + setColor(wo, sib, colorOf(wo, parentOf(wo, x))); + setColor(wo, parentOf(wo, x), BLACK); + setColor(wo, leftOf(wo, sib), BLACK); + rotateRight(wo, parentOf(wo, x)); + + x = this->root.get(wo, &root_orec); + } + } + } + + if (x != nullptr && colorOf(wo, x) != BLACK) { + setColor(wo, x, BLACK); + } + } + + node_t *delete_node(WOSTM &wo, node_t *p) { + + if (leftOf(wo, p) != nullptr && rightOf(wo, p) != nullptr) { + node_t *s = successor(wo, p); + ((p)->key) = ((((((s))->key)))); + ((p)->val) = ((((((s))->val)))); + p = s; + } + + node_t *replacement = + ((leftOf(wo, p) != nullptr) ? leftOf(wo, p) : rightOf(wo, p)); + + if (replacement != nullptr) { + setParent(wo, replacement, parentOf(wo, p)); + node_t *pp = parentOf(wo, p); + if (pp == nullptr) { + this->root.set(wo, &root_orec, replacement); + } else if (p == leftOf(wo, pp)) { + setLeft(wo, pp, replacement); + } else { + setRight(wo, pp, replacement); + } + + setLeft(wo, p, nullptr); + setRight(wo, p, nullptr); + setParent(wo, p, nullptr); + + if (colorOf(wo, p) == BLACK) { + fixAfterDeletion(wo, replacement); + } + } else if (parentOf(wo, p) == nullptr) { + this->root.set(wo, &root_orec, nullptr); + } else { + if (colorOf(wo, p) == BLACK) { + fixAfterDeletion(wo, p); + } + node_t *pp = parentOf(wo, p); + if (pp != nullptr) { + if (p == leftOf(wo, pp)) { + setLeft(wo, pp, nullptr); + } else if (p == rightOf(wo, pp)) { + setRight(wo, pp, nullptr); + } + setParent(wo, p, nullptr); + } + } + return p; + } + + void releaseNode(WOSTM &wo, node_t *n) { wo.reclaim(n); } + + void freeNode(WOSTM &wo, node_t *n) { + if (n) { + freeNode(leftOf(wo, n)); + freeNode(rightOf(wo, n)); + releaseNode(n); + } + } + +public: + rbtree_tl2_omap(HANDSTM *me, auto *cfg) { + BEGIN_WO(me); + root.set(wo, &root_orec, nullptr); + } + + void rbtree_free(rbtree_tl2_omap *r) { + freeNode(r->root); + free(r); + } + + bool insert(HANDSTM *me, const K &key, V &val) { + BEGIN_WO(me); + node_t *node = new node_t(wo, key, val); + node_t *ex = insertOrGet(wo, key, val, node); + if (ex != nullptr) { + releaseNode(wo, node); + } + return ((ex == nullptr) ? true : false); + } + + bool remove(HANDSTM *me, const K &key) { + BEGIN_WO(me); + node_t *node = nullptr; + node = lookup(wo, key); + if (node != nullptr) { + node = delete_node(wo, node); + } + if (node != nullptr) { + releaseNode(wo, node); + } + return ((node != nullptr) ? true : false); + } + + bool rbtree_update(HANDSTM *me, K key, V val) { + BEGIN_WO(me); + node_t *nn = new node_t(key, val); + node_t *ex = insertOrGet(wo, key, val, nn); + if (ex != nullptr) { + ex->val = val; + releaseNode(nn); + return true; + } + return false; + } + + V get(HANDSTM *me, const K &key, V &val) { + BEGIN_RO(me); + node_t *n = lookup(ro, key); + if (n != nullptr) { + val = n->val.get(ro, n); + return true; + } + return false; + } + + bool rbtree_contains(K key) { + node_t *n = lookup(key); + return (n != nullptr); + } +}; diff --git a/artifact/ds/handSTM/skiplist_omap_bigtx.h b/artifact/ds/handSTM/skiplist_omap_bigtx.h new file mode 100644 index 0000000..564ce18 --- /dev/null +++ b/artifact/ds/handSTM/skiplist_omap_bigtx.h @@ -0,0 +1,334 @@ +#pragma once + +#include +#include +#include + +/// NB: in this implementation we use same orec for each node and its chimney +/// nodes + +/// An ordered map, implemented as a doubly-linked skip list. This map supports +/// get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param HANDSTM A thread descriptor type, for safe memory reclamation +/// @param dummy_key A fake key, to use in sentinel nodes +/// @param dummy_val A fake value, to use in sentinel nodes +template +class skiplist_omap_bigtx { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + /// data_t is a node in the skip list. It has a key, a value, an owner, a + /// state, and a "tower" of predecessor and successor pointers + /// + /// NB: Height isn't always the size of tower... it tracks how many levels are + /// fully and correctly stitched, so it changes during insertion and + /// removal. + struct data_t : ownable_t { + /// A pair of data pointers, for the successor and predecessor at a level of + /// the tower + struct level_t { + FIELD next; // Succ at this level + FIELD prev; // Pred at this level + }; + + const K key; // The key stored in this node + FIELD val; // The value stored in this node + const uint8_t height; // # valid tower nodes + level_t tower[0]; // Tower of pointers to pred/succ + + private: + /// Construct a data node. This is private to force the use of our make_* + /// methods, which handle allocating enough space for the tower. + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(K _key, V _val, uint8_t _height) + : key(_key), val(_val), height(_height) {} + + public: + /// Construct a sentinel (head or tail) node. Note that the sentinels can't + /// easily be of a node type that lacks key and value fields, or else the + /// variable-length array would preclude inheriting from it. + /// + /// @param iHeight The max number of index layers this node will have + static data_t *make_sentinel(uint8_t iHeight) { + int node_size = sizeof(data_t) + (iHeight + 1) * sizeof(level_t); + void *region = malloc(node_size); + return new (region) data_t(dummy_key, dummy_val, iHeight); + } + + /// Construct a data node + /// + /// @param iHeight The max number of index layers this node will have + /// @param key The key to store in this node + /// @param val The value to store in this node + static data_t *make_data(WOSTM &wo, uint64_t iHeight, K key, V val) { + int node_size = sizeof(data_t) + (iHeight + 1) * sizeof(level_t); + void *region = malloc(node_size); + return wo.LOG_NEW(new (region) data_t(key, val, iHeight)); + } + }; + + const int NUM_INDEX_LAYERS; // # of index layers. Doesn't count data layer + const int SNAPSHOT_FREQUENCY; // # of nodes between snapshots + data_t *const head; // The head sentinel + data_t *const tail; // The tail sentinel + +public: + /// Default construct a skip list by stitching a head sentinel to a tail + /// sentinel at each level + /// + /// @param _op The operation that is constructing the list + /// @param cfg A configuration object that has a `snapshot_freq` field + skiplist_omap_bigtx(HANDSTM *me, auto *cfg) + : NUM_INDEX_LAYERS(cfg->max_levels), + SNAPSHOT_FREQUENCY(cfg->snapshot_freq), + head(data_t::make_sentinel(NUM_INDEX_LAYERS)), + tail(data_t::make_sentinel(NUM_INDEX_LAYERS)) { + // NB: Even though the constructor is operating on private data, it needs a + // TM context in order to set the head and tail's towers to each other + BEGIN_WO(me); + for (auto i = 0; i <= NUM_INDEX_LAYERS; i++) { + head->tower[i].next.set(wo, head, tail); + tail->tower[i].prev.set(wo, tail, head); + } + } + + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(HANDSTM *me, const K &key, V &val) { + BEGIN_RO(me); + // Do a leq... if head, we fail. n will never be null or tail + auto n = get_leq(ro, key); + if (n == head || n->key != key) + return false; + + val = n->val.get(ro, n); + return true; + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's HANDSTM + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(HANDSTM *me, const K &key, V &val) { + + BEGIN_WO(me); + data_t *new_dn = nullptr; // The node that we insert, if any + int target_height = randomLevel(me); // The target index height of new_dn + // This transaction linearizes the insert by adding the node to the data + // layer + + // Get the insertion point, make sure `key` not already present + auto n = get_leq(wo, key); + + if (n != head && n->key == key) + return false; + + // Stitch new node in at lowest level. Get() can see it immediately. + // Remove() has to wait + auto next = n->tower[0].next.get(wo, n); + new_dn = data_t::make_data(wo, target_height, key, val); + new_dn->tower[0].next.set(wo, new_dn, next); + new_dn->tower[0].prev.set(wo, new_dn, n); + n->tower[0].next.set(wo, n, new_dn); + next->tower[0].prev.set(wo, next, new_dn); + + // If this doesn't have any index nodes, we can unmark it and return + if (target_height == 0) + return true; + + // 'me' did an insert, still owns the node, and needs to stitch it into + // index levels. Do so from bottom to top. Release after the last level. + for (int level = 0; level < target_height; ++level) + index_stitch(wo, new_dn, level); + + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's HANDSTM + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(HANDSTM *me, const K &key) { + BEGIN_WO(me); + // Find the node. Fail if key not present + auto n = get_leq(wo, key); + if (n == head || n->key != key) + return false; + + // Unstitch the node, starting from its topmost level + // + // NB: When depth = 0, it's inefficient to unlock node and then re-lock. + // However, it's also inefficient to abort because we can't lock the + // predecessor or successor. Taking ownership then committing avoids + // re-traversing, so we'll go with it, especially since doing so won't + // block get() or nonconflicting insert(). + index_unstitch(wo, n, n->height); + return true; + } + +private: + /// get_leq uses the towers to skip from the head sentinel to the node + /// with the largest key <= the search key. It can return the head data + /// sentinel, but not the tail sentinel. + /// + /// get_leq can return an OWNED node. + /// + /// @param key The key for which we are doing a predecessor query. + /// + /// @return The data node that was found + data_t *get_leq(STM &tx, const K &key) { + // We always start at the head sentinel. Scan its tower to find the + // highest non-tail level + data_t *curr = head; + int current_level = 0; + for (int i = NUM_INDEX_LAYERS; i > 0; --i) { + if (head->tower[i].next.get(tx, head) != tail) { + current_level = i; + break; + } + } + + // Traverse over and down through the index layers + while (current_level > 0) { + curr = index_leq(tx, key, curr, current_level); + if (curr->key == key) + return curr; + --current_level; + } + + // Search in the data layer. Only return if result valid, not DELETED + return data_leq(tx, key, curr); + } + + /// Traverse forward from `start`, considering only tower level `level`, + /// stopping at the largest key <= `key` + /// + /// This can return nodes that are OWNED. The caller must check. + /// + /// @param key The key for which we are doing a predecessor query. + /// @param start The start position of this traversal. + /// @param level The tower level to consider + /// + /// @return The node that was found (possibly `start`). + data_t *index_leq(STM &tx, K key, data_t *start, uint64_t level) { + auto curr = start; + while (true) { + data_t *next = curr->tower[level].next.get(tx, curr); + if (next == tail) + return curr; + auto next_key = next->key; // not tail => next has a valid key + if (next_key == key) + return next; + if (next_key > key) + return curr; + curr = next; + } + } + + /// Traverse in the data layer to find the largest node with key <= `key`. + /// This can return an OWNED node. + /// + /// @param key The key for which we are doing a predecessor query. + /// @param start The start position of this traversal. This may be the head, + /// or an intermediate point in the list + /// + /// @return The node that was found (possibly `start`). + data_t *data_leq(STM &tx, K key, data_t *start) { + auto curr = start; + auto next = curr->tower[0].next.get(tx, curr); + while (true) { + if (next == tail) + return curr; + auto nkey = next->key; + if (nkey > key) + return curr; + if (nkey == key) + return next; + curr = next; + next = next->tower[0].next.get(tx, next); + } + } + + /// Generate a random level for a new node + /// + /// NB: This code has been verified to produce a nice geometric distribution + /// in constant time per call + /// + /// @param me The caller's HANDSTM operation + /// + /// @return a random number between 0 and NUM_INDEX_LAYERS, inclusive + int randomLevel(HANDSTM *me) { + // Get a random int between 0 and 0xFFFFFFFF + int rr = me->rand(); + // Add 1 to it, then find the lowest nonzero bit. This way, we never return + // a zero for small integers, and the distribution is correct. + int res = __builtin_ffs(rr + 1); + // Now take one off of that, so that we return a zero-based integer + res -= 1; + // But if rr was 0xFFFFFFFF, we've got a problem, so coerce it back + // Also, drop it down to within NUM_INDEX_LAYERS + return (res < 0 || res > NUM_INDEX_LAYERS) ? NUM_INDEX_LAYERS : res; + } + + /// index_stitch is a small atomic operation that stitches a node in at a + /// given index level. + /// + /// @param node The node that was just inserted and stitched into `level` + /// @param level The level below where we're stitching + void index_stitch(WOSTM &wo, data_t *node, uint8_t level) { + // Go backwards, then up, to find a node at current `level` that is tall + // enough. Then get its successor + data_t *pred = node; + while (true) { + pred = pred->tower[level].prev.get(wo, pred); + if (pred->height > level) + break; + } + auto succ = pred->tower[level + 1].next.get(wo, pred); + + // Stitch `node` in between pred and succ, update node's height + node->tower[level + 1].next.set(wo, node, succ); + node->tower[level + 1].prev.set(wo, node, pred); + pred->tower[level + 1].next.set(wo, pred, node); + succ->tower[level + 1].prev.set(wo, succ, node); + } + + /// Unstitch `node`, starting at its topmost index layer. It is currently + /// OWNED. + /// + /// @param node The node that we are unstitching + /// @param height The highest index layer for `node` + void index_unstitch(WOSTM &wo, data_t *node, int height) { + // Work our way downward, unstitching at each level + for (int level = height; level >= 0; --level) { + auto pre = node->tower[level].prev.get(wo, node); + auto nxt = node->tower[level].next.get(wo, node); + pre->tower[level].next.set(wo, pre, nxt); + nxt->tower[level].prev.set(wo, nxt, pre); + } + wo.reclaim(node); + } +}; diff --git a/artifact/ds/handSTM/slist_omap.h b/artifact/ds/handSTM/slist_omap.h new file mode 100644 index 0000000..8b0cffa --- /dev/null +++ b/artifact/ds/handSTM/slist_omap.h @@ -0,0 +1,154 @@ +#pragma once + +/// An ordered map, implemented as a singly-linked list. This map supports +/// get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param HANDSTM A thread descriptor type, for safe memory reclamation +template class slist_omap { + using WOSTM = typename HANDSTM::WOSTM; + using ROSTM = typename HANDSTM::ROSTM; + using STM = typename HANDSTM::STM; + using ownable_t = typename HANDSTM::ownable_t; + template using FIELD = typename HANDSTM::template xField; + + /// A list node. It has a next pointer, but no key or value. It's useful for + /// sentinels, so that K and V don't have to be default constructable. + struct node_t : ownable_t { + FIELD next; // Pointer to successor + + /// Construct a node + node_t() : next(nullptr) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~node_t() {} + }; + + /// A list node that also has a key and value. Note that keys are const. + struct data_t : public node_t { + const K key; // The key of this pair + FIELD val; // The value of this pair + + /// Construct a data_t + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(const K &_key, const V &_val) : node_t(), key(_key), val(_val) {} + }; + + node_t *const head; // The list head pointer + node_t *const tail; // The list tail pointer + +public: + /// Default construct a list by constructing and connecting two sentinel nodes + /// + /// @param me The operation that is constructing the list + /// @param cfg A configuration object that has a `snapshot_freq` field + slist_omap(HANDSTM *me, auto *cfg) : head(new node_t()), tail(new node_t()) { + BEGIN_WO(me); + head->next.set(wo, head, tail); + } + +private: + /// get_leq is an inclusive predecessor query that returns the largest node + /// whose key is <= the provided key. It can return the head sentinel, but + /// not the tail sentinel. + /// + /// @param key The key for which we are doing a predecessor query. + /// @param lt_mode When `true`, this behaves as `get_lt`. When `false`, it + /// behaves as `get_leq`. + /// + /// @return The node that was found, and its orec value + node_t *get_leq(STM &tx, const K key, bool lt_mode = false) { + // Start at the head; read the next now, to avoid reading it in multiple + // iterations of the loop + node_t *curr = head; + + // Starting at `next`, search for key. + while (true) { + // Read the next node, fail if we can't do it consistently + auto next = curr->next.get(tx, curr); + + // Stop if next's key is too big or next is tail + if (next == tail) + return curr; + data_t *dn = static_cast(next); + K k = dn->key; + if (lt_mode ? k >= key : k > key) + return curr; + + // Stop if `next` is the match we were hoping for + if (k == key) + return next; + + // Keep traversing to `next` + curr = next; + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(HANDSTM *me, const K &key, V &val) { + BEGIN_RO(me); + // find the largest node with a key <= `key`. + auto n = get_leq(ro, key); + if (n == head || static_cast(n)->key != key) + return false; + data_t *dn = static_cast(n); + val = dn->val.get(ro, dn); + return true; + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(HANDSTM *me, const K &key, V &val) { + BEGIN_WO(me); + + auto n = get_leq(wo, key); + if (n != head && static_cast(n)->key == key) + return false; + + // stitch in a new node + data_t *new_dn = wo.LOG_NEW(new data_t(key, val)); + new_dn->next.set(wo, new_dn, n->next.get(wo, n)); + n->next.set(wo, n, new_dn); + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(HANDSTM *me, const K &key) { + BEGIN_WO(me); + // NB: this will be a lt query, not a leq query + auto prev = get_leq(wo, key, true); + auto curr = prev->next.get(wo, prev); + // if curr doesn't have a matching key, fail + if (curr == tail || static_cast(curr)->key != key) + return false; + auto next = curr->next.get(wo, curr); + prev->next.set(wo, prev, next); + wo.reclaim(curr); + return true; + } +}; diff --git a/artifact/ds/hybrid/dlist_carumap.h b/artifact/ds/hybrid/dlist_carumap.h new file mode 100644 index 0000000..85667b6 --- /dev/null +++ b/artifact/ds/hybrid/dlist_carumap.h @@ -0,0 +1,696 @@ +#pragma once + +#include +#include +#include +/// An unordered map, implemented as a resizable array of lists (closed +/// addressing, resizable). This map supports get(), insert() and remove() +/// operations. +/// +/// This implementation is based loosely on Liu's nonblocking resizable hash +/// table from PODC 2014. At the current time, we do not support the heuristic +/// for contracting the list, but we do support expanding the list. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param HYPOL The HYPOL implementation (PO or PS) +template class dlist_carumap { + using WOSTM = typename HYPOL::WOSTM; + using ROSTM = typename HYPOL::ROSTM; + using RSTEP = typename HYPOL::RSTEP; + using WSTEP = typename HYPOL::WSTEP; + using ownable_t = typename HYPOL::ownable_t; + template using FIELD = typename HYPOL::template sxField; + + /// A list node. It has prev and next pointers, but no key or value. It's + /// useful for sentinels, so that K and V don't have to be default + /// constructable. + /// + /// NB: we do not need a `valid` bit, because any operation that would clear + /// it would also acquire this node's orec, and thus any node that would + /// encounter a cleared valid bit would also detect an orec inconsistency. + struct node_t : ownable_t { + FIELD prev; // Pointer to predecessor + FIELD next; // Pointer to successor + + /// Construct a node + node_t() : ownable_t(), prev(nullptr), next(nullptr) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~node_t() {} + }; + + /// We need to know if buckets have been rehashed to a new table. We do this + /// by making the head of each bucket a `sentinel_t`, and adding a `closed` + /// bool. Note that the tail of each bucket's list is just a node_t. + struct sentinel_t : node_t { + /// Track if this sentinel is for a bucket that has been rehashed + /// + /// NB: Could we use `prev` to indicated `closed` + FIELD closed; // Has it been rehashed? + + /// Construct a sentinel_t + sentinel_t() : node_t(), closed(false) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~sentinel_t() {} + }; + + /// A list node that also has a key and value. Note that keys are const, and + /// values are only accessed while the node is locked, so neither is a + /// tm_field. + struct data_t : node_t { + const K key; // The key of this key/value pair + V val; // The value of this key/value pair + + /// Construct a data_t + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(const K &_key, const V &_val) : node_t(), key(_key), val(_val) {} + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~data_t() {} + }; + + /// An array of lists, along with its size + /// + /// NB: to avoid indirection, the array is in-lined into the tbl_t. To make + /// this compatible with SMR, tbl_t must be ownable. + class tbl_t : public ownable_t { + using bucket_t = FIELD; + + /// Construct a table + /// + /// @param `_size` The desired size of the table + tbl_t(uint64_t _size) : size(_size) {} + + public: + const uint64_t size; // The size of the table + bucket_t tbl[]; // The buckets of the table + + /// Allocate a tbl_t of size `size` + /// + /// @param size The desired size + /// @param tx The calling operation's descriptor + /// + /// @return A table, all of whose buckets are set to null + static tbl_t *make(uint64_t size, WSTEP &tx) { + tbl_t *tbl = (tbl_t *)malloc(sizeof(tbl_t) + size * sizeof(bucket_t)); + auto ret = new (tbl) tbl_t(size); + for (size_t i = 0; i < size; ++i) + ret->tbl[i].sSet(nullptr, tx); + return ret; + } + }; + + ownable_t *tbl_orec; // An orec for protecting `active` and `frozen` + FIELD active; // The active table + FIELD frozen; // The frozen table + std::hash _pre_hash; // A weak hash function for converting keys to ints + const uint64_t RESIZE_THRESHOLD; // Max bucket size before resizing + + /// A pair consisting of a pointer and an orec version. + struct node_ver_t { + node_t *_obj = nullptr; // The start of a bucket + uint64_t _ver = 0; // NB: _ver may not be related to _obj + }; + + /// Result of trying to resize a bucket + enum resize_result_t { + CANNOT_ACQUIRE, // Couldn't get orec... retry + ALREADY_RESIZED, // Already resized by another thread + RESIZE_OK // Bucket successfully resized + }; + + /// Given a key, determine the bucket into which it should go. As in the Liu + /// hash, we do not change the hash function when we resize, we just change + /// the number of bits to use + /// + /// @param key The key to hash + /// @param size The size of the table into which this should be hashed + /// + /// @return An integer between 0 and size + uint64_t table_hash(HYPOL *me, const K &key, const uint64_t size) const { + return me->hash(_pre_hash(key)) % size; + } + +public: + /// Default construct a map as having a valid active table. + /// + /// NB: This constructor calls std::terminate if the provided size is not a + /// power of 2. + /// + /// @param me The operation that is creating this umap + /// @param cfg A config object with `buckets` and `resize_threshold` + dlist_carumap(HYPOL *me, auto *cfg) + : tbl_orec(new ownable_t()), RESIZE_THRESHOLD(cfg->resize_threshold) { + // Enforce power-of-2 initial size + if (std::popcount(cfg->buckets) != 1) + throw("cfg->buckets should be power of 2"); + + // Create an initial active table in which all of the buckets are + // initialized but empty (null <- head <-> tail -> null). + WSTEP tx(me); + active.sSet(tbl_t::make(cfg->buckets, tx), tx); + for (size_t i = 0; i < cfg->buckets; ++i) + active.sGet(tx)->tbl[i].sSet(create_list(tx), tx); + // NB: since all buckets are initialized, nobody will ever go to the + // frozen table, so we can leave it as null + frozen.sSet(nullptr, tx); + } + +private: + /// Create a dlist with head and tail sentinels + /// + /// @param tx A writing TM context. Even though this code can't fail, we need + /// the context in order to use tm_field correctly. + /// + /// @return A pointer to the head sentinel of the list + sentinel_t *create_list(WSTEP &tx) { + // NB: By default, a node's prev and next will be nullptr, which is what we + // want for head->prev and tail->next. + auto head = new sentinel_t(); + auto tail = new node_t(); + head->next.sSet(tail, tx); + tail->prev.sSet(head, tx); + return head; + } + + /// `resize()` is an internal method for changing the size of the active + /// table. Strictly speaking, it should be called `expand`, because for now we + /// only support expansion, not contraction. When `insert()` discovers that + /// it has made a bucket "too big", it will continue to do its insertion and + /// then, after linearizing, it will call `resize()`. `remove()` does not + /// currently call `resize()`. + /// + /// At a high level, `resize()` is supposed to be open-nested and not to incur + /// any blocking, except due to orec conflicts. We accomplish this through + /// laziness and stealing. resize() finishes the /last/ resize, moves the + /// `active` table to `frozen`, and installs a new `active` table. Subsequent + /// operations will do most of the migrating. + /// + /// @param me The calling thread's descriptor + /// @param a_ver The version of `active` when the resize was triggered + void resize(HYPOL *me, uint64_t a_ver) { + // Get the current active and frozen tables, and the frozen table size + tbl_t *ft = nullptr, *at = nullptr; + { + RSTEP tx(me); + ft = frozen.sGet(tx); + at = active.sGet(tx); + if (!tx.check_continuation(tbl_orec, a_ver)) + return; // someone else must be starting a resize, so we can quit + } + + // If ft is null, then there's no frozen table, so things will be easy + if (ft == nullptr) { + WSTEP tx(me); + + // Make and initialize the table *before* acquiring orecs, to minimize the + // critical section. The table is 2x as big. + auto new_tbl = tbl_t::make(at->size * 2, tx); + + // Lock the table, move it from `active` to `frozen`, then install the new + // table. + if (!tx.acquire_continuation(tbl_orec, a_ver)) { + // NB: new_tbl is private. We don't need SMR + delete new_tbl; + return; // Someone else is resizing, and that's good enough for `me` + } + frozen.sSet(at, tx); + active.sSet(new_tbl, tx); + return; + } + + // Migrate everything out of frozen, remove the frozen table, and retry + // + // NB: prepare_resize removes the frozen table. That will change a_ver, so + // we need to capture the new a_ver value so that our next attempt won't + // fail erroneously. + a_ver = prepare_resize(me, a_ver, ft, at); + if (a_ver == 0) + return; // Someone else finished resizing for `me` + + resize(me, a_ver); // Try again now that it's clean + } + + /// Finish one lazy resize, so that another may begin. + /// + /// This really just boils down to migrating everything from `frozen` to + /// `active` and then nulling `frozen` and reclaiming it. + /// + /// NB: This code takes the "frozen" and "active" tables as arguments. + /// Consequently, we don't care about arbitrary delays. If a thread calls + /// this, rehashes half the table, and then suspends, another thread can + /// rehash everything else and install a new active table. When the first + /// thread wakes, it'll find a bunch of empty buckets, and it'll be safe. + /// + /// @param me The calling thread's descriptor + /// @param a_ver The active table version when this was called + /// @param f_tbl The "frozen table", really the "source" table + /// @param a_tbl The "active table", really the "destination" table + /// + /// @return {0} if another thread stole the job of nulling `frozen`. + /// When this happens, there must be a concurrent resize, + /// and since both are trying to do the same thing (expand), + /// the one who receives {0} can just get out of the other's + /// way + /// {integer} the new orec version of `active` + uint64_t prepare_resize(HYPOL *me, uint64_t a_ver, tbl_t *f_tbl, + tbl_t *a_tbl) { + // NB: Right now, next_index == completed. If we randomized the start + // point, concurrent calls to prepare_resize() would contend less + uint64_t next_index = 0; // Next bucket to migrate + uint64_t completed = 0; // Number of buckets migrated + + // Migrate all data from `frozen` to `active` + while (completed != f_tbl->size) { + uint64_t bucket_orec = 0; + sentinel_t *bucket = nullptr; + { + RSTEP tx(me); + + // Try to rehash the next bucket + bucket = f_tbl->tbl[next_index].sGet(tx); + bucket_orec = tx.check_orec(bucket); + if (bucket_orec == HYPOL::END_OF_TIME) { + continue; + } + } + + auto res = rehash_expand_bucket(me, bucket, bucket_orec, next_index, + f_tbl->size, a_tbl); + { + RSTEP tx(me); + // If we can't acquire all nodes in this bucket, try again, because it + // might just mean someone else was doing an operation in the bucket. + if (res == CANNOT_ACQUIRE) { + continue; + } + // If this bucket is already rehashed by others, there is a chance that + // the current resize phase is finished, so check + if (res == ALREADY_RESIZED) { + // check if the active table version changed since resize() was + // called, if so, we know resize is finished, return + if (!tx.check_continuation(tbl_orec, a_ver)) { + return 0; + } + } + } + + // Move to the next bucket + ++next_index; + ++completed; + } + + // Uninstall the `frozen` table, since it has been emptied. Save the commit + // time, so we can validate tbl_orec later. + tbl_t *old; + { + BEGIN_WO(me); + if (wo.inheritOrec(tbl_orec, a_ver)) { + old = f_tbl; + frozen.xSet(wo, tbl_orec, nullptr); + } else + return 0; + } + auto last_commit_time = me->get_last_wo_end_time(); + + // Reclaim `old`'s buckets, then `old` itself + { + BEGIN_WO(me); + for (size_t i = 0; i < f_tbl->size; i++) { + // use singleton_reclaim to reclaim head and tail of each bucket + auto head = old->tbl[i].xGet(wo, tbl_orec); + auto tail = head->next.xGet(wo, head); + wo.reclaim(head); + wo.reclaim(tail); + } + wo.reclaim(old); + } + return last_commit_time; + } + + /// Get a pointer to the bucket in the active table that holds `key`. This + /// may cause some rehashing to happen. + /// + /// NB: The pattern here is unconventional. get_bucket() is the first step in + /// WSTEP transactions. If it doesn't rehash, then the caller WSTEP + /// continues its operation. If it does rehash, then the caller WSTEP + /// commits and restarts, which is a poor-man's open-nested transaction. + /// If it encounters an inconsistency, the caller WSTEP should "abort" by + /// unwinding and restarting. In the third case, this returns *while + /// holding an orec* + /// + /// @param me The calling thread's descriptor + /// @param key The key whose bucket is sought + /// @param tx An active WSTEP transaction + /// + /// @return On success, a pointer to the head of a bucket, along with + /// `tbl_orec`'s value. {nullptr, 0} on any rehash or inconsistency + std::pair get_bucket(HYPOL *me, const K &key) { + // Get the head of the appropriate bucket in the active table + // + // NB: Validate or else a_tbl[a_idx] could be out of bounds + uint64_t f_bucket_orec = 0, f_idx = 0; + sentinel_t *f_bucket = nullptr; + tbl_t *f_tbl = nullptr, *a_tbl = nullptr; + + while (true) { + RSTEP tx(me); + a_tbl = active.sGet(tx); + uint64_t a_ver = tx.check_orec(tbl_orec); + if (a_ver == HYPOL::END_OF_TIME) { + continue; + } + + auto a_idx = table_hash(me, key, a_tbl->size); + auto a_bucket = a_tbl->tbl[a_idx].sGet(tx); // NB: caller will validate + + if (a_bucket) { + auto b_ver = tx.check_orec(a_bucket); + if (b_ver == HYPOL::END_OF_TIME) { + continue; + } + return {{a_bucket, b_ver}, a_ver}; // not null --> no resize needed + } + + // Find the bucket in the frozen table that needs rehashing + f_tbl = frozen.sGet(tx); + if (tx.check_orec(tbl_orec) == HYPOL::END_OF_TIME) { + continue; + } + + f_idx = table_hash(me, key, f_tbl->size); + f_bucket = f_tbl->tbl[f_idx].sGet(tx); + f_bucket_orec = tx.check_orec(f_bucket); + if (f_bucket_orec == HYPOL::END_OF_TIME) { + continue; + } + break; + } + + // while(true){ + // WSTEP tx(me); + // if(!tx.acquire_continuation(f_bucket, f_bucket_orec)) + // continue; + // } + + // Rehash it, tell caller to commit so the rehash appears to be open nested + // + // NB: if the rehash fails, it's due to someone else rehashing, which is OK + rehash_expand_bucket(me, f_bucket, f_bucket_orec, f_idx, f_tbl->size, + a_tbl); + return {{nullptr, 0}, 0}; + } + + /// Re-hash one list in the frozen table into two lists in the active table + /// + /// @param me The calling thread's descriptor + /// @param f_list A pointer to an (acquired!) list head in the frozen table + /// @param f_idx The index of flist in the frozen table + /// @param f_size The size of the frozen table + /// @param a_tbl A reference to the active table + /// @param tx An active WSTEP transaction + /// + /// @return RESIZE_OK - The frozen bucket was rehashed into `a_tbl` + /// ALREADY_RESIZED - The frozen bucket was empty + /// CANNOT_ACQUIRE - The operation could not acquire all orecs + resize_result_t rehash_expand_bucket(HYPOL *me, sentinel_t *f_list, + uint64_t f_list_orec, uint64_t f_idx, + uint64_t f_size, tbl_t *a_tbl) { + WSTEP tx(me); + if (!tx.acquire_continuation(f_list, f_list_orec)) + return CANNOT_ACQUIRE; + // Stop if this bucket is already rehashed + if (f_list->closed.sGet(tx)) // true is effectively const, skip validation + return ALREADY_RESIZED; + // Fail if we cannot acquire all nodes in f_list + if (!list_acquire_all(f_list, tx)) + return CANNOT_ACQUIRE; + + // Shuffle nodes from f_list into two new lists that will go into `a_tbl` + auto l1 = create_list(tx), l2 = create_list(tx); + auto curr = f_list->next.sGet(tx); + while (curr->next.sGet(tx) != nullptr) { + auto next = curr->next.sGet(tx); + auto data = static_cast(curr); + auto dest = table_hash(me, data->key, a_tbl->size) == f_idx ? l1 : l2; + auto succ = dest->next.sGet(tx); + dest->next.sSet(data, tx); + data->next.sSet(succ, tx); + data->prev.sSet(dest, tx); + succ->prev.sSet(data, tx); + curr = next; + } + // curr is tail, set head->tail + f_list->next.sSet(curr, tx); + // put the lists into the active table, close the frozen bucket + a_tbl->tbl[f_idx].sSet(l1, tx); + a_tbl->tbl[f_idx + f_size].sSet(l2, tx); + f_list->closed.sSet(true, tx); + return RESIZE_OK; + } + + /// Acquire all of the nodes in the list starting at `head`, including the + /// head and tail sentinels + /// + /// @param head The head of the list whose nodes should be acquired + /// @param tail The calling WSTEP transaction + /// + /// @return true if all nodes are acquired, false otherwise + bool list_acquire_all(node_t *head, WSTEP &tx) { + node_t *curr = head; + while (curr) { + if (!tx.acquire_consistent(curr)) + return false; + curr = curr->next.sGet(tx); + } + return true; + } + + /// Given the head sentinel of a list, search through the list to find the + /// node with key `key`, if such a node exists in the list. If it doesn't, + /// then return the head pointer, along with a count of non-sentinel nodes in + /// the list + /// + /// @param key The key for which we are searching + /// @param head The start of the list to search + /// @param tx An active WSTEP transaction + /// + /// @return {nullptr, 0} if the transaction discovered an inconsistency + /// {head, count} if the key was not found + /// {node, 0} if the key was found at `node` + std::pair list_get_or_head(HYPOL *me, const K &key, + sentinel_t *head, + uint64_t head_orec) { + RSTEP tx(me); + // Get the head's successor; on any inconsistency, return. + auto curr = head->next.sGet(tx); + if (!tx.check_continuation(head, head_orec)) { + return {{nullptr, 0}, 0}; + } + // uint64_t head_orec = tx.check_orec(head); + // if (head_orec == HYPOL::END_OF_TIME) + // return {nullptr, 0}; + + uint64_t count = 0; // Number of nodes encountered during the loop + + while (true) { + // if we reached the tail, return the head + if (curr->next.sGet(tx) == nullptr) { + if (!tx.check_continuation(head, head_orec)) { + return {{nullptr, 0}, 0}; + } + return {{head, head_orec}, + count}; // No validation: tail's next is effectively const + } + + // return curr if it has a matching key + if (static_cast(curr)->key == key) { + auto c_orec = tx.check_orec(curr); + if (c_orec == HYPOL::END_OF_TIME) + return {{nullptr, 0}, 0}; + return {{curr, c_orec}, 0}; + } + + // read `next` consistently + // + // NB: We could skip this, and just validate before `return {curr, 0}` + auto next = curr->next.sGet(tx); + if (tx.check_orec(curr) == HYPOL::END_OF_TIME) + return {{nullptr, 0}, 0}; + curr = next; + ++count; + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(HYPOL *me, const K &key, V &val) { + while (true) { + + // Get the bucket in `active` where `key` should be. "Abort" and retry on + // any inconsistency; commit and retry if `get_bucket` resized + auto [bucket_pair, _] = get_bucket(me, key); + auto bucket = bucket_pair._obj; + auto bucket_orec = bucket_pair._ver; + if (!bucket) + continue; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + auto [node_pair, __] = list_get_or_head( + me, key, static_cast(bucket), bucket_orec); + auto node = node_pair._obj; + // If we got back null, there was an inconsistency, so retry + if (!node) { + continue; + } + + // If we got back the head, return false + if (node == bucket) { + return false; + } + + if (std::is_scalar::value) { + RSTEP tx(me); + data_t *dn = static_cast(node); + V val_copy = reinterpret_cast *>(&dn->val)->load( + std::memory_order_acquire); + if (tx.check_orec(node) == HYPOL::END_OF_TIME) { + continue; + } + val = val_copy; + return true; + } else { + WSTEP tx(me); + // Acquire, read, unwind (because no writes!) + if (!tx.acquire_continuation(node, node_pair._ver)) { + continue; + } + val = static_cast(node)->val; + tx.unwind(); + return true; + } + } + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(HYPOL *me, const K &key, V &val) { + // If we discover that a bucket becomes too full, we'll insert, linearize, + // and then resize in a new transaction before returning. Tracking + // `active`'s version prevents double-resizing under concurrency. + uint64_t a_ver = 0; + while (true) { + + auto [bucket_pair, a_version] = get_bucket(me, key); + auto bucket = bucket_pair._obj; + if (!bucket) + continue; + a_ver = a_version; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + auto [node_pair, count] = + list_get_or_head(me, key, static_cast(bucket_pair._obj), + bucket_pair._ver); + auto node = node_pair._obj; + // If we got back null, there was an inconsistency, so retry + if (!node) { + continue; + } + + // If we didn't get the head, the key already exists, so return false + if (node != bucket_pair._obj) { + return false; + } + BEGIN_WO(me); + // Lock the node and its successor + if (!wo.inheritOrec(node, node_pair._ver)) { + continue; + } + auto next = node->next.xGet(wo, node); + + // Stitch in a new node + data_t *new_dn = new data_t(key, val); + new_dn->next.xSet(wo, new_dn, next); + new_dn->prev.xSet(wo, new_dn, node); + node->next.xSet(wo, node, new_dn); + next->prev.xSet(wo, next, new_dn); + if (count >= RESIZE_THRESHOLD) + break; // need to resize! + return true; + } + + resize(me, a_ver); + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(HYPOL *me, const K &key) { + while (true) { + // Get the bucket in `active` where `key` should be. Abort and retry on + // any inconsistency; commit and retry if `get_bucket` resized + auto [bucket_pair, _] = get_bucket(me, key); + auto bucket = bucket_pair._obj; + if (!bucket) + continue; + + // Find the node in `bucket` that matches `key`. If it can't be found, + // we'll get the head node. + // + // NB: While `bucket` has not been reclaimed, `active.tbl` may have + // changed. Fortunately, list_get_or_head will validate it. + auto [node_pair, __] = list_get_or_head( + me, key, static_cast(bucket), bucket_pair._ver); + auto node = node_pair._obj; + WSTEP tx(me); + // If we got back the head, return false + if (node == bucket) { + tx.unwind(); // because we didn't update shared memory + return false; + } + + // If the `node` is null, list_get_or_head failed and we need to retry + // Otherwise, it's unowned and the keys match, so lock `node` and its + // neighbors, else retry + if (!node || !tx.acquire_continuation(node, node_pair._ver) || + !tx.acquire_aggressive(node->prev.sGet(tx)) || + !tx.acquire_aggressive(node->next.sGet(tx))) { + tx.unwind(); + continue; + } + + // unstitch it + auto pred = node->prev.sGet(tx), succ = node->next.sGet(tx); + pred->next.sSet(succ, tx); + succ->prev.sSet(pred, tx); + tx.reclaim(node); + return true; + } + } +}; diff --git a/artifact/ds/hybrid/rbtree_omap_drop.h b/artifact/ds/hybrid/rbtree_omap_drop.h new file mode 100644 index 0000000..4b1aca5 --- /dev/null +++ b/artifact/ds/hybrid/rbtree_omap_drop.h @@ -0,0 +1,599 @@ +#pragma once + +/// An ordered map, implemented as a balanced, internal binary search tree. This +/// map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param HYPOL A thread descriptor type, for safe memory reclamation +/// @param dummy_key A default key to use +/// @param dummy_val A default value to use +template +class rbtree_omap_drop { + using WOSTM = typename HYPOL::WOSTM; + using ROSTM = typename HYPOL::ROSTM; + using RSTEP = typename HYPOL::RSTEP; + using WSTEP = typename HYPOL::WSTEP; + using ownable_t = typename HYPOL::ownable_t; + template using FIELD = typename HYPOL::template sxField; + + /// An easy-to-remember way of indicating the left and right children + enum DIRS { LEFT = 0, RIGHT = 1 }; + + static const int RED = 0; // Enum for red + static const int BLACK = 1; // Enum for black + + /// nodes in a red/black tree + struct node_t : ownable_t { + FIELD key; // Key stored at this node + FIELD val; // Value stored at this node + FIELD color; // color (RED or BLACK) + FIELD parent; // pointer to parent + FIELD ID; // 0/1 for left/right child + FIELD child[2]; // L/R children + + /// basic constructor + node_t(WOSTM &wo, int color, K key, V val, node_t *parent, long ID, + node_t *child0, node_t *child1) + : key(key), val(val), color(color), parent(parent), ID(ID) { + child[0].xSet(wo, this, child0); + child[1].xSet(wo, this, child1); + } + }; + + node_t *sentinel; // The (sentinel) root node of the tree + + /// The pair returned by get_leq; equivalent to the type in snapshots + struct leq_t { + node_t *_obj = nullptr; // The object + uint64_t _ver = 0; // The observed version of the object + }; + + /// A pair holding a child node and its parent, with orec validation info + struct ret_pair_t { + leq_t child; // The child + leq_t parent; // The parent of that child + }; + + /// Search for a `key` in the tree, and return the node holding it. If the + /// key is not found, return the node that ought to be parent of the (not + /// found) `key`. + /// + /// NB: The caller is responsible for clearing the checkpoint stack before + /// calling get_node(). + /// + /// @param me The calling thread's descriptor + /// @param key The key to search for + /// + /// @return {found, orec} if `key` is in the tree; + /// {parent, orec} if `key` is not in the tree + ret_pair_t get_node(HYPOL *me, const K &key) const { + // This loop delineates the search transaction. It commences from the end + // of the longest consistent prefix in the checkpoint stack + while (true) { + // Open a RO transaction to traverse downward to the target node: + leq_t parent = {nullptr, 0}, child = {nullptr, 0}; + RSTEP tx(me); + + // Validate the checkpoints to find a starting point. When this is done, + // there must be at least one entry in the checkpoints (the sentinel), and + // it must be valid. + // + // NB: When this step is done, the curr->child relationship is validated, + // but we haven't read any of child's fields, or checked child's orec. + // Every checkpointed node must be valid at the time of checkpointing. + + // If stack is empty or only holds sentinel, start from {sentinel, root} + if (me->snapshots.size() <= 1) { + parent._obj = sentinel; + child._obj = parent._obj->child[LEFT].sGet(tx); + parent._ver = tx.check_orec(parent._obj); + if (parent._ver == HYPOL::END_OF_TIME) + continue; // retry + me->snapshots.clear(); + me->snapshots.push_back({parent._obj, parent._ver}); + } + // If the stack is larger, we can find the longest valid prefix + else { + // Trim the stack to a set of consistent checkpoints + for (auto cp = me->snapshots.begin(); cp != me->snapshots.end(); ++cp) { + if (!tx.check_continuation(cp->_obj, cp->_ver)) { + me->snapshots.reset(cp - me->snapshots.begin()); + break; // the rest of the checkpoints aren't valid + } + } + // If we don't have more than a sentinel, restart + if (me->snapshots.size() <= 1) + continue; + // Use the key to choose a child of the last good checkpoint + // + // NB: top.key != key, because we never put a matching key into + // snapshots, and if a remove caused a key to change, we'll fail to + // validate that node. + auto top = me->snapshots.top(); + parent = {static_cast(top._obj), top._ver}; + auto parent_key = parent._obj->key.sGet(tx); + child._obj = parent._obj->child[(key < parent_key) ? 0 : 1].sGet(tx); + // Validate that the reads of parent were valid + if (!tx.check_continuation(parent._obj, parent._ver)) + continue; + } + + // Traverse downward from the parent until we find null child or `key` + while (true) { + // nullptr == not found, so stop. Parent was valid, so return it + if (!child._obj) + return {{nullptr, 0}, parent}; + + // It's time to move downward. Read fields of child, then validate it. + // + // NB: we may not use grandchild, but it's better to read it here + auto child_key = child._obj->key.sGet(tx); + auto grandchild = + child._obj->child[(key < child_key) ? LEFT : RIGHT].sGet(tx); + child._ver = tx.check_orec(child._obj); + if (child._ver == HYPOL::END_OF_TIME) + break; // retry + + // If the child key matches, return {child, parent}. We know both are + // valid (parent came from stack; we just checked child) + // + // NB: the snapshot code requires that no node with matching key goes + // into `snapshots` + if (child_key == key) + return {child, parent}; + + // Otherwise add the child to the checkpoint stack and traverse downward + me->snapshots.push_back({child._obj, child._ver}); + parent = child; + child = {grandchild, 0}; + } + } + } + + /// Given a node and its orec value, find the tree node that holds the key + /// that logically succeeds it (i.e., the leftmost descendent of the right + /// child) + /// + /// NB: The caller must ensure that `node` has a valid right child before + /// calling this method + /// + /// @param me The calling thread's descriptor + /// @param node An object and orec value to use as the starting point + /// + /// @return {{found, orec}, {parent, orec}} if no inconsistency occurs + /// {{nullptr, 0}, {nullptr, 0}} on any consistency violation + ret_pair_t get_succ_pair(HYPOL *me, leq_t &node) { + // NB: We expect the successor to be relatively close to the node, so we + // don't bother with checkpoints. However, we are willing to retry, + // since it's unlikely that `node` itself will change. + while (true) { + RSTEP tx(me); + // Ensure `node` is not deleted before reading its fields + if (!tx.check_continuation(node._obj, node._ver)) + return {{nullptr, 0}, {nullptr, 0}}; + + // Read the right child, ensure consistency + leq_t parent = node, child = {node._obj->child[RIGHT].sGet(tx), 0}; + if (!tx.check_continuation(node._obj, node._ver)) + return {{nullptr, 0}, {nullptr, 0}}; + + // Find the leftmost non-null node in the tree rooted at child + while (true) { + auto next = child._obj->child[LEFT].sGet(tx); + child._ver = tx.check_orec(child._obj); + if (child._ver == HYPOL::END_OF_TIME) + break; // retry + // If next is null, `child` is the successor. Otherwise keep traversing + if (!next) + return {child, parent}; + parent = child; + child = {next, 0}; + } + } + } + +public: + /// Construct a tree by creating a sentinel node at the top + rbtree_omap_drop(HYPOL *me, auto *) { + BEGIN_WO(me); + sentinel = new node_t(wo, BLACK, dummy_key, dummy_val, nullptr, 0, nullptr, + nullptr); + } + + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(HYPOL *me, const K &key, V &val) const { + me->snapshots.clear(); + while (true) { + // Get the node that holds `key`, if it is present. If it isn't present, + // we'll get the parent of where it would be. Whatever we get is + // validated, so if it's the sentinel, we're done. + auto curr = get_node(me, key).child; + if (curr._obj == nullptr) + return false; + + // Use an optimistic read if V can be read atomically + if (std::is_scalar::value) { + RSTEP tx(me); + auto *dn = curr._obj; + auto dn_key = dn->key.sGet(tx); + // TODO: Use atomic_ref? + V val_copy = reinterpret_cast *>(&dn->val)->load( + std::memory_order_acquire); + if (!tx.check_continuation(curr._obj, curr._ver)) + continue; + if (dn_key != key) + return false; + val = val_copy; + return true; + } else { + // Using STM here is really easy, and a setjmp is more scalable than + // having to CAS... + // + // TODO: In HandSTM, do we have a hidden requirement that values are + // always scalar? If so, we don't even need this... + BEGIN_RO(me); + if (ro.inheritOrec(curr._obj, curr._ver)) { + val = curr._obj->val.xGet(ro, curr._obj); + return true; + } // else commit and repeat the while loop :) + } + } + } + + // insert a node with k/v as its pair if no such key exists in the tree + bool insert(HYPOL *me, const K &key, V &val) { + me->snapshots.clear(); + while (true) { + // Find insertion point using an STMCAS step. If child isn't null, `key` + // is already present, so we can finish without another STEP or STM + auto [child_, parent_] = get_node(me, key); + if (child_._obj != nullptr) + return false; + + BEGIN_WO(me); + + // Make this WOSTM a continuation of the preceding RSTEP + if (!wo.inheritOrec(parent_._obj, parent_._ver)) + continue; + + // The remaining code needs to know if we're inserting to the left or + // right of leq._obj. If we're at sentinel, it's left. Otherwise, use + // the key to decide. + node_t *curr = parent_._obj; + int cID = curr == sentinel ? 0 : (key < curr->key.xGet(wo, curr) ? 0 : 1); + + node_t *child = + new node_t(wo, RED, key, val, curr, cID, nullptr, nullptr); + curr->child[cID].xSet(wo, curr, child); + + // balance the tree + while (true) { + // Get the parent, grandparent, and their relationship + node_t *parent = child->parent.xGet(wo, child); + int pID = parent->ID.xGet(wo, parent); + node_t *gparent = parent->parent.xGet(wo, parent); + + // Easy exit condition: no more propagation needed + if ((gparent == sentinel) || (BLACK == parent->color.xGet(wo, parent))) + return true; + + // If parent's sibling is also red, we push red up to grandparent + node_t *psib = gparent->child[1 - pID].xGet(wo, gparent); + if ((psib != nullptr) && (RED == psib->color.xGet(wo, psib))) { + parent->color.xSet(wo, parent, BLACK); + psib->color.xSet(wo, psib, BLACK); + gparent->color.xSet(wo, gparent, RED); + child = gparent; + continue; // restart loop at gparent level + } + + int cID = child->ID.xGet(wo, child); + if (cID != pID) { + // set child's child to parent's cID'th child + node_t *baby = child->child[1 - cID].xGet(wo, child); + parent->child[cID].xSet(wo, parent, baby); + if (baby != nullptr) { + baby->parent.xSet(wo, baby, parent); + baby->ID.xSet(wo, baby, cID); + } + // move parent into baby's position as a child of child + child->child[1 - cID].xSet(wo, child, parent); + parent->parent.xSet(wo, parent, child); + parent->ID.xSet(wo, parent, 1 - cID); + // move child into parent's spot as pID'th child of gparent + gparent->child[pID].xSet(wo, gparent, child); + child->parent.xSet(wo, child, gparent); + child->ID.xSet(wo, child, pID); + // now swap child with curr and fall through + node_t *temp = child; + child = parent; + parent = temp; + } + + parent->color.xSet(wo, parent, BLACK); + gparent->color.xSet(wo, gparent, RED); + // promote parent + node_t *ggparent = gparent->parent.xGet(wo, gparent); + int gID = gparent->ID.xGet(wo, gparent); + node_t *ochild = parent->child[1 - pID].xGet(wo, parent); + // make gparent's pIDth child ochild + gparent->child[pID].xSet(wo, gparent, ochild); + if (ochild != nullptr) { + ochild->parent.xSet(wo, ochild, gparent); + ochild->ID.xSet(wo, ochild, pID); + } + // make gparent the 1-pID'th child of parent + parent->child[1 - pID].xSet(wo, parent, gparent); + gparent->parent.xSet(wo, gparent, parent); + gparent->ID.xSet(wo, gparent, 1 - pID); + // make parent the gIDth child of ggparent + ggparent->child[gID].xSet(wo, ggparent, parent); + parent->parent.xSet(wo, parent, ggparent); + parent->ID.xSet(wo, parent, gID); + } + + // now just set the root to black + node_t *root = sentinel->child[0].xGet(wo, sentinel); + if (root->color.xGet(wo, root) != BLACK) + root->color.xSet(wo, root, BLACK); + return true; + } + } + + // remove the node with k as its key if it exists in the tree + bool remove(HYPOL *me, const K &key) { + me->snapshots.clear(); + while (true) { + // Find insertion point using an STMCAS step. If child is null, `key` is + // not present, so we can finish without another STEP or STM. + auto [child_, parent_] = get_node(me, key); + if (child_._obj == nullptr) + return false; + + // If the found node has two children, then we're going to need to swap it + // with its successor. That could mean a big traversal, so let's use an + // RSTEP instead of jumping right into a WOSTM that has to validate its + // read set. + + // First, an RSTEP to see if it has two children + node_t *l = nullptr, *r = nullptr; + { + RSTEP tx(me); + r = child_._obj->child[1].sGet(tx); + l = child_._obj->child[1].sGet(tx); + if (!tx.check_continuation(child_._obj, child_._ver)) + continue; + } + + // If so, then an RSTEP to get the successor and successor parent + leq_t successor = {nullptr, 0}, successor_parent = {nullptr, 0}; + if (r != nullptr && l != nullptr) { + auto [succ, s_parent] = get_succ_pair(me, child_); + if (!succ._obj) + continue; + successor = succ; + successor_parent = s_parent; + } + + { + BEGIN_WO(me); + + // Make this WOSTM a continuation of the preceding RSTEP + if (!wo.inheritOrec(child_._obj, child_._ver)) + continue; + + // NB: We get segfaults if we don't also inheritOrec on the parent. We + // need to investigate this further. + if (!wo.inheritOrec(parent_._obj, parent_._ver)) + continue; + + // find key + node_t *curr = child_._obj; + + // If `curr` has two children, we need to swap it with its successor + if (l != nullptr && r != nullptr) { + // First we have to make `wo` a continuation of the other RSTEP + if (!wo.inheritOrec(successor._obj, successor._ver)) + continue; + if (!wo.inheritOrec(successor_parent._obj, successor_parent._ver)) + continue; + + curr->key.xSet(wo, curr, + successor._obj->key.xGet(wo, successor._obj)); + curr->val.xSet(wo, curr, + successor._obj->val.xGet(wo, successor._obj)); + curr = successor._obj; + parent_ = successor_parent; + } + + // extract x from the tree and prep it for deletion + node_t *parent = parent_._obj; + node_t *child = + curr->child[(curr->child[0].xGet(wo, curr) != nullptr) ? 0 : 1] + .xGet(wo, curr); + int xID = curr->ID.xGet(wo, curr); + parent->child[xID].xSet(wo, parent, child); + if (child != nullptr) { + child->parent.xSet(wo, child, parent); + child->ID.xSet(wo, child, xID); + } + + // fix black height violations + if ((BLACK == curr->color.xGet(wo, curr)) && (child != nullptr)) { + if (RED == child->color.xGet(wo, child)) { + curr->color.xSet(wo, curr, RED); + child->color.xSet(wo, child, BLACK); + } + } + + // rebalance... be sure to save the deletion target! + node_t *to_delete = curr; + while (true) { + parent = curr->parent.xGet(wo, curr); + if ((parent == sentinel) || (RED == curr->color.xGet(wo, curr))) + break; + int cID = curr->ID.xGet(wo, curr); + node_t *sibling = parent->child[1 - cID].xGet(wo, parent); + + // we'd like y's sibling s to be black + // if it's not, promote it and recolor + if (RED == sibling->color.xGet(wo, sibling)) { + /* + Bp Bs + / \ / \ + By Rs => Rp B2 + / \ / \ + B1 B2 By B1 + */ + parent->color.xSet(wo, parent, RED); + sibling->color.xSet(wo, sibling, BLACK); + // promote sibling + node_t *gparent = parent->parent.xGet(wo, parent); + int pID = parent->ID.xGet(wo, parent); + node_t *nephew = sibling->child[cID].xGet(wo, sibling); + // set nephew as 1-cID child of parent + parent->child[1 - cID].xSet(wo, parent, nephew); + nephew->parent.xSet(wo, nephew, parent); + nephew->ID.xSet(wo, nephew, 1 - cID); + // make parent the cID child of the sibling + sibling->child[cID].xSet(wo, sibling, parent); + parent->parent.xSet(wo, parent, sibling); + parent->ID.xSet(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].xSet(wo, gparent, sibling); + sibling->parent.xSet(wo, sibling, gparent); + sibling->ID.xSet(wo, sibling, pID); + // reset sibling + sibling = nephew; + } + + // Handle when the far nephew is red + node_t *n = sibling->child[1 - cID].xGet(wo, sibling); + if ((n != nullptr) && (RED == (n->color.xGet(wo, n)))) { + /* + ?p ?s + / \ / \ + By Bs => Bp Bn + / \ / \ + ?1 Rn By ?1 + */ + sibling->color.xSet(wo, sibling, parent->color.xGet(wo, parent)); + parent->color.xSet(wo, parent, BLACK); + n->color.xSet(wo, n, BLACK); + // promote sibling + node_t *gparent = parent->parent.xGet(wo, parent); + int pID = parent->ID.xGet(wo, parent); + node_t *nephew = sibling->child[cID].xGet(wo, sibling); + // make nephew the 1-cID child of parent + parent->child[1 - cID].xSet(wo, parent, nephew); + if (nephew != nullptr) { + nephew->parent.xSet(wo, nephew, parent); + nephew->ID.xSet(wo, nephew, 1 - cID); + } + // make parent the cID child of the sibling + sibling->child[cID].xSet(wo, sibling, parent); + parent->parent.xSet(wo, parent, sibling); + parent->ID.xSet(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].xSet(wo, gparent, sibling); + sibling->parent.xSet(wo, sibling, gparent); + sibling->ID.xSet(wo, sibling, pID); + break; // problem solved + } + + n = sibling->child[cID].xGet(wo, sibling); + if ((n != nullptr) && (RED == (n->color.xGet(wo, n)))) { + /* + ?p ?p + / \ / \ + By Bs => By Bn + / \ \ + Rn B1 Rs + \ + B1 + */ + sibling->color.xSet(wo, sibling, RED); + n->color.xSet(wo, n, BLACK); + // promote n + node_t *gneph = n->child[1 - cID].xGet(wo, n); + // make gneph the cID child of sibling + sibling->child[cID].xSet(wo, sibling, gneph); + if (gneph != nullptr) { + gneph->parent.xSet(wo, gneph, sibling); + gneph->ID.xSet(wo, gneph, cID); + } + // make sibling the 1-cID child of n + n->child[1 - cID].xSet(wo, n, sibling); + sibling->parent.xSet(wo, sibling, n); + sibling->ID.xSet(wo, sibling, 1 - cID); + // make n the 1-cID child of parent + parent->child[1 - cID].xSet(wo, parent, n); + n->parent.xSet(wo, n, parent); + n->ID.xSet(wo, n, 1 - cID); + // swap sibling and `n` + node_t *temp = sibling; + sibling = n; + n = temp; + + // now the far nephew is red... copy of code from above + sibling->color.xSet(wo, sibling, parent->color.xGet(wo, parent)); + parent->color.xSet(wo, parent, BLACK); + n->color.xSet(wo, n, BLACK); + // promote sibling + node_t *gparent = parent->parent.xGet(wo, parent); + int pID = parent->ID.xGet(wo, parent); + node_t *nephew = sibling->child[cID].xGet(wo, sibling); + // make nephew the 1-cID child of parent + parent->child[1 - cID].xSet(wo, parent, nephew); + if (nephew != nullptr) { + nephew->parent.xSet(wo, nephew, parent); + nephew->ID.xSet(wo, nephew, 1 - cID); + } + // make parent the cID child of the sibling + sibling->child[cID].xSet(wo, sibling, parent); + parent->parent.xSet(wo, parent, sibling); + parent->ID.xSet(wo, parent, cID); + // make sibling the pID child of gparent + gparent->child[pID].xSet(wo, gparent, sibling); + sibling->parent.xSet(wo, sibling, gparent); + sibling->ID.xSet(wo, sibling, pID); + + break; // problem solved + } + + /* + ?p ?p + / \ / \ + Bx Bs => Bp Rs + / \ / \ + B1 B2 B1 B2 + */ + + sibling->color.xSet(wo, sibling, RED); // propagate upwards + + // advance to parent and balance again + curr = parent; + } + + // if curr was red, this fixes the balance + if (curr->color.xGet(wo, curr) == RED) + curr->color.xSet(wo, curr, BLACK); + + // free the node and return + wo.reclaim(to_delete); + + return true; + } + } + } +}; diff --git a/artifact/ds/include/ca_umap_list_adapter.h b/artifact/ds/include/ca_umap_list_adapter.h new file mode 100644 index 0000000..a71edd7 --- /dev/null +++ b/artifact/ds/include/ca_umap_list_adapter.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include +#include + +// STM Non-resizable Hash Table + +/// A straightforward non-resizable hashtable. This map supports +/// get(), insert(), remove(), and size() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param STMCAS The STMCAS implementation (PO or PS) +/// @param OMAP An ordered map type to use as each bucket +/// +/// NB: OMAP must be templated on +template +class ca_umap_list_adapter_t { + OMAP **buckets; // The OMAPs that act as the buckets in the table. + const uint64_t num_buckets; // The number of buckets in the table. + +public: + /// Create a non-resizable hash table with the specified number of buckets. + /// + /// @param me The operation that is constructing the table. + /// @param cfg A configuration object with a `buckets` field + ca_umap_list_adapter_t(STMCAS *me, auto *cfg) : num_buckets(cfg->buckets) { + buckets = (OMAP **)malloc(num_buckets * sizeof(OMAP *)); + + // Fill the "buckets" vector with singly-linked lists. + for (unsigned int i = 0; i < num_buckets; ++i) + buckets[i] = new OMAP(me, cfg); + } + +private: + std::hash pre_hash; + + /// Get the index of the bucket where the provided key belongs + /// + /// @param me The calling thread's descriptor + /// @param key The key to hash. + /// + /// @return The hashed value of the key, modded by the number of buckets + int hash(STMCAS *me, const K key) { + return me->hash(pre_hash(key)) % num_buckets; + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(STMCAS *me, const K &key, V &val) { + return buckets[hash(me, key)]->get(me, key, val); + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(STMCAS *me, const K &key, V &val) { + return buckets[hash(me, key)]->insert(me, key, val); + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(STMCAS *me, const K &key) { + return buckets[hash(me, key)]->remove(me, key); + } +}; diff --git a/artifact/ds/xSTM/dlist_omap.h b/artifact/ds/xSTM/dlist_omap.h new file mode 100644 index 0000000..ad02fc2 --- /dev/null +++ b/artifact/ds/xSTM/dlist_omap.h @@ -0,0 +1,174 @@ +#pragma once + +#include "../../policies/xSTM/common/tm_api.h" + +// NB: We need an operator new(). It can just forward to malloc() +TX_RENAME(_Znwm) void *my_new(std::size_t size) { + void *ptr = malloc(size); + return ptr; +} + +/// An ordered map, implemented as a doubly-linked list. This map supports +/// get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param DESCRIPTOR A thread descriptor type, for safe memory reclamation +template class dlist_omap { + + /// A list node. It has prev and next pointers, but no key or value. It's + /// useful for sentinels, so that K and V don't have to be default + /// constructable. + struct node_t { + node_t *prev; // Pointer to predecessor + node_t *next; // Pointer to successor + + /// Construct a node + node_t() : prev(nullptr), next(nullptr) { TX_CTOR; } + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~node_t() {} + }; + + /// A list node that also has a key and value. Note that keys are const. + struct data_t : public node_t { + const K key; // The key of this key/value pair + V val; // The value of this key/value pair + + /// Construct a data_t + /// + /// @param _key The key that is stored in this node + /// @param _val The value that is stored in this node + data_t(const K &_key, const V &_val) : node_t(), key(_key), val(_val) { + TX_CTOR; + } + + /// Destructor is a no-op, but it needs to be virtual because of inheritance + virtual ~data_t() {} + }; + + node_t *const head; // The list head pointer + node_t *const tail; // The list tail pointer + +public: + /// Default construct a list by constructing and connecting two sentinel nodes + /// + /// @param me The operation that is constructing the list + /// @param cfg A configuration object + dlist_omap(DESCRIPTOR *, auto *cfg) : head(new node_t()), tail(new node_t()) { + head->next = tail; + tail->prev = head; + } + +private: + /// get_leq is an inclusive predecessor query that returns the largest node + /// whose key is <= the provided key. It can return the head sentinel, but + /// not the tail sentinel. + /// + /// @param key The key for which we are doing a predecessor query. + /// + /// @return The node that was found + node_t *get_leq(const K key) { + // Start at the head; read the next now, to avoid reading it in multiple + // iterations of the loop + node_t *curr = head; + auto *next = curr->next; + + // Starting at `next`, search for key. Breaking out of this will take us + // back to the top of the function. + while (true) { + // Case 1: `next` is tail --> stop the search at curr + if (next == tail) + return curr; + + // read next's `next` and `key` + auto next_next = next->next; + auto nkey = static_cast(next)->key; + + // Case 2: `next` is a data node: stop if next->key >= key + if (nkey > key) + return curr; + if (nkey == key) + return next; + + // Case 3: keep traversing to `next` + curr = next; + next = next_next; + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me Unused thread context + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(DESCRIPTOR *, const K &key, V &val) { + TX_RAII; + // get_leq will use a read-only transaction to find the largest node with + // a key <= `key`. + auto n = get_leq(key); + + // Since we have EBR, we can read n.key without validating and fast-fail + // on key-not-found + if (n == head || static_cast(n)->key != key) + return false; + + // NB: given EBR, we don't need to worry about n._obj being deleted, so + // we don't need to validate before looking at the value + data_t *dn = static_cast(n); + val = dn->val; + return true; + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me Unused thread context + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(DESCRIPTOR *, const K &key, V &val) { + TX_RAII; + auto n = get_leq(key); + if (n != head && static_cast(n)->key == key) + return false; + + auto next = n->next; + + // stitch in a new node + data_t *new_dn = new data_t(key, val); + new_dn->next = next; + new_dn->prev = n; + n->next = new_dn; + next->prev = new_dn; + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me Unused thread context + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(DESCRIPTOR *, const K &key) { + TX_RAII; + auto n = get_leq(key); + if (n == head || static_cast(n)->key != key) + return false; + + // unstitch it + auto pred = n->prev, succ = n->next; + pred->next = succ; + succ->prev = pred; + delete (n); + return true; + } +}; diff --git a/artifact/ds/xSTM/ibst_omap.h b/artifact/ds/xSTM/ibst_omap.h new file mode 100644 index 0000000..cd234fa --- /dev/null +++ b/artifact/ds/xSTM/ibst_omap.h @@ -0,0 +1,260 @@ +#pragma once + +#include "../../policies/xSTM/common/tm_api.h" + +// NB: We need an operator new(). It can just forward to malloc() +TX_RENAME(_Znwm) void *my_new(std::size_t size) { + void *ptr = malloc(size); + return ptr; +} + +/// An ordered map, implemented as an unbalanced, internal binary search tree. +/// This map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param DESCRIPTOR A thread descriptor type, for safe memory reclamation +template class ibst_omap { + + /// An easy-to-remember way of indicating the left and right children + enum DIRS { LEFT = 0, RIGHT = 1 }; + + /// node_t is the base type for all tree nodes. It doesn't have key/value + /// fields. + struct node_t { + /// The node's children. Be sure to use LEFT and RIGHT to index it + node_t *children[2]; + + /// Construct a node_t. This should only be called from a writer + /// transaction + /// + /// @param _left The left child of this node + /// @param _right The right child of this node + node_t(node_t *_left = nullptr, node_t *_right = nullptr) { + TX_CTOR; + children[LEFT] = _left; + children[RIGHT] = _right; + } + }; + + /// A pair holding a child node and its parent + struct ret_pair_t { + node_t *child; // The child + node_t *parent; // The parent of that child + }; + + /// Our tree uses a sentinel root node, so that we always have a valid node + /// for which to compute an orec. The sentinel's *LEFT* child is the true + /// root of the tree. That is, logically sentinel has the value "TOP". + node_t *sentinel; + + /// data_t is the type for all internal and leaf nodes in the data structure. + /// It extends the base type with a key and value. + /// + /// NB: keys are *not* const, because we want to overwrite nodes instead of + /// swapping them + struct data_t : public node_t { + K key; // The key stored in this node + V val; // The value stored in this node + + /// Construct a node + /// + /// @param _left left child of the node + /// @param _right right child of the node + /// @param _key the key of the node + /// @param _val the value of the node + data_t(node_t *_left, node_t *_right, const K &_key, V &_val) + : node_t(_left, _right), key(_key), val(_val) { + TX_CTOR; + } + }; + +public: + /// Default construct an empty tree + /// + /// @param me The operation that is constructing the list + /// @param cfg A configuration object + ibst_omap(DESCRIPTOR *, auto *cfg) { + // NB: Even though the constructor is operating on private data, it needs a + // TM context in order to use tm_fields + sentinel = new node_t(); + } + +private: + /// Search for a `key` in the tree, and return the node holding it, as well + /// as the node's parent. If the key is not found, return null, and the + /// node that ought to be parent of the (not found) `key`. + /// + /// NB: The caller is responsible for clearing the checkpoint stack before + /// calling get_node(). + /// + /// @param key The key to search for + /// + /// @return {found, parent} if `key` is in the tree + /// {nullptr, parent} if `key` is not in the tree + ret_pair_t get_node(const K &key) { + // Traverse downward to the target node: + node_t *parent = sentinel; + node_t *child = parent->children[LEFT]; + + // Traverse downward from the parent until we find null child or `key` + while (true) { + // nullptr == not found, so stop. We know parent was valid, so we can + // just return it + if (!child) + return {nullptr, parent}; + + // It's time to move downward. Read fields of child and grandchild + // + // NB: we may not use grandchild, but it's better to read it here + auto child_key = static_cast(child)->key; + auto grandchild = child->children[(key < child_key) ? LEFT : RIGHT]; + + // If the child key matches, return {child, parent}. We know both are + // valid (parent came from stack; we just checked child) + // + // NB: the snapshotting code requires that no node with matching key + // goes into `snapshots` + if (child_key == key) + return {child, parent}; + + // Otherwise traverse downward + parent = child; + child = grandchild; + } + } + + /// Given a node and its orec value, find the tree node that holds the key + /// that logically succeeds it (i.e., the leftmost descendent of the right + /// child) + /// + /// NB: The caller must ensure that `node` has a valid right child before + /// calling this method + /// + /// @param node An object and orec value to use as the starting point + /// + /// @return {{found, orec}, {parent, orec}} if no inconsistency occurs + /// {{nullptr, 0}, {nullptr, 0}} on any consistency violation + ret_pair_t get_succ_pair(node_t *node) { + // Read the right child + node_t *parent = node, *child = node->children[RIGHT]; + + // Find the leftmost non-null node in the tree rooted at child + while (true) { + auto next = child->children[LEFT]; + // If next is null, `child` is the successor. Otherwise keep traversing + if (!next) + return {child, parent}; + parent = child; + child = next; + } + } + +public: + /// Search the data structure for a node with key `key`. If not found, return + /// false. If found, return true, and set `val` to the value associated with + /// `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key to search + /// @param val A ref parameter for returning key's value, if found + /// + /// @return True if the key is found, false otherwise. The reference + /// parameter `val` is only valid when the return value is true. + bool get(DESCRIPTOR *, const K &key, V &val) { + TX_RAII; + // Get the node that holds `key`, if it is present, and also its parent. + // If it isn't present, we'll get a null pointer. That corresponds to a + // consistent read of the parent, which means we already linearized and + // we're done + auto [curr, _] = get_node(key); + if (curr == nullptr) + return false; + + // read the value + auto dn = static_cast(curr); + val = dn->val; + return true; + } + + /// Create a mapping from the provided `key` to the provided `val`, but only + /// if no such mapping already exists. This method does *not* have upsert + /// behavior for keys already present. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to create + /// @param val The value for the mapping to create + /// + /// @return True if the value was inserted, false otherwise. + bool insert(DESCRIPTOR *, const K &key, V &val) { + TX_RAII; + auto [child, parent] = get_node(key); + if (child) + return false; + // We must have a null child and a valid parent. If it's sentinel, we + // must insert as LEFT. Otherwise, compute which child to set. + auto cID = (parent == sentinel ? LEFT : RIGHT) & + (key > static_cast(parent)->key); + auto new_child = new data_t(nullptr, nullptr, key, val); + parent->children[cID] = new_child; + return true; + } + + /// Clear the mapping involving the provided `key`. + /// + /// @param me The calling thread's descriptor + /// @param key The key for the mapping to eliminate + /// + /// @return True if the key was found and removed, false otherwise + bool remove(DESCRIPTOR *, const K &key) { + TX_RAII; + auto [target, parent] = get_node(key); + if (target == nullptr) + return false; + + // Read the target node's children + data_t *t_child[2]; + t_child[RIGHT] = static_cast(target->children[RIGHT]); + t_child[LEFT] = static_cast(target->children[LEFT]); + + // If either child is null, and if the parent is still valid, then we can + // unstitch the target, link the parent to a grandchild and we're done. + if (!t_child[LEFT] || !t_child[RIGHT]) { + // Acquire the (possibly null) grandchild to link to the parent + auto gID = t_child[LEFT] ? LEFT : RIGHT; + + // Which child of the parent is target? + auto cID = parent->children[LEFT] == target ? LEFT : RIGHT; + + // Unstitch and reclaim + parent->children[cID] = t_child[gID]; + delete (target); + return true; + } + + // `target` has two children. WLOG, the leftmost descendent of the right + // child is `target`'s successor, and must have at most one child. We + // want to put that node's key and value into `target`, and then remove + // that node by setting its parent's LEFT to its RIGHT (which might be + // null). + auto [succ, s_parent] = get_succ_pair(target); + + // If target's successor is target's right child, then target._ver must + // equal s_parent._ver. As long as we lock target before we try + // to lock s_parent, we'll get the check for free. + + // Copy `succ`'s key/value into `target` + static_cast(target)->key = static_cast(succ)->key; + static_cast(target)->val = static_cast(succ)->val; + + // Unstitch `succ` by setting its parent's left to its right + // Case 1: there are intermediate nodes between target and successor + if (s_parent != target) + s_parent->children[LEFT] = succ->children[RIGHT]; + // Case 2: target is successor's parent + else + s_parent->children[RIGHT] = succ->children[RIGHT]; + delete (succ); + return true; + } +}; diff --git a/artifact/ds/xSTM/rbtree_omap.h b/artifact/ds/xSTM/rbtree_omap.h new file mode 100644 index 0000000..e964f50 --- /dev/null +++ b/artifact/ds/xSTM/rbtree_omap.h @@ -0,0 +1,370 @@ +#pragma once + +#include "../../policies/xSTM/common/tm_api.h" + +// NB: We need an operator new(). It can just forward to malloc() +TX_RENAME(_Znwm) void *my_new(std::size_t size) { + void *ptr = malloc(size); + return ptr; +} + +/// An ordered map, implemented as a balanced, internal binary search tree. This +/// map supports get(), insert(), and remove() operations. +/// +/// @param K The type of the keys stored in this map +/// @param V The type of the values stored in this map +/// @param DESCRIPTOR A thread descriptor type, for safe memory reclamation +/// @param dummy_key A default key to use +/// @param dummy_val A default value to use +template +class rbtree_omap { + /// An enum for node colors + enum Color { RED, BLACK }; + + /// nodes in a red/black tree + struct node_t { + K key; // Key stored at this node + V val; // Value stored at this node + Color color; // color (RED or BLACK) + node_t *parent; // pointer to parent + int ID; // 0 or 1 to indicate if left or right child + node_t *child[2]; // pointers to children + + /// basic constructor + node_t(Color color, K key, V val, node_t *parent, long ID, node_t *child0, + node_t *child1) + : key(key), val(val), color(color), parent(parent), ID(ID) { + TX_CTOR; + child[0] = child0; + child[1] = child1; + } + }; + + node_t *sentinel; // The (sentinel) root node of the tree + +public: + /// Construct a tree by creating a sentinel node at the head + rbtree_omap(DESCRIPTOR *, auto *) + : sentinel(new node_t(BLACK, dummy_key, dummy_val, nullptr, 0, nullptr, + nullptr)) {} + + // binary search for the node that has v as its value + bool get(DESCRIPTOR *, const K &key, V &val) const { + void *dummy; + bool res = false; + TX_PRIVATE_STACK_REGION(&dummy); + { + TX_RAII; + const node_t *curr = sentinel->child[0]; + while (curr != nullptr && curr->key != key) + curr = curr->child[(key < curr->key) ? 0 : 1]; + res = (curr != nullptr) && (curr->key == key); + if (res) + val = curr->val; + } + return res; + } + + // insert a node with k/v as its pair if no such key exists in the tree + bool insert(DESCRIPTOR *, const K &key, V &val) { + void *dummy; + bool res = false; + TX_PRIVATE_STACK_REGION(&dummy); + { + TX_RAII; + // find insertion point + node_t *curr = sentinel; + int cID = 0; + node_t *child = curr->child[cID]; + while (child != nullptr) { + long ckey = child->key; + if (ckey == key) + return false; + cID = key < ckey ? 0 : 1; + curr = child; + child = curr->child[cID]; + } + + // make a red node and connect it to `curr` + res = true; + child = new node_t(RED, key, val, curr, cID, nullptr, nullptr); + curr->child[cID] = child; + + // balance the tree + while (true) { + // Get the parent, grandparent, and their relationship + node_t *parent = child->parent; + int pID = parent->ID; + node_t *gparent = parent->parent; + + // Easy exit condition: no more propagation needed + if ((gparent == sentinel) || (BLACK == parent->color)) + break; + + // If parent's sibling is also red, we push red up to grandparent + node_t *psib = gparent->child[1 - pID]; + if ((psib != nullptr) && (RED == psib->color)) { + parent->color = BLACK; + psib->color = BLACK; + gparent->color = RED; + child = gparent; + continue; // restart loop at gparent level + } + + int cID = child->ID; + if (cID != pID) { + // set child's child to parent's cID'th child + node_t *baby = child->child[1 - cID]; + parent->child[cID] = baby; + if (baby != nullptr) { + baby->parent = parent; + baby->ID = cID; + } + // move parent into baby's position as a child of child + child->child[1 - cID] = parent; + parent->parent = child; + parent->ID = 1 - cID; + // move child into parent's spot as pID'th child of gparent + gparent->child[pID] = child; + child->parent = gparent; + child->ID = pID; + // now swap child with curr and fall through + node_t *temp = child; + child = parent; + parent = temp; + } + + parent->color = BLACK; + gparent->color = RED; + // promote parent + node_t *ggparent = gparent->parent; + int gID = gparent->ID; + node_t *ochild = parent->child[1 - pID]; + // make gparent's pIDth child ochild + gparent->child[pID] = ochild; + if (ochild != nullptr) { + ochild->parent = gparent; + ochild->ID = pID; + } + // make gparent the 1-pID'th child of parent + parent->child[1 - pID] = gparent; + gparent->parent = parent; + gparent->ID = 1 - pID; + // make parent the gIDth child of ggparent + ggparent->child[gID] = parent; + parent->parent = ggparent; + parent->ID = gID; + } + + // now just set the root to black + node_t *root = sentinel->child[0]; + if (root->color != BLACK) + root->color = BLACK; + } + + return res; + } + + // remove the node with k as its key if it exists in the tree + bool remove(DESCRIPTOR *, const K &key) { + TX_RAII; + // find key + node_t *curr = sentinel->child[0]; + + while (curr != nullptr) { + int ckey = curr->key; + if (ckey == key) + break; + curr = curr->child[key < ckey ? 0 : 1]; + } + + // if we didn't find v, we're done + if (curr == nullptr) + return false; + + // If `curr` has two children, we need to swap it with its successor + if ((curr->child[1] != nullptr) && ((curr->child[0]) != nullptr)) { + node_t *leftmost = curr->child[1]; + while (leftmost->child[0] != nullptr) + leftmost = leftmost->child[0]; + curr->key = leftmost->key; + curr->val = leftmost->val; + curr = leftmost; + } + + // extract x from the tree and prep it for deletion + node_t *parent = curr->parent; + node_t *child = curr->child[(curr->child[0] != nullptr) ? 0 : 1]; + int xID = curr->ID; + parent->child[xID] = child; + if (child != nullptr) { + child->parent = parent; + child->ID = xID; + } + + // fix black height violations + if ((BLACK == curr->color) && (child != nullptr)) { + if (RED == child->color) { + curr->color = RED; + child->color = BLACK; + } + } + + // rebalance... be sure to save the deletion target! + node_t *to_delete = curr; + while (true) { + parent = curr->parent; + if ((parent == sentinel) || (RED == curr->color)) + break; + int cID = curr->ID; + node_t *sibling = parent->child[1 - cID]; + + // we'd like y's sibling s to be black + // if it's not, promote it and recolor + if (RED == sibling->color) { + /* + Bp Bs + / \ / \ + By Rs => Rp B2 + / \ / \ + B1 B2 By B1 + */ + parent->color = RED; + sibling->color = BLACK; + // promote sibling + node_t *gparent = parent->parent; + int pID = parent->ID; + node_t *nephew = sibling->child[cID]; + // set nephew as 1-cID child of parent + parent->child[1 - cID] = nephew; + nephew->parent = parent; + nephew->ID = (1 - cID); + // make parent the cID child of the sibling + sibling->child[cID] = parent; + parent->parent = sibling; + parent->ID = cID; + // make sibling the pID child of gparent + gparent->child[pID] = sibling; + sibling->parent = gparent; + sibling->ID = pID; + // reset sibling + sibling = nephew; + } + + // Handle when the far nephew is red + node_t *n = sibling->child[1 - cID]; + if ((n != nullptr) && (RED == (n->color))) { + /* + ?p ?s + / \ / \ + By Bs => Bp Bn + / \ / \ + ?1 Rn By ?1 + */ + sibling->color = parent->color; + parent->color = BLACK; + n->color = BLACK; + // promote sibling + node_t *gparent = parent->parent; + int pID = parent->ID; + node_t *nephew = sibling->child[cID]; + // make nephew the 1-cID child of parent + parent->child[1 - cID] = nephew; + if (nephew != nullptr) { + nephew->parent = parent; + nephew->ID = 1 - cID; + } + // make parent the cID child of the sibling + sibling->child[cID] = parent; + parent->parent = sibling; + parent->ID = cID; + // make sibling the pID child of gparent + gparent->child[pID] = sibling; + sibling->parent = gparent; + sibling->ID = pID; + break; // problem solved + } + + n = sibling->child[cID]; + if ((n != nullptr) && (RED == (n->color))) { + /* + ?p ?p + / \ / \ + By Bs => By Bn + / \ \ + Rn B1 Rs + \ + B1 + */ + sibling->color = RED; + n->color = BLACK; + // promote n + node_t *gneph = n->child[1 - cID]; + // make gneph the cID child of sibling + sibling->child[cID] = gneph; + if (gneph != nullptr) { + gneph->parent = sibling; + gneph->ID = cID; + } + // make sibling the 1-cID child of n + n->child[1 - cID] = sibling; + sibling->parent = n; + sibling->ID = 1 - cID; + // make n the 1-cID child of parent + parent->child[1 - cID] = n; + n->parent = parent; + n->ID = 1 - cID; + // swap sibling and `n` + node_t *temp = sibling; + sibling = n; + n = temp; + + // now the far nephew is red... copy of code from above + sibling->color = (parent->color); + parent->color = BLACK; + n->color = BLACK; + // promote sibling + node_t *gparent = parent->parent; + int pID = parent->ID; + node_t *nephew = sibling->child[cID]; + // make nephew the 1-cID child of parent + parent->child[1 - cID] = nephew; + if (nephew != nullptr) { + nephew->parent = parent; + nephew->ID = 1 - cID; + } + // make parent the cID child of the sibling + sibling->child[cID] = parent; + parent->parent = sibling; + parent->ID = cID; + // make sibling the pID child of gparent + gparent->child[pID] = sibling; + sibling->parent = gparent; + sibling->ID = pID; + + break; // problem solved + } + + /* + ?p ?p + / \ / \ + Bx Bs => Bp Rs + / \ / \ + B1 B2 B1 B2 + */ + + sibling->color = RED; // propagate upwards + + // advance to parent and balance again + curr = parent; + } + + // if curr was red, this fixes the balance + curr->color = BLACK; + + // free the node and return + free(to_delete); + + return true; + } +}; diff --git a/artifact/ds/xSTM/rbtree_tl2_omap.h b/artifact/ds/xSTM/rbtree_tl2_omap.h new file mode 100644 index 0000000..e1142ae --- /dev/null +++ b/artifact/ds/xSTM/rbtree_tl2_omap.h @@ -0,0 +1,428 @@ +/* ============================================================================= + * + * rbtree.h + * -- Red-black balanced binary search tree + * + * ============================================================================= + * + * Copyright (C) Sun Microsystems Inc., 2006. All Rights Reserved. + * Authors: Dave Dice, Nir Shavit, Ori Shalev. + * + * STM: Transactional Locking for Disjoint Access Parallelism + * + * Transactional Locking II, + * Dave Dice, Ori Shalev, Nir Shavit + * DISC 2006, Sept 2006, Stockholm, Sweden. + */ + +#pragma once + +#include "../../policies/xSTM/common/tm_api.h" + +template +class rbtree_tl2_omap { + + static const int RED = 0; + static const int BLACK = 1; + + struct node_t { + K key; + V val; + node_t *p; + node_t *l; + node_t *r; + long c; + char dummy[64]; + }; + + node_t *root; + char dummy[64]; + + node_t *lookup(K k) { + node_t *p = this->root; + while (p != nullptr) { + if (k == p->key) { + return p; + } + p = (k < p->key) ? p->l : p->r; + } + return nullptr; + } + + void rotateLeft(node_t *x) { + node_t *r = x->r; + node_t *rl = r->l; + x->r = rl; + if (rl != nullptr) { + rl->p = (x); + } + + node_t *xp = (((((x))->p))); + ((r)->p) = (xp); + if (xp == nullptr) { + ((this)->root) = (r); + } else if ((((((xp))->l))) == x) { + ((xp)->l) = (r); + } else { + ((xp)->r) = (r); + } + ((r)->l) = (x); + ((x)->p) = (r); + } + + void rotateRight(node_t *x) { + node_t *l = (((((x))->l))); + node_t *lr = (((((l))->r))); + ((x)->l) = (lr); + if (lr != nullptr) { + ((lr)->p) = (x); + } + node_t *xp = (((((x))->p))); + ((l)->p) = (xp); + if (xp == nullptr) { + ((this)->root) = (l); + } else if ((((((xp))->r))) == x) { + ((xp)->r) = (l); + } else { + ((xp)->l) = (l); + } + ((l)->r) = (x); + ((x)->p) = (l); + } + + node_t *parentOf(node_t *n) { return (n ? (((((n))->p))) : nullptr); } + + node_t *leftOf(node_t *n) { return (n ? (((((n))->l))) : nullptr); } + + node_t *rightOf(node_t *n) { return (n ? (((((n))->r))) : nullptr); } + + long colorOf(node_t *n) { return (n ? (long)(((((n))->c))) : BLACK); } + + void setColor(node_t *n, long c) { + if (n != nullptr) { + ((n)->c) = (c); + } + } + + void fixAfterInsertion(node_t *x) { + ((x)->c) = (RED); + while (x != nullptr && x != (((((this))->root)))) { + node_t *xp = (((((x))->p))); + if (((xp)->c) != RED) { + break; + } + + if (parentOf(x) == leftOf(parentOf(parentOf(x)))) { + node_t *y = rightOf(parentOf(parentOf(x))); + if (colorOf(y) == RED) { + setColor(parentOf(x), BLACK); + setColor(y, BLACK); + setColor(parentOf(parentOf(x)), RED); + x = parentOf(parentOf(x)); + } else { + if (x == rightOf(parentOf(x))) { + x = parentOf(x); + rotateLeft(x); + } + setColor(parentOf(x), BLACK); + setColor(parentOf(parentOf(x)), RED); + if (parentOf(parentOf(x)) != nullptr) { + rotateRight(parentOf(parentOf(x))); + } + } + } else { + node_t *y = leftOf(parentOf(parentOf(x))); + if (colorOf(y) == RED) { + setColor(parentOf(x), BLACK); + setColor(y, BLACK); + setColor(parentOf(parentOf(x)), RED); + x = parentOf(parentOf(x)); + } else { + if (x == leftOf(parentOf(x))) { + x = parentOf(x); + rotateRight(x); + } + setColor(parentOf(x), BLACK); + setColor(parentOf(parentOf(x)), RED); + if (parentOf(parentOf(x)) != nullptr) { + rotateLeft(parentOf(parentOf(x))); + } + } + } + } + node_t *ro = (((((this))->root))); + if (((ro)->c) != BLACK) { + ((ro)->c) = (BLACK); + } + } + + node_t *insertIt(K k, V v, node_t *n) { + node_t *t = (((((this))->root))); + if (t == nullptr) { + if (n == nullptr) { + return nullptr; + } + + ((n)->l) = (nullptr); + ((n)->r) = (nullptr); + ((n)->p) = (nullptr); + ((n)->key) = (k); + ((n)->val) = (v); + ((n)->c) = (BLACK); + ((this)->root) = (n); + return nullptr; + } + + for (;;) { + if (k == t->key) { + return t; + } else if (k < t->key) { + node_t *tl = (((((t))->l))); + if (tl != nullptr) { + t = tl; + } else { + ((n)->l) = (nullptr); + ((n)->r) = (nullptr); + ((n)->key) = (k); + ((n)->val) = (v); + ((n)->p) = (t); + ((t)->l) = (n); + fixAfterInsertion(n); + return nullptr; + } + } else { + node_t *tr = (((((t))->r))); + if (tr != nullptr) { + t = tr; + } else { + ((n)->l) = (nullptr); + ((n)->r) = (nullptr); + ((n)->key) = (k); + ((n)->val) = (v); + ((n)->p) = (t); + ((t)->r) = (n); + fixAfterInsertion(n); + return nullptr; + } + } + } + } + + node_t *successor(node_t *t) { + if (t == nullptr) { + return nullptr; + } else if ((((((t))->r))) != nullptr) { + node_t *p = (((((t))->r))); + while ((((((p))->l))) != nullptr) { + p = (((((p))->l))); + } + return p; + } else { + node_t *p = (((((t))->p))); + node_t *ch = t; + while (p != nullptr && ch == (((((p))->r)))) { + ch = p; + p = (((((p))->p))); + } + return p; + } + } + + void fixAfterDeletion(node_t *x) { + while (x != (((((this))->root))) && colorOf(x) == BLACK) { + if (x == leftOf(parentOf(x))) { + node_t *sib = rightOf(parentOf(x)); + if (colorOf(sib) == RED) { + setColor(sib, BLACK); + setColor(parentOf(x), RED); + rotateLeft(parentOf(x)); + sib = rightOf(parentOf(x)); + } + if (colorOf(leftOf(sib)) == BLACK && colorOf(rightOf(sib)) == BLACK) { + setColor(sib, RED); + x = parentOf(x); + } else { + if (colorOf(rightOf(sib)) == BLACK) { + setColor(leftOf(sib), BLACK); + setColor(sib, RED); + rotateRight(sib); + sib = rightOf(parentOf(x)); + } + setColor(sib, colorOf(parentOf(x))); + setColor(parentOf(x), BLACK); + setColor(rightOf(sib), BLACK); + rotateLeft(parentOf(x)); + + x = (((((this))->root))); + } + } else { + node_t *sib = leftOf(parentOf(x)); + if (colorOf(sib) == RED) { + setColor(sib, BLACK); + setColor(parentOf(x), RED); + rotateRight(parentOf(x)); + sib = leftOf(parentOf(x)); + } + if (colorOf(rightOf(sib)) == BLACK && colorOf(leftOf(sib)) == BLACK) { + setColor(sib, RED); + x = parentOf(x); + } else { + if (colorOf(leftOf(sib)) == BLACK) { + setColor(rightOf(sib), BLACK); + setColor(sib, RED); + rotateLeft(sib); + sib = leftOf(parentOf(x)); + } + setColor(sib, colorOf(parentOf(x))); + setColor(parentOf(x), BLACK); + setColor(leftOf(sib), BLACK); + rotateRight(parentOf(x)); + + x = (((((this))->root))); + } + } + } + + if (x != nullptr && ((x)->c) != BLACK) { + ((x)->c) = (BLACK); + } + } + + node_t *delete_node(node_t *p) { + + if ((((((p))->l))) != nullptr && (((((p))->r))) != nullptr) { + node_t *s = successor(p); + ((p)->key) = ((((((s))->key)))); + ((p)->val) = ((((((s))->val)))); + p = s; + } + + node_t *replacement = + (((((((p))->l))) != nullptr) ? (((((p))->l))) : (((((p))->r)))); + + if (replacement != nullptr) { + + ((replacement)->p) = ((((((p))->p)))); + node_t *pp = (((((p))->p))); + if (pp == nullptr) { + ((this)->root) = (replacement); + } else if (p == (((((pp))->l)))) { + ((pp)->l) = (replacement); + } else { + ((pp)->r) = (replacement); + } + + ((p)->l) = (nullptr); + ((p)->r) = (nullptr); + ((p)->p) = (nullptr); + + if (((p)->c) == BLACK) { + fixAfterDeletion(replacement); + } + } else if ((((((p))->p))) == nullptr) { + ((this)->root) = (nullptr); + } else { + if (((p)->c) == BLACK) { + fixAfterDeletion(p); + } + node_t *pp = (((((p))->p))); + if (pp != nullptr) { + if (p == (((((pp))->l)))) { + ((pp)->l) = (nullptr); + } else if (p == (((((pp))->r)))) { + ((pp)->r) = (nullptr); + } + ((p)->p) = (nullptr); + } + } + return p; + } + + long compareKeysDefault(const void *a, const void *b) { + return ((long)a - (long)b); + } + + void releaseNode(node_t *n) { free(n); } + + void freeNode(node_t *n) { + if (n) { + freeNode(n->l); + freeNode(n->r); + releaseNode(n); + } + } + + node_t *getNode() { + node_t *n = (node_t *)malloc(sizeof(*n)); + return n; + } + +public: + rbtree_tl2_omap(DESCRIPTOR *me, auto *cfg) : root(nullptr) {} + + void rbtree_free(rbtree_tl2_omap *r) { + freeNode(r->root); + free(r); + } + + bool insert(DESCRIPTOR *me, const K &key, V &val) { + void *dummy; + TX_PRIVATE_STACK_REGION(&dummy); + { + TX_RAII; + node_t *node = getNode(); + node_t *ex = insertIt(key, val, node); + if (ex != nullptr) { + releaseNode(node); + } + return ((ex == nullptr) ? true : false); + } + } + + bool remove(DESCRIPTOR *me, const K &key) { + void *dummy; + TX_PRIVATE_STACK_REGION(&dummy); + { + TX_RAII; + node_t *node = nullptr; + node = lookup(key); + if (node != nullptr) { + node = delete_node(node); + } + if (node != nullptr) { + releaseNode(node); + } + return ((node != nullptr) ? true : false); + } + } + + bool rbtree_update(K key, V val) { + node_t *nn = getNode(); + node_t *ex = insert(key, val, nn); + if (ex != nullptr) { + ((ex)->val) = (val); + releaseNode(nn); + return true; + } + return false; + } + + V get(DESCRIPTOR *me, const K &key, V &val) { + void *dummy; + TX_PRIVATE_STACK_REGION(&dummy); + { + TX_RAII; + node_t *n = lookup(key); + if (n != nullptr) { + val = ((n)->val); + return true; + } + return false; + } + } + + long rbtree_contains(K key) { + node_t *n = lookup(key); + return (n != nullptr); + } +}; diff --git a/artifact/policies/README.md b/artifact/policies/README.md new file mode 100644 index 0000000..cf941a2 --- /dev/null +++ b/artifact/policies/README.md @@ -0,0 +1,78 @@ +# Synchronization Policies + +This folder stores the source code for the synchronization policies that are +discussed in the paper. + +## baseline Policy + +This is not really a synchronization policy. It holds the per-thread +functionality that is shared among the baseline lock-free and lock-based +algorithms that we use in our comparison. This includes things like a +per-thread pseudorandom number generator, a hash function, and safe memory +reclamation. Note that these features are also present in each of our +synchronization policies. + +## exoTM Mechanism + +This folder contains the implementation of the exoTM synchronization mechanism. + +## handSTM Policy + +This folder contains the handSTM policy. It uses the exoTM mechanism for its +synchronization. There are five categories of policy: + +- eager_c1: encounter-time locking, undo logging, check-once orecs +- eager_c2: encounter-time locking, undo logging, check-twice orecs +- wb_c1: encounter-time locking, redo logging, check-once orecs +- wb_c2: encounter-time locking, redo logging, check-twice orecs +- lazy: commit-time locking, redo logging, check-once orecs + +Note that each can be instantiated with per-object (PO) or per-stripe (PS) +orecs. + +## hybrid Policy + +This folder contains the hybrid policy (handSTM + STMCAS). It uses the exoTM +mechanism for its synchronization. There are three categories of policy: + +- wb_c1: encounter-time locking, redo logging, check-once orecs +- wb_c2: encounter-time locking, redo logging, check-twice orecs +- lazy: commit-time locking, redo logging, check-once orecs + +Note that each can be instantiated with per-object (PO) or per-stripe (PS) +orecs. + +## STMCAS Policy + +This folder contains the STMCAS policy. It uses the exoTM mechanism for its +synchronization. It can be instantiated with per-object (PO) or per-stripe (PS) +orecs. + +## xSTM Policy + +This folder holds a modified version of the +[llvm-transmem](https://github.com/mfs409/llvm-transmem) plugin, which provides +support for TM in C++. It consists of two sub-components: a plugin (`plugin/`) +for llvm-15, which instruments transactions, and a set of libraries (`libs/`) +that implement various TM algorithms. + +The folder is modified in the following ways: + +- Some names are updated to "xSTM" +- The plugin has been ported to llvm-15 +- The plugin has support for an RAII interface, which is faster than the + original lambda interface. +- Additional libraries have been added, which use the exoTM mechanism instead of + other implementations of orecs and clocks. +- We removed some TM algorithms (PTM, HTM) +- We moved some common libraries out of the folder, if they are shared by our + exoTM-based policies. + +Details about this plugin can be found in the following paper: + +"Simplifying Transactional Memory Support in C++", by PanteA Zardoshti, Tingzhe +Zhou, Pavithra Balaji, Michel L. Scott, and Michael Spear. ACM Transactions on +Architecture and Code Optimization (TACO), 2019. + +Note that these are all per-stripe (PS) policies, because that is the only +appropriate way to map orecs to program data in a language-level STM for C++. \ No newline at end of file diff --git a/artifact/policies/STMCAS/include/base.h b/artifact/policies/STMCAS/include/base.h new file mode 100644 index 0000000..c7bc561 --- /dev/null +++ b/artifact/policies/STMCAS/include/base.h @@ -0,0 +1,98 @@ +#pragma once + +#include +#include +#include + +#include "../../exoTM/exotm.h" +#include "../../include/minivector.h" + +#include "../../include/hash.h" +#include "../../include/orec_policies.h" +#include "../../include/rdtsc_rand.h" +#include "../../include/timestamp_smr.h" + +/// base_t holds common fields and methods for STMCAS policies. +/// +/// base_t is a re-usable descriptor. This means that it can have some +/// dynamic memory allocation internally. +/// +/// @tparam OP The orec policy to use. +template