diff --git a/src/ucs/memory/rcache.c b/src/ucs/memory/rcache.c index 7cf2ae8c50a..e6065dc92bb 100644 --- a/src/ucs/memory/rcache.c +++ b/src/ucs/memory/rcache.c @@ -491,7 +491,11 @@ ucs_rcache_check_overlap(ucs_rcache_t *rcache, ucs_pgt_addr_t *start, ucs_rcache_check_inv_queue(rcache); - ucs_rcache_find_regions(rcache, *start, *end - 1, ®ion_list); + if (rcache->params.merge_adjacent) { + ucs_rcache_find_regions(rcache, *start - 1, *end + 1, ®ion_list); + } else { + ucs_rcache_find_regions(rcache, *start, *end - 1, ®ion_list); + } /* TODO check if any of the regions is locked */ diff --git a/src/ucs/memory/rcache.h b/src/ucs/memory/rcache.h index 995c79d51cb..459b0bc4a19 100644 --- a/src/ucs/memory/rcache.h +++ b/src/ucs/memory/rcache.h @@ -115,6 +115,7 @@ struct ucs_rcache_params { be passed to mem_reg/mem_dereg */ unsigned long max_regions; /**< Maximal number of regions */ size_t max_size; /**< Maximal total size of regions */ + int merge_adjacent; /**< merge adjacent region */ }; diff --git a/src/uct/base/uct_md.c b/src/uct/base/uct_md.c index bdafc06e168..a5b3ae43241 100644 --- a/src/uct/base/uct_md.c +++ b/src/uct/base/uct_md.c @@ -48,6 +48,10 @@ ucs_config_field_t uct_md_config_rcache_table[] = { "Maximal total size of registration cache regions", ucs_offsetof(uct_md_rcache_config_t, max_size), UCS_CONFIG_TYPE_MEMUNITS}, + {"RCACHE_MERGE_ADJACENT", "n", "if yes, merge adjacent regions of rcache, " + "otherwise would only merge regions with intersection", + ucs_offsetof(uct_md_rcache_config_t, merge_adjacent), UCS_CONFIG_TYPE_BOOL}, + {NULL} }; @@ -452,4 +456,5 @@ void uct_md_set_rcache_params(ucs_rcache_params_t *rcache_params, rcache_params->ucm_event_priority = rcache_config->event_prio; rcache_params->max_regions = rcache_config->max_regions; rcache_params->max_size = rcache_config->max_size; + rcache_params->merge_adjacent = rcache_config->merge_adjacent; } diff --git a/src/uct/base/uct_md.h b/src/uct/base/uct_md.h index 7754c457e63..79801051e0a 100644 --- a/src/uct/base/uct_md.h +++ b/src/uct/base/uct_md.h @@ -19,11 +19,12 @@ typedef struct uct_md_rcache_config { - size_t alignment; /**< Force address alignment */ - unsigned event_prio; /**< Memory events priority */ - double overhead; /**< Lookup overhead estimation */ - unsigned long max_regions; /**< Maximal number of rcache regions */ - size_t max_size; /**< Maximal size of mapped memory */ + size_t alignment; /**< Force address alignment */ + unsigned event_prio; /**< Memory events priority */ + double overhead; /**< Lookup overhead estimation */ + unsigned long max_regions; /**< Maximal number of rcache regions */ + size_t max_size; /**< Maximal size of mapped memory */ + int merge_adjacent; /**< merge adjacent region */ } uct_md_rcache_config_t; diff --git a/src/uct/sm/mm/xpmem/mm_xpmem.c b/src/uct/sm/mm/xpmem/mm_xpmem.c index 6c59ac29db4..856688cc2bf 100644 --- a/src/uct/sm/mm/xpmem/mm_xpmem.c +++ b/src/uct/sm/mm/xpmem/mm_xpmem.c @@ -269,6 +269,7 @@ uct_xpmem_rmem_add(xpmem_segid_t xsegid, uct_xpmem_remote_mem_t **rmem_p) rcache_params.context = rmem; rcache_params.max_regions = ULONG_MAX; rcache_params.max_size = SIZE_MAX; + rcache_params.merge_adjacent = 0; status = ucs_rcache_create(&rcache_params, "xpmem_remote_mem", ucs_stats_get_root(), &rmem->rcache); diff --git a/test/gtest/ucs/test_rcache.cc b/test/gtest/ucs/test_rcache.cc index 8e72689f24d..d98c4ccaf0d 100644 --- a/test/gtest/ucs/test_rcache.cc +++ b/test/gtest/ucs/test_rcache.cc @@ -32,7 +32,8 @@ UCS_TEST_F(test_rcache_basic, create_fail) { &ops, NULL, ULONG_MAX, - SIZE_MAX + SIZE_MAX, + 0 }; ucs_rcache_t *rcache; @@ -46,6 +47,7 @@ UCS_TEST_F(test_rcache_basic, create_fail) { class test_rcache : public ucs::test { + friend class test_rcache_merge_adjacent; protected: struct region { @@ -86,7 +88,8 @@ class test_rcache : public ucs::test { &ops, reinterpret_cast(this), ULONG_MAX, - SIZE_MAX + SIZE_MAX, + 0 }; return params; } @@ -198,6 +201,30 @@ class test_rcache : public ucs::test { volatile uint32_t test_rcache::next_id = 1; +class test_rcache_merge_adjacent : public test_rcache { +protected: + + virtual ucs_rcache_params_t rcache_params() { + static const ucs_rcache_ops_t ops = { + mem_reg_cb, + mem_dereg_cb, + dump_region_cb + }; + ucs_rcache_params_t params = { + sizeof(region), + UCS_PGT_ADDR_ALIGN, + ucs_get_page_size(), + UCM_EVENT_VM_UNMAPPED, + 1000, + &ops, + reinterpret_cast(this), + ULONG_MAX, + SIZE_MAX, + 1 + }; + return params; + } +}; static uintptr_t virt_to_phys(uintptr_t address) { @@ -338,6 +365,65 @@ UCS_MT_TEST_F(test_rcache, merge, 6) { munmap(mem, size1 + pad + size2); } +UCS_MT_TEST_F(test_rcache_merge_adjacent, merge_adjacent, 6) { + /* + * -----+---------+-----+---------+-----+----------+---- + * pad | region1 | pad | region2 | pad | region3 | pad + * -----+---------+---------------+-----+----------+---- + * | region4 | + * +---------------------+ + * begin and end pad for avoiding multithread test memory boundary adjacent + */ + static const size_t size1 = 32 * ucs_get_page_size(); + static const size_t size2 = 32 * ucs_get_page_size(); + static const size_t size3 = 32 * ucs_get_page_size(); + static const size_t pad = 16 * ucs_get_page_size(); + static const size_t size4 = 64 * ucs_get_page_size(); + region *region1, *region2, *region3, *region4, *region1_2, *region2_2, *region3_2; + void *ptr1, *ptr2, *ptr3, *ptr4, *mem; + + mem = alloc_pages(pad + size1 + size4 + size3 + pad, PROT_READ|PROT_WRITE); + mem = (char*)mem + pad; + + /* Create region1 */ + ptr1 = (char*)mem; + region1 = get(ptr1, size1); + + /* Create region2 */ + ptr2 = (char*)mem + size1 + pad; + region2 = get(ptr2, size2); + + /* Create region3 */ + ptr3 = (char*)mem + size1 + size4; + region3 = get(ptr3, size3); + + /* Create region4 which should merge region1, region2 and region3 */ + ptr4 = (char*)mem + size1; + region4 = get(ptr4, size4); + + /* Get the same area as region1 - should be a different region now */ + region1_2 = get(ptr1, size1); + region2_2 = get(ptr2, size2); + region3_2 = get(ptr3, size3); + EXPECT_NE(region1, region1_2); /* should be different region because was merged */ + EXPECT_NE(region2, region2_2); /* should be different region because was merged */ + EXPECT_NE(region3, region3_2); /* should be different region because was merged */ + EXPECT_EQ(region4, region1_2); /* it should be the merged region */ + EXPECT_EQ(region4, region2_2); /* it should be the merged region */ + EXPECT_EQ(region4, region3_2); /* it should be the merged region */ + put(region1_2); + put(region2_2); + put(region3_2); + + put(region1); + put(region2); + put(region3); + put(region4); + + mem = (char*)mem - pad; + munmap(mem, pad + size1 + size4 + size3 + pad); +} + UCS_MT_TEST_F(test_rcache, merge_inv, 6) { /* * Merge with another region which causes immediate invalidation of the