From 7454b41d40677342b592952ba89a7c16941adb26 Mon Sep 17 00:00:00 2001 From: Sherin T George Date: Thu, 17 Sep 2026 01:10:04 +0800 Subject: [PATCH] DAOS-19645 common: Enhance dav_v2 alloc-from-bucket APIs for multiple E-buckets Rework the umem alloc-from-bucket APIs to accept a struct of type umem_bucket_req instead of a single bucket id, so an allocation can describe an ordered set of evictable (E) buckets. - Add umem_bucket_req_t { ubr_bkt_ids, ubr_bkt_cnt, ubr_bkt_max } and change umem_{alloc,zalloc,reserve,atomic_alloc}_from_bucket() and the backend ops (mo_tx_alloc/mo_reserve/mo_atomic_alloc) to take a umem_bucket_req_t * (NULL == default/non-evictable bucket). - palloc_reservation_create now allocates by walking the requested evictable buckets in priority order, then the spill-over evictable (SOE) buckets, and finally the non-evictable bucket. - TEMP FIX: ubr_bkt_max == 0 permits unbounded spill-over to SOE buckets, preserving the legacy single-bucket behavior; the vos callers request ubr_bkt_cnt = 1, ubr_bkt_max = 0. Unbounded SOE spill-over will be removed in a future change. - Update umem/vos callers and add a unit test (BMEM017a) covering spill-over across two requested buckets. Allow-unstable-test: true Signed-off-by: Sherin T George --- src/common/ad_tx.c | 9 +- src/common/dav_v2/dav_v2.h | 17 ++- src/common/dav_v2/palloc.c | 189 +++++++++++++++++++----------- src/common/dav_v2/palloc.h | 12 +- src/common/dav_v2/tx.c | 40 +++---- src/common/mem.c | 36 +++--- src/common/tests/umem_test_bmem.c | 145 +++++++++++++++++++---- src/include/daos/mem.h | 56 +++++---- src/vos/tests/vts_wal.c | 13 +- src/vos/vos_internal.h | 20 +++- 10 files changed, 357 insertions(+), 180 deletions(-) diff --git a/src/common/ad_tx.c b/src/common/ad_tx.c index 9ca51db969e..94d1ad2b7a2 100644 --- a/src/common/ad_tx.c +++ b/src/common/ad_tx.c @@ -1,5 +1,6 @@ /** * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1148,7 +1149,7 @@ umo_tx_free(struct umem_instance *umm, umem_off_t umoff) static umem_off_t umo_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { struct ad_tx *tx = tx_get(); struct ad_blob_handle bh = umm2ad_blob_hdl(umm); @@ -1243,7 +1244,7 @@ umo_tx_add_ptr(struct umem_instance *umm, void *ptr, size_t size) static umem_off_t umo_reserve(struct umem_instance *umm, void *act, size_t size, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { struct ad_blob_handle bh = umm2ad_blob_hdl(umm); struct ad_reserv_act *ract = act; @@ -1332,9 +1333,9 @@ umo_atomic_copy(struct umem_instance *umm, void *dest, const void *src, size_t l static umem_off_t umo_atomic_alloc(struct umem_instance *umm, size_t size, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { - return umo_tx_alloc(umm, size, 0, type_num, mbkt_id); + return umo_tx_alloc(umm, size, 0, type_num, req); } static int diff --git a/src/common/dav_v2/dav_v2.h b/src/common/dav_v2/dav_v2.h index a965596459a..bf0256130ba 100644 --- a/src/common/dav_v2/dav_v2.h +++ b/src/common/dav_v2/dav_v2.h @@ -1,6 +1,8 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright 2015-2024, Intel Corporation */ -/* (C) Copyright 2025 Hewlett Packard Enterprise Development LP */ +/* + * Copyright 2015-2024 Intel Corporation. + */ +/* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP */ /* * dav_flags.h -- Interfaces exported by DAOS internal Allocator for VOS (DAV) @@ -17,6 +19,7 @@ typedef struct dav_obj dav_obj_t; struct umem_store; +struct umem_bucket_req; /** * Create and initialize a DAV object and return its handle. @@ -81,7 +84,7 @@ typedef int (*dav_constr)(dav_obj_t *pop, void *ptr, void *arg); */ int dav_alloc_v2(dav_obj_t *pop, uint64_t *offp, size_t size, uint64_t type_num, uint64_t flags, - dav_constr constructor, void *arg); + struct umem_bucket_req *req, dav_constr constructor, void *arg); /** * Frees the memory at specified offset within the DAV object pointed to by hdl. @@ -179,13 +182,15 @@ dav_tx_errno_v2(void); * - POBJ_XALLOC_NO_FLUSH - skip flush on commit * - POBJ_XALLOC_NO_ABORT - if the function does not end successfully, * - DAV_CLASS_ID(id) - id of allocation class to use. - * - DAV_EZONE_ID(id) - id of zone to use. * do not abort the transaction and return the error number. * + * The memory bucket to allocate from is described by \a req (NULL for the + * default bucket). + * * This function must be called during TX_STAGE_WORK. */ uint64_t -dav_tx_alloc_v2(size_t size, uint64_t type_num, uint64_t flags); +dav_tx_alloc_v2(size_t size, uint64_t type_num, uint64_t flags, struct umem_bucket_req *req); /* * Transactionally frees an existing object. @@ -258,7 +263,7 @@ dav_tx_xadd_range_direct_v2(const void *ptr, size_t size, uint64_t flags); struct dav_action; uint64_t dav_reserve_v2(dav_obj_t *pop, struct dav_action *act, size_t size, uint64_t type_num, - uint64_t flags); + uint64_t flags, struct umem_bucket_req *req); void dav_defer_free_v2(dav_obj_t *pop, uint64_t off, struct dav_action *act); void diff --git a/src/common/dav_v2/palloc.c b/src/common/dav_v2/palloc.c index 676591d8a6f..587656c2e72 100644 --- a/src/common/dav_v2/palloc.c +++ b/src/common/dav_v2/palloc.c @@ -1,6 +1,8 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright 2015-2024, Intel Corporation */ -/* (C) Copyright 2025 Hewlett Packard Enterprise Development LP */ +/* + * Copyright 2015-2024 Intel Corporation. + */ +/* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP */ /* * palloc.c -- implementation of pmalloc POSIX-like API @@ -175,54 +177,31 @@ alloc_prep_block(struct palloc_heap *heap, const struct memory_block *m, } /* - * palloc_reservation_create -- creates a volatile reservation of a - * memory block. - * - * The first step in the allocation of a new block is reserving it in - * the transient heap - which is represented by the bucket abstraction. + * palloc_reservation_try_mb -- (internal) attempts a single reservation from a + * specific memory bucket. * - * To provide optimal scaling for multi-threaded applications and reduce - * fragmentation the appropriate bucket is chosen depending on the - * current thread context and to which allocation class the requested - * size falls into. - * - * Once the bucket is selected, just enough memory is reserved for the - * requested size. The underlying block allocation algorithm - * (best-fit, next-fit, ...) varies depending on the bucket container. + * Returns 0 on success or an errno value on failure. ENOMEM indicates the + * bucket has no suitable free space and the caller may retry from a different + * bucket; any other value is a hard failure. */ static int -palloc_reservation_create(struct palloc_heap *heap, size_t size, palloc_constr constructor, - void *arg, uint64_t extra_field, uint16_t object_flags, uint16_t class_id, - uint32_t mb_id, struct dav_action_internal *out) +palloc_reservation_try_mb(struct palloc_heap *heap, struct mbrt *mb, size_t size, + palloc_constr constructor, void *arg, uint64_t extra_field, + uint16_t object_flags, uint16_t class_id, struct dav_action_internal *out) { - int err = 0; struct memory_block *new_block = &out->m; - struct mbrt *mb; + struct alloc_class *c; + struct bucket *b; ssize_t size_idx; + int err; - out->type = DAV_ACTION_TYPE_HEAP; - - ASSERT(class_id < UINT8_MAX); - - mb = heap_mbrt_get_mb(heap, mb_id); - if (mb == NULL) { - ERR("Invalid mb_id %u", mb_id); - errno = EINVAL; - return -1; - } - struct alloc_class *c = class_id == 0 - ? mbrt_get_best_class(mb, size) - : alloc_class_by_id(mbrt_alloc_classes(mb), (uint8_t)class_id); - + c = class_id == 0 ? mbrt_get_best_class(mb, size) + : alloc_class_by_id(mbrt_alloc_classes(mb), (uint8_t)class_id); if (c == NULL) { ERR("no allocation class for size %lu bytes", size); - errno = EINVAL; - return -1; + return EINVAL; } - heap_soemb_active_iter_init(heap); - -retry: /* * The caller provided size in bytes, but buckets operate in * 'size indexes' which are multiples of the block size in the @@ -232,24 +211,19 @@ palloc_reservation_create(struct palloc_heap *heap, size_t size, palloc_constr c * provides 256 byte blocks two memory 'units' are required. */ size_idx = alloc_class_calc_size_idx(c, size); - if (size_idx < 0) { - ERR("allocation class not suitable for size %lu bytes", - size); - errno = EINVAL; - return -1; + ERR("allocation class not suitable for size %lu bytes", size); + return EINVAL; } ASSERT(size_idx <= UINT32_MAX); - *new_block = MEMORY_BLOCK_NONE; + *new_block = MEMORY_BLOCK_NONE; new_block->size_idx = (uint32_t)size_idx; err = heap_mbrt_update_alloc_class_buckets(heap, mb, c); - if (err != 0) { - errno = err; - return -1; - } + if (err != 0) + return err; - struct bucket *b = mbrt_bucket_acquire(mb, c->id); + b = mbrt_bucket_acquire(mb, c->id); err = heap_get_bestfit_block(heap, b, new_block); if (err != 0) @@ -282,36 +256,111 @@ palloc_reservation_create(struct palloc_heap *heap, size_t size, palloc_constr c out: mbrt_bucket_release(b); + return err; +} - if (err == 0) - return 0; +/* + * palloc_reservation_create -- creates a volatile reservation of a + * memory block. + * + * The requested evictable buckets are tried first in priority order, then the + * spill-over evictable buckets (when the object may grow beyond the requested + * set, or ubr_bkt_max == 0 as a temporary fix), and finally the non-evictable + * bucket. + */ +static int +palloc_reservation_create(struct palloc_heap *heap, size_t size, palloc_constr constructor, + void *arg, uint64_t extra_field, uint16_t object_flags, uint16_t class_id, + struct umem_bucket_req *req, struct dav_action_internal *out) +{ + struct mbrt *mb; + uint32_t bkt_cnt = (req != NULL) ? req->ubr_bkt_cnt : 0; + uint32_t bkt_max = (req != NULL) ? req->ubr_bkt_max : 0; + uint32_t mb_id; + uint32_t i; + int err = ENOMEM; + + out->type = DAV_ACTION_TYPE_HEAP; + + ASSERT(class_id < UINT8_MAX); + + heap_soemb_active_iter_init(heap); /* - * If there is no memory in evictable zone then do the allocation - * from non-evictable zone. + * Try the explicitly requested evictable buckets in priority order, + * stopping at the first success or when a zero (terminator) id is seen. */ - if ((mb_id != 0) && (err == ENOMEM)) { - heap_mbrt_log_alloc_failure(heap, mb_id); - mb_id = heap_soemb_active_get(heap); - mb = heap_mbrt_get_mb(heap, mb_id); + for (i = 0; i < bkt_cnt; i++) { + mb_id = req->ubr_bkt_ids[i]; + if (mb_id == UMEM_DEFAULT_MBKT_ID) + break; + + mb = heap_mbrt_get_mb(heap, mb_id); if (mb == NULL) { ERR("Invalid mb_id %u", mb_id); errno = EINVAL; return -1; } - if (mb_id == 0) { - c = class_id == 0 - ? mbrt_get_best_class(mb, size) - : alloc_class_by_id(mbrt_alloc_classes(mb), (uint8_t)class_id); - if (c == NULL) { - ERR("no allocation class for size %lu bytes", size); + + err = palloc_reservation_try_mb(heap, mb, size, constructor, arg, extra_field, + object_flags, class_id, out); + if (err == 0) + return 0; + if (err != ENOMEM) { + errno = err; + return -1; + } + heap_mbrt_log_alloc_failure(heap, mb_id); + } + + /* + * All requested E-buckets are full. If the object is allowed to spill + * into more evictable buckets, try the active spill-over evictable + * (SOE) buckets next. (i > 0) ensures that at least one E-bucket + * allocation was attempted. + * + * TEMP FIX: ubr_bkt_max == 0 means unbounded spill-over to SOE buckets, + * restoring the legacy single-bucket behavior that always spilled to SOE + * buckets. This is temporary; unbounded spill-over to SOE buckets will + * not be allowed in the future. + */ + if (i > 0 && (bkt_max == 0 || i < bkt_max)) { + while ((mb_id = heap_soemb_active_get(heap)) != 0) { + mb = heap_mbrt_get_mb(heap, mb_id); + if (mb == NULL) { + ERR("Invalid mb_id %u", mb_id); errno = EINVAL; return -1; } + + err = palloc_reservation_try_mb(heap, mb, size, constructor, arg, + extra_field, object_flags, class_id, out); + if (err == 0) + return 0; + if (err != ENOMEM) { + errno = err; + return -1; + } + heap_mbrt_log_alloc_failure(heap, mb_id); } - goto retry; } + /* + * Fall back to the non-evictable bucket when no evictable bucket could + * satisfy the request or the spill-over limit has been reached. + */ + mb = heap_mbrt_get_mb(heap, UMEM_DEFAULT_MBKT_ID); + if (mb == NULL) { + ERR("Invalid mb_id %u", UMEM_DEFAULT_MBKT_ID); + errno = EINVAL; + return -1; + } + + err = palloc_reservation_try_mb(heap, mb, size, constructor, arg, extra_field, object_flags, + class_id, out); + if (err == 0) + return 0; + errno = err; return -1; } @@ -654,14 +703,14 @@ palloc_exec_actions(struct palloc_heap *heap, */ int palloc_reserve(struct palloc_heap *heap, size_t size, palloc_constr constructor, void *arg, - uint64_t extra_field, uint16_t object_flags, uint16_t class_id, uint32_t mb_id, - struct dav_action *act) + uint64_t extra_field, uint16_t object_flags, uint16_t class_id, + struct umem_bucket_req *req, struct dav_action *act) { COMPILE_ERROR_ON(sizeof(struct dav_action) != sizeof(struct dav_action_internal)); return palloc_reservation_create(heap, size, constructor, arg, extra_field, object_flags, - class_id, mb_id, (struct dav_action_internal *)act); + class_id, req, (struct dav_action_internal *)act); } /* @@ -805,7 +854,7 @@ palloc_publish(struct palloc_heap *heap, struct dav_action *actv, size_t actvcnt int palloc_operation(struct palloc_heap *heap, uint64_t off, uint64_t *dest_off, size_t size, palloc_constr constructor, void *arg, uint64_t extra_field, uint16_t object_flags, - uint16_t class_id, uint32_t mb_id, struct operation_context *ctx) + uint16_t class_id, struct umem_bucket_req *req, struct operation_context *ctx) { size_t user_size = 0; @@ -836,7 +885,7 @@ palloc_operation(struct palloc_heap *heap, uint64_t off, uint64_t *dest_off, siz if (size != 0) { alloc = &ops[nops++]; if (palloc_reservation_create(heap, size, constructor, arg, extra_field, - object_flags, class_id, mb_id, alloc) != 0) { + object_flags, class_id, req, alloc) != 0) { operation_cancel(ctx); return -1; } diff --git a/src/common/dav_v2/palloc.h b/src/common/dav_v2/palloc.h index 027fb94667b..e5823923438 100644 --- a/src/common/dav_v2/palloc.h +++ b/src/common/dav_v2/palloc.h @@ -1,5 +1,8 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright 2015-2024, Intel Corporation */ +/* + * Copyright 2015-2024 Intel Corporation. + * Copyright 2026 Hewlett Packard Enterprise Development LP + */ /* * palloc.h -- internal definitions for persistent allocator @@ -31,18 +34,19 @@ struct palloc_heap { struct memory_block; struct mbrt; +struct umem_bucket_req; typedef int (*palloc_constr)(void *base, void *ptr, size_t usable_size, void *arg); int palloc_operation(struct palloc_heap *heap, uint64_t off, uint64_t *dest_off, size_t size, palloc_constr constructor, void *arg, uint64_t extra_field, uint16_t object_flags, - uint16_t class_id, uint32_t zset_id, struct operation_context *ctx); + uint16_t class_id, struct umem_bucket_req *req, struct operation_context *ctx); int palloc_reserve(struct palloc_heap *heap, size_t size, palloc_constr constructor, void *arg, - uint64_t extra_field, uint16_t object_flags, uint16_t class_id, uint32_t zset_id, - struct dav_action *act); + uint64_t extra_field, uint16_t object_flags, uint16_t class_id, + struct umem_bucket_req *req, struct dav_action *act); int palloc_action_isalloc(struct dav_action *act); diff --git a/src/common/dav_v2/tx.c b/src/common/dav_v2/tx.c index c6103eb20cb..fa49911ba54 100644 --- a/src/common/dav_v2/tx.c +++ b/src/common/dav_v2/tx.c @@ -1,5 +1,8 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright 2015-2024, Intel Corporation */ +/* + * Copyright 2015-2024 Intel Corporation. + * Copyright 2026 Hewlett Packard Enterprise Development LP + */ /* * tx.c -- transactions implementation @@ -335,8 +338,8 @@ tx_ranges_insert_def(dav_obj_t *pop, struct tx *tx, * tx_alloc_common -- (internal) common function for alloc and zalloc */ static uint64_t -tx_alloc_common(struct tx *tx, size_t size, type_num_t type_num, - palloc_constr constructor, struct tx_alloc_args args) +tx_alloc_common(struct tx *tx, size_t size, type_num_t type_num, palloc_constr constructor, + struct tx_alloc_args args, struct umem_bucket_req *req) { const struct tx_range_def *r; uint64_t off; @@ -354,8 +357,7 @@ tx_alloc_common(struct tx *tx, size_t size, type_num_t type_num, return obj_tx_fail_null(ENOMEM, args.flags); if (palloc_reserve(pop->do_heap, size, constructor, &args, type_num, 0, - CLASS_ID_FROM_FLAG(args.flags), EZONE_ID_FROM_FLAG(args.flags), - action) != 0) + CLASS_ID_FROM_FLAG(args.flags), req, action) != 0) goto err_oom; palloc_get_prange(action, &off, &size, 1); @@ -1272,7 +1274,7 @@ dav_tx_xadd_range_v2(uint64_t hoff, size_t size, uint64_t flags) * dav_tx_alloc -- allocates a new object */ DAV_FUNC_EXPORT uint64_t -dav_tx_alloc_v2(size_t size, uint64_t type_num, uint64_t flags) +dav_tx_alloc_v2(size_t size, uint64_t type_num, uint64_t flags, struct umem_bucket_req *req) { uint64_t off; struct tx *tx = get_tx(); @@ -1299,8 +1301,8 @@ dav_tx_alloc_v2(size_t size, uint64_t type_num, uint64_t flags) return off; } - off = tx_alloc_common(tx, size, (type_num_t)type_num, - constructor_tx_alloc, ALLOC_ARGS(flags)); + off = tx_alloc_common(tx, size, (type_num_t)type_num, constructor_tx_alloc, + ALLOC_ARGS(flags), req); DAV_API_END(); return off; @@ -1506,7 +1508,7 @@ obj_alloc_root(dav_obj_t *pop, size_t size) operation_add_entry(ctx, pop->do_root_sizep, size, ULOG_OPERATION_SET); ret = palloc_operation(pop->do_heap, *pop->do_root_offsetp, pop->do_root_offsetp, size, - constructor_zrealloc_root, &carg, 0, 0, 0, 0, + constructor_zrealloc_root, &carg, 0, 0, 0, NULL, ctx); /* REVISIT: object_flags and type num ignored*/ lw_tx_end(pop, NULL); @@ -1579,9 +1581,8 @@ constructor_alloc(void *ctx, void *ptr, size_t usable_size, void *arg) * obj_alloc_construct -- (internal) allocates a new object with constructor */ static int -obj_alloc_construct(dav_obj_t *pop, uint64_t *offp, size_t size, - type_num_t type_num, uint64_t flags, - dav_constr constructor, void *arg) +obj_alloc_construct(dav_obj_t *pop, uint64_t *offp, size_t size, type_num_t type_num, + uint64_t flags, struct umem_bucket_req *req, dav_constr constructor, void *arg) { struct operation_context *ctx; struct constr_args carg; @@ -1604,7 +1605,7 @@ obj_alloc_construct(dav_obj_t *pop, uint64_t *offp, size_t size, operation_start(ctx); ret = palloc_operation(pop->do_heap, 0, offp, size, constructor_alloc, &carg, type_num, 0, - CLASS_ID_FROM_FLAG(flags), EZONE_ID_FROM_FLAG(flags), ctx); + CLASS_ID_FROM_FLAG(flags), req, ctx); lw_tx_end(pop, NULL); return ret; @@ -1615,7 +1616,7 @@ obj_alloc_construct(dav_obj_t *pop, uint64_t *offp, size_t size, */ DAV_FUNC_EXPORT int dav_alloc_v2(dav_obj_t *pop, uint64_t *offp, size_t size, uint64_t type_num, uint64_t flags, - dav_constr constructor, void *arg) + struct umem_bucket_req *req, dav_constr constructor, void *arg) { DAV_DBG(3, "pop %p offp %p size %zu type_num %llx flags %llx constructor %p arg %p", pop, offp, size, (unsigned long long)type_num, (unsigned long long)flags, constructor, @@ -1634,7 +1635,7 @@ dav_alloc_v2(dav_obj_t *pop, uint64_t *offp, size_t size, uint64_t type_num, uin } DAV_API_START(); - int ret = obj_alloc_construct(pop, offp, size, type_num, flags, constructor, arg); + int ret = obj_alloc_construct(pop, offp, size, type_num, flags, req, constructor, arg); if (ret) { errno = ret; ret = -1; @@ -1667,8 +1668,7 @@ dav_free_v2(dav_obj_t *pop, uint64_t off) ctx = pop->external; operation_start(ctx); - palloc_operation(pop->do_heap, off, NULL, 0, NULL, NULL, - 0, 0, 0, 0, ctx); + palloc_operation(pop->do_heap, off, NULL, 0, NULL, NULL, 0, 0, 0, NULL, ctx); lw_tx_end(pop, NULL); DAV_API_END(); @@ -1702,7 +1702,7 @@ dav_memcpy_persist_v2(dav_obj_t *pop, void *dest, const void *src, */ DAV_FUNC_EXPORT uint64_t dav_reserve_v2(dav_obj_t *pop, struct dav_action *act, size_t size, uint64_t type_num, - uint64_t flags) + uint64_t flags, struct umem_bucket_req *req) { struct constr_args carg; int tx_inprogress = 0; @@ -1732,7 +1732,7 @@ dav_reserve_v2(dav_obj_t *pop, struct dav_action *act, size_t size, uint64_t typ carg.arg = NULL; if (palloc_reserve(pop->do_heap, size, constructor_alloc, &carg, type_num, 0, - CLASS_ID_FROM_FLAG(flags), EZONE_ID_FROM_FLAG(flags), act) != 0) { + CLASS_ID_FROM_FLAG(flags), req, act) != 0) { if (!tx_inprogress) lw_tx_end(pop, NULL); DAV_API_END(); @@ -1902,7 +1902,7 @@ obj_realloc(dav_obj_t *pop, uint64_t *offp, size_t *sizep, size_t size, uint16_t operation_add_entry(ctx, sizep, size, ULOG_OPERATION_SET); ret = palloc_operation(pop->do_heap, *offp, offp, size, constructor_zrealloc_root, &carg, 0, - 0, class_id, 0, ctx); + 0, class_id, NULL, ctx); return ret; } diff --git a/src/common/mem.c b/src/common/mem.c index f64d744be72..2f8ecc9b0a5 100644 --- a/src/common/mem.c +++ b/src/common/mem.c @@ -796,7 +796,7 @@ pmem_tx_free(struct umem_instance *umm, umem_off_t umoff) static umem_off_t pmem_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned int type_num, - unsigned int unused) + struct umem_bucket_req *req) { uint64_t pflags = 0; @@ -1048,7 +1048,7 @@ pmem_tx_stage(void) static umem_off_t pmem_reserve(struct umem_instance *umm, void *act, size_t size, unsigned int type_num, - unsigned int unused) + struct umem_bucket_req *req) { PMEMobjpool *pop = (PMEMobjpool *)umm->umm_pool->up_priv; @@ -1083,7 +1083,7 @@ pmem_atomic_copy(struct umem_instance *umm, void *dest, const void *src, static umem_off_t pmem_atomic_alloc(struct umem_instance *umm, size_t size, unsigned int type_num, - unsigned int unused) + struct umem_bucket_req *req) { PMEMoid oid; PMEMobjpool *pop = (PMEMobjpool *)umm->umm_pool->up_priv; @@ -1234,7 +1234,7 @@ bmem_tx_free(struct umem_instance *umm, umem_off_t umoff) static umem_off_t bmem_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { uint64_t pflags = 0; @@ -1348,7 +1348,7 @@ bmem_defer_free(struct umem_instance *umm, umem_off_t off, void *act) static umem_off_t bmem_reserve(struct umem_instance *umm, void *act, size_t size, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { dav_obj_t *pop = (dav_obj_t *)umm->umm_pool->up_priv; @@ -1388,7 +1388,7 @@ bmem_atomic_copy(struct umem_instance *umm, void *dest, const void *src, static umem_off_t bmem_atomic_alloc(struct umem_instance *umm, size_t size, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { uint64_t off; dav_obj_t *pop = (dav_obj_t *)umm->umm_pool->up_priv; @@ -1473,7 +1473,7 @@ bmem_tx_free_v2(struct umem_instance *umm, umem_off_t umoff) static umem_off_t bmem_tx_alloc_v2(struct umem_instance *umm, size_t size, uint64_t flags, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { uint64_t pflags = 0; @@ -1483,9 +1483,7 @@ bmem_tx_alloc_v2(struct umem_instance *umm, size_t size, uint64_t flags, unsigne pflags |= DAV_FLAG_ZERO; if (flags & UMEM_FLAG_NO_FLUSH) pflags |= DAV_FLAG_NO_FLUSH; - if (mbkt_id != 0) - pflags |= DAV_EZONE_ID(mbkt_id); - return dav_tx_alloc_v2(size, type_num, pflags); + return dav_tx_alloc_v2(size, type_num, pflags, req); } static int @@ -1589,12 +1587,11 @@ bmem_defer_free_v2(struct umem_instance *umm, umem_off_t off, void *act) static umem_off_t bmem_reserve_v2(struct umem_instance *umm, void *act, size_t size, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { dav_obj_t *pop = (dav_obj_t *)umm->umm_pool->up_priv; - uint64_t flags = DAV_EZONE_ID(mbkt_id); - return dav_reserve_v2(pop, (struct dav_action *)act, size, type_num, flags); + return dav_reserve_v2(pop, (struct dav_action *)act, size, type_num, 0, req); } static void @@ -1630,14 +1627,13 @@ bmem_atomic_copy_v2(struct umem_instance *umm, void *dest, const void *src, static umem_off_t bmem_atomic_alloc_v2(struct umem_instance *umm, size_t size, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { uint64_t off; dav_obj_t *pop = (dav_obj_t *)umm->umm_pool->up_priv; - int rc; - uint64_t flags = DAV_EZONE_ID(mbkt_id); + int rc; - rc = dav_alloc_v2(pop, &off, size, type_num, flags, NULL, NULL); + rc = dav_alloc_v2(pop, &off, size, type_num, 0, req, NULL, NULL); if (rc) return UMOFF_NULL; return off; @@ -1719,7 +1715,7 @@ vmem_free(struct umem_instance *umm, umem_off_t umoff) umem_off_t vmem_alloc(struct umem_instance *umm, size_t size, uint64_t flags, unsigned int type_num, - unsigned int unused) + struct umem_bucket_req *req) { return (uint64_t)((flags & UMEM_FLAG_ZERO) ? calloc(1, size) : malloc(size)); @@ -2049,7 +2045,7 @@ umem_rsrvd_act_free(struct umem_rsrvd_act **rsrvd_act) umem_off_t umem_reserve_common(struct umem_instance *umm, struct umem_rsrvd_act *rsrvd_act, size_t size, - unsigned int mbkt_id) + struct umem_bucket_req *req) { if (umm->umm_ops->mo_reserve) { void *act; @@ -2060,7 +2056,7 @@ umem_reserve_common(struct umem_instance *umm, struct umem_rsrvd_act *rsrvd_act, D_ASSERT(rsrvd_act->rs_actv_cnt > rsrvd_act->rs_actv_at); act = rsrvd_act->rs_actv + act_size * rsrvd_act->rs_actv_at; - off = umm->umm_ops->mo_reserve(umm, act, size, UMEM_TYPE_ANY, mbkt_id); + off = umm->umm_ops->mo_reserve(umm, act, size, UMEM_TYPE_ANY, req); if (!UMOFF_IS_NULL(off)) rsrvd_act->rs_actv_at++; D_ASSERTF(umem_off2flags(off) == 0, diff --git a/src/common/tests/umem_test_bmem.c b/src/common/tests/umem_test_bmem.c index a62fe3814a4..791e4990bd7 100644 --- a/src/common/tests/umem_test_bmem.c +++ b/src/common/tests/umem_test_bmem.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2019-2024 Intel Corporation. - * (C) Copyright 2023-2025 Hewlett Packard Enterprise Development LP. + * (C) Copyright 2023-2026 Hewlett Packard Enterprise Development LP. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -32,6 +32,10 @@ #define NEMB_RATIO (0.8) #define MB_SIZE (16 * 1024 * 1024) +/** Build a single-bucket allocation request for the given bucket id. */ +#define BKT_REQ(id) \ + (&(umem_bucket_req_t){.ubr_bkt_ids = &(uint32_t){id}, .ubr_bkt_cnt = 1, .ubr_bkt_max = 0}) + struct test_arg { struct utest_context *ta_utx; uint64_t *ta_root; @@ -1800,10 +1804,10 @@ test_atomic_alloc_mb(void **state) assert_int_not_equal(mb_id, 0); /* zero maps to non-evictable memory bucket */ /* Allocate objects from the memory bucket */ - umoff1 = umem_atomic_alloc_from_bucket(umm, 2048, UMEM_TYPE_ANY, mb_id); + umoff1 = umem_atomic_alloc_from_bucket(umm, 2048, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); assert_true(umem_get_mb_from_offset(umm, umoff1) == mb_id); - umoff2 = umem_atomic_alloc_from_bucket(umm, 1024, UMEM_TYPE_ANY, mb_id); + umoff2 = umem_atomic_alloc_from_bucket(umm, 1024, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff2)); assert_true(umem_get_mb_from_offset(umm, umoff2) == mb_id); @@ -1829,7 +1833,7 @@ test_atomic_alloc_mb(void **state) found = 0; for (i = 0; i < 16 * 1024; i++) { - umoff = umem_atomic_alloc_from_bucket(umm, 2048, UMEM_TYPE_ANY, mb_id); + umoff = umem_atomic_alloc_from_bucket(umm, 2048, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); umem_atomic_free(umm, umoff); @@ -1856,7 +1860,7 @@ test_atomic_alloc_mb(void **state) found = 0; for (i = 0; i < 16 * 1024; i++) { - umoff = umem_atomic_alloc_from_bucket(umm, 1024, UMEM_TYPE_ANY, mb_id); + umoff = umem_atomic_alloc_from_bucket(umm, 1024, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); umem_atomic_free(umm, umoff); @@ -1900,7 +1904,7 @@ test_atomic_alloc_overflow_mb(void **state) hit = 0; /* Allocate objects from the memory bucket */ umoff_prev = umoff1; - umoff1 = umem_atomic_alloc_from_bucket(umm, 2048, UMEM_TYPE_ANY, mb_id); + umoff1 = umem_atomic_alloc_from_bucket(umm, 2048, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); ret_id = umem_get_mb_from_offset(umm, umoff1); if (ret_id == mb_id) @@ -1912,7 +1916,7 @@ test_atomic_alloc_overflow_mb(void **state) } else assert_true(ret_id == mb_id); umoff_prev = umoff2; - umoff2 = umem_atomic_alloc_from_bucket(umm, 1024, UMEM_TYPE_ANY, mb_id); + umoff2 = umem_atomic_alloc_from_bucket(umm, 1024, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff2)); ret_id = umem_get_mb_from_offset(umm, umoff2); if (ret_id == mb_id) @@ -1924,7 +1928,7 @@ test_atomic_alloc_overflow_mb(void **state) } else assert_true(ret_id == mb_id); umoff_prev = umoff3; - umoff3 = umem_atomic_alloc_from_bucket(umm, 128, UMEM_TYPE_ANY, mb_id); + umoff3 = umem_atomic_alloc_from_bucket(umm, 128, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff3)); ret_id = umem_get_mb_from_offset(umm, umoff3); if (ret_id == mb_id) @@ -1946,20 +1950,99 @@ test_atomic_alloc_overflow_mb(void **state) * The only free memory in the MB is that of the offsets freed above. * Subsequent allocation from the same MB should return the same offsets. */ - umoff = umem_atomic_alloc_from_bucket(umm, 2048, UMEM_TYPE_ANY, mb_id); + umoff = umem_atomic_alloc_from_bucket(umm, 2048, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); assert_true(umoff == umoff1); - umoff = umem_atomic_alloc_from_bucket(umm, 1024, UMEM_TYPE_ANY, mb_id); + umoff = umem_atomic_alloc_from_bucket(umm, 1024, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); assert_true(umoff == umoff2); - umoff = umem_atomic_alloc_from_bucket(umm, 128, UMEM_TYPE_ANY, mb_id); + umoff = umem_atomic_alloc_from_bucket(umm, 128, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); assert_true(umoff == umoff3); } +static void +test_atomic_alloc_spill_two_mb(void **state) +{ + struct test_arg *arg = *state; + struct umem_instance *umm = utest_utx2umm(arg->ta_utx); + umem_off_t umoff, prev_umoff = UMOFF_NULL, *ptr = NULL; + uint32_t mb_id1, mb_id2, ret_id; + uint32_t bkt_ids[2]; + umem_bucket_req_t req; + size_t alloc_size = 2048; + uint64_t allocated1, maxsz1; + uint64_t allocated2, maxsz2; + int rc; + + /* + * A freshly allotted evictable bucket is only retired once it crosses + * its utilization threshold, so fill the first bucket up before + * allotting the second one to ensure they are distinct. + */ + mb_id1 = umem_allot_mb_evictable(umm, 0); + assert_int_not_equal(mb_id1, 0); /* zero maps to non-evictable memory bucket */ + + while (1) { + umoff = + umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, BKT_REQ(mb_id1)); + assert_false(UMOFF_IS_NULL(umoff)); + if (umem_get_mb_from_offset(umm, umoff) != mb_id1) { + /* Spilled out of the bucket: it is full. */ + umem_atomic_free(umm, umoff); + break; + } + ptr = (umem_off_t *)umem_off2ptr(umm, umoff); + *ptr = prev_umoff; + prev_umoff = umoff; + } + + mb_id2 = umem_allot_mb_evictable(umm, 0); + assert_int_not_equal(mb_id2, 0); + assert_int_not_equal(mb_id1, mb_id2); + + /* Free everything allocated from the first bucket. */ + while (!UMOFF_IS_NULL(prev_umoff)) { + ptr = (umem_off_t *)umem_off2ptr(umm, prev_umoff); + umoff = *ptr; + umem_atomic_free(umm, prev_umoff); + prev_umoff = umoff; + } + + /* Request mb_id1 as the primary bucket and mb_id2 as the spill-over bucket. */ + bkt_ids[0] = mb_id1; + bkt_ids[1] = mb_id2; + req.ubr_bkt_ids = bkt_ids; + req.ubr_bkt_cnt = 2; + req.ubr_bkt_max = 2; + + /* + * Keep allocating until the request spills over to the non-evictable + * bucket. Each allocation may be served from either requested evictable + * bucket (in any order) until both are full. + */ + do { + umoff = umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, &req); + assert_false(UMOFF_IS_NULL(umoff)); + ret_id = umem_get_mb_from_offset(umm, umoff); + assert_true(ret_id == 0 || ret_id == mb_id1 || ret_id == mb_id2); + } while (ret_id != 0); + + rc = umempobj_get_mbusage(umm->umm_pool, mb_id1, &allocated1, &maxsz1); + assert_int_equal(rc, 0); + print_message("mb_id1: max_size = %lu allocated = %lu\n", maxsz1, allocated1); + rc = umempobj_get_mbusage(umm->umm_pool, mb_id2, &allocated2, &maxsz2); + assert_int_equal(rc, 0); + print_message("mb_id2: max_size = %lu allocated = %lu\n", maxsz2, allocated2); + + /* Both requested buckets must fill up before the request spills to the NE bucket. */ + assert_true(allocated1 * 100 / maxsz1 >= 95); + assert_true(allocated2 * 100 / maxsz2 >= 95); +} + static void test_reserve_from_mb(void **state) { @@ -1977,7 +2060,7 @@ test_reserve_from_mb(void **state) /* Reserve an object and then cancel the allocation */ rc = umem_rsrvd_act_alloc(umm, &rsrvd_act, 1); assert_int_equal(rc, 0); - umoff = umem_reserve_from_bucket(umm, rsrvd_act, rsrv_size, mb_id); + umoff = umem_reserve_from_bucket(umm, rsrvd_act, rsrv_size, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); /* Validate that the object is from the memory bucket of interest. */ assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); @@ -1986,7 +2069,7 @@ test_reserve_from_mb(void **state) /* Validate that the object is really freed */ rc = umem_rsrvd_act_alloc(umm, &rsrvd_act, 1); assert_int_equal(rc, 0); - umoff1 = umem_reserve_from_bucket(umm, rsrvd_act, rsrv_size, mb_id); + umoff1 = umem_reserve_from_bucket(umm, rsrvd_act, rsrv_size, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umoff1 == umoff); umem_cancel(umm, rsrvd_act); @@ -1995,7 +2078,7 @@ test_reserve_from_mb(void **state) /* Reserve an object and publish it within a transaction. */ rc = umem_rsrvd_act_alloc(umm, &rsrvd_act, 1); assert_int_equal(rc, 0); - umoff = umem_reserve_from_bucket(umm, rsrvd_act, rsrv_size, mb_id); + umoff = umem_reserve_from_bucket(umm, rsrvd_act, rsrv_size, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); /* Validate that the object is from the memory bucket of interest. */ assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); @@ -2011,7 +2094,8 @@ test_reserve_from_mb(void **state) * subsequent allocation. */ for (i = 0; i < 32 * 1024; i++) { - umoff1 = umem_atomic_alloc_from_bucket(umm, rsrv_size, UMEM_TYPE_ANY, mb_id); + umoff1 = + umem_atomic_alloc_from_bucket(umm, rsrv_size, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); assert_true(umem_get_mb_from_offset(umm, umoff1) == mb_id); umem_atomic_free(umm, umoff1); @@ -2027,7 +2111,8 @@ test_reserve_from_mb(void **state) umem_rsrvd_act_free(&rsrvd_act); /* Validate that the object is not really freed */ for (i = 0; i < 32 * 1024; i++) { - umoff1 = umem_atomic_alloc_from_bucket(umm, rsrv_size, UMEM_TYPE_ANY, mb_id); + umoff1 = + umem_atomic_alloc_from_bucket(umm, rsrv_size, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); assert_true(umem_get_mb_from_offset(umm, umoff1) == mb_id); umem_atomic_free(umm, umoff1); @@ -2049,7 +2134,8 @@ test_reserve_from_mb(void **state) /* Validate that the object is returned in subsequent allocation */ found = 0; for (i = 0; i < 32 * 1024; i++) { - umoff1 = umem_atomic_alloc_from_bucket(umm, rsrv_size, UMEM_TYPE_ANY, mb_id); + umoff1 = + umem_atomic_alloc_from_bucket(umm, rsrv_size, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); assert_true(umem_get_mb_from_offset(umm, umoff1) == mb_id); umem_atomic_free(umm, umoff1); @@ -2077,14 +2163,15 @@ test_tx_alloc_from_mb(void **state) /* Do a tx alloc and fail the transaction. */ rc = umem_tx_begin(umm, NULL); assert_int_equal(rc, 0); - umoff = umem_alloc_from_bucket(umm, alloc_size, mb_id); + umoff = umem_alloc_from_bucket(umm, alloc_size, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); rc = umem_tx_end(umm, 1); assert_true(rc == umem_tx_errno(1)); found = 0; for (i = 0; i < 32 * 1024; i++) { - umoff1 = umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, mb_id); + umoff1 = + umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); assert_true(umem_get_mb_from_offset(umm, umoff1) == mb_id); umem_atomic_free(umm, umoff1); @@ -2098,13 +2185,14 @@ test_tx_alloc_from_mb(void **state) /* Do a tx alloc and pass the transaction. */ rc = umem_tx_begin(umm, NULL); assert_int_equal(rc, 0); - umoff = umem_alloc_from_bucket(umm, alloc_size, mb_id); + umoff = umem_alloc_from_bucket(umm, alloc_size, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); rc = umem_tx_end(umm, 0); assert_int_equal(rc, 0); for (i = 0; i < 32 * 1024; i++) { - umoff1 = umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, mb_id); + umoff1 = + umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); assert_true(umem_get_mb_from_offset(umm, umoff1) == mb_id); umem_atomic_free(umm, umoff1); @@ -2118,7 +2206,8 @@ test_tx_alloc_from_mb(void **state) rc = umem_tx_end(umm, 1); assert_true(rc == umem_tx_errno(1)); for (i = 0; i < 32 * 1024; i++) { - umoff1 = umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, mb_id); + umoff1 = + umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); assert_true(umem_get_mb_from_offset(umm, umoff1) == mb_id); umem_atomic_free(umm, umoff1); @@ -2133,7 +2222,8 @@ test_tx_alloc_from_mb(void **state) assert_int_equal(rc, 0); found = 0; for (i = 0; i < 32 * 1024; i++) { - umoff1 = umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, mb_id); + umoff1 = + umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff1)); assert_true(umem_get_mb_from_offset(umm, umoff1) == mb_id); umem_atomic_free(umm, umoff1); @@ -2163,7 +2253,7 @@ alloc_bucket_to_full(struct umem_instance *umm, struct bucket_alloc_info *ainfo) if (UMOFF_IS_NULL(ainfo->start_umoff)) { ainfo->start_umoff = - umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, id); + umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, BKT_REQ(id)); assert_false(UMOFF_IS_NULL(ainfo->start_umoff)); ainfo->num_allocs++; assert_true(umem_get_mb_from_offset(umm, ainfo->start_umoff) == id); @@ -2174,7 +2264,7 @@ alloc_bucket_to_full(struct umem_instance *umm, struct bucket_alloc_info *ainfo) assert_true(umem_cache_pin(&umm->umm_pool->up_store, &rg, 1, 0, &p_hdl) == 0); while (1) { - umoff = umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, id); + umoff = umem_atomic_alloc_from_bucket(umm, alloc_size, UMEM_TYPE_ANY, BKT_REQ(id)); assert_false(UMOFF_IS_NULL(umoff)); if (umem_get_mb_from_offset(umm, umoff) != id) { umem_atomic_free(umm, umoff); @@ -2614,7 +2704,8 @@ test_umempobj_heap_mb_stats(void **state) ptr = NULL; /* allocate and consume all of the space */ for (num = 0;; num++) { - umoff = umem_atomic_alloc_from_bucket(&umm, alloc_size, UMEM_TYPE_ANY, mb_id); + umoff = + umem_atomic_alloc_from_bucket(&umm, alloc_size, UMEM_TYPE_ANY, BKT_REQ(mb_id)); if (umem_get_mb_from_offset(&umm, umoff) != mb_id) { umem_atomic_free(&umm, umoff); break; @@ -2737,6 +2828,8 @@ main(int argc, char **argv) setup_pmem_v2, teardown_pmem}, {"BMEM017: Test atomic allocs overflow a memory bucket", test_atomic_alloc_overflow_mb, setup_pmem_v2, teardown_pmem}, + {"BMEM017a: Test atomic allocs spill over to the next requested memory bucket", + test_atomic_alloc_spill_two_mb, setup_pmem_v2, teardown_pmem}, {"BMEM018: Test reserve/defer_free from a memory bucket", test_reserve_from_mb, setup_pmem_v2, teardown_pmem}, {"BMEM019: Test tx alloc/free from a memory bucket", test_tx_alloc_from_mb, diff --git a/src/include/daos/mem.h b/src/include/daos/mem.h index eb4bdbf3552..c8f3d0404f6 100644 --- a/src/include/daos/mem.h +++ b/src/include/daos/mem.h @@ -685,6 +685,19 @@ struct umem_instance; /* Macros associated with Memory buckets */ #define UMEM_DEFAULT_MBKT_ID 0 +/** + * Memory bucket allocation request. + * + * Describes the set of evictable memory buckets an allocation may be served + * from, in priority order. A NULL request is equivalent to allocating from the + * default (non-evictable) bucket. + */ +typedef struct umem_bucket_req { + const uint32_t *ubr_bkt_ids; /* Ordered list of bucket IDs (allocation priority) */ + uint32_t ubr_bkt_cnt; /* Number of valid entries in ubr_bkt_ids */ + uint32_t ubr_bkt_max; /* Max E-buckets allowed for the object */ +} umem_bucket_req_t; + /* type num used by umem ops */ enum { UMEM_TYPE_ANY, @@ -707,10 +720,10 @@ typedef struct { * \param size [IN] size to allocate. * \param flags [IN] flags like zeroing, noflush (for PMDK and BMEM) * \param type_num [IN] struct type (for PMDK and BMEM) - * \param mbkt_id [IN] memory bucket id (for BMEM) + * \param req [IN] memory bucket request (for BMEM), NULL for default bucket */ umem_off_t (*mo_tx_alloc)(struct umem_instance *umm, size_t size, uint64_t flags, - unsigned int type_num, unsigned int mbkt_id); + unsigned int type_num, struct umem_bucket_req *req); /** * Add the specified range of umoff to current memory transaction. * @@ -773,10 +786,10 @@ typedef struct { * \param act [IN|OUT] action used for later cancel/publish. * \param size [IN] size to be reserved. * \param type_num [IN] struct type (for PMDK) - * \param mbkt_id [IN] memory bucket id (for BMEM) + * \param req [IN] memory bucket request (for BMEM), NULL for default bucket */ umem_off_t (*mo_reserve)(struct umem_instance *umm, void *act, size_t size, - unsigned int type_num, unsigned int mbkt_id); + unsigned int type_num, struct umem_bucket_req *req); /** * Defer free til commit. For use with reserved extents that are not @@ -830,10 +843,10 @@ typedef struct { * \param size [IN] size to allocate. * \param flags [IN] flags like zeroing, noflush (for PMDK) * \param type_num [IN] struct type (for PMDK) - * \param mbkt_id [IN] memory bucket id (for BMEM) + * \param req [IN] memory bucket request (for BMEM), NULL for default bucket */ umem_off_t (*mo_atomic_alloc)(struct umem_instance *umm, size_t size, unsigned int type_num, - unsigned int mbkt_id); + struct umem_bucket_req *req); /** * flush data at specific offset to persistent store. @@ -956,11 +969,11 @@ umem_has_tx(struct umem_instance *umm) return umm->umm_ops->mo_tx_add != NULL; } -#define umem_alloc_verb(umm, flags, size, mbkt_id) \ +#define umem_alloc_verb(umm, flags, size, req) \ ({ \ umem_off_t __umoff; \ \ - __umoff = (umm)->umm_ops->mo_tx_alloc(umm, size, flags, UMEM_TYPE_ANY, mbkt_id); \ + __umoff = (umm)->umm_ops->mo_tx_alloc(umm, size, flags, UMEM_TYPE_ANY, req); \ D_ASSERTF(umem_off2flags(__umoff) == 0, \ "Invalid assumption about allocnot using flag bits"); \ D_DEBUG(DB_MEM, \ @@ -971,17 +984,15 @@ umem_has_tx(struct umem_instance *umm) __umoff; \ }) -#define umem_alloc(umm, size) umem_alloc_verb(umm, 0, size, UMEM_DEFAULT_MBKT_ID) +#define umem_alloc(umm, size) umem_alloc_verb(umm, 0, size, NULL) -#define umem_alloc_from_bucket(umm, size, mbkt_id) umem_alloc_verb(umm, 0, size, mbkt_id) +#define umem_alloc_from_bucket(umm, size, req) umem_alloc_verb(umm, 0, size, req) -#define umem_zalloc(umm, size) umem_alloc_verb(umm, UMEM_FLAG_ZERO, size, UMEM_DEFAULT_MBKT_ID) +#define umem_zalloc(umm, size) umem_alloc_verb(umm, UMEM_FLAG_ZERO, size, NULL) -#define umem_zalloc_from_bucket(umm, size, mbkt_id) \ - umem_alloc_verb(umm, UMEM_FLAG_ZERO, size, mbkt_id) +#define umem_zalloc_from_bucket(umm, size, req) umem_alloc_verb(umm, UMEM_FLAG_ZERO, size, req) -#define umem_alloc_noflush(umm, size) \ - umem_alloc_verb(umm, UMEM_FLAG_NO_FLUSH, size, UMEM_DEFAULT_MBKT_ID) +#define umem_alloc_noflush(umm, size) umem_alloc_verb(umm, UMEM_FLAG_NO_FLUSH, size, NULL) #define umem_free(umm, umoff) \ ({ \ @@ -1165,11 +1176,10 @@ int umem_rsrvd_act_free(struct umem_rsrvd_act **act); umem_off_t umem_reserve_common(struct umem_instance *umm, struct umem_rsrvd_act *rsrvd_act, size_t size, - unsigned int mbkt_id); -#define umem_reserve(umm, rsrvd_act, size) \ - umem_reserve_common(umm, rsrvd_act, size, UMEM_DEFAULT_MBKT_ID) -#define umem_reserve_from_bucket(umm, rsrvd_act, size, mbkt_id) \ - umem_reserve_common(umm, rsrvd_act, size, mbkt_id) + struct umem_bucket_req *req); +#define umem_reserve(umm, rsrvd_act, size) umem_reserve_common(umm, rsrvd_act, size, NULL) +#define umem_reserve_from_bucket(umm, rsrvd_act, size, req) \ + umem_reserve_common(umm, rsrvd_act, size, req) void umem_defer_free(struct umem_instance *umm, umem_off_t off, struct umem_rsrvd_act *rsrvd_act); @@ -1190,15 +1200,15 @@ static inline umem_off_t umem_atomic_alloc(struct umem_instance *umm, size_t len, unsigned int type_num) { D_ASSERT(umm->umm_ops->mo_atomic_alloc != NULL); - return umm->umm_ops->mo_atomic_alloc(umm, len, type_num, UMEM_DEFAULT_MBKT_ID); + return umm->umm_ops->mo_atomic_alloc(umm, len, type_num, NULL); } static inline umem_off_t umem_atomic_alloc_from_bucket(struct umem_instance *umm, size_t len, unsigned int type_num, - unsigned int mbkt_id) + struct umem_bucket_req *req) { D_ASSERT(umm->umm_ops->mo_atomic_alloc != NULL); - return umm->umm_ops->mo_atomic_alloc(umm, len, type_num, mbkt_id); + return umm->umm_ops->mo_atomic_alloc(umm, len, type_num, req); } static inline int diff --git a/src/vos/tests/vts_wal.c b/src/vos/tests/vts_wal.c index 05946c0bc8f..eac48377d0c 100644 --- a/src/vos/tests/vts_wal.c +++ b/src/vos/tests/vts_wal.c @@ -23,6 +23,10 @@ #define WAL_IO_MULTI_KEYS 10000 #define WAL_OBJ_KEYS 31 +/** Build a single-bucket allocation request for the given bucket id. */ +#define BKT_REQ(id) \ + (&(umem_bucket_req_t){.ubr_bkt_ids = &(uint32_t){id}, .ubr_bkt_cnt = 1, .ubr_bkt_max = 0}) + /* Define WAL_IO_EXTRA_CHK to one for comprehensive type checking */ #define WAL_IO_EXTRA_CHK 0 @@ -1344,7 +1348,7 @@ wal_mb_tests(void **state) mb_id = umem_allot_mb_evictable(umm, 0); assert_true(mb_id != 0); umem_tx_begin(umm, NULL); - umoff = umem_alloc_from_bucket(umm, 1024, mb_id); + umoff = umem_alloc_from_bucket(umm, 1024, BKT_REQ(mb_id)); assert_false(UMOFF_IS_NULL(umoff)); assert_true(umem_get_mb_from_offset(umm, umoff) == mb_id); ptr = umem_off2ptr(umm, umoff); @@ -1407,7 +1411,7 @@ alloc_bucket_to_full(struct umem_instance *umm, struct bucket_alloc_info *ainfo, if (UMOFF_IS_NULL(ainfo->start_umoff)) { umem_tx_begin(umm, NULL); - ainfo->start_umoff = umem_alloc_from_bucket(umm, alloc_size, id); + ainfo->start_umoff = umem_alloc_from_bucket(umm, alloc_size, BKT_REQ(id)); assert_false(UMOFF_IS_NULL(ainfo->start_umoff)); ainfo->num_allocs++; if (!ainfo->allow_spill) @@ -1429,7 +1433,7 @@ alloc_bucket_to_full(struct umem_instance *umm, struct bucket_alloc_info *ainfo, while (1) { umem_tx_begin(umm, NULL); - umoff = umem_alloc_from_bucket(umm, alloc_size, id); + umoff = umem_alloc_from_bucket(umm, alloc_size, BKT_REQ(id)); if (UMOFF_IS_NULL(umoff) || (!ainfo->allow_spill && (umem_get_mb_from_offset(umm, umoff) != id))) { @@ -2386,7 +2390,8 @@ wal_umempobj_block_reuse_internal(void **state, int restart) for (j = 0; j < free_num[i]; j++) { umem_tx_begin(umm, NULL); - umoff = umem_alloc_from_bucket(umm, ainfo[i].alloc_size, ainfo[i].mb_id); + umoff = umem_alloc_from_bucket(umm, ainfo[i].alloc_size, + BKT_REQ(ainfo[i].mb_id)); assert_true(!UMOFF_IS_NULL(umoff)); umem_tx_commit(umm); assert_true(umoff_in_freelist(free_list[i], free_num[i], umoff, true)); diff --git a/src/vos/vos_internal.h b/src/vos/vos_internal.h index a2be290db34..5cf97aa3421 100644 --- a/src/vos/vos_internal.h +++ b/src/vos/vos_internal.h @@ -1975,11 +1975,18 @@ vos_obj_alloc(struct umem_instance *umm, struct vos_object *obj, size_t size, bo { if (obj != NULL && vos_pool_is_evictable(vos_obj2pool(obj))) { + /* TEMP FIX: ubr_bkt_max = 0 allows unbounded spill-over to SOE buckets. */ + umem_bucket_req_t req = { + .ubr_bkt_ids = &obj->obj_bkt_id0, + .ubr_bkt_cnt = 1, + .ubr_bkt_max = 0, + }; + D_ASSERT(obj->obj_bkt_alloted == 1); if (zeroing) - return umem_zalloc_from_bucket(umm, size, obj->obj_bkt_id0); + return umem_zalloc_from_bucket(umm, size, &req); - return umem_alloc_from_bucket(umm, size, obj->obj_bkt_id0); + return umem_alloc_from_bucket(umm, size, &req); } if (zeroing) @@ -1993,8 +2000,15 @@ vos_obj_reserve(struct umem_instance *umm, struct vos_object *obj, struct umem_rsrvd_act *rsrvd_scm, daos_size_t size) { if (obj != NULL && vos_pool_is_evictable(vos_obj2pool(obj))) { + /* TEMP FIX: ubr_bkt_max = 0 allows unbounded spill-over to SOE buckets. */ + umem_bucket_req_t req = { + .ubr_bkt_ids = &obj->obj_bkt_id0, + .ubr_bkt_cnt = 1, + .ubr_bkt_max = 0, + }; + D_ASSERT(obj->obj_bkt_alloted == 1); - return umem_reserve_from_bucket(umm, rsrvd_scm, size, obj->obj_bkt_id0); + return umem_reserve_from_bucket(umm, rsrvd_scm, size, &req); } return umem_reserve(umm, rsrvd_scm, size);