From da8124e472d353b05e8b383b3243bf5cfadc7cde Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 2 Jul 2018 19:00:55 +0200 Subject: [PATCH] memory: use hash_table to store allocation metadata --- include/villas/hash_table.h | 10 +++ include/villas/memory.h | 42 +++++++---- include/villas/memory_type.h | 9 ++- include/villas/pool.h | 2 - include/villas/queue.h | 1 - lib/Makefile.villas-ext.inc | 2 +- lib/memory.c | 47 +++++++++--- lib/memory/heap.c | 31 +++++--- lib/memory/hugepage.c | 41 +++++++---- lib/memory/ib.c | 36 +++++---- lib/memory/managed.c | 138 +++++++++++++++++------------------ lib/pool.c | 3 +- lib/queue.c | 8 +- lib/shmem.c | 4 +- tests/unit/memory.c | 24 +++--- 15 files changed, 241 insertions(+), 157 deletions(-) diff --git a/include/villas/hash_table.h b/include/villas/hash_table.h index d8a79f873..87726184f 100644 --- a/include/villas/hash_table.h +++ b/include/villas/hash_table.h @@ -20,11 +20,17 @@ * along with this program. If not, see . *********************************************************************************/ +#pragma once + #include #include #include +#ifdef __cplusplus +extern "C" { +#endif + struct hash_table_entry { void *key; void *data; @@ -73,3 +79,7 @@ void * hash_table_lookup(struct hash_table *ht, void *key); /** Dump the contents of the hash table in a human readable format to stdout. */ void hash_table_dump(struct hash_table *ht); + +#ifdef __cplusplus +} +#endif diff --git a/include/villas/memory.h b/include/villas/memory.h index 95ee21972..616377d20 100644 --- a/include/villas/memory.h +++ b/include/villas/memory.h @@ -25,6 +25,7 @@ #include #include +#include #include @@ -35,26 +36,35 @@ extern "C" { /* Forward declarations */ struct node; -enum memblock_flags { - MEMBLOCK_USED = 1, -}; - /** Descriptor of a memory block. Associated block always starts at - * &m + sizeof(struct memblock). */ -struct memblock { - struct memblock *prev; - struct memblock *next; - size_t len; /** #include #include +#include #include +static struct hash_table allocations = { .state = STATE_DESTROYED }; + int memory_init(int hugepages) { + int ret; + + if (allocations.state == STATE_DESTROYED) { + ret = hash_table_init(&allocations, 100); + if (ret) + return ret; + } + #ifdef __linux__ int ret, pagecnt, pagesz; struct rlimit l; @@ -71,25 +82,39 @@ int memory_init(int hugepages) void * memory_alloc(struct memory_type *m, size_t len) { - void *ptr = m->alloc(m, len, sizeof(void *)); - - debug(LOG_MEM | 5, "Allocated %#zx bytes of %s memory: %p", len, m->name, ptr); - - return ptr; + return memory_alloc_aligned(m, len, sizeof(void *)); } void * memory_alloc_aligned(struct memory_type *m, size_t len, size_t alignment) { - void *ptr = m->alloc(m, len, alignment); + struct memory_allocation *ma = m->alloc(m, len, alignment); - debug(LOG_MEM | 5, "Allocated %#zx bytes of %#zx-byte-aligned %s memory: %p", len, alignment, m->name, ptr); + hash_table_insert(&allocations, ma->address, ma); - return ptr; + debug(LOG_MEM | 5, "Allocated %#zx bytes of %#zx-byte-aligned %s memory: %p", ma->length, ma->alignment, ma->type->name, ma->address); + + return ma->address; } -int memory_free(struct memory_type *m, void *ptr, size_t len) +int memory_free(void *ptr) { - debug(LOG_MEM | 5, "Releasing %#zx bytes of %s memory", len, m->name); + int ret; - return m->free(m, ptr, len); + /* Find corresponding memory allocation entry */ + struct memory_allocation *ma = (struct memory_allocation *) hash_table_lookup(&allocations, ptr); + if (!ma) + return -1; + + debug(LOG_MEM | 5, "Releasing %#zx bytes of %s memory", ma->length, ma->type->name); + + ret = ma->type->free(ma->type, ma); + if (ret) + return ret; + + /* Remove allocation entry */ + ret = hash_table_delete(&allocations, ma->address); + if (ret) + return ret; + + return 0; } diff --git a/lib/memory/heap.c b/lib/memory/heap.c index 7a70abf63..1a2c61552 100644 --- a/lib/memory/heap.c +++ b/lib/memory/heap.c @@ -22,24 +22,37 @@ #include -#include +#include +#include -static void * memory_heap_alloc(struct memory_type *m, size_t len, size_t alignment) +static struct memory_allocation * memory_heap_alloc(struct memory_type *m, size_t len, size_t alignment) { - void *ptr; int ret; - if (alignment < sizeof(void *)) - alignment = sizeof(void *); + struct memory_allocation *ma = alloc(sizeof(struct memory_allocation)); + if (!ma) + return NULL; - ret = posix_memalign(&ptr, alignment, len); + ma->alignment = alignment; + ma->type = m; + ma->length = len; - return ret ? NULL : ptr; + if (ma->alignment < sizeof(void *)) + ma->alignment = sizeof(void *); + + ret = posix_memalign(&ma->address, ma->alignment, ma->length); + if (ret) { + free(ma); + return ret; + } + + return ma; } -int memory_heap_free(struct memory_type *m, void *ptr, size_t len) +static int memory_heap_free(struct memory_type *m, struct memory_allocation *ma) { - free(ptr); + free(ma->address); + free(ma); return 0; } diff --git a/lib/memory/hugepage.c b/lib/memory/hugepage.c index 83abbcf15..6bf17719c 100644 --- a/lib/memory/hugepage.c +++ b/lib/memory/hugepage.c @@ -38,15 +38,14 @@ #endif #include -#include +#include #include -#define HUGEPAGESIZE (1 << 21) /* 2 MiB */ +#define HUGEPAGESIZE (1 << 22) /* 2 MiB */ /** Allocate memory backed by hugepages with malloc() like interface */ -static void * memory_hugepage_alloc(struct memory_type *m, size_t len, size_t alignment) +static struct memory_allocation * memory_hugepage_alloc(struct memory_type *m, size_t len, size_t alignment) { - void *ret; int prot = PROT_READ | PROT_WRITE; int flags = MAP_PRIVATE | MAP_ANONYMOUS; @@ -59,22 +58,38 @@ static void * memory_hugepage_alloc(struct memory_type *m, size_t len, size_t al flags |= MAP_LOCKED; #endif - ret = mmap(NULL, len, prot, flags, -1, 0); - if (ret == MAP_FAILED) + struct memory_allocation *ma = alloc(sizeof(struct memory_allocation)); + if (!ma) return NULL; - return ret; -} - -static int memory_hugepage_free(struct memory_type *m, void *ptr, size_t len) -{ /** We must make sure that len is a multiple of the hugepage size * * See: https://lkml.org/lkml/2014/10/22/925 */ - len = ALIGN(len, HUGEPAGESIZE); + ma->length = ALIGN(len, HUGEPAGESIZE); + ma->alignment = alignment; + ma->type = m; - return munmap(ptr, len); + ma->address = mmap(NULL, len, prot, flags, -1, 0); + if (ma->address == MAP_FAILED) { + free(ma); + return NULL; + } + + return ma; +} + +static int memory_hugepage_free(struct memory_type *m, struct memory_allocation *ma) +{ + int ret; + + ret = munmap(ma->address, ma->length); + if (ret) + return ret; + + free(ma); + + return 0; } struct memory_type memory_hugepage = { diff --git a/lib/memory/ib.c b/lib/memory/ib.c index 38955e4d6..fe62e223f 100644 --- a/lib/memory/ib.c +++ b/lib/memory/ib.c @@ -36,33 +36,43 @@ struct ibv_mr * memory_ib_mr(void *ptr) return (mr - 1); } -void * memory_ib_alloc(struct memory_type *m, size_t len, size_t alignment) +static struct memory_allocation * memory_ib_alloc(struct memory_type *m, size_t len, size_t alignment) { struct memory_ib *mi = (struct memory_ib *) m->_vd; - struct ibv_mr **mr = memory_alloc_aligned(mi->parent, len + sizeof(struct ibv_mr *), alignment); - char *ptr = (char *) (mr + 1); + struct memory_allocation *ma = alloc(sizeof(struct memory_allocation)); + if (!ma) + return NULL; - *mr = ibv_reg_mr(mi->pd, ptr, len, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); - if(!*mr) { - free(ptr); + ma->type = m; + ma->length = len; + ma->alignment = alignment; + + ma->parent = mi->parent->alloc(mi->parent, len + sizeof(struct ibv_mr *), alignment); + ma->address = ma->parent->address; + + ma->ib.mr = ibv_reg_mr(mi->pd, ma->address, ma->length, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + if(!ma->ib.mr) { + mi->parent->free(mi->parent, ma->parent); + free(ma); return NULL; } - return ptr; + return ma; } -int memory_ib_free(struct memory_type *m, void *ptr, size_t len) +static int memory_ib_free(struct memory_type *m, struct memory_allocation *ma) { + int ret; struct memory_ib *mi = (struct memory_ib *) m->_vd; - struct ibv_mr *mr = memory_ib_mr(ptr); - ibv_dereg_mr(mr); + ibv_dereg_mr(ma->ib.mr); - ptr -= sizeof(struct ibv_mr *); - len += sizeof(struct ibv_mr *); + ret = mi->parent->free(mi->parent, ma->parent); + if (ret) + return ret; - memory_free(mi->parent, ptr, len); + free(ma); return 0; } diff --git a/lib/memory/managed.c b/lib/memory/managed.c index 05ea6f439..ab8bc4b59 100644 --- a/lib/memory/managed.c +++ b/lib/memory/managed.c @@ -34,18 +34,18 @@ #include #include -void* memory_managed_alloc(struct memory_type *m, size_t len, size_t alignment) +static struct memory_allocation * memory_managed_alloc(struct memory_type *m, size_t len, size_t alignment) { /* Simple first-fit allocation */ - struct memblock *first = (struct memblock *) m->_vd; - struct memblock *block; + struct memory_block *first = (struct memory_block *) m->_vd; + struct memory_block *block; for (block = first; block != NULL; block = block->next) { - if (block->flags & MEMBLOCK_USED) + if (block->used) continue; - char* cptr = (char *) block + sizeof(struct memblock); - size_t avail = block->len; + char* cptr = (char *) block + sizeof(struct memory_block); + size_t avail = block->length; uintptr_t uptr = (uintptr_t) cptr; /* Check alignment first; leave a gap at start of block to assure @@ -62,47 +62,59 @@ void* memory_managed_alloc(struct memory_type *m, size_t len, size_t alignment) } if (avail >= len) { - if (gap > sizeof(struct memblock)) { + if (gap > sizeof(struct memory_block)) { /* The alignment gap is big enough to fit another block. * The original block descriptor is already at the correct * position, so we just change its len and create a new block * descriptor for the actual block we're handling. */ - block->len = gap - sizeof(struct memblock); - struct memblock *newblock = (struct memblock *) (cptr - sizeof(struct memblock)); + block->length = gap - sizeof(struct memory_block); + struct memory_block *newblock = (struct memory_block *) (cptr - sizeof(struct memory_block)); newblock->prev = block; newblock->next = block->next; block->next = newblock; - newblock->flags = 0; - newblock->len = len; + newblock->used = false; + newblock->length = len; block = newblock; } else { /* The gap is too small to fit another block descriptor, so we * must account for the gap length in the block length. */ - block->len = len + gap; + block->length = len + gap; } - if (avail > len + sizeof(struct memblock)) { + if (avail > len + sizeof(struct memory_block)) { /* Imperfect fit, so create another block for the remaining part */ - struct memblock *newblock = (struct memblock *) (cptr + len); + struct memory_block *newblock = (struct memory_block *) (cptr + len); newblock->prev = block; newblock->next = block->next; block->next = newblock; + if (newblock->next) newblock->next->prev = newblock; - newblock->flags = 0; - newblock->len = avail - len - sizeof(struct memblock); + + newblock->used = false; + newblock->length = avail - len - sizeof(struct memory_block); } else { /* If this block was larger than the requested length, but only - * by less than sizeof(struct memblock), we may have wasted - * memory by previous assignments to block->len. */ - block->len = avail; + * by less than sizeof(struct memory_block), we may have wasted + * memory by previous assignments to block->length. */ + block->length = avail; } - block->flags |= MEMBLOCK_USED; + block->used = true; - return (void *) cptr; + struct memory_allocation *ma = alloc(sizeof(struct memory_allocation)); + if (!ma) + return NULL; + + ma->address = cptr; + ma->type = m; + ma->alignment = alignment; + ma->length = len; + ma->managed.block = block; + + return ma; } } @@ -110,60 +122,48 @@ void* memory_managed_alloc(struct memory_type *m, size_t len, size_t alignment) return NULL; } -int memory_managed_free(struct memory_type *m, void *ptr, size_t len) +static int memory_managed_free(struct memory_type *m, struct memory_allocation *ma) { - struct memblock *first = (struct memblock *) m->_vd; - struct memblock *block; - char *cptr = ptr; + struct memory_block *block = ma->managed.block; - for (block = first; block != NULL; block = block->next) { - if (!(block->flags & MEMBLOCK_USED)) - continue; - - /* Since we may waste some memory at the start of a block to ensure - * alignment, ptr may not actually be the start of the block */ - if ((char *) block + sizeof(struct memblock) <= cptr && - cptr < (char *) block + sizeof(struct memblock) + block->len) { - /* Try to merge it with neighbouring free blocks */ - if (block->prev && !(block->prev->flags & MEMBLOCK_USED) && - block->next && !(block->next->flags & MEMBLOCK_USED)) { - /* Special case first: both previous and next block are unused */ - block->prev->len += block->len + block->next->len + 2 * sizeof(struct memblock); - block->prev->next = block->next->next; - if (block->next->next) - block->next->next->prev = block->prev; - } - else if (block->prev && !(block->prev->flags & MEMBLOCK_USED)) { - block->prev->len += block->len + sizeof(struct memblock); - block->prev->next = block->next; - if (block->next) - block->next->prev = block->prev; - } - else if (block->next && !(block->next->flags & MEMBLOCK_USED)) { - block->len += block->next->len + sizeof(struct memblock); - block->next = block->next->next; - if (block->next) - block->next->prev = block; - } - else { - /* no neighbouring free block, so just mark it as free */ - block->flags &= ~MEMBLOCK_USED; - } - - return 0; - } + /* Try to merge it with neighbouring free blocks */ + if (block->prev && !block->prev->used && + block->next && !block->next->used) { + /* Special case first: both previous and next block are unused */ + block->prev->length += block->length + block->next->length + 2 * sizeof(struct memory_block); + block->prev->next = block->next->next; + if (block->next->next) + block->next->next->prev = block->prev; + } + else if (block->prev && !block->prev->used) { + block->prev->length += block->length + sizeof(struct memory_block); + block->prev->next = block->next; + if (block->next) + block->next->prev = block->prev; + } + else if (block->next && !block->next->used) { + block->length += block->next->length + sizeof(struct memory_block); + block->next = block->next->next; + if (block->next) + block->next->prev = block; + } + else { + /* no neighbouring free block, so just mark it as free */ + block->used = false; } - return -1; + free(ma); + + return 0; } struct memory_type * memory_managed(void *ptr, size_t len) { struct memory_type *mt = ptr; - struct memblock *mb; + struct memory_block *mb; char *cptr = ptr; - if (len < sizeof(struct memory_type) + sizeof(struct memblock)) { + if (len < sizeof(struct memory_type) + sizeof(struct memory_block)) { info("memory_managed: passed region too small"); return NULL; } @@ -177,15 +177,15 @@ struct memory_type * memory_managed(void *ptr, size_t len) cptr += ALIGN(sizeof(struct memory_type), sizeof(void *)); - /* Initialize first free memblock */ - mb = (struct memblock *) cptr; + /* Initialize first free memory block */ + mb = (struct memory_block *) cptr; mb->prev = NULL; mb->next = NULL; - mb->flags = 0; + mb->used = false; - cptr += ALIGN(sizeof(struct memblock), sizeof(void *)); + cptr += ALIGN(sizeof(struct memory_block), sizeof(void *)); - mb->len = len - (cptr - (char *) ptr); + mb->length = len - (cptr - (char *) ptr); mt->_vd = (void *) mb; diff --git a/lib/pool.c b/lib/pool.c index 78b47f2c8..1a7ce726f 100644 --- a/lib/pool.c +++ b/lib/pool.c @@ -35,7 +35,6 @@ int pool_init(struct pool *p, size_t cnt, size_t blocksz, struct memory_type *m) p->alignment = kernel_get_cacheline_size(); p->blocksz = p->alignment * CEIL(blocksz, p->alignment); p->len = cnt * p->blocksz; - p->mem = m; void *buffer = memory_alloc_aligned(m, p->len, p->alignment); if (!buffer) @@ -66,7 +65,7 @@ int pool_destroy(struct pool *p) queue_destroy(&p->queue); void *buffer = (char*) p + p->buffer_off; - ret = memory_free(p->mem, buffer, p->len); + ret = memory_free(buffer); if (ret == 0) p->state = STATE_DESTROYED; diff --git a/lib/queue.c b/lib/queue.c index 438a2c523..0b020b192 100644 --- a/lib/queue.c +++ b/lib/queue.c @@ -36,7 +36,7 @@ #include /** Initialize MPMC queue */ -int queue_init(struct queue *q, size_t size, struct memory_type *mem) +int queue_init(struct queue *q, size_t size, struct memory_type *m) { assert(q->state == STATE_DESTROYED); @@ -47,9 +47,8 @@ int queue_init(struct queue *q, size_t size, struct memory_type *mem) warn("A queue size was changed from %lu to %lu", old_size, size); } - q->mem = mem; q->buffer_mask = size - 1; - struct queue_cell *buffer = (struct queue_cell *) memory_alloc(q->mem, sizeof(struct queue_cell) * size); + struct queue_cell *buffer = (struct queue_cell *) memory_alloc(m, sizeof(struct queue_cell) * size); if (!buffer) return -2; @@ -74,8 +73,7 @@ int queue_destroy(struct queue *q) if (q->state == STATE_DESTROYED) return 0; - ret = memory_free(q->mem, buffer, (q->buffer_mask + 1) * sizeof(struct queue_cell)); - + ret = memory_free(buffer); if (ret == 0) q->state = STATE_DESTROYED; diff --git a/lib/shmem.c b/lib/shmem.c index 65042a155..294e7ae67 100644 --- a/lib/shmem.c +++ b/lib/shmem.c @@ -44,7 +44,7 @@ size_t shmem_total_size(int queuelen, int samplelen) /* the size of the pool */ + queuelen * kernel_get_cacheline_size() * CEIL(SAMPLE_LEN(samplelen), kernel_get_cacheline_size()) /* a memblock for each allocation (1 shmem_shared, 2 queues, 1 pool) */ - + 4 * sizeof(struct memblock) + + 4 * sizeof(struct memory_block) /* and some extra buffer for alignment */ + 1024; } @@ -144,7 +144,7 @@ retry: fd = shm_open(wname, O_RDWR|O_CREAT|O_EXCL, 0600); if (base == MAP_FAILED) return -10; - cptr = (char *) base + sizeof(struct memory_type) + sizeof(struct memblock); + cptr = (char *) base + sizeof(struct memory_type) + sizeof(struct memory_block); shared = (struct shmem_shared *) cptr; shm->read.base = base; shm->read.name = rname; diff --git a/tests/unit/memory.c b/tests/unit/memory.c index 38ccbe51a..821795afd 100644 --- a/tests/unit/memory.c +++ b/tests/unit/memory.c @@ -28,7 +28,7 @@ #include #include -#define HUGEPAGESIZE (1<<22) +#define HUGEPAGESIZE (1 << 22) TheoryDataPoints(memory, aligned) = { DataPoints(size_t, 1, 32, 55, 1 << 10, 1 << 20), @@ -40,6 +40,9 @@ Theory((size_t len, size_t align, struct memory_type *m), memory, aligned) { int ret; void *ptr; + ret = memory_init(100); + cr_assert(!ret); + ptr = memory_alloc_aligned(m, len, align); cr_assert_neq(ptr, NULL, "Failed to allocate memory"); @@ -49,7 +52,7 @@ Theory((size_t len, size_t align, struct memory_type *m), memory, aligned) { cr_assert(IS_ALIGNED(ptr, HUGEPAGESIZE)); } - ret = memory_free(m, ptr, len); + ret = memory_free(ptr); cr_assert_eq(ret, 0, "Failed to release memory: ret=%d, ptr=%p, len=%zu: %s", ret, ptr, len, strerror(errno)); } @@ -62,7 +65,10 @@ Test(memory, manager) { struct memory_type *m; total_size = 1 << 10; - max_block = total_size - sizeof(struct memory_type) - sizeof(struct memblock); + max_block = total_size - sizeof(struct memory_type) - sizeof(struct memory_block); + + ret = memory_init(0); + cr_assert(!ret); p = memory_alloc(&memory_type_heap, total_size); cr_assert_not_null(p); @@ -76,7 +82,7 @@ Test(memory, manager) { p2 = memory_alloc(m, 32); cr_assert_not_null(p2); - ret = memory_free(m, p1, 16); + ret = memory_free(p1); cr_assert(ret == 0); p1 = memory_alloc_aligned(m, 128, 128); @@ -87,21 +93,21 @@ Test(memory, manager) { cr_assert(p3); cr_assert(IS_ALIGNED(p3, 256)); - ret = memory_free(m, p2, 32); + ret = memory_free(p2); cr_assert(ret == 0); - ret = memory_free(m, p1, 128); + ret = memory_free(p1); cr_assert(ret == 0); - ret = memory_free(m, p3, 128); + ret = memory_free(p3); cr_assert(ret == 0); p1 = memory_alloc(m, max_block); cr_assert_not_null(p1); - ret = memory_free(m, p1, max_block); + ret = memory_free(p1); cr_assert(ret == 0); - ret = memory_free(&memory_type_heap, p, total_size); + ret = memory_free(p); cr_assert(ret == 0); }