diff --git a/include/villas/hash_table.h b/include/villas/hash_table.h
index d8a79f873..87726184f 100644
--- a/include/villas/hash_table.h
+++ b/include/villas/hash_table.h
@@ -20,11 +20,17 @@
* along with this program. If not, see .
*********************************************************************************/
+#pragma once
+
#include
#include
#include
+#ifdef __cplusplus
+extern "C" {
+#endif
+
struct hash_table_entry {
void *key;
void *data;
@@ -73,3 +79,7 @@ void * hash_table_lookup(struct hash_table *ht, void *key);
/** Dump the contents of the hash table in a human readable format to stdout. */
void hash_table_dump(struct hash_table *ht);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/villas/memory.h b/include/villas/memory.h
index 95ee21972..616377d20 100644
--- a/include/villas/memory.h
+++ b/include/villas/memory.h
@@ -25,6 +25,7 @@
#include
#include
+#include
#include
@@ -35,26 +36,35 @@ extern "C" {
/* Forward declarations */
struct node;
-enum memblock_flags {
- MEMBLOCK_USED = 1,
-};
-
/** Descriptor of a memory block. Associated block always starts at
- * &m + sizeof(struct memblock). */
-struct memblock {
- struct memblock *prev;
- struct memblock *next;
- size_t len; /**
#include
#include
+#include
#include
+static struct hash_table allocations = { .state = STATE_DESTROYED };
+
int memory_init(int hugepages)
{
+ int ret;
+
+ if (allocations.state == STATE_DESTROYED) {
+ ret = hash_table_init(&allocations, 100);
+ if (ret)
+ return ret;
+ }
+
#ifdef __linux__
int ret, pagecnt, pagesz;
struct rlimit l;
@@ -71,25 +82,39 @@ int memory_init(int hugepages)
void * memory_alloc(struct memory_type *m, size_t len)
{
- void *ptr = m->alloc(m, len, sizeof(void *));
-
- debug(LOG_MEM | 5, "Allocated %#zx bytes of %s memory: %p", len, m->name, ptr);
-
- return ptr;
+ return memory_alloc_aligned(m, len, sizeof(void *));
}
void * memory_alloc_aligned(struct memory_type *m, size_t len, size_t alignment)
{
- void *ptr = m->alloc(m, len, alignment);
+ struct memory_allocation *ma = m->alloc(m, len, alignment);
- debug(LOG_MEM | 5, "Allocated %#zx bytes of %#zx-byte-aligned %s memory: %p", len, alignment, m->name, ptr);
+ hash_table_insert(&allocations, ma->address, ma);
- return ptr;
+ debug(LOG_MEM | 5, "Allocated %#zx bytes of %#zx-byte-aligned %s memory: %p", ma->length, ma->alignment, ma->type->name, ma->address);
+
+ return ma->address;
}
-int memory_free(struct memory_type *m, void *ptr, size_t len)
+int memory_free(void *ptr)
{
- debug(LOG_MEM | 5, "Releasing %#zx bytes of %s memory", len, m->name);
+ int ret;
- return m->free(m, ptr, len);
+ /* Find corresponding memory allocation entry */
+ struct memory_allocation *ma = (struct memory_allocation *) hash_table_lookup(&allocations, ptr);
+ if (!ma)
+ return -1;
+
+ debug(LOG_MEM | 5, "Releasing %#zx bytes of %s memory", ma->length, ma->type->name);
+
+ ret = ma->type->free(ma->type, ma);
+ if (ret)
+ return ret;
+
+ /* Remove allocation entry */
+ ret = hash_table_delete(&allocations, ma->address);
+ if (ret)
+ return ret;
+
+ return 0;
}
diff --git a/lib/memory/heap.c b/lib/memory/heap.c
index 7a70abf63..1a2c61552 100644
--- a/lib/memory/heap.c
+++ b/lib/memory/heap.c
@@ -22,24 +22,37 @@
#include
-#include
+#include
+#include
-static void * memory_heap_alloc(struct memory_type *m, size_t len, size_t alignment)
+static struct memory_allocation * memory_heap_alloc(struct memory_type *m, size_t len, size_t alignment)
{
- void *ptr;
int ret;
- if (alignment < sizeof(void *))
- alignment = sizeof(void *);
+ struct memory_allocation *ma = alloc(sizeof(struct memory_allocation));
+ if (!ma)
+ return NULL;
- ret = posix_memalign(&ptr, alignment, len);
+ ma->alignment = alignment;
+ ma->type = m;
+ ma->length = len;
- return ret ? NULL : ptr;
+ if (ma->alignment < sizeof(void *))
+ ma->alignment = sizeof(void *);
+
+ ret = posix_memalign(&ma->address, ma->alignment, ma->length);
+ if (ret) {
+ free(ma);
+ return ret;
+ }
+
+ return ma;
}
-int memory_heap_free(struct memory_type *m, void *ptr, size_t len)
+static int memory_heap_free(struct memory_type *m, struct memory_allocation *ma)
{
- free(ptr);
+ free(ma->address);
+ free(ma);
return 0;
}
diff --git a/lib/memory/hugepage.c b/lib/memory/hugepage.c
index 83abbcf15..6bf17719c 100644
--- a/lib/memory/hugepage.c
+++ b/lib/memory/hugepage.c
@@ -38,15 +38,14 @@
#endif
#include
-#include
+#include
#include
-#define HUGEPAGESIZE (1 << 21) /* 2 MiB */
+#define HUGEPAGESIZE (1 << 22) /* 2 MiB */
/** Allocate memory backed by hugepages with malloc() like interface */
-static void * memory_hugepage_alloc(struct memory_type *m, size_t len, size_t alignment)
+static struct memory_allocation * memory_hugepage_alloc(struct memory_type *m, size_t len, size_t alignment)
{
- void *ret;
int prot = PROT_READ | PROT_WRITE;
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
@@ -59,22 +58,38 @@ static void * memory_hugepage_alloc(struct memory_type *m, size_t len, size_t al
flags |= MAP_LOCKED;
#endif
- ret = mmap(NULL, len, prot, flags, -1, 0);
- if (ret == MAP_FAILED)
+ struct memory_allocation *ma = alloc(sizeof(struct memory_allocation));
+ if (!ma)
return NULL;
- return ret;
-}
-
-static int memory_hugepage_free(struct memory_type *m, void *ptr, size_t len)
-{
/** We must make sure that len is a multiple of the hugepage size
*
* See: https://lkml.org/lkml/2014/10/22/925
*/
- len = ALIGN(len, HUGEPAGESIZE);
+ ma->length = ALIGN(len, HUGEPAGESIZE);
+ ma->alignment = alignment;
+ ma->type = m;
- return munmap(ptr, len);
+ ma->address = mmap(NULL, len, prot, flags, -1, 0);
+ if (ma->address == MAP_FAILED) {
+ free(ma);
+ return NULL;
+ }
+
+ return ma;
+}
+
+static int memory_hugepage_free(struct memory_type *m, struct memory_allocation *ma)
+{
+ int ret;
+
+ ret = munmap(ma->address, ma->length);
+ if (ret)
+ return ret;
+
+ free(ma);
+
+ return 0;
}
struct memory_type memory_hugepage = {
diff --git a/lib/memory/ib.c b/lib/memory/ib.c
index 38955e4d6..fe62e223f 100644
--- a/lib/memory/ib.c
+++ b/lib/memory/ib.c
@@ -36,33 +36,43 @@ struct ibv_mr * memory_ib_mr(void *ptr)
return (mr - 1);
}
-void * memory_ib_alloc(struct memory_type *m, size_t len, size_t alignment)
+static struct memory_allocation * memory_ib_alloc(struct memory_type *m, size_t len, size_t alignment)
{
struct memory_ib *mi = (struct memory_ib *) m->_vd;
- struct ibv_mr **mr = memory_alloc_aligned(mi->parent, len + sizeof(struct ibv_mr *), alignment);
- char *ptr = (char *) (mr + 1);
+ struct memory_allocation *ma = alloc(sizeof(struct memory_allocation));
+ if (!ma)
+ return NULL;
- *mr = ibv_reg_mr(mi->pd, ptr, len, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
- if(!*mr) {
- free(ptr);
+ ma->type = m;
+ ma->length = len;
+ ma->alignment = alignment;
+
+ ma->parent = mi->parent->alloc(mi->parent, len + sizeof(struct ibv_mr *), alignment);
+ ma->address = ma->parent->address;
+
+ ma->ib.mr = ibv_reg_mr(mi->pd, ma->address, ma->length, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
+ if(!ma->ib.mr) {
+ mi->parent->free(mi->parent, ma->parent);
+ free(ma);
return NULL;
}
- return ptr;
+ return ma;
}
-int memory_ib_free(struct memory_type *m, void *ptr, size_t len)
+static int memory_ib_free(struct memory_type *m, struct memory_allocation *ma)
{
+ int ret;
struct memory_ib *mi = (struct memory_ib *) m->_vd;
- struct ibv_mr *mr = memory_ib_mr(ptr);
- ibv_dereg_mr(mr);
+ ibv_dereg_mr(ma->ib.mr);
- ptr -= sizeof(struct ibv_mr *);
- len += sizeof(struct ibv_mr *);
+ ret = mi->parent->free(mi->parent, ma->parent);
+ if (ret)
+ return ret;
- memory_free(mi->parent, ptr, len);
+ free(ma);
return 0;
}
diff --git a/lib/memory/managed.c b/lib/memory/managed.c
index 05ea6f439..ab8bc4b59 100644
--- a/lib/memory/managed.c
+++ b/lib/memory/managed.c
@@ -34,18 +34,18 @@
#include
#include
-void* memory_managed_alloc(struct memory_type *m, size_t len, size_t alignment)
+static struct memory_allocation * memory_managed_alloc(struct memory_type *m, size_t len, size_t alignment)
{
/* Simple first-fit allocation */
- struct memblock *first = (struct memblock *) m->_vd;
- struct memblock *block;
+ struct memory_block *first = (struct memory_block *) m->_vd;
+ struct memory_block *block;
for (block = first; block != NULL; block = block->next) {
- if (block->flags & MEMBLOCK_USED)
+ if (block->used)
continue;
- char* cptr = (char *) block + sizeof(struct memblock);
- size_t avail = block->len;
+ char* cptr = (char *) block + sizeof(struct memory_block);
+ size_t avail = block->length;
uintptr_t uptr = (uintptr_t) cptr;
/* Check alignment first; leave a gap at start of block to assure
@@ -62,47 +62,59 @@ void* memory_managed_alloc(struct memory_type *m, size_t len, size_t alignment)
}
if (avail >= len) {
- if (gap > sizeof(struct memblock)) {
+ if (gap > sizeof(struct memory_block)) {
/* The alignment gap is big enough to fit another block.
* The original block descriptor is already at the correct
* position, so we just change its len and create a new block
* descriptor for the actual block we're handling. */
- block->len = gap - sizeof(struct memblock);
- struct memblock *newblock = (struct memblock *) (cptr - sizeof(struct memblock));
+ block->length = gap - sizeof(struct memory_block);
+ struct memory_block *newblock = (struct memory_block *) (cptr - sizeof(struct memory_block));
newblock->prev = block;
newblock->next = block->next;
block->next = newblock;
- newblock->flags = 0;
- newblock->len = len;
+ newblock->used = false;
+ newblock->length = len;
block = newblock;
}
else {
/* The gap is too small to fit another block descriptor, so we
* must account for the gap length in the block length. */
- block->len = len + gap;
+ block->length = len + gap;
}
- if (avail > len + sizeof(struct memblock)) {
+ if (avail > len + sizeof(struct memory_block)) {
/* Imperfect fit, so create another block for the remaining part */
- struct memblock *newblock = (struct memblock *) (cptr + len);
+ struct memory_block *newblock = (struct memory_block *) (cptr + len);
newblock->prev = block;
newblock->next = block->next;
block->next = newblock;
+
if (newblock->next)
newblock->next->prev = newblock;
- newblock->flags = 0;
- newblock->len = avail - len - sizeof(struct memblock);
+
+ newblock->used = false;
+ newblock->length = avail - len - sizeof(struct memory_block);
}
else {
/* If this block was larger than the requested length, but only
- * by less than sizeof(struct memblock), we may have wasted
- * memory by previous assignments to block->len. */
- block->len = avail;
+ * by less than sizeof(struct memory_block), we may have wasted
+ * memory by previous assignments to block->length. */
+ block->length = avail;
}
- block->flags |= MEMBLOCK_USED;
+ block->used = true;
- return (void *) cptr;
+ struct memory_allocation *ma = alloc(sizeof(struct memory_allocation));
+ if (!ma)
+ return NULL;
+
+ ma->address = cptr;
+ ma->type = m;
+ ma->alignment = alignment;
+ ma->length = len;
+ ma->managed.block = block;
+
+ return ma;
}
}
@@ -110,60 +122,48 @@ void* memory_managed_alloc(struct memory_type *m, size_t len, size_t alignment)
return NULL;
}
-int memory_managed_free(struct memory_type *m, void *ptr, size_t len)
+static int memory_managed_free(struct memory_type *m, struct memory_allocation *ma)
{
- struct memblock *first = (struct memblock *) m->_vd;
- struct memblock *block;
- char *cptr = ptr;
+ struct memory_block *block = ma->managed.block;
- for (block = first; block != NULL; block = block->next) {
- if (!(block->flags & MEMBLOCK_USED))
- continue;
-
- /* Since we may waste some memory at the start of a block to ensure
- * alignment, ptr may not actually be the start of the block */
- if ((char *) block + sizeof(struct memblock) <= cptr &&
- cptr < (char *) block + sizeof(struct memblock) + block->len) {
- /* Try to merge it with neighbouring free blocks */
- if (block->prev && !(block->prev->flags & MEMBLOCK_USED) &&
- block->next && !(block->next->flags & MEMBLOCK_USED)) {
- /* Special case first: both previous and next block are unused */
- block->prev->len += block->len + block->next->len + 2 * sizeof(struct memblock);
- block->prev->next = block->next->next;
- if (block->next->next)
- block->next->next->prev = block->prev;
- }
- else if (block->prev && !(block->prev->flags & MEMBLOCK_USED)) {
- block->prev->len += block->len + sizeof(struct memblock);
- block->prev->next = block->next;
- if (block->next)
- block->next->prev = block->prev;
- }
- else if (block->next && !(block->next->flags & MEMBLOCK_USED)) {
- block->len += block->next->len + sizeof(struct memblock);
- block->next = block->next->next;
- if (block->next)
- block->next->prev = block;
- }
- else {
- /* no neighbouring free block, so just mark it as free */
- block->flags &= ~MEMBLOCK_USED;
- }
-
- return 0;
- }
+ /* Try to merge it with neighbouring free blocks */
+ if (block->prev && !block->prev->used &&
+ block->next && !block->next->used) {
+ /* Special case first: both previous and next block are unused */
+ block->prev->length += block->length + block->next->length + 2 * sizeof(struct memory_block);
+ block->prev->next = block->next->next;
+ if (block->next->next)
+ block->next->next->prev = block->prev;
+ }
+ else if (block->prev && !block->prev->used) {
+ block->prev->length += block->length + sizeof(struct memory_block);
+ block->prev->next = block->next;
+ if (block->next)
+ block->next->prev = block->prev;
+ }
+ else if (block->next && !block->next->used) {
+ block->length += block->next->length + sizeof(struct memory_block);
+ block->next = block->next->next;
+ if (block->next)
+ block->next->prev = block;
+ }
+ else {
+ /* no neighbouring free block, so just mark it as free */
+ block->used = false;
}
- return -1;
+ free(ma);
+
+ return 0;
}
struct memory_type * memory_managed(void *ptr, size_t len)
{
struct memory_type *mt = ptr;
- struct memblock *mb;
+ struct memory_block *mb;
char *cptr = ptr;
- if (len < sizeof(struct memory_type) + sizeof(struct memblock)) {
+ if (len < sizeof(struct memory_type) + sizeof(struct memory_block)) {
info("memory_managed: passed region too small");
return NULL;
}
@@ -177,15 +177,15 @@ struct memory_type * memory_managed(void *ptr, size_t len)
cptr += ALIGN(sizeof(struct memory_type), sizeof(void *));
- /* Initialize first free memblock */
- mb = (struct memblock *) cptr;
+ /* Initialize first free memory block */
+ mb = (struct memory_block *) cptr;
mb->prev = NULL;
mb->next = NULL;
- mb->flags = 0;
+ mb->used = false;
- cptr += ALIGN(sizeof(struct memblock), sizeof(void *));
+ cptr += ALIGN(sizeof(struct memory_block), sizeof(void *));
- mb->len = len - (cptr - (char *) ptr);
+ mb->length = len - (cptr - (char *) ptr);
mt->_vd = (void *) mb;
diff --git a/lib/pool.c b/lib/pool.c
index 78b47f2c8..1a7ce726f 100644
--- a/lib/pool.c
+++ b/lib/pool.c
@@ -35,7 +35,6 @@ int pool_init(struct pool *p, size_t cnt, size_t blocksz, struct memory_type *m)
p->alignment = kernel_get_cacheline_size();
p->blocksz = p->alignment * CEIL(blocksz, p->alignment);
p->len = cnt * p->blocksz;
- p->mem = m;
void *buffer = memory_alloc_aligned(m, p->len, p->alignment);
if (!buffer)
@@ -66,7 +65,7 @@ int pool_destroy(struct pool *p)
queue_destroy(&p->queue);
void *buffer = (char*) p + p->buffer_off;
- ret = memory_free(p->mem, buffer, p->len);
+ ret = memory_free(buffer);
if (ret == 0)
p->state = STATE_DESTROYED;
diff --git a/lib/queue.c b/lib/queue.c
index 438a2c523..0b020b192 100644
--- a/lib/queue.c
+++ b/lib/queue.c
@@ -36,7 +36,7 @@
#include
/** Initialize MPMC queue */
-int queue_init(struct queue *q, size_t size, struct memory_type *mem)
+int queue_init(struct queue *q, size_t size, struct memory_type *m)
{
assert(q->state == STATE_DESTROYED);
@@ -47,9 +47,8 @@ int queue_init(struct queue *q, size_t size, struct memory_type *mem)
warn("A queue size was changed from %lu to %lu", old_size, size);
}
- q->mem = mem;
q->buffer_mask = size - 1;
- struct queue_cell *buffer = (struct queue_cell *) memory_alloc(q->mem, sizeof(struct queue_cell) * size);
+ struct queue_cell *buffer = (struct queue_cell *) memory_alloc(m, sizeof(struct queue_cell) * size);
if (!buffer)
return -2;
@@ -74,8 +73,7 @@ int queue_destroy(struct queue *q)
if (q->state == STATE_DESTROYED)
return 0;
- ret = memory_free(q->mem, buffer, (q->buffer_mask + 1) * sizeof(struct queue_cell));
-
+ ret = memory_free(buffer);
if (ret == 0)
q->state = STATE_DESTROYED;
diff --git a/lib/shmem.c b/lib/shmem.c
index 65042a155..294e7ae67 100644
--- a/lib/shmem.c
+++ b/lib/shmem.c
@@ -44,7 +44,7 @@ size_t shmem_total_size(int queuelen, int samplelen)
/* the size of the pool */
+ queuelen * kernel_get_cacheline_size() * CEIL(SAMPLE_LEN(samplelen), kernel_get_cacheline_size())
/* a memblock for each allocation (1 shmem_shared, 2 queues, 1 pool) */
- + 4 * sizeof(struct memblock)
+ + 4 * sizeof(struct memory_block)
/* and some extra buffer for alignment */
+ 1024;
}
@@ -144,7 +144,7 @@ retry: fd = shm_open(wname, O_RDWR|O_CREAT|O_EXCL, 0600);
if (base == MAP_FAILED)
return -10;
- cptr = (char *) base + sizeof(struct memory_type) + sizeof(struct memblock);
+ cptr = (char *) base + sizeof(struct memory_type) + sizeof(struct memory_block);
shared = (struct shmem_shared *) cptr;
shm->read.base = base;
shm->read.name = rname;
diff --git a/tests/unit/memory.c b/tests/unit/memory.c
index 38ccbe51a..821795afd 100644
--- a/tests/unit/memory.c
+++ b/tests/unit/memory.c
@@ -28,7 +28,7 @@
#include
#include
-#define HUGEPAGESIZE (1<<22)
+#define HUGEPAGESIZE (1 << 22)
TheoryDataPoints(memory, aligned) = {
DataPoints(size_t, 1, 32, 55, 1 << 10, 1 << 20),
@@ -40,6 +40,9 @@ Theory((size_t len, size_t align, struct memory_type *m), memory, aligned) {
int ret;
void *ptr;
+ ret = memory_init(100);
+ cr_assert(!ret);
+
ptr = memory_alloc_aligned(m, len, align);
cr_assert_neq(ptr, NULL, "Failed to allocate memory");
@@ -49,7 +52,7 @@ Theory((size_t len, size_t align, struct memory_type *m), memory, aligned) {
cr_assert(IS_ALIGNED(ptr, HUGEPAGESIZE));
}
- ret = memory_free(m, ptr, len);
+ ret = memory_free(ptr);
cr_assert_eq(ret, 0, "Failed to release memory: ret=%d, ptr=%p, len=%zu: %s", ret, ptr, len, strerror(errno));
}
@@ -62,7 +65,10 @@ Test(memory, manager) {
struct memory_type *m;
total_size = 1 << 10;
- max_block = total_size - sizeof(struct memory_type) - sizeof(struct memblock);
+ max_block = total_size - sizeof(struct memory_type) - sizeof(struct memory_block);
+
+ ret = memory_init(0);
+ cr_assert(!ret);
p = memory_alloc(&memory_type_heap, total_size);
cr_assert_not_null(p);
@@ -76,7 +82,7 @@ Test(memory, manager) {
p2 = memory_alloc(m, 32);
cr_assert_not_null(p2);
- ret = memory_free(m, p1, 16);
+ ret = memory_free(p1);
cr_assert(ret == 0);
p1 = memory_alloc_aligned(m, 128, 128);
@@ -87,21 +93,21 @@ Test(memory, manager) {
cr_assert(p3);
cr_assert(IS_ALIGNED(p3, 256));
- ret = memory_free(m, p2, 32);
+ ret = memory_free(p2);
cr_assert(ret == 0);
- ret = memory_free(m, p1, 128);
+ ret = memory_free(p1);
cr_assert(ret == 0);
- ret = memory_free(m, p3, 128);
+ ret = memory_free(p3);
cr_assert(ret == 0);
p1 = memory_alloc(m, max_block);
cr_assert_not_null(p1);
- ret = memory_free(m, p1, max_block);
+ ret = memory_free(p1);
cr_assert(ret == 0);
- ret = memory_free(&memory_type_heap, p, total_size);
+ ret = memory_free(p);
cr_assert(ret == 0);
}