diff --git a/lib/memory_ib.c b/lib/memory_ib.c new file mode 100644 index 000000000..80f2c868c --- /dev/null +++ b/lib/memory_ib.c @@ -0,0 +1,96 @@ +/** Memory allocators. + * + * @author Steffen Vogel + * @copyright 2017, Institute for Automation of Complex Power Systems, EONERC + * @license GNU General Public License (version 3) + * + * VILLASnode + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + *********************************************************************************/ + +#include + +struct memory_ib { + struct ibv_pd *pd; + struct memtype *parent; +}; + +struct ibv_mr * memory_ib_mr(void *ptr) +{ + struct ibv_mr *mr = (struct ibv_mr *) ptr; + + return (mr - 1); +} + +void * memory_ib_alloc(struct memtype *m, size_t len, size_t alignment) +{ + struct memtype_ib *mi = (struct memtype_ib *) m->_vd; + + struct ibv_mr **mr = memory_alloc_aligned(m->parent, len + sizeof(struct ibv_mr *), alignment); + char *ptr = (char *) (mr + 1); + + *mr = ibv_reg_mr(mi->pd, ptr, len, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + if(!*mr) { + free(ptr); + return NULL; + } + + return ptr; +} + +int memory_ib_free(struct memtype *m, void *ptr, size_t len) +{ + struct ibv_mr *mr = memory_ib_mr(ptr); + + ibv_dereg_mr(mr); + + ptr -= sizeof(struct ibv_mr *); + len += sizeof(struct ibv_mr *); + + memory_free(m->parent, ptr, len); + + return 0; +} + +struct memtype * ib_memtype(struct node *n, struct memtype *parent) +{ + struct infiniband *i = (struct infiniband *) n->_vd; + struct memtype *mt = alloc(struct memtype); + + mt->name = "ib"; + mt->flags = 0; + mt->alloc = memory_ib_alloc; + mt->free = memory_ib_free; + mt->alignment = 1; + + mt->_vd = malloc(sizeof(struct memory_ib)); + + struct memory_ib *mi = (struct memory_ib *) mt->_vd; + + mi->pd = i->ctx.pd; + mi->parent = parent; + + return mt; +} + +/* Ausserhalb von lib/nodes/infiniband.c */ +struct pool p = { .state = STATE_DESTROYED }; +struct node *n = ..; + +pool_init(&p, 100, 32, node_get_memtype(n)); + + + diff --git a/lib/nodes/infiniband.c b/lib/nodes/infiniband.c index 416449e73..963c26376 100644 --- a/lib/nodes/infiniband.c +++ b/lib/nodes/infiniband.c @@ -837,7 +837,8 @@ static struct plugin p = { .deinit = ib_deinit, .read = ib_read, .write = ib_write, - .fd = ib_fd + .fd = ib_fd, + .memtype = ib_memtype } }; diff --git a/src/pipe.c b/src/pipe.c index ee27df078..b9e967d97 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -132,7 +132,7 @@ static void * send_loop(void *ctx) struct sample *smps[node->out.vectorize]; /* Initialize memory */ - ret = pool_init(&sendd.pool, LOG2_CEIL(node->out.vectorize), SAMPLE_LEN(DEFAULT_SAMPLELEN), &memtype_hugepage); + ret = pool_init(&sendd.pool, LOG2_CEIL(node->out.vectorize), SAMPLE_LEN(DEFAULT_SAMPLELEN), node_memtype(node)); if (ret < 0) error("Failed to allocate memory for receive pool."); @@ -196,7 +196,7 @@ static void * recv_loop(void *ctx) struct sample *smps[node->in.vectorize]; /* Initialize memory */ - ret = pool_init(&recvv.pool, LOG2_CEIL(node->in.vectorize), SAMPLE_LEN(DEFAULT_SAMPLELEN), &memtype_hugepage); + ret = pool_init(&recvv.pool, LOG2_CEIL(node->in.vectorize), SAMPLE_LEN(DEFAULT_SAMPLELEN), node_memtype(node)); if (ret < 0) error("Failed to allocate memory for receive pool.");