diff --git a/include/villas/memory_ib.h b/include/villas/memory_ib.h deleted file mode 100644 index 5dfdf854e..000000000 --- a/include/villas/memory_ib.h +++ /dev/null @@ -1,40 +0,0 @@ -/** Memory allocators. - * - * @file - * @author Dennis Potter - * @copyright 2018, Institute for Automation of Complex Power Systems, EONERC - * @license GNU General Public License (version 3) - * - * VILLASnode - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - *********************************************************************************/ - -#include -#include - -struct memory_ib { - struct ibv_pd *pd; - struct memtype *parent; -}; - -struct ibv_mr* memory_ib_mr(void*); -void* memory_ib_alloc(struct memtype*, size_t, size_t); -int memory_ib_free(struct memtype*, void*, size_t); -struct memtype* ib_memtype(struct node*, struct memtype*); - - - - - diff --git a/lib/memory.c b/lib/memory.c index 4257d14aa..d60c8d698 100644 --- a/lib/memory.c +++ b/lib/memory.c @@ -118,3 +118,9 @@ int memory_free(void *ptr) return 0; } + +struct memory_allocation * memory_get_allocation(void *ptr) +{ + struct memory_allocation *ma = (struct memory_allocation *) hash_table_lookup(&allocations, ptr); + return ma; +} diff --git a/lib/memory/ib.c b/lib/memory/ib.c index 5679c6e6a..573487477 100644 --- a/lib/memory/ib.c +++ b/lib/memory/ib.c @@ -24,17 +24,20 @@ #include #include #include +#include -struct memory_ib { - struct ibv_pd *pd; - struct memory_type *parent; -}; -struct ibv_mr * memory_ib_mr(void *ptr) +struct ibv_mr * memory_ib_get_mr(struct sample *smps) { - struct ibv_mr *mr = (struct ibv_mr *) ptr; + struct memory_allocation *ma; + struct pool *p; + struct ibv_mr *mr; - return (mr - 1); + p = sample_pool(smps); + + ma = memory_get_allocation((char *)(p)+p->buffer_off); + mr = ma->ib.mr; + return mr; } static struct memory_allocation * memory_ib_alloc(struct memory_type *m, size_t len, size_t alignment) diff --git a/lib/nodes/infiniband.c b/lib/nodes/infiniband.c index b5483c114..0cc37af2a 100644 --- a/lib/nodes/infiniband.c +++ b/lib/nodes/infiniband.c @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -695,15 +696,15 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt) struct ibv_wc wc[n->in.vectorize]; struct ibv_recv_wr wr[cnt], *bad_wr = NULL; struct ibv_sge sge[cnt]; - struct ibv_mr ** mr; - struct pool *p; + struct ibv_mr * mr; int ret; + + if(ib->conn.available_recv_wrs <= ib->qp_init.cap.max_recv_wr && cnt==n->in.vectorize) { // Get Memory Region - p = sample_pool(smps[0]); - mr = (struct ibv_mr **)((char *)(p)+p->buffer_off-8); + mr = memory_ib_get_mr(smps[0]); for(int i=0; idata; sge[i].length = SAMPLE_DATA_LEN(DEFAULT_SAMPLELEN); - sge[i].lkey = (*mr)->lkey; + sge[i].lkey = mr->lkey; // Prepare a receive Work Request wr[i].wr_id = (uintptr_t)smps[i]; @@ -744,7 +745,8 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt) for(int i=0; i_vd; struct ibv_send_wr wr[cnt], *bad_wr = NULL; struct ibv_sge sge[cnt]; - struct ibv_mr ** mr; - struct pool *p; + struct ibv_mr * mr; int ret; memset(&wr, 0, sizeof(wr)); @@ -776,8 +777,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt) int send_inline = 1; // Get Memory Region - p = sample_pool(smps[0]); - mr = (struct ibv_mr **)((char *)(p)+p->buffer_off-8); + mr = memory_ib_get_mr(smps[0]); for(int i=0; idata; sge[i].length = smps[i]->length*sizeof(double); - sge[i].lkey = (*mr)->lkey; + sge[i].lkey = mr->lkey; // Set Send Work Request wr[i].wr_id = (uintptr_t)smps[i]; //This way the sample can be release in WC