Mirrors/VILLASnode
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/node/ synced 2025-03-09 00:00:00 +01:00
Code Issues Releases Wiki Activity

Implemented new memory system implementation in infiniband node

Browse source
This commit is contained in:
Dennis Potter 2018-07-04 15:15:24 +02:00
parent 070e2c2bde
commit 51519c06df
4 changed files with 27 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
include/villas/memory_ib.h
View file

@ -1,40 +0,0 @@
/** Memory allocators.
*
* @file
* @author Dennis Potter <dennis@dennispotter.eu>
* @copyright 2018, Institute for Automation of Complex Power Systems, EONERC
* @license GNU General Public License (version 3)
*
* VILLASnode
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*********************************************************************************/
#include <villas/memory.h>
#include <villas/node.h>
struct memory_ib {
struct ibv_pd *pd;
struct memtype *parent;
};
struct ibv_mr* memory_ib_mr(void*);
void* memory_ib_alloc(struct memtype*, size_t, size_t);
int memory_ib_free(struct memtype*, void*, size_t);
struct memtype* ib_memtype(struct node*, struct memtype*);

6
lib/memory.c
View file

@ -118,3 +118,9 @@ int memory_free(void *ptr)
return 0;
}
struct memory_allocation * memory_get_allocation(void *ptr)
{
struct memory_allocation *ma = (struct memory_allocation *) hash_table_lookup(&allocations, ptr);
return ma;
}

17
lib/memory/ib.c
View file

@ -24,17 +24,20 @@
#include <villas/memory.h>
#include <villas/utils.h>
#include <rdma/rdma_cma.h>
#include <villas/ib.h>
struct memory_ib {
struct ibv_pd *pd;
struct memory_type *parent;
};
struct ibv_mr * memory_ib_mr(void *ptr)
struct ibv_mr * memory_ib_get_mr(struct sample *smps)
{
struct ibv_mr *mr = (struct ibv_mr *) ptr;
struct memory_allocation *ma;
struct pool *p;
struct ibv_mr *mr;
return (mr - 1);
p = sample_pool(smps);
ma = memory_get_allocation((char *)(p)+p->buffer_off);
mr = ma->ib.mr;
return mr;
}
static struct memory_allocation * memory_ib_alloc(struct memory_type *m, size_t len, size_t alignment)

22
lib/nodes/infiniband.c
View file

@ -30,6 +30,7 @@
#include <villas/memory.h>
#include <villas/pool.h>
#include <villas/memory.h>
#include <villas/ib.h>
#include <rdma/rdma_cma.h>
@ -695,15 +696,15 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt)
struct ibv_wc wc[n->in.vectorize];
struct ibv_recv_wr wr[cnt], *bad_wr = NULL;
struct ibv_sge sge[cnt];
struct ibv_mr ** mr;
struct pool *p;
struct ibv_mr * mr;
int ret;
if(ib->conn.available_recv_wrs <= ib->qp_init.cap.max_recv_wr && cnt==n->in.vectorize)
{
// Get Memory Region
p = sample_pool(smps[0]);
mr = (struct ibv_mr **)((char *)(p)+p->buffer_off-8);
mr = memory_ib_get_mr(smps[0]);
for(int i=0; i<cnt; i++)
{
@ -713,7 +714,7 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt)
// Prepare receive Scatter/Gather element
sge[i].addr = (uint64_t)&smps[i]->data;
sge[i].length = SAMPLE_DATA_LEN(DEFAULT_SAMPLELEN);
sge[i].lkey = (*mr)->lkey;
sge[i].lkey = mr->lkey;
// Prepare a receive Work Request
wr[i].wr_id = (uintptr_t)smps[i];
@ -744,7 +745,8 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt)
for(int i=0; i<ret; i++)
{
if(wc[i].status != IBV_WC_SUCCESS)
error("Work Completion status was not IBV_WC_SUCCES in node %s", node_name(n));
warn("Work Completion status was not IBV_WC_SUCCES in node %s: %i",
node_name(n), wc[i].status);
else if(wc[i].opcode & IBV_WC_RECV && wc[i].status != IBV_WC_WR_FLUSH_ERR)
{
smps[i] = (struct sample*)(wc[i].wr_id);
@ -766,8 +768,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt)
struct infiniband *ib = (struct infiniband *) n->_vd;
struct ibv_send_wr wr[cnt], *bad_wr = NULL;
struct ibv_sge sge[cnt];
struct ibv_mr ** mr;
struct pool *p;
struct ibv_mr * mr;
int ret;
memset(&wr, 0, sizeof(wr));
@ -776,8 +777,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt)
int send_inline = 1;
// Get Memory Region
p = sample_pool(smps[0]);
mr = (struct ibv_mr **)((char *)(p)+p->buffer_off-8);
mr = memory_ib_get_mr(smps[0]);
for(int i=0; i<cnt; i++)
{
@ -787,7 +787,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt)
//Set Scatter/Gather element to data of sample
sge[i].addr = (uint64_t)&smps[i]->data;
sge[i].length = smps[i]->length*sizeof(double);
sge[i].lkey = (*mr)->lkey;
sge[i].lkey = mr->lkey;
// Set Send Work Request
wr[i].wr_id = (uintptr_t)smps[i]; //This way the sample can be release in WC