mirror of
https://git.rwth-aachen.de/acs/public/villas/node/
synced 2025-03-09 00:00:00 +01:00
Implemented new memory system implementation in infiniband node
This commit is contained in:
parent
070e2c2bde
commit
51519c06df
4 changed files with 27 additions and 58 deletions
|
@ -1,40 +0,0 @@
|
|||
/** Memory allocators.
|
||||
*
|
||||
* @file
|
||||
* @author Dennis Potter <dennis@dennispotter.eu>
|
||||
* @copyright 2018, Institute for Automation of Complex Power Systems, EONERC
|
||||
* @license GNU General Public License (version 3)
|
||||
*
|
||||
* VILLASnode
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*********************************************************************************/
|
||||
|
||||
#include <villas/memory.h>
|
||||
#include <villas/node.h>
|
||||
|
||||
struct memory_ib {
|
||||
struct ibv_pd *pd;
|
||||
struct memtype *parent;
|
||||
};
|
||||
|
||||
struct ibv_mr* memory_ib_mr(void*);
|
||||
void* memory_ib_alloc(struct memtype*, size_t, size_t);
|
||||
int memory_ib_free(struct memtype*, void*, size_t);
|
||||
struct memtype* ib_memtype(struct node*, struct memtype*);
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -118,3 +118,9 @@ int memory_free(void *ptr)
|
|||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct memory_allocation * memory_get_allocation(void *ptr)
|
||||
{
|
||||
struct memory_allocation *ma = (struct memory_allocation *) hash_table_lookup(&allocations, ptr);
|
||||
return ma;
|
||||
}
|
||||
|
|
|
@ -24,17 +24,20 @@
|
|||
#include <villas/memory.h>
|
||||
#include <villas/utils.h>
|
||||
#include <rdma/rdma_cma.h>
|
||||
#include <villas/ib.h>
|
||||
|
||||
struct memory_ib {
|
||||
struct ibv_pd *pd;
|
||||
struct memory_type *parent;
|
||||
};
|
||||
|
||||
struct ibv_mr * memory_ib_mr(void *ptr)
|
||||
struct ibv_mr * memory_ib_get_mr(struct sample *smps)
|
||||
{
|
||||
struct ibv_mr *mr = (struct ibv_mr *) ptr;
|
||||
struct memory_allocation *ma;
|
||||
struct pool *p;
|
||||
struct ibv_mr *mr;
|
||||
|
||||
return (mr - 1);
|
||||
p = sample_pool(smps);
|
||||
|
||||
ma = memory_get_allocation((char *)(p)+p->buffer_off);
|
||||
mr = ma->ib.mr;
|
||||
return mr;
|
||||
}
|
||||
|
||||
static struct memory_allocation * memory_ib_alloc(struct memory_type *m, size_t len, size_t alignment)
|
||||
|
|
|
@ -30,6 +30,7 @@
|
|||
#include <villas/memory.h>
|
||||
#include <villas/pool.h>
|
||||
#include <villas/memory.h>
|
||||
#include <villas/ib.h>
|
||||
|
||||
#include <rdma/rdma_cma.h>
|
||||
|
||||
|
@ -695,15 +696,15 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt)
|
|||
struct ibv_wc wc[n->in.vectorize];
|
||||
struct ibv_recv_wr wr[cnt], *bad_wr = NULL;
|
||||
struct ibv_sge sge[cnt];
|
||||
struct ibv_mr ** mr;
|
||||
struct pool *p;
|
||||
struct ibv_mr * mr;
|
||||
int ret;
|
||||
|
||||
|
||||
|
||||
if(ib->conn.available_recv_wrs <= ib->qp_init.cap.max_recv_wr && cnt==n->in.vectorize)
|
||||
{
|
||||
// Get Memory Region
|
||||
p = sample_pool(smps[0]);
|
||||
mr = (struct ibv_mr **)((char *)(p)+p->buffer_off-8);
|
||||
mr = memory_ib_get_mr(smps[0]);
|
||||
|
||||
for(int i=0; i<cnt; i++)
|
||||
{
|
||||
|
@ -713,7 +714,7 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt)
|
|||
// Prepare receive Scatter/Gather element
|
||||
sge[i].addr = (uint64_t)&smps[i]->data;
|
||||
sge[i].length = SAMPLE_DATA_LEN(DEFAULT_SAMPLELEN);
|
||||
sge[i].lkey = (*mr)->lkey;
|
||||
sge[i].lkey = mr->lkey;
|
||||
|
||||
// Prepare a receive Work Request
|
||||
wr[i].wr_id = (uintptr_t)smps[i];
|
||||
|
@ -744,7 +745,8 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt)
|
|||
for(int i=0; i<ret; i++)
|
||||
{
|
||||
if(wc[i].status != IBV_WC_SUCCESS)
|
||||
error("Work Completion status was not IBV_WC_SUCCES in node %s", node_name(n));
|
||||
warn("Work Completion status was not IBV_WC_SUCCES in node %s: %i",
|
||||
node_name(n), wc[i].status);
|
||||
else if(wc[i].opcode & IBV_WC_RECV && wc[i].status != IBV_WC_WR_FLUSH_ERR)
|
||||
{
|
||||
smps[i] = (struct sample*)(wc[i].wr_id);
|
||||
|
@ -766,8 +768,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt)
|
|||
struct infiniband *ib = (struct infiniband *) n->_vd;
|
||||
struct ibv_send_wr wr[cnt], *bad_wr = NULL;
|
||||
struct ibv_sge sge[cnt];
|
||||
struct ibv_mr ** mr;
|
||||
struct pool *p;
|
||||
struct ibv_mr * mr;
|
||||
int ret;
|
||||
|
||||
memset(&wr, 0, sizeof(wr));
|
||||
|
@ -776,8 +777,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt)
|
|||
int send_inline = 1;
|
||||
|
||||
// Get Memory Region
|
||||
p = sample_pool(smps[0]);
|
||||
mr = (struct ibv_mr **)((char *)(p)+p->buffer_off-8);
|
||||
mr = memory_ib_get_mr(smps[0]);
|
||||
|
||||
for(int i=0; i<cnt; i++)
|
||||
{
|
||||
|
@ -787,7 +787,7 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt)
|
|||
//Set Scatter/Gather element to data of sample
|
||||
sge[i].addr = (uint64_t)&smps[i]->data;
|
||||
sge[i].length = smps[i]->length*sizeof(double);
|
||||
sge[i].lkey = (*mr)->lkey;
|
||||
sge[i].lkey = mr->lkey;
|
||||
|
||||
// Set Send Work Request
|
||||
wr[i].wr_id = (uintptr_t)smps[i]; //This way the sample can be release in WC
|
||||
|
|
Loading…
Add table
Reference in a new issue