Mirrors/libhermit
1
0
Fork 0
mirror of https://github.com/hermitcore/libhermit.git synced 2025-03-09 00:00:03 +01:00
Code Issues Releases Wiki Activity

Added conversion for wr->sg_list->addr fields. write_bw benchmark failing at ibv_poll_cq(), error code 10: IBV_WC_REM_ACCESS_ERR

Browse source
This commit is contained in:
Annika Wierichs 2018-01-24 13:49:37 +01:00
parent f0421c94ef
commit 647b8e758d
11 changed files with 209 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
include/hermit/verbs.h
View file

@ -1650,15 +1650,15 @@ struct verbs_context {
//NULL : container_of(ctx, struct verbs_context, context);
//}
// #define verbs_get_ctx_op(ctx, op) ({ \
// struct verbs_context *__vctx = verbs_get_ctx(ctx); \
// (!__vctx || (__vctx->sz < sizeof(*__vctx) - offsetof(struct verbs_context, op)) || \
// !__vctx->op) ? NULL : __vctx; })
/* #define verbs_get_ctx_op(ctx, op) ({ \
struct verbs_context *__vctx = verbs_get_ctx(ctx); \
(!__vctx || (__vctx->sz < sizeof(*__vctx) - offsetof(struct verbs_context, op)) || \
!__vctx->op) ? NULL : __vctx; })
#define verbs_set_ctx_op(_vctx, op, ptr) ({ \
struct verbs_context *vctx = _vctx; \
if (vctx && (vctx->sz >= sizeof(*vctx) - offsetof(struct verbs_context, op))) \
vctx->op = ptr; })
vctx->op = ptr; }) */
///**
// * ibv_get_device_list - Get list of IB devices currently available
@ -1745,8 +1745,8 @@ struct verbs_context {
// return ibv_query_port(context, port_num, port_attr);
// }
// #define ibv_query_port(context, port_num, port_attr) \
// ___ibv_query_port(context, port_num, port_attr)
/* #define ibv_query_port(context, port_num, port_attr) \
___ibv_query_port(context, port_num, port_attr) */
///**
// * ibv_query_gid - Get a GID table entry

64
kernel/ibv.c
View file

@ -607,7 +607,7 @@ typedef struct {
struct ibv_device ** ibv_get_device_list(int * num_devices) {
uhyve_ibv_get_device_list_t uhyve_args;
uhyve_args.num_devices = num_devices;
uhyve_args.num_devices = (int *) guest_to_host((size_t) num_devices);
uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned) virt_to_phys((size_t) &uhyve_args));
@ -1004,10 +1004,10 @@ typedef struct {
struct ibv_mr * ret;
} __attribute__((packed)) uhyve_ibv_reg_mr_t;
struct ibv_mr * ibv_reg_mr(struct ibv_pd * pd, void * addr, int length, int access) {
struct ibv_mr * ibv_reg_mr(struct ibv_pd * pd, void * addr, int length, int access) { // !!!
uhyve_ibv_reg_mr_t uhyve_args;
uhyve_args.pd = pd;
uhyve_args.addr = addr;
uhyve_args.addr = (void *) guest_to_host((size_t) addr);
uhyve_args.length = length;
uhyve_args.access = access;
@ -1033,12 +1033,13 @@ typedef struct {
int ret;
} __attribute__((packed)) uhyve_ibv_rereg_mr_t;
int ibv_rereg_mr(struct ibv_mr * mr, int flags, struct ibv_pd * pd, void * addr, int length, int access) {
int ibv_rereg_mr(struct ibv_mr * mr, int flags, struct ibv_pd * pd, void * addr,
int length, int access) { // !!!
uhyve_ibv_rereg_mr_t uhyve_args;
uhyve_args.mr = mr;
uhyve_args.flags = flags;
uhyve_args.pd = pd;
uhyve_args.addr = addr;
uhyve_args.mr = mr;
uhyve_args.flags = flags;
uhyve_args.pd = pd;
uhyve_args.addr = (void *) guest_to_host((size_t) addr);
uhyve_args.length = length;
uhyve_args.access = access;
@ -1918,7 +1919,7 @@ int ibv_destroy_rwq_ind_table(struct ibv_rwq_ind_table * rwq_ind_table) {
typedef struct {
// Parameters:
struct ibv_qp * qp;
struct ibv_qp * qp;
struct ibv_send_wr * wr;
struct ibv_send_wr ** bad_wr;
// Return value:
@ -1933,19 +1934,27 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w
struct ibv_send_wr * curr_wr;
int num_wrs;
int num_sges_max;
int is_rdma, is_atomic, is_bind_mw, is_tso;
// Number of work requests in linked list
curr_wr = wr->next;
num_wrs = 1;
while (curr_wr) {
for (curr_wr = wr->next; curr_wr; curr_wr = curr_wr->next) {
num_wrs++;
curr_wr = curr_wr->next;
}
// Number of SGEs in each WR
num_sges_max = 0;
for (curr_wr = wr; curr_wr; curr_wr = curr_wr->next) {
if (num_sges_max < curr_wr->num_sge) {
num_sges_max = curr_wr->num_sge;
}
}
// Backup arrays for original guest memory pointers
struct ibv_send_wr * wr__next[num_wrs];
struct ibv_sge * wr__sg_list[num_wrs];
uint64_t wr__sg_list__addr[num_wrs][num_sges_max];
uint64_t wr__wr__rdma__remote_addr[num_wrs];
uint64_t wr__wr__atomic__remote_addr[num_wrs];
uint64_t wr__bind_mw__bind_info__addr[num_wrs];
@ -1982,10 +1991,19 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w
// Next pointer and SGE array
wr__next[i] = curr_wr->next;
curr_wr->next = (struct ibv_send_wr *) guest_to_host((size_t) curr_wr->next);
for (int s = 0; s < curr_wr->num_sge; s++) {
wr__sg_list__addr[i][s] = curr_wr->sg_list[s].addr;
curr_wr->sg_list[s].addr = (uint64_t) guest_to_host((size_t) curr_wr->sg_list[s].addr);
}
wr__sg_list[i] = curr_wr->sg_list;
curr_wr->sg_list = (struct ibv_sge *) guest_to_host((size_t) curr_wr->sg_list);
curr_wr = curr_wr->next;
/* LOG_INFO("\tsg_list - guest backup address: %p", wr__sg_list[i]); */
/* LOG_INFO("\tsg_list - host address: %p", curr_wr->sg_list); */
}
uhyve_send(UHYVE_PORT_IBV_POST_SEND, (unsigned) virt_to_phys((size_t) &uhyve_args));
@ -2022,9 +2040,13 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w
if (*bad_wr && *bad_wr == curr_wr->next) {
*bad_wr = wr__next[i];
}
// Next pointer and SGE array
curr_wr->next = wr__next[i];
curr_wr->sg_list = wr__sg_list[i];
for (int s = 0; s < curr_wr->num_sge; s++) {
curr_wr->sg_list[s].addr = wr__sg_list__addr[i][s];
}
curr_wr = curr_wr->next;
}
@ -2053,6 +2075,7 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w
uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr);
struct ibv_recv_wr * curr_wr;
int num_sges_max;
int num_wrs;
// Number of work requests in linked list
@ -2063,15 +2086,29 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w
curr_wr = curr_wr->next;
}
// Number of SGEs in each WR
num_sges_max = 0;
for (curr_wr = wr; curr_wr; curr_wr = curr_wr->next) {
if (num_sges_max < curr_wr->num_sge) {
num_sges_max = curr_wr->num_sge;
}
}
// Backup arrays for original guest memory pointers
struct ibv_recv_wr * wr__next[num_wrs];
struct ibv_sge * wr__sg_list[num_wrs];
uint64_t wr__sg_list__addr[num_wrs][num_sges_max];
curr_wr = wr;
for (int i = 0; i < num_wrs; i++) {
wr__next[i] = curr_wr->next;
curr_wr->next = (struct ibv_recv_wr *) guest_to_host((size_t) curr_wr->next);
for (int s = 0; s < curr_wr->num_sge; s++) {
wr__sg_list__addr[i][s] = curr_wr->sg_list[s].addr;
curr_wr->sg_list[s].addr = (uint64_t) guest_to_host((size_t) curr_wr->sg_list[s].addr);
}
wr__sg_list[i] = curr_wr->sg_list;
curr_wr->sg_list = (struct ibv_sge *) guest_to_host((size_t) curr_wr->sg_list);
@ -2091,6 +2128,9 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w
}
curr_wr->next = wr__next[i];
curr_wr->sg_list = wr__sg_list[i];
for (int s = 0; s < curr_wr->num_sge; s++) {
curr_wr->sg_list[s].addr = wr__sg_list__addr[i][s];
}
curr_wr = curr_wr->next;
}

2
tools/ibv_code_generator/generate-code.py
View file

@ -87,6 +87,8 @@ Required in: ./tool/uhyve-ibv.h
"""
# TODO: Add ibv_resolve_eth_l2_from_gid function back in. Not linking right now.
# TODO: int * num devices --> guest to host
# TODO: void * as well
from __future__ import print_function
from parser import generate_struct_conversions

11
tools/uhyve-ibv.c
View file

@ -1226,6 +1226,17 @@ void call_ibv_post_send(struct kvm_run * run, uint8_t * guest_mem) {
unsigned data = *((unsigned*) ((size_t) run + run->io.data_offset));
uhyve_ibv_post_send_t * args = (uhyve_ibv_post_send_t *) (guest_mem + data);
/* printf("\tqp->context: %p\n", args->qp->context); */
/* printf("\tqp->state: %d\n", args->qp->state); */
/* printf("\twr->id: %lu\n", args->wr->wr_id); */
/* printf("\twr->next: %p\n", args->wr->next); */
/* printf("\twr->num_sge: %d\n", args->wr->num_sge); */
/* printf("\twr->sg_list: %p\n", args->wr->sg_list); */
/* printf("\t ->sg_list[0].length: %lu\n", args->wr->sg_list[0].length); */
/* printf("\t ->sg_list[0].lkey: %lu\n", args->wr->sg_list[0].lkey); */
/* printf("\t ->sg_list[0].addr: %p\n", (char *) args->wr->sg_list[0].addr); */
printf("\t ->sg_list[0].addr deref 1: %u\n", *((uint8_t *) args->wr->sg_list[0].addr));
use_ib_mem_pool = true;
args->ret = ibv_post_send(args->qp, args->wr, args->bad_wr);
use_ib_mem_pool = false;

138
tools/uhyve-ibv.h
View file

@ -51,75 +51,75 @@ typedef enum {
UHYVE_PORT_IBV_WC_READ_CVLAN = 0x625,
UHYVE_PORT_IBV_WC_READ_FLOW_TAG = 0x626,
UHYVE_PORT_IBV_POST_WQ_RECV = 0x627,
UHYVE_PORT_IBV_GET_DEVICE_LIST = 0x629,
UHYVE_PORT_IBV_FREE_DEVICE_LIST = 0x62A,
UHYVE_PORT_IBV_GET_DEVICE_NAME = 0x62B,
UHYVE_PORT_IBV_GET_DEVICE_GUID = 0x62C,
UHYVE_PORT_IBV_OPEN_DEVICE = 0x62D,
UHYVE_PORT_IBV_CLOSE_DEVICE = 0x62E,
UHYVE_PORT_IBV_GET_ASYNC_EVENT = 0x62F,
UHYVE_PORT_IBV_ACK_ASYNC_EVENT = 0x630,
UHYVE_PORT_IBV_QUERY_DEVICE = 0x631,
UHYVE_PORT_IBV_QUERY_PORT = 0x632,
UHYVE_PORT_IBV_QUERY_GID = 0x634,
UHYVE_PORT_IBV_QUERY_PKEY = 0x635,
UHYVE_PORT_IBV_ALLOC_PD = 0x636,
UHYVE_PORT_IBV_DEALLOC_PD = 0x637,
UHYVE_PORT_IBV_CREATE_FLOW = 0x638,
UHYVE_PORT_IBV_DESTROY_FLOW = 0x639,
UHYVE_PORT_IBV_OPEN_XRCD = 0x63A,
UHYVE_PORT_IBV_CLOSE_XRCD = 0x63B,
UHYVE_PORT_IBV_REG_MR = 0x63C,
UHYVE_PORT_IBV_REREG_MR = 0x63D,
UHYVE_PORT_IBV_DEREG_MR = 0x63E,
UHYVE_PORT_IBV_ALLOC_MW = 0x63F,
UHYVE_PORT_IBV_DEALLOC_MW = 0x640,
UHYVE_PORT_IBV_INC_RKEY = 0x641,
UHYVE_PORT_IBV_BIND_MW = 0x642,
UHYVE_PORT_IBV_CREATE_COMP_CHANNEL = 0x643,
UHYVE_PORT_IBV_DESTROY_COMP_CHANNEL = 0x644,
UHYVE_PORT_IBV_CREATE_CQ = 0x645,
UHYVE_PORT_IBV_CREATE_CQ_EX = 0x646,
UHYVE_PORT_IBV_RESIZE_CQ = 0x647,
UHYVE_PORT_IBV_DESTROY_CQ = 0x648,
UHYVE_PORT_IBV_GET_CQ_EVENT = 0x649,
UHYVE_PORT_IBV_ACK_CQ_EVENTS = 0x64A,
UHYVE_PORT_IBV_POLL_CQ = 0x64B,
UHYVE_PORT_IBV_REQ_NOTIFY_CQ = 0x64C,
UHYVE_PORT_IBV_CREATE_SRQ = 0x64D,
UHYVE_PORT_IBV_CREATE_SRQ_EX = 0x64E,
UHYVE_PORT_IBV_MODIFY_SRQ = 0x64F,
UHYVE_PORT_IBV_QUERY_SRQ = 0x650,
UHYVE_PORT_IBV_GET_SRQ_NUM = 0x651,
UHYVE_PORT_IBV_DESTROY_SRQ = 0x652,
UHYVE_PORT_IBV_POST_SRQ_RECV = 0x653,
UHYVE_PORT_IBV_CREATE_QP = 0x654,
UHYVE_PORT_IBV_CREATE_QP_EX = 0x655,
UHYVE_PORT_IBV_QUERY_RT_VALUES_EX = 0x656,
UHYVE_PORT_IBV_QUERY_DEVICE_EX = 0x657,
UHYVE_PORT_IBV_OPEN_QP = 0x658,
UHYVE_PORT_IBV_MODIFY_QP = 0x659,
UHYVE_PORT_IBV_QUERY_QP = 0x65A,
UHYVE_PORT_IBV_DESTROY_QP = 0x65B,
UHYVE_PORT_IBV_CREATE_WQ = 0x65C,
UHYVE_PORT_IBV_MODIFY_WQ = 0x65D,
UHYVE_PORT_IBV_DESTROY_WQ = 0x65E,
UHYVE_PORT_IBV_CREATE_RWQ_IND_TABLE = 0x65F,
UHYVE_PORT_IBV_DESTROY_RWQ_IND_TABLE = 0x660,
UHYVE_PORT_IBV_POST_SEND = 0x661,
UHYVE_PORT_IBV_POST_RECV = 0x662,
UHYVE_PORT_IBV_CREATE_AH = 0x663,
UHYVE_PORT_IBV_INIT_AH_FROM_WC = 0x664,
UHYVE_PORT_IBV_CREATE_AH_FROM_WC = 0x665,
UHYVE_PORT_IBV_DESTROY_AH = 0x666,
UHYVE_PORT_IBV_ATTACH_MCAST = 0x667,
UHYVE_PORT_IBV_DETACH_MCAST = 0x668,
UHYVE_PORT_IBV_FORK_INIT = 0x669,
UHYVE_PORT_IBV_NODE_TYPE_STR = 0x66A,
UHYVE_PORT_IBV_PORT_STATE_STR = 0x66B,
UHYVE_PORT_IBV_EVENT_TYPE_STR = 0x66C,
// UHYVE_PORT_IBV_RESOLVE_ETH_L2_FROM_GID = 0x66D,
UHYVE_PORT_IBV_IS_QPT_SUPPORTED = 0x66E,
UHYVE_PORT_IBV_GET_DEVICE_LIST = 0x628,
UHYVE_PORT_IBV_FREE_DEVICE_LIST = 0x629,
UHYVE_PORT_IBV_GET_DEVICE_NAME = 0x62A,
UHYVE_PORT_IBV_GET_DEVICE_GUID = 0x62B,
UHYVE_PORT_IBV_OPEN_DEVICE = 0x62C,
UHYVE_PORT_IBV_CLOSE_DEVICE = 0x62D,
UHYVE_PORT_IBV_GET_ASYNC_EVENT = 0x62E,
UHYVE_PORT_IBV_ACK_ASYNC_EVENT = 0x62F,
UHYVE_PORT_IBV_QUERY_DEVICE = 0x630,
UHYVE_PORT_IBV_QUERY_PORT = 0x631,
UHYVE_PORT_IBV_QUERY_GID = 0x632,
UHYVE_PORT_IBV_QUERY_PKEY = 0x633,
UHYVE_PORT_IBV_ALLOC_PD = 0x634,
UHYVE_PORT_IBV_DEALLOC_PD = 0x635,
UHYVE_PORT_IBV_CREATE_FLOW = 0x636,
UHYVE_PORT_IBV_DESTROY_FLOW = 0x637,
UHYVE_PORT_IBV_OPEN_XRCD = 0x638,
UHYVE_PORT_IBV_CLOSE_XRCD = 0x639,
UHYVE_PORT_IBV_REG_MR = 0x63A,
UHYVE_PORT_IBV_REREG_MR = 0x63B,
UHYVE_PORT_IBV_DEREG_MR = 0x63C,
UHYVE_PORT_IBV_ALLOC_MW = 0x63D,
UHYVE_PORT_IBV_DEALLOC_MW = 0x63E,
UHYVE_PORT_IBV_INC_RKEY = 0x63F,
UHYVE_PORT_IBV_BIND_MW = 0x640,
UHYVE_PORT_IBV_CREATE_COMP_CHANNEL = 0x641,
UHYVE_PORT_IBV_DESTROY_COMP_CHANNEL = 0x642,
UHYVE_PORT_IBV_CREATE_CQ = 0x643,
UHYVE_PORT_IBV_CREATE_CQ_EX = 0x644,
UHYVE_PORT_IBV_RESIZE_CQ = 0x645,
UHYVE_PORT_IBV_DESTROY_CQ = 0x646,
UHYVE_PORT_IBV_GET_CQ_EVENT = 0x647,
UHYVE_PORT_IBV_ACK_CQ_EVENTS = 0x648,
UHYVE_PORT_IBV_POLL_CQ = 0x649,
UHYVE_PORT_IBV_REQ_NOTIFY_CQ = 0x64A,
UHYVE_PORT_IBV_CREATE_SRQ = 0x64B,
UHYVE_PORT_IBV_CREATE_SRQ_EX = 0x64C,
UHYVE_PORT_IBV_MODIFY_SRQ = 0x64D,
UHYVE_PORT_IBV_QUERY_SRQ = 0x64E,
UHYVE_PORT_IBV_GET_SRQ_NUM = 0x64F,
UHYVE_PORT_IBV_DESTROY_SRQ = 0x650,
UHYVE_PORT_IBV_POST_SRQ_RECV = 0x651,
UHYVE_PORT_IBV_CREATE_QP = 0x652,
UHYVE_PORT_IBV_CREATE_QP_EX = 0x653,
UHYVE_PORT_IBV_QUERY_RT_VALUES_EX = 0x654,
UHYVE_PORT_IBV_QUERY_DEVICE_EX = 0x655,
UHYVE_PORT_IBV_OPEN_QP = 0x656,
UHYVE_PORT_IBV_MODIFY_QP = 0x657,
UHYVE_PORT_IBV_QUERY_QP = 0x658,
UHYVE_PORT_IBV_DESTROY_QP = 0x659,
UHYVE_PORT_IBV_CREATE_WQ = 0x65A,
UHYVE_PORT_IBV_MODIFY_WQ = 0x65B,
UHYVE_PORT_IBV_DESTROY_WQ = 0x65C,
UHYVE_PORT_IBV_CREATE_RWQ_IND_TABLE = 0x65D,
UHYVE_PORT_IBV_DESTROY_RWQ_IND_TABLE = 0x65E,
UHYVE_PORT_IBV_POST_SEND = 0x65F,
UHYVE_PORT_IBV_POST_RECV = 0x660,
UHYVE_PORT_IBV_CREATE_AH = 0x661,
UHYVE_PORT_IBV_INIT_AH_FROM_WC = 0x662,
UHYVE_PORT_IBV_CREATE_AH_FROM_WC = 0x663,
UHYVE_PORT_IBV_DESTROY_AH = 0x664,
UHYVE_PORT_IBV_ATTACH_MCAST = 0x665,
UHYVE_PORT_IBV_DETACH_MCAST = 0x666,
UHYVE_PORT_IBV_FORK_INIT = 0x667,
UHYVE_PORT_IBV_NODE_TYPE_STR = 0x668,
UHYVE_PORT_IBV_PORT_STATE_STR = 0x669,
UHYVE_PORT_IBV_EVENT_TYPE_STR = 0x66A,
// UHYVE_PORT_IBV_RESOLVE_ETH_L2_FROM_GID = 0x66B,
UHYVE_PORT_IBV_IS_QPT_SUPPORTED = 0x66C,
UHYVE_PORT_KERNEL_IBV_LOG = 0x66F,
} uhyve_ibv_t;

32
tools/uhyve.c
View file

@ -172,7 +172,6 @@
#define APIC_DEFAULT_BASE 0xfee00000
#define IB_POOL_SIZE 0x400000ULL
#define IB_MEM_DEBUG
static bool restart = false;
static bool cap_tsc_deadline = false;
@ -226,6 +225,8 @@ typedef struct {
// DLSYM
/* #define IB_MEM_DEBUG */
/* static uint8_t * ib_pool_top = NULL; */
/* uint64_t ib_pool_addr = 0; // TODO: static? */
static uint8_t* ib_pool_addr = 0;
@ -259,32 +260,24 @@ static void init_ib_mem_functions(void)
if (real_malloc == NULL && real_calloc == NULL && real_realloc == NULL && real_free == NULL) {
init_real_calloc_active = true;
/* size_t dlsym_mem_len = 1024 * 8; */
/* dlsym_mem = mmap(NULL, dlsym_mem_len, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANONYMOUS | MAP_SHARED, -1, 0); */
real_calloc = dlsym(RTLD_NEXT, "calloc");
init_real_calloc_active = false;
/* munmap(dlsym_mem, dlsym_mem_len); */
/* dlsym_mem = NULL; */
real_malloc = dlsym(RTLD_NEXT, "malloc");
real_realloc = dlsym(RTLD_NEXT, "realloc");
real_free = dlsym(RTLD_NEXT, "free");
}
if (!real_malloc || !real_calloc || !real_free || !real_realloc /* || !real_free_alt */ ) {
if (!real_malloc || !real_calloc || !real_free || !real_realloc) {
fprintf(stderr, "Error in `dlsym`: %s\n", dlerror());
}
pthread_mutex_unlock(&ib_pool_mutex);
#ifdef IB_MEM_DEBUG
printf("\tInit finished.\n");
#endif
}
/*
* new_ib_malloc_region
*/
@ -314,14 +307,13 @@ void * new_ib_malloc_region(size_t size)
}
if (ib_pool_top > ib_pool_addr + IB_POOL_SIZE) { // TODO: make ib addr a uint8* as well?
#ifdef IB_MEM_DEBUG
fprintf(stderr, "Error: IB Memory Pool overflow.\n");
#endif
fprintf(stderr, "\nError: IB Memory Pool overflow.------------------------------\n\n");
}
return result;
}
// TODO: locks ok?
/*
* malloc
@ -330,7 +322,7 @@ void * new_ib_malloc_region(size_t size)
void * malloc(size_t size)
{
#ifdef IB_MEM_DEBUG
printf("FUNCTION: malloc()\n");
printf("FUNCTION: malloc() ------------- size: %lu\n", size);
#endif
if (real_malloc == NULL) {
@ -341,8 +333,14 @@ void * malloc(size_t size)
void * result;
if (use_ib_mem_pool) {
#ifdef IB_MEM_DEBUG
printf("\tuse_ib_mem_pool == true.\n");
#endif
result = new_ib_malloc_region(size);
} else { // !use_ib_mem_pool
#ifdef IB_MEM_DEBUG
printf("\tcalling real_malloc()\n");
#endif
result = real_malloc(size);
}
@ -358,7 +356,7 @@ void * malloc(size_t size)
void * calloc(size_t nitems, size_t size)
{
#ifdef IB_MEM_DEBUG
printf("FUNCTION: calloc()\n");
printf("FUNCTION: calloc() ------------- nitems: %lu size: %lu\n", nitems, size);
#endif
void * result;
@ -375,11 +373,11 @@ void * calloc(size_t nitems, size_t size)
result = dlsym_mem_buffer;
} else if (use_ib_mem_pool) {
#ifdef IB_MEM_DEBUG
printf("\tuse_ib == true\n");
printf("\tuse_ib_mem_pool == true\n");
#endif
result = new_ib_malloc_region(full_size);
memset(result, 0, full_size);
} else {
} else { // !use_ib_mem_pool && !init_real_calloc_active
#ifdef IB_MEM_DEBUG
printf("\tcalling real_calloc()\n");
#endif

20
usr/benchmarks/ib/perftest_communication_write_bw.c
View file

@ -442,17 +442,17 @@ static int ethernet_client_connect(struct perftest_comm *comm)
{
struct addrinfo *res, *t;
struct addrinfo hints;
/* char *service; */
char *service;
int sockfd = -1;
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
/* if (check_add_port(&service, comm->rdma_params->port, comm->rdma_params->servername, &hints, &res)) { */
/* fprintf(stderr, "Problem in resolving basic address and port\n"); */
/* return 1; */
/* } */
if (check_add_port(&service, comm->rdma_params->port, comm->rdma_params->servername, &hints, &res)) {
fprintf(stderr, "Problem in resolving basic address and port\n");
return 1;
}
for (t = res; t; t = t->ai_next) {
sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
@ -483,7 +483,7 @@ static int ethernet_server_connect(struct perftest_comm *comm)
{
struct addrinfo *res, *t;
struct addrinfo hints;
/* char *service; */
char *service;
int n;
int sockfd = -1, connfd;
@ -492,10 +492,10 @@ static int ethernet_server_connect(struct perftest_comm *comm)
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
/* if (check_add_port(&service, comm->rdma_params->port, NULL, &hints, &res)) { */
/* fprintf(stderr, "Problem in resolving basic address and port\n"); */
/* return 1; */
/* } */
if (check_add_port(&service, comm->rdma_params->port, NULL, &hints, &res)) {
fprintf(stderr, "Problem in resolving basic address and port\n");
return 1;
}
for (t = res; t; t = t->ai_next) {
sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);

32
usr/benchmarks/ib/perftest_resources.c
View file

@ -525,26 +525,26 @@ static struct ibv_qp *ctx_rss_eth_qp_create(struct pingpong_context *ctx,struct
/******************************************************************************
*
******************************************************************************/
/* int check_add_port(char **service,int port, */
/* const char *servername, */
/* struct addrinfo *hints, */
/* struct addrinfo **res) */
/* { */
/* int number; */
int check_add_port(char **service,int port,
const char *servername,
struct addrinfo *hints,
struct addrinfo **res)
{
int number;
/* if (asprintf(service,"%d", port) < 0) { */
/* return FAILURE; */
/* } */
if (asprintf(service,"%d", port) < 0) {
return FAILURE;
}
/* number = getaddrinfo(servername,*service,hints,res); */
number = getaddrinfo(servername,*service,hints,res);
/* if (number < 0) { */
/* fprintf(stderr, "%s for %s:%d\n", gai_strerror(number), servername, port); */
/* return FAILURE; */
/* } */
if (number < 0) {
fprintf(stderr, "%s for %s:%d\n", gai_strerror(number), servername, port);
return FAILURE;
}
/* return SUCCESS; */
/* } */
return SUCCESS;
}
/******************************************************************************
+ *

22
usr/benchmarks/ib/perftest_resources_write_bw.c
View file

@ -711,6 +711,28 @@ int verify_params_with_device_context(struct ibv_context *context,
// -----------------------------------------------------------------------------
int check_add_port(char **service,int port,
const char *servername,
struct addrinfo *hints,
struct addrinfo **res)
{
int number;
if (asprintf(service,"%d", port) < 0) {
return FAILURE;
}
number = getaddrinfo(servername,*service,hints,res);
if (number < 0) {
/* fprintf(stderr, "%s for %s:%d\n", gai_strerror(number), servername, port); */
fprintf(stderr, "Error");
return FAILURE;
}
return SUCCESS;
}
struct ibv_device* ctx_find_dev(const char *ib_devname)
{
int num_of_device;

8
usr/benchmarks/ib/perftest_resources_write_bw.h
View file

@ -212,10 +212,10 @@ struct raw_ethernet_info { // !
*
* Return Value : SUCCESS, FAILURE.
*/
// int check_add_port(char **service,int port,
// const char *servername,
// struct addrinfo *hints,
// struct addrinfo **res);
int check_add_port(char **service,int port,
const char *servername,
struct addrinfo *hints,
struct addrinfo **res);
/* ctx_find_dev
*

94
usr/tests/ib-test.c
View file

@ -43,101 +43,9 @@
int main(int argc, char** argv)
{
// CONTAINING HOST PTRS:
struct ibv_wq_init_attr {
void *wq_context; // host
enum ibv_wq_type wq_type;
uint32_t max_wr;
uint32_t max_sge;
struct ibv_pd *pd; // host
struct ibv_cq *cq; // host
uint32_t comp_mask; /* Use ibv_wq_init_attr_mask */
uint32_t create_flags; /* use ibv_wq_flags */
};
struct ibv_qp_init_attr {
void *qp_context; // host
struct ibv_cq *send_cq; // host
struct ibv_cq *recv_cq; // host
struct ibv_srq *srq; // host
struct ibv_qp_cap cap;
enum ibv_qp_type qp_type;
int sq_sig_all;
};
struct ibv_qp_init_attr_ex {
void *qp_context; // host
struct ibv_cq *send_cq; // host
struct ibv_cq *recv_cq; // host
struct ibv_srq *srq; // host
struct ibv_qp_cap cap;
enum ibv_qp_type qp_type;
int sq_sig_all;
uint32_t comp_mask;
struct ibv_pd *pd; // host
struct ibv_xrcd *xrcd; // host
uint32_t create_flags;
uint16_t max_tso_header;
struct ibv_rwq_ind_table *rwq_ind_tbl; // host
struct ibv_rx_hash_conf rx_hash_conf;
uint32_t source_qpn;
};
struct ibv_srq_init_attr_ex {
void *srq_context; // host
struct ibv_srq_attr attr;
uint32_t comp_mask;
enum ibv_srq_type srq_type;
struct ibv_pd *pd; // host
struct ibv_xrcd *xrcd; // host
struct ibv_cq *cq; // host
};
struct ibv_srq_init_attr {
void *srq_context; // host
struct ibv_srq_attr attr;
};
struct ibv_qp_open_attr {
uint32_t comp_mask;
uint32_t qp_num;
struct ibv_xrcd *xrcd; // host
void *qp_context; // host
enum ibv_qp_type qp_type;
};
struct ibv_cq_init_attr_ex {
/* Minimum number of entries required for CQ */
uint32_t cqe;
void *cq_context; // host
struct ibv_comp_channel *channel; // host
/* Completion vector used to signal completion events.
* Must be < context->num_comp_vectors.
*/
uint32_t comp_vector;
uint64_t wc_flags;
uint32_t comp_mask;
uint32_t flags;
};
struct ibv_async_event {
union {
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_srq *srq;
struct ibv_wq *wq;
int port_num;
} element;
enum ibv_event_type event_type;
};
int i, random;
int num_devices;
printf("\t\tib-test.c: before get dev list.\n");
struct ibv_device ** dev_list = ibv_get_device_list(&num_devices);
printf("\t\tib-test.c: after get device list -- ib-test.c: num devices: %d \n", num_devices);