diff --git a/include/hermit/verbs.h b/include/hermit/verbs.h index 2856e27c6..8b79f728c 100644 --- a/include/hermit/verbs.h +++ b/include/hermit/verbs.h @@ -1650,15 +1650,15 @@ struct verbs_context { //NULL : container_of(ctx, struct verbs_context, context); //} -// #define verbs_get_ctx_op(ctx, op) ({ \ - // struct verbs_context *__vctx = verbs_get_ctx(ctx); \ - // (!__vctx || (__vctx->sz < sizeof(*__vctx) - offsetof(struct verbs_context, op)) || \ - // !__vctx->op) ? NULL : __vctx; }) +/* #define verbs_get_ctx_op(ctx, op) ({ \ + struct verbs_context *__vctx = verbs_get_ctx(ctx); \ + (!__vctx || (__vctx->sz < sizeof(*__vctx) - offsetof(struct verbs_context, op)) || \ + !__vctx->op) ? NULL : __vctx; }) #define verbs_set_ctx_op(_vctx, op, ptr) ({ \ struct verbs_context *vctx = _vctx; \ if (vctx && (vctx->sz >= sizeof(*vctx) - offsetof(struct verbs_context, op))) \ - vctx->op = ptr; }) + vctx->op = ptr; }) */ ///** // * ibv_get_device_list - Get list of IB devices currently available @@ -1745,8 +1745,8 @@ struct verbs_context { // return ibv_query_port(context, port_num, port_attr); // } -// #define ibv_query_port(context, port_num, port_attr) \ - // ___ibv_query_port(context, port_num, port_attr) +/* #define ibv_query_port(context, port_num, port_attr) \ + ___ibv_query_port(context, port_num, port_attr) */ ///** // * ibv_query_gid - Get a GID table entry diff --git a/kernel/ibv.c b/kernel/ibv.c index 336bae3a0..043d1fe83 100644 --- a/kernel/ibv.c +++ b/kernel/ibv.c @@ -607,7 +607,7 @@ typedef struct { struct ibv_device ** ibv_get_device_list(int * num_devices) { uhyve_ibv_get_device_list_t uhyve_args; - uhyve_args.num_devices = num_devices; + uhyve_args.num_devices = (int *) guest_to_host((size_t) num_devices); uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned) virt_to_phys((size_t) &uhyve_args)); @@ -1004,10 +1004,10 @@ typedef struct { struct ibv_mr * ret; } __attribute__((packed)) uhyve_ibv_reg_mr_t; -struct ibv_mr * ibv_reg_mr(struct ibv_pd * pd, void * addr, int length, int access) { +struct ibv_mr * ibv_reg_mr(struct ibv_pd * pd, void * addr, int length, int access) { // !!! uhyve_ibv_reg_mr_t uhyve_args; uhyve_args.pd = pd; - uhyve_args.addr = addr; + uhyve_args.addr = (void *) guest_to_host((size_t) addr); uhyve_args.length = length; uhyve_args.access = access; @@ -1033,12 +1033,13 @@ typedef struct { int ret; } __attribute__((packed)) uhyve_ibv_rereg_mr_t; -int ibv_rereg_mr(struct ibv_mr * mr, int flags, struct ibv_pd * pd, void * addr, int length, int access) { +int ibv_rereg_mr(struct ibv_mr * mr, int flags, struct ibv_pd * pd, void * addr, + int length, int access) { // !!! uhyve_ibv_rereg_mr_t uhyve_args; - uhyve_args.mr = mr; - uhyve_args.flags = flags; - uhyve_args.pd = pd; - uhyve_args.addr = addr; + uhyve_args.mr = mr; + uhyve_args.flags = flags; + uhyve_args.pd = pd; + uhyve_args.addr = (void *) guest_to_host((size_t) addr); uhyve_args.length = length; uhyve_args.access = access; @@ -1918,7 +1919,7 @@ int ibv_destroy_rwq_ind_table(struct ibv_rwq_ind_table * rwq_ind_table) { typedef struct { // Parameters: - struct ibv_qp * qp; + struct ibv_qp * qp; struct ibv_send_wr * wr; struct ibv_send_wr ** bad_wr; // Return value: @@ -1933,19 +1934,27 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w struct ibv_send_wr * curr_wr; int num_wrs; + int num_sges_max; int is_rdma, is_atomic, is_bind_mw, is_tso; // Number of work requests in linked list - curr_wr = wr->next; num_wrs = 1; - while (curr_wr) { + for (curr_wr = wr->next; curr_wr; curr_wr = curr_wr->next) { num_wrs++; - curr_wr = curr_wr->next; + } + + // Number of SGEs in each WR + num_sges_max = 0; + for (curr_wr = wr; curr_wr; curr_wr = curr_wr->next) { + if (num_sges_max < curr_wr->num_sge) { + num_sges_max = curr_wr->num_sge; + } } // Backup arrays for original guest memory pointers struct ibv_send_wr * wr__next[num_wrs]; struct ibv_sge * wr__sg_list[num_wrs]; + uint64_t wr__sg_list__addr[num_wrs][num_sges_max]; uint64_t wr__wr__rdma__remote_addr[num_wrs]; uint64_t wr__wr__atomic__remote_addr[num_wrs]; uint64_t wr__bind_mw__bind_info__addr[num_wrs]; @@ -1982,10 +1991,19 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w // Next pointer and SGE array wr__next[i] = curr_wr->next; curr_wr->next = (struct ibv_send_wr *) guest_to_host((size_t) curr_wr->next); + + for (int s = 0; s < curr_wr->num_sge; s++) { + wr__sg_list__addr[i][s] = curr_wr->sg_list[s].addr; + curr_wr->sg_list[s].addr = (uint64_t) guest_to_host((size_t) curr_wr->sg_list[s].addr); + } + wr__sg_list[i] = curr_wr->sg_list; curr_wr->sg_list = (struct ibv_sge *) guest_to_host((size_t) curr_wr->sg_list); curr_wr = curr_wr->next; + + /* LOG_INFO("\tsg_list - guest backup address: %p", wr__sg_list[i]); */ + /* LOG_INFO("\tsg_list - host address: %p", curr_wr->sg_list); */ } uhyve_send(UHYVE_PORT_IBV_POST_SEND, (unsigned) virt_to_phys((size_t) &uhyve_args)); @@ -2022,9 +2040,13 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w if (*bad_wr && *bad_wr == curr_wr->next) { *bad_wr = wr__next[i]; } + // Next pointer and SGE array curr_wr->next = wr__next[i]; curr_wr->sg_list = wr__sg_list[i]; + for (int s = 0; s < curr_wr->num_sge; s++) { + curr_wr->sg_list[s].addr = wr__sg_list__addr[i][s]; + } curr_wr = curr_wr->next; } @@ -2053,6 +2075,7 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w uhyve_args.bad_wr = (struct ibv_recv_wr **) guest_to_host((size_t) bad_wr); struct ibv_recv_wr * curr_wr; + int num_sges_max; int num_wrs; // Number of work requests in linked list @@ -2063,15 +2086,29 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w curr_wr = curr_wr->next; } + // Number of SGEs in each WR + num_sges_max = 0; + for (curr_wr = wr; curr_wr; curr_wr = curr_wr->next) { + if (num_sges_max < curr_wr->num_sge) { + num_sges_max = curr_wr->num_sge; + } + } + // Backup arrays for original guest memory pointers struct ibv_recv_wr * wr__next[num_wrs]; struct ibv_sge * wr__sg_list[num_wrs]; + uint64_t wr__sg_list__addr[num_wrs][num_sges_max]; curr_wr = wr; for (int i = 0; i < num_wrs; i++) { wr__next[i] = curr_wr->next; curr_wr->next = (struct ibv_recv_wr *) guest_to_host((size_t) curr_wr->next); + for (int s = 0; s < curr_wr->num_sge; s++) { + wr__sg_list__addr[i][s] = curr_wr->sg_list[s].addr; + curr_wr->sg_list[s].addr = (uint64_t) guest_to_host((size_t) curr_wr->sg_list[s].addr); + } + wr__sg_list[i] = curr_wr->sg_list; curr_wr->sg_list = (struct ibv_sge *) guest_to_host((size_t) curr_wr->sg_list); @@ -2091,6 +2128,9 @@ int ibv_post_recv(struct ibv_qp * qp, struct ibv_recv_wr * wr, struct ibv_recv_w } curr_wr->next = wr__next[i]; curr_wr->sg_list = wr__sg_list[i]; + for (int s = 0; s < curr_wr->num_sge; s++) { + curr_wr->sg_list[s].addr = wr__sg_list__addr[i][s]; + } curr_wr = curr_wr->next; } diff --git a/tools/ibv_code_generator/generate-code.py b/tools/ibv_code_generator/generate-code.py index d6ece1524..3e9201519 100755 --- a/tools/ibv_code_generator/generate-code.py +++ b/tools/ibv_code_generator/generate-code.py @@ -87,6 +87,8 @@ Required in: ./tool/uhyve-ibv.h """ # TODO: Add ibv_resolve_eth_l2_from_gid function back in. Not linking right now. +# TODO: int * num devices --> guest to host +# TODO: void * as well from __future__ import print_function from parser import generate_struct_conversions diff --git a/tools/uhyve-ibv.c b/tools/uhyve-ibv.c index 9cb6d404d..26dc9178f 100644 --- a/tools/uhyve-ibv.c +++ b/tools/uhyve-ibv.c @@ -1226,6 +1226,17 @@ void call_ibv_post_send(struct kvm_run * run, uint8_t * guest_mem) { unsigned data = *((unsigned*) ((size_t) run + run->io.data_offset)); uhyve_ibv_post_send_t * args = (uhyve_ibv_post_send_t *) (guest_mem + data); + /* printf("\tqp->context: %p\n", args->qp->context); */ + /* printf("\tqp->state: %d\n", args->qp->state); */ + /* printf("\twr->id: %lu\n", args->wr->wr_id); */ + /* printf("\twr->next: %p\n", args->wr->next); */ + /* printf("\twr->num_sge: %d\n", args->wr->num_sge); */ + /* printf("\twr->sg_list: %p\n", args->wr->sg_list); */ + /* printf("\t ->sg_list[0].length: %lu\n", args->wr->sg_list[0].length); */ + /* printf("\t ->sg_list[0].lkey: %lu\n", args->wr->sg_list[0].lkey); */ + /* printf("\t ->sg_list[0].addr: %p\n", (char *) args->wr->sg_list[0].addr); */ + printf("\t ->sg_list[0].addr deref 1: %u\n", *((uint8_t *) args->wr->sg_list[0].addr)); + use_ib_mem_pool = true; args->ret = ibv_post_send(args->qp, args->wr, args->bad_wr); use_ib_mem_pool = false; diff --git a/tools/uhyve-ibv.h b/tools/uhyve-ibv.h index abba7f8a5..5ae3de12c 100644 --- a/tools/uhyve-ibv.h +++ b/tools/uhyve-ibv.h @@ -51,75 +51,75 @@ typedef enum { UHYVE_PORT_IBV_WC_READ_CVLAN = 0x625, UHYVE_PORT_IBV_WC_READ_FLOW_TAG = 0x626, UHYVE_PORT_IBV_POST_WQ_RECV = 0x627, - UHYVE_PORT_IBV_GET_DEVICE_LIST = 0x629, - UHYVE_PORT_IBV_FREE_DEVICE_LIST = 0x62A, - UHYVE_PORT_IBV_GET_DEVICE_NAME = 0x62B, - UHYVE_PORT_IBV_GET_DEVICE_GUID = 0x62C, - UHYVE_PORT_IBV_OPEN_DEVICE = 0x62D, - UHYVE_PORT_IBV_CLOSE_DEVICE = 0x62E, - UHYVE_PORT_IBV_GET_ASYNC_EVENT = 0x62F, - UHYVE_PORT_IBV_ACK_ASYNC_EVENT = 0x630, - UHYVE_PORT_IBV_QUERY_DEVICE = 0x631, - UHYVE_PORT_IBV_QUERY_PORT = 0x632, - UHYVE_PORT_IBV_QUERY_GID = 0x634, - UHYVE_PORT_IBV_QUERY_PKEY = 0x635, - UHYVE_PORT_IBV_ALLOC_PD = 0x636, - UHYVE_PORT_IBV_DEALLOC_PD = 0x637, - UHYVE_PORT_IBV_CREATE_FLOW = 0x638, - UHYVE_PORT_IBV_DESTROY_FLOW = 0x639, - UHYVE_PORT_IBV_OPEN_XRCD = 0x63A, - UHYVE_PORT_IBV_CLOSE_XRCD = 0x63B, - UHYVE_PORT_IBV_REG_MR = 0x63C, - UHYVE_PORT_IBV_REREG_MR = 0x63D, - UHYVE_PORT_IBV_DEREG_MR = 0x63E, - UHYVE_PORT_IBV_ALLOC_MW = 0x63F, - UHYVE_PORT_IBV_DEALLOC_MW = 0x640, - UHYVE_PORT_IBV_INC_RKEY = 0x641, - UHYVE_PORT_IBV_BIND_MW = 0x642, - UHYVE_PORT_IBV_CREATE_COMP_CHANNEL = 0x643, - UHYVE_PORT_IBV_DESTROY_COMP_CHANNEL = 0x644, - UHYVE_PORT_IBV_CREATE_CQ = 0x645, - UHYVE_PORT_IBV_CREATE_CQ_EX = 0x646, - UHYVE_PORT_IBV_RESIZE_CQ = 0x647, - UHYVE_PORT_IBV_DESTROY_CQ = 0x648, - UHYVE_PORT_IBV_GET_CQ_EVENT = 0x649, - UHYVE_PORT_IBV_ACK_CQ_EVENTS = 0x64A, - UHYVE_PORT_IBV_POLL_CQ = 0x64B, - UHYVE_PORT_IBV_REQ_NOTIFY_CQ = 0x64C, - UHYVE_PORT_IBV_CREATE_SRQ = 0x64D, - UHYVE_PORT_IBV_CREATE_SRQ_EX = 0x64E, - UHYVE_PORT_IBV_MODIFY_SRQ = 0x64F, - UHYVE_PORT_IBV_QUERY_SRQ = 0x650, - UHYVE_PORT_IBV_GET_SRQ_NUM = 0x651, - UHYVE_PORT_IBV_DESTROY_SRQ = 0x652, - UHYVE_PORT_IBV_POST_SRQ_RECV = 0x653, - UHYVE_PORT_IBV_CREATE_QP = 0x654, - UHYVE_PORT_IBV_CREATE_QP_EX = 0x655, - UHYVE_PORT_IBV_QUERY_RT_VALUES_EX = 0x656, - UHYVE_PORT_IBV_QUERY_DEVICE_EX = 0x657, - UHYVE_PORT_IBV_OPEN_QP = 0x658, - UHYVE_PORT_IBV_MODIFY_QP = 0x659, - UHYVE_PORT_IBV_QUERY_QP = 0x65A, - UHYVE_PORT_IBV_DESTROY_QP = 0x65B, - UHYVE_PORT_IBV_CREATE_WQ = 0x65C, - UHYVE_PORT_IBV_MODIFY_WQ = 0x65D, - UHYVE_PORT_IBV_DESTROY_WQ = 0x65E, - UHYVE_PORT_IBV_CREATE_RWQ_IND_TABLE = 0x65F, - UHYVE_PORT_IBV_DESTROY_RWQ_IND_TABLE = 0x660, - UHYVE_PORT_IBV_POST_SEND = 0x661, - UHYVE_PORT_IBV_POST_RECV = 0x662, - UHYVE_PORT_IBV_CREATE_AH = 0x663, - UHYVE_PORT_IBV_INIT_AH_FROM_WC = 0x664, - UHYVE_PORT_IBV_CREATE_AH_FROM_WC = 0x665, - UHYVE_PORT_IBV_DESTROY_AH = 0x666, - UHYVE_PORT_IBV_ATTACH_MCAST = 0x667, - UHYVE_PORT_IBV_DETACH_MCAST = 0x668, - UHYVE_PORT_IBV_FORK_INIT = 0x669, - UHYVE_PORT_IBV_NODE_TYPE_STR = 0x66A, - UHYVE_PORT_IBV_PORT_STATE_STR = 0x66B, - UHYVE_PORT_IBV_EVENT_TYPE_STR = 0x66C, - // UHYVE_PORT_IBV_RESOLVE_ETH_L2_FROM_GID = 0x66D, - UHYVE_PORT_IBV_IS_QPT_SUPPORTED = 0x66E, + UHYVE_PORT_IBV_GET_DEVICE_LIST = 0x628, + UHYVE_PORT_IBV_FREE_DEVICE_LIST = 0x629, + UHYVE_PORT_IBV_GET_DEVICE_NAME = 0x62A, + UHYVE_PORT_IBV_GET_DEVICE_GUID = 0x62B, + UHYVE_PORT_IBV_OPEN_DEVICE = 0x62C, + UHYVE_PORT_IBV_CLOSE_DEVICE = 0x62D, + UHYVE_PORT_IBV_GET_ASYNC_EVENT = 0x62E, + UHYVE_PORT_IBV_ACK_ASYNC_EVENT = 0x62F, + UHYVE_PORT_IBV_QUERY_DEVICE = 0x630, + UHYVE_PORT_IBV_QUERY_PORT = 0x631, + UHYVE_PORT_IBV_QUERY_GID = 0x632, + UHYVE_PORT_IBV_QUERY_PKEY = 0x633, + UHYVE_PORT_IBV_ALLOC_PD = 0x634, + UHYVE_PORT_IBV_DEALLOC_PD = 0x635, + UHYVE_PORT_IBV_CREATE_FLOW = 0x636, + UHYVE_PORT_IBV_DESTROY_FLOW = 0x637, + UHYVE_PORT_IBV_OPEN_XRCD = 0x638, + UHYVE_PORT_IBV_CLOSE_XRCD = 0x639, + UHYVE_PORT_IBV_REG_MR = 0x63A, + UHYVE_PORT_IBV_REREG_MR = 0x63B, + UHYVE_PORT_IBV_DEREG_MR = 0x63C, + UHYVE_PORT_IBV_ALLOC_MW = 0x63D, + UHYVE_PORT_IBV_DEALLOC_MW = 0x63E, + UHYVE_PORT_IBV_INC_RKEY = 0x63F, + UHYVE_PORT_IBV_BIND_MW = 0x640, + UHYVE_PORT_IBV_CREATE_COMP_CHANNEL = 0x641, + UHYVE_PORT_IBV_DESTROY_COMP_CHANNEL = 0x642, + UHYVE_PORT_IBV_CREATE_CQ = 0x643, + UHYVE_PORT_IBV_CREATE_CQ_EX = 0x644, + UHYVE_PORT_IBV_RESIZE_CQ = 0x645, + UHYVE_PORT_IBV_DESTROY_CQ = 0x646, + UHYVE_PORT_IBV_GET_CQ_EVENT = 0x647, + UHYVE_PORT_IBV_ACK_CQ_EVENTS = 0x648, + UHYVE_PORT_IBV_POLL_CQ = 0x649, + UHYVE_PORT_IBV_REQ_NOTIFY_CQ = 0x64A, + UHYVE_PORT_IBV_CREATE_SRQ = 0x64B, + UHYVE_PORT_IBV_CREATE_SRQ_EX = 0x64C, + UHYVE_PORT_IBV_MODIFY_SRQ = 0x64D, + UHYVE_PORT_IBV_QUERY_SRQ = 0x64E, + UHYVE_PORT_IBV_GET_SRQ_NUM = 0x64F, + UHYVE_PORT_IBV_DESTROY_SRQ = 0x650, + UHYVE_PORT_IBV_POST_SRQ_RECV = 0x651, + UHYVE_PORT_IBV_CREATE_QP = 0x652, + UHYVE_PORT_IBV_CREATE_QP_EX = 0x653, + UHYVE_PORT_IBV_QUERY_RT_VALUES_EX = 0x654, + UHYVE_PORT_IBV_QUERY_DEVICE_EX = 0x655, + UHYVE_PORT_IBV_OPEN_QP = 0x656, + UHYVE_PORT_IBV_MODIFY_QP = 0x657, + UHYVE_PORT_IBV_QUERY_QP = 0x658, + UHYVE_PORT_IBV_DESTROY_QP = 0x659, + UHYVE_PORT_IBV_CREATE_WQ = 0x65A, + UHYVE_PORT_IBV_MODIFY_WQ = 0x65B, + UHYVE_PORT_IBV_DESTROY_WQ = 0x65C, + UHYVE_PORT_IBV_CREATE_RWQ_IND_TABLE = 0x65D, + UHYVE_PORT_IBV_DESTROY_RWQ_IND_TABLE = 0x65E, + UHYVE_PORT_IBV_POST_SEND = 0x65F, + UHYVE_PORT_IBV_POST_RECV = 0x660, + UHYVE_PORT_IBV_CREATE_AH = 0x661, + UHYVE_PORT_IBV_INIT_AH_FROM_WC = 0x662, + UHYVE_PORT_IBV_CREATE_AH_FROM_WC = 0x663, + UHYVE_PORT_IBV_DESTROY_AH = 0x664, + UHYVE_PORT_IBV_ATTACH_MCAST = 0x665, + UHYVE_PORT_IBV_DETACH_MCAST = 0x666, + UHYVE_PORT_IBV_FORK_INIT = 0x667, + UHYVE_PORT_IBV_NODE_TYPE_STR = 0x668, + UHYVE_PORT_IBV_PORT_STATE_STR = 0x669, + UHYVE_PORT_IBV_EVENT_TYPE_STR = 0x66A, + // UHYVE_PORT_IBV_RESOLVE_ETH_L2_FROM_GID = 0x66B, + UHYVE_PORT_IBV_IS_QPT_SUPPORTED = 0x66C, UHYVE_PORT_KERNEL_IBV_LOG = 0x66F, } uhyve_ibv_t; diff --git a/tools/uhyve.c b/tools/uhyve.c index 847065de7..928281eea 100644 --- a/tools/uhyve.c +++ b/tools/uhyve.c @@ -172,7 +172,6 @@ #define APIC_DEFAULT_BASE 0xfee00000 #define IB_POOL_SIZE 0x400000ULL -#define IB_MEM_DEBUG static bool restart = false; static bool cap_tsc_deadline = false; @@ -226,6 +225,8 @@ typedef struct { // DLSYM +/* #define IB_MEM_DEBUG */ + /* static uint8_t * ib_pool_top = NULL; */ /* uint64_t ib_pool_addr = 0; // TODO: static? */ static uint8_t* ib_pool_addr = 0; @@ -259,32 +260,24 @@ static void init_ib_mem_functions(void) if (real_malloc == NULL && real_calloc == NULL && real_realloc == NULL && real_free == NULL) { init_real_calloc_active = true; - /* size_t dlsym_mem_len = 1024 * 8; */ - /* dlsym_mem = mmap(NULL, dlsym_mem_len, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANONYMOUS | MAP_SHARED, -1, 0); */ - real_calloc = dlsym(RTLD_NEXT, "calloc"); - init_real_calloc_active = false; - /* munmap(dlsym_mem, dlsym_mem_len); */ - /* dlsym_mem = NULL; */ real_malloc = dlsym(RTLD_NEXT, "malloc"); real_realloc = dlsym(RTLD_NEXT, "realloc"); real_free = dlsym(RTLD_NEXT, "free"); } - if (!real_malloc || !real_calloc || !real_free || !real_realloc /* || !real_free_alt */ ) { + if (!real_malloc || !real_calloc || !real_free || !real_realloc) { fprintf(stderr, "Error in `dlsym`: %s\n", dlerror()); } pthread_mutex_unlock(&ib_pool_mutex); - #ifdef IB_MEM_DEBUG printf("\tInit finished.\n"); #endif } - /* * new_ib_malloc_region */ @@ -314,14 +307,13 @@ void * new_ib_malloc_region(size_t size) } if (ib_pool_top > ib_pool_addr + IB_POOL_SIZE) { // TODO: make ib addr a uint8* as well? -#ifdef IB_MEM_DEBUG - fprintf(stderr, "Error: IB Memory Pool overflow.\n"); -#endif + fprintf(stderr, "\nError: IB Memory Pool overflow.------------------------------\n\n"); } return result; } +// TODO: locks ok? /* * malloc @@ -330,7 +322,7 @@ void * new_ib_malloc_region(size_t size) void * malloc(size_t size) { #ifdef IB_MEM_DEBUG - printf("FUNCTION: malloc()\n"); + printf("FUNCTION: malloc() ------------- size: %lu\n", size); #endif if (real_malloc == NULL) { @@ -341,8 +333,14 @@ void * malloc(size_t size) void * result; if (use_ib_mem_pool) { +#ifdef IB_MEM_DEBUG + printf("\tuse_ib_mem_pool == true.\n"); +#endif result = new_ib_malloc_region(size); } else { // !use_ib_mem_pool +#ifdef IB_MEM_DEBUG + printf("\tcalling real_malloc()\n"); +#endif result = real_malloc(size); } @@ -358,7 +356,7 @@ void * malloc(size_t size) void * calloc(size_t nitems, size_t size) { #ifdef IB_MEM_DEBUG - printf("FUNCTION: calloc()\n"); + printf("FUNCTION: calloc() ------------- nitems: %lu size: %lu\n", nitems, size); #endif void * result; @@ -375,11 +373,11 @@ void * calloc(size_t nitems, size_t size) result = dlsym_mem_buffer; } else if (use_ib_mem_pool) { #ifdef IB_MEM_DEBUG - printf("\tuse_ib == true\n"); + printf("\tuse_ib_mem_pool == true\n"); #endif result = new_ib_malloc_region(full_size); memset(result, 0, full_size); - } else { + } else { // !use_ib_mem_pool && !init_real_calloc_active #ifdef IB_MEM_DEBUG printf("\tcalling real_calloc()\n"); #endif diff --git a/usr/benchmarks/ib/perftest_communication_write_bw.c b/usr/benchmarks/ib/perftest_communication_write_bw.c index cf2f14491..457a6525e 100755 --- a/usr/benchmarks/ib/perftest_communication_write_bw.c +++ b/usr/benchmarks/ib/perftest_communication_write_bw.c @@ -442,17 +442,17 @@ static int ethernet_client_connect(struct perftest_comm *comm) { struct addrinfo *res, *t; struct addrinfo hints; - /* char *service; */ + char *service; int sockfd = -1; memset(&hints, 0, sizeof hints); hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; - /* if (check_add_port(&service, comm->rdma_params->port, comm->rdma_params->servername, &hints, &res)) { */ - /* fprintf(stderr, "Problem in resolving basic address and port\n"); */ - /* return 1; */ - /* } */ + if (check_add_port(&service, comm->rdma_params->port, comm->rdma_params->servername, &hints, &res)) { + fprintf(stderr, "Problem in resolving basic address and port\n"); + return 1; + } for (t = res; t; t = t->ai_next) { sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); @@ -483,7 +483,7 @@ static int ethernet_server_connect(struct perftest_comm *comm) { struct addrinfo *res, *t; struct addrinfo hints; - /* char *service; */ + char *service; int n; int sockfd = -1, connfd; @@ -492,10 +492,10 @@ static int ethernet_server_connect(struct perftest_comm *comm) hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; - /* if (check_add_port(&service, comm->rdma_params->port, NULL, &hints, &res)) { */ - /* fprintf(stderr, "Problem in resolving basic address and port\n"); */ - /* return 1; */ - /* } */ + if (check_add_port(&service, comm->rdma_params->port, NULL, &hints, &res)) { + fprintf(stderr, "Problem in resolving basic address and port\n"); + return 1; + } for (t = res; t; t = t->ai_next) { sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); diff --git a/usr/benchmarks/ib/perftest_resources.c b/usr/benchmarks/ib/perftest_resources.c index 8dd6c2e39..24bb1c5d2 100755 --- a/usr/benchmarks/ib/perftest_resources.c +++ b/usr/benchmarks/ib/perftest_resources.c @@ -525,26 +525,26 @@ static struct ibv_qp *ctx_rss_eth_qp_create(struct pingpong_context *ctx,struct /****************************************************************************** * ******************************************************************************/ -/* int check_add_port(char **service,int port, */ - /* const char *servername, */ - /* struct addrinfo *hints, */ - /* struct addrinfo **res) */ -/* { */ - /* int number; */ +int check_add_port(char **service,int port, + const char *servername, + struct addrinfo *hints, + struct addrinfo **res) +{ + int number; - /* if (asprintf(service,"%d", port) < 0) { */ - /* return FAILURE; */ - /* } */ + if (asprintf(service,"%d", port) < 0) { + return FAILURE; + } - /* number = getaddrinfo(servername,*service,hints,res); */ + number = getaddrinfo(servername,*service,hints,res); - /* if (number < 0) { */ - /* fprintf(stderr, "%s for %s:%d\n", gai_strerror(number), servername, port); */ - /* return FAILURE; */ - /* } */ + if (number < 0) { + fprintf(stderr, "%s for %s:%d\n", gai_strerror(number), servername, port); + return FAILURE; + } - /* return SUCCESS; */ -/* } */ + return SUCCESS; +} /****************************************************************************** + * diff --git a/usr/benchmarks/ib/perftest_resources_write_bw.c b/usr/benchmarks/ib/perftest_resources_write_bw.c index dcdc354d5..05fd26b16 100755 --- a/usr/benchmarks/ib/perftest_resources_write_bw.c +++ b/usr/benchmarks/ib/perftest_resources_write_bw.c @@ -711,6 +711,28 @@ int verify_params_with_device_context(struct ibv_context *context, // ----------------------------------------------------------------------------- +int check_add_port(char **service,int port, + const char *servername, + struct addrinfo *hints, + struct addrinfo **res) +{ + int number; + + if (asprintf(service,"%d", port) < 0) { + return FAILURE; + } + + number = getaddrinfo(servername,*service,hints,res); + + if (number < 0) { + /* fprintf(stderr, "%s for %s:%d\n", gai_strerror(number), servername, port); */ + fprintf(stderr, "Error"); + return FAILURE; + } + + return SUCCESS; +} + struct ibv_device* ctx_find_dev(const char *ib_devname) { int num_of_device; diff --git a/usr/benchmarks/ib/perftest_resources_write_bw.h b/usr/benchmarks/ib/perftest_resources_write_bw.h index 03a3d09ad..e21ecc56a 100755 --- a/usr/benchmarks/ib/perftest_resources_write_bw.h +++ b/usr/benchmarks/ib/perftest_resources_write_bw.h @@ -212,10 +212,10 @@ struct raw_ethernet_info { // ! * * Return Value : SUCCESS, FAILURE. */ -// int check_add_port(char **service,int port, - // const char *servername, - // struct addrinfo *hints, - // struct addrinfo **res); +int check_add_port(char **service,int port, + const char *servername, + struct addrinfo *hints, + struct addrinfo **res); /* ctx_find_dev * diff --git a/usr/tests/ib-test.c b/usr/tests/ib-test.c index 86131737c..cf1b0d349 100644 --- a/usr/tests/ib-test.c +++ b/usr/tests/ib-test.c @@ -43,101 +43,9 @@ int main(int argc, char** argv) { -// CONTAINING HOST PTRS: - -struct ibv_wq_init_attr { - void *wq_context; // host - enum ibv_wq_type wq_type; - uint32_t max_wr; - uint32_t max_sge; - struct ibv_pd *pd; // host - struct ibv_cq *cq; // host - uint32_t comp_mask; /* Use ibv_wq_init_attr_mask */ - uint32_t create_flags; /* use ibv_wq_flags */ -}; - -struct ibv_qp_init_attr { - void *qp_context; // host - struct ibv_cq *send_cq; // host - struct ibv_cq *recv_cq; // host - struct ibv_srq *srq; // host - struct ibv_qp_cap cap; - enum ibv_qp_type qp_type; - int sq_sig_all; -}; - -struct ibv_qp_init_attr_ex { - void *qp_context; // host - struct ibv_cq *send_cq; // host - struct ibv_cq *recv_cq; // host - struct ibv_srq *srq; // host - struct ibv_qp_cap cap; - enum ibv_qp_type qp_type; - int sq_sig_all; - - uint32_t comp_mask; - struct ibv_pd *pd; // host - struct ibv_xrcd *xrcd; // host - uint32_t create_flags; - uint16_t max_tso_header; - struct ibv_rwq_ind_table *rwq_ind_tbl; // host - struct ibv_rx_hash_conf rx_hash_conf; - uint32_t source_qpn; -}; - -struct ibv_srq_init_attr_ex { - void *srq_context; // host - struct ibv_srq_attr attr; - - uint32_t comp_mask; - enum ibv_srq_type srq_type; - struct ibv_pd *pd; // host - struct ibv_xrcd *xrcd; // host - struct ibv_cq *cq; // host -}; - -struct ibv_srq_init_attr { - void *srq_context; // host - struct ibv_srq_attr attr; -}; - -struct ibv_qp_open_attr { - uint32_t comp_mask; - uint32_t qp_num; - struct ibv_xrcd *xrcd; // host - void *qp_context; // host - enum ibv_qp_type qp_type; -}; - -struct ibv_cq_init_attr_ex { - /* Minimum number of entries required for CQ */ - uint32_t cqe; - void *cq_context; // host - struct ibv_comp_channel *channel; // host - /* Completion vector used to signal completion events. - * Must be < context->num_comp_vectors. - */ - uint32_t comp_vector; - uint64_t wc_flags; - uint32_t comp_mask; - uint32_t flags; -}; - -struct ibv_async_event { - union { - struct ibv_cq *cq; - struct ibv_qp *qp; - struct ibv_srq *srq; - struct ibv_wq *wq; - int port_num; - } element; - enum ibv_event_type event_type; -}; - int i, random; - - int num_devices; + printf("\t\tib-test.c: before get dev list.\n"); struct ibv_device ** dev_list = ibv_get_device_list(&num_devices); printf("\t\tib-test.c: after get device list -- ib-test.c: num devices: %d \n", num_devices);