Mirrors/libhermit
1
0
Fork 0
mirror of https://github.com/hermitcore/libhermit.git synced 2025-03-09 00:00:03 +01:00
Code Issues Releases Wiki Activity

perftest not linking

Browse source
This commit is contained in:
Annika Wierichs 2018-01-01 19:01:39 +01:00
parent 98c9dad011
commit 36e69bdc12
9 changed files with 1588 additions and 1007 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

612
usr/benchmarks/ib/perftest_communication.c
View file

@ -5,9 +5,7 @@
#include <unistd.h>
#include <getopt.h>
#include <malloc.h>
/* #include <byteswap.h> */
#include <errno.h>
/* #include <arpa/inet.h> */
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>
@ -26,12 +24,12 @@ static inline int valid_mtu_size(int mtu_size)
static inline int ipv6_addr_v4mapped(const struct in6_addr *a)
{
return ((a->s6_addr32[0] | a->s6_addr32[1]) |
(a->s6_addr32[2] ^ htonl(0x0000ffff))) == 0UL ||
return ((a->un.u32_addr[0] | a->un.u32_addr[1]) | // !
(a->un.u32_addr[2] ^ htonl(0x0000ffff))) == 0UL ||
/* IPv4 encoded multicast addresses */
(a->s6_addr32[0] == htonl(0xff0e0000) &&
((a->s6_addr32[1] |
(a->s6_addr32[2] ^ htonl(0x0000ffff))) == 0UL));
(a->un.u32_addr[0] == htonl(0xff0e0000) &&
((a->un.u32_addr[1] |
(a->un.u32_addr[2] ^ htonl(0x0000ffff))) == 0UL));
}
@ -158,30 +156,31 @@ static int send_qp_num_for_ah(struct pingpong_context *ctx,
/******************************************************************************
*
******************************************************************************/
static int create_ah_from_wc_recv(struct pingpong_context *ctx,
struct perftest_parameters *user_param)
{
struct ibv_qp_attr attr;
struct ibv_qp_init_attr init_attr;
struct ibv_wc wc;
int ne;
/* static int create_ah_from_wc_recv(struct pingpong_context *ctx, */
/* struct perftest_parameters *user_param) */
/* { */
/* struct ibv_qp_attr attr; */
/* struct ibv_qp_init_attr init_attr; */
/* struct ibv_wc wc; */
/* int ne; */
do {
ne = ibv_poll_cq(ctx->recv_cq,1,&wc);
} while (ne == 0);
/* do { */
/* ne = ibv_poll_cq(ctx->recv_cq,1,&wc); */
/* } while (ne == 0); */
if (wc.status || !(wc.opcode & IBV_WC_RECV) || wc.wr_id != 0) {
fprintf(stderr, "Bad wc status when trying to create AH -- %d -- %d \n",(int)wc.status,(int)wc.wr_id);
return 1;
}
/* if (wc.status || !(wc.opcode & IBV_WC_RECV) || wc.wr_id != 0) { */
/* fprintf(stderr, "Bad wc status when trying to create AH -- %d -- %d \n",(int)wc.status,(int)wc.wr_id); */
/* return 1; */
/* } */
ctx->ah[0] = ibv_create_ah_from_wc(ctx->pd, &wc, (struct ibv_grh*)ctx->buf[0], ctx->cm_id->port_num);
user_param->rem_ud_qpn = ntohl(wc.imm_data);
ibv_query_qp(ctx->qp[0],&attr, IBV_QP_QKEY,&init_attr);
user_param->rem_ud_qkey = attr.qkey;
/* ctx->ah[0] = ibv_create_ah_from_wc(ctx->pd, &wc, (struct ibv_grh*)ctx->buf[0], */
/* ctx->cm_id->port_num); */
/* user_param->rem_ud_qpn = ntohl(wc.imm_data); */
/* ibv_query_qp(ctx->qp[0],&attr, IBV_QP_QKEY,&init_attr); */
/* user_param->rem_ud_qkey = attr.qkey; */
return 0;
}
/* return 0; */
/* } */
/******************************************************************************
@ -330,107 +329,64 @@ static int ethernet_read_keys(struct pingpong_dest *rem_dest,
/******************************************************************************
*
******************************************************************************/
static int rdma_write_keys(struct pingpong_dest *my_dest,
struct perftest_comm *comm)
{
struct ibv_send_wr wr;
struct ibv_send_wr *bad_wr;
struct ibv_sge list;
struct ibv_wc wc;
int ne;
/* static int rdma_write_keys(struct pingpong_dest *my_dest, */
/* struct perftest_comm *comm) */
/* { */
/* struct ibv_send_wr wr; */
/* struct ibv_send_wr *bad_wr; */
/* struct ibv_sge list; */
/* struct ibv_wc wc; */
/* int ne; */
#ifdef HAVE_ENDIAN
int i;
struct pingpong_dest m_my_dest;
/* #ifdef HAVE_ENDIAN */
/* int i; */
/* struct pingpong_dest m_my_dest; */
m_my_dest.lid = htobe32(my_dest->lid);
m_my_dest.out_reads = htobe32(my_dest->out_reads);
m_my_dest.qpn = htobe32(my_dest->qpn);
m_my_dest.psn = htobe32(my_dest->psn);
m_my_dest.rkey = htobe32(my_dest->rkey);
m_my_dest.srqn = htobe32(my_dest->srqn);
m_my_dest.gid_index = htobe32(my_dest->gid_index);
m_my_dest.vaddr = htobe64(my_dest->vaddr);
/* m_my_dest.lid = htobe32(my_dest->lid); */
/* m_my_dest.out_reads = htobe32(my_dest->out_reads); */
/* m_my_dest.qpn = htobe32(my_dest->qpn); */
/* m_my_dest.psn = htobe32(my_dest->psn); */
/* m_my_dest.rkey = htobe32(my_dest->rkey); */
/* m_my_dest.srqn = htobe32(my_dest->srqn); */
/* m_my_dest.gid_index = htobe32(my_dest->gid_index); */
/* m_my_dest.vaddr = htobe64(my_dest->vaddr); */
for(i=0; i<16; i++) {
m_my_dest.gid.raw[i] = my_dest->gid.raw[i];
}
/* for(i=0; i<16; i++) { */
/* m_my_dest.gid.raw[i] = my_dest->gid.raw[i]; */
/* } */
memcpy(comm->rdma_ctx->buf[0], &m_my_dest, sizeof(struct pingpong_dest));
#else
memcpy(comm->rdma_ctx->buf[0], &my_dest, sizeof(struct pingpong_dest));
#endif
list.addr = (uintptr_t)comm->rdma_ctx->buf[0];
list.length = sizeof(struct pingpong_dest);
list.lkey = comm->rdma_ctx->mr[0]->lkey;
/* memcpy(comm->rdma_ctx->buf[0], &m_my_dest, sizeof(struct pingpong_dest)); */
/* #else */
/* memcpy(comm->rdma_ctx->buf[0], &my_dest, sizeof(struct pingpong_dest)); */
/* #endif */
/* list.addr = (uintptr_t)comm->rdma_ctx->buf[0]; */
/* list.length = sizeof(struct pingpong_dest); */
/* list.lkey = comm->rdma_ctx->mr[0]->lkey; */
wr.wr_id = SYNC_SPEC_ID;
wr.sg_list = &list;
wr.num_sge = 1;
wr.opcode = IBV_WR_SEND;
wr.send_flags = IBV_SEND_SIGNALED;
wr.next = NULL;
/* wr.wr_id = SYNC_SPEC_ID; */
/* wr.sg_list = &list; */
/* wr.num_sge = 1; */
/* wr.opcode = IBV_WR_SEND; */
/* wr.send_flags = IBV_SEND_SIGNALED; */
/* wr.next = NULL; */
if (ibv_post_send(comm->rdma_ctx->qp[0],&wr,&bad_wr)) {
fprintf(stderr, "Function ibv_post_send failed\n");
return 1;
}
/* if (ibv_post_send(comm->rdma_ctx->qp[0],&wr,&bad_wr)) { */
/* fprintf(stderr, "Function ibv_post_send failed\n"); */
/* return 1; */
/* } */
do {
ne = ibv_poll_cq(comm->rdma_ctx->send_cq, 1,&wc);
} while (ne == 0);
/* do { */
/* ne = ibv_poll_cq(comm->rdma_ctx->send_cq, 1,&wc); */
/* } while (ne == 0); */
if (wc.status || wc.opcode != IBV_WC_SEND || wc.wr_id != SYNC_SPEC_ID) {
fprintf(stderr, " Bad wc status %d\n",(int)wc.status);
return 1;
}
/* if (wc.status || wc.opcode != IBV_WC_SEND || wc.wr_id != SYNC_SPEC_ID) { */
/* fprintf(stderr, " Bad wc status %d\n",(int)wc.status); */
/* return 1; */
/* } */
return 0;
}
/******************************************************************************
*
******************************************************************************/
static int rdma_read_keys(struct pingpong_dest *rem_dest,
struct perftest_comm *comm)
{
#ifdef HAVE_ENDIAN
struct pingpong_dest a_rem_dest;
#endif
struct ibv_wc wc;
int ne;
do {
ne = ibv_poll_cq(comm->rdma_ctx->recv_cq,1,&wc);
} while (ne == 0);
if (wc.status || !(wc.opcode & IBV_WC_RECV) || wc.wr_id != SYNC_SPEC_ID) {
fprintf(stderr, "Bad wc status -- %d -- %d \n",(int)wc.status,(int)wc.wr_id);
return 1;
}
#ifdef HAVE_ENDIAN
memcpy(&a_rem_dest,comm->rdma_ctx->buf[0],sizeof(struct pingpong_dest));
rem_dest->lid = ntohl(a_rem_dest.lid);
rem_dest->out_reads = ntohl(a_rem_dest.out_reads);
rem_dest->qpn = ntohl(a_rem_dest.qpn);
rem_dest->psn = ntohl(a_rem_dest.psn);
rem_dest->rkey = ntohl(a_rem_dest.rkey);
rem_dest->vaddr = be64toh(a_rem_dest.vaddr);
memcpy(rem_dest->gid.raw, &(a_rem_dest.gid), 16*sizeof(uint8_t));
#else
memcpy(&rem_dest,comm->rdma_ctx->buf[0],sizeof(struct pingpong_dest));
#endif
if (post_one_recv_wqe(comm->rdma_ctx)) {
fprintf(stderr, "Couldn't post send \n");
return 1;
}
return 0;
}
/* return 0; */
/* } */
#ifdef HAVE_GID_ATTR
@ -494,7 +450,7 @@ static int get_best_gid_index (struct pingpong_context *ctx,
return -1;
}
is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)temp_gid.raw);
is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)temp_gid.raw);
is_ipv4_rival = ipv6_addr_v4mapped((struct in6_addr *)temp_gid_rival.raw);
if (is_ipv4_rival && !is_ipv4 && !user_param->ipv6)
@ -531,17 +487,17 @@ static int ethernet_client_connect(struct perftest_comm *comm)
{
struct addrinfo *res, *t;
struct addrinfo hints;
char *service;
/* char *service; */
int sockfd = -1;
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if (check_add_port(&service,comm->rdma_params->port,comm->rdma_params->servername,&hints,&res)) {
fprintf(stderr, "Problem in resolving basic address and port\n");
return 1;
}
/* if (check_add_port(&service, comm->rdma_params->port, comm->rdma_params->servername, &hints, &res)) { */
/* fprintf(stderr, "Problem in resolving basic address and port\n"); */
/* return 1; */
/* } */
for (t = res; t; t = t->ai_next) {
sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
@ -572,7 +528,7 @@ static int ethernet_server_connect(struct perftest_comm *comm)
{
struct addrinfo *res, *t;
struct addrinfo hints;
char *service;
/* char *service; */
int n;
int sockfd = -1, connfd;
@ -581,10 +537,10 @@ static int ethernet_server_connect(struct perftest_comm *comm)
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if (check_add_port(&service,comm->rdma_params->port,NULL,&hints,&res)) {
fprintf(stderr, "Problem in resolving basic address and port\n");
return 1;
}
/* if (check_add_port(&service, comm->rdma_params->port, NULL, &hints, &res)) { */
/* fprintf(stderr, "Problem in resolving basic address and port\n"); */
/* return 1; */
/* } */
for (t = res; t; t = t->ai_next) {
sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
@ -758,316 +714,6 @@ int set_up_connection(struct pingpong_context *ctx,
return 0;
}
/******************************************************************************
*
******************************************************************************/
int rdma_client_connect(struct pingpong_context *ctx,struct perftest_parameters *user_param)
{
char *service;
int temp,num_of_retry= NUM_OF_RETRIES;
struct sockaddr_in sin;
struct addrinfo *res;
struct rdma_cm_event *event;
struct rdma_conn_param conn_param;
struct addrinfo hints;
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if (check_add_port(&service,user_param->port,user_param->servername,&hints,&res)) {
fprintf(stderr, "Problem in resolving basic address and port\n");
return FAILURE;
}
if (res->ai_family != PF_INET) {
return FAILURE;
}
memcpy(&sin, res->ai_addr, sizeof(sin));
sin.sin_port = htons((unsigned short)user_param->port);
while (1) {
if (num_of_retry == 0) {
fprintf(stderr, "Received %d times ADDR_ERROR\n",NUM_OF_RETRIES);
return FAILURE;
}
if (rdma_resolve_addr(ctx->cm_id, NULL,(struct sockaddr *)&sin,2000)) {
fprintf(stderr, "rdma_resolve_addr failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event == RDMA_CM_EVENT_ADDR_ERROR) {
num_of_retry--;
rdma_ack_cm_event(event);
continue;
}
if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
fprintf(stderr, "unexpected CM event %d\n",event->event);
rdma_ack_cm_event(event);
return FAILURE;
}
rdma_ack_cm_event(event);
break;
}
if (user_param->tos != DEF_TOS) {
if (rdma_set_option(ctx->cm_id,RDMA_OPTION_ID,RDMA_OPTION_ID_TOS,&user_param->tos,sizeof(uint8_t))) {
fprintf(stderr, " Set TOS option failed: %d\n",event->event);
return FAILURE;
}
}
while (1) {
if (num_of_retry <= 0) {
fprintf(stderr, "Received %d times ADDR_ERROR - aborting\n",NUM_OF_RETRIES);
return FAILURE;
}
if (rdma_resolve_route(ctx->cm_id,2000)) {
fprintf(stderr, "rdma_resolve_route failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event == RDMA_CM_EVENT_ROUTE_ERROR) {
num_of_retry--;
rdma_ack_cm_event(event);
continue;
}
if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
fprintf(stderr, "unexpected CM event %d\n",event->event);
rdma_ack_cm_event(event);
return FAILURE;
}
rdma_ack_cm_event(event);
break;
}
ctx->context = ctx->cm_id->verbs;
temp = user_param->work_rdma_cm;
user_param->work_rdma_cm = ON;
if (ctx_init(ctx, user_param)) {
fprintf(stderr," Unable to create the resources needed by comm struct\n");
return FAILURE;
}
memset(&conn_param, 0, sizeof conn_param);
if (user_param->verb == READ || user_param->verb == ATOMIC) {
conn_param.responder_resources = user_param->out_reads;
conn_param.initiator_depth = user_param->out_reads;
}
user_param->work_rdma_cm = temp;
conn_param.retry_count = user_param->retry_count;
conn_param.rnr_retry_count = 7;
if (user_param->work_rdma_cm == OFF) {
if (post_one_recv_wqe(ctx)) {
fprintf(stderr, "Couldn't post send \n");
return 1;
}
}
if (rdma_connect(ctx->cm_id,&conn_param)) {
fprintf(stderr, "Function rdma_connect failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event != RDMA_CM_EVENT_ESTABLISHED) {
fprintf(stderr, "Unexpected CM event bl blka %d\n", event->event);
rdma_ack_cm_event(event);
return FAILURE;
}
if (user_param->connection_type == UD) {
user_param->rem_ud_qpn = event->param.ud.qp_num;
user_param->rem_ud_qkey = event->param.ud.qkey;
ctx->ah[0] = ibv_create_ah(ctx->pd,&event->param.ud.ah_attr);
if (!ctx->ah[0]) {
printf(" Unable to create address handler for UD QP\n");
return FAILURE;
}
if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) {
if (send_qp_num_for_ah(ctx,user_param)) {
printf(" Unable to send my QP number\n");
return FAILURE;
}
}
}
rdma_ack_cm_event(event);
return SUCCESS;
}
/******************************************************************************
*
******************************************************************************/
int retry_rdma_connect(struct pingpong_context *ctx,
struct perftest_parameters *user_param)
{
int i, max_retries = 10;
int delay = 100000; /* 100 millisec */
for (i = 0; i < max_retries; i++) {
if (create_rdma_resources(ctx,user_param)) {
fprintf(stderr," Unable to create rdma resources\n");
return FAILURE;
}
if (rdma_client_connect(ctx,user_param) == SUCCESS)
return SUCCESS;
if (destroy_rdma_resources(ctx,user_param)) {
fprintf(stderr,"Unable to destroy rdma resources\n");
return FAILURE;
}
usleep(delay);
}
fprintf(stderr,"Unable to connect (retries = %d)\n", max_retries);
return FAILURE;
}
/******************************************************************************
+ *
+ ******************************************************************************/
int rdma_server_connect(struct pingpong_context *ctx,
struct perftest_parameters *user_param)
{
int temp;
struct addrinfo *res;
struct rdma_cm_event *event;
struct rdma_conn_param conn_param;
struct addrinfo hints;
char *service;
struct sockaddr_in sin;
memset(&hints, 0, sizeof hints);
hints.ai_flags = AI_PASSIVE;
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if (check_add_port(&service,user_param->port,user_param->servername,&hints,&res)) {
fprintf(stderr, "Problem in resolving basic address and port\n");
return FAILURE;
}
if (res->ai_family != PF_INET) {
return FAILURE;
}
memcpy(&sin, res->ai_addr, sizeof(sin));
sin.sin_port = htons((unsigned short)user_param->port);
if (rdma_bind_addr(ctx->cm_id_control,(struct sockaddr *)&sin)) {
fprintf(stderr," rdma_bind_addr failed\n");
return 1;
}
if (rdma_listen(ctx->cm_id_control,0)) {
fprintf(stderr, "rdma_listen failed\n");
return 1;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return 1;
}
if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
fprintf(stderr, "bad event waiting for connect request %d\n",event->event);
return 1;
}
ctx->cm_id = (struct rdma_cm_id*)event->id;
ctx->context = ctx->cm_id->verbs;
if (user_param->work_rdma_cm == ON)
alloc_ctx(ctx,user_param);
temp = user_param->work_rdma_cm;
user_param->work_rdma_cm = ON;
if (ctx_init(ctx,user_param)) {
fprintf(stderr," Unable to create the resources needed by comm struct\n");
return FAILURE;
}
memset(&conn_param, 0, sizeof conn_param);
if (user_param->verb == READ || user_param->verb == ATOMIC) {
conn_param.responder_resources = user_param->out_reads;
conn_param.initiator_depth = user_param->out_reads;
}
if (user_param->connection_type == UD)
conn_param.qp_num = ctx->qp[0]->qp_num;
conn_param.retry_count = user_param->retry_count;
conn_param.rnr_retry_count = 7;
user_param->work_rdma_cm = temp;
if (user_param->work_rdma_cm == OFF) {
if (post_one_recv_wqe(ctx)) {
fprintf(stderr, "Couldn't post send \n");
return 1;
}
} else if (user_param->connection_type == UD) {
if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) {
if (post_recv_to_get_ah(ctx)) {
fprintf(stderr, "Couldn't post send \n");
return 1;
}
}
}
if (rdma_accept(ctx->cm_id, &conn_param)) {
fprintf(stderr, "Function rdma_accept failed\n");
return 1;
}
if (user_param->work_rdma_cm && user_param->connection_type == UD) {
if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) {
if (create_ah_from_wc_recv(ctx,user_param)) {
fprintf(stderr, "Unable to create AH from WC\n");
return 1;
}
}
}
rdma_ack_cm_event(event);
rdma_destroy_id(ctx->cm_id_control);
freeaddrinfo(res);
return 0;
}
/******************************************************************************
*
******************************************************************************/
@ -1081,7 +727,7 @@ int create_comm_struct(struct perftest_comm *comm,
comm->rdma_params->sockfd = -1;
comm->rdma_params->gid_index = user_param->gid_index;
comm->rdma_params->gid_index2 = user_param->gid_index2;
comm->rdma_params->use_rdma_cm = user_param->use_rdma_cm;
comm->rdma_params->use_rdma_cm = OFF;
comm->rdma_params->servername = user_param->servername;
comm->rdma_params->machine = user_param->machine;
comm->rdma_params->side = LOCAL;
@ -1097,30 +743,6 @@ int create_comm_struct(struct perftest_comm *comm,
comm->rdma_params->mr_per_qp = user_param->mr_per_qp;
comm->rdma_params->dlid = user_param->dlid;
if (user_param->use_rdma_cm) {
ALLOCATE(comm->rdma_ctx, struct pingpong_context, 1);
memset(comm->rdma_ctx, 0, sizeof(struct pingpong_context));
comm->rdma_params->tx_depth = 1;
comm->rdma_params->rx_depth = 1;
comm->rdma_params->connection_type = RC;
comm->rdma_params->num_of_qps = 1;
comm->rdma_params->verb = SEND;
comm->rdma_params->size = sizeof(struct pingpong_dest);
comm->rdma_ctx->context = NULL;
ALLOCATE(comm->rdma_ctx->mr, struct ibv_mr*, user_param->num_of_qps);
ALLOCATE(comm->rdma_ctx->buf, void* , user_param->num_of_qps);
ALLOCATE(comm->rdma_ctx->qp,struct ibv_qp*,comm->rdma_params->num_of_qps);
comm->rdma_ctx->buff_size = user_param->cycle_buffer;
if (create_rdma_resources(comm->rdma_ctx,comm->rdma_params)) {
fprintf(stderr," Unable to create the resources needed by comm struct\n");
return FAILURE;
}
}
return SUCCESS;
}
@ -1131,26 +753,13 @@ int establish_connection(struct perftest_comm *comm)
{
int (*ptr)(struct perftest_comm*);
if (comm->rdma_params->use_rdma_cm) {
if (comm->rdma_params->machine == CLIENT) {
if (rdma_client_connect(comm->rdma_ctx,comm->rdma_params)) {
fprintf(stderr," Unable to perform rdma_client function\n");
return 1;
}
} else {
if (rdma_server_connect(comm->rdma_ctx,comm->rdma_params)) {
fprintf(stderr," Unable to perform rdma_server function\n");
return 1;
}
}
} else {
ptr = comm->rdma_params->servername ? &ethernet_client_connect : &ethernet_server_connect;
ptr = comm->rdma_params->servername ? &ethernet_client_connect : &ethernet_server_connect;
if ((*ptr)(comm)) {
fprintf(stderr,"Unable to open file descriptor for socket connection");
return 1;
}
if ((*ptr)(comm)) {
fprintf(stderr,"Unable to open file descriptor for socket connection");
return 1;
}
return 0;
}
@ -1164,15 +773,8 @@ int ctx_hand_shake(struct perftest_comm *comm,
int (*read_func_ptr) (struct pingpong_dest*,struct perftest_comm*);
int (*write_func_ptr)(struct pingpong_dest*,struct perftest_comm*);
if (comm->rdma_params->use_rdma_cm || comm->rdma_params->work_rdma_cm) {
read_func_ptr = &rdma_read_keys;
write_func_ptr = &rdma_write_keys;
} else {
read_func_ptr = &ethernet_read_keys;
write_func_ptr = &ethernet_write_keys;
}
read_func_ptr = &ethernet_read_keys;
write_func_ptr = &ethernet_write_keys;
rem_dest->gid_index = my_dest->gid_index;
if (comm->rdma_params->servername) {
@ -1378,11 +980,7 @@ int ctx_xchg_data( struct perftest_comm *comm,
void *my_data,
void *rem_data,int size)
{
if (comm->rdma_params->use_rdma_cm || comm->rdma_params->work_rdma_cm)
ctx_xchg_data_rdma(comm,my_data,rem_data,size);
else
ctx_xchg_data_ethernet(comm,my_data,rem_data,size);
ctx_xchg_data_ethernet(comm,my_data,rem_data,size);
return 0;
}
@ -1533,18 +1131,14 @@ int ctx_close_connection(struct perftest_comm *comm,
return 1;
}
if (!comm->rdma_params->use_rdma_cm && !comm->rdma_params->work_rdma_cm) {
if (write(comm->rdma_params->sockfd,"done",sizeof "done") != sizeof "done") {
perror(" Client write");
fprintf(stderr,"Couldn't write to socket\n");
return -1;
}
close(comm->rdma_params->sockfd);
return 0;
if (write(comm->rdma_params->sockfd,"done",sizeof "done") != sizeof "done") {
perror(" Client write");
fprintf(stderr,"Couldn't write to socket\n");
return -1;
}
close(comm->rdma_params->sockfd);
return 0;
}
@ -1594,12 +1188,6 @@ void check_sys_data(struct perftest_comm *user_comm, struct perftest_parameters
/*take the max and update user_param*/
user_param->cycle_buffer = (rem_cycle_buffer > user_param->cycle_buffer) ? rem_cycle_buffer : user_param->cycle_buffer;
user_param->cache_line_size = (rem_cache_line_size > user_param->cache_line_size) ? rem_cache_line_size : user_param->cache_line_size;
/*update user_comm as well*/
if (user_param->use_rdma_cm) {
user_comm->rdma_ctx->buff_size = user_param->cycle_buffer;
}
}
/******************************************************************************

33
usr/benchmarks/ib/perftest_communication.h
View file

@ -41,11 +41,9 @@
#include <netinet/in.h>
// #include <infiniband/verbs.h>
#include <hermit/ibv.h>
#include <hermit/verbs.h>
// #include <rdma/rdma_cma.h>
#include "perftest_resources.h"
/* Macro for 64 bit variables to switch to/from net */
@ -126,7 +124,7 @@ double bswap_double(double x);
/* create_comm_struct
*
* Description : Creating the communication struct for Etherent or rdma_cm options.
* Description : Creating the communication struct for Etherent.
*
* Parameters :
* comm - An empty Communication struct.
@ -158,9 +156,9 @@ int set_up_connection(struct pingpong_context *ctx,
*
* Description :
*
* Connect the client the a well known server to a requested port.
* Connect the client to a well known server to a requested port.
* It assumes the Server is waiting for request on the port.
* It uses Ethernet sockets or rdma_cm as mentioned in use_rdma_cm.
* It uses Ethernet sockets.
*
* Parameters :
* comm - The communication struct with all the data.
@ -169,19 +167,6 @@ int set_up_connection(struct pingpong_context *ctx,
*/
int establish_connection(struct perftest_comm *comm);
/* rdma_client_connect .
*
* Description : Connects the client to a QP on the other machine with rdma_cm.
*
* Parameters :
* ctx - An empty resources struct to fill the resources created for this QP.
* user_param - Perftest parameters.
*
* Return Value : SUCCESS,FAILURE.
*/
int rdma_client_connect(struct pingpong_context *ctx,
struct perftest_parameters *user_param);
/* retry_rdma_connect .
*
* Description : Retries rdma_client_connect() because the listener may not be ready
@ -196,18 +181,6 @@ int rdma_client_connect(struct pingpong_context *ctx,
int retry_rdma_connect(struct pingpong_context *ctx,
struct perftest_parameters *user_param);
/* rdma_server_connect .
*
* Description : Assinging a server to listen on a rdma_cm port and connect to it.
*
* Parameters :
* ctx - An empty resources struct to fill the resources created for this QP.
* user_param - Perftest parameters.
*
* Return Value : SUCCESS,FAILURE.
*/
int rdma_server_connect(struct pingpong_context *ctx,
struct perftest_parameters *user_param);
/* ctx_hand_shake .
*
* Description :

111
usr/benchmarks/ib/perftest_parameters.c
View file

@ -3,23 +3,20 @@
#include <string.h>
#include <getopt.h>
#include <limits.h>
/* #include <arpa/inet.h> */
#include <netinet/in.h>
#if defined(__FreeBSD__)
#include <netinet/in.h>
#include <sys/socket.h>
#endif
/* #include <netinet/in.h> */
#include <math.h>
#include "perftest_parameters.h"
#include "raw_ethernet_resources.h"
#include<math.h>
#define MAC_LEN (17)
#define ETHERTYPE_LEN (6)
#define MAC_ARR_LEN (6)
#define HEX_BASE (16)
static const char *connStr[] = {"RC","UC","UD","RawEth","XRC","DC"};
static const char *testsStr[] = {"Send","RDMA_Write","RDMA_Read","Atomic"};
static const char *portStates[] = {"Nop","Down","Init","Armed","","Active Defer"};
static const char *qp_state[] = {"OFF","ON"};
static const char *connStr[] = {"RC","UC","UD","RawEth","XRC","DC"};
static const char *testsStr[] = {"Send","RDMA_Write","RDMA_Read","Atomic"};
static const char *portStates[] = {"Nop","Down","Init","Armed","","Active Defer"};
static const char *qp_state[] = {"OFF","ON"};
static const char *exchange_state[] = {"Ethernet","rdma_cm"};
static const char *atomicTypesStr[] = {"CMP_AND_SWAP","FETCH_AND_ADD"};
@ -34,19 +31,6 @@ static const char *atomicTypesStr[] = {"CMP_AND_SWAP","FETCH_AND_ADD"};
*
* Return Value : SUCCESS, FAILURE.
******************************************************************************/
#if defined(__FreeBSD__)
#define strdupa(_s) \
({ \
char *_d; \
int _len; \
\
_len = strlen(_s) + 1; \
_d = alloca(_len); \
if (_d) \
memcpy(_d, _s, _len); \
_d; \
})
#endif
static int parse_mac_from_str(char *mac, u_int8_t *addr)
{
@ -86,6 +70,7 @@ static int parse_mac_from_str(char *mac, u_int8_t *addr)
}
return SUCCESS;
}
static int parse_ethertype_from_str(char *ether_str, uint16_t *ethertype_val)
{
if (strlen(ether_str) != ETHERTYPE_LEN) {
@ -130,13 +115,13 @@ int check_if_valid_udp_port(int udp_port)
{
return ON;
}
/******************************************************************************
get cache line size from system
******************************************************************************/
static int get_cache_line_size()
{
int size = 0;
#if !defined(__FreeBSD__)
size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
if (size == 0) {
#if defined(__sparc__) && defined(__arch64__)
@ -158,7 +143,6 @@ static int get_cache_line_size()
fclose(fp);
}
}
#endif
if (size <= 0)
size = DEF_CACHE_LINE_SIZE;
@ -672,7 +656,7 @@ static void init_perftest_params(struct perftest_parameters *user_param)
user_param->reply_every = 1;
user_param->vlan_en = OFF;
user_param->vlan_pcp = 1;
user_param->print_eth_func = &print_ethernet_header;
/* user_param->print_eth_func = &print_ethernet_header; */
if (user_param->tst == LAT) {
user_param->r_flag->unsorted = OFF;
@ -981,11 +965,6 @@ static void force_dependecies(struct perftest_parameters *user_param)
exit(1);
}
if (user_param->use_rdma_cm == ON || user_param->work_rdma_cm == ON) {
fprintf(stderr," RDMA CM isn't supported for Raw Ethernet tests\n");
exit(1);
}
if (user_param->use_gid_user) {
fprintf(stderr," GID index isn't supported for Raw Ethernet tests\n");
exit(1);
@ -1063,34 +1042,7 @@ static void force_dependecies(struct perftest_parameters *user_param)
exit(1);
}
if (user_param->work_rdma_cm) {
if (user_param->connection_type == UC) {
printf(RESULT_LINE);
printf(" UC is not supported in librdmacm\n");
exit(1);
}
if (user_param->use_mcg) {
printf(RESULT_LINE);
printf(" Perftest still doesn't support Multicast with rdma_cm\n");
exit(1);
}
if (user_param->dualport) {
printf(RESULT_LINE);
printf(" Perftest still doesn't support Dual Port with rdma_cm\n");
exit(1);
}
if (user_param->num_of_qps > 1) {
printf(RESULT_LINE);
fprintf(stdout," Perftest only supports 1 rmda_cm QP for now\n");
exit(1);
}
user_param->use_rdma_cm = ON;
} else if (user_param->tos != DEF_TOS && user_param->connection_type != RawEth) {
if (user_param->tos != DEF_TOS && user_param->connection_type != RawEth) {
fprintf(stdout," TOS only valid for rdma_cm based QP and RawEth QP \n");
exit(1);
}
@ -1154,23 +1106,10 @@ static void force_dependecies(struct perftest_parameters *user_param)
/* XRC Part */
if (user_param->connection_type == XRC) {
if (user_param->work_rdma_cm == ON) {
printf(RESULT_LINE);
fprintf(stderr," XRC does not support RDMA_CM\n");
exit(1);
}
user_param->use_xrc = ON;
user_param->use_srq = ON;
}
if(user_param->connection_type == DC) {
if (user_param->work_rdma_cm == ON) {
printf(RESULT_LINE);
fprintf(stderr," DC does not support RDMA_CM\n");
exit(1);
}
}
#ifndef HAVE_RSS_EXP
if (user_param->use_rss) {
printf(RESULT_LINE);
@ -1203,10 +1142,6 @@ static void force_dependecies(struct perftest_parameters *user_param)
exit(1);
}
} else if (user_param->rate_limit_type == HW_RATE_LIMIT) {
if (user_param->use_rdma_cm == ON || user_param->work_rdma_cm == ON) {
fprintf(stderr," HW rate limit isn't supported yet with rdma_cm scenarios\n");
exit(1);
}
double rate_limit_gbps = 0;
switch (user_param->rate_units) {
case MEGA_BYTE_PS:
@ -2506,7 +2441,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
user_param->use_ooo = 1;
if(vlan_en) {
user_param->vlan_en = ON;
user_param->print_eth_func = &print_ethernet_vlan_header;
/* user_param->print_eth_func = &print_ethernet_vlan_header; */
vlan_en = 0;
}
if (optind == argc - 1) {
@ -2746,25 +2681,7 @@ void ctx_print_test_info(struct perftest_parameters *user_param)
printf(" Outstand reads : %d\n",user_param->out_reads);
printf(" rdma_cm QPs : %s\n",qp_state[user_param->work_rdma_cm]);
if (user_param->use_rdma_cm)
temp = 1;
printf(" Data ex. method : %s",exchange_state[temp]);
if (user_param->work_rdma_cm) {
if (user_param->tos != DEF_TOS) {
printf(" \tTOS : %d",user_param->tos);
}
if (user_param->machine == SERVER) {
putchar('\n');
printf(RESULT_LINE);
printf(" Waiting for client rdma_cm QP to connect\n");
printf(" Please run the same command with the IB/RoCE interface IP");
}
}
putchar('\n');
printf(RESULT_LINE);

243
usr/benchmarks/ib/perftest_parameters.h
View file

@ -54,13 +54,16 @@
#ifndef PERFTEST_PARAMETERS_H
#define PERFTEST_PARAMETERS_H
// #include <infiniband/verbs.h>
#include <hermit/ibv.h>
#include <hermit/verbs.h>
#include <unistd.h>
#include <malloc.h>
#include <stddef.h>
#include <netinet/in.h>
#include <sys/socket.h>
// #include <endian.h>
#include "get_clock.h"
@ -68,13 +71,54 @@
// #include <config.h>
// #endif
/* ------------------------------------------------------------------- */
/* Config TODO */
// #define HAVE_VERBS_EXP
// #define HAVE_RAW_ETH_EXP
// #define HAVE_RAW_ETH_REG
// #define HAVE_XRCD
#define HAVE_ENDIAN
// #define HAVE_SCIF
// #define HAVE_MASKED_ATOMICS
// #define HAVE_RSS_EXP
// #define HAVE_DC
// #define HAVE_ACCL_VERBS
#define HAVE_IPV6
#define HAVE_IPV4_EXT
#define HAVE_SNIFFER
// #define HAVE_SNIFFER_EXP
#define HAVE_EX
#define HAVE_EX_ODP
// #define HAVE_EXP_ODP
// #define HAVE_CUDA
// #define HAVE_SCATTER_FCS
// #define HAVE_GID_ATTR
// #define HAVE_PACKET_PACING_EXP
#define HAVE_PACKET_PACING
// #define HAVE_OOO_ATTR
// #define HAVE_EXP_OOO_ATTR
#define VERSION "1"
/* -------------------------------------------------------------------
/* Connection types available. */
#define RC (0)
// #define UC (1)
#define UC (1)
#define UD (2)
// #define RawEth (3)
// #define XRC (4)
// #define DC (5)
#define RawEth (3)
#define XRC (4)
#define DC (5)
/* Genral control definitions */
#define OFF (0)
@ -158,12 +202,12 @@
#define MAX_EQ_NUM (2048)
/* Raw etherent defines */
// #define RAWETH_MIN_MSG_SIZE (64)
// #define MIN_MTU_RAW_ETERNET (64)
// #define MAX_MTU_RAW_ETERNET (9600)
// #define MIN_FS_PORT (5000)
// #define MAX_FS_PORT (65536)
// #define VLAN_PCP_VARIOUS (8)
#define RAWETH_MIN_MSG_SIZE (64)
#define MIN_MTU_RAW_ETERNET (64)
#define MAX_MTU_RAW_ETERNET (9600)
#define MIN_FS_PORT (5000)
#define MAX_FS_PORT (65536)
#define VLAN_PCP_VARIOUS (8)
#define RESULT_LINE "---------------------------------------------------------------------------------------\n"
@ -324,41 +368,43 @@ struct ETH_vlan_header {
}__attribute__((packed));*/
struct perftest_parameters {
int port;
char *ib_devname;
char *servername;
uint8_t ib_port;
uint8_t ib_port2;
int mtu;
enum ibv_mtu curr_mtu;
uint64_t size;
uint64_t dct_key;
char *ib_devname;
char *servername;
uint8_t ib_port;
uint8_t ib_port2;
int mtu; // minimum translation unit
enum ibv_mtu curr_mtu;
uint64_t size; // BW: 65536, LAT: 2
uint64_t dct_key;
int iters;
uint64_t iters_per_port[2];
uint64_t *port_by_qp;
int tx_depth;
uint8_t qp_timeout;
uint8_t sl;
uint64_t iters_per_port[2];
uint64_t *port_by_qp;
int tx_depth; // <= iters
uint8_t qp_timeout;
uint8_t sl; // service lvl
int gid_index;
int gid_index2;
int use_gid_user;
uint8_t source_mac[6];
uint8_t dest_mac[6];
uint8_t source_mac[6];
uint8_t dest_mac[6];
int is_source_mac;
int is_dest_mac;
uint8_t server_ip6[16];
uint8_t client_ip6[16];
uint8_t local_ip6[16];
uint8_t remote_ip6[16];
uint8_t local_mac[6];
uint8_t remote_mac[6];
uint32_t client_ip;
uint32_t server_ip;
uint8_t server_ip6[16];
uint8_t client_ip6[16];
uint8_t local_ip6[16];
uint8_t remote_ip6[16];
uint8_t local_mac[6];
uint8_t remote_mac[6];
uint32_t client_ip;
uint32_t server_ip;
int is_server_ip;
int is_client_ip;
uint32_t local_ip;
uint32_t remote_ip;
uint32_t local_ip;
uint32_t remote_ip;
int server_port;
int client_port;
int tcp;
@ -366,9 +412,9 @@ struct perftest_parameters {
int is_client_port;
int local_port;
int remote_port;
int is_old_raw_eth_param;
int is_new_raw_eth_param;
uint16_t ethertype;
int is_old_raw_eth_param;
int is_new_raw_eth_param;
uint16_t ethertype;
int is_ethertype;
int cpu_freq_f;
int connection_type;
@ -376,86 +422,86 @@ struct perftest_parameters {
int use_event;
int eq_num;
int use_eq_num;
int inline_size;
int inline_size;
int inline_recv_size;
int out_reads;
int rx_depth;
int duplex;
int noPeak;
int cq_mod;
int spec;
int dualport;
int post_list;
int spec;
int dualport;
int post_list;
int duration;
int use_srq;
int use_srq;
int use_xrc;
int use_rss;
int srq_exists;
int tos;
int margin;
int is_bw_limit_passed;
int is_msgrate_limit_passed;
int is_limit_bw;
int is_limit_msgrate;
float limit_bw;
float limit_msgrate;
uint32_t rem_ud_qpn;
uint32_t rem_ud_qkey;
int8_t link_type;
int8_t link_type2;
MachineType machine;
PrintDataSide side;
VerbType verb;
TestType tst;
AtomicType atomicType;
TestMethod test_type;
DurationStates state;
int sockfd;
char version[MAX_VERSION];
char rem_version[MAX_VERSION];
cycles_t *tposted;
cycles_t *tcompleted;
int is_bw_limit_passed;
int is_msgrate_limit_passed;
int is_limit_bw;
int is_limit_msgrate;
float limit_bw;
float limit_msgrate;
uint32_t rem_ud_qpn;
uint32_t rem_ud_qkey;
int8_t link_type;
int8_t link_type2;
MachineType machine;
PrintDataSide side;
VerbType verb;
TestType tst;
AtomicType atomicType;
TestMethod test_type;
DurationStates state;
int sockfd;
char version[MAX_VERSION];
char rem_version[MAX_VERSION];
cycles_t *tposted;
cycles_t *tcompleted;
int use_mcg;
int use_rdma_cm;
int use_rdma_cm;
int is_reversed;
int work_rdma_cm;
char *user_mgid;
int buff_size;
int pkey_index;
int pkey_index;
int raw_qos;
int use_cuda;
char *mmap_file;
unsigned long mmap_offset;
/* New test params format pilot. will be used in all flags soon,. */
enum ctx_test_method test_method;
enum ibv_transport_type transport_type;
enum ctx_report_fmt report_fmt;
struct report_options *r_flag ;
int mac_fwd;
int report_both; /* in bidirectional tests, report tx and rx separately */
enum ctx_test_method test_method;
enum ibv_transport_type transport_type;
enum ctx_report_fmt report_fmt;
struct report_options *r_flag ;
int mac_fwd;
int report_both; /* in bidirectional tests, report tx and rx separately */
/* results limits */
float min_bw_limit;
float min_msgRate_limit;
float min_bw_limit;
float min_msgRate_limit;
/* Rate Limiter */
char *rate_limit_str;
double rate_limit;
double rate_limit;
int valid_hw_rate_limit;
int burst_size;
enum rate_limiter_units rate_units;
enum rate_limiter_types rate_limit_type;
int burst_size;
enum rate_limiter_units rate_units;
enum rate_limiter_types rate_limit_type;
int is_rate_limit_type;
enum verbosity_level output;
int cpu_util;
struct cpu_util_data cpu_util_data;
int latency_gap;
int flow_label;
int retry_count;
int dont_xchg_versions;
int use_exp;
int ipv6;
int raw_ipv6;
int report_per_port;
int use_odp;
enum verbosity_level output;
int cpu_util;
struct cpu_util_data cpu_util_data;
int latency_gap;
int flow_label;
int retry_count;
int dont_xchg_versions;
int use_exp;
int ipv6;
int raw_ipv6;
int report_per_port;
int use_odp;
int use_hugepages;
int use_promiscuous;
int use_sniffer;
@ -464,7 +510,7 @@ struct perftest_parameters {
int masked_atomics;
int cycle_buffer;
int cache_line_size;
enum verbs_intf verb_type;
enum verbs_intf verb_type;
int is_exp_cq;
int is_exp_qp;
int use_res_domain;
@ -478,10 +524,9 @@ struct perftest_parameters {
uint32_t reply_every;
int perform_warm_up;
int use_ooo;
int vlan_en;
uint32_t vlan_pcp;
void (*print_eth_func)(void*);
int vlan_en;
uint32_t vlan_pcp;
// void (*print_eth_func)(void*);
};
struct report_options {

280
usr/benchmarks/ib/perftest_resources.c
View file

@ -2,26 +2,22 @@
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#if !defined(__FreeBSD__)
#include <malloc.h>
#endif
#include <getopt.h>
#include <limits.h>
#include <errno.h>
#include <signal.h>
#include <string.h>
#include <ctype.h>
#include <sys/mman.h>
/* #include <sys/mman.h> */
#include <sys/ipc.h>
#include <sys/shm.h>
#include <pthread.h>
#if defined(__FreeBSD__)
#include <sys/stat.h>
#endif
#include "perftest_resources.h"
#include "raw_ethernet_resources.h"
#include "config.h"
#include "perftest_parameters.h"
/* #include "config.h" */
#ifdef HAVE_VERBS_EXP
static enum ibv_exp_wr_opcode exp_opcode_verbs_array[] = {IBV_EXP_WR_SEND,IBV_EXP_WR_RDMA_WRITE,IBV_EXP_WR_RDMA_READ};
@ -137,34 +133,33 @@ static int pp_free_gpu(struct pingpong_context *ctx)
}
#endif
static int pp_init_mmap(struct pingpong_context *ctx, size_t size,
const char *fname, unsigned long offset)
{
int fd = open(fname, O_RDWR);
if (fd < 0) {
printf("Unable to open '%s': %s\n", fname, strerror(errno));
return 1;
}
/* static int pp_init_mmap(struct pingpong_context *ctx, size_t size, */
/* const char *fname, unsigned long offset) */
/* { */
/* int fd = open(fname, O_RDWR); */
/* if (fd < 0) { */
/* printf("Unable to open '%s': %s\n", fname, strerror(errno)); */
/* return 1; */
/* } */
ctx->buf[0] = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd,
offset);
close(fd);
/* ctx->buf[0] = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd, offset); */
/* close(fd); */
if (ctx->buf[0] == MAP_FAILED) {
printf("Unable to mmap '%s': %s\n", fname, strerror(errno));
return 1;
}
/* if (ctx->buf[0] == MAP_FAILED) { */
/* printf("Unable to mmap '%s': %s\n", fname, strerror(errno)); */
/* return 1; */
/* } */
printf("allocated mmap buffer of size %zd at %p\n", size, ctx->buf[0]);
/* printf("allocated mmap buffer of size %zd at %p\n", size, ctx->buf[0]); */
return 0;
}
/* return 0; */
/* } */
static int pp_free_mmap(struct pingpong_context *ctx)
{
munmap(ctx->buf[0], ctx->buff_size);
return 0;
}
/* static int pp_free_mmap(struct pingpong_context *ctx) */
/* { */
/* munmap(ctx->buf[0], ctx->buff_size); */
/* return 0; */
/* } */
#ifdef HAVE_VERBS_EXP
static void get_verbs_pointers(struct pingpong_context *ctx)
@ -530,67 +525,27 @@ static struct ibv_qp *ctx_rss_eth_qp_create(struct pingpong_context *ctx,struct
/******************************************************************************
*
******************************************************************************/
int check_add_port(char **service,int port,
const char *servername,
struct addrinfo *hints,
struct addrinfo **res)
{
int number;
if (asprintf(service,"%d", port) < 0) {
return FAILURE;
}
number = getaddrinfo(servername,*service,hints,res);
if (number < 0) {
fprintf(stderr, "%s for %s:%d\n", gai_strerror(number), servername, port);
return FAILURE;
}
return SUCCESS;
}
/******************************************************************************
*
******************************************************************************/
/* int create_rdma_resources(struct pingpong_context *ctx, */
/* struct perftest_parameters *user_param) */
/* int check_add_port(char **service,int port, */
/* const char *servername, */
/* struct addrinfo *hints, */
/* struct addrinfo **res) */
/* { */
/* int is_udp_ps = user_param->connection_type == UD || user_param->connection_type == RawEth; */
/* enum rdma_port_space port_space = (is_udp_ps) ? RDMA_PS_UDP : RDMA_PS_TCP; */
/* struct rdma_cm_id **cm_id = (user_param->machine == CLIENT) ? &ctx->cm_id : &ctx->cm_id_control; */
/* int number; */
/* ctx->cm_channel = rdma_create_event_channel(); */
/* if (ctx->cm_channel == NULL) { */
/* fprintf(stderr, " rdma_create_event_channel failed\n"); */
/* if (asprintf(service,"%d", port) < 0) { */
/* return FAILURE; */
/* } */
/* if (rdma_create_id(ctx->cm_channel,cm_id,NULL,port_space)) { */
/* fprintf(stderr,"rdma_create_id failed\n"); */
/* number = getaddrinfo(servername,*service,hints,res); */
/* if (number < 0) { */
/* fprintf(stderr, "%s for %s:%d\n", gai_strerror(number), servername, port); */
/* return FAILURE; */
/* } */
/* return SUCCESS; */
/* } */
/******************************************************************************
*
******************************************************************************/
/* int destroy_rdma_resources(struct pingpong_context *ctx, */
/* struct perftest_parameters *user_param) */
/* { */
/* int ret; */
/* if (user_param->machine == CLIENT) { */
/* ret = rdma_destroy_id(ctx->cm_id); */
/* } else { */
/* ret = rdma_destroy_id(ctx->cm_id_control); */
/* } */
/* rdma_destroy_event_channel(ctx->cm_channel); */
/* return ret; */
/* } */
/******************************************************************************
+ *
+ ******************************************************************************/
@ -737,9 +692,6 @@ int destroy_ctx(struct pingpong_context *ctx,
dereg_counter = (user_param->mr_per_qp) ? user_param->num_of_qps : 1;
if (user_param->work_rdma_cm == ON)
rdma_disconnect(ctx->cm_id);
/* in dc with bidirectional,
* there are send qps and recv qps. the actual number of send/recv qps
* is num_of_qps / 2.
@ -837,16 +789,6 @@ int destroy_ctx(struct pingpong_context *ctx,
}
}
if (user_param->verb == SEND && user_param->work_rdma_cm == ON && ctx->send_rcredit) {
if (ibv_dereg_mr(ctx->credit_mr)) {
fprintf(stderr, "Failed to deregister send credit MR\n");
test_result = 1;
}
free(ctx->ctrl_buf);
free(ctx->ctrl_sge_list);
free(ctx->ctrl_wr);
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "Failed to deallocate PD - %s\n", strerror(errno));
test_result = 1;
@ -858,12 +800,9 @@ int destroy_ctx(struct pingpong_context *ctx,
test_result = 1;
}
}
if (user_param->use_rdma_cm == OFF) {
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "Failed to close device context\n");
test_result = 1;
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "Failed to close device context\n");
test_result = 1;
}
#ifdef HAVE_CUDA
@ -872,9 +811,10 @@ int destroy_ctx(struct pingpong_context *ctx,
}
else
#endif
if (user_param->mmap_file != NULL) {
pp_free_mmap(ctx);
} else if (ctx->is_contig_supported == FAILURE) {
/* if (user_param->mmap_file != NULL) { */
/* pp_free_mmap(ctx); */
/* } else if (ctx->is_contig_supported == FAILURE) { */
if (ctx->is_contig_supported == FAILURE) {
for (i = 0; i < dereg_counter; i++) {
if (user_param->use_hugepages) {
shmdt(ctx->buf[i]);
@ -1174,50 +1114,42 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
}
#endif
if (user_param->mmap_file != NULL) {
#if defined(__FreeBSD__)
posix_memalign(ctx->buf, user_param->cycle_buffer, ctx->buff_size);
#else
ctx->buf = memalign(user_param->cycle_buffer, ctx->buff_size);
#endif
if (pp_init_mmap(ctx, ctx->buff_size, user_param->mmap_file,
user_param->mmap_offset))
{
/* if (user_param->mmap_file != NULL) { */
/* ctx->buf = memalign(user_param->cycle_buffer, ctx->buff_size); */
/* if (pp_init_mmap(ctx, ctx->buff_size, user_param->mmap_file, */
/* user_param->mmap_offset)) */
/* { */
/* fprintf(stderr, "Couldn't allocate work buf.\n"); */
/* return FAILURE; */
/* } */
/* } else { */
/* Allocating buffer for data, in case driver not support contig pages. */
if (ctx->is_contig_supported == FAILURE) {
if (user_param->use_hugepages) {
if (alloc_hugepage_region(ctx) != SUCCESS){
fprintf(stderr, "Failed to allocate hugepage region.\n");
return FAILURE;
}
memset(ctx->buf[qp_index], 0, ctx->buff_size);
} else if (ctx->is_contig_supported == FAILURE) {
ctx->buf[qp_index] = memalign(user_param->cycle_buffer, ctx->buff_size);
}
if (!ctx->buf[qp_index]) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return FAILURE;
}
memset(ctx->buf[qp_index], 0, ctx->buff_size);
} else {
/* Allocating buffer for data, in case driver not support contig pages. */
if (ctx->is_contig_supported == FAILURE) {
#if defined(__FreeBSD__)
posix_memalign(ctx->buf[qp_index], user_param->cycle_buffer, ctx->buff_size);
#else
if (user_param->use_hugepages) {
if (alloc_hugepage_region(ctx) != SUCCESS){
fprintf(stderr, "Failed to allocate hugepage region.\n");
return FAILURE;
}
memset(ctx->buf[qp_index], 0, ctx->buff_size);
} else if (ctx->is_contig_supported == FAILURE) {
ctx->buf[qp_index] = memalign(user_param->cycle_buffer, ctx->buff_size);
}
#endif
if (!ctx->buf[qp_index]) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return FAILURE;
}
memset(ctx->buf[qp_index], 0, ctx->buff_size);
} else {
ctx->buf[qp_index] = NULL;
#ifdef HAVE_VERBS_EXP
exp_flags |= IBV_EXP_ACCESS_ALLOCATE_MR;
#else
flags |= (1 << 5);
#endif
}
ctx->buf[qp_index] = NULL;
#ifdef HAVE_VERBS_EXP
exp_flags |= IBV_EXP_ACCESS_ALLOCATE_MR;
#else
flags |= (1 << 5);
#endif
}
/* } */
if (user_param->verb == WRITE) {
flags |= IBV_ACCESS_REMOTE_WRITE;
@ -1319,6 +1251,7 @@ int create_mr(struct pingpong_context *ctx, struct perftest_parameters *user_par
#define HUGEPAGE_ALIGN (2*1024*1024)
#define SHMAT_ADDR (void *)(0x0UL)
#define SHMAT_FLAGS (0)
#define SHM_HUGETLB (2048) // !
int alloc_hugepage_region (struct pingpong_context *ctx)
{
@ -1328,7 +1261,7 @@ int alloc_hugepage_region (struct pingpong_context *ctx)
/* create hugepage shared region */
ctx->huge_shmid = shmget(IPC_PRIVATE, buf_size,
SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W);
SHM_HUGETLB | IPC_CREAT /* | IPC_R | IPC_W */); // !
if (ctx->huge_shmid < 0) {
fprintf(stderr, "Failed to allocate hugepages. Please configure hugepages\n");
return FAILURE;
@ -1561,23 +1494,20 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
fprintf(stderr, "Failed to create QP.\n");
return FAILURE;
}
if (user_param->work_rdma_cm == OFF) {
modify_qp_to_init(ctx, user_param, i, num_of_qps);
#ifdef HAVE_ACCL_VERBS
if (user_param->verb_type == ACCL_INTF) {
memset(&intf_params, 0, sizeof(intf_params));
intf_params.intf_scope = IBV_EXP_INTF_GLOBAL;
intf_params.intf = IBV_EXP_INTF_QP_BURST;
intf_params.obj = ctx->qp[i];
ctx->qp_burst_family[i] = ibv_exp_query_intf(ctx->context, &intf_params, &intf_status);
if (!ctx->qp_burst_family[i]) {
fprintf(stderr, "Couldn't get QP burst family.\n");
return FAILURE;
}
modify_qp_to_init(ctx, user_param, i, num_of_qps);
#ifdef HAVE_ACCL_VERBS
if (user_param->verb_type == ACCL_INTF) {
memset(&intf_params, 0, sizeof(intf_params));
intf_params.intf_scope = IBV_EXP_INTF_GLOBAL;
intf_params.intf = IBV_EXP_INTF_QP_BURST;
intf_params.obj = ctx->qp[i];
ctx->qp_burst_family[i] = ibv_exp_query_intf(ctx->context, &intf_params, &intf_status);
if (!ctx->qp_burst_family[i]) {
fprintf(stderr, "Couldn't get QP burst family.\n");
return FAILURE;
}
#endif
}
#endif
}
return SUCCESS;
@ -1871,16 +1801,6 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
return NULL;
}
/* if (user_param->work_rdma_cm) { */
/* if (rdma_create_qp(ctx->cm_id,ctx->pd,&attr)) { */
/* fprintf(stderr, " Couldn't create rdma QP - %s\n",strerror(errno)); */
/* return NULL; */
/* } */
/* qp = ctx->cm_id->qp; */
/* } else { */
/* qp = ibv_create_qp(ctx->pd,&attr); */
/* } */
qp = ibv_create_qp(ctx->pd,&attr);
return qp;
}
@ -2333,7 +2253,7 @@ static int ctx_modify_qp_to_rts(struct ibv_qp *qp,
flags |= IBV_EXP_QP_RATE_LIMIT;
return ibv_exp_modify_qp(qp, (struct ibv_exp_qp_attr*)_attr, flags);
}
#elif HAVE_PACKET_PACING
#elif defined(HAVE_PACKET_PACING)
if (user_param->rate_limit_type == PP_RATE_LIMIT) {
attr->rate_limit = user_param->rate_limit;
flags |= IBV_QP_RATE_LIMIT;
@ -2628,15 +2548,8 @@ void ctx_set_send_exp_wqes(struct pingpong_context *ctx,
if (user_param->connection_type == UD) {
ctx->exp_wr[i*user_param->post_list + j].wr.ud.ah = ctx->ah[i];
if (user_param->work_rdma_cm) {
ctx->exp_wr[i*user_param->post_list + j].wr.ud.remote_qkey = user_param->rem_ud_qkey;
ctx->exp_wr[i*user_param->post_list + j].wr.ud.remote_qpn = user_param->rem_ud_qpn;
} else {
ctx->exp_wr[i*user_param->post_list + j].wr.ud.remote_qkey = DEF_QKEY;
ctx->exp_wr[i*user_param->post_list + j].wr.ud.remote_qpn = rem_dest[xrc_offset + i].qpn;
}
ctx->exp_wr[i*user_param->post_list + j].wr.ud.remote_qkey = DEF_QKEY;
ctx->exp_wr[i*user_param->post_list + j].wr.ud.remote_qpn = rem_dest[xrc_offset + i].qpn;
#ifdef HAVE_DC
} else if (user_param->connection_type == DC) {
@ -2787,15 +2700,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
if (user_param->connection_type == UD) {
ctx->wr[i*user_param->post_list + j].wr.ud.ah = ctx->ah[i];
if (user_param->work_rdma_cm) {
ctx->wr[i*user_param->post_list + j].wr.ud.remote_qkey = user_param->rem_ud_qkey;
ctx->wr[i*user_param->post_list + j].wr.ud.remote_qpn = user_param->rem_ud_qpn;
} else {
ctx->wr[i*user_param->post_list + j].wr.ud.remote_qkey = DEF_QKEY;
ctx->wr[i*user_param->post_list + j].wr.ud.remote_qpn = rem_dest[xrc_offset + i].qpn;
}
ctx->wr[i*user_param->post_list + j].wr.ud.remote_qkey = DEF_QKEY;
ctx->wr[i*user_param->post_list + j].wr.ud.remote_qpn = rem_dest[xrc_offset + i].qpn;
}
}
@ -4992,7 +4898,7 @@ int check_packet_pacing_support(struct pingpong_context *ctx)
return MASK_IS_SET(IBV_EXP_DEVICE_ATTR_PACKET_PACING_CAPS, attr.comp_mask) ?
SUCCESS : FAILURE;
}
#elif HAVE_PACKET_PACING
#elif defined(HAVE_PACKET_PACING)
int check_packet_pacing_support(struct pingpong_context *ctx)
{
struct ibv_device_attr_ex attr;

74
usr/benchmarks/ib/perftest_resources.h
View file

@ -51,20 +51,17 @@
#ifndef PERFTEST_RESOURCES_H
#define PERFTEST_RESOURCES_H
// #include <infiniband/verbs.h>
#include <hermit/ibv.h>
#include <hermit/verbs.h>
// #include <rdma/rdma_cma.h>
#include <stdint.h>
#include <unistd.h>
// #include <byteswap.h>
#include <math.h>
// #include <arpa/inet.h>
#include <netinet/in.h>
// #include <netinet/in.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>
// #include <sys/socket.h>
#include <netdb.h>
#include <fcntl.h> // ?
@ -128,6 +125,7 @@
/******************************************************************************
* Perftest resources Structures and data types.
******************************************************************************/
struct pingpong_context {
// struct rdma_event_channel *cm_channel;
// struct rdma_cm_id *cm_id_control;
@ -177,20 +175,26 @@ struct pingpong_context {
struct ibv_exp_cq_family *recv_cq_family;
struct ibv_exp_qp_burst_family **qp_burst_family;
#endif
};
struct pingpong_dest {
int lid;
int out_reads;
int qpn;
int psn;
unsigned rkey;
unsigned long long vaddr;
union ibv_gid gid;
unsigned srqn;
int gid_index;
};
struct pingpong_dest {
int lid;
int out_reads;
int qpn;
int psn;
unsigned rkey;
unsigned long long vaddr;
union ibv_gid gid;
unsigned srqn;
int gid_index;
};
struct raw_ethernet_info { // !
uint8_t mac[6];
uint32_t ip;
uint8_t ip6[16];
int port;
};
/******************************************************************************
* Perftest resources Methods and interface utilitizes.
@ -208,10 +212,10 @@ struct pingpong_context {
*
* Return Value : SUCCESS, FAILURE.
*/
int check_add_port(char **service,int port,
const char *servername,
struct addrinfo *hints,
struct addrinfo **res);
// int check_add_port(char **service,int port,
// const char *servername,
// struct addrinfo *hints,
// struct addrinfo **res);
/* ctx_find_dev
*
@ -226,32 +230,6 @@ int check_add_port(char **service,int port,
*/
struct ibv_device* ctx_find_dev(const char *ib_devname);
/* create_rdma_resources
*
* Description : Creates the rdma_cm_id and rdma_channel for the rdma_cm QPs.
*
* Parameters :
* ctx - Resources sructure.
* user_param - the perftest parameters.
*
* Return Value : SUCCESS, FAILURE.
*/
// int create_rdma_resources(struct pingpong_context *ctx,
// struct perftest_parameters *user_param);
/* destroy_rdma_resources
*
* Description : Destroys the rdma_cm_id and rdma_channel for the rdma_cm QPs.
*
* Parameters :
* ctx - Resources sructure.
* user_param - the perftest parameters.
*
* Return Value : SUCCESS, FAILURE.
*/
// int destroy_rdma_resources(struct pingpong_context *ctx,
// struct perftest_parameters *user_param);
/* alloc_ctx
*
* Description : allocate all perftest resources.

401
usr/benchmarks/ib/wr_bw.c Normal file
View file

@ -0,0 +1,401 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2009 HNR Consulting. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* $Id$
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(int argc, char *argv[])
{
int ret_parser,i = 0;
struct ibv_device *ib_dev = NULL;
struct pingpong_context ctx;
struct pingpong_dest *my_dest,*rem_dest;
struct perftest_parameters user_param;
struct perftest_comm user_comm;
struct bw_report_data my_bw_rep, rem_bw_rep;
/* init default values to user's parameters */
memset(&user_param,0,sizeof(struct perftest_parameters));
memset(&user_comm,0,sizeof(struct perftest_comm));
memset(&ctx,0,sizeof(struct pingpong_context));
user_param.verb = WRITE;
user_param.tst = BW;
strncpy(user_param.version, VERSION, sizeof(user_param.version));
/* Configure the parameters values according to user arguments or default values. */
ret_parser = parser(&user_param,argv,argc);
if (ret_parser) {
if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
fprintf(stderr," Parser function exited with Error\n");
return FAILURE;
}
if((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
user_param.num_of_qps *= 2;
}
/* Finding the IB device selected (or default if none is selected). */
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
return FAILURE;
}
/* Getting the relevant context from the device */
ctx.context = ibv_open_device(ib_dev);
if (!ctx.context) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
/* See if MTU and link type are valid and supported. */
if (check_link(ctx.context,&user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return FAILURE;
}
if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
printf("\n************************************\n");
printf("* Waiting for client to connect... *\n");
printf("************************************\n");
}
/* Initialize the connection and print the local data. */
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
return FAILURE;
}
sleep(1);
exchange_versions(&user_comm, &user_param);
check_sys_data(&user_comm, &user_param);
/* See if MTU and link type are valid and supported. */
if (check_mtu(ctx.context,&user_param, &user_comm)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
/* Allocating arrays needed for the test. */
alloc_ctx(&ctx,&user_param);
/* Create (if necessary) the rdma_cm ids and channel. */
if (user_param.work_rdma_cm == ON) {
if (user_param.machine == CLIENT) {
if (retry_rdma_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
if (ctx_init(&ctx, &user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
}
/* Set up the Connection. */
if (set_up_connection(&ctx,&user_param,my_dest)) {
fprintf(stderr," Unable to set up socket connection\n");
return FAILURE;
}
/* Print basic test information. */
ctx_print_test_info(&user_param);
/* Print this machine QP information */
for (i=0; i < user_param.num_of_qps; i++)
ctx_print_pingpong_data(&my_dest[i],&user_comm);
user_comm.rdma_params->side = REMOTE;
for (i=0; i < user_param.num_of_qps; i++) {
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
return FAILURE;
}
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
}
if (user_param.work_rdma_cm == OFF) {
if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
fprintf(stderr," Please Try to use a different IP version.\n\n");
return FAILURE;
}
}
if (user_param.work_rdma_cm == OFF) {
if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
return FAILURE;
}
}
/* An additional handshake is required after moving qp to RTR. */
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
return FAILURE;
}
if (user_param.output == FULL_VERBOSITY) {
if (user_param.report_per_port) {
printf(RESULT_LINE_PER_PORT);
printf((user_param.report_fmt == MBS ? RESULT_FMT_PER_PORT : RESULT_FMT_G_PER_PORT));
}
else {
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
}
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
}
/* For half duplex tests, server just waits for client to exit */
if (user_param.machine == SERVER && !user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
return FAILURE;
}
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
print_full_bw_report(&user_param, &rem_bw_rep, NULL);
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
return FAILURE;
}
if (user_param.output == FULL_VERBOSITY) {
if (user_param.report_per_port)
printf(RESULT_LINE_PER_PORT);
else
printf(RESULT_LINE);
}
if (user_param.work_rdma_cm == ON) {
if (destroy_ctx(&ctx,&user_param)) {
fprintf(stderr, "Failed to destroy resources\n");
return FAILURE;
}
user_comm.rdma_params->work_rdma_cm = ON;
return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
}
return destroy_ctx(&ctx,&user_param);
}
if (user_param.test_method == RUN_ALL) {
for (i = 1; i < 24 ; ++i) {
user_param.size = (uint64_t)1 << i;
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if (user_param.perform_warm_up) {
if(perform_warm_up(&ctx, &user_param)) {
fprintf(stderr, "Problems with warm up\n");
return FAILURE;
}
}
if(user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
return FAILURE;
}
}
if(run_iter_bw(&ctx,&user_param)) {
fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
return FAILURE;
}
if (user_param.duplex && (atof(user_param.version) >= 4.6)) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
return FAILURE;
}
}
print_report_bw(&user_param,&my_bw_rep);
if (user_param.duplex) {
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
}
}
} else if (user_param.test_method == RUN_REGULAR) {
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if (user_param.verb != SEND) {
if (user_param.perform_warm_up) {
if(perform_warm_up(&ctx, &user_param)) {
fprintf(stderr, "Problems with warm up\n");
return FAILURE;
}
}
}
if(user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
return FAILURE;
}
}
if(run_iter_bw(&ctx,&user_param)) {
fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
return FAILURE;
}
print_report_bw(&user_param,&my_bw_rep);
if (user_param.duplex) {
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
}
if (user_param.report_both && user_param.duplex) {
printf(RESULT_LINE);
printf("\n Local results: \n");
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
print_full_bw_report(&user_param, &my_bw_rep, NULL);
printf(RESULT_LINE);
printf("\n Remote results: \n");
printf(RESULT_LINE);
printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
print_full_bw_report(&user_param, &rem_bw_rep, NULL);
}
} else if (user_param.test_method == RUN_INFINITELY) {
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
if(run_iter_bw_infinitely(&ctx,&user_param)) {
fprintf(stderr," Error occurred while running infinitely! aborting ...\n");
return FAILURE;
}
}
if (user_param.output == FULL_VERBOSITY) {
if (user_param.report_per_port)
printf(RESULT_LINE_PER_PORT);
else
printf(RESULT_LINE);
}
/* For half duplex tests, server just waits for client to exit */
if (user_param.machine == CLIENT && !user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
return FAILURE;
}
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
}
/* Closing connection. */
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
return FAILURE;
}
if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
fprintf(stderr,"Error: BW result is below bw limit\n");
return FAILURE;
}
if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
fprintf(stderr,"Error: Msg rate is below msg_rate limit\n");
return FAILURE;
}
free(my_dest);
free(rem_dest);
if (user_param.work_rdma_cm == ON) {
if (destroy_ctx(&ctx,&user_param)) {
fprintf(stderr, "Failed to destroy resources\n");
return FAILURE;
}
user_comm.rdma_params->work_rdma_cm = ON;
return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
}
return destroy_ctx(&ctx,&user_param);
}

799
usr/benchmarks/ib/wr_bw.h Normal file
View file

@ -0,0 +1,799 @@
/*
* Copyright (c) 2009 Mellanox Technologies Ltd. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* Author: Ido Shamay <idos@dev.mellanox.co.il>
*
* Description :
*
* This API gathres the Socket interface methods for all perftest benchmarks
* and can be used for any benchmark for IB.
* It passes messages between 2 end points through sockets interface methods,
* while passing the rellevant information for the IB entities.
*
* Methods :
*
* ctx_get_local_lid - Receives the Local id from the subnet manager.
* ctx_client_connect - Connects the client through sockets interface.
* ctx_server_connect - Connects the Server to client through sockets.
* ctx_hand_shake - Passes the data between 2 end points machines.
* ctx_print_pingpong_data - Prints the data that was passed.
* ctx_close_connection - Closing the sockets interface.
*/
#ifndef WR_BW_H
#define WR_BW_H
#include <hermit/ibv.h>
#include <hermit/verbs.h>
#include <stdint.h>
#include <math.h>
#include <netinet/in.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <fcntl.h> // ?
#define NUM_OF_RETRIES (10)
/* Outstanding reads for "read" verb only. */
#define MAX_SEND_SGE (1)
#define MAX_RECV_SGE (1)
#define CTX_POLL_BATCH (16)
#define PL (1)
#define ATOMIC_ADD_VALUE (1)
#define ATOMIC_SWAP_VALUE (0)
/* Space for GRH when we scatter the packet in UD. */
#define PINGPONG_SEND_WRID (60)
#define PINGPONG_RDMA_WRID (3)
#define PINGPONG_READ_WRID (1)
#define PINGPONG_ATOMIC_WRID (22)
#define DEFF_QKEY (0x11111111)
#define NOTIFY_COMP_ERROR_SEND(wc,scnt,ccnt) \
{ fprintf(stderr," Completion with error at client\n"); \
fprintf(stderr," Failed status %d: wr_id %d syndrom 0x%x\n",wc.status,(int) wc.wr_id,wc.vendor_err); \
fprintf(stderr, "scnt=%lu, ccnt=%lu\n",scnt, ccnt); }
#define NOTIFY_COMP_ERROR_RECV(wc,rcnt) \
{ fprintf(stderr," Completion with error at server\n"); \
fprintf(stderr," Failed status %d: wr_id %d syndrom 0x%x\n",wc.status,(int) wc.wr_id,wc.vendor_err); \
fprintf(stderr," rcnt=%lu\n",rcnt); }
/* Macro to determine packet size in case of UD. The UD addition is for the GRH . */
#define SIZE(type,size,valid) ((type == UD && valid) ? (size + UD_ADDITION) : (size))
/* Macro to define the buffer size (according to "Nahalem" chip set).
* for small message size (under 4K) , we allocate 4K buffer , and the RDMA write
* verb will write in cycle on the buffer. this improves the BW in "Nahalem" systems.
*/
#define BUFF_SIZE(size,cycle_buffer) ((size < cycle_buffer) ? (cycle_buffer) : (size))
/* UD addition to the buffer. */
#define IF_UD_ADD(type,cache_line_size) ((type == UD) ? (cache_line_size) : (0))
/* Macro that defines the address where we write in RDMA.
* If message size is smaller then CACHE_LINE size then we write in CACHE_LINE jumps.
*/
#define INC(size,cache_line_size) ((size > cache_line_size) ? ((size%cache_line_size == 0) ? \
(size) : (cache_line_size*(size/cache_line_size+1))) : (cache_line_size))
#define UD_MSG_2_EXP(size) ((log(size))/(log(2)))
#define MASK_IS_SET(mask, attr) (((mask)&(attr))!=0)
/******************************************************************************
* Perftest resources Structures and data types.
******************************************************************************/
struct pingpong_context {
struct ibv_context *context;
struct ibv_comp_channel *channel;
struct ibv_pd *pd;
struct ibv_mr **mr;
struct ibv_cq *send_cq;
struct ibv_cq *recv_cq;
void **buf;
struct ibv_ah **ah;
struct ibv_qp **qp;
struct ibv_srq *srq;
struct ibv_sge *sge_list;
struct ibv_sge *recv_sge_list;
struct ibv_send_wr *wr;
struct ibv_recv_wr *rwr;
uint64_t size;
uint64_t *my_addr;
uint64_t *rx_buffer_addr;
uint64_t *rem_addr;
uint64_t buff_size;
uint64_t send_qp_buff_size;
uint64_t flow_buff_size;
int tx_depth;
int huge_shmid;
uint64_t *scnt;
uint64_t *ccnt;
int is_contig_supported;
uint32_t *ctrl_buf;
uint32_t *credit_buf;
struct ibv_mr *credit_mr;
struct ibv_sge *ctrl_sge_list;
struct ibv_send_wr *ctrl_wr;
int send_rcredit;
int credit_cnt;
int cache_line_size;
int cycle_buffer;
};
struct pingpong_dest {
int lid;
// int out_reads;
int qpn;
int psn;
// unsigned rkey;
// unsigned long long vaddr;
union ibv_gid gid;
// unsigned srqn;
// int gid_index;
};
/******************************************************************************
* Perftest resources Methods and interface utilitizes.
******************************************************************************/
/* check_add_port
*
* Description : Creating a service struct from a given port and servername.
*
* Parameters :
* service - an empty char** to contain the service name.
* port - The selected port on which the server will listen.
* hints - The requested ai_* params for the connection.
* res - Holds the result.
*
* Return Value : SUCCESS, FAILURE.
*/
int check_add_port(char **service,int port,
const char *servername,
struct addrinfo *hints,
struct addrinfo **res);
/* ctx_find_dev
*
* Description : Returns the device corresponding to ib_devname
* or the first one found , in case ib_devname == NULL
*
* Parameters :
* ib_devname - The name of the device requested or NULL for the first one.
*
* Return Value : the device or NULL in case of failure.
*/
struct ibv_device* ctx_find_dev(const char *ib_devname);
/* alloc_ctx
*
* Description : allocate all perftest resources.
*
* Parameters :
* ctx - Resources sructure.
* user_param - the perftest parameters.
*/
void alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_param);
/* destroy_ctx
*
* Description : Deallocate all perftest resources.
*
* Parameters :
* ctx - Resources sructure.
* user_param - the perftest parameters.
*
* Return Value : SUCCESS, FAILURE.
*/
int destroy_ctx(struct pingpong_context *ctx,
struct perftest_parameters *user_param);
/* ctx_init
*
* Description :
* Creates all the test resources.
* It creates Buffer, PD, MR, CQ QPs and moves the QPs to INIT.
*
* Parameters :
* ctx - An empty resources sructure to fill inside the resources.
* user_param - the perftest parameters.
*
* Return Value : SUCCESS, FAILURE.
*/
int ctx_init(struct pingpong_context *ctx,struct perftest_parameters *user_param);
/* ctx_qp_create.
*
* Description :
* Creates a QP , according to the attributes given in param.
* The relevent attributes are tx_depth,rx_depth,inline_size and connection_type.
*
* Parameters :
* pd - The Protection domain , each the qp will be assigned to.
* send_cq - The CQ that will produce send CQE.
* recv_qp - The CQ that will produce recv CQE.
* param - The parameters for the QP.
*
* Return Value : Adress of the new QP.
*/
struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
struct perftest_parameters *user_param);
/* ctx_modify_qp_to_init.
*
* Description :
*
* Modifies the given QP to INIT state , according to attributes in param.
* The relevent attributes are ib_port, connection_type and verb.
*
* Parameters :
*
* qp - The QP that will be moved to INIT.
* param - The parameters for the QP.
*
* Return Value : SUCCESS, FAILURE.
*
*/
int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param, uint64_t init_flag);
/* ctx_connect.
*
* Description :
*
* Modifies the given QP to RTR and then RTS states, given it's transport type and feature.
*
* Parameters :
*
* ctx - Test Context.
* dest - pingpong_dest struct of the remote side.
* user_param - user_parameters struct for this test.
* my_dest - pingpong_dest struct of this side.
*
* Return Value : SUCCESS, FAILURE.
*
*/
int ctx_connect(struct pingpong_context *ctx,
struct pingpong_dest *dest,
struct perftest_parameters *user_param,
struct pingpong_dest *my_dest);
/* ctx_set_send_exp_wqes.
*
* Description :
*
* Prepare the exp send work request templates for all QPs
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
* rem_dest - pingpong_dest struct of the remote side.
*
*/
void ctx_set_send_exp_wqes(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *rem_dest);
/* ctx_set_send_regwqes.
*
* Description :
*
* Prepare the regular send work request templates for all QPs
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
* rem_dest - pingpong_dest struct of the remote side.
*
*/
void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *rem_dest);
/* ctx_set_send_wqes.
*
* Description :
*
* Prepare the send work request templates for all QPs
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
* rem_dest - pingpong_dest struct of the remote side.
*
*/
void ctx_set_send_wqes(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *rem_dest);
/* ctx_set_recv_wqes.
*
* Description :
*
* Prepare the receives work request templates for all QPs in SEND receive test.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*
*/
int ctx_set_recv_wqes(struct pingpong_context *ctx,struct perftest_parameters *user_param);
/* ctx_alloc_credit
*
* Description :
*
* Allocate resources to support the credit exchange mechanism,
* which allows ib_send_bw to work with iWARP
*
* Parameters :
*
* ctx - test context
* user_param - user parameters struct for this test
* my_dest - pingpong_dest struct of the this side
*
* my_dest is updated to store the credit buf vaddr and rkey
* which need to be exchanged with the remote side
* to enable RDMA WRITE op
*/
int ctx_alloc_credit(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *my_dest);
/* ctx_set_credit_wqes
*
* Description :
*
* Prepare the send credit work request templates for all QPs
* RDMA WRITE op is used for sending credit
* Credit exchange is necessary for ib_send_bw to work with iWARP
*
* Parameters :
*
* ctx - test context
* user_param - user parameters struct for this test
* rem_dest - pingpong_dest struct of the remote side.
*/
int ctx_set_credit_wqes(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *rem_dest);
/* run_iter_bw.
*
* Description :
*
* The main testing method in BW tests.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*
*/
int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_param);
/* run_iter_bw_infinitely
*
* Description :
*
* Infinite BW method, that prints BW every 5 seconds and never stops.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*
*/
int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_parameters *user_param);
/* run_iter_bw_infinitely_server
*
* Description :
*
* Infinite on SEND server BW method, will not print anything and will post receive wqes for each incoming packet.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*
*/
int run_iter_bw_infinitely_server(struct pingpong_context *ctx, struct perftest_parameters *user_param);
/* run_iter_bw_server.
*
* Description :
*
* The main testing method for Receiver in SEND test.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*
*/
int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters *user_param);
/* run_iter_bi.
*
* Description :
*
* The main testing method for bidirrectional.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*
*/
int run_iter_bi(struct pingpong_context *ctx,struct perftest_parameters *user_param);
/* run_iter_lat_write
*
* Description :
*
* This is the latency test function for WRITE verb.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*/
int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *user_param);
/* run_iter_lat
*
* Description :
*
* This is the latency test function for READ or ATOMIC verb latency tests.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*/
int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_param);
/* run_iter_lat_send
*
* Description :
*
* This is the latency test function for SEND verb latency test.
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*/
int run_iter_lat_send(struct pingpong_context *ctx, struct perftest_parameters *user_param);
/* run_iter_lat_burst
*
* Description :
*
* This is the latency test function for SEND verb latency test in burst mode
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*/
int run_iter_lat_burst(struct pingpong_context *ctx, struct perftest_parameters *user_param);
/* run_iter_lat_burst_server
*
* Description :
*
* This is the latency test function for server side latency test in burst mode
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*/
int run_iter_lat_burst_server(struct pingpong_context *ctx, struct perftest_parameters *user_param);
/* ctx_get_local_lid .
*
* Description :
*
* This method find and returns the local Id in IB subnet manager of
* the selected port and HCA given.The lid identifies the port.
*
* Parameters :
*
* context - the context of the HCA device.
* ib_port - The port of the HCA (1 or 2).
*
* Return Value : The Lid itself. (No error values).
*/
uint16_t ctx_get_local_lid(struct ibv_context *context, int ib_port);
/* ctx_notify_events
*
* Description : Prepare the test to work with events instead of polling the CQ.
* This is the way to work in un interipted mode.
*
* Parameters :
* channel - (Mandotory) the created event channel.
*
* Return Value : SUCCESS, FAILURE.
*/
static __inline int ctx_notify_events(struct ibv_comp_channel *channel)
{
struct ibv_cq *ev_cq;
void *ev_ctx;
if (ibv_get_cq_event(channel,&ev_cq,&ev_ctx)) {
fprintf(stderr, "Failed to get cq_event\n");
return 1;
}
ibv_ack_cq_events(ev_cq,1);
if (ibv_req_notify_cq(ev_cq, 0)) {
fprintf(stderr, "Couldn't request CQ notification\n");
return 1;
}
return 0;
}
#if defined(HAVE_VERBS_EXP)
static __inline void increase_exp_rem_addr(struct ibv_exp_send_wr *wr,int size,uint64_t scnt,uint64_t prim_addr,VerbType verb, int cache_line_size, int cycle_buffer)
{
if (verb == ATOMIC)
wr->wr.atomic.remote_addr += INC(size,cache_line_size);
else
wr->wr.rdma.remote_addr += INC(size,cache_line_size);
if ( ((scnt+1) % (cycle_buffer/ INC(size,cache_line_size))) == 0) {
if (verb == ATOMIC)
wr->wr.atomic.remote_addr = prim_addr;
else
wr->wr.rdma.remote_addr = prim_addr;
}
}
#endif
static __inline void increase_rem_addr(struct ibv_send_wr *wr,int size,uint64_t scnt,uint64_t prim_addr,VerbType verb, int cache_line_size, int cycle_buffer)
{
if (verb == ATOMIC)
wr->wr.atomic.remote_addr += INC(size,cache_line_size);
else
wr->wr.rdma.remote_addr += INC(size,cache_line_size);
if ( ((scnt+1) % (cycle_buffer/ INC(size,cache_line_size))) == 0) {
if (verb == ATOMIC)
wr->wr.atomic.remote_addr = prim_addr;
else
wr->wr.rdma.remote_addr = prim_addr;
}
}
/* increase_loc_addr.
*
* Description :
* Increases the local address in all verbs ,
* (at least 64 CACHE_LINE size) , so that the system will be a able to cahce the data
* in an orginzed way.
*
* Parameters :
* sg - The scatter element of the wqe.
* size - size of the message to send.
* rcnt - The ammount of post_send or post_receive we called.
* prim_addr - The address of the original buffer.
* server_is_ud - Indication to weather we are in UD mode.
*/
static __inline void increase_loc_addr(struct ibv_sge *sg,int size,uint64_t rcnt,uint64_t prim_addr,int server_is_ud, int cache_line_size, int cycle_buffer)
{
sg->addr += INC(size,cache_line_size);
if ( ((rcnt+1) % (cycle_buffer/ INC(size,cache_line_size))) == 0 )
sg->addr = prim_addr;
}
/* catch_alarm.
*
* Description :
* Signal catcher for duration feature.
* run_iter_bw in start will set it to triger at MARGIN (parameter), it will then start counting packets
* and triger it back to SAMPLE TIME, in that time the test will count packets and completion and
* will calculate BW accordingley.
*
*/
void catch_alarm(int sig);
void check_alive(int sig);
/* catch_alarm.
*
* Description :
* Signal catcher for run_infinitely feature.
* Will be triggered every 5 sec and measure BW in this time frame.
*
*/
void catch_alarm_infintely();
/* handle_signal_print_thread
*
* Description :
* Handle thread creation for signal catching in run_infinitely mode
*
*/
void *handle_signal_print_thread(void *sig_mask);
/* ctx_modify_dc_qp_to_init.
*
* Description :
*
* Modifies the given QP to INIT state , according to attributes in param.
* The relevent attributes are ib_port, connection_type and verb.
*
* Parameters :
*
* qp - The QP that will be moved to INIT.
* param - The parameters for the QP.
*
* Return Value : SUCCESS, FAILURE.
*
*/
int ctx_modify_dc_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param);
int perform_warm_up(struct pingpong_context *ctx,struct perftest_parameters *user_param);
#ifdef HAVE_MASKED_ATOMICS
struct ibv_qp* ctx_atomic_qp_create(struct pingpong_context *ctx,
struct perftest_parameters *user_param);
int check_masked_atomics_support(struct pingpong_context *ctx);
#endif
#if defined (HAVE_PACKET_PACING_EXP) || defined (HAVE_PACKET_PACING)
int check_packet_pacing_support(struct pingpong_context *ctx);
#endif
#ifdef HAVE_ACCL_VERBS
struct ibv_exp_res_domain* create_res_domain(struct pingpong_context *ctx,
struct perftest_parameters *user_param);
#endif
int create_reg_qp_main(struct pingpong_context *ctx,
struct perftest_parameters *user_param, int i, int num_of_qps);
#ifdef HAVE_VERBS_EXP
int create_exp_qp_main(struct pingpong_context *ctx,
struct perftest_parameters *user_param, int i, int num_of_qps);
#endif
int create_qp_main(struct pingpong_context *ctx,
struct perftest_parameters *user_param, int i, int num_of_qps);
#ifdef HAVE_VERBS_EXP
struct ibv_qp* ctx_exp_qp_create(struct pingpong_context *ctx,
struct perftest_parameters *user_param, int qp_index);
#endif
int modify_qp_to_init(struct pingpong_context *ctx,
struct perftest_parameters *user_param, int qp_index, int num_of_qps);
/* create_single_mr
*
* Description :
*
* Creates a single MR for a specific QP index.
*
* Parameters :
* ctx - Resources sructure.
* user_param - the perftest parameters.
* qp_index - QP index to register a MR
*
* Return Value : SUCCESS, FAILURE.
*
*/
int create_single_mr(struct pingpong_context *ctx,
struct perftest_parameters *user_param, int qp_index);
/* create_mr
*
* Description :
*
* Creates Memory Regions for the test.
* Takes into consideration all user parameters and test type.
*
* Parameters :
* ctx - Resources sructure.
* user_param - the perftest parameters.
*
* Return Value : SUCCESS, FAILURE.
*
*/
int create_mr(struct pingpong_context *ctx,
struct perftest_parameters *user_param);
/* alloc_hugapage_region
*
* Description :
*
* Creates hugepage memory Regions for the test.
*
* Parameters :
* ctx - Resources sructure.
*
* Return Value : SUCCESS, FAILURE.
*
*/
int alloc_hugepage_region (struct pingpong_context *ctx);
/* run_iter_fs_rate
*
* Description :
*
* The main testing method for Flow steering creation
*
* Parameters :
*
* ctx - Test Context.
* user_param - user_parameters struct for this test.
*
*/
int run_iter_fs(struct pingpong_context *ctx, struct perftest_parameters *user_param);
#endif /* WR_BW_H */

42
usr/benchmarks/ib/write_bw.c
View file

@ -34,7 +34,6 @@
* $Id$
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -46,8 +45,9 @@
int main(int argc, char *argv[])
{
int ret_parser,i = 0;
int ret_parser,i = 0;
struct ibv_device *ib_dev = NULL;
struct pingpong_context ctx;
struct pingpong_dest *my_dest,*rem_dest;
struct perftest_parameters user_param;
@ -95,11 +95,8 @@ int main(int argc, char *argv[])
return FAILURE;
}
/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
if (create_comm_struct(&user_comm,&user_param)) {
fprintf(stderr," Unable to create RDMA_CM resources\n");
return FAILURE;
}
/* copy the relevant user parameters to the comm struct */
create_comm_struct(&user_comm,&user_param);
if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
printf("\n************************************\n");
@ -131,33 +128,10 @@ int main(int argc, char *argv[])
/* Allocating arrays needed for the test. */
alloc_ctx(&ctx,&user_param);
/* Create (if necessary) the rdma_cm ids and channel. */
if (user_param.work_rdma_cm == ON) {
if (user_param.machine == CLIENT) {
if (retry_rdma_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
} else {
if (create_rdma_resources(&ctx,&user_param)) {
fprintf(stderr," Unable to create the rdma_resources\n");
return FAILURE;
}
if (rdma_server_connect(&ctx,&user_param)) {
fprintf(stderr,"Unable to perform rdma_client function\n");
return FAILURE;
}
}
} else {
/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
if (ctx_init(&ctx, &user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
if (ctx_init(&ctx, &user_param)) {
fprintf(stderr, " Couldn't create IB resources\n");
return FAILURE;
}
/* Set up the Connection. */