Mirrors/libhermit
1
0
Fork 0
mirror of https://github.com/hermitcore/libhermit.git synced 2025-03-09 00:00:03 +01:00
Code Issues Releases Wiki Activity

went back to cmd line options for benchmark

Browse source
This commit is contained in:
Annika Wierichs 2018-01-04 13:19:48 +01:00
parent db151e597f
commit d020c7251f
6 changed files with 1371 additions and 493 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
usr/benchmarks/ib/perftest_communication_write_bw.c
View file

@ -40,7 +40,7 @@ enum ctx_device ib_dev_name(struct ibv_context *context)
enum ctx_device dev_fname = UNKNOWN;
struct ibv_device_attr attr;
if (ibv_query_device(context,&attr)) {
if (ibv_query_device(context, &attr)) {
dev_fname = DEVICE_ERROR;
}
@ -128,16 +128,16 @@ int ethernet_read_data(struct perftest_comm *comm, char *recv_msg, size_t size)
int ctx_xchg_data_ethernet( struct perftest_comm *comm,
void *my_data,
void *rem_data,int size)
void *rem_data, int size)
{
if (comm->rdma_params->servername) {
if (ethernet_write_data(comm, (char *) my_data, size)) {
fprintf(stderr," Unable to write to socket/rdam_cm\n");
fprintf(stderr, " Unable to write to socket/rdam_cm\n");
return 1;
}
if (ethernet_read_data(comm, (char *) rem_data, size)) {
fprintf(stderr," Unable to read from socket/rdam_cm\n");
fprintf(stderr, " Unable to read from socket/rdam_cm\n");
return 1;
}
@ -145,12 +145,12 @@ int ctx_xchg_data_ethernet( struct perftest_comm *comm,
} else {
if (ethernet_read_data(comm, (char *) rem_data, size)) {
fprintf(stderr," Unable to read to socket/rdam_cm\n");
fprintf(stderr, " Unable to read to socket/rdam_cm\n");
return 1;
}
if (ethernet_write_data(comm, (char *) my_data, size)) {
fprintf(stderr," Unable to write from socket/rdam_cm\n");
fprintf(stderr, " Unable to write from socket/rdam_cm\n");
return 1;
}
}
@ -468,7 +468,7 @@ static int ethernet_client_connect(struct perftest_comm *comm)
freeaddrinfo(res);
if (sockfd < 0) {
fprintf(stderr, "Couldn't connect to %s:%d\n",comm->rdma_params->servername,comm->rdma_params->port);
fprintf(stderr, "Couldn't connect to %s:%d\n", comm->rdma_params->servername, comm->rdma_params->port);
return 1;
}

4
usr/benchmarks/ib/perftest_parameters.c
View file

@ -491,6 +491,7 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
#endif
putchar('\n');
}
/******************************************************************************
usage
******************************************************************************/
@ -580,10 +581,9 @@ void usage_raw_ethernet(TestType tst)
printf(" send IPv6 Packets.\n");
#endif
printf("\n");
}
/******************************************************************************
*
******************************************************************************/

1427
usr/benchmarks/ib/perftest_parameters_write_bw.c
View file

File diff suppressed because it is too large Load diff

2
usr/benchmarks/ib/perftest_parameters_write_bw.h
View file

@ -577,7 +577,7 @@ static const struct rate_gbps_string RATE_VALUES[RATE_VALUES_COUNT] = {
{IBV_RATE_MAX, "MAX"}
};
char * duplicate_str(const char * given_str);
/* link_layer_str
*

293
usr/benchmarks/ib/perftest_resources_write_bw.c
View file

@ -18,12 +18,8 @@
#include "perftest_parameters_write_bw.h"
/* #ifdef HAVE_VERBS_EXP */
/* static enum ibv_exp_wr_opcode exp_opcode_verbs_array[] = {IBV_EXP_WR_SEND,IBV_EXP_WR_RDMA_WRITE,IBV_EXP_WR_RDMA_READ}; */
/* static enum ibv_exp_wr_opcode exp_opcode_atomic_array[] = {IBV_EXP_WR_ATOMIC_CMP_AND_SWP,IBV_EXP_WR_ATOMIC_FETCH_AND_ADD}; */
/* #endif */
static enum ibv_wr_opcode opcode_verbs_array[] = {IBV_WR_SEND,IBV_WR_RDMA_WRITE,IBV_WR_RDMA_READ};
static enum ibv_wr_opcode opcode_atomic_array[] = {IBV_WR_ATOMIC_CMP_AND_SWP,IBV_WR_ATOMIC_FETCH_AND_ADD};
static enum ibv_wr_opcode opcode_verbs_array[] = {IBV_WR_SEND, IBV_WR_RDMA_WRITE, IBV_WR_RDMA_READ};
static enum ibv_wr_opcode opcode_atomic_array[] = {IBV_WR_ATOMIC_CMP_AND_SWP, IBV_WR_ATOMIC_FETCH_AND_ADD};
#define CPU_UTILITY "/proc/stat"
@ -65,7 +61,7 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
return NULL;
}
qp = ibv_create_qp(ctx->pd,&attr);
qp = ibv_create_qp(ctx->pd, &attr);
return qp;
}
@ -106,11 +102,6 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
int i;
int flags = IBV_ACCESS_LOCAL_WRITE;
/* #ifdef HAVE_VERBS_EXP */
/* struct ibv_exp_reg_mr_in reg_mr_exp_in; */
/* uint64_t exp_flags = IBV_EXP_ACCESS_LOCAL_WRITE; */
/* #endif */
/* ODP */
#if defined HAVE_EX_ODP || defined HAVE_EXP_ODP
if (user_param->use_odp) {
@ -168,53 +159,22 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
} else {
ctx->buf[qp_index] = NULL;
/* #ifdef HAVE_VERBS_EXP */
/* exp_flags |= IBV_EXP_ACCESS_ALLOCATE_MR; */
/* #else */
flags |= (1 << 5);
/* #endif */
}
/* } */
if (user_param->verb == WRITE) {
flags |= IBV_ACCESS_REMOTE_WRITE;
/* #ifdef HAVE_VERBS_EXP */
/* exp_flags |= IBV_EXP_ACCESS_REMOTE_WRITE; */
/* #endif */
} else if (user_param->verb == READ) {
flags |= IBV_ACCESS_REMOTE_READ;
/* #ifdef HAVE_VERBS_EXP */
/* exp_flags |= IBV_EXP_ACCESS_REMOTE_READ; */
/* #endif */
if (user_param->transport_type == IBV_TRANSPORT_IWARP)
flags |= IBV_ACCESS_REMOTE_WRITE;
/* #ifdef HAVE_VERBS_EXP */
/* exp_flags |= IBV_EXP_ACCESS_REMOTE_WRITE; */
/* #endif */
/* } else if (user_param->verb == ATOMIC) { */
/* flags |= IBV_ACCESS_REMOTE_ATOMIC; */
/* #ifdef HAVE_VERBS_EXP */
/* exp_flags |= IBV_EXP_ACCESS_REMOTE_ATOMIC; */
/* #endif */
}
/* Allocating Memory region and assigning our buffer to it. */
/* #ifdef HAVE_VERBS_EXP */
/* if (ctx->is_contig_supported == SUCCESS || user_param->use_odp) { */
/* reg_mr_exp_in.pd = ctx->pd; */
/* reg_mr_exp_in.addr = ctx->buf[qp_index]; */
/* reg_mr_exp_in.length = ctx->buff_size; */
/* reg_mr_exp_in.exp_access = exp_flags; */
/* reg_mr_exp_in.comp_mask = 0; */
/* ctx->mr[qp_index] = ibv_exp_reg_mr(&reg_mr_exp_in); */
/* } */
/* else */
/* ctx->mr[qp_index] = ibv_reg_mr(ctx->pd, ctx->buf[qp_index], ctx->buff_size, flags); */
/* #else */
ctx->mr[qp_index] = ibv_reg_mr(ctx->pd, ctx->buf[qp_index], ctx->buff_size, flags);
/* #endif */
if (!ctx->mr[qp_index]) {
fprintf(stderr, "Couldn't allocate MR\n");
@ -244,7 +204,7 @@ int create_reg_cqs(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
int tx_buffer_depth, int need_recv_cq)
{
ctx->send_cq = ibv_create_cq(ctx->context,tx_buffer_depth *
ctx->send_cq = ibv_create_cq(ctx->context, tx_buffer_depth *
user_param->num_of_qps, NULL, ctx->channel, user_param->eq_num);
if (!ctx->send_cq) {
fprintf(stderr, "Couldn't create CQ\n");
@ -252,7 +212,7 @@ int create_reg_cqs(struct pingpong_context *ctx,
}
if (need_recv_cq) {
ctx->recv_cq = ibv_create_cq(ctx->context,user_param->rx_depth *
ctx->recv_cq = ibv_create_cq(ctx->context, user_param->rx_depth *
user_param->num_of_qps, NULL, ctx->channel, user_param->eq_num);
if (!ctx->recv_cq) {
fprintf(stderr, "Couldn't create a receiver CQ\n");
@ -283,7 +243,7 @@ int create_reg_qp_main(struct pingpong_context *ctx,
return SUCCESS;
}
int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param, uint64_t init_flag)
int ctx_modify_qp_to_init(struct ibv_qp *qp, struct perftest_parameters *user_param, uint64_t init_flag)
{
int num_of_qps = user_param->num_of_qps;
int num_of_qps_per_port = user_param->num_of_qps / 2;
@ -291,11 +251,6 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
struct ibv_qp_attr attr;
int flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
/* #ifdef HAVE_VERBS_EXP */
/* struct ibv_exp_qp_attr exp_attr; */
/* uint64_t exp_flags = 0; */
/* #endif */
static int portindex=0; /* for dual-port support */
int ret = 0;
@ -303,12 +258,6 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
attr.qp_state = IBV_QPS_INIT;
attr.pkey_index = user_param->pkey_index;
/* #ifdef HAVE_VERBS_EXP */
/* memset(&exp_attr, 0, sizeof(struct ibv_exp_qp_attr)); */
/* exp_attr.qp_state = attr.qp_state; */
/* exp_attr.pkey_index = attr.pkey_index; */
/* #endif */
if ( user_param->use_xrc && (user_param->duplex || user_param->tst == LAT)) {
num_of_qps /= 2;
num_of_qps_per_port = num_of_qps / 2;
@ -329,16 +278,8 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
attr.port_num = user_param->ib_port;
}
/* #ifdef HAVE_VERBS_EXP */
/* exp_attr.port_num = attr.port_num; */
/* #endif */
if (user_param->connection_type == RawEth) {
flags = IBV_QP_STATE | IBV_QP_PORT;
/* #ifdef HAVE_VERBS_EXP */
/* exp_flags = init_flag | IBV_EXP_QP_STATE | IBV_EXP_QP_PORT; */
/* #endif */
} else if (user_param->connection_type == UD) {
attr.qkey = DEFF_QKEY;
flags |= IBV_QP_QKEY;
@ -360,16 +301,10 @@ int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_par
}
#endif
/* #ifdef HAVE_VERBS_EXP */
/* if ( (init_flag != 0 && user_param->use_rss) || user_param->masked_atomics ) { */
/* ret = ibv_exp_modify_qp(qp,&exp_attr,exp_flags); */
/* } */
/* else */
/* #endif */
ret = ibv_modify_qp(qp,&attr,flags);
ret = ibv_modify_qp(qp, &attr, flags);
if (ret) {
fprintf(stderr, "Failed to modify QP to INIT, ret=%d\n",ret);
fprintf(stderr, "Failed to modify QP to INIT, ret=%d\n", ret);
return 1;
}
return 0;
@ -423,12 +358,7 @@ int create_cqs(struct pingpong_context *ctx, struct perftest_parameters *user_pa
if ((user_param->connection_type == DC && !dct_only) || (user_param->verb == SEND))
need_recv_cq = 1;
/* #ifdef HAVE_VERBS_EXP */
/* if (user_param->is_exp_cq) */
/* ret = create_exp_cqs(ctx, user_param, tx_buffer_depth, need_recv_cq); */
/* else */
/* #endif */
ret = create_reg_cqs(ctx, user_param, tx_buffer_depth, need_recv_cq);
ret = create_reg_cqs(ctx, user_param, tx_buffer_depth, need_recv_cq);
return ret;
}
@ -437,28 +367,7 @@ int create_qp_main(struct pingpong_context *ctx,
struct perftest_parameters *user_param, int i, int num_of_qps)
{
int ret;
/* #ifdef HAVE_VERBS_EXP */
/* int query; */
/* [> flag that indicates that we are going to use exp QP <] */
/* query = (user_param->connection_type == DC); */
/* query |= (user_param->use_rss && user_param->connection_type == RawEth); */
/* query |= user_param->use_xrc; */
/* query |= user_param->inline_recv_size != 0; */
/* query |= user_param->masked_atomics; */
/* query |= user_param->verb_type != NORMAL_INTF; */
/* query |= user_param->use_res_domain; */
/* query |= user_param->use_exp; */
/* if (query == 1) */
/* user_param->is_exp_qp = 1; */
/* if (user_param->is_exp_qp) */
/* ret = create_exp_qp_main(ctx, user_param, i, num_of_qps); */
/* else */
/* #endif */
ret = create_reg_qp_main(ctx, user_param, i, num_of_qps);
ret = create_reg_qp_main(ctx, user_param, i, num_of_qps);
return ret;
}
@ -479,14 +388,14 @@ int modify_qp_to_init(struct pingpong_context *ctx,
if ( !((!(user_param->duplex || user_param->tst == LAT) && (user_param->machine == SERVER) )
|| ((user_param->duplex || user_param->tst == LAT) && (qp_index >= num_of_qps)))) {
#ifdef HAVE_DC
if (ctx_modify_dc_qp_to_init(ctx->qp[qp_index],user_param)) {
fprintf(stderr," Unable to create DC QP.\n");
if (ctx_modify_dc_qp_to_init(ctx->qp[qp_index], user_param)) {
fprintf(stderr, " Unable to create DC QP.\n");
return FAILURE;
}
#endif
}
} else {
if (ctx_modify_qp_to_init(ctx->qp[qp_index],user_param,init_flag)) {
if (ctx_modify_qp_to_init(ctx->qp[qp_index], user_param, init_flag)) {
fprintf(stderr, "Failed to modify QP to INIT\n");
return FAILURE;
}
@ -599,7 +508,7 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *rem_dest)
{
int i,j;
int i, j;
int num_of_qps = user_param->num_of_qps;
int xrc_offset = 0;
@ -609,13 +518,13 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
}
for (i = 0; i < num_of_qps ; i++) {
memset(&ctx->wr[i*user_param->post_list],0,sizeof(struct ibv_send_wr));
memset(&ctx->wr[i*user_param->post_list], 0, sizeof(struct ibv_send_wr));
ctx->sge_list[i*user_param->post_list].addr = (uintptr_t)ctx->buf[i];
if (user_param->mac_fwd) {
if (user_param->mr_per_qp) {
ctx->sge_list[i*user_param->post_list].addr =
(uintptr_t)ctx->buf[0] + (num_of_qps + i)*BUFF_SIZE(ctx->size,ctx->cycle_buffer);
(uintptr_t)ctx->buf[0] + (num_of_qps + i)*BUFF_SIZE(ctx->size, ctx->cycle_buffer);
} else {
ctx->sge_list[i*user_param->post_list].addr = (uintptr_t)ctx->buf[i];
}
@ -648,8 +557,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
ctx->sge_list[i*user_param->post_list +j].addr = ctx->sge_list[i*user_param->post_list + (j-1)].addr;
if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && user_param->size <= (ctx->cycle_buffer / 2))
increase_loc_addr(&ctx->sge_list[i*user_param->post_list +j],user_param->size,
j-1,ctx->my_addr[i],0,ctx->cache_line_size,ctx->cycle_buffer);
increase_loc_addr(&ctx->sge_list[i*user_param->post_list +j], user_param->size,
j-1, ctx->my_addr[i], 0, ctx->cache_line_size, ctx->cycle_buffer);
}
ctx->wr[i*user_param->post_list + j].sg_list = &ctx->sge_list[i*user_param->post_list + j];
@ -685,8 +594,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
ctx->wr[i*user_param->post_list + (j-1)].wr.rdma.remote_addr;
if ((user_param->tst == BW || user_param->tst == LAT_BY_BW ) && user_param->size <= (ctx->cycle_buffer / 2))
increase_rem_addr(&ctx->wr[i*user_param->post_list + j],user_param->size,
j-1,ctx->rem_addr[i],WRITE,ctx->cache_line_size,ctx->cycle_buffer);
increase_rem_addr(&ctx->wr[i*user_param->post_list + j], user_param->size,
j-1, ctx->rem_addr[i], WRITE, ctx->cache_line_size, ctx->cycle_buffer);
}
} else if (user_param->verb == ATOMIC) {
@ -698,8 +607,8 @@ void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
ctx->wr[i*user_param->post_list + j].wr.atomic.remote_addr =
ctx->wr[i*user_param->post_list + j-1].wr.atomic.remote_addr;
if (user_param->tst == BW || user_param->tst == LAT_BY_BW)
increase_rem_addr(&ctx->wr[i*user_param->post_list + j],user_param->size,
j-1,ctx->rem_addr[i],ATOMIC,ctx->cache_line_size,ctx->cycle_buffer);
increase_rem_addr(&ctx->wr[i*user_param->post_list + j], user_param->size,
j-1, ctx->rem_addr[i], ATOMIC, ctx->cache_line_size, ctx->cycle_buffer);
}
if (user_param->atomicType == FETCH_AND_ADD)
@ -835,19 +744,15 @@ void alloc_ctx(struct pingpong_context *ctx, struct perftest_parameters *user_pa
ALLOCATE(ctx->mr, struct ibv_mr*, user_param->num_of_qps);
ALLOCATE(ctx->buf, void* , user_param->num_of_qps);
#ifdef HAVE_ACCL_VERBS
ALLOCATE(ctx->qp_burst_family, struct ibv_exp_qp_burst_family*, user_param->num_of_qps);
#endif
/* #ifdef HAVE_ACCL_VERBS */
/* ALLOCATE(ctx->qp_burst_family, struct ibv_exp_qp_burst_family*, user_param->num_of_qps); */
/* #endif */
#ifdef HAVE_DC
if (user_param->connection_type == DC) {
#ifdef HAVE_VERBS_EXP
ALLOCATE(ctx->dct, struct ibv_exp_dct*, user_param->num_of_qps);
#else
ALLOCATE(ctx->dct, struct ibv_dct*, user_param->num_of_qps);
#endif
}
#endif
/* #ifdef HAVE_DC */
/* if (user_param->connection_type == DC) { */
/* ALLOCATE(ctx->dct, struct ibv_dct*, user_param->num_of_qps); */
/* } */
/* #endif */
if ((user_param->tst == BW || user_param->tst == LAT_BY_BW) && (user_param->machine == CLIENT || user_param->duplex)) {
ALLOCATE(user_param->tcompleted, cycles_t, tarr_size);
@ -872,9 +777,6 @@ void alloc_ctx(struct pingpong_context *ctx, struct perftest_parameters *user_pa
if (user_param->machine == CLIENT || user_param->tst == LAT || user_param->duplex) {
ALLOCATE(ctx->sge_list, struct ibv_sge, user_param->num_of_qps * user_param->post_list);
#ifdef HAVE_VERBS_EXP
ALLOCATE(ctx->exp_wr, struct ibv_exp_send_wr, user_param->num_of_qps * user_param->post_list);
#endif
ALLOCATE(ctx->wr, struct ibv_send_wr, user_param->num_of_qps * user_param->post_list);
if ((user_param->verb == SEND && user_param->connection_type == UD ) || user_param->connection_type == DC) {
@ -915,16 +817,10 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
ctx->is_contig_supported = FAILURE;
#ifdef HAVE_ACCL_VERBS
enum ibv_exp_query_intf_status intf_status;
struct ibv_exp_query_intf_params intf_params;
#endif
#ifdef HAVE_VERBS_EXP
struct ibv_exp_device_attr dattr;
memset(&dattr, 0, sizeof(dattr));
get_verbs_pointers(ctx);
#endif
/* #ifdef HAVE_ACCL_VERBS */
/* enum ibv_exp_query_intf_status intf_status; */
/* struct ibv_exp_query_intf_params intf_params; */
/* #endif */
/* #if defined HAVE_OOO_ATTR || defined HAVE_EXP_OOO_ATTR */
/* if (user_param->use_ooo) { */
@ -935,10 +831,6 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
/* } */
/* #endif */
/* #ifdef HAVE_VERBS_EXP */
/* ctx->is_contig_supported = check_for_contig_pages_support(ctx->context); */
/* #endif */
if (user_param->use_hugepages)
ctx->is_contig_supported = FAILURE;
@ -1071,11 +963,7 @@ int ctx_connect(struct pingpong_context *ctx,
int i;
/* #ifdef HAVE_DC */
/* #ifdef HAVE_VERBS_EXP */
/* struct ibv_exp_qp_attr attr_ex; */
/* #else */
/* struct ibv_qp_attr_ex attr_ex; */
/* #endif */
/* #elif HAVE_PACKET_PACING_EXP */
/* struct ibv_exp_qp_attr attr_ex; */
/* #endif */
@ -1389,9 +1277,6 @@ int destroy_ctx(struct pingpong_context *ctx,
if (user_param->machine == CLIENT || user_param->tst == LAT || user_param->duplex) {
free(ctx->sge_list);
free(ctx->wr);
/* #ifdef HAVE_VERBS_EXP */
/* free(ctx->exp_wr); */
/* #endif */
}
if (user_param->verb == SEND && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex)) {
@ -1409,16 +1294,6 @@ void ctx_set_send_wqes(struct pingpong_context *ctx,
{
ctx_set_send_reg_wqes(ctx, user_param, rem_dest);
/* #ifdef HAVE_VERBS_EXP */
/* if (user_param->use_exp == 1) { */
/* ctx_set_send_exp_wqes(ctx, user_param, rem_dest); */
/* } */
/* else { */
/* #endif */
/* ctx_set_send_reg_wqes(ctx, user_param, rem_dest); */
/* #ifdef HAVE_VERBS_EXP */
/* } */
/* #endif */
}
int perform_warm_up(struct pingpong_context *ctx, struct perftest_parameters *user_param)
@ -1426,10 +1301,6 @@ int perform_warm_up(struct pingpong_context *ctx, struct perftest_parameters *us
int ne, index, warmindex, warmupsession;
int err = 0;
/* #ifdef HAVE_VERBS_EXP */
/* struct ibv_exp_send_wr *bad_exp_wr = NULL; */
/* #endif */
struct ibv_send_wr *bad_wr = NULL;
struct ibv_wc wc;
struct ibv_wc *wc_for_cleaning = NULL;
@ -1448,16 +1319,6 @@ int perform_warm_up(struct pingpong_context *ctx, struct perftest_parameters *us
for (index=0 ; index < num_of_qps ; index++) {
for (warmindex = 0 ;warmindex < warmupsession ;warmindex += user_param->post_list) {
/* #ifdef HAVE_VERBS_EXP */
/* if (user_param->use_exp == 1) */
/* err = (ctx->exp_post_send_func_pointer)(ctx->qp[index], */
/* &ctx->exp_wr[index*user_param->post_list], &bad_exp_wr); */
/* else */
/* err = (ctx->post_send_func_pointer)(ctx->qp[index], &ctx->wr[index*user_param->post_list], &bad_wr); */
/* #else */
/* err = ibv_post_send(ctx->qp[index], &ctx->wr[index*user_param->post_list], &bad_wr); */
/* #endif */
err = ibv_post_send(ctx->qp[index], &ctx->wr[index*user_param->post_list], &bad_wr);
if (err) {
fprintf(stderr, "Couldn't post send during warm up: qp %d scnt=%d \n", index, warmindex);
@ -1496,14 +1357,6 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p
uint64_t tot_iters;
int err = 0;
/* #ifdef HAVE_VERBS_EXP */
/* struct ibv_exp_send_wr *bad_exp_wr = NULL; */
/* #ifdef HAVE_ACCL_VERBS */
/* int pl_index; */
/* struct ibv_sge *sg_l; */
/* #endif */
/* #endif */
struct ibv_send_wr *bad_wr = NULL;
struct ibv_wc *wc = NULL;
int num_of_qps = user_param->num_of_qps;
@ -1596,20 +1449,6 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p
!(ctx->scnt[index] == (user_param->iters - 1) && user_param->test_type == ITERATIONS)) {
ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED;
/* #ifdef HAVE_VERBS_EXP */
/* #ifdef HAVE_ACCL_VERBS */
/* if (user_param->verb_type == ACCL_INTF) */
/* ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_QP_BURST_SIGNALED; */
/* else { */
/* #endif */
/* if (user_param->use_exp == 1) */
/* ctx->exp_wr[index].exp_send_flags &= ~IBV_EXP_SEND_SIGNALED; */
/* else */
/* #endif */
/* ctx->wr[index].send_flags &= ~IBV_SEND_SIGNALED; */
/* #ifdef HAVE_ACCL_VERBS */
/* } */
/* #endif */
}
if (user_param->noPeak == OFF)
@ -1618,32 +1457,6 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p
/* if (user_param->test_type == DURATION && user_param->state == END_STATE) */
/* break; */
/* #ifdef HAVE_VERBS_EXP */
/* #ifdef HAVE_ACCL_VERBS */
/* if (user_param->verb_type == ACCL_INTF) { */
/* for (pl_index = 0; pl_index < user_param->post_list; pl_index++) { */
/* sg_l = ctx->exp_wr[index*user_param->post_list + pl_index].sg_list; */
/* ctx->qp_burst_family[index]->send_pending(ctx->qp[index], sg_l->addr, sg_l->length, sg_l->lkey, */
/* ctx->exp_wr[index*user_param->post_list + pl_index].exp_send_flags); */
/* } */
/* ctx->qp_burst_family[index]->send_flush(ctx->qp[index]); */
/* } else { */
/* #endif */
/* if (user_param->use_exp == 1) { */
/* err = (ctx->exp_post_send_func_pointer)(ctx->qp[index], */
/* &ctx->exp_wr[index*user_param->post_list], &bad_exp_wr); */
/* } */
/* else { */
/* err = (ctx->post_send_func_pointer)(ctx->qp[index], */
/* &ctx->wr[index*user_param->post_list], &bad_wr); */
/* } */
/* #ifdef HAVE_ACCL_VERBS */
/* } */
/* #endif */
/* #else */
/* err = ibv_post_send(ctx->qp[index], &ctx->wr[index*user_param->post_list], &bad_wr); */
/* #endif */
err = ibv_post_send(ctx->qp[index], &ctx->wr[index*user_param->post_list], &bad_wr);
if (err) {
fprintf(stderr, "Couldn't post send: qp %d scnt=%lu \n", index, ctx->scnt[index]);
@ -1665,26 +1478,11 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p
/* in multiple flow scenarios we will go to next cycle buffer address in the main buffer*/
if (user_param->post_list == 1 && user_param->size <= (ctx->cycle_buffer / 2)) {
/* #ifdef HAVE_VERBS_EXP */
/* if (user_param->use_exp == 1) */
/* increase_loc_addr(ctx->exp_wr[index].sg_list, user_param->size, */
/* ctx->scnt[index], ctx->my_addr[index] + address_offset, 0, */
/* ctx->cache_line_size, ctx->cycle_buffer); */
/* else */
/* #endif */
increase_loc_addr(ctx->wr[index].sg_list, user_param->size, ctx->scnt[index],
ctx->my_addr[index] + address_offset , 0, ctx->cache_line_size,
ctx->cycle_buffer);
increase_loc_addr(ctx->wr[index].sg_list, user_param->size,
ctx->scnt[index], ctx->my_addr[index] + address_offset , 0,
ctx->cache_line_size, ctx->cycle_buffer);
if (user_param->verb != SEND) {
/* #ifdef HAVE_VERBS_EXP */
/* if (user_param->use_exp == 1) */
/* increase_exp_rem_addr(&ctx->exp_wr[index], user_param->size, */
/* ctx->scnt[index], ctx->rem_addr[index], user_param->verb, */
/* ctx->cache_line_size, ctx->cycle_buffer); */
/* else */
/* #endif */
increase_rem_addr(&ctx->wr[index], user_param->size,
ctx->scnt[index], ctx->rem_addr[index], user_param->verb,
ctx->cache_line_size, ctx->cycle_buffer);
@ -1697,21 +1495,8 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p
/* ask for completion on this wr */
if (user_param->post_list == 1 &&
(ctx->scnt[index]%user_param->cq_mod == user_param->cq_mod - 1 ||
(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
/* #ifdef HAVE_VERBS_EXP */
/* #ifdef HAVE_ACCL_VERBS */
/* if (user_param->verb_type == ACCL_INTF) */
/* ctx->exp_wr[index].exp_send_flags |= IBV_EXP_QP_BURST_SIGNALED; */
/* else { */
/* #endif */
/* if (user_param->use_exp == 1) */
/* ctx->exp_wr[index].exp_send_flags |= IBV_EXP_SEND_SIGNALED; */
/* else */
/* #endif */
ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
/* #ifdef HAVE_ACCL_VERBS */
/* } */
/* #endif */
(user_param->test_type == ITERATIONS && ctx->scnt[index] == user_param->iters - 1))) {
ctx->wr[index].send_flags |= IBV_SEND_SIGNALED;
}
/* Check if a full burst was sent. */

124
usr/benchmarks/ib/write_bw.c
View file

@ -46,29 +46,29 @@
int main(int argc, char *argv[])
{
int ret_parser,i = 0;
int ret_parser, i = 0;
struct ibv_device *ib_dev = NULL;
struct pingpong_context ctx;
struct pingpong_dest *my_dest,*rem_dest;
struct pingpong_dest *my_dest, *rem_dest;
struct perftest_parameters user_param;
struct perftest_comm user_comm;
struct bw_report_data my_bw_rep, rem_bw_rep;
/* init default values to user's parameters */
memset(&user_param,0,sizeof(struct perftest_parameters));
memset(&user_comm,0,sizeof(struct perftest_comm));
memset(&ctx,0,sizeof(struct pingpong_context));
memset(&user_param, 0, sizeof(struct perftest_parameters));
memset(&user_comm, 0, sizeof(struct perftest_comm));
memset(&ctx, 0, sizeof(struct pingpong_context));
user_param.verb = WRITE;
user_param.tst = BW;
strncpy(user_param.version, VERSION, sizeof(user_param.version));
/* Configure the parameters values according to user arguments or default values. */
ret_parser = parser(&user_param,argv,argc);
ret_parser = parser(&user_param, argv, argc);
if (ret_parser) {
if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
fprintf(stderr," Parser function exited with Error\n");
fprintf(stderr, " Parser function exited with Error\n");
return FAILURE;
}
@ -79,7 +79,7 @@ int main(int argc, char *argv[])
/* Finding the IB device selected (or default if none is selected). */
ib_dev = ctx_find_dev(user_param.ib_devname);
if (!ib_dev) {
fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
fprintf(stderr, " Unable to find the Infiniband/RoCE device\n");
return FAILURE;
}
@ -91,13 +91,13 @@ int main(int argc, char *argv[])
}
/* See if MTU and link type are valid and supported. */
if (check_link(ctx.context,&user_param)) {
if (check_link(ctx.context, &user_param)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
/* copy the relevant user parameters to the comm struct */
create_comm_struct(&user_comm,&user_param);
create_comm_struct(&user_comm, &user_param);
if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
printf("\n************************************\n");
@ -107,7 +107,7 @@ int main(int argc, char *argv[])
/* Initialize the connection and print the local data. */
if (establish_connection(&user_comm)) {
fprintf(stderr," Unable to init the socket connection\n");
fprintf(stderr, " Unable to init the socket connection\n");
return FAILURE;
}
@ -124,7 +124,7 @@ int main(int argc, char *argv[])
check_sys_data(&user_comm, &user_param);
/* See if MTU and link type are valid and supported. */
if (check_mtu(ctx.context,&user_param, &user_comm)) {
if (check_mtu(ctx.context, &user_param, &user_comm)) {
fprintf(stderr, " Couldn't get context for the device\n");
return FAILURE;
}
@ -135,7 +135,7 @@ int main(int argc, char *argv[])
memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
/* Allocating arrays needed for the test. */
alloc_ctx(&ctx,&user_param);
alloc_ctx(&ctx, &user_param);
/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
if (ctx_init(&ctx, &user_param)) {
@ -144,8 +144,8 @@ int main(int argc, char *argv[])
}
/* Set up the Connection. */
if (set_up_connection(&ctx,&user_param,my_dest)) {
fprintf(stderr," Unable to set up socket connection\n");
if (set_up_connection(&ctx, &user_param, my_dest)) {
fprintf(stderr, " Unable to set up socket connection\n");
return FAILURE;
}
@ -155,38 +155,38 @@ int main(int argc, char *argv[])
/* Print this machine QP information */
for (i=0; i < user_param.num_of_qps; i++)
ctx_print_pingpong_data(&my_dest[i],&user_comm);
ctx_print_pingpong_data(&my_dest[i], &user_comm);
user_comm.rdma_params->side = REMOTE;
for (i=0; i < user_param.num_of_qps; i++) {
if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
if (ctx_hand_shake(&user_comm, &my_dest[i], &rem_dest[i])) {
fprintf(stderr, " Failed to exchange data between server and clients\n");
return FAILURE;
}
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
ctx_print_pingpong_data(&rem_dest[i], &user_comm);
}
if (user_param.work_rdma_cm == OFF) { // TODO:
if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
fprintf(stderr," Please Try to use a different IP version.\n\n");
fprintf(stderr, "\n Found Incompatibility issue with GID types.\n");
fprintf(stderr, " Please Try to use a different IP version.\n\n");
return FAILURE;
}
}
if (user_param.work_rdma_cm == OFF) {
if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
fprintf(stderr," Unable to Connect the HCA's through the link\n");
if (ctx_connect(&ctx, rem_dest, &user_param, my_dest)) {
fprintf(stderr, " Unable to Connect the HCA's through the link\n");
return FAILURE;
}
}
/* An additional handshake is required after moving qp to RTR. */
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
fprintf(stderr, " Failed to exchange data between server and clients\n");
return FAILURE;
}
@ -206,16 +206,16 @@ int main(int argc, char *argv[])
/* For half duplex tests, server just waits for client to exit */
if (user_param.machine == SERVER && !user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
fprintf(stderr, " Failed to exchange data between server and clients\n");
return FAILURE;
}
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version));
print_full_bw_report(&user_param, &rem_bw_rep, NULL);
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
if (ctx_close_connection(&user_comm, &my_dest[0], &rem_dest[0])) {
fprintf(stderr, "Failed to close connection between server and client\n");
return FAILURE;
}
@ -227,15 +227,15 @@ int main(int argc, char *argv[])
}
/* if (user_param.work_rdma_cm == ON) { */
/* if (destroy_ctx(&ctx,&user_param)) { */
/* if (destroy_ctx(&ctx, &user_param)) { */
/* fprintf(stderr, "Failed to destroy resources\n"); */
/* return FAILURE; */
/* } */
/* user_comm.rdma_params->work_rdma_cm = ON; */
/* return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params); */
/* return destroy_ctx(user_comm.rdma_ctx, user_comm.rdma_params); */
/* } */
return destroy_ctx(&ctx,&user_param);
return destroy_ctx(&ctx, &user_param);
}
if (user_param.test_method == RUN_ALL) {
@ -243,7 +243,7 @@ int main(int argc, char *argv[])
for (i = 1; i < 24 ; ++i) {
user_param.size = (uint64_t)1 << i;
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
ctx_set_send_wqes(&ctx, &user_param, rem_dest);
if (user_param.perform_warm_up) {
if(perform_warm_up(&ctx, &user_param)) {
@ -253,35 +253,35 @@ int main(int argc, char *argv[])
}
if(user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
fprintf(stderr, "Failed to sync between server and client between different msg sizes\n");
return FAILURE;
}
}
if(run_iter_bw(&ctx,&user_param)) {
fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
if(run_iter_bw(&ctx, &user_param)) {
fprintf(stderr, " Failed to complete run_iter_bw function successfully\n");
return FAILURE;
}
if (user_param.duplex && (atof(user_param.version) >= 4.6)) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
fprintf(stderr, "Failed to sync between server and client between different msg sizes\n");
return FAILURE;
}
}
print_report_bw(&user_param,&my_bw_rep);
print_report_bw(&user_param, &my_bw_rep);
if (user_param.duplex) {
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version));
print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
}
}
} else if (user_param.test_method == RUN_REGULAR) {
ctx_set_send_wqes(&ctx,&user_param,rem_dest);
ctx_set_send_wqes(&ctx, &user_param, rem_dest);
if (user_param.verb != SEND) {
@ -294,21 +294,21 @@ int main(int argc, char *argv[])
}
if(user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
fprintf(stderr, "Failed to sync between server and client between different msg sizes\n");
return FAILURE;
}
}
if(run_iter_bw(&ctx,&user_param)) {
fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
if(run_iter_bw(&ctx, &user_param)) {
fprintf(stderr, " Failed to complete run_iter_bw function successfully\n");
return FAILURE;
}
print_report_bw(&user_param,&my_bw_rep);
print_report_bw(&user_param, &my_bw_rep);
if (user_param.duplex) {
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version));
print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
}
@ -329,10 +329,10 @@ int main(int argc, char *argv[])
}
/* } else if (user_param.test_method == RUN_INFINITELY) { */
/* ctx_set_send_wqes(&ctx,&user_param,rem_dest); */
/* ctx_set_send_wqes(&ctx, &user_param, rem_dest); */
/* if(run_iter_bw_infinitely(&ctx,&user_param)) { */
/* fprintf(stderr," Error occurred while running infinitely! aborting ...\n"); */
/* if(run_iter_bw_infinitely(&ctx, &user_param)) { */
/* fprintf(stderr, " Error occurred while running infinitely! aborting ...\n"); */
/* return FAILURE; */
/* } */
}
@ -347,27 +347,27 @@ int main(int argc, char *argv[])
/* For half duplex tests, server just waits for client to exit */
if (user_param.machine == CLIENT && !user_param.duplex) {
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr," Failed to exchange data between server and clients\n");
if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
fprintf(stderr, " Failed to exchange data between server and clients\n");
return FAILURE;
}
xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version));
}
/* Closing connection. */
if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
fprintf(stderr,"Failed to close connection between server and client\n");
if (ctx_close_connection(&user_comm, &my_dest[0], &rem_dest[0])) {
fprintf(stderr, "Failed to close connection between server and client\n");
return FAILURE;
}
if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
fprintf(stderr,"Error: BW result is below bw limit\n");
fprintf(stderr, "Error: BW result is below bw limit\n");
return FAILURE;
}
if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
fprintf(stderr,"Error: Msg rate is below msg_rate limit\n");
fprintf(stderr, "Error: Msg rate is below msg_rate limit\n");
return FAILURE;
}
@ -375,13 +375,13 @@ int main(int argc, char *argv[])
free(rem_dest);
/* if (user_param.work_rdma_cm == ON) { */
/* if (destroy_ctx(&ctx,&user_param)) { */
/* if (destroy_ctx(&ctx, &user_param)) { */
/* fprintf(stderr, "Failed to destroy resources\n"); */
/* return FAILURE; */
/* } */
/* user_comm.rdma_params->work_rdma_cm = ON; */
/* return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params); */
/* return destroy_ctx(user_comm.rdma_ctx, user_comm.rdma_params); */
/* } */
return destroy_ctx(&ctx,&user_param);
return destroy_ctx(&ctx, &user_param);
}