From b381b3a54a8f1392f2c2254547c6a7ad4ee86e3e Mon Sep 17 00:00:00 2001 From: Annika Wierichs Date: Fri, 5 Jan 2018 11:37:09 +0100 Subject: [PATCH] all 4 benchmarks linking --- usr/benchmarks/CMakeLists.txt | 9 + usr/benchmarks/ib/perftest_parameters.c | 5 +- .../ib/perftest_parameters_write_bw.c | 131 +++++++++++- .../ib/perftest_resources_write_bw.c | 188 +++++++++++++++++ usr/benchmarks/ib/read_bw.c | 196 ++++++++---------- usr/benchmarks/ib/read_lat.c | 174 +++++++--------- usr/benchmarks/ib/write_lat.c | 186 +++++++---------- 7 files changed, 553 insertions(+), 336 deletions(-) diff --git a/usr/benchmarks/CMakeLists.txt b/usr/benchmarks/CMakeLists.txt index 90bc5a607..a0b628ffe 100644 --- a/usr/benchmarks/CMakeLists.txt +++ b/usr/benchmarks/CMakeLists.txt @@ -14,6 +14,15 @@ add_executable(hg hg.c hist.c rdtsc.c run.c init.c opt.c report.c setup.c) add_executable(ib_write_bw ib/write_bw.c ib/get_clock.c ib/perftest_parameters_write_bw.c ib/perftest_resources_write_bw.c ib/perftest_communication_write_bw.c) +add_executable(ib_write_lat ib/write_lat.c ib/get_clock.c + ib/perftest_parameters_write_bw.c ib/perftest_resources_write_bw.c + ib/perftest_communication_write_bw.c) +add_executable(ib_read_bw ib/read_bw.c ib/get_clock.c + ib/perftest_parameters_write_bw.c ib/perftest_resources_write_bw.c + ib/perftest_communication_write_bw.c) +add_executable(ib_read_lat ib/read_lat.c ib/get_clock.c + ib/perftest_parameters_write_bw.c ib/perftest_resources_write_bw.c + ib/perftest_communication_write_bw.c) add_executable(netio netio.c) diff --git a/usr/benchmarks/ib/perftest_parameters.c b/usr/benchmarks/ib/perftest_parameters.c index 79f743d91..054f11635 100755 --- a/usr/benchmarks/ib/perftest_parameters.c +++ b/usr/benchmarks/ib/perftest_parameters.c @@ -2851,6 +2851,7 @@ void print_full_bw_report (struct perftest_parameters *user_param, struct bw_rep fprintf(stdout, user_param->cpu_util_data.enable ? REPORT_EXT_CPU_UTIL : REPORT_EXT , calc_cpu_util(user_param)); } } + /****************************************************************************** * ******************************************************************************/ @@ -2881,7 +2882,6 @@ static int cycles_compare(const void *aptr, const void *bptr) #define LAT_MEASURE_TAIL (2) void print_report_lat (struct perftest_parameters *user_param) { - int i; int rtt_factor; double cycles_to_units, cycles_rtt_quotient, temp_var, pow_var; @@ -2980,9 +2980,6 @@ void print_report_lat (struct perftest_parameters *user_param) free(delta); } -/****************************************************************************** - * - ******************************************************************************/ void print_report_lat_duration (struct perftest_parameters *user_param) { int rtt_factor; diff --git a/usr/benchmarks/ib/perftest_parameters_write_bw.c b/usr/benchmarks/ib/perftest_parameters_write_bw.c index c480d1121..81046b760 100755 --- a/usr/benchmarks/ib/perftest_parameters_write_bw.c +++ b/usr/benchmarks/ib/perftest_parameters_write_bw.c @@ -27,16 +27,6 @@ static const char *atomicTypesStr[] = {"CMP_AND_SWAP", "FETCH_AND_ADD"}; // ----------------------------------------------------------------------------- -/* char * duplicate_str(const char * given_str) */ -/* { */ - /* char * new_str = malloc(strlen(given_str) + 1); */ - /* if (new_str == NULL) */ - /* return NULL; */ - /* strcpy(new_str, given_str); */ - - /* return new_str; */ -/* } */ - const int str_link_layer(const char *str) { if (strncasecmp("IB", str, 2) == 0) @@ -1430,8 +1420,28 @@ int parse_ip6_from_str(char *ip6, struct in6_addr *addr) return inet_pton(AF_INET6, ip6, addr); } +static int cycles_compare(const void *aptr, const void *bptr) +{ + const cycles_t *a = aptr; + const cycles_t *b = bptr; + if (*a < *b) return -1; + if (*a > *b) return 1; + + return 0; +} + +static inline cycles_t get_median(int n, cycles_t delta[]) +{ + if ((n - 1) % 2) + return(delta[n / 2] + delta[n / 2 - 1]) / 2; + else + return delta[n / 2]; +} + + // ----------------------------------------------------------------------------- + int parser(struct perftest_parameters *user_param, char *argv[], int argc) { int c, size_len; @@ -2453,6 +2463,107 @@ void print_report_bw(struct perftest_parameters *user_param, struct bw_report_da } } +#define LAT_MEASURE_TAIL (2) +void print_report_lat (struct perftest_parameters *user_param) +{ + int i; + int rtt_factor; + double cycles_to_units, cycles_rtt_quotient, temp_var, pow_var; + cycles_t median ; + cycles_t *delta = NULL; + const char* units; + double latency, stdev, average_sum = 0 , average, stdev_sum = 0; + int iters_99, iters_99_9; + int measure_cnt; + + measure_cnt = (user_param->tst == LAT) ? user_param->iters - 1 : (user_param->iters) / user_param->reply_every; + rtt_factor = (user_param->verb == READ || user_param->verb == ATOMIC) ? 1 : 2; + ALLOCATE(delta, cycles_t, measure_cnt); + + if (user_param->r_flag->cycles) { + cycles_to_units = 1; + units = "cycles"; + } else { + cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f); + units = "usec"; + } + + if (user_param->tst == LAT) { + for (i = 0; i < measure_cnt; ++i) { + delta[i] = user_param->tposted[i + 1] - user_param->tposted[i]; + } + } else if (user_param->tst == LAT_BY_BW) { + for (i = 0; i < measure_cnt; ++i) { + delta[i] = user_param->tcompleted[i] - user_param->tposted[i]; + } + } + else { + fprintf(stderr,"print report LAT is support in LAT and LAT_BY_BW tests only\n"); + exit(1); + } + + cycles_rtt_quotient = cycles_to_units * rtt_factor; + if (user_param->r_flag->unsorted) { + printf("#, %s\n", units); + for (i = 0; i < measure_cnt; ++i) + printf("%d, %g\n", i + 1, delta[i] / cycles_rtt_quotient); + } + + qsort(delta, measure_cnt, sizeof *delta, cycles_compare); + measure_cnt = measure_cnt - LAT_MEASURE_TAIL; + median = get_median(measure_cnt, delta); + + /* calcualte average sum on sorted array*/ + for (i = 0; i < measure_cnt; ++i) + average_sum += (delta[i] / cycles_rtt_quotient); + + average = average_sum / measure_cnt; + + /* Calculate stdev by variance*/ + for (i = 0; i < measure_cnt; ++i) { + temp_var = average - (delta[i] / cycles_rtt_quotient); + pow_var = pow(temp_var, 2 ); + stdev_sum += pow_var; + } + + if (user_param->r_flag->histogram) { + printf("#, %s\n", units); + for (i = 0; i < measure_cnt; ++i) + printf("%d, %g\n", i + 1, delta[i] / cycles_rtt_quotient); + } + + if (user_param->r_flag->unsorted || user_param->r_flag->histogram) { + if (user_param->output == FULL_VERBOSITY) { + printf(RESULT_LINE); + printf("%s",(user_param->test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR); + printf((user_param->cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT)); + } + } + + latency = median / cycles_rtt_quotient; + stdev = sqrt(stdev_sum / measure_cnt); + iters_99 = ceil((measure_cnt) * 0.99); + iters_99_9 = ceil((measure_cnt) * 0.999); + + if (user_param->output == OUTPUT_LAT) + printf("%lf\n",average); + else { + printf(REPORT_FMT_LAT, + (unsigned long)user_param->size, + user_param->iters, + delta[0] / cycles_rtt_quotient, + delta[measure_cnt] / cycles_rtt_quotient, + latency, + average, + stdev, + delta[iters_99] / cycles_rtt_quotient, + delta[iters_99_9] / cycles_rtt_quotient); + printf( user_param->cpu_util_data.enable ? REPORT_EXT_CPU_UTIL : REPORT_EXT , calc_cpu_util(user_param)); + } + + free(delta); +} + void print_full_bw_report (struct perftest_parameters *user_param, struct bw_report_data *my_bw_rep, struct bw_report_data *rem_bw_rep) { double bw_peak = my_bw_rep->bw_peak; diff --git a/usr/benchmarks/ib/perftest_resources_write_bw.c b/usr/benchmarks/ib/perftest_resources_write_bw.c index a6440bd4a..dcdc354d5 100755 --- a/usr/benchmarks/ib/perftest_resources_write_bw.c +++ b/usr/benchmarks/ib/perftest_resources_write_bw.c @@ -692,6 +692,22 @@ static int ctx_modify_qp_to_rts(struct ibv_qp *qp, return ibv_modify_qp(qp, attr, flags); } +int verify_params_with_device_context(struct ibv_context *context, + struct perftest_parameters *user_param) +{ + if(user_param->use_event) { + if(user_param->eq_num > context->num_comp_vectors) { + fprintf(stderr, " Completion vector specified is invalid\n"); + fprintf(stderr, " Max completion vector = %d\n", + context->num_comp_vectors - 1); + return FAILURE; + } + } + + return SUCCESS; +} + + // ----------------------------------------------------------------------------- @@ -1573,6 +1589,178 @@ cleaning: return return_value; } +int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_param) +{ + uint64_t scnt = 0; + int ne; + int err = 0; + struct ibv_send_wr *bad_wr = NULL; + struct ibv_wc wc; + + int cpu_mhz = get_cpu_mhz(user_param->cpu_freq_f); + int total_gap_cycles = user_param->latency_gap * cpu_mhz; + cycles_t end_cycle, start_gap=0; + + ctx->wr[0].sg_list->length = user_param->size; + ctx->wr[0].send_flags = IBV_SEND_SIGNALED; + + /* Duration support in latency tests. */ + /* if (user_param->test_type == DURATION) { */ + /* duration_param=user_param; */ + /* duration_param->state = START_STATE; */ + /* signal(SIGALRM, catch_alarm); */ + /* user_param->iters = 0; */ + /* if (user_param->margin > 0) */ + /* alarm(user_param->margin); */ + /* else */ + /* catch_alarm(0); */ + /* } */ + + while (scnt < user_param->iters || (user_param->test_type == DURATION && user_param->state != END_STATE)) { + if (user_param->latency_gap) { + start_gap = get_cycles(); + end_cycle = start_gap + total_gap_cycles; + while (get_cycles() < end_cycle) { + continue; + } + } + if (user_param->test_type == ITERATIONS) + user_param->tposted[scnt++] = get_cycles(); + + err = ibv_post_send(ctx->qp[0],&ctx->wr[0],&bad_wr); + if (err) { + fprintf(stderr,"Couldn't post send: scnt=%lu\n",scnt); + return 1; + } + + if (user_param->test_type == DURATION && user_param->state == END_STATE) + break; + + if (user_param->use_event) { + if (ctx_notify_events(ctx->channel)) { + fprintf(stderr, "Couldn't request CQ notification\n"); + return 1; + } + } + + do { + ne = ibv_poll_cq(ctx->send_cq, 1, &wc); + if(ne > 0) { + if (wc.status != IBV_WC_SUCCESS) { + NOTIFY_COMP_ERROR_SEND(wc,scnt,scnt); + return 1; + } + if (user_param->test_type==DURATION && user_param->state == SAMPLE_STATE) + user_param->iters++; + + } else if (ne < 0) { + fprintf(stderr, "poll CQ failed %d\n", ne); + return FAILURE; + } + + } while (!user_param->use_event && ne == 0); + } + + return 0; +} + +int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *user_param) +{ + uint64_t scnt = 0; + uint64_t ccnt = 0; + uint64_t rcnt = 0; + int ne; + int err = 0; + int poll_buf_offset = 0; + volatile char *poll_buf = NULL; + volatile char *post_buf = NULL; + struct ibv_send_wr *bad_wr = NULL; + struct ibv_wc wc; + + int cpu_mhz = get_cpu_mhz(user_param->cpu_freq_f); + int total_gap_cycles = user_param->latency_gap * cpu_mhz; + cycles_t end_cycle, start_gap=0; + + ctx->wr[0].sg_list->length = user_param->size; + ctx->wr[0].send_flags = IBV_SEND_SIGNALED; + if (user_param->size <= user_param->inline_size) + ctx->wr[0].send_flags |= IBV_SEND_INLINE; + + if((user_param->use_xrc || user_param->connection_type == DC)) + poll_buf_offset = 1; + + post_buf = (char*)ctx->buf[0] + user_param->size - 1; + poll_buf = (char*)ctx->buf[0] + (user_param->num_of_qps + poll_buf_offset)*BUFF_SIZE(ctx->size, ctx->cycle_buffer) + user_param->size - 1; + + /* Duration support in latency tests. */ + /* if (user_param->test_type == DURATION) { */ + /* duration_param=user_param; */ + /* duration_param->state = START_STATE; */ + /* signal(SIGALRM, catch_alarm); */ + /* user_param->iters = 0; */ + /* if (user_param->margin > 0) */ + /* alarm(user_param->margin); */ + /* else */ + /* catch_alarm(0); */ + /* } */ + + /* Done with setup. Start the test. */ + while (scnt < user_param->iters || ccnt < user_param->iters || rcnt < user_param->iters + || ((user_param->test_type == DURATION && user_param->state != END_STATE))) { + + if ((rcnt < user_param->iters || user_param->test_type == DURATION) && + !(scnt < 1 && user_param->machine == SERVER)) { + rcnt++; + while (*poll_buf != (char)rcnt && user_param->state != END_STATE); + } + + if (scnt < user_param->iters || user_param->test_type == DURATION) { + if (user_param->latency_gap) { + start_gap = get_cycles(); + end_cycle = start_gap + total_gap_cycles; + while (get_cycles() < end_cycle) { + continue; + } + } + + if (user_param->test_type == ITERATIONS) + user_param->tposted[scnt] = get_cycles(); + + *post_buf = (char)++scnt; + err = ibv_post_send(ctx->qp[0],&ctx->wr[0],&bad_wr); + if (err) { + fprintf(stderr,"Couldn't post send: scnt=%lu\n",scnt); + return 1; + } + } + + if (user_param->test_type == DURATION && user_param->state == END_STATE) + break; + + if (ccnt < user_param->iters || user_param->test_type == DURATION) { + do { + ne = ibv_poll_cq(ctx->send_cq, 1, &wc); + } while (ne == 0); + + if(ne > 0) { + if (wc.status != IBV_WC_SUCCESS) { + NOTIFY_COMP_ERROR_SEND(wc,scnt,ccnt); + return 1; + } + ccnt++; + if (user_param->test_type==DURATION && user_param->state == SAMPLE_STATE) { + user_param->iters++; + } + + } else if (ne < 0) { + fprintf(stderr, "poll CQ failed %d\n", ne); + return FAILURE; + } + } + } + + return 0; +} diff --git a/usr/benchmarks/ib/read_bw.c b/usr/benchmarks/ib/read_bw.c index 4b0386392..1e80f1e6b 100755 --- a/usr/benchmarks/ib/read_bw.c +++ b/usr/benchmarks/ib/read_bw.c @@ -1,26 +1,26 @@ /* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2006 Mellanox Technologies Ltd. All rights reserved. - * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2006 Mellanox Technologies Ltd. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU + * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF @@ -42,33 +42,31 @@ #include "perftest_parameters.h" #include "perftest_communication.h" -/****************************************************************************** - * - ******************************************************************************/ + int main(int argc, char *argv[]) { - int ret_parser,i = 0; - struct ibv_device *ib_dev = NULL; + int ret_parser, i = 0; + struct ibv_device *ib_dev = NULL; struct pingpong_context ctx; struct pingpong_dest *my_dest = NULL; struct pingpong_dest *rem_dest = NULL; struct perftest_parameters user_param; - struct perftest_comm user_comm; + struct perftest_comm user_comm; struct bw_report_data my_bw_rep, rem_bw_rep; /* init default values to user's parameters */ - memset(&ctx,0,sizeof(struct pingpong_context)); - memset(&user_param , 0 , sizeof(struct perftest_parameters)); - memset(&user_comm,0,sizeof(struct perftest_comm)); + memset(&ctx, 0, sizeof(struct pingpong_context)); + memset(&user_param, 0, sizeof(struct perftest_parameters)); + memset(&user_comm, 0, sizeof(struct perftest_comm)); - user_param.verb = READ; - user_param.tst = BW; + user_param.verb = READ; + user_param.tst = BW; strncpy(user_param.version, VERSION, sizeof(user_param.version)); - ret_parser = parser(&user_param,argv,argc); + ret_parser = parser(&user_param, argv, argc); if (ret_parser) { if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT) - fprintf(stderr," Parser function exited with Error\n"); + fprintf(stderr, " Parser function exited with Error\n"); return FAILURE; } @@ -94,14 +92,14 @@ int main(int argc, char *argv[]) } /* See if MTU and link type are valid and supported. */ - if (check_link(ctx.context,&user_param)) { + if (check_link(ctx.context, &user_param)) { fprintf(stderr, " Couldn't get context for the device\n"); return FAILURE; } /* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */ - if (create_comm_struct(&user_comm,&user_param)) { - fprintf(stderr," Unable to create RDMA_CM resources\n"); + if (create_comm_struct(&user_comm, &user_param)) { + fprintf(stderr, " Unable to create RDMA_CM resources\n"); return FAILURE; } @@ -113,7 +111,7 @@ int main(int argc, char *argv[]) /* Initialize the connection and print the local data. */ if (establish_connection(&user_comm)) { - fprintf(stderr," Unable to init the socket connection\n"); + fprintf(stderr, " Unable to init the socket connection\n"); return FAILURE; } @@ -122,7 +120,7 @@ int main(int argc, char *argv[]) check_sys_data(&user_comm, &user_param); /* See if MTU and link type are valid and supported. */ - if (check_mtu(ctx.context,&user_param, &user_comm)) { + if (check_mtu(ctx.context, &user_param, &user_comm)) { fprintf(stderr, " Couldn't get context for the device\n"); return FAILURE; } @@ -133,39 +131,17 @@ int main(int argc, char *argv[]) memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps); /* Allocating arrays needed for the test. */ - alloc_ctx(&ctx,&user_param); + alloc_ctx(&ctx, &user_param); - /* Create (if nessacery) the rdma_cm ids and channel. */ - if (user_param.work_rdma_cm == ON) { - - if (user_param.machine == CLIENT) { - if (retry_rdma_connect(&ctx,&user_param)) { - fprintf(stderr,"Unable to perform rdma_client function\n"); - return FAILURE; - } - - } else { - if (create_rdma_resources(&ctx,&user_param)) { - fprintf(stderr," Unable to create the rdma_resources\n"); - return FAILURE; - } - if (rdma_server_connect(&ctx,&user_param)) { - fprintf(stderr,"Unable to perform rdma_client function\n"); - return FAILURE; - } - } - - } else { - /* create all the basic IB resources. */ - if (ctx_init(&ctx,&user_param)) { - fprintf(stderr, " Couldn't create IB resources\n"); - return FAILURE; - } + /* create all the basic IB resources. */ + if (ctx_init(&ctx, &user_param)) { + fprintf(stderr, " Couldn't create IB resources\n"); + return FAILURE; } /* Set up the Connection. */ - if (set_up_connection(&ctx,&user_param,my_dest)) { - fprintf(stderr," Unable to set up socket connection\n"); + if (set_up_connection(&ctx, &user_param, my_dest)) { + fprintf(stderr, " Unable to set up socket connection\n"); return FAILURE; } @@ -174,39 +150,39 @@ int main(int argc, char *argv[]) /* Print this machine QP information */ for (i=0; i < user_param.num_of_qps; i++) - ctx_print_pingpong_data(&my_dest[i],&user_comm); + ctx_print_pingpong_data(&my_dest[i], &user_comm); user_comm.rdma_params->side = REMOTE; for (i=0; i < user_param.num_of_qps; i++) { /* shaking hands and gather the other side info. */ - if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) { - fprintf(stderr,"Failed to exchange data between server and clients\n"); + if (ctx_hand_shake(&user_comm, &my_dest[i], &rem_dest[i])) { + fprintf(stderr, "Failed to exchange data between server and clients\n"); return FAILURE; } - ctx_print_pingpong_data(&rem_dest[i],&user_comm); + ctx_print_pingpong_data(&rem_dest[i], &user_comm); } if (user_param.work_rdma_cm == OFF) { if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) { - fprintf(stderr,"\n Found Incompatibility issue with GID types.\n"); - fprintf(stderr," Please Try to use a different IP version.\n\n"); + fprintf(stderr, "\n Found Incompatibility issue with GID types.\n"); + fprintf(stderr, " Please Try to use a different IP version.\n\n"); return FAILURE; } } if (user_param.work_rdma_cm == OFF) { - if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) { - fprintf(stderr," Unable to Connect the HCA's through the link\n"); + if (ctx_connect(&ctx, rem_dest, &user_param, my_dest)) { + fprintf(stderr, " Unable to Connect the HCA's through the link\n"); return FAILURE; } } /* An additional handshake is required after moving qp to RTR. */ - if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) { - fprintf(stderr,"Failed to exchange data between server and clients\n"); + if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) { + fprintf(stderr, "Failed to exchange data between server and clients\n"); return FAILURE; } @@ -225,16 +201,16 @@ int main(int argc, char *argv[]) /* For half duplex tests, server just waits for client to exit */ if (user_param.machine == SERVER && !user_param.duplex) { - if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) { - fprintf(stderr," Failed to exchange data between server and clients\n"); + if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) { + fprintf(stderr, " Failed to exchange data between server and clients\n"); return FAILURE; } - xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version)); + xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version)); print_full_bw_report(&user_param, &rem_bw_rep, NULL); - if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) { - fprintf(stderr,"Failed to close connection between server and client\n"); + if (ctx_close_connection(&user_comm, &my_dest[0], &rem_dest[0])) { + fprintf(stderr, "Failed to close connection between server and client\n"); return FAILURE; } if (user_param.output == FULL_VERBOSITY) { @@ -245,15 +221,15 @@ int main(int argc, char *argv[]) } if (user_param.work_rdma_cm == ON) { - if (destroy_ctx(&ctx,&user_param)) { + if (destroy_ctx(&ctx, &user_param)) { fprintf(stderr, "Failed to destroy resources\n"); return FAILURE; } user_comm.rdma_params->work_rdma_cm = ON; - return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params); + return destroy_ctx(user_comm.rdma_ctx, user_comm.rdma_params); } - return destroy_ctx(&ctx,&user_param); + return destroy_ctx(&ctx, &user_param); } @@ -269,7 +245,7 @@ int main(int argc, char *argv[]) for (i = 1; i < 24 ; ++i) { user_param.size = (uint64_t)1 << i; - ctx_set_send_wqes(&ctx,&user_param,rem_dest); + ctx_set_send_wqes(&ctx, &user_param, rem_dest); if (user_param.perform_warm_up) { if(perform_warm_up(&ctx, &user_param)) { @@ -279,33 +255,33 @@ int main(int argc, char *argv[]) } if(user_param.duplex) { - if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) { - fprintf(stderr,"Failed to sync between server and client between different msg sizes\n"); + if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) { + fprintf(stderr, "Failed to sync between server and client between different msg sizes\n"); return FAILURE; } } - if(run_iter_bw(&ctx,&user_param)) + if(run_iter_bw(&ctx, &user_param)) return 17; if (user_param.duplex && (atof(user_param.version) >= 4.6)) { - if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) { - fprintf(stderr,"Failed to sync between server and client between different msg sizes\n"); + if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) { + fprintf(stderr, "Failed to sync between server and client between different msg sizes\n"); return FAILURE; } } - print_report_bw(&user_param,&my_bw_rep); + print_report_bw(&user_param, &my_bw_rep); if (user_param.duplex) { - xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version)); + xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version)); print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep); } } } else if (user_param.test_method == RUN_REGULAR) { - ctx_set_send_wqes(&ctx,&user_param,rem_dest); + ctx_set_send_wqes(&ctx, &user_param, rem_dest); if (user_param.perform_warm_up) { if(perform_warm_up(&ctx, &user_param)) { fprintf(stderr, "Problems with warm up\n"); @@ -314,21 +290,21 @@ int main(int argc, char *argv[]) } if(user_param.duplex) { - if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) { - fprintf(stderr,"Failed to sync between server and client between different msg sizes\n"); + if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) { + fprintf(stderr, "Failed to sync between server and client between different msg sizes\n"); return FAILURE; } } - if(run_iter_bw(&ctx,&user_param)) { - fprintf(stderr," Failed to complete run_iter_bw function successfully\n"); + if(run_iter_bw(&ctx, &user_param)) { + fprintf(stderr, " Failed to complete run_iter_bw function successfully\n"); return FAILURE; } - print_report_bw(&user_param,&my_bw_rep); + print_report_bw(&user_param, &my_bw_rep); if (user_param.duplex) { - xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version)); + xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version)); print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep); } @@ -347,14 +323,14 @@ int main(int argc, char *argv[]) printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT)); print_full_bw_report(&user_param, &rem_bw_rep, NULL); } - } else if (user_param.test_method == RUN_INFINITELY) { - ctx_set_send_wqes(&ctx,&user_param,rem_dest); + /* } else if (user_param.test_method == RUN_INFINITELY) { */ + /* ctx_set_send_wqes(&ctx, &user_param, rem_dest); */ - if(run_iter_bw_infinitely(&ctx,&user_param)) { - fprintf(stderr," Error occurred while running! aborting ...\n"); - return FAILURE; - } + /* if(run_iter_bw_infinitely(&ctx, &user_param)) { */ + /* fprintf(stderr, " Error occurred while running! aborting ...\n"); */ + /* return FAILURE; */ + /* } */ } if (user_param.output == FULL_VERBOSITY) { @@ -367,37 +343,37 @@ int main(int argc, char *argv[]) /* For half duplex tests, server just waits for client to exit */ if (user_param.machine == CLIENT && !user_param.duplex) { - if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) { - fprintf(stderr," Failed to exchange data between server and clients\n"); + if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) { + fprintf(stderr, " Failed to exchange data between server and clients\n"); return FAILURE; } - xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version)); + xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version)); } - if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) { - fprintf(stderr,"Failed to close connection between server and client\n"); + if (ctx_close_connection(&user_comm, &my_dest[0], &rem_dest[0])) { + fprintf(stderr, "Failed to close connection between server and client\n"); return FAILURE; } if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) { - fprintf(stderr,"Error: BW result is below bw limit\n"); + fprintf(stderr, "Error: BW result is below bw limit\n"); return FAILURE; } if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) { - fprintf(stderr,"Error: Msg rate is below msg_rate limit\n"); + fprintf(stderr, "Error: Msg rate is below msg_rate limit\n"); return FAILURE; } if (user_param.work_rdma_cm == ON) { - if (destroy_ctx(&ctx,&user_param)) { + if (destroy_ctx(&ctx, &user_param)) { fprintf(stderr, "Failed to destroy resources\n"); return FAILURE; } user_comm.rdma_params->work_rdma_cm = ON; - return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params); + return destroy_ctx(user_comm.rdma_ctx, user_comm.rdma_params); } - return destroy_ctx(&ctx,&user_param); + return destroy_ctx(&ctx, &user_param); } diff --git a/usr/benchmarks/ib/read_lat.c b/usr/benchmarks/ib/read_lat.c index 5149988fd..d7719d232 100755 --- a/usr/benchmarks/ib/read_lat.c +++ b/usr/benchmarks/ib/read_lat.c @@ -1,27 +1,27 @@ /* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler) - * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU + * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF @@ -39,44 +39,38 @@ #include #include #include -#if !defined(__FreeBSD__) -#include -#endif #include "get_clock.h" #include "perftest_resources.h" #include "perftest_parameters.h" #include "perftest_communication.h" -/****************************************************************************** - * - ******************************************************************************/ int main(int argc, char *argv[]) { - int ret_parser,i = 0; - struct report_options report; - struct pingpong_context ctx; - struct ibv_device *ib_dev; - struct perftest_parameters user_param; - struct pingpong_dest *my_dest = NULL; - struct pingpong_dest *rem_dest = NULL; - struct perftest_comm user_comm; + int ret_parser, i = 0; + struct report_options report; + struct pingpong_context ctx; + struct ibv_device *ib_dev; + struct perftest_parameters user_param; + struct pingpong_dest *my_dest = NULL; + struct pingpong_dest *rem_dest = NULL; + struct perftest_comm user_comm; /* init default values to user's parameters */ - memset(&ctx,0,sizeof(struct pingpong_context)); - memset(&user_param,0,sizeof(struct perftest_parameters)); - memset(&user_comm,0,sizeof(struct perftest_comm)); + memset(&ctx, 0, sizeof(struct pingpong_context)); + memset(&user_param, 0, sizeof(struct perftest_parameters)); + memset(&user_comm, 0, sizeof(struct perftest_comm)); - user_param.verb = READ; - user_param.tst = LAT; - user_param.r_flag = &report; + user_param.verb = READ; + user_param.tst = LAT; + user_param.r_flag = &report; strncpy(user_param.version, VERSION, sizeof(user_param.version)); /* Configure the parameters values according to user arguments or defalut values. */ - ret_parser = parser(&user_param,argv,argc); + ret_parser = parser(&user_param, argv, argc); if (ret_parser) { if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT) - fprintf(stderr," Parser function exited with Error\n"); + fprintf(stderr, " Parser function exited with Error\n"); return FAILURE; } @@ -87,7 +81,7 @@ int main(int argc, char *argv[]) /* Finding the IB device selected (or defalut if no selected). */ ib_dev = ctx_find_dev(user_param.ib_devname); if (!ib_dev) { - fprintf(stderr," Unable to find the Infiniband/RoCE device\n"); + fprintf(stderr, " Unable to find the Infiniband/RoCE device\n"); return FAILURE; } @@ -105,14 +99,14 @@ int main(int argc, char *argv[]) } /* See if MTU and link type are valid and supported. */ - if (check_link(ctx.context,&user_param)) { + if (check_link(ctx.context, &user_param)) { fprintf(stderr, " Couldn't get context for the device\n"); return FAILURE; } /* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */ - if (create_comm_struct(&user_comm,&user_param)) { - fprintf(stderr," Unable to create RDMA_CM resources\n"); + if (create_comm_struct(&user_comm, &user_param)) { + fprintf(stderr, " Unable to create RDMA_CM resources\n"); return FAILURE; } @@ -124,7 +118,7 @@ int main(int argc, char *argv[]) /* Initialize the connection and print the local data. */ if (establish_connection(&user_comm)) { - fprintf(stderr," Unable to init the socket connection\n"); + fprintf(stderr, " Unable to init the socket connection\n"); return FAILURE; } @@ -133,7 +127,7 @@ int main(int argc, char *argv[]) check_sys_data(&user_comm, &user_param); /* See if MTU and link type are valid and supported. */ - if (check_mtu(ctx.context,&user_param, &user_comm)) { + if (check_mtu(ctx.context, &user_param, &user_comm)) { fprintf(stderr, " Couldn't get context for the device\n"); return FAILURE; } @@ -144,40 +138,17 @@ int main(int argc, char *argv[]) memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps); /* Allocate arrays */ - alloc_ctx(&ctx,&user_param); + alloc_ctx(&ctx, &user_param); - /* Create (if nessacery) the rdma_cm ids and channel. */ - if (user_param.work_rdma_cm == ON) { - - if (user_param.machine == CLIENT) { - if (retry_rdma_connect(&ctx,&user_param)) { - fprintf(stderr,"Unable to perform rdma_client function\n"); - return FAILURE; - } - - } else { - if (create_rdma_resources(&ctx,&user_param)) { - fprintf(stderr," Unable to create the rdma_resources\n"); - return FAILURE; - } - if (rdma_server_connect(&ctx,&user_param)) { - fprintf(stderr,"Unable to perform rdma_client function\n"); - return FAILURE; - } - } - - } else { - - /* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */ - if (ctx_init(&ctx,&user_param)) { - fprintf(stderr, " Couldn't create IB resources\n"); - return FAILURE; - } + /* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */ + if (ctx_init(&ctx, &user_param)) { + fprintf(stderr, " Couldn't create IB resources\n"); + return FAILURE; } /* Set up the Connection. */ - if (set_up_connection(&ctx,&user_param,my_dest)) { - fprintf(stderr," Unable to set up socket connection\n"); + if (set_up_connection(&ctx, &user_param, my_dest)) { + fprintf(stderr, " Unable to set up socket connection\n"); return FAILURE; } @@ -185,11 +156,11 @@ int main(int argc, char *argv[]) ctx_print_test_info(&user_param); for (i=0; i < user_param.num_of_qps; i++) - ctx_print_pingpong_data(&my_dest[i],&user_comm); + ctx_print_pingpong_data(&my_dest[i], &user_comm); - /* shaking hands and gather the other side info. */ - if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) { - fprintf(stderr,"Failed to exchange data between server and clients\n"); + /* shaking hands and gather the other side info. */ + if (ctx_hand_shake(&user_comm, my_dest, rem_dest)) { + fprintf(stderr, "Failed to exchange data between server and clients\n"); return FAILURE; } @@ -197,41 +168,36 @@ int main(int argc, char *argv[]) for (i=0; i < user_param.num_of_qps; i++) { /* shaking hands and gather the other side info. */ - if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) { - fprintf(stderr,"Failed to exchange data between server and clients\n"); + if (ctx_hand_shake(&user_comm, &my_dest[i], &rem_dest[i])) { + fprintf(stderr, "Failed to exchange data between server and clients\n"); return FAILURE; } - ctx_print_pingpong_data(&rem_dest[i],&user_comm); + ctx_print_pingpong_data(&rem_dest[i], &user_comm); } - if (user_param.work_rdma_cm == OFF) { - if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) { - fprintf(stderr,"\n Found Incompatibility issue with GID types.\n"); - fprintf(stderr," Please Try to use a different IP version.\n\n"); - return FAILURE; - } + if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) { + fprintf(stderr, "\n Found Incompatibility issue with GID types.\n"); + fprintf(stderr, " Please Try to use a different IP version.\n\n"); + return FAILURE; } - if (user_param.work_rdma_cm == OFF) { - - if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) { - fprintf(stderr," Unable to Connect the HCA's through the link\n"); - return FAILURE; - } + if (ctx_connect(&ctx, rem_dest, &user_param, my_dest)) { + fprintf(stderr, " Unable to Connect the HCA's through the link\n"); + return FAILURE; } /* An additional handshake is required after moving qp to RTR. */ - if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) { - fprintf(stderr,"Failed to exchange data between server and clients\n"); + if (ctx_hand_shake(&user_comm, my_dest, rem_dest)) { + fprintf(stderr, "Failed to exchange data between server and clients\n"); return FAILURE; } /* Only Client post read request. */ if (user_param.machine == SERVER) { - if (ctx_close_connection(&user_comm,my_dest,rem_dest)) { - fprintf(stderr,"Failed to close connection between server and client\n"); + if (ctx_close_connection(&user_comm, my_dest, rem_dest)) { + fprintf(stderr, "Failed to close connection between server and client\n"); return FAILURE; } if (user_param.output == FULL_VERBOSITY) { @@ -250,29 +216,31 @@ int main(int argc, char *argv[]) if (user_param.output == FULL_VERBOSITY) { printf(RESULT_LINE); - printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR); + printf("%s", (user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR); printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT)); } - ctx_set_send_wqes(&ctx,&user_param,rem_dest); + ctx_set_send_wqes(&ctx, &user_param, rem_dest); if (user_param.test_method == RUN_ALL) { for (i = 1; i < 24 ; ++i) { user_param.size = (uint64_t)1 << i; - if(run_iter_lat(&ctx,&user_param)) + if(run_iter_lat(&ctx, &user_param)); return 17; - user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); + /* user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); */ + print_report_lat(&user_param); } } else { - if(run_iter_lat(&ctx,&user_param)) + if(run_iter_lat(&ctx, &user_param)) return 18; - user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); + /* user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); */ + print_report_lat(&user_param); } - if (ctx_close_connection(&user_comm,my_dest,rem_dest)) { - fprintf(stderr,"Failed to close connection between server and client\n"); + if (ctx_close_connection(&user_comm, my_dest, rem_dest)) { + fprintf(stderr, "Failed to close connection between server and client\n"); return FAILURE; } diff --git a/usr/benchmarks/ib/write_lat.c b/usr/benchmarks/ib/write_lat.c index 03d661012..4a6278d4d 100755 --- a/usr/benchmarks/ib/write_lat.c +++ b/usr/benchmarks/ib/write_lat.c @@ -1,27 +1,27 @@ /* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler) - * Copyright (c) 2009 HNR Consulting. All rights reserved. + * Copyright (c) 2009 HNR Consulting. All rights reserved. * * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU + * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF @@ -39,44 +39,39 @@ #include #include #include -#if !defined(__FreeBSD__) -#include -#endif #include "get_clock.h" -#include "perftest_parameters.h" -#include "perftest_resources.h" -#include "perftest_communication.h" +#include "perftest_parameters_write_bw.h" +#include "perftest_resources_write_bw.h" +#include "perftest_communication_write_bw.h" + -/****************************************************************************** - * - ******************************************************************************/ int main(int argc, char *argv[]) { - int ret_parser,i = 0; - struct report_options report; - struct pingpong_context ctx; - struct pingpong_dest *my_dest = NULL; - struct pingpong_dest *rem_dest = NULL; - struct ibv_device *ib_dev; - struct perftest_parameters user_param; - struct perftest_comm user_comm; + int ret_parser, i = 0; + struct report_options report; + struct pingpong_context ctx; + struct pingpong_dest *my_dest = NULL; + struct pingpong_dest *rem_dest = NULL; + struct ibv_device *ib_dev; + struct perftest_parameters user_param; + struct perftest_comm user_comm; /* init default values to user's parameters */ - memset(&ctx,0,sizeof(struct pingpong_context)); + memset(&ctx, 0, sizeof(struct pingpong_context)); memset(&user_param, 0, sizeof(struct perftest_parameters)); - memset(&user_comm,0,sizeof(struct perftest_comm)); + memset(&user_comm, 0, sizeof(struct perftest_comm)); - user_param.verb = WRITE; - user_param.tst = LAT; - user_param.r_flag = &report; + user_param.verb = WRITE; + user_param.tst = LAT; + user_param.r_flag = &report; strncpy(user_param.version, VERSION, sizeof(user_param.version)); /* Configure the parameters values according to user arguments or defalut values. */ - ret_parser = parser(&user_param,argv,argc); + ret_parser = parser(&user_param, argv, argc); if (ret_parser) { if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT) - fprintf(stderr," Parser function exited with Error\n"); + fprintf(stderr, " Parser function exited with Error\n"); return FAILURE; } @@ -87,7 +82,7 @@ int main(int argc, char *argv[]) /* Finding the IB device selected (or defalut if no selected). */ ib_dev = ctx_find_dev(user_param.ib_devname); if (!ib_dev) { - fprintf(stderr," Unable to find the Infiniband/RoCE device\n"); + fprintf(stderr, " Unable to find the Infiniband/RoCE device\n"); return FAILURE; } @@ -99,14 +94,14 @@ int main(int argc, char *argv[]) } /* See if MTU and link type are valid and supported. */ - if (check_link(ctx.context,&user_param)) { + if (check_link(ctx.context, &user_param)) { fprintf(stderr, " Couldn't get context for the device\n"); return FAILURE; } /* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */ - if (create_comm_struct(&user_comm,&user_param)) { - fprintf(stderr," Unable to create RDMA_CM resources\n"); + if (create_comm_struct(&user_comm, &user_param)) { + fprintf(stderr, " Unable to create RDMA_CM resources\n"); return FAILURE; } @@ -118,7 +113,7 @@ int main(int argc, char *argv[]) /* Initialize the connection and print the local data. */ if (establish_connection(&user_comm)) { - fprintf(stderr," Unable to init the socket connection\n"); + fprintf(stderr, " Unable to init the socket connection\n"); return FAILURE; } @@ -127,7 +122,7 @@ int main(int argc, char *argv[]) check_sys_data(&user_comm, &user_param); /* See if MTU and link type are valid and supported. */ - if (check_mtu(ctx.context,&user_param, &user_comm)) { + if (check_mtu(ctx.context, &user_param, &user_comm)) { fprintf(stderr, " Couldn't get context for the device\n"); return FAILURE; } @@ -138,40 +133,17 @@ int main(int argc, char *argv[]) memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps); /* Allocating arrays needed for the test. */ - alloc_ctx(&ctx,&user_param); + alloc_ctx(&ctx, &user_param); - /* Create (if nessacery) the rdma_cm ids and channel. */ - if (user_param.work_rdma_cm == ON) { - - if (user_param.machine == CLIENT) { - if (retry_rdma_connect(&ctx,&user_param)) { - fprintf(stderr,"Unable to perform rdma_client function\n"); - return FAILURE; - } - - } else { - if (create_rdma_resources(&ctx,&user_param)) { - fprintf(stderr," Unable to create the rdma_resources\n"); - return FAILURE; - } - if (rdma_server_connect(&ctx,&user_param)) { - fprintf(stderr,"Unable to perform rdma_client function\n"); - return FAILURE; - } - } - - } else { - - /* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */ - if (ctx_init(&ctx,&user_param)) { - fprintf(stderr, " Couldn't create IB resources\n"); - return FAILURE; - } + /* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */ + if (ctx_init(&ctx, &user_param)) { + fprintf(stderr, " Couldn't create IB resources\n"); + return FAILURE; } /* Set up the Connection. */ - if (set_up_connection(&ctx,&user_param,my_dest)) { - fprintf(stderr," Unable to set up socket connection\n"); + if (set_up_connection(&ctx, &user_param, my_dest)) { + fprintf(stderr, " Unable to set up socket connection\n"); return FAILURE; } @@ -179,11 +151,11 @@ int main(int argc, char *argv[]) ctx_print_test_info(&user_param); for (i=0; i < user_param.num_of_qps; i++) - ctx_print_pingpong_data(&my_dest[i],&user_comm); + ctx_print_pingpong_data(&my_dest[i], &user_comm); /* shaking hands and gather the other side info. */ - if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) { - fprintf(stderr,"Failed to exchange data between server and clients\n"); + if (ctx_hand_shake(&user_comm, my_dest, rem_dest)) { + fprintf(stderr, "Failed to exchange data between server and clients\n"); return FAILURE; } @@ -191,62 +163,58 @@ int main(int argc, char *argv[]) for (i=0; i < user_param.num_of_qps; i++) { /* shaking hands and gather the other side info. */ - if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) { - fprintf(stderr,"Failed to exchange data between server and clients\n"); + if (ctx_hand_shake(&user_comm, &my_dest[i], &rem_dest[i])) { + fprintf(stderr, "Failed to exchange data between server and clients\n"); return FAILURE; } - ctx_print_pingpong_data(&rem_dest[i],&user_comm); + ctx_print_pingpong_data(&rem_dest[i], &user_comm); }; - if (user_param.work_rdma_cm == OFF) { - if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) { - fprintf(stderr,"\n Found Incompatibility issue with GID types.\n"); - fprintf(stderr," Please Try to use a different IP version.\n\n"); - return FAILURE; - } - } - - if (user_param.work_rdma_cm == OFF) { - if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) { - fprintf(stderr," Unable to Connect the HCA's through the link\n"); - return FAILURE; - } - } - - /* An additional handshake is required after moving qp to RTR. */ - if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) { - fprintf(stderr,"Failed to exchange data between server and clients\n"); + if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) { + fprintf(stderr, "\n Found Incompatibility issue with GID types.\n"); + fprintf(stderr, " Please Try to use a different IP version.\n\n"); return FAILURE; } - ctx_set_send_wqes(&ctx,&user_param,rem_dest); + if (ctx_connect(&ctx, rem_dest, &user_param, my_dest)) { + fprintf(stderr, " Unable to Connect the HCA's through the link\n"); + return FAILURE; + } + + /* An additional handshake is required after moving qp to RTR. */ + if (ctx_hand_shake(&user_comm, my_dest, rem_dest)) { + fprintf(stderr, "Failed to exchange data between server and clients\n"); + return FAILURE; + } + + ctx_set_send_wqes(&ctx, &user_param, rem_dest); if (user_param.output == FULL_VERBOSITY) { printf(RESULT_LINE); - printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR); + printf("%s", (user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR); printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT)); } if (user_param.test_method == RUN_ALL) { - for (i = 1; i < 24 ; ++i) { user_param.size = (uint64_t)1 << i; - if(run_iter_lat_write(&ctx,&user_param)) { - fprintf(stderr,"Test exited with Error\n"); + if(run_iter_lat_write(&ctx, &user_param)) { + fprintf(stderr, "Test exited with Error\n"); return FAILURE; } - user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); + /* user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); */ + print_report_lat(&user_param); } - } else { - - if(run_iter_lat_write(&ctx,&user_param)) { - fprintf(stderr,"Test exited with Error\n"); + } else { // user_param.test_method != RUN_ALL + if(run_iter_lat_write(&ctx, &user_param)) { + fprintf(stderr, "Test exited with Error\n"); return FAILURE; } - user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); + /* user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param); */ + print_report_lat(&user_param); } if (user_param.output == FULL_VERBOSITY) {