#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <getopt.h>
#include <limits.h>
#include <netinet/in.h>
#include <math.h>

#include "perftest_parameters_write_bw.h"

#define MAC_LEN (17)
#define ETHERTYPE_LEN (6)
#define MAC_ARR_LEN (6)
#define HEX_BASE (16)

#define inet_pton(af, src, dst) \
	(((af) == AF_INET) ? ip4addr_aton((src), (ip4_addr_t*)(dst)) : 0)

static const char *connStr[] = {"RC", "UC", "UD", "RawEth", "XRC", "DC"};
static const char *testsStr[] = {"Send", "RDMA_Write", "RDMA_Read", "Atomic"};
static const char *portStates[] = {"Nop", "Down", "Init", "Armed", "", "Active Defer"};
static const char *qp_state[] = {"OFF", "ON"};
static const char *exchange_state[] = {"Ethernet", "rdma_cm"};
static const char *atomicTypesStr[] = {"CMP_AND_SWAP", "FETCH_AND_ADD"};


// -----------------------------------------------------------------------------


const int str_link_layer(const char *str)
{
	if (strncasecmp("IB", str, 2) == 0)
		return IBV_LINK_LAYER_INFINIBAND;
	else if (strncasecmp("Ethernet", str, 8) == 0)
		return IBV_LINK_LAYER_ETHERNET;
	else
		return LINK_FAILURE;
}

static int parse_mac_from_str(char *mac, u_int8_t *addr)
{
	char tmpMac[MAC_LEN+1];
	char *tmpField;
	int fieldNum = 0;

	if (strlen(mac) != MAC_LEN) {
		fprintf(stderr, "invalid MAC length\n");
		return FAILURE;
	}
	if (addr == NULL) {
		fprintf(stderr, "invalid output addr array\n");
		return FAILURE;
	}

	strcpy(tmpMac, mac);
	tmpField = strtok(tmpMac, ":");
	while (tmpField != NULL && fieldNum < MAC_ARR_LEN) {
		char *chk;
		int tmpVal;
		tmpVal = strtoul(tmpField, &chk, HEX_BASE);
		if (tmpVal > 0xff) {
			fprintf(stderr, "field %d value %X out of range\n", fieldNum, tmpVal);
			return FAILURE;
		}
		if (*chk != 0) {
			fprintf(stderr, "Non-digit character %c (%0x) detected in field %d\n", *chk, *chk, fieldNum);
			return FAILURE;
		}
		addr[fieldNum++] = (u_int8_t) tmpVal;
		tmpField = strtok(NULL, ":");
	}
	if (tmpField != NULL || fieldNum != MAC_ARR_LEN) {
		fprintf(stderr, "MAC address longer than six fields\n");
		return FAILURE;
	}
	return SUCCESS;
}

int parse_ip_from_str(char *ip, u_int32_t *addr)
{
	return inet_pton(AF_INET, ip, addr);
}

const char *link_layer_str(int8_t link_layer)
{
	switch (link_layer) {
		case IBV_LINK_LAYER_UNSPECIFIED:
		case IBV_LINK_LAYER_INFINIBAND:
			return "IB";
		case IBV_LINK_LAYER_ETHERNET:
			return "Ethernet";
		default:
			return "Unknown";
	}
}

void flow_rules_force_dependecies(struct perftest_parameters *user_param)
{
	int min_iter_req = 0;
	if (user_param->flows != DEF_FLOWS) {
		if (user_param->is_server_port == OFF) {
			fprintf(stderr, " Flows feature works with UDP/TCP packets only for now\n");
			exit(1);
		}
		if (user_param->test_type == ITERATIONS) {
			min_iter_req = user_param->flows * user_param->flows_burst;
			if (user_param->iters / min_iter_req < 1) {
				fprintf(stderr, " Current iteration number will not complete full cycle on all flows, it need to be multiple of the product between flows and flows_burst\n");
				fprintf(stderr, " Set N*%d Iterations \n", user_param->flows * user_param->flows_burst);
				exit(FAILURE);
			}
		}
		if (user_param->tst == FS_RATE) {
			fprintf(stderr, "FS rate test not requiring flows parameter\n");
			exit(FAILURE);
		}
		if (user_param->duplex) {
			fprintf(stderr, " Flows is currently designed to work with unidir tests only\n");
			exit(FAILURE);
		}
	} else {
		if (user_param->flows_burst > 1) {
			fprintf(stderr, " Flows burst is designed to work with more then single flow\n");
			exit(FAILURE);
		}
	}
	return;
}

void get_gbps_str_by_ibv_rate(char *rate_input_value, int *rate)
{
	int i;
	for (i = 0; i < RATE_VALUES_COUNT; i++) {
		if (strcmp(rate_input_value, RATE_VALUES[i].rate_gbps_str) == 0) {
			*rate = (int)RATE_VALUES[i].rate_gbps_enum;
			return;
		}
	}
	printf("\x1b[31mThe input value for hw rate limit is not supported\x1b[0m\n");
	/* print_supported_ibv_rate_values(); */
}

static void change_conn_type(int *cptr, VerbType verb, const char *optarg)
{
	if (*cptr == RawEth)
		return;

	if (strcmp(connStr[0], optarg) == 0)
		*cptr = RC;

	else if (strcmp(connStr[1], optarg) == 0) {
		*cptr = UC;
		if (verb == READ || verb == ATOMIC) {
			fprintf(stderr, " UC connection not possible in READ/ATOMIC verbs\n");
			exit(1);
		}

	} else if (strcmp(connStr[2], optarg)==0) {
		*cptr = UD;
		if (verb != SEND) {
			fprintf(stderr, " UD connection only possible in SEND verb\n");
			exit(1);
		}
	} else if(strcmp(connStr[3], optarg)==0) {
		*cptr = RawEth;

	} else if(strcmp(connStr[4], optarg)==0) {
		/* #ifdef HAVE_XRCD */
		/* *cptr = XRC; */
		/* #else */
		fprintf(stderr, " XRC not detected in libibverbs\n");
		exit(1);
		/* #endif */
	} else if (strcmp(connStr[5], optarg)==0) {
		/* #ifdef HAVE_DC */
		/* *cptr = DC; */
		/* #else */
		fprintf(stderr, " DC not detected in libibverbs\n");
		exit(1);
		/* #endif */
	} else {
		fprintf(stderr, " Invalid Connection type . please choose from {RC, UC, UD}\n");
		exit(1);
	}
}


// -----------------------------------------------------------------------------


static int get_cache_line_size()
{
	// HermitCore's cache line size is 64 (== DEF_CACHE_LINE_SIZE)
	int size = DEF_CACHE_LINE_SIZE;

	/* int size = 0; */
	/* size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); */
	/* if (size == 0) { */
		/* #if defined(__sparc__) && defined(__arch64__) */
		/* char* file_name = */
			/* "/sys/devices/system/cpu/cpu0/l2_cache_line_size"; */
		/* #else */
		/* char* file_name = */
			/* "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size"; */
		/* #endif */

		/* FILE *fp; */
		/* char line[10]; */
		/* fp = fopen(file_name, "r"); */
		/* if (fp == NULL) { */
			/* return DEF_CACHE_LINE_SIZE; */
		/* } */
		/* if(fgets(line, 10, fp) != NULL) { */
			/* size = atoi(line); */
			/* fclose(fp); */
		/* } */
	/* } */
	/* if (size <= 0) */
		/* size = DEF_CACHE_LINE_SIZE; */

	return size;
}

static void usage(const char *argv0, VerbType verb, TestType tst, int connection_type)
{
	printf("Usage:\n");

	if (tst != FS_RATE) {
		printf(" %s start a server and wait for connection\n", argv0);
		printf(" %s <host> connect to server at <host>\n", argv0);
	} else
		printf(" %s run a server to measure FS rate \n", argv0);

	printf("\n");
	printf("Options:\n");

	if (verb != ATOMIC && connection_type != RawEth) {
		printf(" -a, --all ");
		printf(" Run sizes from 2 till 2^23\n");
	}

	if (verb == ATOMIC) {
		printf(" -A, --atomic_type=<type> ");
		printf(" type of atomic operation from {CMP_AND_SWAP, FETCH_AND_ADD} (default FETCH_AND_ADD)\n");
	}

	if (tst == BW) {
		printf(" -b, --bidirectional ");
		printf(" Measure bidirectional bandwidth (default unidirectional)\n");
	}

	if (connection_type != RawEth) {
		if (verb == SEND) {
			printf(" -c, --connection=<RC/XRC/UC/UD/DC> ");
			printf(" Connection type RC/XRC/UC/UD/DC (default RC)\n");
		} else if (verb == WRITE) {
			printf(" -c, --connection=<RC/XRC/UC/DC> ");
			printf(" Connection type RC/XRC/UC/DC (default RC)\n");
		} else if (verb == READ || verb == ATOMIC) {
			printf(" -c, --connection=<RC/XRC/DC> ");
			printf(" Connection type RC/XRC/DC (default RC)\n");
		}
	}

	if (tst == LAT) {
		printf(" -C, --report-cycles ");
		printf(" report times in cpu cycle units (default microseconds)\n");
	}

	printf(" -d, --ib-dev=<dev> ");
	printf(" Use IB device <dev> (default first device found)\n");

	printf(" -D, --duration ");
	printf(" Run test for a customized period of seconds.\n");

	if (verb != WRITE && connection_type != RawEth) {
		printf(" -e, --events ");
		printf(" Sleep on CQ events (default poll)\n");

		printf(" -X, --vector=<completion vector> ");
		printf(" Set <completion vector> used for events\n");
	}

	printf(" -f, --margin ");
	printf(" measure results within margins. (default=2sec)\n");

	printf(" -F, --CPU-freq ");
	printf(" Do not show a warning even if cpufreq_ondemand module is loaded, and cpu-freq is not on max.\n");

	if (verb == SEND && tst != FS_RATE) {
		printf(" -g, --mcg ");
		printf(" Send messages to multicast group with 1 QP attached to it.\n");
	}

	printf(" -h, --help ");
	printf(" Show this help screen.\n");

	if (tst == LAT || tst == LAT_BY_BW || tst == FS_RATE) {
		printf(" -H, --report-histogram ");
		printf(" Print out all results (default print summary only)\n");
	}

	printf(" -i, --ib-port=<port> ");
	printf(" Use port <port> of IB device (default %d)\n", DEF_IB_PORT);

	if (verb != READ && verb != ATOMIC) {
		printf(" -I, --inline_size=<size> ");
		printf(" Max size of message to be sent in inline\n");
	}

	if (tst == BW || tst == LAT_BY_BW) {
		printf(" -l, --post_list=<list size>");
		printf(" Post list of WQEs of <list size> size (instead of single post)\n");
	}

	if (tst != FS_RATE) {
		if (connection_type == RawEth) {
			printf(" -m, --mtu=<mtu> ");
			printf(" MTU size : 64 - 9600 (default port mtu)\n");
		} else {
			printf(" -m, --mtu=<mtu> ");
			printf(" MTU size : 256 - 4096 (default port mtu)\n");
		}

		if (verb == SEND) {
			printf(" -M, --MGID=<multicast_gid> ");
			printf(" In multicast, uses <multicast_gid> as the group MGID.\n");
		}
	}

	printf(" -n, --iters=<iters> ");
	printf(" Number of exchanges (at least %d, default %d)\n", MIN_ITER, ((verb == WRITE) && (tst == BW)) ? DEF_ITERS_WB : DEF_ITERS);

	if (tst == BW) {
		printf(" -N, --noPeak");
		printf(" Cancel peak-bw calculation (default with peak up to iters=20000)\n");
	}

	if (verb == READ || verb == ATOMIC) {
		printf(" -o, --outs=<num> ");
		printf(" num of outstanding read/atom(default max of device)\n");
	}

	if (tst == BW && connection_type != RawEth) {
		printf(" -O, --dualport ");
		printf(" Run test in dual-port mode.\n");
	}

	printf(" -p, --port=<port> ");
	printf(" Listen on/connect to port <port> (default %d)\n", DEF_PORT);

	if (tst == BW ) {
		printf(" -q, --qp=<num of qp's> Num of qp's(default %d)\n", DEF_NUM_QPS);
	}

	if (tst == BW) {
		printf(" -Q, --cq-mod ");
		printf(" Generate Cqe only after <--cq-mod> completion\n");
	}

	if (verb == SEND && tst != FS_RATE) {
		printf(" -r, --rx-depth=<dep> ");
		printf(" Rx queue size (default %d).", DEF_RX_SEND);
		printf(" If using srq, rx-depth controls max-wr size of the srq\n");
	}

	if (connection_type != RawEth) {
		printf(" -R, --rdma_cm ");
		printf(" Connect QPs with rdma_cm and run test on those QPs\n");
	}

	if (verb != ATOMIC) {
		printf(" -s, --size=<size> ");
		printf(" Size of message to exchange (default %d)\n", tst == LAT ? DEF_SIZE_LAT : DEF_SIZE_BW);
	}

	if (tst != FS_RATE) {
		printf(" -S, --sl=<sl> ");
		printf(" SL (default %d)\n", DEF_SL);

		if (tst == BW || tst == LAT_BY_BW) {
			printf(" -t, --tx-depth=<dep> ");
			printf(" Size of tx queue (default %d)\n", tst == LAT ? DEF_TX_LAT : DEF_TX_BW);
		}

		printf(" -T, --tos=<tos value> ");
		printf(" Set <tos_value> to RDMA-CM QPs. available only with -R flag. values 0-256 (default off)\n");
	}

	printf(" -u, --qp-timeout=<timeout> ");
	printf(" QP timeout, timeout value is 4 usec * 2 ^(timeout), default %d\n", DEF_QP_TIME);

	if (tst == LAT || tst == LAT_BY_BW || tst == FS_RATE) {
		printf(" -U, --report-unsorted ");
		printf(" (implies -H) print out unsorted results (default sorted)\n");
	}

	printf(" -V, --version ");
	printf(" Display version number\n");

	if (tst == BW) {
		printf(" -w, --limit_bw=<value> ");
		printf(" Set verifier limit for bandwidth\n");
	}

	if (connection_type != RawEth) {
		printf(" -x, --gid-index=<index> ");
		printf(" Test uses GID with GID index (Default : IB - no gid . ETH - 0)\n");
	}

	if (tst == BW) {
		printf(" -y, --limit_msgrate=<value> ");
		printf(" Set verifier limit for Msg Rate\n");
	}

	if (connection_type != RawEth) {
		printf(" -z, --com_rdma_cm ");
		printf(" Communicate with rdma_cm module to exchange data - use regular QPs\n");
	}

	/*Long flags*/
	putchar('\n');

	printf(" --cpu_util ");
	printf(" Show CPU Utilization in report, valid only in Duration mode \n");

	if (tst != FS_RATE) {
		printf(" --dlid ");
		printf(" Set a Destination LID instead of getting it from the other side.\n");
	}

	if (connection_type != RawEth) {
		printf(" --dont_xchg_versions ");
		printf(" Do not exchange versions and MTU with other side \n");
	}

	if (tst != FS_RATE) {
		printf(" --force-link=<value> ");
		printf(" Force the link(s) to a specific type: IB or Ethernet.\n");
	}

	if (verb != WRITE) {
		printf(" --inline_recv=<size> ");
		printf(" Max size of message to be sent in inline receive\n");
	}

	if (connection_type != RawEth) {
		printf(" --ipv6 ");
		printf(" Use IPv6 GID. Default is IPv4\n");
	}

	if (tst == LAT) {
		printf(" --latency_gap=<delay_time> ");
		printf(" delay time between each post send\n");
	}

	if (connection_type != RawEth) {
		printf(" --mmap=file ");
		printf(" Use an mmap'd file as the buffer for testing P2P transfers.\n");
		printf(" --mmap-offset=<offset> ");
		printf(" Use an mmap'd file as the buffer for testing P2P transfers.\n");
	}

	if (tst == BW) {
		printf(" --mr_per_qp ");
		printf(" Create memory region for each qp.\n");
	}

	#if defined HAVE_EX_ODP || defined HAVE_EXP_ODP
	printf(" --odp ");
	printf(" Use On Demand Paging instead of Memory Registration.\n");
	#endif

	printf(" --output=<units>");
	printf(" Set verbosity output level: bandwidth , message_rate, latency \n");
	printf(" Latency measurement is Average calculation \n");

	if (tst != FS_RATE) {
		printf(" --perform_warm_up");
		printf(" Perform some iterations before start measuring in order to warming-up memory cache, valid in Atomic, Read and Write BW tests\n");

		printf(" --pkey_index=<pkey index> PKey index to use for QP\n");
	}

	if ( tst == BW ) {
		printf(" --report-both ");
		printf(" Report RX & TX results separately on Bidirectinal BW tests\n");

		printf(" --report_gbits ");
		printf(" Report Max/Average BW of test in Gbit/sec (instead of MB/sec)\n");

		if (connection_type != RawEth) {
			printf(" --report-per-port ");
			printf(" Report BW data on both ports when running Dualport and Duration mode\n");
		}

		printf(" --reversed ");
		printf(" Reverse traffic direction - Server send to client\n");

		printf(" --run_infinitely ");
		printf(" Run test forever, print results every <duration> seconds\n");
	}

	if (connection_type != RawEth) {
		printf(" --retry_count=<value> ");
		printf(" Set retry count value in rdma_cm mode\n");
	}

	if (tst != FS_RATE) {
		printf(" --tclass=<value> ");
		printf(" Set the Traffic Class in GRH (if GRH is in use)\n");

		#ifdef HAVE_CUDA
		printf(" --use_cuda ");
		printf(" Use CUDA lib for GPU-Direct testing.\n");
		#endif

		#ifdef HAVE_VERBS_EXP
		printf(" --use_exp ");
		printf(" Use Experimental verbs in data path. Default is OFF.\n");
		#endif

		printf(" --use_hugepages ");
		printf(" Use Hugepages instead of contig, memalign allocations.\n");


		#ifdef HAVE_ACCL_VERBS
		printf(" --use_res_domain ");
		printf(" Use shared resource domain\n");

		printf(" --verb_type=<option> ");
		printf(" Set verb type: normal, accl. Default is normal.\n");
		#endif
	}

	if (tst == BW || tst == LAT_BY_BW) {
		printf(" --wait_destroy=<seconds> ");
		printf(" Wait <seconds> before destroying allocated resources (QP/CQ/PD/MR..)\n");

		printf("\n Rate Limiter:\n");
		printf(" --burst_size=<size>");
		printf(" Set the amount of messages to send in a burst when using rate limiter\n");

		printf(" --rate_limit=<rate>");
		printf(" Set the maximum rate of sent packages. default unit is [Gbps]. use --rate_units to change that.\n");

		printf(" --rate_units=<units>");
		printf(" [Mgp] Set the units for rate limit to MBps (M), Gbps (g) or pps (p). default is Gbps (g).\n");
		printf(" Note (1): pps not supported with HW limit.\n");
		printf(" Note (2): When using PP rate_units is forced to Kbps.\n");

		printf(" --rate_limit_type=<type>");
		printf(" [HW/SW/PP] Limit the QP's by HW, PP or by SW. Disabled by default. When rate_limit Not is specified HW limit is Default.\n");
		printf(" Note (1) in Latency under load test SW rate limit is forced\n");

	}
	#if defined HAVE_OOO_ATTR || defined HAVE_EXP_OOO_ATTR
	printf(" --use_ooo ");
	printf(" Use out of order data placement\n");
	#endif
	putchar('\n');
}

/******************************************************************************
  usage
 ******************************************************************************/
void usage_raw_ethernet(TestType tst)

{
	printf(" Raw Ethernet options :\n");
	printf(" -B, --source_mac ");
	printf(" source MAC address by this format XX:XX:XX:XX:XX:XX **MUST** be entered \n");

	printf(" -E, --dest_mac ");
	printf(" destination MAC address by this format XX:XX:XX:XX:XX:XX **MUST** be entered \n");

	printf(" -G, --use_rss ");
	printf(" use RSS on server side. need to open 2^x qps (using -q flag. default is -q 2). open 2^x clients that transmit to this server\n");

	printf(" -J, --dest_ip ");
	#ifdef HAVE_IPV6
	printf(" destination ip address by this format X.X.X.X for IPv4 or X:X:X:X:X:X for IPv6 (using to send packets with IP header)\n");
	#else
	printf(" destination ip address by this format X.X.X.X (using to send packets with IP header)\n");
	#endif

	printf(" -j, --source_ip ");
	#ifdef HAVE_IPV6
	printf(" source ip address by this format X.X.X.X for IPv4 or X:X:X:X:X:X for IPv6 (using to send packets with IP header)\n");
	#else
	printf(" source ip address by this format X.X.X.X (using to send packets with IP header)\n");
	#endif

	printf(" -K, --dest_port ");
	printf(" destination port number (using to send packets with UDP header as default, or you can use --tcp flag to send TCP Header)\n");

	printf(" -k, --source_port ");
	printf(" source port number (using to send packets with UDP header as default, or you can use --tcp flag to send TCP Header)\n");

	printf(" -Y, --ethertype ");
	printf(" ethertype value in the ethernet frame by this format 0xXXXX\n");

	printf(" -Z, --server ");
	printf(" choose server side for the current machine (--server/--client must be selected )\n");

	printf(" --vlan_en ");
	printf(" insert vlan tag in ethernet header.\n");

	printf(" --vlan_pcp ");
	printf(" specify vlan_pcp value for vlan tag, 0~7. 8 means different vlan_pcp for each packet\n");

	if (tst != FS_RATE) {
		printf(" -P, --client ");
		printf(" choose client side for the current machine (--server/--client must be selected)\n");

		printf(" -v, --mac_fwd ");
		printf(" run mac forwarding test \n");

		#ifdef HAVE_SCATTER_FCS
		printf(" --disable_fcs ");
		printf(" Disable Scatter FCS feature. (Scatter FCS is enabled by default when using --use_exp flag). \n");
		#endif

		printf(" --flows");
		printf(" set number of TCP/UDP flows, starting from <src_port, dst_port>. \n");

		printf(" --flows_burst");
		printf(" set number of burst size per TCP/UDP flow. \n");

		printf(" --promiscuous");
		printf(" run promiscuous mode.\n");

		printf(" --reply_every ");
		printf(" in latency test, receiver pong after number of received pings\n");

		#if defined HAVE_SNIFFER || defined HAVE_SNIFFER_EXP
		printf(" --sniffer");
		printf(" run sniffer mode.\n");
		#endif

		printf(" --flow_label ");
		printf(" IPv6 flow label\n");

	}

	printf(" --tcp ");
	printf(" send TCP Packets. must include IP and Ports information.\n");

	#ifdef HAVE_IPV6
	printf(" --raw_ipv6 ");
	printf(" send IPv6 Packets.\n");
	#endif

	printf("\n");
}
static void init_perftest_params(struct perftest_parameters *user_param)
{
	user_param->port                 = DEF_PORT;
	user_param->ib_port              = DEF_IB_PORT;
	user_param->ib_port2             = DEF_IB_PORT2;
	user_param->link_type            = LINK_UNSPEC;
	user_param->link_type2           = LINK_UNSPEC;
	user_param->size                 = (user_param->tst == BW ) ? DEF_SIZE_BW : DEF_SIZE_LAT;
	user_param->tx_depth             = (user_param->tst == BW || user_param->tst == LAT_BY_BW ) ? DEF_TX_BW : DEF_TX_LAT;
	user_param->qp_timeout           = DEF_QP_TIME;
	user_param->test_method          = RUN_REGULAR;
	user_param->cpu_freq_f           = OFF;
	user_param->connection_type      = (user_param->connection_type == RawEth) ? RawEth : RC;
	if (user_param->connection_type == RawEth)
		user_param->machine = UNCHOSEN;
	user_param->use_event            = OFF;
	user_param->eq_num               = 0;
	user_param->use_eq_num           = OFF;
	user_param->num_of_qps           = DEF_NUM_QPS;
	user_param->gid_index            = DEF_GID_INDEX;
	user_param->gid_index2           = DEF_GID_INDEX;
	user_param->use_gid_user         = 0;
	user_param->inline_size          = DEF_INLINE;
	user_param->use_mcg              = OFF;
	user_param->use_rdma_cm          = OFF;
	user_param->work_rdma_cm         = OFF;
	user_param->rx_depth             = user_param->verb == SEND ? DEF_RX_SEND : DEF_RX_RDMA;
	user_param->duplex               = OFF;
	user_param->noPeak               = OFF;
	user_param->cq_mod               = DEF_CQ_MOD;
	user_param->iters                = (user_param->tst == BW && user_param->verb == WRITE) ? DEF_ITERS_WB : DEF_ITERS;
	user_param->dualport             = OFF;
	user_param->post_list            = 1;
	user_param->use_srq              = OFF;
	user_param->use_xrc              = OFF;
	user_param->use_rss              = OFF;
	user_param->srq_exists           = OFF;
	user_param->duration             = DEF_DURATION;
	user_param->margin               = DEF_INIT_MARGIN;
	user_param->test_type            = ITERATIONS;
	user_param->state                = START_STATE;
	user_param->tos                  = DEF_TOS;
	user_param->mac_fwd              = OFF;
	user_param->report_fmt           = MBS;
	user_param->report_both          = OFF;
	user_param->is_reversed          = OFF;
	user_param->is_limit_bw          = OFF;
	user_param->limit_bw             = 0;
	user_param->is_limit_msgrate     = OFF;
	user_param->limit_msgrate        = 0;
	user_param->pkey_index           = 0;
	user_param->raw_qos              = 0;
	user_param->inline_recv_size     = 0;
	user_param->tcp                  = 0;
	user_param->burst_size           = 0;
	user_param->rate_limit           = 0;
	user_param->valid_hw_rate_limit  = 0;
	user_param->rate_units           = GIGA_BIT_PS;
	user_param->rate_limit_type      = DISABLE_RATE_LIMIT;
	user_param->is_rate_limit_type   = 0;
	user_param->output               = -1;
	user_param->use_cuda             = 0;
	user_param->mmap_file            = NULL;
	user_param->mmap_offset          = 0;
	user_param->iters_per_port[0]    = 0;
	user_param->iters_per_port[1]    = 0;
	user_param->wait_destroy         = 0;
	user_param->is_old_raw_eth_param = 0;
	user_param->is_new_raw_eth_param = 0;
	user_param->reply_every          = 1;
	user_param->vlan_en              = OFF;
	user_param->vlan_pcp             = 1;
	/* user_param->print_eth_func = &print_ethernet_header; */

	if (user_param->tst == LAT) {
		user_param->r_flag->unsorted  = OFF;
		user_param->r_flag->histogram = OFF;
		user_param->r_flag->cycles    = OFF;
	}
	if (user_param->verb == ATOMIC) {
		user_param->atomicType = FETCH_AND_ADD;
		user_param->size       = DEF_SIZE_ATOMIC;
	}

	user_param->cpu_util             = 0;
	user_param->cpu_util_data.enable = 0;
	user_param->retry_count          = DEF_RETRY_COUNT;
	user_param->dont_xchg_versions   = 0;
	user_param->use_exp              = 0;
	user_param->ipv6                 = 0;
	user_param->report_per_port      = 0;
	user_param->use_odp              = 0;
	user_param->use_hugepages        = 0;
	user_param->use_promiscuous      = 0;
	user_param->use_sniffer          = 0;
	user_param->check_alive_exited   = 0;
	user_param->raw_mcast            = 0;
	user_param->masked_atomics       = 0;
	user_param->cache_line_size      = get_cache_line_size();
	user_param->cycle_buffer         = DEF_PAGE_SIZE;
	user_param->verb_type            = NORMAL_INTF;
	user_param->is_exp_cq            = 0;
	user_param->is_exp_qp            = 0;
	user_param->use_res_domain       = 0;
	user_param->mr_per_qp            = 0;
	user_param->dlid                 = 0;
	user_param->traffic_class        = 0;
	user_param->disable_fcs          = 0;
	user_param->flows                = DEF_FLOWS;
	user_param->flows_burst          = 1;
	user_param->perform_warm_up      = 0;
	user_param->use_ooo              = 0;
}

static int set_link_layer(struct ibv_context *context, struct perftest_parameters *params)
{
	struct ibv_port_attr port_attr;
	int8_t curr_link = params->link_type;

	if (ibv_query_port(context, params->ib_port, &port_attr)) {
		fprintf(stderr, " Unable to query port %d attributes\n", params->ib_port);
		return FAILURE;
	}

	if (curr_link == LINK_UNSPEC)
		params->link_type = port_attr.link_layer;

	if (port_attr.state != IBV_PORT_ACTIVE) {
		fprintf(stderr, " Port number %d state is %s\n"
				, params->ib_port
				, portStates[port_attr.state]);
		return FAILURE;
	}

	if (strcmp("Unknown", link_layer_str(params->link_type)) == 0) {
		fprintf(stderr, "Link layer on port %d is Unknown\n", params->ib_port);
		return FAILURE;
	}

	if (params->dualport == ON) {
		curr_link = params->link_type2;
		if (ibv_query_port(context, params->ib_port2, &port_attr)) {
			fprintf(stderr, " Unable to query port %d attributes\n", params->ib_port2);
			return FAILURE;
		}

		if (curr_link == LINK_UNSPEC)
			params->link_type2 = port_attr.link_layer;

		if (port_attr.state != IBV_PORT_ACTIVE) {
			fprintf(stderr, " Port number %d state is %s\n"
				, params->ib_port2
				, portStates[port_attr.state]);
			return FAILURE;
		}

		if (strcmp("Unknown", link_layer_str(params->link_type2)) == 0) {
			fprintf(stderr, "Link layer on port %d is Unknown\n", params->ib_port2);
			return FAILURE;
		}
	}

	return SUCCESS;
}

static void ctx_set_max_inline(struct ibv_context *context, struct perftest_parameters *user_param)
{
	enum ctx_device current_dev = ib_dev_name(context);

	if (current_dev == UNKNOWN || current_dev == DEVICE_ERROR) {

		if (user_param->inline_size != DEF_INLINE) {
			printf(RESULT_LINE);
			fprintf(stderr, "Device not recognized to implement inline feature. Disabling it\n");
		}
		user_param->inline_size = 0;
		return;
	}

	if (user_param->inline_size == DEF_INLINE) {

		if (user_param->tst ==LAT) {

			switch(user_param->verb) {

				case WRITE: user_param->inline_size = (user_param->connection_type == DC)? DEF_INLINE_DC : DEF_INLINE_WRITE; break;
				case SEND : user_param->inline_size = (user_param->connection_type == DC)? DEF_INLINE_DC : (user_param->connection_type == UD)? DEF_INLINE_SEND_UD :
					    ((user_param->connection_type == XRC) ? DEF_INLINE_SEND_XRC : DEF_INLINE_SEND_RC_UC) ; break;
				default : user_param->inline_size = 0;
			}

		} else {
			user_param->inline_size = 0;
		}
	}

	return;
}

static int ctx_set_out_reads(struct ibv_context *context, int num_user_reads)
{
	int max_reads = 0;
	struct ibv_device_attr attr;

	if (!ibv_query_device(context, &attr)) {
		max_reads = attr.max_qp_rd_atom;
	}

	if (num_user_reads > max_reads) {
		printf(RESULT_LINE);
		fprintf(stderr, " Number of outstanding reads is above max = %d\n", max_reads);
		fprintf(stderr, " Changing to that max value\n");
		num_user_reads = max_reads;
	}
	else if (num_user_reads <= 0) {
		num_user_reads = max_reads;
	}

	return num_user_reads;
}

static int ctx_chk_pkey_index(struct ibv_context *context, int pkey_idx)
{
	int idx = 0;
	struct ibv_device_attr attr;

	if (!ibv_query_device(context, &attr)) {
		if (pkey_idx > attr.max_pkeys - 1) {
			printf(RESULT_LINE);
			fprintf(stderr, " Specified PKey Index, %i, greater than allowed max, %i\n", pkey_idx, attr.max_pkeys - 1);
			fprintf(stderr, " Changing to 0\n");
			idx = 0;
		} else
			idx = pkey_idx;
	} else {
		fprintf(stderr, " Unable to validata PKey Index, changing to 0\n");
		idx = 0;
	}

	return idx;

}

void set_raw_eth_parameters(struct perftest_parameters *user_param)
{
	int i;

	if (user_param->is_new_raw_eth_param == 1 && user_param->is_old_raw_eth_param == 1) {
		printf(RESULT_LINE);
		fprintf(stderr, " Invalid Command line.\nMix of source with local|remote and dest with local|remote is not supported.\n");
		fprintf(stderr, "For L2 tests you must enter local and remote mac by this format --local_mac XX:XX:XX:XX:XX:XX --remote_mac XX:XX:XX:XX:XX:XX\n");
		fprintf(stderr, "For L3 tests You must add also local and remote ip by this format --local_ip X.X.X.X --remote_ip X.X.X.X\n");
		fprintf(stderr, "For L4 you need to add also local and remote port by this format --local_port XXXX --remote_port XXXX\n");
		exit(1);
	}

	if (user_param->is_new_raw_eth_param) {
		for (i = 0; i < MAC_ARR_LEN; i++) {
			user_param->source_mac[i] = user_param->local_mac[i];
			user_param->dest_mac[i] = user_param->remote_mac[i];
		}

		if (user_param->machine == SERVER) {
			user_param->server_ip   = user_param->local_ip;
			user_param->client_ip   = user_param->remote_ip;
			user_param->server_port = user_param->local_port;
			user_param->client_port = user_param->remote_port;
		} else if (user_param->machine == CLIENT) {
			user_param->server_ip   = user_param->remote_ip;
			user_param->client_ip   = user_param->local_ip;
			user_param->server_port = user_param->remote_port;
			user_param->client_port = user_param->local_port;
		}
	}
}

static void force_dependecies(struct perftest_parameters *user_param)
{
	/*Additional configuration and assignments.*/
	if (user_param->test_type == ITERATIONS) {
		if (user_param->tx_depth > user_param->iters) {
			user_param->tx_depth = user_param->iters;
		}

		if (user_param->verb == SEND && user_param->rx_depth > user_param->iters) {
			user_param->rx_depth = user_param->iters;
		}

		if (user_param->connection_type == UD || user_param->connection_type == UC) {
			if (user_param->rx_depth == DEF_RX_SEND) {
				user_param->rx_depth = (user_param->iters < UC_MAX_RX) ? user_param->iters : UC_MAX_RX;
			}
		}
	}

	if (user_param->tst == LAT_BY_BW && user_param->rate_limit_type == DISABLE_RATE_LIMIT) {
		if (user_param->output == FULL_VERBOSITY)
			printf("rate_limit type is forced to SW.\n");
		user_param->rate_limit_type = SW_RATE_LIMIT;
	}

	if (user_param->cq_mod > user_param->tx_depth) {
		user_param->cq_mod = user_param->tx_depth;
	}

	if (user_param->verb == READ || user_param->verb == ATOMIC)
		user_param->inline_size = 0;

	if (user_param->test_method == RUN_ALL)
		user_param->size = MAX_SIZE;

	if (user_param->verb == ATOMIC && user_param->size != DEF_SIZE_ATOMIC) {
		user_param->size = DEF_SIZE_ATOMIC;
	}

	if (user_param->use_srq && user_param->verb != SEND) {
		printf(RESULT_LINE);
		printf(" Using SRQ only avavilible in SEND tests.\n");
		exit (1);
	}

	if (user_param->dualport == ON) {

		user_param->num_of_qps *= 2;
		if (user_param->tst != BW) {
			printf(" Dual-port mode only supports BW tests.\n");
			exit (1);
		}

		if (user_param->use_mcg){
			printf(" Dual-port mode not supported in multicast feature\n");
			exit (1);
		}
		if (user_param->link_type != LINK_UNSPEC)
			user_param->link_type2 = user_param->link_type;
	}

	if (user_param->post_list > 1) {
		user_param->cq_mod = user_param->post_list;
		printf(RESULT_LINE);
		printf("Post List requested - CQ moderation will be the size of the post list\n");
	}

	if (user_param->test_type==DURATION) {

		/* When working with Duration, iters=0 helps us to satisfy loop cond. in run_iter_bw.
		We also use it for "global" counter of packets.
		*/
		user_param->iters = 0;
		user_param->noPeak = ON;

		if (user_param->use_event) {
			printf(RESULT_LINE);
			fprintf(stderr, "Duration mode doesn't work with events.\n");
			exit(1);
		}

		if (user_param->test_method == RUN_ALL) {
			printf(RESULT_LINE);
			fprintf(stderr, "Duration mode currently doesn't support running on all sizes.\n");
			exit(1);
		}
		if (user_param->cpu_util) {
			user_param->cpu_util_data.enable = 1;
		}
	}

	if ( (user_param->test_type != DURATION) && user_param->cpu_util ) {
		printf(RESULT_LINE);
		fprintf(stderr, " CPU Utilization works only with Duration mode.\n");
	}

	if (user_param->connection_type == RawEth) {

		if (user_param->test_method == RUN_ALL) {
			fprintf(stderr, "Raw Ethernet tests do not support -a / --all flag.\n");
			exit(1);
		}

		if (user_param->use_gid_user) {
			fprintf(stderr, " GID index isn't supported for Raw Ethernet tests\n");
			exit(1);
		}

		if (user_param->mmap_file != NULL || user_param->mmap_offset) {
			fprintf(stderr, " mmaped files aren't supported for Raw Ethernet tests\n");
			exit(1);
		}

		if(user_param->machine == UNCHOSEN) {
			printf(RESULT_LINE);
			fprintf(stderr, " Invalid Command line.\n you must choose test side --client or --server\n");
			exit(1);
		}

		/* Verify the packet */
		if(user_param->is_source_mac == OFF) {
			printf(RESULT_LINE);
			fprintf(stderr, " Invalid Command line.\n you must enter source mac by this format -B XX:XX:XX:XX:XX:XX\n");
			exit(1);
		}

		if(user_param->is_dest_mac == OFF && (user_param->tst == LAT || (user_param->machine == CLIENT && !user_param->raw_mcast))) {
			printf(RESULT_LINE);
			fprintf(stderr, " Invalid Command line.\n you must enter dest mac by this format -E XX:XX:XX:XX:XX:XX\n");
			exit(1);
		}

		if((user_param->is_server_port == ON && user_param->is_client_port == OFF) || (user_param->is_server_port == OFF && user_param->is_client_port == ON)) {
			printf(RESULT_LINE);
			fprintf(stderr, " Invalid Command line.\n if you would like to send UDP header, \n you must enter server&client port --server_port X --client_port X\n");
			exit(1);
		}

		if ((user_param->is_server_port == ON) && (user_param->is_server_ip == OFF || user_param->is_client_ip == OFF)) {
			printf(RESULT_LINE);
			fprintf(stderr, " Invalid Command line.\nPlease provide source_ip and/or dest_ip when using UDP\n");
			exit(1);
		}
		/* UDP packet is ok by now. check tcp flag */
		if (user_param->tcp == ON && user_param->is_server_port == OFF) {
			printf(RESULT_LINE);
			fprintf(stderr, "Invalid Command line.\nPlease provide UDP information (IP & UDP Port src/dest) in order to use TCP\n");
			exit(1);
		}

		/* Mac forwarding dependencies */
		if (user_param->duplex == OFF && user_param->mac_fwd == ON) {
			printf("mac_fwd should run in duplex mode only. changing to duplex mode.\n");
			user_param->duplex = ON;
		}
		if (user_param->mac_fwd == ON && user_param->cq_mod >= user_param->rx_depth) {
			fprintf(stderr, " CQ moderation can't be grater than rx depth.\n");
			user_param->cq_mod = user_param->rx_depth < user_param->tx_depth ? user_param->rx_depth : user_param->tx_depth;
			fprintf(stderr, " Changing CQ moderation to min( rx depth , tx depth) = %d.\n", user_param->cq_mod);
		}

		if (user_param->raw_mcast && user_param->duplex) {
			fprintf(stderr, " Multicast feature works on unidirectional traffic only\n");
			exit(1);
		}

		flow_rules_force_dependecies(user_param);
	}

	if (user_param->use_mcg && user_param->gid_index == -1) {
		user_param->gid_index = 0;
	}


	if (user_param->verb == ATOMIC && user_param->connection_type == DC) {
		printf(RESULT_LINE);
		fprintf(stderr, " ATOMIC tests don't support DC transport\n");
		exit(1);
	}

	if (user_param->tos != DEF_TOS && user_param->connection_type != RawEth) {
		fprintf(stdout, " TOS only valid for rdma_cm based QP and RawEth QP \n");
		exit(1);
	}

	if (user_param->use_mcg) {

		if (user_param->connection_type != UD)
			user_param->connection_type = UD;

		if (user_param->duplex) {
			fprintf(stdout, "Bidirectional mode not supported in multicast\n");
			exit (1);
		}

		if (user_param->num_of_qps > 1) {
			fprintf(stdout, "Only 1 QP supported in multicast\n");
			exit(1);
		}
	}

	if(user_param->verb == ATOMIC && user_param->use_odp) {
		printf(RESULT_LINE);
		fprintf(stderr, " ODP does not support ATOMICS for now\n");
		exit(1);
	}

	if (user_param->verb == SEND && user_param->tst == BW && user_param->machine == SERVER && !user_param->duplex )
		user_param->noPeak = ON;

	/* Run infinitely dependencies */
	if (user_param->test_method == RUN_INFINITELY) {
		user_param->noPeak = ON;
		user_param->test_type = DURATION;
		if (user_param->use_event) {
			printf(RESULT_LINE);
			fprintf(stderr, " run_infinitely does not support events feature yet.\n");
			exit(1);
		}

		if (user_param->tst == LAT) {
			printf(RESULT_LINE);
			fprintf(stderr, " run_infinitely exists only in BW tests for now.\n");
			exit(1);

		}

		if (user_param->duplex && user_param->verb == SEND) {
			printf(RESULT_LINE);
			fprintf(stderr, " run_infinitely mode is not supported in SEND Bidirectional BW test\n");
			exit(1);
		}
		if (user_param->rate_limit_type != DISABLE_RATE_LIMIT) {
			printf(RESULT_LINE);
			fprintf(stderr, " run_infinitely does not support rate limit feature yet\n");
			exit(1);
		}
	}

	if (user_param->connection_type == DC && !user_param->use_srq)
		user_param->use_srq = 1;

	/* XRC Part */
	if (user_param->connection_type == XRC) {
		user_param->use_xrc = ON;
		user_param->use_srq = ON;
	}

	#ifndef HAVE_RSS_EXP
	if (user_param->use_rss) {
		printf(RESULT_LINE);
		fprintf(stderr, " RSS feature is not available in libibverbs\n");
		exit(1);
	}
	#endif

	if ((user_param->use_srq && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex == ON)) || user_param->use_xrc)
		user_param->srq_exists = 1;

	if (user_param->burst_size > 0) {
		if (user_param->rate_limit_type == DISABLE_RATE_LIMIT && user_param->tst != LAT_BY_BW ) {
			printf(RESULT_LINE);
			fprintf(stderr, " Can't enable burst mode when rate limiter is off\n");
			exit(1);
		}
	}

	if (user_param->burst_size <= 0) {
		if (user_param->rate_limit_type == SW_RATE_LIMIT)
			fprintf(stderr, " Setting burst size to tx depth = %d\n", user_param->tx_depth);
		user_param->burst_size = user_param->tx_depth;
	}

	if (user_param->rate_limit_type == SW_RATE_LIMIT) {
		if (user_param->tst != BW || user_param->verb == ATOMIC || (user_param->verb == SEND && user_param->duplex)) {
			printf(RESULT_LINE);
			fprintf(stderr, "SW Rate limiter cann't be executed on non-BW, ATOMIC or bidirectional SEND tests\n");
			exit(1);
		}
	} else if (user_param->rate_limit_type == HW_RATE_LIMIT) {
		double rate_limit_gbps = 0;
		switch (user_param->rate_units) {
			case MEGA_BYTE_PS:
				rate_limit_gbps =((double)(((user_param->rate_limit)*8*1024*1024) / 1000000000));
				break;
			case GIGA_BIT_PS:
				rate_limit_gbps = user_param->rate_limit;
				break;
			case PACKET_PS:
				printf(RESULT_LINE);
				fprintf(stderr, " Failed: pps rate limit units is not supported when setting HW rate limit\n");
				exit(1);
			default:
				printf(RESULT_LINE);
				fprintf(stderr, " Failed: Unknown rate limit units\n");
				exit(1);
		}
		if (rate_limit_gbps > 0) {
			int rate_to_set = -1;
			get_gbps_str_by_ibv_rate(user_param->rate_limit_str, &rate_to_set);
			if (rate_to_set == -1) {
				printf(RESULT_LINE);
				fprintf(stderr, " Failed: Unknown rate limit value\n");
				exit(1);
			}
			user_param->valid_hw_rate_limit = rate_to_set;
		}
	} else if (user_param->rate_limit_type == PP_RATE_LIMIT) {
		if (user_param->rate_limit < 0) {
			fprintf(stderr, " Must specify a rate limit when using Packet Pacing.\n Please add --rate_limit=<limit>.\n");
			exit(1);
		}
		if (user_param->connection_type != RawEth) {
			fprintf(stderr, "Packet Pacing is only supported for Raw Ethernet.\n");
			exit(1);
		}

		if (user_param->rate_units != MEGA_BYTE_PS) {
			fprintf(stderr, "Packet Pacing only supports MEGA_BYTE_PS.\n");
			exit(1);
		}

		user_param->rate_limit = user_param->rate_limit * 8 * 1024;
	}

	if (user_param->tst == LAT_BY_BW) {
		if ( user_param->test_type == DURATION) {
			fprintf(stderr, "Latency under load test is currently support iteration mode only.\n");
			exit(1);
		}
		if (user_param->num_of_qps > 1) {
			fprintf(stderr, "Multi QP is not supported in LAT under load test\n");
			exit(1);
		}
		if (user_param->duplex) {
			fprintf(stderr, "Bi-Dir is not supported in LAT under load test\n");
			exit(1);
		}
		if(user_param->output != FULL_VERBOSITY && user_param->output != OUTPUT_LAT) {
			printf(RESULT_LINE);
			fprintf(stderr, " Output verbosity level for BW can be latency\n");
			exit(1);
		}
	}

	if (user_param->output != FULL_VERBOSITY) {
		if (user_param->tst == BW && !(user_param->output == OUTPUT_BW || user_param->output == OUTPUT_MR)) {
			printf(RESULT_LINE);
			fprintf(stderr, " Output verbosity level for BW can be: bandwidth, message_rate\n");
			exit(1);
		}

		if (user_param->tst == LAT && !(user_param->output == OUTPUT_LAT)) {
			printf(RESULT_LINE);
			fprintf(stderr, " Output verbosity level for LAT can be: latency\n");
			exit(1);
		}
	}

	if ( (user_param->latency_gap > 0) && user_param->tst != LAT ) {
		printf(RESULT_LINE);
		fprintf(stderr, " Latency gap feature is only for latency tests\n");
		exit(1);
	}

	if ( user_param->test_type == DURATION && user_param->margin == DEF_INIT_MARGIN) {
		user_param->margin = user_param->duration / 4;
	}

	if ( (user_param->connection_type == UD) && (user_param->inline_size > MAX_INLINE_UD) ) {
		printf(RESULT_LINE);
		fprintf(stderr, "Setting inline size to %d (Max inline size in UD)\n", MAX_INLINE_UD);
		user_param->inline_size = MAX_INLINE_UD;
	}

	if (user_param->report_per_port && (user_param->test_type != DURATION || !user_param->dualport)) {
		printf(RESULT_LINE);
		fprintf(stderr, "report per port feature work only with Duration and Dualport\n");
		exit(1);
	}

	/* WA for a bug when rx_depth is odd in SEND */
	if (user_param->verb == SEND && (user_param->rx_depth % 2 == 1) && user_param->test_method == RUN_REGULAR)
		user_param->rx_depth += 1;

	if (user_param->test_type == ITERATIONS && user_param->iters > 20000 && user_param->noPeak == OFF && user_param->tst == BW)
		user_param->noPeak = ON;

	if (!(user_param->duration > 2*user_param->margin)) {
		printf(RESULT_LINE);
		fprintf(stderr, "please check that DURATION > 2*MARGIN\n");
		exit(1);
	}

	if((user_param->use_event == OFF) && user_param->use_eq_num == ON) {
		fprintf(stderr, " Events must be enabled to select a completion vector\n");
		exit(1);
	}

	/* #ifdef HAVE_ACCL_VERBS */
	/* if (user_param->verb_type != NORMAL_INTF || user_param->use_res_domain) { */
		/* user_param->is_exp_cq = 1; */
		/* user_param->use_exp = 1; */
	/* } */

	/* if (user_param->verb_type == ACCL_INTF) { */
		/* if (user_param->connection_type != RC && */
			/* user_param->connection_type != UC && user_param->connection_type != RawEth) { */
			/* fprintf(stderr, "Accelerated verbs support RC/UC/RAW_ETH connections only.\n"); */
			/* exit(1); */
		/* } */

		/* if (user_param->verb != SEND) { */
			/* fprintf(stderr, "Accelerated verbs support SEND opcode only.\n"); */
			/* exit(1); */
		/* } */

		/* if (user_param->num_of_qps > 1) { */
			/* fprintf(stderr, "Accelerated verbs in perftest support only 1 qp for now.\n"); */
			/* exit(1); */
		/* } */

		/* if (user_param->tst != BW) { */
			/* fprintf(stderr, "Accelerated verbs in perftest supports only BW tests for now.\n"); */
			/* exit(1); */
		/* } */

		/* if (user_param->duplex) { */
			/* fprintf(stderr, "Accelerated verbs in perftest supports only unidir tests for now\n"); */
			/* exit(1); */
		/* } */

		/* if (user_param->test_method == RUN_INFINITELY) { */
			/* fprintf(stderr, "Accelerated verbs in perftest does not support Run Infinitely mode for now\n"); */
			/* exit(1); */
		/* } */
	/* } */
	/* if (user_param->perform_warm_up && */
			/* !(user_param->tst == BW && */
			/* (user_param->verb == ATOMIC || user_param->verb == WRITE || user_param->verb == READ))) { */
		/* fprintf(stderr, "Perform warm up is available in ATOMIC, READ and WRITE BW tests only"); */
		/* exit(1); */
	/* } */
	/* #endif */

	return;
}

static float calc_cpu_util(struct perftest_parameters *user_param)
{
	long long ustat_diff, idle_diff;
	ustat_diff = user_param->cpu_util_data.ustat[1] - user_param->cpu_util_data.ustat[0];
	idle_diff = user_param->cpu_util_data.idle[1] - user_param->cpu_util_data.idle[0];

	if ((ustat_diff + idle_diff) != 0)
		return ((float)ustat_diff / (ustat_diff + idle_diff)) * 100;
	else
		return 0;
}

const char *transport_str(enum ibv_transport_type type)
{
	switch (type) {
		case IBV_TRANSPORT_IB:
			return "IB";
			break;
		case IBV_TRANSPORT_IWARP:
			return "IW";
			break;
		default:
			return "Unknown";
	}
}

static int parse_ethertype_from_str(char *ether_str, uint16_t *ethertype_val)
{
	if (strlen(ether_str) != ETHERTYPE_LEN) {
		fprintf(stderr, "invalid ethertype length\n");
		return FAILURE;
	}
	*ethertype_val = strtoul(ether_str, NULL, HEX_BASE);
	if (!*ethertype_val)
		return FAILURE;
	return SUCCESS;
}

int parse_ip6_from_str(char *ip6, struct in6_addr *addr)
{
	return inet_pton(AF_INET6, ip6, addr);
}

static int cycles_compare(const void *aptr, const void *bptr)
{
	const cycles_t *a = aptr;
	const cycles_t *b = bptr;
	if (*a < *b) return -1;
	if (*a > *b) return 1;

	return 0;
}

static inline cycles_t get_median(int n, cycles_t delta[])
{
	if ((n - 1) % 2)
		return(delta[n / 2] + delta[n / 2 - 1]) / 2;
	else
		return delta[n / 2];
}


// -----------------------------------------------------------------------------


int parser(struct perftest_parameters *user_param, char *argv[], int argc)
{
	int c, size_len;
	int size_factor = 1;
	static int run_inf_flag = 0;
	static int report_fmt_flag = 0;
	static int srq_flag = 0;
	static int report_both_flag = 0;
	static int is_reversed_flag = 0;
	static int pkey_flag = 0;
	static int inline_recv_flag = 0;
	static int tcp_flag = 0;
	static int burst_size_flag = 0;
	static int rate_limit_flag = 0;
	static int rate_units_flag = 0;
	static int rate_limit_type_flag = 0;
	static int verbosity_output_flag = 0;
	static int cpu_util_flag = 0;
	static int latency_gap_flag = 0;
	static int flow_label_flag = 0;
	static int retry_count_flag = 0;
	static int dont_xchg_versions_flag = 0;
	static int use_exp_flag = 0;
	static int use_cuda_flag = 0;
	static int mmap_file_flag = 0;
	static int mmap_offset_flag = 0;
	static int ipv6_flag = 0;
	static int raw_ipv6_flag = 0;
	static int report_per_port_flag = 0;
	static int odp_flag = 0;
	static int hugepages_flag = 0;
	static int use_promiscuous_flag = 0;
	static int use_sniffer_flag = 0;
	static int raw_mcast_flag = 0;
	static int verb_type_flag = 0;
	static int use_res_domain_flag = 0;
	static int mr_per_qp_flag = 0;
	static int dlid_flag = 0;
	static int tclass_flag = 0;
	static int wait_destroy_flag = 0;
	static int disable_fcs_flag = 0;
	static int flows_flag = 0;
	static int flows_burst_flag = 0;
	static int force_link_flag = 0;
	static int local_ip_flag = 0;
	static int remote_ip_flag = 0;
	static int local_port_flag = 0;
	static int remote_port_flag = 0;
	static int local_mac_flag = 0;
	static int remote_mac_flag = 0;
	static int reply_every_flag = 0;
	static int perform_warm_up_flag = 0;
	static int use_ooo_flag = 0;
	static int vlan_en = 0;
	static int vlan_pcp_flag = 0;

	char *server_ip = NULL;
	char *client_ip = NULL;
	char *local_ip = NULL;
	char *remote_ip = NULL;

	init_perftest_params(user_param);

	if(user_param->connection_type == RawEth)
		user_param->machine = UNCHOSEN;

	while (1) {
		static const struct option long_options[] = {
			{ .name = "port", .has_arg = 1, .val = 'p' },
			{ .name = "ib-dev", .has_arg = 1, .val = 'd' },
			{ .name = "ib-port", .has_arg = 1, .val = 'i' },
			{ .name = "mtu", .has_arg = 1, .val = 'm' },
			{ .name = "size", .has_arg = 1, .val = 's' },
			{ .name = "iters", .has_arg = 1, .val = 'n' },
			{ .name = "tx-depth", .has_arg = 1, .val = 't' },
			{ .name = "qp-timeout", .has_arg = 1, .val = 'u' },
			{ .name = "sl", .has_arg = 1, .val = 'S' },
			{ .name = "gid-index", .has_arg = 1, .val = 'x' },
			{ .name = "all", .has_arg = 0, .val = 'a' },
			{ .name = "CPU-freq", .has_arg = 0, .val = 'F' },
			{ .name = "connection", .has_arg = 1, .val = 'c' },
			{ .name = "qp", .has_arg = 1, .val = 'q' },
			{ .name = "events", .has_arg = 0, .val = 'e' },
			{ .name = "vector", .has_arg = 1, .val = 'X' },
			{ .name = "inline_size", .has_arg = 1, .val = 'I' },
			{ .name = "outs", .has_arg = 1, .val = 'o' },
			{ .name = "mcg", .has_arg = 0, .val = 'g' },
			{ .name = "comm_rdma_cm", .has_arg = 0, .val = 'z' },
			{ .name = "rdma_cm", .has_arg = 0, .val = 'R' },
			{ .name = "tos", .has_arg = 1, .val = 'T' },
			{ .name = "help", .has_arg = 0, .val = 'h' },
			{ .name = "MGID", .has_arg = 1, .val = 'M' },
			{ .name = "rx-depth", .has_arg = 1, .val = 'r' },
			{ .name = "bidirectional", .has_arg = 0, .val = 'b' },
			{ .name = "cq-mod", .has_arg = 1, .val = 'Q' },
			{ .name = "noPeak", .has_arg = 0, .val = 'N' },
			{ .name = "version", .has_arg = 0, .val = 'V' },
			{ .name = "report-cycles", .has_arg = 0, .val = 'C' },
			{ .name = "report-histogrm", .has_arg = 0, .val = 'H' },
			{ .name = "report-unsorted", .has_arg = 0, .val = 'U' },
			{ .name = "atomic_type", .has_arg = 1, .val = 'A' },
			{ .name = "dualport", .has_arg = 0, .val = 'O' },
			{ .name = "post_list", .has_arg = 1, .val = 'l' },
			{ .name = "duration", .has_arg = 1, .val = 'D' },
			{ .name = "margin", .has_arg = 1, .val = 'f' },
			{ .name = "source_mac", .has_arg = 1, .val = 'B' },
			{ .name = "dest_mac", .has_arg = 1, .val = 'E' },
			{ .name = "dest_ip", .has_arg = 1, .val = 'J' },
			{ .name = "source_ip", .has_arg = 1, .val = 'j' },
			{ .name = "dest_port", .has_arg = 1, .val = 'K' },
			{ .name = "source_port", .has_arg = 1, .val = 'k' },
			{ .name = "ethertype", .has_arg = 1, .val = 'Y' },
			{ .name = "limit_bw", .has_arg = 1, .val = 'w' },
			{ .name = "limit_msgrate", .has_arg = 1, .val = 'y' },
			{ .name = "server", .has_arg = 0, .val = 'Z' },
			{ .name = "client", .has_arg = 0, .val = 'P' },
			{ .name = "mac_fwd", .has_arg = 0, .val = 'v' },
			{ .name = "use_rss", .has_arg = 0, .val = 'G' },
			{ .name = "force-link", .has_arg = 1, .flag = &force_link_flag, .val = 1},
			{ .name = "remote_mac", .has_arg = 1, .flag = &remote_mac_flag, .val = 1 },
			{ .name = "local_mac", .has_arg = 1, .flag = &local_mac_flag, .val = 1 },
			{ .name = "remote_ip", .has_arg = 1, .flag = &remote_ip_flag, .val = 1 },
			{ .name = "local_ip", .has_arg = 1, .flag = &local_ip_flag, .val = 1 },
			{ .name = "remote_port", .has_arg = 1, .flag = &remote_port_flag, .val = 1 },
			{ .name = "local_port", .has_arg = 1, .flag = &local_port_flag, .val = 1 },
			{ .name = "run_infinitely", .has_arg = 0, .flag = &run_inf_flag, .val = 1 },
			{ .name = "report_gbits", .has_arg = 0, .flag = &report_fmt_flag, .val = 1},
			{ .name = "use-srq", .has_arg = 0, .flag = &srq_flag, .val = 1},
			{ .name = "report-both", .has_arg = 0, .flag = &report_both_flag, .val = 1},
			{ .name = "reversed", .has_arg = 0, .flag = &is_reversed_flag, .val = 1},
			{ .name = "pkey_index", .has_arg = 1, .flag = &pkey_flag, .val = 1},
			{ .name = "inline_recv", .has_arg = 1, .flag = &inline_recv_flag, .val = 1},
			{ .name = "tcp", .has_arg = 0, .flag = &tcp_flag, .val = 1},
			{ .name = "burst_size", .has_arg = 1, .flag = &burst_size_flag, .val = 1},
			{ .name = "rate_limit", .has_arg = 1, .flag = &rate_limit_flag, .val = 1},
			{ .name = "rate_limit_type", .has_arg = 1, .flag = &rate_limit_type_flag, .val = 1},
			{ .name = "rate_units", .has_arg = 1, .flag = &rate_units_flag, .val = 1},
			{ .name = "output", .has_arg = 1, .flag = &verbosity_output_flag, .val = 1},
			{ .name = "cpu_util", .has_arg = 0, .flag = &cpu_util_flag, .val = 1},
			{ .name = "latency_gap", .has_arg = 1, .flag = &latency_gap_flag, .val = 1},
			{ .name = "flow_label", .has_arg = 1, .flag = &flow_label_flag, .val = 1},
			{ .name = "retry_count", .has_arg = 1, .flag = &retry_count_flag, .val = 1},
			{ .name = "dont_xchg_versions", .has_arg = 0, .flag = &dont_xchg_versions_flag, .val = 1},
			{ .name = "use_cuda", .has_arg = 0, .flag = &use_cuda_flag, .val = 1},
			{ .name = "mmap", .has_arg = 1, .flag = &mmap_file_flag, .val = 1},
			{ .name = "mmap-offset", .has_arg = 1, .flag = &mmap_offset_flag, .val = 1},
			{ .name = "ipv6", .has_arg = 0, .flag = &ipv6_flag, .val = 1},
			#ifdef HAVE_IPV6
			{ .name = "raw_ipv6", .has_arg = 0, .flag = &raw_ipv6_flag, .val = 1},
			#endif
			{ .name = "report-per-port", .has_arg = 0, .flag = &report_per_port_flag, .val = 1},
			{ .name = "odp", .has_arg = 0, .flag = &odp_flag, .val = 1},
			{ .name = "use_hugepages", .has_arg = 0, .flag = &hugepages_flag, .val = 1},
			{ .name = "promiscuous", .has_arg = 0, .flag = &use_promiscuous_flag, .val = 1},
			#if defined HAVE_SNIFFER || defined HAVE_SNIFFER_EXP
			{ .name = "sniffer", .has_arg = 0, .flag = &use_sniffer_flag, .val = 1},
			#endif
			{ .name = "raw_mcast", .has_arg = 0, .flag = &raw_mcast_flag, .val = 1},
			#ifdef HAVE_VERBS_EXP
			{ .name = "use_exp", .has_arg = 0, .flag = &use_exp_flag, .val = 1},
			#endif
			#ifdef HAVE_ACCL_VERBS
			{ .name = "verb_type", .has_arg = 1, .flag = &verb_type_flag, .val = 1},
			{ .name = "use_res_domain", .has_arg = 0, .flag = &use_res_domain_flag, .val = 1},
			#endif
			{ .name = "mr_per_qp", .has_arg = 0, .flag = &mr_per_qp_flag, .val = 1},
			{ .name = "dlid", .has_arg = 1, .flag = &dlid_flag, .val = 1},
			{ .name = "tclass", .has_arg = 1, .flag = &tclass_flag, .val = 1},
			{ .name = "wait_destroy", .has_arg = 1, .flag = &wait_destroy_flag, .val = 1},
			#ifdef HAVE_SCATTER_FCS
			{ .name = "disable_fcs", .has_arg = 0, .flag = &disable_fcs_flag, .val = 1},
			#endif
			{ .name = "flows", .has_arg = 1, .flag = &flows_flag, .val = 1},
			{ .name = "flows_burst", .has_arg = 1, .flag = &flows_burst_flag, .val = 1},
			{ .name = "reply_every", .has_arg = 1, .flag = &reply_every_flag, .val = 1},
			{ .name = "perform_warm_up", .has_arg = 0, .flag = &perform_warm_up_flag, .val = 1},
			{ .name = "vlan_en", .has_arg = 0, .flag = &vlan_en, .val = 1 },
			{ .name = "vlan_pcp", .has_arg = 1, .flag = &vlan_pcp_flag, .val = 1 },

			#if defined HAVE_OOO_ATTR || defined HAVE_EXP_OOO_ATTR
			{ .name = "use_ooo", .has_arg = 0, .flag = &use_ooo_flag, .val = 1},
			#endif
			{ 0 }
		};
		c = getopt_long(argc, argv, "w:y:p:d:i:m:s:n:t:u:S:x:c:q:I:o:M:r:Q:A:l:D:f:B:T:E:J:j:K:k:X:aFegzRvhbNVCHUOZP", long_options, NULL);

		if (c == -1)
			break;

		switch (c) {

			case 'p': user_param->port = strtol(optarg, NULL, 0); break;
			case 'd': GET_STRING(user_param->ib_devname, optarg); break;
			case 'i': CHECK_VALUE(user_param->ib_port, uint8_t, MIN_IB_PORT, MAX_IB_PORT, "IB Port"); break;
			case 'm': user_param->mtu = strtol(optarg, NULL, 0); break;
			case 'n': CHECK_VALUE(user_param->iters, int, MIN_ITER, MAX_ITER, "Iteration num"); break;
			case 't': CHECK_VALUE(user_param->tx_depth, int, MIN_TX, MAX_TX, "Tx depth"); break;
			case 'T': CHECK_VALUE(user_param->tos, int, MIN_TOS, MAX_TOS, "TOS"); break;
			case 'u': user_param->qp_timeout = (uint8_t)strtol(optarg, NULL, 0); break;
			case 'S': user_param->sl = (uint8_t)strtol(optarg, NULL, 0);
				  if (user_param->sl > MAX_SL) {
					  fprintf(stderr, " Only %d Service levels\n", MAX_SL);
					  return 1;
				  }
				  if (user_param->connection_type == RawEth)
					  user_param->raw_qos = 1;
				  break;
			case 'x': CHECK_VALUE(user_param->gid_index, uint8_t, MIN_GID_IX, MAX_GID_IX, "Gid index");
				  user_param->use_gid_user = 1; break;
			case 'c': change_conn_type(&user_param->connection_type, user_param->verb, optarg); break;
			case 'q': if (user_param->tst != BW) {
					fprintf(stderr, " Multiple QPs only available on bw tests\n");
					return 1;
				  }
				  CHECK_VALUE(user_param->num_of_qps, int, MIN_QP_NUM, MAX_QP_NUM, "num of Qps");
				  break;
			case 'I': CHECK_VALUE(user_param->inline_size, int, 0, MAX_INLINE, "Max inline");
				  if (user_param->verb == READ || user_param->verb ==ATOMIC) {
					  fprintf(stderr, " Inline feature not available on READ/Atomic verbs\n");
					  return 1;
				  } break;
			case 'o': user_param->out_reads = strtol(optarg, NULL, 0);
				  if (user_param->verb != READ && user_param->verb != ATOMIC) {
					  fprintf(stderr, " Setting Outstanding reads only available on READ verb\n");
					  return 1;
				  } break;
			case 'M': GET_STRING(user_param->user_mgid, optarg); break;
			case 'r': CHECK_VALUE(user_param->rx_depth, int, MIN_RX, MAX_RX, " Rx depth");
				  if (user_param->verb != SEND && user_param->rx_depth > DEF_RX_RDMA) {
					  fprintf(stderr, " On RDMA verbs rx depth can be only 1\n");
					  return 1;
				  } break;
			case 'Q': CHECK_VALUE(user_param->cq_mod, int, MIN_CQ_MOD, MAX_CQ_MOD, "CQ moderation"); break;
			case 'A':
				  if (user_param->verb != ATOMIC) {
					  fprintf(stderr, " You are not running the atomic_lat/bw test!\n");
					  fprintf(stderr, " To change the atomic action type, you must run one of the atomic tests\n");
					  return 1;
				  }

				  if (strcmp(atomicTypesStr[0], optarg)==0)
					  user_param->atomicType = CMP_AND_SWAP;

				  else if (strcmp(atomicTypesStr[1], optarg)==0)
					  user_param->atomicType = FETCH_AND_ADD;

				  else {
					  fprintf(stderr, " Invalid Atomic type! please choose from {CMP_AND_SWAP, FETCH_AND_ADD}\n");
					  exit(1);
				  }
				  break;
			case 'l': user_param->post_list = strtol(optarg, NULL, 0); break;
			case 'D': user_param->duration = strtol(optarg, NULL, 0);
				  if (user_param->duration <= 0) {
					  fprintf(stderr, " Duration period must be greater than 0\n");
					  return 1;
				  }
				  user_param->test_type = DURATION;
				  break;
			case 'f': user_param->margin = strtol(optarg, NULL, 0);
				  if (user_param->margin < 0) {
					  fprintf(stderr, " margin must be positive.\n");
					  return 1;
				  } break;
			case 'O':
				  user_param->ib_port = DEF_IB_PORT;
				  user_param->ib_port2 = DEF_IB_PORT2;
				  user_param->dualport = ON;
				  break;
			case 'a': user_param->test_method = RUN_ALL; break;
			case 'F': user_param->cpu_freq_f = ON; break;
			case 'V': printf("Version: %s\n", user_param->version); return VERSION_EXIT;
			case 'h': usage(argv[0], user_param->verb, user_param->tst, user_param->connection_type);
				  if(user_param->connection_type == RawEth) {
					  usage_raw_ethernet(user_param->tst);
				  }
				  return HELP_EXIT;
			case 'z': user_param->use_rdma_cm = ON; break;
			case 'R': user_param->work_rdma_cm = ON; break;
			case 's': size_len = (int)strlen(optarg);
				  if (optarg[size_len-1] == 'K') {
					  optarg[size_len-1] = '\0';
					  size_factor = 1024;
				  }
				  if (optarg[size_len-1] == 'M') {
					  optarg[size_len-1] = '\0';
					  size_factor = 1024*1024;
				  }
				  user_param->size = (uint64_t)strtol(optarg, NULL, 0) * size_factor;
				  if (user_param->size < 1 || user_param->size > (UINT_MAX / 2)) {
					  fprintf(stderr, " Message Size should be between %d and %d\n", 1, UINT_MAX/2);
					  return 1;
				  }
				  break;
			case 'e': user_param->use_event = ON;
				  if (user_param->verb == WRITE) {
					  fprintf(stderr, " Events feature not available on WRITE verb\n");
					  return 1;
				  }
				  break;
			case 'X':
				  if (user_param->verb == WRITE) {
					  fprintf(stderr, " Events feature not available on WRITE verb\n");
					  return 1;
				  }
				  user_param->use_eq_num = ON;
				  CHECK_VALUE(user_param->eq_num, int, MIN_EQ_NUM, MAX_EQ_NUM, "EQN");
				  break;
			case 'b': user_param->duplex = ON;
				  if (user_param->tst == LAT) {
					  fprintf(stderr, " Bidirectional is only available in BW test\n");
					  return 1;
				  } break;
			case 'N': user_param->noPeak = ON;
				  if (user_param->tst == LAT) {
					  fprintf(stderr, " NoPeak only valid for BW tests\n");
					  return 1;
				  } break;
			case 'C':
				  if (user_param->tst != LAT) {
					  fprintf(stderr, " Available only on Latency tests\n");
					  return 1;
				  }
				  user_param->r_flag->cycles = ON;
				  break;
			case 'g': user_param->use_mcg = ON;
				  if (user_param->verb != SEND) {
					  fprintf(stderr, " MultiCast feature only available on SEND verb\n");
					  return 1;
				  } break;
			case 'H':
				  if (user_param->tst == BW) {
					  fprintf(stderr, " Available only on Latency tests\n");
					  return 1;
				  }
				  user_param->r_flag->histogram = ON;
				  break;
			case 'U':
				  if (user_param->tst == BW) {
					fprintf(stderr, " is Available only on Latency tests\n");
					  return 1;
				  }
				  user_param->r_flag->unsorted = ON;
				  break;
			case 'B':
				  user_param->is_old_raw_eth_param = 1;
				  user_param->is_source_mac = ON;
				  if(parse_mac_from_str(optarg, user_param->source_mac))
					  return FAILURE;
				  break;
			case 'E':
				  user_param->is_old_raw_eth_param = 1;
				  user_param->is_dest_mac = ON;
				  if(parse_mac_from_str(optarg, user_param->dest_mac))
					  return FAILURE;
				  break;
			case 'J':
				  user_param->is_old_raw_eth_param = 1;
				  user_param->is_server_ip = ON;
				  server_ip = optarg;
				  break;
			case 'j':
				  user_param->is_old_raw_eth_param = 1;
				  user_param->is_client_ip = ON;
				  client_ip = optarg;
				  break;
			case 'K':
				  user_param->is_old_raw_eth_param = 1;
				  user_param->is_server_port = ON;
				  user_param->server_port = strtol(optarg, NULL, 0);
				  break;
			case 'k':
				  user_param->is_old_raw_eth_param = 1;
				  user_param->is_client_port = ON;
				  user_param->client_port = strtol(optarg, NULL, 0);
				  break;
			case 'Y':
				  user_param->is_ethertype = ON;
				  if (parse_ethertype_from_str(optarg, &user_param->ethertype)) {
					  fprintf(stderr, " Invalid ethertype value\n");
					  return FAILURE;
				  }
				  break;
			case 'w':
				  user_param->is_limit_bw = ON;
				  user_param->limit_bw = strtof(optarg, NULL);
				  if (user_param->limit_bw < 0) {
					  fprintf(stderr, " Invalid Minimum BW Limit\n");
					  return FAILURE;
				  }
				  break;
			case 'y':
				  user_param->is_limit_msgrate = ON;
				  user_param->limit_msgrate = strtof(optarg, NULL);
				  if (user_param->limit_msgrate < 0) {
					  fprintf(stderr, " Invalid Minimum msgRate Limit\n");
					  return FAILURE;
				  }
				  break;
			case 'P': user_param->machine = CLIENT; break;
			case 'Z': user_param->machine = SERVER; break;
			case 'v': user_param->mac_fwd = ON; break;
			case 'G': user_param->use_rss = ON; break;
			case 0: /* required for long options to work. */
				if (pkey_flag) {
					user_param->pkey_index = strtol(optarg, NULL, 0);
					pkey_flag = 0;
				}
				if (inline_recv_flag) {
					user_param->inline_recv_size = strtol(optarg, NULL, 0);
					inline_recv_flag = 0;
				}
				if (rate_limit_flag) {
					GET_STRING(user_param->rate_limit_str , optarg);
					user_param->rate_limit = atof(optarg);
					if (user_param->rate_limit <= 0) {
						fprintf(stderr, " Rate limit must be non-negative\n");
						return FAILURE;
					}
					/* if not specified, choose HW rate limiter as default */
					if (user_param->rate_limit_type == DISABLE_RATE_LIMIT)
						user_param->rate_limit_type = HW_RATE_LIMIT;

					rate_limit_flag = 0;
				}
				if (burst_size_flag) {
					user_param->burst_size = strtol(optarg, NULL, 0);
					if (user_param->burst_size < 0) {
						fprintf(stderr, " Burst size must be non-negative\n");
						return FAILURE;
					}
					burst_size_flag = 0;
				}
				if (rate_units_flag) {
					if (strcmp("M", optarg) == 0) {
						user_param->rate_units = MEGA_BYTE_PS;
					} else if (strcmp("g", optarg) == 0) {
						user_param->rate_units = GIGA_BIT_PS;
					} else if (strcmp("p", optarg) == 0) {
						user_param->rate_units = PACKET_PS;
					} else {
						fprintf(stderr, " Invalid rate limit units. Please use M, g or p\n");
						return FAILURE;
					}
					rate_units_flag = 0;
				}
				if (rate_limit_type_flag) {
					user_param->is_rate_limit_type = 1;
					if(strcmp("SW", optarg) == 0)
						user_param->rate_limit_type = SW_RATE_LIMIT;
					else if(strcmp("HW", optarg) == 0)
						user_param->rate_limit_type = HW_RATE_LIMIT;
					else if(strcmp("PP", optarg) == 0)
						user_param->rate_limit_type = PP_RATE_LIMIT;
					else {
						fprintf(stderr, " Invalid rate limit type flag. Please use HW, SW or PP.\n");
						return FAILURE;
					}
					rate_limit_type_flag = 0;
				}
				if (verbosity_output_flag) {
					if (strcmp("bandwidth", optarg) == 0) {
						user_param->output = OUTPUT_BW;
					} else if (strcmp("message_rate", optarg) == 0) {
						user_param->output = OUTPUT_MR;
					} else if (strcmp("latency", optarg) == 0) {
						user_param->output = OUTPUT_LAT;
					} else {
						fprintf(stderr, " Invalid verbosity level output flag. Please use bandwidth, latency, message_rate\n");
						return FAILURE;
					}
					verbosity_output_flag = 0;
				}
				if (latency_gap_flag) {
					user_param->latency_gap = strtol(optarg, NULL, 0);
					if (user_param->latency_gap < 0) {
						fprintf(stderr, " Latency gap time must be non-negative\n");
						return FAILURE;
					}
					latency_gap_flag = 0;
				}
				if (flow_label_flag) {
					user_param->flow_label = strtol(optarg, NULL, 0);
					if (user_param->flow_label < 0) {
						fprintf(stderr, "flow label must be non-negative\n");
						return FAILURE;
					}
					flow_label_flag = 0;
				}
				if (retry_count_flag) {
					user_param->retry_count = strtol(optarg, NULL, 0);
					if (user_param->retry_count < 0) {
						fprintf(stderr, " Retry Count value must be positive\n");
						return FAILURE;
					}
					retry_count_flag = 0;
				}
				if (verb_type_flag) {
					if (strcmp("normal", optarg) == 0) {
						user_param->verb_type = NORMAL_INTF;
					} else if (strcmp("accl", optarg) == 0) {
						user_param->verb_type = ACCL_INTF;
					} else {
						fprintf(stderr, " Invalid verb type. Please choose normal/accl.\n");
						return FAILURE;
					}
					verb_type_flag = 0;
				}
				if (mmap_file_flag) {
					user_param->mmap_file = strdup(optarg);
					mmap_file_flag = 0;
				}
				if (mmap_offset_flag) {
					user_param->mmap_offset = strtol(optarg, NULL, 0);
					mmap_offset_flag = 0;
				}
				if (dlid_flag) {
					user_param->dlid = (uint16_t)strtol(optarg, NULL, 0);
					dlid_flag = 0;
				}
				if (tclass_flag) {
					user_param->traffic_class = (uint16_t)strtol(optarg, NULL, 0);
					tclass_flag = 0;
				}
				if (wait_destroy_flag) {
					user_param->wait_destroy = (uint32_t)strtol(optarg, NULL, 0);
					wait_destroy_flag = 0;
				}
				if (flows_flag) {
					user_param->flows = (uint16_t)strtol(optarg, NULL, 0);
					if (user_param->flows == 0) {
						fprintf(stderr, "Invalid flows value. Please set a positive number\n");
						return FAILURE;
					}
					flows_flag = 0;
				}
				if (flows_burst_flag) {
					user_param->flows_burst = (uint16_t)strtol(optarg, NULL, 0);
					if (user_param->flows_burst == 0) {
						fprintf(stderr, "Invalid burst flow value. Please set a positive number\n");
						return FAILURE;
					}
					flows_burst_flag = 0;

				}
				if (force_link_flag) {
					user_param->link_type = str_link_layer(optarg);
					if (user_param->link_type == LINK_FAILURE) {
						fprintf(stderr, "Invalid link layer value should be IB or ETHERNET.\n");
						return FAILURE;
					}
					force_link_flag = 0;
				}
				if (remote_mac_flag) {
					user_param->is_new_raw_eth_param = 1;
					user_param->is_dest_mac = 1;
					if(parse_mac_from_str(optarg, user_param->remote_mac))
						return FAILURE;
					remote_mac_flag = 0;
				}
				if (local_mac_flag) {
					user_param->is_new_raw_eth_param = 1;
					user_param->is_source_mac = 1;
					if(parse_mac_from_str(optarg, user_param->local_mac))
						return FAILURE;
					local_mac_flag = 0;
				}
				if (remote_ip_flag) {
					user_param->is_new_raw_eth_param = 1;
					user_param->is_client_ip = 1;
					remote_ip = optarg;
					remote_ip_flag = 0;
				}
				if (local_ip_flag) {
					user_param->is_new_raw_eth_param = 1;
					user_param->is_server_ip = 1;
					local_ip = optarg;
					local_ip_flag = 0;
				}
				if (remote_port_flag) {
					user_param->is_new_raw_eth_param = 1;
					user_param->is_client_port = 1;
					user_param->remote_port = strtol(optarg, NULL, 0);
					remote_port_flag = 0;
				}
				if (local_port_flag) {
					user_param->is_new_raw_eth_param = 1;
					user_param->is_server_port = 1;
					user_param->local_port = strtol(optarg, NULL, 0);
					local_port_flag = 0;
				}
				if (reply_every_flag) {
					user_param->reply_every = strtol(optarg, NULL, 0);
					reply_every_flag = 0;
				}
				if(vlan_pcp_flag) {
					user_param->vlan_pcp = strtol(optarg, NULL, 0);
					user_param->vlan_en = ON;
					if (user_param->vlan_pcp > 8) {
						fprintf(stderr, "Invalid vlan priority value. Please set a number in 0~8\n");
						return FAILURE;
					}
					vlan_pcp_flag = 0;
				}
				break;

			default:
				  fprintf(stderr, " Invalid Command or flag.\n");
				  fprintf(stderr, " Please check command line and run again.\n\n");
				  usage(argv[0], user_param->verb, user_param->tst, user_param->connection_type);
				  if(user_param->connection_type == RawEth) {
					  usage_raw_ethernet(user_param->tst);
				  }
				  return 1;
		}
	}

	if (tcp_flag) {
		user_param->tcp = 1;
	}
	if (run_inf_flag) {
		user_param->test_method = RUN_INFINITELY;
	}

	if (srq_flag) {
		user_param->use_srq = 1;
	}

	if (report_fmt_flag) {
		user_param->report_fmt = GBS;
	}

	if (dont_xchg_versions_flag) {
		user_param->dont_xchg_versions = 1;
	}

	if (use_exp_flag) {
		user_param->use_exp = 1;
	}

	if (use_res_domain_flag) {
		user_param->use_res_domain = 1;
	}

	if (use_cuda_flag) {
		user_param->use_cuda = 1;
	}
	if (report_both_flag) {
		user_param->report_both = 1;
	}

	if (is_reversed_flag) {
		user_param->is_reversed = 1;
	}

	if (cpu_util_flag) {
		user_param->cpu_util = 1;
	}

	if (report_per_port_flag) {
		user_param->report_per_port = 1;
	}

	if (ipv6_flag) {
		user_param->ipv6 = 1;
	}

	if (raw_ipv6_flag) {
		if (user_param->is_new_raw_eth_param) {
			if (user_param->is_server_ip) {
				if(1 != parse_ip6_from_str(local_ip,
							  (struct in6_addr *)&(user_param->local_ip6))) {
					fprintf(stderr, " Invalid local IP address\n");
					return FAILURE;
				}
			}
			if (user_param->is_client_ip) {
				if(1 != parse_ip6_from_str(remote_ip,
							  (struct in6_addr *)&(user_param->remote_ip6))) {
					fprintf(stderr, " Invalid remote IP address\n");
					return FAILURE;
				}
			}
		} else {
			if (user_param->is_server_ip) {
				if(1 != parse_ip6_from_str(server_ip,
							  (struct in6_addr *)&(user_param->server_ip6))) {
					fprintf(stderr, " Invalid server IP address\n");
					return FAILURE;
				}
			}
			if (user_param->is_client_ip) {
				if(1 != parse_ip6_from_str(client_ip,
							  (struct in6_addr *)&(user_param->client_ip6))) {
					fprintf(stderr, " Invalid client IP address\n");
					return FAILURE;
				}
			}
		}
		user_param->raw_ipv6 = 1;
	} else {
		if (user_param->is_new_raw_eth_param) {
			if (user_param->is_server_ip) {
				if(1 != parse_ip_from_str(local_ip,
							  &(user_param->local_ip))) {
					fprintf(stderr, " Invalid local IP address\n");
					return FAILURE;
				}
			}
			if (user_param->is_client_ip) {
				if(1 != parse_ip_from_str(remote_ip,
							  &(user_param->remote_ip))) {
					fprintf(stderr, " Invalid remote IP address\n");
					return FAILURE;
				}
			}
		} else {
			if (user_param->is_server_ip) {
				if(1 != parse_ip_from_str(server_ip,
							  &(user_param->server_ip))) {
					fprintf(stderr, " Invalid server IP address\n");
					return FAILURE;
				}
			}
			if (user_param->is_client_ip) {
				if(1 != parse_ip_from_str(client_ip,
							  &(user_param->client_ip))) {
					fprintf(stderr, " Invalid client IP address\n");
					return FAILURE;
				}
			}
		}
	}

	if(odp_flag) {
		user_param->use_odp = 1;
	}

	if(hugepages_flag) {
		user_param->use_hugepages = 1;
	}

	if (use_promiscuous_flag) {
		user_param->use_promiscuous = 1;
	}

	if (use_sniffer_flag) {
		user_param->use_sniffer = 1;
	}

	if (raw_mcast_flag) {
		user_param->raw_mcast = 1;
	}

	if (mr_per_qp_flag) {
		user_param->mr_per_qp = 1;
	}

	if (disable_fcs_flag) {
		user_param->disable_fcs = 1;
	}
	if (perform_warm_up_flag) {
		user_param->perform_warm_up = 1;
	}
	if (use_ooo_flag)
		user_param->use_ooo = 1;
	if(vlan_en) {
		user_param->vlan_en = ON;
		/* user_param->print_eth_func = &print_ethernet_vlan_header; */
		vlan_en = 0;
	}
	if (optind == argc - 1) {
		GET_STRING(user_param->servername, argv[optind]);

	} else if (optind < argc) {
		fprintf(stderr, " Invalid Command line. Please check command rerun \n");
		return 1;
	}

	if(user_param->connection_type != RawEth)
		user_param->machine = user_param->servername ? CLIENT : SERVER;

	/* fan-in addition */
	if (user_param->is_reversed) {
		if (user_param->machine == SERVER)
			user_param->machine = CLIENT;
		else
			user_param->machine = SERVER;
	}
	set_raw_eth_parameters(user_param);
	force_dependecies(user_param);
	return 0;
}

int check_link(struct ibv_context *context, struct perftest_parameters *user_param)
{
	user_param->transport_type = context->device->transport_type;

	if (set_link_layer(context, user_param) == FAILURE){
		fprintf(stderr, " Couldn't set the link layer\n");
		return FAILURE;
	}

	if (user_param->link_type == IBV_LINK_LAYER_ETHERNET && user_param->gid_index == -1) {
		user_param->gid_index = 0;
	}

	if (user_param->connection_type == RawEth) {

		if (user_param->link_type != IBV_LINK_LAYER_ETHERNET) {
			fprintf(stderr, " Raw Etherent test can only run on Ethernet link! exiting ...\n");
			return FAILURE;
		}
	}

	/* in case of dual-port mode */
	if (user_param->dualport==ON) {
		if (user_param->link_type2 == IBV_LINK_LAYER_ETHERNET && user_param->gid_index2 == -1) {
			user_param->gid_index2 = 1;
		}
	}

	/* Compute Max inline size with pre found statistics values */
	ctx_set_max_inline(context, user_param);

	if (user_param->verb == READ || user_param->verb == ATOMIC)
		user_param->out_reads = ctx_set_out_reads(context, user_param->out_reads);
	else
		user_param->out_reads = 1;


	if (!user_param->ib_devname)
		GET_STRING(user_param->ib_devname, ibv_get_device_name(context->device))

	if (user_param->pkey_index > 0)
		user_param->pkey_index = ctx_chk_pkey_index(context, user_param->pkey_index);

	return SUCCESS;
}

void ctx_print_test_info(struct perftest_parameters *user_param)
{
	int temp = 0;

	if (user_param->output != FULL_VERBOSITY)
		return;

	printf(RESULT_LINE);
	printf(" ");
	printf("%s ", testsStr[user_param->verb]);

	if (user_param->verb == ATOMIC) {
		printf("%s ", atomicTypesStr[user_param->atomicType]);
	}

	if (user_param->tst == BW) {

		if (user_param->duplex) {
			printf("Bidirectional ");
		}

		if (user_param->post_list > 1) {
			printf("Post List ");
		}

		printf("BW ");

	} else if (user_param->tst == LAT) {
		printf("Latency ");
	}

	if (user_param->mac_fwd) {
		printf("forwarding ");
	}

	if (user_param->use_mcg)
		printf("Multicast ");

	printf("Test\n");

	if (user_param->use_event) {
		printf(" Test with events. Using %s_comp%d\n", user_param->ib_devname, user_param->eq_num);
	}

	if (user_param->use_mcg)
		printf(" MultiCast runs on UD!\n");

	printf(" Dual-port : %s\t\tDevice : %s\n", user_param->dualport ? "ON" : "OFF", user_param->ib_devname);
	printf(" Number of qps : %d\t\tTransport type : %s\n", user_param->num_of_qps, transport_str(user_param->transport_type));
	printf(" Connection type : %s\t\tUsing SRQ : %s\n", connStr[user_param->connection_type], user_param->use_srq ? "ON" : "OFF");

	if (user_param->machine == CLIENT || user_param->duplex) {
		printf(" TX depth : %d\n", user_param->tx_depth);
	}

	if (user_param->post_list > 1)
		printf(" Post List : %d\n", user_param->post_list);

	if (user_param->verb == SEND && (user_param->machine == SERVER || user_param->duplex)) {
		printf(" RX depth : %d\n", user_param->rx_depth);
	}

	if (user_param->tst == BW) {
		printf(" CQ Moderation : %d\n", user_param->cq_mod);
	}

	printf(" Mtu : %lu[B]\n", user_param->connection_type == RawEth ? user_param->curr_mtu : MTU_SIZE(user_param->curr_mtu));
	printf(" Link type : %s\n" , link_layer_str(user_param->link_type));

	/* we use the receive buffer only for mac forwarding. */
	if (user_param->mac_fwd == ON)
		printf(" Buffer size : %d[B]\n" , user_param->buff_size/2);

	if (user_param->gid_index != DEF_GID_INDEX)
		printf(" GID index : %d\n", user_param->gid_index);
	if ((user_param->dualport == ON) && (user_param->gid_index2 != DEF_GID_INDEX))
		printf(" GID index2 : %d\n", user_param->gid_index2);

	if (user_param->verb != READ && user_param->verb != ATOMIC)
		printf(" Max inline data : %d[B]\n", user_param->inline_size);

	else
		printf(" Outstand reads : %d\n", user_param->out_reads);

	printf(" rdma_cm QPs : %s\n", qp_state[user_param->work_rdma_cm]);
	printf(" Data ex. method : %s", exchange_state[temp]);
	putchar('\n');

	printf(RESULT_LINE);
}

void print_report_bw(struct perftest_parameters *user_param, struct bw_report_data *my_bw_rep)
{
	double cycles_to_units, sum_of_test_cycles;
	int location_arr;
	int opt_completed = 0;
	int opt_posted = 0;
	int i, j;
	int run_inf_bi_factor;
	int num_of_qps = user_param->num_of_qps;
	long format_factor;
	long num_of_calculated_iters = user_param->iters;

	int free_my_bw_rep = 0;
	if (user_param->test_method == RUN_INFINITELY)
		user_param->tcompleted[opt_posted] = get_cycles();

	cycles_t t, opt_delta, peak_up, peak_down, tsize;

	opt_delta = user_param->tcompleted[opt_posted] - user_param->tposted[opt_completed];

	if((user_param->connection_type == DC || user_param->use_xrc) && user_param->duplex)
		num_of_qps /= 2;

	if (user_param->noPeak == OFF) {
		/* Find the peak bandwidth unless asked not to in command line */
		for (i = 0; i < num_of_calculated_iters * num_of_qps; ++i) {
			for (j = i; j < num_of_calculated_iters * num_of_qps; ++j) {
				t = (user_param->tcompleted[j] - user_param->tposted[i]) / (j - i + 1);
				if (t < opt_delta) {
					opt_delta = t;
					opt_posted = i;
					opt_completed = j;
				}
			}
		}
	}

	cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f) * 1000000;
	if ((cycles_to_units == 0 && !user_param->cpu_freq_f)) {
		fprintf(stderr, "Can't produce a report\n");
		exit(1);
	}

	run_inf_bi_factor = (user_param->duplex && user_param->test_method == RUN_INFINITELY) ? (user_param->verb == SEND ? 1 : 2) : 1 ;
	tsize = run_inf_bi_factor * user_param->size;
	num_of_calculated_iters *= (user_param->test_type == DURATION) ? 1 : num_of_qps;
	location_arr = (user_param->noPeak) ? 0 : num_of_calculated_iters - 1;
	/* support in GBS format */
	format_factor = (user_param->report_fmt == MBS) ? 0x100000 : 125000000;

	sum_of_test_cycles = ((double)(user_param->tcompleted[location_arr] - user_param->tposted[0]));

	double bw_avg = ((double)tsize*num_of_calculated_iters * cycles_to_units) / (sum_of_test_cycles * format_factor);
	double msgRate_avg = ((double)num_of_calculated_iters * cycles_to_units * run_inf_bi_factor) / (sum_of_test_cycles * 1000000);

	double bw_avg_p1 = ((double)tsize*user_param->iters_per_port[0] * cycles_to_units) / (sum_of_test_cycles * format_factor);
	double msgRate_avg_p1 = ((double)user_param->iters_per_port[0] * cycles_to_units * run_inf_bi_factor) / (sum_of_test_cycles * 1000000);

	double bw_avg_p2 = ((double)tsize*user_param->iters_per_port[1] * cycles_to_units) / (sum_of_test_cycles * format_factor);
	double msgRate_avg_p2 = ((double)user_param->iters_per_port[1] * cycles_to_units * run_inf_bi_factor) / (sum_of_test_cycles * 1000000);

	peak_up = !(user_param->noPeak)*(cycles_t)tsize*(cycles_t)cycles_to_units;
	peak_down = (cycles_t)opt_delta * format_factor;

	if (my_bw_rep == NULL) {
		free_my_bw_rep = 1;
		ALLOCATE(my_bw_rep , struct bw_report_data , 1);
		memset(my_bw_rep, 0, sizeof(struct bw_report_data));
	}

	my_bw_rep->size = (unsigned long)user_param->size;
	my_bw_rep->iters = num_of_calculated_iters;
	my_bw_rep->bw_peak = (double)peak_up/peak_down;
	my_bw_rep->bw_avg = bw_avg;
	my_bw_rep->msgRate_avg = msgRate_avg;
	my_bw_rep->bw_avg_p1 = bw_avg_p1;
	my_bw_rep->msgRate_avg_p1 = msgRate_avg_p1;
	my_bw_rep->bw_avg_p2 = bw_avg_p2;
	my_bw_rep->msgRate_avg_p2 = msgRate_avg_p2;
	my_bw_rep->sl = user_param->sl;

	if (!user_param->duplex || (user_param->verb == SEND && user_param->test_type == DURATION)
			|| user_param->test_method == RUN_INFINITELY || user_param->connection_type == RawEth)
		print_full_bw_report(user_param, my_bw_rep, NULL);

	if (free_my_bw_rep == 1) {
		free(my_bw_rep);
	}
}

#define LAT_MEASURE_TAIL (2)
void print_report_lat (struct perftest_parameters *user_param)
{
	int i;
	int rtt_factor;
	double cycles_to_units, cycles_rtt_quotient, temp_var, pow_var;
	cycles_t median ;
	cycles_t *delta = NULL;
	const char* units;
	double latency, stdev, average_sum = 0 , average, stdev_sum = 0;
	int iters_99, iters_99_9;
	int measure_cnt;

	measure_cnt = (user_param->tst == LAT) ? user_param->iters - 1 : (user_param->iters) / user_param->reply_every;
	rtt_factor = (user_param->verb == READ || user_param->verb == ATOMIC) ? 1 : 2;
	ALLOCATE(delta, cycles_t, measure_cnt);

	if (user_param->r_flag->cycles) {
		cycles_to_units = 1;
		units = "cycles";
	} else {
		cycles_to_units = get_cpu_mhz(user_param->cpu_freq_f);
		units = "usec";
	}

	if (user_param->tst == LAT) {
		for (i = 0; i < measure_cnt; ++i) {
			delta[i] = user_param->tposted[i + 1] - user_param->tposted[i];
		}
	} else if (user_param->tst == LAT_BY_BW) {
		for (i = 0; i < measure_cnt; ++i) {
			delta[i] = user_param->tcompleted[i] - user_param->tposted[i];
		}
	}
	else {
		fprintf(stderr,"print report LAT is support in LAT and LAT_BY_BW tests only\n");
		exit(1);
	}

	cycles_rtt_quotient = cycles_to_units * rtt_factor;
	if (user_param->r_flag->unsorted) {
		printf("#, %s\n", units);
		for (i = 0; i < measure_cnt; ++i)
			printf("%d, %g\n", i + 1, delta[i] / cycles_rtt_quotient);
	}

	qsort(delta, measure_cnt, sizeof *delta, cycles_compare);
	measure_cnt = measure_cnt - LAT_MEASURE_TAIL;
	median = get_median(measure_cnt, delta);

	/* calcualte average sum on sorted array*/
	for (i = 0; i < measure_cnt; ++i)
		average_sum += (delta[i] / cycles_rtt_quotient);

	average = average_sum / measure_cnt;

	/* Calculate stdev by variance*/
	for (i = 0; i < measure_cnt; ++i) {
		temp_var = average - (delta[i] / cycles_rtt_quotient);
		pow_var = pow(temp_var, 2 );
		stdev_sum += pow_var;
	}

	if (user_param->r_flag->histogram) {
		printf("#, %s\n", units);
		for (i = 0; i < measure_cnt; ++i)
			printf("%d, %g\n", i + 1, delta[i] / cycles_rtt_quotient);
	}

	if (user_param->r_flag->unsorted || user_param->r_flag->histogram) {
		if (user_param->output == FULL_VERBOSITY) {
			printf(RESULT_LINE);
			printf("%s",(user_param->test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
			printf((user_param->cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
		}
	}

	latency = median / cycles_rtt_quotient;
	stdev = sqrt(stdev_sum / measure_cnt);
	iters_99 = ceil((measure_cnt) * 0.99);
	iters_99_9 = ceil((measure_cnt) * 0.999);

	if (user_param->output == OUTPUT_LAT)
		printf("%lf\n",average);
	else {
		printf(REPORT_FMT_LAT,
				(unsigned long)user_param->size,
				user_param->iters,
				delta[0] / cycles_rtt_quotient,
				delta[measure_cnt] / cycles_rtt_quotient,
				latency,
				average,
				stdev,
				delta[iters_99] / cycles_rtt_quotient,
				delta[iters_99_9] / cycles_rtt_quotient);
		printf( user_param->cpu_util_data.enable ? REPORT_EXT_CPU_UTIL : REPORT_EXT , calc_cpu_util(user_param));
	}

	free(delta);
}

void print_full_bw_report (struct perftest_parameters *user_param, struct bw_report_data *my_bw_rep, struct bw_report_data *rem_bw_rep)
{
	double bw_peak        = my_bw_rep->bw_peak;
	double bw_avg         = my_bw_rep->bw_avg;
	double bw_avg_p1      = my_bw_rep->bw_avg_p1;
	double bw_avg_p2      = my_bw_rep->bw_avg_p2;
	double msgRate_avg    = my_bw_rep->msgRate_avg;
	double msgRate_avg_p1 = my_bw_rep->msgRate_avg_p1;
	double msgRate_avg_p2 = my_bw_rep->msgRate_avg_p2;
	int inc_accuracy = ((bw_avg < 0.1) && (user_param->report_fmt == GBS));

	if (rem_bw_rep != NULL) {
		bw_peak        += rem_bw_rep->bw_peak;
		bw_avg         += rem_bw_rep->bw_avg;
		bw_avg_p1      += rem_bw_rep->bw_avg_p1;
		bw_avg_p2      += rem_bw_rep->bw_avg_p2;
		msgRate_avg    += rem_bw_rep->msgRate_avg;
		msgRate_avg_p1 += rem_bw_rep->msgRate_avg_p1;
		msgRate_avg_p2 += rem_bw_rep->msgRate_avg_p2;
	}

	if ( (user_param->duplex && rem_bw_rep != NULL) || (!user_param->duplex && rem_bw_rep == NULL)) {
		/* Verify Limits */
		if ( ((user_param->is_limit_bw == ON )&& (user_param->limit_bw > bw_avg)) )
			user_param->is_bw_limit_passed |= 0;
		else
			user_param->is_bw_limit_passed |= 1;

		if ( (user_param->is_limit_msgrate) && (user_param->limit_msgrate > msgRate_avg) )
			user_param->is_msgrate_limit_passed |= 0;
		else
			user_param->is_msgrate_limit_passed |= 1;
	}

	if (user_param->output == OUTPUT_BW)
		printf("%lf\n", bw_avg);
	else if (user_param->output == OUTPUT_MR)
		printf("%lf\n", msgRate_avg);
	else if (user_param->raw_qos)
		printf( REPORT_FMT_QOS, my_bw_rep->size, my_bw_rep->sl, my_bw_rep->iters, bw_peak, bw_avg, msgRate_avg);
	else if (user_param->report_per_port)
		printf(REPORT_FMT_PER_PORT, my_bw_rep->size, my_bw_rep->iters, bw_peak, bw_avg, msgRate_avg, bw_avg_p1, msgRate_avg_p1, bw_avg_p2, msgRate_avg_p2);
	else
		printf( inc_accuracy ? REPORT_FMT_EXT : REPORT_FMT, my_bw_rep->size, my_bw_rep->iters, bw_peak, bw_avg, msgRate_avg);
	if (user_param->output == FULL_VERBOSITY) {
		fflush(stdout);
		fprintf(stdout, user_param->cpu_util_data.enable ? REPORT_EXT_CPU_UTIL : REPORT_EXT , calc_cpu_util(user_param));
	}
}