added generic write bandwidth benchmarks.

usr/benchmarks/CMakeLists.txt

@@ -8,6 +8,10 @@ target_link_libraries(basic pthread)
 
 add_executable(hg hg.c hist.c rdtsc.c run.c init.c opt.c report.c setup.c)
 
+# add_executable(ib-pingpong-ud ib/pingpong.c ib/pingpong-ud.c)
+add_executable(ib_write_bw ib/write_bw.c ib/get_clock.c ib/perftest_parameters.c
+	 ib/perftest_resources.c ib/perftest_communication.c)
+
 add_executable(netio netio.c)
 
 add_executable(RCCE_pingpong RCCE_pingpong.c)

usr/benchmarks/ib/atomic_bw.c

@@ -0,0 +1,341 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2006 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "perftest_communication.h"
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	int ret_parser, i;
+	struct ibv_device		*ib_dev = NULL;
+	struct pingpong_context		ctx;
+	struct pingpong_dest		*my_dest = NULL;
+	struct pingpong_dest		*rem_dest = NULL;
+	struct perftest_parameters	user_param;
+	struct perftest_comm		user_comm;
+	struct bw_report_data		my_bw_rep, rem_bw_rep;
+
+	/* init default values to user's parameters */
+	memset(&ctx, 0, sizeof(struct pingpong_context));
+	memset(&user_param, 0, sizeof(struct perftest_parameters));
+	memset(&user_comm, 0, sizeof(struct perftest_comm));
+
+	user_param.verb    = ATOMIC;
+	user_param.tst     = BW;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+
+	ret_parser = parser(&user_param, argv, argc);
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr, " Parser function exited with Error\n");
+		return FAILURE;
+	}
+
+	if (user_param.use_xrc && user_param.duplex) {
+		user_param.num_of_qps *= 2;
+	}
+
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev)
+		return 7;
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	#ifdef HAVE_MASKED_ATOMICS
+	if (check_masked_atomics_support(&ctx)) {
+		user_param.masked_atomics = 1;
+		user_param.use_exp = 1;
+	}
+
+	if (user_param.masked_atomics && (user_param.work_rdma_cm || user_param.use_rdma_cm)) {
+		fprintf(stderr, "atomic test is not supported with -R/-z flag (rdma_cm) with this device.\n");
+		return FAILURE;
+	}
+	#endif
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link(ctx.context, &user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
+	if (create_comm_struct(&user_comm, &user_param)) {
+		fprintf(stderr, " Unable to create RDMA_CM resources\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
+		printf("\n************************************\n");
+		printf("* Waiting for client to connect... *\n");
+		printf("************************************\n");
+	}
+
+	/* Initialize the connection and print the local data. */
+	if (establish_connection(&user_comm)) {
+		fprintf(stderr, " Unable to init the socket connection\n");
+		return FAILURE;
+	}
+
+	exchange_versions(&user_comm, &user_param);
+
+	check_sys_data(&user_comm, &user_param);
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_mtu(ctx.context, &user_param, &user_comm)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	ALLOCATE(my_dest, struct pingpong_dest, user_param.num_of_qps);
+	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+	ALLOCATE(rem_dest, struct pingpong_dest, user_param.num_of_qps);
+	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx, &user_param);
+
+	/* Create (if nessacery) the rdma_cm ids and channel. */
+	if (user_param.work_rdma_cm == ON) {
+
+		if (user_param.machine == CLIENT) {
+			if (retry_rdma_connect(&ctx, &user_param)) {
+				fprintf(stderr, "Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+
+		} else {
+			if (create_rdma_resources(&ctx, &user_param)) {
+				fprintf(stderr, " Unable to create the rdma_resources\n");
+				return FAILURE;
+			}
+			if (rdma_server_connect(&ctx, &user_param)) {
+				fprintf(stderr, "Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+		}
+
+	} else {
+
+		/* create all the basic IB resources. */
+		if (ctx_init(&ctx, &user_param)) {
+			fprintf(stderr, " Couldn't create IB resources\n");
+			return FAILURE;
+		}
+	}
+
+	/* Set up the Connection. */
+	if (set_up_connection(&ctx, &user_param, my_dest)) {
+		fprintf(stderr, " Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	/* Print this machine QP information */
+	for (i = 0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i], &user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+	for (i = 0; i < user_param.num_of_qps; i++) {
+
+		/* shaking hands and gather the other side info. */
+		if (ctx_hand_shake(&user_comm, &my_dest[i], &rem_dest[i])) {
+			fprintf(stderr, "Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+		ctx_print_pingpong_data(&rem_dest[i], &user_comm);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
+			fprintf(stderr, "\n Found Incompatibility issue with GID types.\n");
+			fprintf(stderr, " Please Try to use a different IP version.\n\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_connect(&ctx, rem_dest, &user_param, my_dest)) {
+			fprintf(stderr, " Unable to Connect the HCA's through the link\n");
+			return FAILURE;
+		}
+	}
+
+	/* An additional handshake is required after moving qp to RTR. */
+	if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
+		fprintf(stderr, "Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	/* For half duplex tests, server just waits for client to exit */
+	if (user_param.machine == SERVER && !user_param.duplex) {
+		if (user_param.output == FULL_VERBOSITY) {
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+		}
+
+		if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
+			fprintf(stderr, " Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version));
+		print_full_bw_report(&user_param, &rem_bw_rep, NULL);
+
+		if (user_param.output == FULL_VERBOSITY) {
+			printf(RESULT_LINE);
+		}
+
+		if (ctx_close_connection(&user_comm, &my_dest[0], &rem_dest[0])) {
+			fprintf(stderr, "Failed to close connection between server and client\n");
+			return FAILURE;
+		}
+
+		return destroy_ctx(&ctx, &user_param);
+	}
+
+	if (user_param.use_event) {
+		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return FAILURE;
+		}
+	}
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+		printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	ctx_set_send_wqes(&ctx, &user_param, rem_dest);
+
+	if (user_param.test_method == RUN_REGULAR || user_param.test_method == RUN_ALL) {
+
+		if (user_param.perform_warm_up) {
+			if (perform_warm_up(&ctx, &user_param)) {
+				fprintf(stderr, "Problems with warm up\n");
+				return FAILURE;
+			}
+		}
+
+		if (user_param.duplex) {
+			if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
+				fprintf(stderr, "Failed to sync between server and client between different msg sizes\n");
+				return FAILURE;
+			}
+		}
+
+		if (run_iter_bw(&ctx, &user_param)) {
+			fprintf(stderr, " Error occurred in run_iter function\n");
+			return FAILURE;
+		}
+
+		print_report_bw(&user_param, &my_bw_rep);
+
+		if (user_param.duplex) {
+			xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version));
+			print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
+		}
+
+		if (user_param.report_both && user_param.duplex) {
+			printf(RESULT_LINE);
+			printf("\n Local results:\n");
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+			print_full_bw_report(&user_param, &my_bw_rep, NULL);
+			printf(RESULT_LINE);
+
+			printf("\n Remote results:\n");
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+			print_full_bw_report(&user_param, &rem_bw_rep, NULL);
+		}
+	} else if (user_param.test_method == RUN_INFINITELY) {
+
+		if (run_iter_bw_infinitely(&ctx, &user_param)) {
+			fprintf(stderr, " Error occurred while running infinitely! aborting ...\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+	}
+	/* For half duplex tests, server just waits for client to exit */
+	if (user_param.machine == CLIENT && !user_param.duplex) {
+
+		if (ctx_hand_shake(&user_comm, &my_dest[0], &rem_dest[0])) {
+			fprintf(stderr, " Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		xchg_bw_reports(&user_comm, &my_bw_rep, &rem_bw_rep, atof(user_param.rem_version));
+	}
+
+	if (ctx_close_connection(&user_comm, &my_dest[0], &rem_dest[0])) {
+		fprintf(stderr, "Failed to close connection between server and client\n");
+		return FAILURE;
+	}
+
+	if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON)) {
+		fprintf(stderr, "Error: BW result is below bw limit\n");
+		return FAILURE;
+	}
+
+	if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON)) {
+		fprintf(stderr, "Error: Msg rate  is below msg_rate limit\n");
+		return FAILURE;
+	}
+
+	return destroy_ctx(&ctx, &user_param);
+}

usr/benchmarks/ib/atomic_lat.c

@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler)
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#if !defined(__FreeBSD__)
+#include <malloc.h>
+#endif
+
+#include "get_clock.h"
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "perftest_communication.h"
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	int				ret_parser, i;
+	struct report_options		report;
+	struct pingpong_context		ctx;
+	struct pingpong_dest		*my_dest  = NULL;
+	struct pingpong_dest		*rem_dest = NULL;
+	struct ibv_device		*ib_dev;
+	struct perftest_parameters	user_param;
+	struct perftest_comm		user_comm;
+
+	/* init default values to user's parameters */
+	memset(&ctx,0,sizeof(struct pingpong_context));
+	memset(&user_param, 0, sizeof(struct perftest_parameters));
+	memset(&user_comm,0,sizeof(struct perftest_comm));
+
+	user_param.verb    = ATOMIC;
+	user_param.tst     = LAT;
+	user_param.r_flag  = &report;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+
+	ret_parser = parser(&user_param,argv,argc);
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		return FAILURE;
+	}
+
+	if(user_param.use_xrc)
+		user_param.num_of_qps *= 2;
+
+	/* Finding the IB device selected (or defalut if no selected). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		return FAILURE;
+	}
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param)) {
+		return FAILURE;
+	}
+
+	#ifdef HAVE_MASKED_ATOMICS
+	if (check_masked_atomics_support(&ctx)) {
+		user_param.masked_atomics = 1;
+		user_param.use_exp = 1;
+	}
+
+	if (user_param.masked_atomics && (user_param.work_rdma_cm || user_param.use_rdma_cm)) {
+		fprintf(stderr, "atomic test is not supported with -R/-z flag (rdma_cm) with this device.\n");
+		return FAILURE;
+	}
+
+	#endif
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link(ctx.context,&user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
+	if (create_comm_struct(&user_comm,&user_param)) {
+		fprintf(stderr," Unable to create RDMA_CM resources\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
+		printf("\n************************************\n");
+		printf("* Waiting for client to connect... *\n");
+		printf("************************************\n");
+	}
+
+	/* Initialize the connection and print the local data. */
+	if (establish_connection(&user_comm)) {
+		fprintf(stderr," Unable to init the socket connection\n");
+		return FAILURE;
+	}
+
+	exchange_versions(&user_comm, &user_param);
+
+	check_sys_data(&user_comm, &user_param);
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_mtu(ctx.context,&user_param, &user_comm)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx,&user_param);
+
+	/* Create (if nessacery) the rdma_cm ids and channel. */
+	if (user_param.work_rdma_cm == ON) {
+
+		if (user_param.machine == CLIENT) {
+			if (retry_rdma_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+
+		} else {
+			if (create_rdma_resources(&ctx,&user_param)) {
+				fprintf(stderr," Unable to create the rdma_resources\n");
+				return FAILURE;
+			}
+			if (rdma_server_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+		}
+
+	} else {
+
+		/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
+		if (ctx_init(&ctx,&user_param)) {
+			fprintf(stderr, " Couldn't create IB resources\n");
+			return FAILURE;
+		}
+	}
+
+	/* Set up the Connection. */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	/* shaking hands and gather the other side info. */
+	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	user_comm.rdma_params->side = REMOTE;
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		/* shaking hands and gather the other side info. */
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
+			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
+			fprintf(stderr," Please Try to use a different IP version.\n\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
+			fprintf(stderr," Unable to Connect the HCA's through the link\n");
+			return FAILURE;
+		}
+	}
+
+	/* An additional handshake is required after moving qp to RTR. */
+	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	/* Only Client post read request. */
+	if (user_param.machine == SERVER) {
+		if (ctx_close_connection(&user_comm,my_dest,rem_dest)) {
+			fprintf(stderr,"Failed to close connection between server and client\n");
+			return FAILURE;
+		}
+		if (user_param.output == FULL_VERBOSITY) {
+			printf(RESULT_LINE);
+		}
+		return 0;
+	}
+
+	if (user_param.use_event) {
+		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return FAILURE;
+		}
+	}
+
+	ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+		printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+	if(run_iter_lat(&ctx,&user_param))
+		return 17;
+
+	user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
+
+	if (ctx_close_connection(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to close connection between server and client\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+	}
+
+	return 0;
+}

usr/benchmarks/ib/clock_test.c

@@ -0,0 +1,24 @@
+#include <unistd.h>
+#include <stdio.h>
+#include "get_clock.h"
+
+int main()
+{
+	int no_cpu_freq_fail = 0;
+	double mhz;
+	mhz = get_cpu_mhz(no_cpu_freq_fail);
+	cycles_t c1, c2;
+
+	if (!mhz) {
+		printf("Unable to calibrate cycles. Exiting.\n");
+		return 2;
+	}
+
+	printf("Type CTRL-C to cancel.\n");
+	for (;;) {
+		c1 = get_cycles();
+		sleep(1);
+		c2 = get_cycles();
+		printf("1 sec = %g usec\n", (c2 - c1) / mhz);
+	}
+}

usr/benchmarks/ib/get_clock.c

@@ -0,0 +1,234 @@
+/*
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ *
+ * Author: Michael S. Tsirkin <mst@mellanox.co.il>
+ */
+
+/* #define DEBUG 1 */
+/* #define DEBUG_DATA 1 */
+/* #define GET_CPU_MHZ_FROM_PROC 1 */
+
+/* For gettimeofday */
+#define _BSD_SOURCE
+#include <sys/time.h>
+
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include "get_clock.h"
+
+#ifndef DEBUG
+#define DEBUG 0
+#endif
+#ifndef DEBUG_DATA
+#define DEBUG_DATA 0
+#endif
+
+#define MEASUREMENTS 200
+#define USECSTEP 10
+#define USECSTART 100
+
+/*
+   Use linear regression to calculate cycles per microsecond.
+http://en.wikipedia.org/wiki/Linear_regression#Parameter_estimation
+*/
+static double sample_get_cpu_mhz(void)
+{
+	struct timeval tv1, tv2;
+	cycles_t start;
+	double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0;
+	double tx, ty;
+	int i;
+
+	/* Regression: y = a + b x */
+	long x[MEASUREMENTS];
+	cycles_t y[MEASUREMENTS];
+	double a; /* system call overhead in cycles */
+	double b; /* cycles per microsecond */
+	double r_2;
+
+	for (i = 0; i < MEASUREMENTS; ++i) {
+		start = get_cycles();
+
+		if (gettimeofday(&tv1, NULL)) {
+			fprintf(stderr, "gettimeofday failed.\n");
+			return 0;
+		}
+
+		do {
+			if (gettimeofday(&tv2, NULL)) {
+				fprintf(stderr, "gettimeofday failed.\n");
+				return 0;
+			}
+		} while ((tv2.tv_sec - tv1.tv_sec) * 1000000 +
+				(tv2.tv_usec - tv1.tv_usec) < USECSTART + i * USECSTEP);
+
+		x[i] = (tv2.tv_sec - tv1.tv_sec) * 1000000 +
+			tv2.tv_usec - tv1.tv_usec;
+		y[i] = get_cycles() - start;
+		if (DEBUG_DATA)
+			fprintf(stderr, "x=%ld y=%Ld\n", x[i], (long long)y[i]);
+	}
+
+	for (i = 0; i < MEASUREMENTS; ++i) {
+		tx = x[i];
+		ty = y[i];
+		sx += tx;
+		sy += ty;
+		sxx += tx * tx;
+		syy += ty * ty;
+		sxy += tx * ty;
+	}
+
+	b = (MEASUREMENTS * sxy - sx * sy) / (MEASUREMENTS * sxx - sx * sx);
+	a = (sy - b * sx) / MEASUREMENTS;
+
+	if (DEBUG)
+		fprintf(stderr, "a = %g\n", a);
+	if (DEBUG)
+		fprintf(stderr, "b = %g\n", b);
+	if (DEBUG)
+		fprintf(stderr, "a / b = %g\n", a / b);
+	r_2 = (MEASUREMENTS * sxy - sx * sy) * (MEASUREMENTS * sxy - sx * sy) /
+		(MEASUREMENTS * sxx - sx * sx) /
+		(MEASUREMENTS * syy - sy * sy);
+
+	if (DEBUG)
+		fprintf(stderr, "r^2 = %g\n", r_2);
+	if (r_2 < 0.9) {
+		fprintf(stderr,"Correlation coefficient r^2: %g < 0.9\n", r_2);
+		return 0;
+	}
+
+	return b;
+}
+
+#if !defined(__s390x__) && !defined(__s390__)
+static double proc_get_cpu_mhz(int no_cpu_freq_warn)
+{
+	FILE* f;
+	char buf[256];
+	double mhz = 0.0;
+	int print_flag = 0;
+	double delta;
+
+	#if defined(__FreeBSD__)
+	f = popen("/sbin/sysctl hw.clockrate","r");
+	#else
+	f = fopen("/proc/cpuinfo","r");
+	#endif
+
+	if (!f)
+		return 0.0;
+	while(fgets(buf, sizeof(buf), f)) {
+		double m;
+		int rc;
+
+		#if defined (__ia64__)
+		/* Use the ITC frequency on IA64 */
+		rc = sscanf(buf, "itc MHz : %lf", &m);
+		#elif defined (__PPC__) || defined (__PPC64__)
+		/* PPC has a different format as well */
+		rc = sscanf(buf, "clock : %lf", &m);
+		#elif defined (__sparc__) && defined (__arch64__)
+		/*
+		 * on sparc the /proc/cpuinfo lines that hold
+		 * the cpu freq in HZ are as follow:
+		 * Cpu{cpu-num}ClkTck      : 00000000a9beeee4
+		 */
+		char *s;
+
+		s = strstr(buf, "ClkTck\t: ");
+		if (!s)
+			continue;
+		s += (strlen("ClkTck\t: ") - strlen("0x"));
+		strncpy(s, "0x", strlen("0x"));
+		rc = sscanf(s, "%lf", &m);
+		m /= 1000000;
+		#else
+		#if defined (__FreeBSD__)
+		rc = sscanf(buf, "hw.clockrate: %lf", &m);
+		#else
+		rc = sscanf(buf, "cpu MHz : %lf", &m);
+		#endif
+		#endif
+
+		if (rc != 1)
+			continue;
+
+		if (mhz == 0.0) {
+			mhz = m;
+			continue;
+		}
+		delta = mhz > m ? mhz - m : m - mhz;
+		if ((delta / mhz > 0.02) && (print_flag ==0)) {
+			print_flag = 1;
+			if (!no_cpu_freq_warn) {
+				fprintf(stderr, "Conflicting CPU frequency values"
+						" detected: %lf != %lf. CPU Frequency is not max.\n", mhz, m);
+			}
+			continue;
+		}
+	}
+
+#if defined(__FreeBSD__)
+	pclose(f);
+#else
+	fclose(f);
+#endif
+	return mhz;
+}
+#endif
+
+double get_cpu_mhz(int no_cpu_freq_warn)
+{
+	#if defined(__s390x__) || defined(__s390__)
+	return sample_get_cpu_mhz();
+	#else
+	double sample, proc, delta;
+	sample = sample_get_cpu_mhz();
+	proc = proc_get_cpu_mhz(no_cpu_freq_warn);
+	#ifdef __aarch64__
+	if (proc < 1)
+		proc = sample;
+	#endif
+	if (!proc || !sample)
+		return 0;
+
+	delta = proc > sample ? proc - sample : sample - proc;
+	if (delta / proc > 0.02) {
+		return sample;
+	}
+	return proc;
+#endif
+}

usr/benchmarks/ib/get_clock.h

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ *
+ * Author: Michael S. Tsirkin <mst@mellanox.co.il>
+ */
+
+#ifndef GET_CLOCK_H
+#define GET_CLOCK_H
+
+#if defined (__x86_64__) || defined(__i386__)
+/* Note: only x86 CPUs which have rdtsc instruction are supported. */
+typedef unsigned long long cycles_t;
+static inline cycles_t get_cycles()
+{
+	unsigned low, high;
+	unsigned long long val;
+	asm volatile ("rdtsc" : "=a" (low), "=d" (high));
+	val = high;
+	val = (val << 32) | low;
+	return val;
+}
+#elif defined(__PPC__) || defined(__PPC64__)
+/* Note: only PPC CPUs which have mftb instruction are supported. */
+/* PPC64 has mftb */
+typedef unsigned long cycles_t;
+static inline cycles_t get_cycles()
+{
+	cycles_t ret;
+
+	__asm__ __volatile__ ("\n\t isync" "\n\t mftb %0" : "=r"(ret));
+	return ret;
+}
+#elif defined(__ia64__)
+/* Itanium2 and up has ar.itc (Itanium1 has errata) */
+typedef unsigned long cycles_t;
+static inline cycles_t get_cycles()
+{
+	cycles_t ret;
+
+	asm volatile ("mov %0=ar.itc" : "=r" (ret));
+	return ret;
+}
+#elif defined(__ARM_ARCH_7A__)
+typedef unsigned long long cycles_t;
+static inline cycles_t get_cycles(void)
+{
+	cycles_t        clk;
+	asm volatile("mrrc p15, 0, %Q0, %R0, c14" : "=r" (clk));
+	return clk;
+}
+#elif defined(__s390x__) || defined(__s390__)
+typedef unsigned long long cycles_t;
+static inline cycles_t get_cycles(void)
+{
+	cycles_t        clk;
+	asm volatile("stck %0" : "=Q" (clk) : : "cc");
+	return clk >> 2;
+}
+#elif defined(__sparc__) && defined(__arch64__)
+typedef unsigned long long cycles_t;
+static inline cycles_t get_cycles(void)
+{
+	cycles_t v;
+	asm volatile ("rd %%tick, %0" : "=r" (v) : );
+	return v;
+}
+#elif defined(__aarch64__)
+
+typedef unsigned long cycles_t;
+static inline cycles_t get_cycles()
+{
+	cycles_t cval;
+	asm volatile("isb" : : : "memory");
+	asm volatile("mrs %0, cntvct_el0" : "=r" (cval));
+	return cval;
+}
+
+#else
+#warning get_cycles not implemented for this architecture: attempt asm/timex.h
+#include <asm/timex.h>
+#endif
+
+extern double get_cpu_mhz(int);
+
+#endif

usr/benchmarks/ib/multicast_resources.c

@@ -0,0 +1,266 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <netdb.h>
+#include <time.h>
+#include <limits.h>
+#include <unistd.h>
+#if !defined(__FreeBSD__)
+#include <malloc.h>
+#endif
+#include <getopt.h>
+#include <errno.h>
+#if defined(__FreeBSD__)
+#include <infiniband/byteswap.h>
+#else
+#include <byteswap.h>
+#endif
+#include <signal.h>
+#include <pthread.h>
+#include "multicast_resources.h"
+
+/* This is when we get sig handler from the user before we remove the join request. */
+struct mcast_parameters *sighandler_params;
+
+/******************************************************************************
+ * signalCatcher - cacth user signal in order to reregiser the mcast group
+ ******************************************************************************/
+static void signalCatcher (int sig)
+{
+	if (sig == SIGINT) {
+
+		if (join_multicast_group(SUBN_ADM_METHOD_DELETE,sighandler_params))
+			fprintf(stderr,"Couldn't Unregister the Mcast group on the SM\n");
+
+		if (sighandler_params->is_2nd_mgid_used) {
+			memcpy(sighandler_params->mgid.raw,sighandler_params->base_mgid.raw,16);
+			if (join_multicast_group(SUBN_ADM_METHOD_DELETE,sighandler_params))
+				fprintf(stderr,"Couldn't Unregister the Base Mcast group on the SM\n");
+		}
+	}
+	exit(1);
+}
+
+/******************************************************************************
+ * prepare_mcast_mad
+ ******************************************************************************/
+static void prepare_mcast_mad(uint8_t method,
+		struct mcast_parameters *params,
+		struct sa_mad_packet_t *samad_packet)
+{
+	uint8_t *ptr;
+	uint64_t comp_mask;
+
+	memset(samad_packet,0,sizeof(*samad_packet));
+
+	/* prepare the MAD header. according to Table 145 in IB spec 1.2.1 */
+	ptr = samad_packet->mad_header_buf;
+	ptr[0]                     = 0x01;					/* BaseVersion */
+	ptr[1]                     = MANAGMENT_CLASS_SUBN_ADM;			/* MgmtClass */
+	ptr[2]                     = 0x02; 					/* ClassVersion */
+	ptr[3]                     = INSERTF(ptr[3], 0, method, 0, 7); 		/* Method */
+	(*(uint64_t *)(ptr + 8))   = htonll((uint64_t)DEF_TRANS_ID);             /* TransactionID */
+	(*(uint16_t *)(ptr + 16))  = htons(SUBN_ADM_ATTR_MC_MEMBER_RECORD);      /* AttributeID */
+
+	ptr = samad_packet->SubnetAdminData;
+
+	memcpy(&ptr[0],params->mgid.raw, 16);
+	memcpy(&ptr[16],params->port_gid.raw, 16);
+
+	(*(uint32_t *)(ptr + 32)) = htonl(DEF_QKEY);
+	(*(uint16_t *)(ptr + 40)) = htons(params->pkey);
+	ptr[39]                    = DEF_TCLASS;
+	ptr[44]                    = INSERTF(ptr[44], 4, DEF_SLL, 0, 4);
+	ptr[44]                    = INSERTF(ptr[44], 0, DEF_FLOW_LABLE, 16, 4);
+	ptr[45]                    = INSERTF(ptr[45], 0, DEF_FLOW_LABLE, 8, 8);
+	ptr[46]                    = INSERTF(ptr[46], 0, DEF_FLOW_LABLE, 0, 8);
+	ptr[48]                    = INSERTF(ptr[48], 0, MCMEMBER_JOINSTATE_FULL_MEMBER, 0, 4);
+
+	comp_mask = SUBN_ADM_COMPMASK_MGID | SUBN_ADM_COMPMASK_PORT_GID | SUBN_ADM_COMPMASK_Q_KEY |
+		SUBN_ADM_COMPMASK_P_KEY | SUBN_ADM_COMPMASK_TCLASS | SUBN_ADM_COMPMASK_SL |
+		SUBN_ADM_COMPMASK_FLOW_LABEL | SUBN_ADM_COMPMASK_JOIN_STATE;
+
+	samad_packet->ComponentMask = htonll(comp_mask);
+}
+
+/******************************************************************************
+ * check_mad_status
+ ******************************************************************************/
+static int check_mad_status(struct sa_mad_packet_t *samad_packet)
+{
+	uint8_t *ptr;
+	uint32_t user_trans_id;
+	uint16_t mad_header_status;
+
+	ptr = samad_packet->mad_header_buf;
+
+	/* the upper 32 bits of TransactionID were set by the kernel */
+	user_trans_id = ntohl(*(uint32_t *)(ptr + 12));
+
+	/* check the TransactionID to make sure this is the response */
+	/* for the join/leave multicast group request we posted */
+	if (user_trans_id != DEF_TRANS_ID) {
+		fprintf(stderr, "received a mad with TransactionID 0x%x, when expecting 0x%x\n",
+				(unsigned int)user_trans_id, (unsigned int)DEF_TRANS_ID);;
+		return 1;
+	}
+
+	mad_header_status = 0x0;
+	mad_header_status = INSERTF(mad_header_status, 8, ptr[4], 0, 7);
+	mad_header_status = INSERTF(mad_header_status, 0, ptr[5], 0, 8);
+
+	if (mad_header_status) {
+		fprintf(stderr,"received UMAD with an error: 0x%x\n", mad_header_status);
+		return 1;
+	}
+
+	return 0;
+}
+
+
+/******************************************************************************
+ * get_mlid_from_mad
+ ******************************************************************************/
+static void get_mlid_from_mad(struct sa_mad_packet_t *samad_packet,uint16_t *mlid)
+{
+	uint8_t *ptr;
+	ptr = samad_packet->SubnetAdminData;
+	*mlid = ntohs(*(uint16_t *)(ptr + 36));
+}
+
+/******************************************************************************
+ * set_multicast_gid
+ ******************************************************************************/
+void set_multicast_gid(struct mcast_parameters *params,uint32_t qp_num,int is_client)
+{
+	uint8_t mcg_gid[16] = MCG_GID;
+	const char *pstr = params->user_mgid;
+	char *term = NULL;
+	char tmp[20];
+	int i;
+
+	if (params->user_mgid) {
+		term = strpbrk(pstr, ":");
+		memcpy(tmp, pstr, term - pstr+1);
+		tmp[term - pstr] = 0;
+
+		mcg_gid[0] = (unsigned char)strtoll(tmp, NULL, 0);
+
+		for (i = 1; i < 15; ++i) {
+			pstr += term - pstr + 1;
+			term = strpbrk(pstr, ":");
+			memcpy(tmp, pstr, term - pstr+1);
+			tmp[term - pstr] = 0;
+
+			mcg_gid[i] = (unsigned char)strtoll(tmp, NULL, 0);
+		}
+		pstr += term - pstr + 1;
+
+		strcpy(tmp, pstr);
+		mcg_gid[15] = (unsigned char)strtoll(tmp, NULL, 0);
+	}
+
+	memcpy(params->mgid.raw,mcg_gid,16);
+	if (is_client && params->user_mgid==NULL)
+		params->mgid.raw[15]++;
+}
+
+/******************************************************************************
+ * join_multicast_group
+ ******************************************************************************/
+int join_multicast_group(subn_adm_method method,struct mcast_parameters *params)
+{
+	int portid = -1;
+	int agentid = -1;
+	void *umad_buff = NULL;
+	void *mad = NULL;
+	int length = MAD_SIZE;
+	int test_result = 0;
+
+	/* mlid will be assigned to the new LID after the join */
+	if (umad_init() < 0) {
+		fprintf(stderr, "failed to init the UMAD library\n");
+		goto cleanup;
+	}
+	/* use casting to loose the "const char0 *" */
+	portid = umad_open_port((char*)params->ib_devname,params->ib_port);
+	if (portid < 0) {
+		fprintf(stderr,"failed to open UMAD port %d\n",params->ib_port);
+		goto cleanup;
+	}
+
+	agentid = umad_register(portid,MANAGMENT_CLASS_SUBN_ADM, 2, 0, 0);
+	if (agentid < 0) {
+		fprintf(stderr,"failed to register UMAD agent for MADs\n");
+		goto cleanup;
+	}
+
+	umad_buff = umad_alloc(1, umad_size() + MAD_SIZE);
+	if (!umad_buff) {
+		fprintf(stderr, "failed to allocate MAD buffer\n");
+		goto cleanup;
+	}
+
+	mad = umad_get_mad(umad_buff);
+	prepare_mcast_mad(method,params,(struct sa_mad_packet_t *)mad);
+
+	if (umad_set_addr(umad_buff,params->sm_lid,1,params->sm_sl,QP1_WELL_KNOWN_Q_KEY) < 0) {
+		fprintf(stderr, "failed to set the destination address of the SMP\n");
+		goto cleanup;
+	}
+
+	if (umad_send(portid,agentid,umad_buff,MAD_SIZE,100,5) < 0) {
+		fprintf(stderr, "failed to send MAD\n");
+		goto cleanup;
+	}
+
+	if (umad_recv(portid,umad_buff,&length,5000) < 0) {
+		fprintf(stderr, "failed to receive MAD response\n");
+		goto cleanup;
+	}
+
+	if (check_mad_status((struct sa_mad_packet_t*)mad)) {
+		fprintf(stderr, "failed to get mlid from MAD\n");
+		goto cleanup;
+	}
+
+	/*  "Join multicast group" message was sent */
+	if (method == SUBN_ADM_METHOD_SET) {
+		get_mlid_from_mad((struct sa_mad_packet_t*)mad,&params->mlid);
+		params->mcast_state |= MCAST_IS_JOINED;
+		if (params->is_2nd_mgid_used == 0) {
+			sighandler_params = params;
+			signal(SIGINT,signalCatcher);
+		}
+	} else {
+		params->mcast_state &= ~MCAST_IS_JOINED;
+	}
+
+cleanup:
+	if (umad_buff)
+		umad_free(umad_buff);
+
+	if (portid >= 0) {
+		if (agentid >= 0) {
+			if (umad_unregister(portid, agentid)) {
+				fprintf(stderr, "failed to deregister UMAD agent for MADs\n");
+				test_result = 1;
+			}
+		}
+
+		if (umad_close_port(portid)) {
+			fprintf(stderr, "failed to close UMAD portid\n");
+			test_result = 1;
+		}
+	}
+
+	return test_result;
+}
+
+/******************************************************************************
+ * End
+ ******************************************************************************/

usr/benchmarks/ib/multicast_resources.h

@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2009 Mellanox Technologies Ltd.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Author: Ido Shamay <idos@dev.mellanox.co.il>
+ */
+
+#ifndef MULTICAST_RESOURCES_H
+#define MULTICAST_RESOURCES_H
+
+/* Multicast Module for perftest.
+ *
+ * Description :
+ *
+ *   This file contains the structures and methods for implementing a multiple
+ *   multicast groups in user space enviroment.
+ *	  The module is in use in "send_bw" and "send_lat" ,but can be used on other
+ *	  applications and can generate more methods and serve more benchmarks.
+ *   The Module uses only the structire defined here , enabling generic use of it.
+ *
+ * Defined Types :
+ *
+ *   mcast_parameters - Contains all the parameters needed for this module.
+ *   mcast_group      - The multicast group entitiy itself.
+ *   mcg_qp		   - Is a QP structure that is attahced to the group.
+ *
+ */
+
+
+/************************************************************************
+ *   Macros , Defines and Files included for work.	    			    *
+ ************************************************************************/
+
+#include <infiniband/verbs.h>
+#include <infiniband/umad.h>
+#include "get_clock.h"
+
+#define QPNUM_MCAST 				0xffffff
+#define DEF_QKEY     			    0x11111111
+#define DEF_PKEY_IDX        		0
+#define DEF_SLL              		0
+#define MAX_POLL_ITERATION_TIMEOUT  1000000
+#define MCG_GID {255,1,0,0,0,2,201,133,0,0,0,0,0,0,0,0}
+
+/* Definitions section for MADs */
+#define SUBN_ADM_ATTR_MC_MEMBER_RECORD 0x38
+#define MANAGMENT_CLASS_SUBN_ADM       0x03 	  /* Subnet Administration class */
+#define MCMEMBER_JOINSTATE_FULL_MEMBER 0x1
+#define MAD_SIZE                       256	      /* The size of a MAD is 256 bytes */
+#define QP1_WELL_KNOWN_Q_KEY           0x80010000 /* Q_Key value of QP1 */
+#define DEF_TRANS_ID                   0x12345678 /* TransactionID */
+#define DEF_TCLASS                     0
+#define DEF_FLOW_LABLE                 0
+
+/* Macro for 64 bit variables to switch to from net */
+#ifndef ntohll
+#define ntohll(x) (((uint64_t)(ntohl((int)((x << 32) >> 32))) << 32) | (unsigned int)ntohl(((int)(x >> 32))))
+#endif
+#ifndef htonll
+#define htonll(x) ntohll(x)
+#endif
+
+/* generate a bit mask S bits width */
+#define MASK32(S)  ( ((uint32_t) ~0L) >> (32-(S)) )
+
+/* generate a bit mask with bits O+S..O set (assumes 32 bit integer). */
+#define BITS32(O,S) ( MASK32(S) << (O) )
+
+/* extract S bits from (u_int32_t)W with offset O and shifts them O places to the right */
+#define EXTRACT32(W,O,S) ( ((W)>>(O)) & MASK32(S) )
+
+/* insert S bits with offset O from field F into word W (u_int32_t) */
+#define INSERT32(W,F,O,S) (/*(W)=*/ ( ((W) & (~BITS32(O,S)) ) | (((F) & MASK32(S))<<(O)) ))
+
+#ifndef INSERTF
+#define INSERTF(W,O1,F,O2,S) (INSERT32(W, EXTRACT32(F, O2, S), O1, S) )
+#endif
+
+
+/* according to Table 187 in the IB spec 1.2.1 */
+typedef enum {
+	SUBN_ADM_METHOD_SET    = 0x2,
+	SUBN_ADM_METHOD_DELETE = 0x15
+} subn_adm_method;
+
+/* Utilities for Umad Usage. */
+typedef enum {
+	SUBN_ADM_COMPMASK_MGID         = (1ULL << 0),
+	SUBN_ADM_COMPMASK_PORT_GID     = (1ULL << 1),
+	SUBN_ADM_COMPMASK_Q_KEY	       = (1ULL << 2),
+	SUBN_ADM_COMPMASK_P_KEY	       = (1ULL << 7),
+	SUBN_ADM_COMPMASK_TCLASS       = (1ULL << 6),
+	SUBN_ADM_COMPMASK_SL           = (1ULL << 12),
+	SUBN_ADM_COMPMASK_FLOW_LABEL   = (1ULL << 13),
+	SUBN_ADM_COMPMASK_JOIN_STATE   = (1ULL << 16),
+} subn_adm_component_mask;
+
+typedef enum {
+	MCAST_IS_JOINED   = 1,
+	MCAST_IS_ATTACHED = (1 << 1)
+} mcast_state;
+
+
+/************************************************************************
+ *   Multicast data structures.						    			    *
+ ************************************************************************/
+
+/* Needed parameters for creating a multiple multicast group entity. */
+struct mcast_parameters {
+	int             	num_qps_on_group;
+	int			is_user_mgid;
+	int			mcast_state;
+	int 			ib_port;
+	uint16_t		mlid;
+	uint16_t		base_mlid;
+	const char		*user_mgid;
+	char			*ib_devname;
+	uint16_t 		pkey;
+	uint16_t		sm_lid;
+	uint8_t 		sm_sl;
+	union ibv_gid 		port_gid;
+	union ibv_gid		mgid;
+	/* In case it's a latency test. */
+	union ibv_gid		base_mgid;
+	int 			is_2nd_mgid_used;
+};
+
+/* according to Table 195 in the IB spec 1.2.1 */
+
+struct sa_mad_packet_t {
+	u_int8_t		mad_header_buf[24];
+	u_int8_t		rmpp_header_buf[12];
+	u_int64_t		SM_Key;
+	u_int16_t		AttributeOffset;
+	u_int16_t		Reserved1;
+	u_int64_t		ComponentMask;
+	u_int8_t		SubnetAdminData[200];
+}__attribute__((packed));
+
+/************************************************************************
+ *   Multicast resources methods.					    			    *
+ ************************************************************************/
+
+/* set_multicast_gid .
+ *
+ * Description :
+ *
+ *  Sets the Multicast GID , and stores it in the "mgid" value of
+ *  mcast resourcs. If the user requested for a specific MGID, which
+ *  is stored in params->user_mgid (in this case params->is_user_mgid should be 1)
+ *  than it will be his MGID, if not the library choose a default one.
+ *
+ * Parameters :
+ *
+ *  params            - The parameters of the machine
+ *  my_dest ,rem_dest - The 2 sides that ends the connection.
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+void set_multicast_gid(struct mcast_parameters *params,uint32_t qp_num,int is_client);
+
+
+/* ctx_close_connection .
+ *
+ * Description :
+ *
+ *  Close the connection between the 2 machines.
+ *  It performs an handshake to ensure the 2 sides are there.
+ *
+ * Parameters :
+ *
+ *  params            - The parameters of the machine
+ *  my_dest ,rem_dest - The 2 sides that ends the connection.
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int join_multicast_group(subn_adm_method method,struct mcast_parameters *params);
+
+
+#endif /* MULTICAST_RESOURCES_H */

usr/benchmarks/ib/perftest_communication.h

@@ -0,0 +1,444 @@
+/*
+ * Copyright (c) 2011 Mellanox Technologies Ltd.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Author: Ido Shamay <idos@dev.mellanox.co.il>
+ *
+ * Description : ...
+ *
+ * Methods : ...
+ */
+
+#ifndef PERFTEST_COMMUNICATION_H
+#define PERFTEST_COMMUNICATION_H
+
+#include <netinet/in.h>
+
+// #include <infiniband/verbs.h>
+#include <hermit/ibv.h>
+#include <hermit/verbs.h>
+
+// #include <rdma/rdma_cma.h>
+#include "perftest_resources.h"
+
+/* Macro for 64 bit variables to switch to/from net */
+#if __BYTE_ORDER == __BIG_ENDIAN || __BYTE_ORDER == __LITTLE_ENDIAN
+#if __BYTE_ORDER == __BIG_ENDIAN
+#define ntoh_64(x) (x)
+#define hton_64(x) (x)
+#define ntoh_double(x) (x)
+#define hton_double(x) (x)
+#else
+#define ntoh_64(x) bswap_64(x)
+#define hton_64(x) bswap_64(x)
+#define ntoh_double(x) bswap_double(x)
+#define hton_double(x) bswap_double(x)
+#endif
+#else
+#error "Only BIG_ENDIAN and LITTLE_ENDIAN are supported."
+#endif
+
+/* long is 64-bit in LP64 mode, 32-bit in LLP64 mode. */
+#if defined(_LP64) || defined(__LP64__)
+#define ntoh_long(x) ntoh_64(x)
+#define hton_long(x) hton_64(x)
+#else
+#define ntoh_long(x) ntohl(x)
+#define hton_long(x) htonl(x)
+#endif
+
+/* int is 32-bit in both LP64 and LLP64 modes. */
+#define ntoh_int(x) (int) ntohl((uint32_t) (x))
+#define hton_int(x) (int) htonl((uint32_t) (x))
+
+#define KEY_MSG_SIZE 	 (59)   /* Message size without gid. */
+#define KEY_MSG_SIZE_GID (108)   /* Message size with gid (MGID as well). */
+#define SYNC_SPEC_ID	 (5)
+
+/* The Format of the message we pass through sockets , without passing Gid. */
+#define KEY_PRINT_FMT "%04x:%04x:%06x:%06x:%08x:%016Lx:%08x"
+
+/* The Format of the message we pass through sockets (With Gid). */
+#define KEY_PRINT_FMT_GID "%04x:%04x:%06x:%06x:%08x:%016Lx:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%08x:"
+
+/* The Basic print format for all verbs. */
+#define BASIC_ADDR_FMT " %s address: LID %#04x QPN %#06x PSN %#06x"
+
+/* Addition format string for READ - the outstanding reads. */
+#define READ_FMT       " OUT %#04x"
+
+/* The print format of the pingpong_dest element for RDMA verbs. */
+#define RDMA_FMT       " RKey %#08x VAddr %#016Lx"
+
+/* The print number of SRQ in case of XRC */
+#define XRC_FMT 	   " SRQn %#08x"
+#define DC_FMT         " SRQn %#08x"
+
+/* The print format of a global address or a multicast address. */
+#define PERF_GID_FMT " %s: %02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d\n"
+
+/* The print format of a global address or a multicast address. */
+#define PERF_RAW_MGID_FMT " %s: %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n"
+
+struct perftest_comm {
+	struct pingpong_context    *rdma_ctx;
+	struct perftest_parameters *rdma_params;
+};
+
+/* bswap_double
+ *
+ * Description : swap byte order for double.
+ *
+ * Parameters :
+ *  x -	input double variable
+ *
+ *  Return Value : double after byte order being swapped.
+ */
+double bswap_double(double x);
+
+
+/* create_comm_struct
+ *
+ * Description : Creating the communication struct for Etherent or rdma_cm options.
+ *
+ * Parameters :
+ *	comm - An empty Communication struct.
+ *  user_param -  Perftest parameters.
+ *
+ * Return Value : SUCCESS,FAILURE.
+ */
+int create_comm_struct (struct perftest_comm *comm,
+		struct perftest_parameters *user_param);
+
+
+/* set_up_connection .
+ *
+ * Description : Fills the my_dest with all of the machine proporties.
+ *
+ *
+ * Parameters :
+ *  ctx 		- Pingoong context after the ctx_init function.
+ *  user_param   - Perftest parameters.
+ *  my_dest		- An empty pingpong_dest struct.
+ *
+ * Return Value : SUCCESS,FAILURE.
+ */
+int set_up_connection(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct pingpong_dest *my_dest);
+
+/* establish_connection .
+ *
+ * Description :
+ *
+ *  Connect the client the a well known server to a requested port.
+ *  It assumes the Server is waiting for request on the port.
+ *  It uses Ethernet sockets or rdma_cm as mentioned in use_rdma_cm.
+ *
+ * Parameters :
+ *		comm - The communication struct with all the data.
+ *
+ * Return Value : SUCCESS,FAILURE.
+ */
+int establish_connection(struct perftest_comm *comm);
+
+/* rdma_client_connect .
+ *
+ * Description : Connects the client to a QP on the other machine with rdma_cm.
+ *
+ * Parameters :
+ *		ctx - An empty resources struct to fill the resources created for this QP.
+ *		user_param - Perftest parameters.
+ *
+ * Return Value : SUCCESS,FAILURE.
+ */
+int rdma_client_connect(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param);
+
+/* retry_rdma_connect .
+ *
+ * Description : Retries rdma_client_connect() because the listener may not be ready
+ *               when the rdma client attempts to connect
+ *
+ * Parameters :
+ *		ctx - An empty resources struct to fill the resources created for this QP.
+ *		user_param - Perftest parameters.
+ *
+ * Return Value : SUCCESS,FAILURE.
+ */
+int retry_rdma_connect(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param);
+
+/* rdma_server_connect .
+ *
+ * Description : Assinging a server to listen on a rdma_cm port and connect to it.
+ *
+ * Parameters :
+ *		ctx - An empty resources struct to fill the resources created for this QP.
+ *		user_param - Perftest parameters.
+ *
+ * Return Value : SUCCESS,FAILURE.
+ */
+int rdma_server_connect(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param);
+/* ctx_hand_shake .
+ *
+ * Description :
+ *
+ *  Exchanging the data , represented in struct pingpong_dest , between
+ *  a server and client that performed the ctx_server/clinet_connect.
+ *  The method fills in rem_dest the remote machine data , and passed the data
+ *  in my_dest to other machine.
+ *
+ * Parameters :
+ *
+ *  params   - The parameters needed for this method. Are mentioned above ,and
+ *             contains standard IB info. (exists on perftest).
+ *  my_dest  - Contains the data you want to pass to the other side.
+ *  rem_dest - The other side data.
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int ctx_hand_shake(struct perftest_comm *comm,
+		struct pingpong_dest *my_dest,
+		struct pingpong_dest *rem_dest);
+
+
+
+/* ctx_print_pingpong_data.
+ *
+ * Description :
+ *
+ *  Prints the data stored in the struct pingpong_dest.
+ *
+ * Parameters :
+ *
+ *  params  - The parameters of the machine.
+ *  element - The element to print.
+ */
+void ctx_print_pingpong_data(struct pingpong_dest *element,
+		struct perftest_comm *comm);
+
+/* ctx_close_connection .
+ *
+ * Description :
+ *
+ *  Close the connection between the 2 machines.
+ *  It performs an handshake to ensure the 2 sides are there.
+ *
+ * Parameters :
+ *
+ *  params            - The parameters of the machine
+ *  my_dest ,rem_dest - The 2 sides that ends the connection.
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int ctx_close_connection(struct perftest_comm *comm,
+		struct pingpong_dest *my_dest,
+		struct pingpong_dest *rem_dest);
+
+/* ctx_xchg_data .
+ *
+ * Description :
+ *
+ *  Exchanging data between
+ *  a server and client after performing ctx_server/client_connect.
+ *  The method fills in rem_data the remote machine data , and passed the data
+ *  in my_dest to other machine.
+ *
+ * Parameters :
+ *
+ *  comm   - contains connections info
+ *  my_data  - Contains the data you want to pass to the other side.
+ *  rem_data - The other side data.
+ *  size - size of my_data (after casting is made)
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int ctx_xchg_data( struct perftest_comm *comm,
+		void *my_data,
+		void *rem_data,int size);
+
+/* ethernet_write_data .
+ *
+ * Description :
+ *
+ *  Sends data that is written in msg using ethernet
+ *  This functions can send any basic type (int,float,double,char*,string, etc..).
+ *	If you need to send a struct, decoder and encoder must be implemented to convert
+ *  the struct to a string
+ *
+ * Parameters :
+ *
+ *  comm   - contains connections info
+ *  msg    - message that will be sent
+ *  size   - size of the message
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int ethernet_write_data(struct perftest_comm *comm, char *msg, size_t size);
+
+/* ethernet_read_data .
+ *
+ * Description :
+ *
+ *  Read data from remote machine using ethernet.
+ *
+ * Parameters :
+ *
+ *  comm   - contains connections info
+ *  recv_msg    - function will return, in this argument, the message from remote machine
+ *  size   - size of the message
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int ethernet_read_data(struct perftest_comm *comm, char *recv_msg, size_t size);
+
+/* rdma_write_data .
+ *
+ * Description :
+ *
+ *  Sends data that to remote machine using RDMA.
+ *  This functions can send any variable type
+ *
+ * Parameters :
+ *
+ *  data   - data that will be sent
+ *  comm   - contains connections info
+ *  size    - size of data
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int rdma_write_data(void *data,	struct perftest_comm *comm, int size);
+
+/* rdma_read_data .
+ *
+ * Description :
+ *
+ *  Reads  data from remote machine using RDMA.
+ *  This functions can read any variable type
+ *
+ * Parameters :
+ *
+ *  data   - data that will be sent
+ *  comm   - contains connections info
+ *  size    - size of data
+ * Return Value : 0 upon success. -1 if it fails.
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int rdma_read_data(void *data, struct perftest_comm *comm, int size);
+
+/* ctx_xchg_data .
+ *
+ * Description :
+ *
+ *  Implements ctx_xchg_data for ethernet
+ *
+ * Parameters :
+ *
+ *  comm   - contains connections info
+ *  my_data  - Contains the data you want to pass to the other side.
+ *  rem_data - The other side data.
+ *  size - size of my_data (after casting is made)
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int ctx_xchg_data_ethernet( struct perftest_comm *comm, void *my_data, void *rem_data,int size);
+
+/* ctx_xchg_data .
+ *
+ * Description :
+ *
+ *  Implements ctx_xchg_data for RDMA
+ *
+ * Parameters :
+ *
+ *  comm   - contains connections info
+ *  my_data  - Contains the data you want to pass to the other side.
+ *  rem_data - The other side data.
+ *  size - size of my_data (after casting is made)
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int ctx_xchg_data_rdma( struct perftest_comm *comm, void *my_data, void *rem_data,int size);
+
+/* ctx_xchg_data .
+ *
+ * Description :
+ *
+ *  Exchanging bw reports between
+ *  a server and client after performing ctx_server/client_connect.
+ *  The method fills in rem_data the remote machine data , and passed the data
+ *  in my_dest to other machine.
+ *
+ * Parameters :
+ *
+ *  comm   - contains connections info
+ *  my_bw_rep  - Contains the data you want to pass to the other side.
+ *  rem_bw_rep - The other side data.
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+void xchg_bw_reports (struct perftest_comm *comm, struct bw_report_data *my_bw_rep,
+		struct bw_report_data *rem_bw_rep, float remote_version);
+
+/* exchange_versions.
+ *
+ * Description :
+ * 	Exchange versions between sides.
+ *
+ */
+void exchange_versions (struct perftest_comm *user_comm, struct perftest_parameters *user_param);
+
+/* check_sys_data.
+ *
+ * Description :
+ * 	Exchange system data between sides.
+ *
+ */
+void check_sys_data(struct perftest_comm *user_comm, struct perftest_parameters *user_param);
+
+/* check_mtu
+ *
+ * Description : Configures test MTU.
+ *
+ * Parameters :
+ *
+ *	 context    - Context of the device.
+ *	 user_param - Perftest parameters.
+ *   user_comm	- user communication struct.
+ * Return Value : SUCCESS, FAILURE.
+ */
+int check_mtu(struct ibv_context *context,struct perftest_parameters *user_param, struct perftest_comm *user_comm);
+
+int ctx_check_gid_compatibility(struct pingpong_dest *my_dest,
+		struct pingpong_dest *rem_dest);
+
+#endif /* PERFTEST_COMMUNICATION_H */
+
+
+

usr/benchmarks/ib/perftest_parameters.h

@@ -0,0 +1,707 @@
+/*
+ * Copyright (c) 2009 Mellanox Technologies Ltd.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Author: Ido Shamay <idos@dev.mellanox.co.il>
+ *
+ * Description :
+ *
+ *  This API defines structs, formats and enums for all perftest benchmarks.
+ *  It includes parameters parser, and a report generator.
+ *
+ * Methods :
+ *
+ *  link_layer_str  - Return a String representation of the link type.
+ *  parser - Setting default test parameters and parsing the user choices.
+ *  check_link - Configures test MTU,inline and link layer of the test.
+ *  check_link_and_mtu     - Configures test MTU,inline and link layer of the test.
+ *  print_report_bw - Calculate the peak and average throughput of the BW test.
+ *  print_full_bw_report    - Print the peak and average throughput of the BW test.
+ *  print_report_lat - Print the min/max/median latency samples taken from a latency test.
+ *  print_report_lat_duration     - Prints only the avergae latency for samples taken from
+ *									a latency test with Duration..
+ *  set_mtu - set MTU from the port or user.
+ *  set_eth_mtu    - set MTU for Raw Ethernet tests.
+ */
+
+#ifndef PERFTEST_PARAMETERS_H
+#define PERFTEST_PARAMETERS_H
+
+// #include <infiniband/verbs.h>
+#include <hermit/ibv.h>
+#include <hermit/verbs.h>
+
+#include <unistd.h>
+
+#include <malloc.h>
+
+#include "get_clock.h"
+
+// #ifdef HAVE_CONFIG_H
+// #include <config.h>
+// #endif
+
+/* Connection types available. */
+#define RC  (0)
+// #define UC  (1)
+#define UD  (2)
+// #define RawEth  (3)
+// #define XRC (4)
+// #define DC  (5)
+
+/* Genral control definitions */
+#define OFF	     (0)
+#define ON 	     (1)
+#define SUCCESS	     (0)
+#define FAILURE	     (1)
+#define VERSION_EXIT (10)
+#define HELP_EXIT	 (11)
+#define MTU_FIX	     (7)
+#define MAX_SIZE     (8388608)
+#define LINK_FAILURE (-1)
+#define LINK_UNSPEC (-2)
+#define MAX_OUT_READ_HERMON (16)
+#define MAX_OUT_READ        (4)
+#define UD_ADDITION         (40)
+#define RAWETH_ADDITION    (18)
+#define HW_CRC_ADDITION    (4)
+
+/* Default Values of perftest parameters */
+#define DEF_PORT            (18515)
+#define DEF_IB_PORT         (1)
+#define DEF_IB_PORT2        (2)
+#define DEF_SIZE_BW         (65536)
+#define DEF_SIZE_LAT        (2)
+#define DEF_ITERS           (1000)
+#define DEF_ITERS_WB        (5000)
+#define DEF_TX_BW           (128)
+#define DEF_TX_LAT          (1)
+#define DEF_QP_TIME         (14)
+#define DEF_SL              (0)
+#define DEF_GID_INDEX       (-1)
+#define DEF_NUM_QPS         (1)
+#define DEF_RX_RDMA         (1)
+#define DEF_RX_SEND         (512)
+#define DEF_CQ_MOD          (100)
+#define DEF_SIZE_ATOMIC     (8)
+#define DEF_QKEY            0x11111111
+#define DEF_DURATION        (5)
+#define	DEF_MARGIN          (2)
+#define DEF_INIT_MARGIN     (-1)
+#define DEF_INLINE          (-1)
+#define DEF_TOS             (-1)
+#define DEF_RETRY_COUNT     (7)
+#define DEF_CACHE_LINE_SIZE (64)
+#define DEF_PAGE_SIZE       (4096)
+#define DEF_FLOWS           (1)
+#define RATE_VALUES_COUNT   (18)
+
+/* Optimal Values for Inline */
+#define DEF_INLINE_WRITE (220)
+#define DEF_INLINE_SEND_RC_UC (236)
+#define DEF_INLINE_SEND_XRC (236)
+#define DEF_INLINE_SEND_UD (188)
+#define DEF_INLINE_DC (150)
+
+/* Max and Min allowed values for perftest parameters. */
+#define MIN_TOS		(0)
+#define MAX_TOS		(255)
+#define MIN_IB_PORT   (1)
+#define MAX_IB_PORT   (3)
+#define MIN_ITER      (5)
+#define MAX_ITER      (100000000)
+#define MIN_TX 	      (1)
+#define MAX_TX	      (15000)
+#define MIN_SL	      (0)
+#define MAX_SL	      (15)
+#define MIN_GID_IX    (0)
+#define MAX_GID_IX    (64)
+#define MIN_QP_NUM    (1)
+#define MAX_QP_NUM    (16384)
+#define MIN_QP_MCAST  (1)
+#define MAX_QP_MCAST  (56)
+#define MIN_RX	      (1)
+#define MAX_RX	      (16384)
+#define UC_MAX_RX     (16000)
+#define MIN_CQ_MOD    (1)
+#define MAX_CQ_MOD    (1024)
+#define MAX_INLINE    (912)
+#define MAX_INLINE_UD (884)
+#define MIN_EQ_NUM    (0)
+#define MAX_EQ_NUM    (2048)
+
+/* Raw etherent defines */
+// #define RAWETH_MIN_MSG_SIZE	(64)
+// #define MIN_MTU_RAW_ETERNET	(64)
+// #define MAX_MTU_RAW_ETERNET	(9600)
+// #define MIN_FS_PORT		(5000)
+// #define MAX_FS_PORT		(65536)
+// #define VLAN_PCP_VARIOUS        (8)
+
+#define RESULT_LINE "---------------------------------------------------------------------------------------\n"
+
+#define RESULT_LINE_PER_PORT "-------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
+#define CYCLES	"cycles"
+#define USEC	"usec"
+
+/* The format of the results */
+#define RESULT_FMT            " #bytes     #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]"
+#define RESULT_FMT_PER_PORT   " #bytes     #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]   BW Port1[MB/sec]   MsgRate Port1[Mpps]   BW Port2[MB/sec]   MsgRate Port2[Mpps]"
+#define RESULT_FMT_G          " #bytes     #iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]"
+#define RESULT_FMT_G_PER_PORT " #bytes     #iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]   BW Port1[Gb/sec]   MsgRate Port1[Mpps]   BW Port2[Gb/sec]   MsgRate Port2[Mpps]"
+#define RESULT_FMT_QOS        " #bytes    #sl      #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]"
+#define RESULT_FMT_G_QOS      " #bytes    #sl      #iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]"
+#define RESULT_FMT_LAT        " #bytes #iterations    t_min[usec]    t_max[usec]  t_typical[usec]    t_avg[usec]    t_stdev[usec]   99""%"" percentile[usec]   99.9""%"" percentile[usec] "
+#define RESULT_FMT_LAT_DUR    " #bytes        #iterations       t_avg[usec]    tps average"
+
+#define RESULT_EXT          "\n"
+#define RESULT_EXT_CPU_UTIL "    CPU_Util[%%]\n"
+
+#define RESULT_FMT_FS_RATE     " #flows	fs_min_time[usec]	fs_max_time[usec]	fs_typical_time[usec]	fs_avg_time[usec]	fps[flow per sec]"
+#define RESULT_FMT_FS_RATE_DUR " #flows		fs_avg_time[usec]    	fps[flow per sec]"
+
+/* Result print format */
+#define REPORT_FMT     " %-7lu    %-10lu       %-7.2lf            %-7.2lf		   %-7.6lf"
+#define REPORT_FMT_EXT     " %-7lu    %lu           %-7.6lf            %-7.6lf            %-7.6lf"
+#define REPORT_FMT_PER_PORT     " %-7lu    %-10lu     %-7.2lf            %-7.2lf		   %-7.6lf        %-7.2lf            %-7.6lf              %-7.2lf            %-7.6lf"
+#define REPORT_EXT	"\n"
+#define REPORT_EXT_CPU_UTIL	"	    %-3.2f\n"
+#define REPORT_FMT_QOS " %-7lu    %d           %lu           %-7.2lf            %-7.2lf                  %-7.6lf\n"
+
+/* Result print format for latency tests. */
+#define REPORT_FMT_LAT " %-7lu %d          %-7.2f        %-7.2f      %-7.2f  	       %-7.2f     	%-7.2f		%-7.2f 		%-7.2f"
+#define REPORT_FMT_LAT_DUR " %-7lu       %d            %-7.2f        %-7.2f"
+#define REPORT_FMT_FS_RATE " %d          %-7.2f        		%-7.2f      	%-7.2f  	       		%-7.2f     	%-7.2f"
+#define REPORT_FMT_FS_RATE_DUR " %d               %-7.2f		%-7.2f"
+
+#define CHECK_VALUE(arg,type,minv,maxv,name) 						    					\
+{ arg = (type)strtol(optarg, NULL, 0); if ((arg < minv) || (arg > maxv))                \
+	{ fprintf(stderr," %s should be between %d and %d\n",name,minv,maxv); return 1; }}
+
+/* Macro for allocating. */
+#define ALLOCATE(var,type,size)                                     \
+{ if((var = (type*)malloc(sizeof(type)*(size))) == NULL)        \
+	{ fprintf(stderr," Cannot Allocate\n"); exit(1);}}
+
+/* This is our string builder */
+#define GET_STRING(orig,temp) 						            \
+{ ALLOCATE(orig,char,(strlen(temp) + 1)); strcpy(orig,temp); }
+
+#define MTU_SIZE(mtu_ind) (((uint64_t)1 << (MTU_FIX + mtu_ind)))
+
+#define MAX_VERSION 16	/* Reserve 15 bytes for version numbers */
+
+#define GET_ARRAY_SIZE(arr) (sizeof((arr)) / sizeof((arr[0])))
+
+/* The Verb of the benchmark. */
+typedef enum { SEND , WRITE, READ, ATOMIC } VerbType;
+
+/* The type of the test */
+typedef enum { LAT , BW , LAT_BY_BW, FS_RATE } TestType;
+
+/* The type of the machine ( server or client actually). */
+typedef enum { SERVER , CLIENT , UNCHOSEN} MachineType;
+
+/* The type of the machine ( server or client actually). */
+typedef enum { LOCAL , REMOTE } PrintDataSide;
+
+/* The atomic test type */
+typedef enum {CMP_AND_SWAP, FETCH_AND_ADD} AtomicType;
+
+/* Type of test method. */
+typedef enum { ITERATIONS , DURATION } TestMethod;
+
+/* for duration calculation */
+typedef enum { START_STATE, SAMPLE_STATE, STOP_SAMPLE_STATE, END_STATE} DurationStates;
+
+/* Report format (Gbit/s VS MB/s) */
+enum ctx_report_fmt { GBS, MBS };
+
+/* Test method */
+enum ctx_test_method {RUN_REGULAR, RUN_ALL, RUN_INFINITELY};
+
+/* The type of the device */
+enum ctx_device {
+	DEVICE_ERROR		= -1,
+	UNKNOWN			= 0,
+	CONNECTX 		= 1,
+	CONNECTX2 		= 2,
+	CONNECTX3 		= 3,
+	CONNECTIB 		= 4,
+	LEGACY 			= 5,
+	CHELSIO_T4 		= 6,
+	CHELSIO_T5 		= 7,
+	CONNECTX3_PRO		= 8,
+	SKYHAWK			= 9,
+	CONNECTX4		= 10,
+	CONNECTX4LX		= 11,
+	QLOGIC_E4		= 12,
+	QLOGIC_AH		= 13,
+	CHELSIO_T6		= 14,
+	CONNECTX5		= 15,
+	CONNECTX5EX		= 16,
+	CONNECTX6		= 17,
+	BLUEFIELD		= 18
+};
+
+/* Units for rate limiter */
+enum rate_limiter_units {MEGA_BYTE_PS, GIGA_BIT_PS, PACKET_PS};
+
+/*Types rate limit*/
+enum rate_limiter_types {HW_RATE_LIMIT, SW_RATE_LIMIT, PP_RATE_LIMIT, DISABLE_RATE_LIMIT};
+
+/* Verbosity Levels for test report */
+enum verbosity_level {FULL_VERBOSITY=-1, OUTPUT_BW=0, OUTPUT_MR, OUTPUT_LAT };
+
+/*Accelerated verbs */
+enum verbs_intf {
+	NORMAL_INTF,
+	ACCL_INTF,
+};
+
+struct cpu_util_data {
+	int enable;
+	long long ustat[2];
+	long long idle[2];
+};
+
+struct check_alive_data {
+	int current_totrcnt;
+	int last_totrcnt;
+	int g_total_iters;
+	int to_exit;
+	int is_events;
+};
+
+/* gen_eth_header .
+ * Description :create raw Ethernet header on buffer
+ *
+ * Parameters :
+ *	 	eth_header - Pointer to output
+ *	 	src_mac - source MAC address of the packet
+ *	 	dst_mac - destination MAC address of the packet
+ *	 	eth_type - IP/or size of ptk
+ *
+ *
+struct ETH_header {
+	uint8_t dst_mac[6];
+	uint8_t src_mac[6];
+	uint16_t eth_type;
+}__attribute__((packed));
+
+struct ETH_vlan_header {
+        uint8_t dst_mac[6];
+        uint8_t src_mac[6];
+        uint32_t vlan_header;
+        uint16_t eth_type;
+}__attribute__((packed));*/
+
+struct perftest_parameters {
+
+	int				port;
+	char				*ib_devname;
+	char				*servername;
+	uint8_t				ib_port;
+	uint8_t				ib_port2;
+	int				mtu;
+	enum ibv_mtu			curr_mtu;
+	uint64_t			size;
+	uint64_t			dct_key;
+	int				iters;
+	uint64_t			iters_per_port[2];
+	uint64_t			*port_by_qp;
+	int				tx_depth;
+	uint8_t				qp_timeout;
+	uint8_t				sl;
+	int				gid_index;
+	int				gid_index2;
+	int				use_gid_user;
+	uint8_t				source_mac[6];
+	uint8_t				dest_mac[6];
+	int				is_source_mac;
+	int				is_dest_mac;
+	uint8_t				server_ip6[16];
+	uint8_t				client_ip6[16];
+	uint8_t				local_ip6[16];
+	uint8_t				remote_ip6[16];
+	uint8_t				local_mac[6];
+	uint8_t				remote_mac[6];
+	uint32_t			client_ip;
+	uint32_t			server_ip;
+	int				is_server_ip;
+	int				is_client_ip;
+	uint32_t			local_ip;
+	uint32_t			remote_ip;
+	int				server_port;
+	int				client_port;
+	int				tcp;
+	int				is_server_port;
+	int				is_client_port;
+	int				local_port;
+	int				remote_port;
+	int 				is_old_raw_eth_param;
+	int 				is_new_raw_eth_param;
+	uint16_t			ethertype;
+	int				is_ethertype;
+	int				cpu_freq_f;
+	int				connection_type;
+	int				num_of_qps;
+	int				use_event;
+	int				eq_num;
+	int				use_eq_num;
+	int 				inline_size;
+	int				inline_recv_size;
+	int				out_reads;
+	int				rx_depth;
+	int				duplex;
+	int				noPeak;
+	int				cq_mod;
+	int 				spec;
+	int 				dualport;
+	int 				post_list;
+	int				duration;
+	int 				use_srq;
+	int				use_xrc;
+	int				use_rss;
+	int				srq_exists;
+	int				tos;
+	int				margin;
+	int 				is_bw_limit_passed;
+	int 				is_msgrate_limit_passed;
+	int 				is_limit_bw;
+	int 				is_limit_msgrate;
+	float				limit_bw;
+	float				limit_msgrate;
+	uint32_t			rem_ud_qpn;
+	uint32_t			rem_ud_qkey;
+	int8_t				link_type;
+	int8_t				link_type2;
+	MachineType			machine;
+	PrintDataSide			side;
+	VerbType			verb;
+	TestType			tst;
+	AtomicType			atomicType;
+	TestMethod			test_type;
+	DurationStates			state;
+	int				sockfd;
+	char				version[MAX_VERSION];
+	char				rem_version[MAX_VERSION];
+	cycles_t			*tposted;
+	cycles_t			*tcompleted;
+	int				use_mcg;
+	int 				use_rdma_cm;
+	int				is_reversed;
+	int				work_rdma_cm;
+	char				*user_mgid;
+	int				buff_size;
+	int             		pkey_index;
+	int				raw_qos;
+	int				use_cuda;
+	char				*mmap_file;
+	unsigned long			mmap_offset;
+	/* New test params format pilot. will be used in all flags soon,. */
+	enum ctx_test_method 		test_method;
+	enum ibv_transport_type 	transport_type;
+	enum ctx_report_fmt		report_fmt;
+	struct report_options  		*r_flag	;
+	int 				mac_fwd;
+	int report_both; /* in bidirectional tests, report tx and rx separately */
+	/* results limits */
+	float 				min_bw_limit;
+	float 				min_msgRate_limit;
+	/* Rate Limiter */
+	char				*rate_limit_str;
+	double 				rate_limit;
+	int				valid_hw_rate_limit;
+	int 				burst_size;
+	enum 				rate_limiter_units rate_units;
+	enum 				rate_limiter_types rate_limit_type;
+	int				is_rate_limit_type;
+	enum verbosity_level 		output;
+	int 				cpu_util;
+	struct cpu_util_data 		cpu_util_data;
+	int 				latency_gap;
+	int 				flow_label;
+	int 				retry_count;
+	int 				dont_xchg_versions;
+	int 				use_exp;
+	int 				ipv6;
+	int 				raw_ipv6;
+	int 				report_per_port;
+	int 				use_odp;
+	int				use_hugepages;
+	int				use_promiscuous;
+	int				use_sniffer;
+	int				check_alive_exited;
+	int				raw_mcast;
+	int				masked_atomics;
+	int				cycle_buffer;
+	int				cache_line_size;
+	enum verbs_intf			verb_type;
+	int				is_exp_cq;
+	int				is_exp_qp;
+	int				use_res_domain;
+	int				mr_per_qp;
+	uint16_t			dlid;
+	uint8_t				traffic_class;
+	uint32_t			wait_destroy;
+	int				disable_fcs;
+	int				flows;
+	int				flows_burst;
+	uint32_t			reply_every;
+	int				perform_warm_up;
+	int				use_ooo;
+	int                             vlan_en;
+	uint32_t			vlan_pcp;
+	void 				(*print_eth_func)(void*);
+
+};
+
+struct report_options {
+	int unsorted;
+	int histogram;
+	int cycles;
+};
+
+struct bw_report_data {
+	unsigned long size;
+	uint64_t iters;
+	double bw_peak;
+	double bw_avg;
+	double bw_avg_p1;
+	double bw_avg_p2;
+	double msgRate_avg;
+	double msgRate_avg_p1;
+	double msgRate_avg_p2;
+	int sl;
+};
+
+struct rate_gbps_string {
+	enum ibv_rate rate_gbps_enum;
+	char* rate_gbps_str;
+};
+/*
+ *Enums taken from verbs.h
+ */
+static const struct rate_gbps_string RATE_VALUES[RATE_VALUES_COUNT] = {
+	{IBV_RATE_2_5_GBPS, "2.5"},
+	{IBV_RATE_5_GBPS,   "5"},
+	{IBV_RATE_10_GBPS,  "10"},
+	{IBV_RATE_14_GBPS,  "14"},
+	{IBV_RATE_20_GBPS,  "20"},
+	{IBV_RATE_25_GBPS,  "25"},
+	{IBV_RATE_30_GBPS,  "30"},
+	{IBV_RATE_40_GBPS,  "40"},
+	{IBV_RATE_56_GBPS,  "56"},
+	{IBV_RATE_60_GBPS,  "60"},
+	{IBV_RATE_80_GBPS,  "80"},
+	{IBV_RATE_100_GBPS, "100"},
+	{IBV_RATE_112_GBPS, "112"},
+	{IBV_RATE_120_GBPS, "120"},
+	{IBV_RATE_168_GBPS, "168"},
+	{IBV_RATE_200_GBPS, "200"},
+	{IBV_RATE_300_GBPS, "300"},
+	{IBV_RATE_MAX,      "MAX"}
+};
+
+
+
+
+/* link_layer_str
+ *
+ * Description : Return a String representation of the link type.
+ *
+ * link_layer   : (According to verbs.h) :
+ *		IBV_LINK_LAYER_UNSPECIFIED.
+ *      IBV_LINK_LAYER_INFINIBAND.
+ *      IBV_LINK_LAYER_ETHERNET.
+ *
+ * Return Value :"IB", "Etherent" or "Unknown".
+ */
+const char *link_layer_str(int8_t link_layer);
+
+/* str_link_layer
+ *
+ * Description : Try to parse a string into a verbs link layer type.
+ *
+ * link_layer   : (According to verbs.h) :
+ *      "IB"       -> IBV_LINK_LAYER_INFINIBAND.
+ *      "Ethernet" -> IBV_LINK_LAYER_ETHERNET.
+ *      otherwise  -> LINK_FAILURE.
+ *
+ * Return Value : IBV_LINK_LAYER or LINK_FAILURE
+ */
+const int str_link_layer(const char *str);
+
+/* parser
+ *
+ * Description : Setting default test parameters and parsing the user choises
+ *
+ * Parameters :
+ *
+ *      user_param  - the parameters element.
+ *      argv & argc - from the user prompt.
+ *
+ * Return Value : 0 upon success. -1 if it fails.
+ */
+int parser(struct perftest_parameters *user_param,char *argv[], int argc);
+
+/* check_link_and_mtu
+ *
+ * Description : Configures test MTU,inline and link layer of the test.
+ *
+ * Parameters :
+ *
+ *      context    - Context of the device.
+ *      user_param - Perftest parameters.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+int check_link(struct ibv_context *context,struct perftest_parameters *user_param);
+
+/* check_link_and_mtu
+ *
+ * Description : Configures test MTU,inline and link layer of the test.
+ *
+ * Parameters :
+ *
+
+ *	 context    - Context of the device.
+ *	 user_param - Perftest parameters.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+int check_link_and_mtu(struct ibv_context *context,struct perftest_parameters *user_param);
+
+/* ctx_print_test_info
+ *
+ * Description : Prints all the parameters selected for this run.
+ *
+ * Parameters :
+ *
+ *	 user_param  - the parameters parameters.
+ *
+ */
+void ctx_print_test_info(struct perftest_parameters *user_param);
+
+/* print_report_bw
+ *
+ * Description : Calculate the peak and average throughput of the BW test.
+ *				 The function will print when not in duplex mode.
+ *
+ * Parameters :
+ *
+ *	 user_param  - the parameters parameters.
+ *   my_bw_rep   - get my bw test report.
+ *
+ */
+void print_report_bw (struct perftest_parameters *user_param, struct bw_report_data *my_bw_rep);
+
+/* print_full_bw_report
+ *
+ * Description : Print the peak and average throughput of the BW test.
+ *				 If rem_bw_rep is not NULL, the function will sum the server and client results.
+ *
+ * Parameters :
+ *
+ *	 user_param  - the parameters parameters.
+ *   my_bw_rep   - my bw test report.
+ *   rem_bw_rep   - remote's bw test report.
+ *
+ */
+void print_full_bw_report (struct perftest_parameters *user_param, struct bw_report_data *my_bw_rep, struct bw_report_data *rem_bw_rep);
+
+/* print_report_lat
+ *
+ * Description : Print the min/max/median latency samples taken from a latency test.
+ * 				 It also support a unsorted/histogram report of all samples.
+ *
+ * Parameters :
+ *
+ *   user_param  - the parameters parameters.
+ *
+ */
+void print_report_lat (struct perftest_parameters *user_param);
+
+/* print_report_lat_duration
+ *
+ * Description : Prints only the avergae latency for samples taken from a latency test
+ *				 With Duration.
+ *
+ * Parameters :
+ *
+ *   user_param  - the parameters parameters.
+ *
+ */
+void print_report_lat_duration (struct perftest_parameters *user_param);
+
+/* print_report_fs_rate
+ *
+ * Description : Prints the Flow steering rate and avarage latency to create flow
+ *
+ * Parameters :
+ *
+ *   user_param  - the parameters parameters.
+ *
+ */
+void print_report_fs_rate (struct perftest_parameters *user_param);
+
+/* set_mtu
+ *
+ * Description : set MTU from the port or user
+ *
+ * Parameters :
+ *
+ *   context  - Context of the device.
+ *   ib_port  - ib port number that's in use.
+ *   user_mtu  - MTU that the user supplied.
+ *
+ * Return Value : MTU size
+ */
+enum ibv_mtu set_mtu(struct ibv_context *context,uint8_t ib_port,int user_mtu);
+
+/* set_eth_mtu
+ *
+ * Description : set MTU for Raw Ethernet tests
+ *
+ * Parameters :
+ *
+ *   user_param  - the parameters parameters.
+ *
+ * Return Value : MTU size
+ */
+int set_eth_mtu(struct perftest_parameters *user_param);
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+enum ctx_device ib_dev_name(struct ibv_context *context);
+
+#endif /* PERFTEST_RESOURCES_H */

usr/benchmarks/ib/perftest_resources.h

@@ -0,0 +1,858 @@
+/*
+ * Copyright (c) 2009 Mellanox Technologies Ltd.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Author: Ido Shamay <idos@dev.mellanox.co.il>
+ *
+ * Description :
+ *
+ *  This API gathres the Socket interface methods for all perftest benchmarks
+ *  and can be used for any benchmark for IB.
+ *  It passes messages between 2 end points through sockets interface methods,
+ *  while passing the rellevant information for the IB entities.
+ *
+ * Methods :
+ *
+ *  ctx_get_local_lid  - Receives the Local id from the subnet manager.
+ *  ctx_client_connect - Connects the client through sockets interface.
+ *  ctx_server_connect - Connects the Server to client through sockets.
+ *  ctx_hand_shake     - Passes the data between 2 end points machines.
+ *  ctx_print_pingpong_data - Prints the data that was passed.
+ *  ctx_close_connection    - Closing the sockets interface.
+ */
+
+#ifndef PERFTEST_RESOURCES_H
+#define PERFTEST_RESOURCES_H
+
+// #include <infiniband/verbs.h>
+#include <hermit/ibv.h>
+#include <hermit/verbs.h>
+
+// #include <rdma/rdma_cma.h>
+#include <stdint.h>
+// #include <byteswap.h>
+
+#include <math.h>
+// #include <arpa/inet.h>
+#include <netinet/in.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <fcntl.h> // ?
+
+#include "perftest_parameters.h"
+
+#define NUM_OF_RETRIES		(10)
+
+/* Outstanding reads for "read" verb only. */
+#define MAX_SEND_SGE      (1)
+#define MAX_RECV_SGE      (1)
+#define CTX_POLL_BATCH    (16)
+#define PL                (1)
+#define ATOMIC_ADD_VALUE  (1)
+#define ATOMIC_SWAP_VALUE (0)
+
+/* Space for GRH when we scatter the packet in UD. */
+#define PINGPONG_SEND_WRID   (60)
+#define PINGPONG_RDMA_WRID   (3)
+#define PINGPONG_READ_WRID   (1)
+#define PINGPONG_ATOMIC_WRID (22)
+#define DEFF_QKEY            (0x11111111)
+
+#ifdef HAVE_XRCD
+#define SERVER_FD "/tmp/xrc_domain_server"
+#define CLIENT_FD "/tmp/xrc_domain_client"
+#endif
+
+
+#define NOTIFY_COMP_ERROR_SEND(wc,scnt,ccnt)                     											\
+	{ fprintf(stderr," Completion with error at client\n");      											\
+	  fprintf(stderr," Failed status %d: wr_id %d syndrom 0x%x\n",wc.status,(int) wc.wr_id,wc.vendor_err);	\
+	  fprintf(stderr, "scnt=%lu, ccnt=%lu\n",scnt, ccnt); }
+
+#define NOTIFY_COMP_ERROR_RECV(wc,rcnt)                     											    \
+	{ fprintf(stderr," Completion with error at server\n");      											\
+	  fprintf(stderr," Failed status %d: wr_id %d syndrom 0x%x\n",wc.status,(int) wc.wr_id,wc.vendor_err);	\
+	  fprintf(stderr," rcnt=%lu\n",rcnt); }
+
+/* Macro to determine packet size in case of UD. The UD addition is for the GRH . */
+#define SIZE(type,size,valid) ((type == UD && valid) ? (size + UD_ADDITION) : (size))
+
+/* Macro to define the buffer size (according to "Nahalem" chip set).
+ * for small message size (under 4K) , we allocate 4K buffer , and the RDMA write
+ * verb will write in cycle on the buffer. this improves the BW in "Nahalem" systems.
+ */
+#define BUFF_SIZE(size,cycle_buffer) ((size < cycle_buffer) ? (cycle_buffer) : (size))
+
+/* UD addition to the buffer. */
+#define IF_UD_ADD(type,cache_line_size) ((type == UD) ? (cache_line_size) : (0))
+
+/* Macro that defines the address where we write in RDMA.
+ * If message size is smaller then CACHE_LINE size then we write in CACHE_LINE jumps.
+ */
+#define INC(size,cache_line_size) ((size > cache_line_size) ? ((size%cache_line_size == 0) ?  \
+	       (size) : (cache_line_size*(size/cache_line_size+1))) : (cache_line_size))
+
+#define UD_MSG_2_EXP(size) ((log(size))/(log(2)))
+
+#define MASK_IS_SET(mask, attr)      (((mask)&(attr))!=0)
+
+/******************************************************************************
+ * Perftest resources Structures and data types.
+ ******************************************************************************/
+struct pingpong_context {
+	// struct rdma_event_channel		*cm_channel;
+	// struct rdma_cm_id			*cm_id_control;
+	// struct rdma_cm_id			*cm_id;
+	struct ibv_context			*context;
+	struct ibv_comp_channel			*channel;
+	struct ibv_pd				*pd;
+	struct ibv_mr				**mr;
+	struct ibv_cq				*send_cq;
+	struct ibv_cq				*recv_cq;
+	void					**buf;
+	struct ibv_ah				**ah;
+	struct ibv_qp				**qp;
+	struct ibv_srq				*srq;
+	struct ibv_sge				*sge_list;
+	struct ibv_sge				*recv_sge_list;
+	struct ibv_send_wr			*wr;
+	struct ibv_recv_wr			*rwr;
+	uint64_t				size;
+	uint64_t				*my_addr;
+	uint64_t				*rx_buffer_addr;
+	uint64_t				*rem_addr;
+	uint64_t				buff_size;
+	uint64_t				send_qp_buff_size;
+	uint64_t				flow_buff_size;
+	int					tx_depth;
+	int					huge_shmid;
+	uint64_t				*scnt;
+	uint64_t				*ccnt;
+	int					is_contig_supported;
+	uint32_t                                *ctrl_buf;
+	uint32_t                                *credit_buf;
+	struct ibv_mr                           *credit_mr;
+	struct ibv_sge                          *ctrl_sge_list;
+	struct ibv_send_wr                      *ctrl_wr;
+	int                                     send_rcredit;
+	int                                     credit_cnt;
+	int					cache_line_size;
+	int					cycle_buffer;
+	#ifdef HAVE_XRCD
+	struct ibv_xrcd				*xrc_domain;
+	int 					fd;
+	#endif
+	#ifdef HAVE_ACCL_VERBS
+	struct ibv_exp_res_domain		*res_domain;
+	struct ibv_exp_cq_family		*send_cq_family;
+	struct ibv_exp_cq_family		*recv_cq_family;
+	struct ibv_exp_qp_burst_family		**qp_burst_family;
+	#endif
+
+};
+
+ struct pingpong_dest {
+	int 				lid;
+	int 				out_reads;
+	int 				qpn;
+	int 				psn;
+	unsigned			rkey;
+	unsigned long long		vaddr;
+	union ibv_gid			gid;
+	unsigned			srqn;
+	int				gid_index;
+ };
+
+/******************************************************************************
+ * Perftest resources Methods and interface utilitizes.
+ ******************************************************************************/
+
+/* check_add_port
+ *
+ * Description : Creating a service struct from a given port and servername.
+ *
+ * Parameters :
+ *	service - an empty char** to contain the service name.
+ *  port - The selected port on which the server will listen.
+ *  hints - The requested ai_* params for the connection.
+ *  res - Holds the result.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+int check_add_port(char **service,int port,
+				   const char *servername,
+				   struct addrinfo *hints,
+				   struct addrinfo **res);
+
+/* ctx_find_dev
+ *
+ * Description : Returns the device corresponding to ib_devname
+ *	or the first one found , in case ib_devname == NULL
+ *
+ * Parameters :
+ *
+ *	ib_devname - The name of the device requested or NULL for the first one.
+ *
+ * Return Value : the device or NULL in case of failure.
+ */
+struct ibv_device* ctx_find_dev(const char *ib_devname);
+
+/* create_rdma_resources
+ *
+ * Description : Creates the rdma_cm_id and rdma_channel for the rdma_cm QPs.
+ *
+ * Parameters :
+ *	ctx - Resources sructure.
+ * 	user_param - the perftest parameters.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+// int create_rdma_resources(struct pingpong_context *ctx,
+                          // struct perftest_parameters *user_param);
+
+/* destroy_rdma_resources
+ *
+ * Description : Destroys the rdma_cm_id and rdma_channel for the rdma_cm QPs.
+ *
+ * Parameters :
+ *	ctx - Resources sructure.
+ * 	user_param - the perftest parameters.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+// int destroy_rdma_resources(struct pingpong_context *ctx,
+                          // struct perftest_parameters *user_param);
+
+/* alloc_ctx
+ *
+ * Description : allocate all perftest resources.
+ *
+ * Parameters :
+ *	ctx - Resources sructure.
+ * 	user_param - the perftest parameters.
+ */
+void alloc_ctx(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* destroy_ctx
+ *
+ * Description : Deallocate all perftest resources.
+ *
+ * Parameters :
+ *	ctx - Resources sructure.
+ * 	user_param - the perftest parameters.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+int destroy_ctx(struct pingpong_context *ctx,
+				struct perftest_parameters *user_param);
+
+/* verify_params_with_device_context
+ *
+ * Description :
+ * 		Verify user params that require information from the ibv_context
+ *
+ * Parameters :
+ *	context - ibv_context
+ * 	user_param - the perftest parameters.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+int verify_params_with_device_context(struct ibv_context *ctx,
+				      struct perftest_parameters *user_param);
+
+
+/* ctx_init
+ *
+ * Description :
+ * 		Creates all the test resources.
+ *		It creates Buffer, PD, MR, CQ QPs and moves the QPs to INIT.
+ *
+ * Parameters :
+ *	ctx - An empty resources sructure to fill inside the resources.
+ * 	user_param - the perftest parameters.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+int ctx_init(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* ctx_qp_create.
+ *
+ * Description :
+ *
+ *	Creates a QP , according to the attributes given in param.
+ *	The relevent attributes are tx_depth,rx_depth,inline_size and connection_type.
+ *
+ * Parameters :
+ *
+ *	pd      - The Protection domain , each the qp will be assigned to.
+ *	send_cq - The CQ that will produce send CQE.
+ *	recv_qp - The CQ that will produce recv CQE.
+ *	param   - The parameters for the QP.
+ *
+ * Return Value : Adress of the new QP.
+ */
+struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx,
+							 struct perftest_parameters *user_param);
+
+/* ctx_modify_qp_to_init.
+ *
+ * Description :
+ *
+ *	Modifies the given QP to INIT state , according to attributes in param.
+ *  The relevent attributes are ib_port, connection_type and verb.
+ *
+ * Parameters :
+ *
+ *	qp     - The QP that will be moved to INIT.
+ *	param  - The parameters for the QP.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ *
+ */
+int ctx_modify_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param, uint64_t init_flag);
+
+/* ctx_connect.
+ *
+ * Description :
+ *
+ *	Modifies the given QP to RTR and then RTS states, given it's transport type and feature.
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *  dest    - pingpong_dest struct of the remote side.
+ *	user_param  - user_parameters struct for this test.
+ *  my_dest - pingpong_dest struct of this side.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ *
+ */
+int ctx_connect(struct pingpong_context *ctx,
+				struct pingpong_dest *dest,
+				struct perftest_parameters *user_param,
+				struct pingpong_dest *my_dest);
+
+/* ctx_set_send_exp_wqes.
+ *
+ * Description :
+ *
+ *	Prepare the exp send work request templates for all QPs
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *  rem_dest   - pingpong_dest struct of the remote side.
+ *
+ */
+void ctx_set_send_exp_wqes(struct pingpong_context *ctx,
+					   struct perftest_parameters *user_param,
+					   struct pingpong_dest *rem_dest);
+
+
+/* ctx_set_send_regwqes.
+ *
+ * Description :
+ *
+ *	Prepare the regular send work request templates for all QPs
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *  rem_dest   - pingpong_dest struct of the remote side.
+ *
+ */
+void ctx_set_send_reg_wqes(struct pingpong_context *ctx,
+					   struct perftest_parameters *user_param,
+					   struct pingpong_dest *rem_dest);
+
+/* ctx_set_send_wqes.
+ *
+ * Description :
+ *
+ *	Prepare the send work request templates for all QPs
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *  rem_dest   - pingpong_dest struct of the remote side.
+ *
+ */
+void ctx_set_send_wqes(struct pingpong_context *ctx,
+					   struct perftest_parameters *user_param,
+					   struct pingpong_dest *rem_dest);
+
+
+/* ctx_set_recv_wqes.
+ *
+ * Description :
+ *
+ *	Prepare the receives work request templates for all QPs in SEND receive test.
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *
+ */
+int ctx_set_recv_wqes(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* ctx_alloc_credit
+ *
+ * Description :
+ *
+ * 	Allocate resources to support the credit exchange mechanism,
+ * 	which allows ib_send_bw to work with iWARP
+ *
+ * Parameters :
+ *
+ *	ctx        - test context
+ *  	user_param - user parameters struct for this test
+ *  	my_dest    - pingpong_dest struct of the this side
+ *
+ *  	my_dest is updated to store the credit buf vaddr and rkey
+ *  	which need to be exchanged with the remote side
+ *  	to enable RDMA WRITE op
+ */
+int ctx_alloc_credit(struct pingpong_context *ctx,
+				struct perftest_parameters *user_param,
+				struct pingpong_dest *my_dest);
+/* ctx_set_credit_wqes
+ *
+ * Description :
+ *
+ * 	Prepare the send credit work request templates for all QPs
+ * 	RDMA WRITE op is used for sending credit
+ * 	Credit exchange is necessary for ib_send_bw to work with iWARP
+ *
+ * Parameters :
+ *
+ *	ctx        - test context
+ *  	user_param - user parameters struct for this test
+ *  	rem_dest   - pingpong_dest struct of the remote side.
+ */
+int ctx_set_credit_wqes(struct pingpong_context *ctx,
+				struct perftest_parameters *user_param,
+				struct pingpong_dest *rem_dest);
+/* run_iter_bw.
+ *
+ * Description :
+ *
+ *	The main testing method in BW tests.
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *
+ */
+int run_iter_bw(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* run_iter_bw_infinitely
+ *
+ * Description :
+ *
+ *	Infinite BW method, that prints BW every 5 seconds and never stops.
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *
+ */
+int run_iter_bw_infinitely(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* run_iter_bw_infinitely_server
+ *
+ * Description :
+ *
+ *	Infinite on SEND server BW method, will not print anything and will post receive wqes for each incoming packet.
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *
+ */
+int run_iter_bw_infinitely_server(struct pingpong_context *ctx, struct perftest_parameters *user_param);
+
+/* run_iter_bw_server.
+ *
+ * Description :
+ *
+ *	The main testing method for Receiver in SEND test.
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *
+ */
+int run_iter_bw_server(struct pingpong_context *ctx, struct perftest_parameters *user_param);
+
+/* run_iter_bi.
+ *
+ * Description :
+ *
+ *	The main testing method for bidirrectional.
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ *
+ */
+int run_iter_bi(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* run_iter_lat_write
+ *
+ * Description :
+ *
+ *  This is the latency test function for WRITE verb.
+ *
+ * Parameters :
+ *
+ *	ctx     - Test Context.
+ *	user_param  - user_parameters struct for this test.
+ */
+int run_iter_lat_write(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* run_iter_lat
+ *
+ * Description :
+ *
+ *  This is the latency test function for READ or ATOMIC verb latency tests.
+ *
+ * Parameters :
+ *
+ *  ctx     - Test Context.
+ *  user_param  - user_parameters struct for this test.
+ */
+int run_iter_lat(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* run_iter_lat_send
+ *
+ * Description :
+ *
+ *  This is the latency test function for SEND verb latency test.
+ *
+ * Parameters :
+ *
+ *  ctx     - Test Context.
+ *  user_param  - user_parameters struct for this test.
+ */
+int run_iter_lat_send(struct pingpong_context *ctx, struct perftest_parameters *user_param);
+
+/* run_iter_lat_burst
+ *
+ * Description :
+ *
+ *  This is the latency test function for SEND verb latency test in burst mode
+ *
+ * Parameters :
+ *
+ *  ctx     - Test Context.
+ *  user_param  - user_parameters struct for this test.
+ */
+
+int run_iter_lat_burst(struct pingpong_context *ctx, struct perftest_parameters *user_param);
+
+/* run_iter_lat_burst_server
+ *
+ * Description :
+ *
+ *  This is the latency test function for server side latency test in burst mode
+ *
+ * Parameters :
+ *
+ *  ctx     - Test Context.
+ *  user_param  - user_parameters struct for this test.
+ */
+
+int run_iter_lat_burst_server(struct pingpong_context *ctx, struct perftest_parameters *user_param);
+
+/* ctx_get_local_lid .
+ *
+ * Description :
+ *
+ *  This method find and returns the local Id in IB subnet manager of
+ *  the selected port and HCA given.The lid identifies the port.
+ *
+ * Parameters :
+ *
+ *  context - the context of the HCA device.
+ *  ib_port - The port of the HCA (1 or 2).
+ *
+ * Return Value : The Lid itself. (No error values).
+ */
+uint16_t ctx_get_local_lid(struct ibv_context *context, int ib_port);
+
+/* ctx_notify_events
+ *
+ * Description : Prepare the test to work with events instead of polling the CQ.
+ *	This is the way to work in un interipted mode.
+ *
+ * Parameters :
+ *  channel - (Mandotory) the created event channel.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ */
+static __inline int ctx_notify_events(struct ibv_comp_channel *channel)
+{
+
+	struct ibv_cq       *ev_cq;
+	void                *ev_ctx;
+
+	if (ibv_get_cq_event(channel,&ev_cq,&ev_ctx)) {
+		fprintf(stderr, "Failed to get cq_event\n");
+		return 1;
+	}
+
+	ibv_ack_cq_events(ev_cq,1);
+
+	if (ibv_req_notify_cq(ev_cq, 0)) {
+		fprintf(stderr, "Couldn't request CQ notification\n");
+		return 1;
+	}
+	return 0;
+}
+
+#if defined(HAVE_VERBS_EXP)
+static __inline void increase_exp_rem_addr(struct ibv_exp_send_wr *wr,int size,uint64_t scnt,uint64_t prim_addr,VerbType verb, int cache_line_size, int cycle_buffer)
+{
+	if (verb == ATOMIC)
+		wr->wr.atomic.remote_addr += INC(size,cache_line_size);
+
+	else
+		wr->wr.rdma.remote_addr += INC(size,cache_line_size);
+
+	if ( ((scnt+1) % (cycle_buffer/ INC(size,cache_line_size))) == 0) {
+
+		if (verb == ATOMIC)
+			wr->wr.atomic.remote_addr = prim_addr;
+
+		else
+			wr->wr.rdma.remote_addr = prim_addr;
+	}
+}
+#endif
+static __inline void increase_rem_addr(struct ibv_send_wr *wr,int size,uint64_t scnt,uint64_t prim_addr,VerbType verb, int cache_line_size, int cycle_buffer)
+{
+	if (verb == ATOMIC)
+		wr->wr.atomic.remote_addr += INC(size,cache_line_size);
+
+	else
+		wr->wr.rdma.remote_addr += INC(size,cache_line_size);
+
+	if ( ((scnt+1) % (cycle_buffer/ INC(size,cache_line_size))) == 0) {
+
+		if (verb == ATOMIC)
+			wr->wr.atomic.remote_addr = prim_addr;
+
+		else
+			wr->wr.rdma.remote_addr = prim_addr;
+	}
+}
+
+/* increase_loc_addr.
+ *
+ * Description :
+ * 	Increases the local address in all verbs ,
+ *  (at least 64 CACHE_LINE size) , so that the system will be a able to cahce the data
+ *  in an orginzed way.
+ *
+ * Parameters :
+ *		sg - The scatter element of the wqe.
+ *		size - size of the message to send.
+ *		rcnt - The ammount of post_send or post_receive we called.
+ *		prim_addr - The address of the original buffer.
+ *		server_is_ud - Indication to weather we are in UD mode.
+ */
+static __inline void increase_loc_addr(struct ibv_sge *sg,int size,uint64_t rcnt,uint64_t prim_addr,int server_is_ud, int cache_line_size, int cycle_buffer)
+{
+	sg->addr  += INC(size,cache_line_size);
+
+	if ( ((rcnt+1) % (cycle_buffer/ INC(size,cache_line_size))) == 0 )
+		sg->addr = prim_addr;
+
+}
+
+/* catch_alarm.
+ *
+ * Description :
+ * 	Signal catcher for duration feature.
+ *  run_iter_bw in start will set it to triger at MARGIN (parameter), it will then start counting packets
+ *  and triger it back to SAMPLE TIME, in that time the test will count packets and completion and
+ *  will calculate BW accordingley.
+ *
+ */
+void catch_alarm(int sig);
+
+void check_alive(int sig);
+
+/* catch_alarm.
+ *
+ * Description :
+ * 	Signal catcher for run_infinitely feature.
+ *  Will be triggered every 5 sec and measure BW in this time frame.
+ *
+ */
+void catch_alarm_infintely();
+
+/* handle_signal_print_thread
+*
+* Description :
+* 	Handle thread creation for signal catching in run_infinitely mode
+*
+*/
+void *handle_signal_print_thread(void *sig_mask);
+
+/* ctx_modify_dc_qp_to_init.
+ *
+ * Description :
+ *
+ *	Modifies the given QP to INIT state , according to attributes in param.
+ *  The relevent attributes are ib_port, connection_type and verb.
+ *
+ * Parameters :
+ *
+ *	qp     - The QP that will be moved to INIT.
+ *	param  - The parameters for the QP.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ *
+ */
+int ctx_modify_dc_qp_to_init(struct ibv_qp *qp,struct perftest_parameters *user_param);
+
+int perform_warm_up(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+#ifdef HAVE_MASKED_ATOMICS
+struct ibv_qp* ctx_atomic_qp_create(struct pingpong_context *ctx,
+					struct perftest_parameters *user_param);
+int check_masked_atomics_support(struct pingpong_context *ctx);
+#endif
+
+#if defined (HAVE_PACKET_PACING_EXP) || defined (HAVE_PACKET_PACING)
+int check_packet_pacing_support(struct pingpong_context *ctx);
+#endif
+
+#ifdef HAVE_ACCL_VERBS
+struct ibv_exp_res_domain* create_res_domain(struct pingpong_context *ctx,
+						struct perftest_parameters *user_param);
+#endif
+
+int create_reg_qp_main(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param, int i, int num_of_qps);
+
+#ifdef HAVE_VERBS_EXP
+int create_exp_qp_main(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param, int i, int num_of_qps);
+#endif
+
+int create_qp_main(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param, int i, int num_of_qps);
+
+#ifdef HAVE_VERBS_EXP
+struct ibv_qp* ctx_exp_qp_create(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param, int qp_index);
+#endif
+
+int modify_qp_to_init(struct pingpong_context *ctx,
+                struct perftest_parameters *user_param, int qp_index, int num_of_qps);
+
+
+/* create_single_mr
+ *
+ * Description :
+ *
+ *	Creates a single MR for a specific QP index.
+ *
+ *	Parameters :
+ *      	ctx - Resources sructure.
+ *		user_param - the perftest parameters.
+ *		qp_index  - QP index to register a MR
+ *
+ * Return Value : SUCCESS, FAILURE.
+ *
+ */
+int create_single_mr(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param, int qp_index);
+
+/* create_mr
+ *
+ * Description :
+ *
+ *	Creates Memory Regions for the test.
+ *	Takes into consideration all user parameters and test type.
+ *
+ *	Parameters :
+ *      	ctx - Resources sructure.
+ *		user_param - the perftest parameters.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ *
+ */
+int create_mr(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param);
+
+/* alloc_hugapage_region
+ *
+ * Description :
+ *
+ *	Creates hugepage memory Regions for the test.
+ *
+ *	Parameters :
+ *      	ctx - Resources sructure.
+ *
+ * Return Value : SUCCESS, FAILURE.
+ *
+ */
+int alloc_hugepage_region (struct pingpong_context *ctx);
+
+/* run_iter_fs_rate
+ *
+ * Description :
+ *
+ *	The main testing method for Flow steering creation
+ *
+ * Parameters :
+ *
+ *	ctx		- Test Context.
+ *	user_param	- user_parameters struct for this test.
+ *
+ */
+
+int run_iter_fs(struct pingpong_context *ctx, struct perftest_parameters *user_param);
+
+#endif /* PERFTEST_RESOURCES_H */

usr/benchmarks/ib/raw_ethernet_fs_rate.c

@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#if defined(__FreeBSD__)
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <getopt.h>
+#include </usr/include/netinet/ip.h>
+#include <poll.h>
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "multicast_resources.h"
+#include "perftest_communication.h"
+#include "raw_ethernet_resources.h"
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	struct ibv_device		*ib_dev = NULL;
+	struct pingpong_context		ctx;
+	struct raw_ethernet_info	*my_dest_info = NULL;
+	struct raw_ethernet_info	*rem_dest_info = NULL;
+	int				ret_parser;
+	struct perftest_parameters	user_param;
+	struct report_options		report;
+
+	/* init default values to user's parameters */
+	memset(&ctx, 0, sizeof(struct pingpong_context));
+	memset(&user_param, 0, sizeof(struct perftest_parameters));
+
+	user_param.tst     = FS_RATE;
+	user_param.verb    = SEND;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+	user_param.connection_type = RawEth;
+	user_param.r_flag  = &report;
+
+	ret_parser = parser(&user_param, argv, argc);
+
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT) {
+			fprintf(stderr, " Parser function exited with Error\n");
+		}
+		DEBUG_LOG(TRACE,"<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+	user_param.machine = SERVER;
+
+	/* Allocate user input dependable structs */
+	ALLOCATE(my_dest_info, struct raw_ethernet_info, user_param.num_of_qps);
+	memset(my_dest_info, 0, sizeof(struct raw_ethernet_info) * user_param.num_of_qps);
+	ALLOCATE(rem_dest_info, struct raw_ethernet_info, user_param.num_of_qps);
+	memset(rem_dest_info, 0, sizeof(struct raw_ethernet_info) * user_param.num_of_qps);
+
+	/* Finding the IB device selected (or default if no selected). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr, "Unable to find the Infiniband/RoCE device\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+	GET_STRING(user_param.ib_devname, ibv_get_device_name(ib_dev));
+
+	if (check_flow_steering_support(user_param.ib_devname)) {
+		return FAILURE;
+	}
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, "Couldn't get context for the device\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link_and_mtu(ctx.context, &user_param)) {
+		fprintf(stderr, "Couldn't get context for the device\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx, &user_param);
+
+	/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
+	if (ctx_init(&ctx, &user_param)) {
+		fprintf(stderr, "Couldn't create IB resources\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	if(run_iter_fs(&ctx, &user_param)){
+		fprintf(stderr, "Unable to run iter fs rate\n");
+		return FAILURE;
+	}
+
+	print_report_fs_rate(&user_param);
+
+	if (destroy_ctx(&ctx, &user_param)) {
+		fprintf(stderr, "Failed to destroy_ctx\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY)
+		printf(RESULT_LINE);
+
+	DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+	return SUCCESS;
+}

usr/benchmarks/ib/raw_ethernet_resources.h

@@ -0,0 +1,356 @@
+#ifndef RAW_ETHERNET_RESOURCES_H
+#define RAW_ETHERNET_RESOURCES_H
+
+
+#if defined(__FreeBSD__)
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <getopt.h>
+#include </usr/include/netinet/ip.h>
+#include <poll.h>
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "multicast_resources.h"
+#include "perftest_communication.h"
+#if defined(__FreeBSD__)
+#include <infiniband/byteorder.h>
+#else
+#include <asm/byteorder.h>
+#endif
+
+#define INFO "INFO"
+#define TRACE "TRACE"
+
+#ifdef DEBUG
+#define DEBUG_LOG(type,fmt, args...) fprintf(stderr,"file:%s: %d ""["type"]"fmt"\n",__FILE__,__LINE__,args)
+#else
+#define DEBUG_LOG(type,fmt, args...)
+#endif
+
+#define PERF_MAC_FMT " %02X:%02X:%02X:%02X:%02X:%02X"
+
+#define IP_ETHER_TYPE (0x800)
+#define IP6_ETHER_TYPE (0x86DD)
+#define PRINT_ON (1)
+#define PRINT_OFF (0)
+#define UDP_PROTOCOL (0x11)
+#define TCP_PROTOCOL (0x06)
+#define IP_HEADER_LEN (20)
+#define DEFAULT_TTL (128)
+#define DEFAULT_IPV6_NEXT_HDR (0x3b)
+
+struct raw_ethernet_info {
+	uint8_t mac[6];
+	uint32_t ip;
+	uint8_t ip6[16];
+	int port;
+};
+
+
+
+/* gen_eth_header .
+ * Description :create raw Ethernet header on buffer
+ *
+ * Parameters :
+ *	 	eth_header - Pointer to output
+ *	 	src_mac - source MAC address of the packet
+ *	 	dst_mac - destination MAC address of the packet
+ *	 	eth_type - IP/or size of ptk
+ *
+ */
+struct ETH_header {
+	uint8_t dst_mac[6];
+	uint8_t src_mac[6];
+	uint16_t eth_type;
+}__attribute__((packed));
+
+struct ETH_vlan_header {
+        uint8_t dst_mac[6];
+        uint8_t src_mac[6];
+        uint32_t vlan_header;
+        uint16_t eth_type;
+}__attribute__((packed));
+
+#define VLAN_TPID (0x8100)
+#define VLAN_VID (0x001)
+#define VLAN_CFI (0)
+
+#if defined(__FreeBSD__)
+#if BYTE_ORDER == BIG_ENDIAN
+#define __BIG_ENDIAN_BITFIELD
+#define htobe32_const(x) (x)
+#elif BYTE_ORDER == LITTLE_ENDIAN
+#define __LITTLE_ENDIAN_BITFIELD
+#define htobe32_const(x) (((x) >> 24) | (((x) >> 8) & 0xff00) | \
+    ((((x) & 0xffffff) << 8) & 0xff0000) | ((((x) & 0xff) << 24) & 0xff000000))
+#else
+#error "Must set BYTE_ORDER"
+#endif
+#endif
+
+struct IP_V6_header {
+#if defined(__LITTLE_ENDIAN_BITFIELD)
+	__u8			priority:4,
+				version:4;
+#elif defined(__BIG_ENDIAN_BITFIELD)
+	__u8			version:4,
+				priority:4;
+#endif
+	__u8			flow_lbl[3];
+
+	__be16			payload_len;
+	__u8			nexthdr;
+	__u8			hop_limit;
+
+	struct	in6_addr	saddr;
+	struct	in6_addr	daddr;
+}__attribute__((packed));
+
+struct IP_V4_header{
+	#if defined(__LITTLE_ENDIAN_BITFIELD)
+	uint8_t		ihl:4;
+	uint8_t		version:4;
+	#elif defined(__BIG_ENDIAN_BITFIELD)
+	uint8_t		version:4;
+	uint8_t		ihl:4;
+	#endif
+	uint8_t		tos;
+	uint16_t 	tot_len;
+	uint16_t	id;
+	uint16_t	frag_off;
+	uint8_t		ttl;
+	uint8_t		protocol;
+	uint16_t	check;
+	uint32_t	saddr;
+	uint32_t	daddr;
+}__attribute__((packed));
+
+struct UDP_header {
+	u_short	uh_sport;		/* source port */
+	u_short	uh_dport;		/* destination port */
+	u_short	uh_ulen;		/* udp length */
+	u_short	uh_sum;			/* udp checksum */
+}__attribute__((packed));
+
+struct TCP_header {
+	uint16_t        th_sport;               /* source port */
+	uint16_t        th_dport;               /* destination port */
+	uint32_t        th_seq;
+	uint32_t        th_ack;
+	uint8_t         th_rsv:4;
+	uint8_t         th_doff:4;
+	uint8_t         th_falgs;
+	uint16_t        th_window;
+	uint16_t        th_check;
+	uint16_t        th_urgptr;
+}__attribute__((packed));
+
+void gen_eth_header(struct ETH_header* eth_header,uint8_t* src_mac,uint8_t* dst_mac, uint16_t eth_type);
+#ifdef HAVE_RAW_ETH_EXP
+void print_spec(struct ibv_exp_flow_attr* flow_rules,struct perftest_parameters* user_param);
+#else
+void print_spec(struct ibv_flow_attr* flow_rules,struct perftest_parameters* user_param);
+#endif
+//void print_ethernet_header(struct ETH_header* p_ethernet_header);
+void print_ethernet_header(void* p_ethernet_header);
+//void print_ethernet_vlan_header(struct ETH_vlan_header* p_ethernet_header);
+void print_ethernet_vlan_header(void* p_ethernet_header);
+void print_ip_header(struct IP_V4_header* ip_header);
+void print_udp_header(struct UDP_header* udp_header);
+void print_pkt(void* pkt,struct perftest_parameters *user_param);
+
+int check_flow_steering_support(char *dev_name);
+
+/* build_pkt_on_buffer
+ * Description: build single Ethernet packet on ctx buffer
+ *
+ * Parameters:
+ *		eth_header - Pointer to output
+ *		my_dest_info - ethernet information of me
+ *		rem_dest_info - ethernet information of the remote
+ *		user_param - user_parameters struct for this test
+ *		eth_type -
+ *		ip_next_protocol -
+ *		print_flag - if print_flag == TRUE : print the packet after it's done
+ *		pkt_size - size of the requested packet
+ *		flows_offset - current offset from the base flow
+ */
+void build_pkt_on_buffer(struct ETH_header* eth_header,
+		struct raw_ethernet_info *my_dest_info,
+		struct raw_ethernet_info *rem_dest_info,
+		struct perftest_parameters *user_param,
+		uint16_t eth_type,
+		uint16_t ip_next_protocol,
+		int print_flag,
+		int pkt_size,
+		int flows_offset);
+
+/*  create_raw_eth_pkt
+ * 	Description: build raw Ethernet packet by user arguments
+ *				 on bw test, build one packet and duplicate it on the buffer
+ *				 on lat test, build only one packet on the buffer (for the ping pong method)
+ *
+ *	Parameters:
+ *				user_param - user_parameters struct for this test
+ *				ctx - Test Context.
+ *				buf - The QP's packet buffer.
+ *				my_dest_info - ethernet information of me
+ *				rem_dest_info - ethernet information of the remote
+ */
+void create_raw_eth_pkt( struct perftest_parameters *user_param,
+		struct pingpong_context 	*ctx ,
+		void		 		*eth_header,
+		struct raw_ethernet_info	*my_dest_info,
+		struct raw_ethernet_info	*rem_dest_info);
+
+/*calc_flow_rules_size
+ * Description: calculate the size of the flow(size of headers - ib, ethernet and ip/udp if available)
+ * Parameters:
+ *				is_ip_header - if ip header is exist, count the header's size
+ *				is_udp_header - if udp header is exist, count the header's size
+ *
+ */
+int calc_flow_rules_size(struct perftest_parameters *user_param, int is_ip_header,int is_udp_header);
+
+/* send_set_up_connection
+ * Description: init raw_ethernet_info and ibv_flow_spec to user args
+ *
+ *	Parameters:
+ *				flow_rules - Pointer to output, is set to header buffer and specification information
+ *				ctx - Test Context.
+ *				user_param - user_parameters struct for this test
+ *				my_dest_info - ethernet information of me
+ *				rem_dest_info - ethernet information of the remote
+ *
+ */
+
+int send_set_up_connection(
+		#ifdef HAVE_RAW_ETH_EXP
+		struct ibv_exp_flow_attr **flow_rules,
+		#else
+		struct ibv_flow_attr **flow_rules,
+		#endif
+		struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct raw_ethernet_info* my_dest_info,
+		struct raw_ethernet_info* rem_dest_info);
+
+/* gen_ip_header .
+
+ * Description :create IP header on buffer
+ *
+ * Parameters :
+ * 		ip_header_buff - Pointer to output
+ * 		saddr - source IP address of the packet(network order)
+ * 		daddr - destination IP address of the packet(network order)
+ * 		pkt_size - size of the packet
+ *		flows_offset - current offset from the base flow
+ */
+void gen_ip_header(void* ip_header_buff, uint32_t* saddr, uint32_t* daddr,
+		   uint8_t protocol, int pkt_size, int tos, int flows_offset);
+
+/* gen_udp_header .
+
+ * Description :create UDP header on buffer
+ *
+ * Parameters :
+ * 		UDP_header_buffer - Pointer to output
+ *		src_port - source UDP port of the packet
+ *		dst_port -destination UDP port of the packet
+ *		pkt_size - size of the packet
+ */
+void gen_udp_header(void* UDP_header_buffer, int src_port, int dst_port, int pkt_size);
+
+/* gen_tcp_header .
+
+ * Description :create TCP header on buffer
+ *
+ * Parameters :
+ * 		TCP_header_buffer - Pointer to output
+ *		src_port - source TCP port of the packet
+ *		dst_port -destination TCP port of the packet
+ */
+void gen_tcp_header(void* TCP_header_buffer,int src_port ,int dst_port);
+
+/* run_iter_fw
+ *
+ * Description :
+ *
+ *  In this method we receive packets and "turn them around"
+ *  this is done by changing the dmac with the smac
+ *
+ * Parameters :
+ *
+ *  ctx     - Test Context.
+ *  user_param  - user_parameters struct for this test.
+ */
+int run_iter_fw(struct pingpong_context *ctx,struct perftest_parameters *user_param);
+
+/* switch_smac_dmac
+ *
+ * Description : In this method we receive buffer and change it's dmac and smac
+ *
+ * Parameters :
+ *
+ *  sg     - sg->addr is pointer to the buffer.
+ */
+static __inline void switch_smac_dmac(struct ibv_sge *sg)
+{
+	struct ETH_header* eth_header;
+	eth_header = (struct ETH_header*)sg->addr;
+	uint8_t tmp_mac[6] = {0} ;
+	memcpy(tmp_mac , (uint8_t *)eth_header + sizeof(eth_header->src_mac) ,sizeof(eth_header->src_mac));
+	memcpy((uint8_t *)eth_header->src_mac , (uint8_t *)eth_header->dst_mac ,sizeof(eth_header->src_mac));
+	memcpy((uint8_t *)eth_header->dst_mac  , tmp_mac ,sizeof(tmp_mac));
+}
+
+/* set_up_flow_rules
+ * Description: set the flow rules objects
+ *
+ *	Parameters:
+ *				flow_rules 	- Pointer to output, is set to header buffer and specification information
+ *				ctx 		- Test Context.
+ *				user_param 	- user_parameters struct for this test
+ *				local_port 	- the local port in the flow rule
+ *				remote_port 	- the remote port in the flow rule
+ */
+
+int set_up_flow_rules(
+		#ifdef HAVE_RAW_ETH_EXP
+		struct ibv_exp_flow_attr **flow_rules,
+		#else
+		struct ibv_flow_attr **flow_rules,
+		#endif
+		struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		int local_port,
+		int remote_port);
+/* set_up_fs_rules
+ * Description: set the flow rules objects  for FS rate test
+ *
+ *	Parameters:
+ *				flow_rules 	- Pointer to output, is set to header buffer and specification information
+ *				ctx 		- Test Context.
+ *				user_param 	- user_parameters struct for this test
+ *				allocated_flows	- number of flow ruled that are allocated and ready to be set
+ *
+ */
+
+int set_up_fs_rules(
+		#ifdef HAVE_RAW_ETH_EXP
+		struct ibv_exp_flow_attr **flow_rules,
+		#else
+		struct ibv_flow_attr **flow_rules,
+		#endif
+		struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		uint64_t allocated_flows);
+
+#endif /* RAW_ETHERNET_RESOURCES_H */

usr/benchmarks/ib/raw_ethernet_send_burst_lat.c

@@ -0,0 +1,301 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#if defined(__FreeBSD__)
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <getopt.h>
+#include </usr/include/netinet/ip.h>
+#include <poll.h>
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "multicast_resources.h"
+#include "perftest_communication.h"
+#include "raw_ethernet_resources.h"
+
+
+/*
+ * Main function. implements raw_ethernet_send_lat
+ */
+int main(int argc, char *argv[])
+{
+
+	struct ibv_device		*ib_dev = NULL;
+	struct pingpong_context		ctx;
+	struct raw_ethernet_info	my_dest_info,rem_dest_info;
+	int				ret_parser;
+	struct perftest_parameters	user_param;
+	#ifdef HAVE_RAW_ETH_EXP
+	struct ibv_exp_flow		**flow_create_result;
+	struct ibv_exp_flow_attr	**flow_rules;
+	struct ibv_exp_flow 		*flow_promisc = NULL;
+	#else
+	struct ibv_flow			**flow_create_result;
+	struct ibv_flow_attr		**flow_rules;
+	struct ibv_flow 		*flow_promisc = NULL;
+	#endif
+	struct report_options		report;
+	int				i;
+
+	/* allocate memory space for user parameters &*/
+	memset(&ctx,		0, sizeof(struct pingpong_context));
+	memset(&user_param, 0, sizeof(struct perftest_parameters));
+	memset(&my_dest_info, 0 , sizeof(struct raw_ethernet_info));
+	memset(&rem_dest_info, 0 , sizeof(struct raw_ethernet_info));
+
+	/* init default values to user's parameters that's relvant for this test:
+	 * Raw Ethernet Send Latency Test
+	 */
+	user_param.verb    = SEND;
+	user_param.tst     = LAT_BY_BW;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+	user_param.connection_type = RawEth;
+	user_param.r_flag  = &report;
+
+
+	/* Configure the parameters values according to user
+	   arguments or default values. */
+	ret_parser = parser(&user_param, argv,argc);
+
+	/* check for parsing errors */
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+	#ifdef HAVE_RAW_ETH_EXP
+	ALLOCATE(flow_create_result, struct ibv_exp_flow*, user_param.flows);
+	ALLOCATE(flow_rules, struct ibv_exp_flow_attr*, user_param.flows);
+	#else
+	ALLOCATE(flow_create_result, struct ibv_flow*, user_param.flows);
+	ALLOCATE(flow_rules, struct ibv_flow_attr*, user_param.flows);
+	#endif
+
+
+	/*this is a bidirectional test, so we need to let the init functions
+	 * think we are in duplex mode
+	 */
+	user_param.duplex  = 1;
+
+	/* Find the selected IB device (or default if the user didn't select one). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+	GET_STRING(user_param.ib_devname, ibv_get_device_name(ib_dev));
+
+	if (check_flow_steering_support(user_param.ib_devname)) {
+		return FAILURE;
+	}
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param)) {
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link_and_mtu(ctx.context, &user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx, &user_param);
+
+	/*set up the connection, return the required flow rules (notice that user_param->duplex == TRUE)
+	 * so the function will setup like it's a bidirectional test
+	 */
+	if (send_set_up_connection(flow_rules, &ctx, &user_param, &my_dest_info, &rem_dest_info)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	for (i = 0; i < user_param.flows; i++)
+		print_spec(flow_rules[i], &user_param);
+
+	/* initalize IB resources (data buffer, PD, MR, CQ and events channel) */
+	if (ctx_init(&ctx, &user_param)) {
+		fprintf(stderr, " Couldn't create IB resources\n");
+		return FAILURE;
+	}
+
+
+	/* attaching the qp to the spec */
+	for (i = 0; i < user_param.flows; i++) {
+		#ifdef HAVE_RAW_ETH_EXP
+		flow_create_result[i] = ibv_exp_create_flow(ctx.qp[0], flow_rules[i]);
+		#else
+		flow_create_result[i] = ibv_create_flow(ctx.qp[0], flow_rules[i]);
+		#endif
+
+		if (!flow_create_result[i]){
+			perror("error");
+			fprintf(stderr, "Couldn't attach QP\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.use_promiscuous) {
+		#ifdef HAVE_RAW_ETH_EXP
+		struct ibv_exp_flow_attr attr = {
+			.type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
+			.num_of_specs = 0,
+			.port = user_param.ib_port,
+			.flags = 0
+		};
+
+		if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) {
+			perror("error");
+			fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
+		}
+		#else
+		struct ibv_flow_attr attr = {
+			.type = IBV_FLOW_ATTR_ALL_DEFAULT,
+			.num_of_specs = 0,
+			.port = user_param.ib_port,
+			.flags = 0
+		};
+
+		if ((flow_promisc = ibv_create_flow(ctx.qp[0], &attr)) == NULL) {
+			perror("error");
+			fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
+		}
+		#endif
+	}
+
+	/* build ONE Raw Ethernet packets on ctx buffer */
+	create_raw_eth_pkt(&user_param, &ctx, (void*)ctx.buf[0], &my_dest_info , &rem_dest_info);
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+		printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	/* modify QPs to rtr/rts */
+	if (ctx_connect(&ctx, NULL, &user_param, NULL)) {
+		fprintf(stderr," Unable to Connect the HCA's through the link\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+
+	ctx_set_send_wqes(&ctx,&user_param,NULL);
+
+	if (ctx_set_recv_wqes(&ctx,&user_param)) {
+		fprintf(stderr," Failed to post receive recv_wqes\n");
+		return FAILURE;
+	}
+
+	/* latency test function for SEND verb latency test. */
+	if (user_param.machine == CLIENT) {
+		if (run_iter_lat_burst(&ctx, &user_param))
+			return FAILURE;
+	}
+	else {
+		if (run_iter_lat_burst_server(&ctx, &user_param))
+			return FAILURE;
+	}
+
+	/* print report (like print_report_bw) in the correct format
+	 * (as set before: FMT_LAT or FMT_LAT_DUR)
+	 */
+	if (user_param.machine == CLIENT)
+		print_report_lat(&user_param);
+
+	/* destroy promisc flow */
+	if (user_param.use_promiscuous) {
+		#ifdef HAVE_RAW_ETH_EXP
+		if (ibv_exp_destroy_flow(flow_promisc)) {
+		#else
+		if (ibv_destroy_flow(flow_promisc)) {
+		#endif
+			perror("error");
+			fprintf(stderr, "Couldn't destroy promisc flow\n");
+			return FAILURE;
+		}
+	}
+
+	/* destroy flow */
+	for (i = 0; i < user_param.flows; i++) {
+		#ifdef HAVE_RAW_ETH_EXP
+		if (ibv_exp_destroy_flow(flow_create_result[i])) {
+		#else
+		if (ibv_destroy_flow(flow_create_result[i])) {
+		#endif
+			perror("error");
+			fprintf(stderr, "Couldn't destroy flow\n");
+			return FAILURE;
+		}
+
+		free(flow_rules[i]);
+	}
+
+	/* Deallocate all perftest resources. */
+	if (destroy_ctx(&ctx, &user_param)) {
+		fprintf(stderr,"Failed to destroy_ctx\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY)
+		printf(RESULT_LINE);
+
+	DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+	return SUCCESS;
+}

usr/benchmarks/ib/raw_ethernet_send_bw.c

@@ -0,0 +1,493 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#if defined(__FreeBSD__)
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <getopt.h>
+#include </usr/include/netinet/ip.h>
+#include <poll.h>
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "multicast_resources.h"
+#include "perftest_communication.h"
+#include "raw_ethernet_resources.h"
+
+
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	struct ibv_device		*ib_dev = NULL;
+	struct pingpong_context		ctx;
+	struct raw_ethernet_info	*my_dest_info = NULL;
+	struct raw_ethernet_info	*rem_dest_info = NULL;
+	int				ret_parser;
+	struct perftest_parameters	user_param;
+
+	#ifdef HAVE_RAW_ETH_EXP
+	struct ibv_exp_flow		**flow_create_result;
+	struct ibv_exp_flow_attr	**flow_rules;
+	struct ibv_exp_flow 		**flow_promisc = NULL ;
+	#ifdef HAVE_SNIFFER_EXP
+	struct ibv_exp_flow 		**flow_sniffer = NULL;
+	#endif
+	#else
+	struct ibv_flow			**flow_create_result;
+	struct ibv_flow_attr		**flow_rules;
+	struct ibv_flow 		**flow_promisc = NULL ;
+	#ifdef HAVE_SNIFFER
+	struct ibv_flow 		**flow_sniffer = NULL;
+	#endif
+	#endif
+	int 				flow_index, qp_index;
+	union ibv_gid mgid;
+
+	/* init default values to user's parameters */
+	memset(&ctx, 0, sizeof(struct pingpong_context));
+	memset(&user_param, 0 , sizeof(struct perftest_parameters));
+
+	user_param.verb    = SEND;
+	user_param.tst     = BW;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+	user_param.connection_type = RawEth;
+
+	ret_parser = parser(&user_param, argv, argc);
+
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT) {
+			fprintf(stderr, " Parser function exited with Error\n");
+		}
+		DEBUG_LOG(TRACE,"<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+
+	/* Allocate user input dependable structs */
+	ALLOCATE(my_dest_info, struct raw_ethernet_info, user_param.num_of_qps);
+	memset(my_dest_info, 0, sizeof(struct raw_ethernet_info) * user_param.num_of_qps);
+	ALLOCATE(rem_dest_info, struct raw_ethernet_info, user_param.num_of_qps);
+	memset(rem_dest_info, 0, sizeof(struct raw_ethernet_info) * user_param.num_of_qps);
+
+	#ifdef HAVE_RAW_ETH_EXP
+        ALLOCATE(flow_create_result, struct ibv_exp_flow*, user_param.flows * user_param.num_of_qps);
+        ALLOCATE(flow_rules, struct ibv_exp_flow_attr*, user_param.flows * user_param.num_of_qps);
+	#ifdef HAVE_SNIFFER_EXP
+	ALLOCATE(flow_sniffer, struct ibv_exp_flow*, user_param.num_of_qps);
+	#endif
+	ALLOCATE(flow_promisc, struct ibv_exp_flow*, user_param.num_of_qps);
+	#else
+        ALLOCATE(flow_create_result, struct ibv_flow*, user_param.flows * user_param.num_of_qps);
+        ALLOCATE(flow_rules, struct ibv_flow_attr*, user_param.flows * user_param.num_of_qps);
+	#ifdef HAVE_SNIFFER
+	ALLOCATE(flow_sniffer, struct ibv_flow*, user_param.num_of_qps);
+	#endif
+        ALLOCATE(flow_promisc, struct ibv_flow*, user_param.num_of_qps);
+        #endif
+
+	if (user_param.raw_mcast) {
+		/* Transform IPv4 to Multicast MAC */
+		user_param.dest_mac[0] = 0x01;
+		user_param.dest_mac[1] = 0x00;
+		user_param.dest_mac[2] = 0x5e;
+		user_param.dest_mac[3] = (user_param.server_ip >> 8) & 0x7f;
+		user_param.dest_mac[4] = (user_param.server_ip >> 16) & 0xff;
+		user_param.dest_mac[5] = (user_param.server_ip >> 24) & 0xff;
+
+		/* Build up MGID (128bits, 16bytes) */
+		memset (&mgid, 0, sizeof (union ibv_gid));
+		memcpy (&mgid.raw[10], &user_param.dest_mac[0], 6);
+
+		/* Multicast send so no response UDP port */
+		user_param.client_port = 0;
+
+	}
+
+	if (user_param.use_rss) {
+		/* if num_of_qps is not even, set it to 2. */
+		if (user_param.num_of_qps % 2)
+			user_param.num_of_qps = 2;
+
+		/* add another one for rss parent QP */
+		user_param.num_of_qps += 1;
+	}
+
+	/* Finding the IB device selected (or default if no selected). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+	GET_STRING(user_param.ib_devname, ibv_get_device_name(ib_dev));
+
+	if (check_flow_steering_support(user_param.ib_devname)) {
+		return FAILURE;
+	}
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param)) {
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link_and_mtu(ctx.context, &user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx, &user_param);
+
+	/* set mac address by user choose */
+	for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+		if (send_set_up_connection(&flow_rules[qp_index * user_param.flows],
+						&ctx, &user_param, &my_dest_info[qp_index], &rem_dest_info[qp_index])) {
+			fprintf(stderr, " Unable to set up socket connection\n");
+			return FAILURE;
+		}
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	if ( !user_param.raw_mcast && (user_param.machine == SERVER || user_param.duplex)) {
+		for (flow_index = 0; flow_index < user_param.flows; flow_index++)
+			print_spec(flow_rules[flow_index], &user_param);
+	}
+
+	/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
+	if (ctx_init(&ctx, &user_param)) {
+		fprintf(stderr, " Couldn't create IB resources\n");
+		return FAILURE;
+	}
+
+	/* build raw Ethernet packets on ctx buffer */
+	if ((user_param.machine == CLIENT || user_param.duplex) && !user_param.mac_fwd) {
+		for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+			create_raw_eth_pkt(&user_param, &ctx, (void*)ctx.buf[qp_index], &my_dest_info[qp_index], &rem_dest_info[qp_index]);
+		}
+	}
+
+	/* create flow rules for servers/duplex clients ,  that not test raw_mcast */
+	if (!user_param.raw_mcast && (user_param.machine == SERVER || user_param.duplex)) {
+
+		/* attaching the qp to the spec */
+		for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+			for (flow_index = 0; flow_index < user_param.flows; flow_index++) {
+				#ifdef HAVE_RAW_ETH_EXP
+				flow_create_result[flow_index + qp_index * user_param.flows] =
+					ibv_exp_create_flow(ctx.qp[qp_index], flow_rules[flow_index]);
+				#else
+				flow_create_result[flow_index + qp_index * user_param.flows] =
+					ibv_create_flow(ctx.qp[qp_index], flow_rules[(qp_index * user_param.flows) + flow_index]);
+				#endif
+
+				if (!flow_create_result[flow_index + qp_index * user_param.flows]){
+					perror("error");
+					fprintf(stderr, "Couldn't attach QP\n");
+					return FAILURE;
+				}
+			}
+		}
+
+		if (user_param.use_promiscuous) {
+			#ifdef HAVE_RAW_ETH_EXP
+			struct ibv_exp_flow_attr attr = {
+				.type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
+				.num_of_specs = 0,
+				.port = user_param.ib_port,
+				.flags = 0
+			};
+
+			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+				if ((flow_promisc[qp_index] = ibv_exp_create_flow(ctx.qp[qp_index], &attr)) == NULL) {
+					perror("error");
+					fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
+				}
+			}
+			#else
+			struct ibv_flow_attr attr = {
+				.type = IBV_FLOW_ATTR_ALL_DEFAULT,
+				.num_of_specs = 0,
+				.port = user_param.ib_port,
+				.flags = 0
+			};
+
+			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+				if ((flow_promisc[qp_index] = ibv_create_flow(ctx.qp[qp_index], &attr)) == NULL) {
+					perror("error");
+					fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
+				}
+			}
+			#endif
+		}
+		#if defined HAVE_SNIFFER || defined HAVE_SNIFFER_EXP
+		if (user_param.use_sniffer) {
+			#ifdef HAVE_RAW_ETH_EXP
+			struct ibv_exp_flow_attr attr = {
+				.type = IBV_EXP_FLOW_ATTR_SNIFFER,
+				.num_of_specs = 0,
+				.port = user_param.ib_port,
+				.flags = 0
+			};
+			#else
+			struct ibv_flow_attr attr = {
+				.type = IBV_FLOW_ATTR_SNIFFER,
+				.num_of_specs = 0,
+				.port = user_param.ib_port,
+				.flags = 0
+			};
+			#endif
+
+			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+				#ifdef HAVE_RAW_ETH_EXP
+				if ((flow_sniffer[qp_index] = ibv_exp_create_flow(ctx.qp[qp_index], &attr)) == NULL) {
+				#else
+				if ((flow_sniffer[qp_index] = ibv_create_flow(ctx.qp[qp_index], &attr)) == NULL) {
+				#endif
+					perror("error");
+					fprintf(stderr, "Couldn't attach SNIFFER rule QP\n");
+				}
+			}
+		}
+		#endif /* HAVE_SNIFFER */
+	}
+
+	/* Prepare IB resources for rtr/rts. */
+	if (ctx_connect(&ctx, NULL, &user_param, NULL)) {
+		fprintf(stderr, " Unable to Connect the HCA's through the link\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+
+	if (user_param.raw_mcast) {
+		if (user_param.machine == SERVER) {
+			/* join Multicast group by MGID */
+			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+				ibv_attach_mcast(ctx.qp[qp_index], &mgid, 0);
+				printf(PERF_RAW_MGID_FMT, "MGID",
+						mgid.raw[0], mgid.raw[1],
+						mgid.raw[2], mgid.raw[3],
+						mgid.raw[4], mgid.raw[5],
+						mgid.raw[6], mgid.raw[7],
+						mgid.raw[8], mgid.raw[9],
+						mgid.raw[10],mgid.raw[11],
+						mgid.raw[12],mgid.raw[13],
+						mgid.raw[14],mgid.raw[15]);
+			}
+		}
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+		if (user_param.raw_qos)
+			printf((user_param.report_fmt == MBS ? RESULT_FMT_QOS : RESULT_FMT_G_QOS));
+		else
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	if (user_param.test_method == RUN_REGULAR) {
+		if (user_param.machine == CLIENT || user_param.duplex) {
+			ctx_set_send_wqes(&ctx,	&user_param, NULL);
+		}
+
+		if (user_param.machine == SERVER || user_param.duplex) {
+			if (ctx_set_recv_wqes(&ctx, &user_param)) {
+				fprintf(stderr," Failed to post receive recv_wqes\n");
+				DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+				return FAILURE;
+			}
+		}
+
+		if (user_param.mac_fwd) {
+
+			if(run_iter_fw(&ctx, &user_param)) {
+				DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+				return FAILURE;
+			}
+
+		} else if (user_param.duplex) {
+
+			if(run_iter_bi(&ctx, &user_param)) {
+				DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+				return FAILURE;
+			}
+
+		} else if (user_param.machine == CLIENT) {
+
+			if(run_iter_bw(&ctx, &user_param)) {
+				DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+				return FAILURE;
+			}
+
+		} else {
+
+			if(run_iter_bw_server(&ctx, &user_param)) {
+				DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+				return FAILURE;
+			}
+		}
+
+		print_report_bw(&user_param, NULL);
+	} else if (user_param.test_method == RUN_INFINITELY) {
+
+		if (user_param.machine == CLIENT)
+			ctx_set_send_wqes(&ctx, &user_param, NULL);
+
+		else if (user_param.machine == SERVER) {
+
+			if (ctx_set_recv_wqes(&ctx, &user_param)) {
+				fprintf(stderr, "Failed to post receive recv_wqes\n");
+				return FAILURE;
+			}
+		}
+
+		if (user_param.machine == CLIENT) {
+
+			if(run_iter_bw_infinitely(&ctx, &user_param)) {
+				fprintf(stderr, " Error occurred while running infinitely! aborting ...\n");
+				return FAILURE;
+			}
+
+		} else if (user_param.machine == SERVER) {
+
+			if(run_iter_bw_infinitely_server(&ctx, &user_param)) {
+				fprintf(stderr, " Error occurred while running infinitely on server! aborting ...\n");
+				return FAILURE;
+			}
+		}
+	}
+
+	if(user_param.machine == SERVER || user_param.duplex) {
+		/* destroy open flows */
+		for (flow_index = 0; flow_index < user_param.flows; flow_index++) {
+			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+				#ifdef HAVE_RAW_ETH_EXP
+				if (ibv_exp_destroy_flow(flow_create_result[flow_index + qp_index * user_param.flows])) {
+				#else
+				if (ibv_destroy_flow(flow_create_result[flow_index + qp_index * user_param.flows])) {
+				#endif
+					perror("error");
+					fprintf(stderr, "Couldn't destroy flow\n");
+					return FAILURE;
+				}
+			}
+		}
+		free(flow_rules);
+
+		if (user_param.use_promiscuous) {
+			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+				#ifdef HAVE_RAW_ETH_EXP
+				if (ibv_exp_destroy_flow(flow_promisc[qp_index])) {
+				#else
+				if (ibv_destroy_flow(flow_promisc[qp_index])) {
+				#endif
+					perror("error");
+					fprintf(stderr, "Couldn't destroy flow\n");
+					return FAILURE;
+				}
+			}
+			free(flow_promisc);
+		}
+
+		#if defined HAVE_SNIFFER || defined HAVE_SNIFFER_EXP
+		if (user_param.use_sniffer) {
+			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+				#ifdef HAVE_RAW_ETH_EXP
+				if (ibv_exp_destroy_flow(flow_sniffer[qp_index])) {
+				#else
+				if (ibv_destroy_flow(flow_sniffer[qp_index])) {
+				#endif
+					perror("error");
+					fprintf(stderr, "Couldn't destroy sniffer flow\n");
+					return FAILURE;
+				}
+			}
+			free(flow_sniffer);
+		}
+		#endif
+
+		if (user_param.raw_mcast) {
+			for (qp_index = 0; qp_index < user_param.num_of_qps; qp_index++) {
+				if (ibv_detach_mcast(ctx.qp[qp_index], &mgid, 0)) {
+					perror("error");
+					fprintf(stderr, "Couldn't leave multicast group\n");
+				}
+			}
+		}
+	}
+
+	if (destroy_ctx(&ctx, &user_param)) {
+		fprintf(stderr, "Failed to destroy_ctx\n");
+		DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+		return FAILURE;
+	}
+
+	free(my_dest_info);
+	free(rem_dest_info);
+
+	/* limit verifier */
+	if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
+		fprintf(stderr, "Error: BW result is below bw limit\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY)
+		printf(RESULT_LINE);
+
+	DEBUG_LOG(TRACE, "<<<<<<%s", __FUNCTION__);
+	return SUCCESS;
+}

usr/benchmarks/ib/raw_ethernet_send_lat.c

@@ -0,0 +1,296 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#if defined(__FreeBSD__)
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <getopt.h>
+#include </usr/include/netinet/ip.h>
+#include <poll.h>
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "multicast_resources.h"
+#include "perftest_communication.h"
+#include "raw_ethernet_resources.h"
+
+
+/*
+ * Main function. implements raw_ethernet_send_lat
+ */
+int main(int argc, char *argv[])
+{
+
+	struct ibv_device		*ib_dev = NULL;
+	struct pingpong_context		ctx;
+	struct raw_ethernet_info	my_dest_info,rem_dest_info;
+	int				ret_parser;
+	struct perftest_parameters	user_param;
+	#ifdef HAVE_RAW_ETH_EXP
+	struct ibv_exp_flow		**flow_create_result;
+	struct ibv_exp_flow_attr	**flow_rules;
+	struct ibv_exp_flow 		*flow_promisc = NULL;
+	#else
+	struct ibv_flow			**flow_create_result;
+	struct ibv_flow_attr		**flow_rules;
+	struct ibv_flow 		*flow_promisc = NULL;
+	#endif
+	struct report_options		report;
+	int				i;
+
+	/* allocate memory space for user parameters &*/
+	memset(&ctx,		0, sizeof(struct pingpong_context));
+	memset(&user_param, 0, sizeof(struct perftest_parameters));
+	memset(&my_dest_info, 0 , sizeof(struct raw_ethernet_info));
+	memset(&rem_dest_info, 0 , sizeof(struct raw_ethernet_info));
+
+	/* init default values to user's parameters that's relvant for this test:
+	 * Raw Ethernet Send Latency Test
+	 */
+	user_param.verb    = SEND;
+	user_param.tst     = LAT;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+	user_param.connection_type = RawEth;
+	user_param.r_flag  = &report;
+
+
+	/* Configure the parameters values according to user
+	   arguments or default values. */
+	ret_parser = parser(&user_param, argv,argc);
+
+	/* check for parsing errors */
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+	#ifdef HAVE_RAW_ETH_EXP
+	ALLOCATE(flow_create_result, struct ibv_exp_flow*, user_param.flows);
+	ALLOCATE(flow_rules, struct ibv_exp_flow_attr*, user_param.flows);
+	#else
+	ALLOCATE(flow_create_result, struct ibv_flow*, user_param.flows);
+	ALLOCATE(flow_rules, struct ibv_flow_attr*, user_param.flows);
+	#endif
+
+
+	/*this is a bidirectional test, so we need to let the init functions
+	 * think we are in duplex mode
+	 */
+	user_param.duplex  = 1;
+
+	/* Find the selected IB device (or default if the user didn't select one). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+	GET_STRING(user_param.ib_devname, ibv_get_device_name(ib_dev));
+
+	if (check_flow_steering_support(user_param.ib_devname)) {
+		return FAILURE;
+	}
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param)) {
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link_and_mtu(ctx.context, &user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx, &user_param);
+
+	/*set up the connection, return the required flow rules (notice that user_param->duplex == TRUE)
+	 * so the function will setup like it's a bidirectional test
+	 */
+	if (send_set_up_connection(flow_rules, &ctx, &user_param, &my_dest_info, &rem_dest_info)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	for (i = 0; i < user_param.flows; i++)
+		print_spec(flow_rules[i], &user_param);
+
+	/* initalize IB resources (data buffer, PD, MR, CQ and events channel) */
+	if (ctx_init(&ctx, &user_param)) {
+		fprintf(stderr, " Couldn't create IB resources\n");
+		return FAILURE;
+	}
+
+	/* attaching the qp to the spec */
+	for (i = 0; i < user_param.flows; i++) {
+		#ifdef HAVE_RAW_ETH_EXP
+		flow_create_result[i] = ibv_exp_create_flow(ctx.qp[0], flow_rules[i]);
+		#else
+		flow_create_result[i] = ibv_create_flow(ctx.qp[0], flow_rules[i]);
+		#endif
+
+		if (!flow_create_result[i]){
+			perror("error");
+			fprintf(stderr, "Couldn't attach QP\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.use_promiscuous) {
+		#ifdef HAVE_RAW_ETH_EXP
+		struct ibv_exp_flow_attr attr = {
+			.type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT,
+			.num_of_specs = 0,
+			.port = user_param.ib_port,
+			.flags = 0
+		};
+
+		if ((flow_promisc = ibv_exp_create_flow(ctx.qp[0], &attr)) == NULL) {
+			perror("error");
+			fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
+		}
+		#else
+		struct ibv_flow_attr attr = {
+			.type = IBV_FLOW_ATTR_ALL_DEFAULT,
+			.num_of_specs = 0,
+			.port = user_param.ib_port,
+			.flags = 0
+		};
+
+		if ((flow_promisc = ibv_create_flow(ctx.qp[0], &attr)) == NULL) {
+			perror("error");
+			fprintf(stderr, "Couldn't attach promiscuous rule QP\n");
+		}
+		#endif
+	}
+
+	/* build ONE Raw Ethernet packets on ctx buffer */
+	create_raw_eth_pkt(&user_param, &ctx, ctx.buf[0], &my_dest_info , &rem_dest_info);
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+		printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	/* modify QPs to rtr/rts */
+	if (ctx_connect(&ctx, NULL, &user_param, NULL)) {
+		fprintf(stderr," Unable to Connect the HCA's through the link\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+
+
+	ctx_set_send_wqes(&ctx,&user_param,NULL);
+
+	if (ctx_set_recv_wqes(&ctx,&user_param)) {
+		fprintf(stderr," Failed to post receive recv_wqes\n");
+		return FAILURE;
+	}
+
+	/* latency test function for SEND verb latency test. */
+	if (run_iter_lat_send(&ctx, &user_param)) {
+		return FAILURE;
+	}
+
+	/* print report (like print_report_bw) in the correct format
+	 * (as set before: FMT_LAT or FMT_LAT_DUR)
+	 */
+	user_param.test_type == ITERATIONS ? print_report_lat(&user_param) :
+		print_report_lat_duration(&user_param);
+
+	/* destroy promisc flow */
+	if (user_param.use_promiscuous) {
+		#ifdef HAVE_RAW_ETH_EXP
+		if (ibv_exp_destroy_flow(flow_promisc)) {
+		#else
+		if (ibv_destroy_flow(flow_promisc)) {
+		#endif
+			perror("error");
+			fprintf(stderr, "Couldn't destroy promisc flow\n");
+			return FAILURE;
+		}
+	}
+
+	/* destroy flow */
+	for (i = 0; i < user_param.flows; i++) {
+		#ifdef HAVE_RAW_ETH_EXP
+		if (ibv_exp_destroy_flow(flow_create_result[i])) {
+		#else
+		if (ibv_destroy_flow(flow_create_result[i])) {
+		#endif
+			perror("error");
+			fprintf(stderr, "Couldn't destroy flow\n");
+			return FAILURE;
+		}
+
+		free(flow_rules[i]);
+	}
+
+	/* Deallocate all perftest resources. */
+	if (destroy_ctx(&ctx, &user_param)) {
+		fprintf(stderr,"Failed to destroy_ctx\n");
+		DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY)
+		printf(RESULT_LINE);
+
+	DEBUG_LOG(TRACE,"<<<<<<%s",__FUNCTION__);
+	return SUCCESS;
+}

usr/benchmarks/ib/read_bw.c

@@ -0,0 +1,403 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2006 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "perftest_resources.h"
+#include "perftest_parameters.h"
+#include "perftest_communication.h"
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	int                        ret_parser,i = 0;
+	struct ibv_device	   *ib_dev = NULL;
+	struct pingpong_context    ctx;
+	struct pingpong_dest       *my_dest = NULL;
+	struct pingpong_dest       *rem_dest = NULL;
+	struct perftest_parameters user_param;
+	struct perftest_comm	   user_comm;
+	struct bw_report_data      my_bw_rep, rem_bw_rep;
+
+	/* init default values to user's parameters */
+	memset(&ctx,0,sizeof(struct pingpong_context));
+	memset(&user_param , 0 , sizeof(struct perftest_parameters));
+	memset(&user_comm,0,sizeof(struct perftest_comm));
+
+	user_param.verb    = READ;
+	user_param.tst     = BW;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+
+	ret_parser = parser(&user_param,argv,argc);
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		return FAILURE;
+	}
+
+	if((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
+		user_param.num_of_qps *= 2;
+	}
+
+	ib_dev =ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev)
+		return 7;
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param)) {
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link(ctx.context,&user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
+	if (create_comm_struct(&user_comm,&user_param)) {
+		fprintf(stderr," Unable to create RDMA_CM resources\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
+		printf("\n************************************\n");
+		printf("* Waiting for client to connect... *\n");
+		printf("************************************\n");
+	}
+
+	/* Initialize the connection and print the local data. */
+	if (establish_connection(&user_comm)) {
+		fprintf(stderr," Unable to init the socket connection\n");
+		return FAILURE;
+	}
+
+	exchange_versions(&user_comm, &user_param);
+
+	check_sys_data(&user_comm, &user_param);
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_mtu(ctx.context,&user_param, &user_comm)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx,&user_param);
+
+	/* Create (if nessacery) the rdma_cm ids and channel. */
+	if (user_param.work_rdma_cm == ON) {
+
+		if (user_param.machine == CLIENT) {
+			if (retry_rdma_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+
+		} else {
+			if (create_rdma_resources(&ctx,&user_param)) {
+				fprintf(stderr," Unable to create the rdma_resources\n");
+				return FAILURE;
+			}
+			if (rdma_server_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+		}
+
+	} else {
+		/* create all the basic IB resources. */
+		if (ctx_init(&ctx,&user_param)) {
+			fprintf(stderr, " Couldn't create IB resources\n");
+			return FAILURE;
+		}
+	}
+
+	/* Set up the Connection. */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		/* shaking hands and gather the other side info. */
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
+			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
+			fprintf(stderr," Please Try to use a different IP version.\n\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+
+		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
+			fprintf(stderr," Unable to Connect the HCA's through the link\n");
+			return FAILURE;
+		}
+	}
+
+	/* An additional handshake is required after moving qp to RTR. */
+	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		if (user_param.report_per_port) {
+			printf(RESULT_LINE_PER_PORT);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT_PER_PORT : RESULT_FMT_G_PER_PORT));
+		}
+		else {
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+		}
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	/* For half duplex tests, server just waits for client to exit */
+	if (user_param.machine == SERVER && !user_param.duplex) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+		print_full_bw_report(&user_param, &rem_bw_rep, NULL);
+
+		if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
+			fprintf(stderr,"Failed to close connection between server and client\n");
+			return FAILURE;
+		}
+		if (user_param.output == FULL_VERBOSITY) {
+			if (user_param.report_per_port)
+				printf(RESULT_LINE_PER_PORT);
+			else
+				printf(RESULT_LINE);
+		}
+
+		if (user_param.work_rdma_cm == ON) {
+			if (destroy_ctx(&ctx,&user_param)) {
+				fprintf(stderr, "Failed to destroy resources\n");
+				return FAILURE;
+			}
+			user_comm.rdma_params->work_rdma_cm = ON;
+			return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
+		}
+
+		return destroy_ctx(&ctx,&user_param);
+
+	}
+
+	if (user_param.use_event) {
+		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.test_method == RUN_ALL) {
+
+		for (i = 1; i < 24 ; ++i) {
+
+			user_param.size = (uint64_t)1 << i;
+			ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+			if (user_param.perform_warm_up) {
+				if(perform_warm_up(&ctx, &user_param)) {
+					fprintf(stderr, "Problems with warm up\n");
+					return FAILURE;
+				}
+			}
+
+			if(user_param.duplex) {
+				if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+					fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
+					return FAILURE;
+				}
+			}
+
+			if(run_iter_bw(&ctx,&user_param))
+				return 17;
+
+			if (user_param.duplex && (atof(user_param.version) >= 4.6)) {
+				if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+					fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
+					return FAILURE;
+				}
+			}
+
+			print_report_bw(&user_param,&my_bw_rep);
+
+			if (user_param.duplex) {
+				xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+				print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
+			}
+		}
+
+	} else if (user_param.test_method == RUN_REGULAR) {
+
+		ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+		if (user_param.perform_warm_up) {
+			if(perform_warm_up(&ctx, &user_param)) {
+				fprintf(stderr, "Problems with warm up\n");
+				return FAILURE;
+			}
+		}
+
+		if(user_param.duplex) {
+			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+				fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
+				return FAILURE;
+			}
+		}
+
+		if(run_iter_bw(&ctx,&user_param)) {
+			fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
+			return FAILURE;
+		}
+
+		print_report_bw(&user_param,&my_bw_rep);
+
+		if (user_param.duplex) {
+			xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+			print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
+		}
+
+		if (user_param.report_both && user_param.duplex) {
+			printf(RESULT_LINE);
+			printf("\n Local results: \n");
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+			print_full_bw_report(&user_param, &my_bw_rep, NULL);
+			printf(RESULT_LINE);
+
+			printf("\n Remote results: \n");
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+			print_full_bw_report(&user_param, &rem_bw_rep, NULL);
+		}
+	} else if (user_param.test_method == RUN_INFINITELY) {
+
+		ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+		if(run_iter_bw_infinitely(&ctx,&user_param)) {
+			fprintf(stderr," Error occurred while running! aborting ...\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		if (user_param.report_per_port)
+			printf(RESULT_LINE_PER_PORT);
+		else
+			printf(RESULT_LINE);
+	}
+
+	/* For half duplex tests, server just waits for client to exit */
+	if (user_param.machine == CLIENT && !user_param.duplex) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+	}
+
+	if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
+		fprintf(stderr,"Failed to close connection between server and client\n");
+		return FAILURE;
+	}
+
+	if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
+		fprintf(stderr,"Error: BW result is below bw limit\n");
+		return FAILURE;
+	}
+
+	if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
+		fprintf(stderr,"Error: Msg rate  is below msg_rate limit\n");
+		return FAILURE;
+	}
+
+	if (user_param.work_rdma_cm == ON) {
+		if (destroy_ctx(&ctx,&user_param)) {
+			fprintf(stderr, "Failed to destroy resources\n");
+			return FAILURE;
+		}
+		user_comm.rdma_params->work_rdma_cm = ON;
+		return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
+	}
+
+	return destroy_ctx(&ctx,&user_param);
+}

usr/benchmarks/ib/read_lat.c

@@ -0,0 +1,283 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler)
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#if !defined(__FreeBSD__)
+#include <malloc.h>
+#endif
+
+#include "get_clock.h"
+#include "perftest_resources.h"
+#include "perftest_parameters.h"
+#include "perftest_communication.h"
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	int                         ret_parser,i = 0;
+	struct report_options       report;
+	struct pingpong_context     ctx;
+	struct ibv_device           *ib_dev;
+	struct perftest_parameters  user_param;
+	struct pingpong_dest	    *my_dest  = NULL;
+	struct pingpong_dest	    *rem_dest = NULL;
+	struct perftest_comm		user_comm;
+
+	/* init default values to user's parameters */
+	memset(&ctx,0,sizeof(struct pingpong_context));
+	memset(&user_param,0,sizeof(struct perftest_parameters));
+	memset(&user_comm,0,sizeof(struct perftest_comm));
+
+	user_param.verb    = READ;
+	user_param.tst     = LAT;
+	user_param.r_flag  = &report;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+
+	/* Configure the parameters values according to user arguments or defalut values. */
+	ret_parser = parser(&user_param,argv,argc);
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		return FAILURE;
+	}
+
+	if(user_param.use_xrc || user_param.connection_type == DC) {
+		user_param.num_of_qps *= 2;
+	}
+
+	/* Finding the IB device selected (or defalut if no selected). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		return FAILURE;
+	}
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param)) {
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link(ctx.context,&user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
+	if (create_comm_struct(&user_comm,&user_param)) {
+		fprintf(stderr," Unable to create RDMA_CM resources\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
+		printf("\n************************************\n");
+		printf("* Waiting for client to connect... *\n");
+		printf("************************************\n");
+	}
+
+	/* Initialize the connection and print the local data. */
+	if (establish_connection(&user_comm)) {
+		fprintf(stderr," Unable to init the socket connection\n");
+		return FAILURE;
+	}
+
+	exchange_versions(&user_comm, &user_param);
+
+	check_sys_data(&user_comm, &user_param);
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_mtu(ctx.context,&user_param, &user_comm)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+
+	/* Allocate arrays */
+	alloc_ctx(&ctx,&user_param);
+
+	/* Create (if nessacery) the rdma_cm ids and channel. */
+	if (user_param.work_rdma_cm == ON) {
+
+		if (user_param.machine == CLIENT) {
+			if (retry_rdma_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+
+		} else {
+			if (create_rdma_resources(&ctx,&user_param)) {
+				fprintf(stderr," Unable to create the rdma_resources\n");
+				return FAILURE;
+			}
+			if (rdma_server_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+		}
+
+	} else {
+
+		/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
+		if (ctx_init(&ctx,&user_param)) {
+			fprintf(stderr, " Couldn't create IB resources\n");
+			return FAILURE;
+		}
+	}
+
+	/* Set up the Connection. */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	/*  shaking hands and gather the other side info. */
+	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	user_comm.rdma_params->side = REMOTE;
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		/* shaking hands and gather the other side info. */
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
+			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
+			fprintf(stderr," Please Try to use a different IP version.\n\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+
+		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
+			fprintf(stderr," Unable to Connect the HCA's through the link\n");
+			return FAILURE;
+		}
+	}
+
+	/* An additional handshake is required after moving qp to RTR. */
+	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	/* Only Client post read request. */
+	if (user_param.machine == SERVER) {
+
+		if (ctx_close_connection(&user_comm,my_dest,rem_dest)) {
+			fprintf(stderr,"Failed to close connection between server and client\n");
+			return FAILURE;
+		}
+		if (user_param.output == FULL_VERBOSITY) {
+			printf(RESULT_LINE);
+		}
+		return 0;
+
+	}
+
+	if (user_param.use_event) {
+		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+		printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+	if (user_param.test_method == RUN_ALL) {
+		for (i = 1; i < 24 ; ++i) {
+			user_param.size = (uint64_t)1 << i;
+			if(run_iter_lat(&ctx,&user_param))
+				return 17;
+
+			user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
+		}
+	} else {
+		if(run_iter_lat(&ctx,&user_param))
+			return 18;
+
+		user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
+	}
+
+	if (ctx_close_connection(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to close connection between server and client\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+	}
+	return 0;
+}

usr/benchmarks/ib/send_bw.c

@@ -0,0 +1,581 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "multicast_resources.h"
+#include "perftest_communication.h"
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+static int set_mcast_group(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct mcast_parameters *mcg_params)
+{
+	struct ibv_port_attr port_attr;
+
+	if (ibv_query_gid(ctx->context,user_param->ib_port,user_param->gid_index,&mcg_params->port_gid)) {
+		return FAILURE;
+	}
+
+	if (ibv_query_pkey(ctx->context,user_param->ib_port,DEF_PKEY_IDX,&mcg_params->pkey)) {
+		return FAILURE;
+	}
+
+	if (ibv_query_port(ctx->context,user_param->ib_port,&port_attr)) {
+		return FAILURE;
+	}
+	mcg_params->sm_lid  = port_attr.sm_lid;
+	mcg_params->sm_sl   = port_attr.sm_sl;
+	mcg_params->ib_port = user_param->ib_port;
+
+	if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
+		/* Request for Mcast group create registery in SM. */
+		if (join_multicast_group(SUBN_ADM_METHOD_SET,mcg_params)) {
+			fprintf(stderr,"Couldn't Register the Mcast group on the SM\n");
+			return FAILURE;
+		}
+	}
+	return 0;
+}
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+static int send_set_up_connection(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct pingpong_dest *my_dest,
+		struct mcast_parameters *mcg_params,
+		struct perftest_comm *comm)
+{
+	int i;
+
+	if (set_up_connection(ctx,user_param,my_dest)) {
+		fprintf(stderr," Unable to set up my IB connection parameters\n");
+		return FAILURE;
+	}
+
+	if (user_param->use_mcg && (user_param->duplex || user_param->machine == SERVER)) {
+
+		mcg_params->user_mgid = user_param->user_mgid;
+		set_multicast_gid(mcg_params,ctx->qp[0]->qp_num,(int)user_param->machine);
+		if (set_mcast_group(ctx,user_param,mcg_params)) {
+			return FAILURE;
+		}
+
+		for (i=0; i < user_param->num_of_qps; i++) {
+			if (ibv_attach_mcast(ctx->qp[i],&mcg_params->mgid,mcg_params->mlid)) {
+				fprintf(stderr, "Couldn't attach QP to MultiCast group");
+				return FAILURE;
+			}
+		}
+
+		mcg_params->mcast_state |= MCAST_IS_ATTACHED;
+		my_dest->gid = mcg_params->mgid;
+		my_dest->lid = mcg_params->mlid;
+		my_dest->qpn = QPNUM_MCAST;
+	}
+	return 0;
+}
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+static int send_destroy_ctx(
+		struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct mcast_parameters *mcg_params)
+{
+	int i;
+	if (user_param->use_mcg) {
+
+		if (user_param->duplex || user_param->machine == SERVER) {
+			for (i=0; i < user_param->num_of_qps; i++) {
+				if (ibv_detach_mcast(ctx->qp[i],&mcg_params->mgid,mcg_params->mlid)) {
+					fprintf(stderr, "Couldn't attach QP to MultiCast group");
+					return FAILURE;
+				}
+			}
+		}
+
+		/* Removal Request for Mcast group in SM if needed. */
+		if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
+			if (join_multicast_group(SUBN_ADM_METHOD_DELETE,mcg_params)) {
+				fprintf(stderr,"Couldn't Unregister the Mcast group on the SM\n");
+				return FAILURE;
+			}
+		}
+	}
+	return destroy_ctx(ctx,user_param);
+}
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	struct ibv_device		*ib_dev = NULL;
+	struct pingpong_context  	ctx;
+	struct pingpong_dest	 	*my_dest  = NULL;
+	struct pingpong_dest		*rem_dest = NULL;
+	struct perftest_parameters  	user_param;
+	struct perftest_comm		user_comm;
+	struct mcast_parameters     	mcg_params;
+	struct bw_report_data		my_bw_rep, rem_bw_rep;
+	int                      	ret_parser,i = 0;
+	int                      	size_max_pow = 24;
+
+	/* init default values to user's parameters */
+	memset(&ctx, 0,sizeof(struct pingpong_context));
+	memset(&user_param, 0 , sizeof(struct perftest_parameters));
+	memset(&mcg_params, 0 , sizeof(struct mcast_parameters));
+	memset(&user_comm, 0,sizeof(struct perftest_comm));
+
+	user_param.verb    = SEND;
+	user_param.tst     = BW;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+
+	/* Configure the parameters values according to user arguments or defalut values. */
+	ret_parser = parser(&user_param,argv,argc);
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		return FAILURE;
+	}
+	if((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
+		user_param.num_of_qps *= 2;
+	}
+	/* Checking that the user did not run with RawEth. for this we have raw_etherent_bw test. */
+	if (user_param.connection_type == RawEth) {
+		fprintf(stderr," This test cannot run Raw Ethernet QPs (you have chosen RawEth as connection type\n");
+		fprintf(stderr," For this we have raw_ethernet_bw test in this package.\n");
+		return FAILURE;
+	}
+
+	/* Finding the IB device selected (or defalut if no selected). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		return FAILURE;
+	}
+
+	if (user_param.use_mcg)
+		GET_STRING(mcg_params.ib_devname,ibv_get_device_name(ib_dev));
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param)) {
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link(ctx.context,&user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
+	if (create_comm_struct(&user_comm,&user_param)) {
+		fprintf(stderr," Unable to create RDMA_CM resources\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
+		printf("\n************************************\n");
+		printf("* Waiting for client to connect... *\n");
+		printf("************************************\n");
+	}
+
+	/* Initialize the connection and print the local data. */
+	if (establish_connection(&user_comm)) {
+		fprintf(stderr," Unable to init the socket connection\n");
+		return FAILURE;
+	}
+
+	exchange_versions(&user_comm, &user_param);
+
+	check_sys_data(&user_comm, &user_param);
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_mtu(ctx.context,&user_param, &user_comm)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	ALLOCATE(my_dest, struct pingpong_dest, user_param.num_of_qps);
+	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+	ALLOCATE(rem_dest, struct pingpong_dest, user_param.num_of_qps);
+	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+
+	if (user_param.transport_type == IBV_TRANSPORT_IWARP)
+		ctx.send_rcredit = 1;
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx,&user_param);
+
+	/* Create (if nessacery) the rdma_cm ids and channel. */
+	if (user_param.work_rdma_cm == ON) {
+
+		if (user_param.machine == CLIENT) {
+			if (retry_rdma_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+
+		} else {
+			if (create_rdma_resources(&ctx,&user_param)) {
+				fprintf(stderr," Unable to create the rdma_resources\n");
+				return FAILURE;
+			}
+			if (rdma_server_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+		}
+
+	} else {
+
+		/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
+		if (ctx_init(&ctx,&user_param)) {
+			fprintf(stderr, " Couldn't create IB resources\n");
+			return FAILURE;
+		}
+	}
+
+	/* Set up the Connection. */
+	if (send_set_up_connection(&ctx,&user_param,my_dest,&mcg_params,&user_comm)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	if (ctx.send_rcredit)
+		ctx_alloc_credit(&ctx,&user_param,my_dest);
+
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		/* shaking hands and gather the other side info. */
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
+			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
+			fprintf(stderr," Please Try to use a different IP version.\n\n");
+			return FAILURE;
+		}
+	}
+
+	/* If credit for available recieve buffers is necessary,
+	 * the credit sending is done via RDMA WRITE ops and the ctx_hand_shake above
+	 * is used to exchange the rkeys and buf addresses for the RDMA WRITEs
+	 */
+	if (ctx.send_rcredit)
+		ctx_set_credit_wqes(&ctx,&user_param,rem_dest);
+
+	/* Joining the Send side port the Mcast gid */
+	if (user_param.use_mcg && (user_param.machine == CLIENT || user_param.duplex)) {
+
+		memcpy(mcg_params.mgid.raw, rem_dest[0].gid.raw, 16);
+		if (set_mcast_group(&ctx,&user_param,&mcg_params)) {
+			fprintf(stderr," Unable to Join Sender to Mcast gid\n");
+			return FAILURE;
+		}
+		/*
+		 * The next stall in code (50 ms sleep) is a work around for fixing the
+		 * the bug this test had in Multicast for the past 1 year.
+		 * It appears, that when a switch involved, it takes ~ 10 ms for the join
+		 * request to propogate on the IB fabric, thus we need to wait for it.
+		 * what happened before this fix was client reaching the post_send
+		 * code segment in about 350 ns from here, and the switch(es) dropped
+		 * the packet because join request wasn't finished.
+		 */
+		usleep(50000);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+
+		/* Prepare IB resources for rtr/rts. */
+		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
+			fprintf(stderr," Unable to Connect the HCA's through the link\n");
+			return FAILURE;
+		}
+	}
+
+	/* shaking hands and gather the other side info. */
+	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	if (user_param.use_event) {
+
+		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
+			fprintf(stderr, " Couldn't request CQ notification\n");
+			return FAILURE;
+		}
+
+		if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
+			fprintf(stderr, " Couldn't request CQ notification\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		if (user_param.report_per_port) {
+			printf(RESULT_LINE_PER_PORT);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT_PER_PORT : RESULT_FMT_G_PER_PORT));
+		}
+		else {
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+		}
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	if (user_param.test_method == RUN_ALL) {
+
+		if (user_param.connection_type == UD)
+			size_max_pow =  (int)UD_MSG_2_EXP(MTU_SIZE(user_param.curr_mtu)) + 1;
+
+		for (i = 1; i < size_max_pow ; ++i) {
+
+			user_param.size = (uint64_t)1 << i;
+
+			if (user_param.machine == CLIENT || user_param.duplex)
+				ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+			if (user_param.machine == SERVER || user_param.duplex) {
+				if (ctx_set_recv_wqes(&ctx,&user_param)) {
+					fprintf(stderr," Failed to post receive recv_wqes\n");
+					return FAILURE;
+				}
+			}
+
+			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+				fprintf(stderr,"Failed to exchange data between server and clients\n");
+				return FAILURE;
+			}
+
+			if (ctx.send_rcredit) {
+				int j;
+				for (j = 0; j < user_param.num_of_qps; j++)
+					ctx.credit_buf[j] = 0;
+			}
+
+			if (user_param.duplex) {
+				if(run_iter_bi(&ctx,&user_param))
+					return 17;
+
+			} else if (user_param.machine == CLIENT) {
+
+				if(run_iter_bw(&ctx,&user_param)) {
+					return 17;
+				}
+
+			} else	{
+
+				if(run_iter_bw_server(&ctx,&user_param)) {
+					return 17;
+				}
+			}
+
+			print_report_bw(&user_param,&my_bw_rep);
+
+			if (user_param.duplex && user_param.test_type != DURATION) {
+				xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+				print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
+			}
+			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+				fprintf(stderr,"Failed to exchange data between server and clients\n");
+				return FAILURE;
+			}
+
+			/* Check if last iteration ended well in UC/UD */
+			if (user_param.check_alive_exited) {
+				break;
+			}
+		}
+
+	} else if (user_param.test_method == RUN_REGULAR) {
+
+		if (user_param.machine == CLIENT || user_param.duplex)
+			ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+		if (user_param.machine == SERVER || user_param.duplex) {
+			if (ctx_set_recv_wqes(&ctx,&user_param)) {
+				fprintf(stderr," Failed to post receive recv_wqes\n");
+				return FAILURE;
+			}
+		}
+
+		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		if (user_param.duplex) {
+
+			if(run_iter_bi(&ctx,&user_param))
+				return 17;
+
+		} else if (user_param.machine == CLIENT) {
+
+			if(run_iter_bw(&ctx,&user_param)) {
+				return 17;
+			}
+
+		} else if(run_iter_bw_server(&ctx,&user_param)) {
+
+			return 17;
+		}
+
+		print_report_bw(&user_param,&my_bw_rep);
+
+		if (user_param.duplex && user_param.test_type != DURATION) {
+			xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+			print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
+		}
+
+		if (user_param.report_both && user_param.duplex) {
+			printf(RESULT_LINE);
+			printf("\n Local results: \n");
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+			print_full_bw_report(&user_param, &my_bw_rep, NULL);
+			printf(RESULT_LINE);
+
+			printf("\n Remote results: \n");
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+			print_full_bw_report(&user_param, &rem_bw_rep, NULL);
+		}
+	} else if (user_param.test_method == RUN_INFINITELY) {
+
+		if (user_param.machine == CLIENT)
+			ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+		else if (user_param.machine == SERVER) {
+
+			if (ctx_set_recv_wqes(&ctx,&user_param)) {
+				fprintf(stderr," Failed to post receive recv_wqes\n");
+				return FAILURE;
+			}
+		}
+
+		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		if (user_param.machine == CLIENT) {
+
+			if(run_iter_bw_infinitely(&ctx,&user_param)) {
+				fprintf(stderr," Error occurred while running infinitely! aborting ...\n");
+				return FAILURE;
+			}
+
+		} else if (user_param.machine == SERVER) {
+
+			if(run_iter_bw_infinitely_server(&ctx,&user_param)) {
+				fprintf(stderr," Error occurred while running infinitely on server! aborting ...\n");
+				return FAILURE;
+			}
+		}
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		if (user_param.report_per_port)
+			printf(RESULT_LINE_PER_PORT);
+		else
+			printf(RESULT_LINE);
+	}
+
+	if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
+		fprintf(stderr," Failed to close connection between server and client\n");
+		fprintf(stderr," Trying to close this side resources\n");
+	}
+
+	/* Destroy all test resources, including Mcast if exists */
+	if (send_destroy_ctx(&ctx,&user_param,&mcg_params)) {
+		fprintf(stderr,"Couldn't destroy all SEND resources\n");
+		return FAILURE;
+	}
+	if (user_param.work_rdma_cm == ON) {
+		user_comm.rdma_params->work_rdma_cm = ON;
+		if (destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params)) {
+			fprintf(stderr,"Failed to destroy resources\n");
+			return FAILURE;
+		}
+	}
+
+	if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
+		fprintf(stderr,"Error: BW result is below bw limit\n");
+		return FAILURE;
+	}
+
+	if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
+		fprintf(stderr,"Error: Msg rate  is below msg_rate limit\n");
+		return FAILURE;
+	}
+
+	return 0;
+}

usr/benchmarks/ib/send_lat.c

@@ -0,0 +1,471 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler)
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#if !defined(__FreeBSD__)
+#include <malloc.h>
+#endif
+
+#include "get_clock.h"
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "multicast_resources.h"
+#include "perftest_communication.h"
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+static int set_mcast_group(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct mcast_parameters *mcg_params)
+{
+	int i;
+	struct ibv_port_attr port_attr;
+
+	if (ibv_query_gid(ctx->context,user_param->ib_port,user_param->gid_index,&mcg_params->port_gid)) {
+		return FAILURE;
+	}
+
+	if (ibv_query_pkey(ctx->context,user_param->ib_port,DEF_PKEY_IDX,&mcg_params->pkey)) {
+		return FAILURE;
+	}
+
+	if (ibv_query_port(ctx->context,user_param->ib_port,&port_attr)) {
+		return FAILURE;
+	}
+	mcg_params->sm_lid  = port_attr.sm_lid;
+	mcg_params->sm_sl   = port_attr.sm_sl;
+	mcg_params->ib_port = user_param->ib_port;
+	mcg_params->user_mgid = user_param->user_mgid;
+	set_multicast_gid(mcg_params,ctx->qp[0]->qp_num,(int)user_param->machine);
+
+	if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
+		/* Request for Mcast group create registery in SM. */
+		if (join_multicast_group(SUBN_ADM_METHOD_SET,mcg_params)) {
+			fprintf(stderr," Failed to Join Mcast request\n");
+			return FAILURE;
+		}
+	}
+
+	for (i=0; i < user_param->num_of_qps; i++) {
+
+		if (ibv_attach_mcast(ctx->qp[i],&mcg_params->mgid,mcg_params->mlid)) {
+			fprintf(stderr, "Couldn't attach QP to MultiCast group");
+			return FAILURE;
+		}
+	}
+	mcg_params->mcast_state |= MCAST_IS_ATTACHED;
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+static int send_set_up_connection(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct pingpong_dest *my_dest,
+		struct mcast_parameters *mcg_params,
+		struct perftest_comm *comm)
+{
+	int i;
+
+	if (set_up_connection(ctx,user_param,my_dest)) {
+		fprintf(stderr," Unable to set up my IB connection parameters\n");
+		return FAILURE;
+	}
+
+	if (user_param->use_mcg && (user_param->duplex || user_param->machine == SERVER)) {
+
+		mcg_params->user_mgid = user_param->user_mgid;
+		set_multicast_gid(mcg_params,ctx->qp[0]->qp_num,(int)user_param->machine);
+		if (set_mcast_group(ctx,user_param,mcg_params)) {
+			return FAILURE;
+		}
+
+		for (i=0; i < user_param->num_of_qps; i++) {
+			if (ibv_attach_mcast(ctx->qp[i],&mcg_params->mgid,mcg_params->mlid)) {
+				fprintf(stderr, "Couldn't attach QP to MultiCast group");
+				return FAILURE;
+			}
+		}
+
+		mcg_params->mcast_state |= MCAST_IS_ATTACHED;
+		my_dest->gid = mcg_params->mgid;
+		my_dest->lid = mcg_params->mlid;
+		my_dest->qpn = QPNUM_MCAST;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+static int send_destroy_ctx(struct pingpong_context *ctx,
+		struct perftest_parameters *user_param,
+		struct mcast_parameters *mcg_params)
+{
+	int i;
+	if (user_param->use_mcg) {
+
+		for (i=0; i < user_param->num_of_qps; i++) {
+			if (ibv_detach_mcast(ctx->qp[i],&mcg_params->base_mgid,mcg_params->base_mlid)) {
+				fprintf(stderr, "Couldn't dettach QP to MultiCast group\n");
+				return FAILURE;
+			}
+		}
+
+		if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
+
+			if (join_multicast_group(SUBN_ADM_METHOD_DELETE,mcg_params)) {
+				fprintf(stderr,"Couldn't Unregister the Mcast group on the SM\n");
+				return FAILURE;
+			}
+
+			memcpy(mcg_params->mgid.raw,mcg_params->base_mgid.raw,16);
+
+			if (join_multicast_group(SUBN_ADM_METHOD_DELETE,mcg_params)) {
+				fprintf(stderr,"Couldn't Unregister the Mcast group on the SM\n");
+				return FAILURE;
+			}
+		}
+	}
+	return destroy_ctx(ctx,user_param);
+}
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	int                        i = 0;
+	int                        size_max_pow = 24;
+	int			   ret_val;
+	struct report_options      report;
+	struct pingpong_context    ctx;
+	struct pingpong_dest	   *my_dest  = NULL;
+	struct pingpong_dest	   *rem_dest = NULL;
+	struct mcast_parameters	   mcg_params;
+	struct ibv_device          *ib_dev = NULL;
+	struct perftest_parameters user_param;
+	struct perftest_comm	   user_comm;
+
+	/* init default values to user's parameters */
+	memset(&ctx,		0, sizeof(struct pingpong_context));
+	memset(&user_param, 0, sizeof(struct perftest_parameters));
+	memset(&user_comm , 0, sizeof(struct perftest_comm));
+	memset(&mcg_params, 0, sizeof(struct mcast_parameters));
+
+	user_param.verb    = SEND;
+	user_param.tst     = LAT;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+	user_param.r_flag  = &report;
+
+	/* Configure the parameters values according to user arguments or defalut values. */
+	ret_val = parser(&user_param,argv,argc);
+	if (ret_val) {
+		if (ret_val != VERSION_EXIT && ret_val != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		return FAILURE;
+	}
+
+	if(user_param.use_xrc || user_param.connection_type == DC) {
+		user_param.num_of_qps *= 2;
+	}
+
+	/* Checking that the user did not run with RawEth. for this we have raw_etherent_bw test. */
+	if (user_param.connection_type == RawEth) {
+		fprintf(stderr," This test cannot run Raw Ethernet QPs (you have chosen RawEth as connection type\n");
+		return FAILURE;
+	}
+
+	/* Finding the IB device selected (or defalut if no selected). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		return FAILURE;
+	}
+
+	if (user_param.use_mcg)
+		GET_STRING(mcg_params.ib_devname,ibv_get_device_name(ib_dev));
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* Verify user parameters that require the device context,
+	 * the function will print the relevent error info. */
+	if (verify_params_with_device_context(ctx.context, &user_param)) {
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link(ctx.context,&user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
+	if (create_comm_struct(&user_comm,&user_param)) {
+		fprintf(stderr," Unable to create RDMA_CM resources\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
+		printf("\n************************************\n");
+		printf("* Waiting for client to connect... *\n");
+		printf("************************************\n");
+	}
+
+	/* Initialize the connection and print the local data. */
+	if (establish_connection(&user_comm)) {
+		fprintf(stderr," Unable to init the socket connection\n");
+		return FAILURE;
+	}
+
+	exchange_versions(&user_comm, &user_param);
+
+	check_sys_data(&user_comm, &user_param);
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_mtu(ctx.context,&user_param, &user_comm)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx,&user_param);
+
+	/* Create (if nessacery) the rdma_cm ids and channel. */
+	if (user_param.work_rdma_cm == ON) {
+
+		if (user_param.machine == CLIENT) {
+			if (retry_rdma_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+
+		} else {
+			if (create_rdma_resources(&ctx,&user_param)) {
+				fprintf(stderr," Unable to create the rdma_resources\n");
+				return FAILURE;
+			}
+			if (rdma_server_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+		}
+
+	} else {
+
+		/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
+		if (ctx_init(&ctx,&user_param)) {
+			fprintf(stderr, " Couldn't create IB resources\n");
+			return FAILURE;
+		}
+	}
+
+	/* Set up the Connection. */
+	if (send_set_up_connection(&ctx,&user_param,my_dest,&mcg_params,&user_comm)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	user_comm.rdma_params->side = REMOTE;
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		/* shaking hands and gather the other side info. */
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
+			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
+			fprintf(stderr," Please Try to use a different IP version.\n\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.use_mcg) {
+
+		memcpy(mcg_params.base_mgid.raw,mcg_params.mgid.raw,16);
+		memcpy(mcg_params.mgid.raw,rem_dest[0].gid.raw,16);
+		mcg_params.base_mlid = mcg_params.mlid;
+		mcg_params.is_2nd_mgid_used = ON;
+		if (!strcmp(link_layer_str(user_param.link_type),"IB")) {
+			/* Request for Mcast group create registery in SM. */
+			if (join_multicast_group(SUBN_ADM_METHOD_SET,&mcg_params)) {
+				fprintf(stderr," Failed to Join Mcast request\n");
+				return FAILURE;
+			}
+		}
+
+		/*
+		 * The next stall in code (50 ms sleep) is a work around for fixing the
+		 * the bug this test had in Multicast for the past 1 year.
+		 * It appears, that when a switch involved, it takes ~ 10 ms for the join
+		 * request to propogate on the IB fabric, thus we need to wait for it.
+		 * what happened before this fix was  reaching the post_send
+		 * code segment in about 350 ns from here, and the switch(es) dropped
+		 * the packet because join request wasn't finished.
+		 */
+		usleep(50000);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+
+		/* Prepare IB resources for rtr/rts. */
+		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
+			fprintf(stderr," Unable to Connect the HCA's through the link\n");
+			return FAILURE;
+		}
+	}
+
+	/* shaking hands and gather the other side info. */
+	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	if (user_param.use_event) {
+
+		if (ibv_req_notify_cq(ctx.send_cq, 0)) {
+			fprintf(stderr, "Couldn't request RCQ notification\n");
+			return FAILURE;
+		}
+
+		if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
+			fprintf(stderr, "Couldn't request RCQ notification\n");
+			return FAILURE;
+		}
+	}
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+		printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+	if (user_param.test_method == RUN_ALL) {
+
+		if (user_param.connection_type == UD)
+			size_max_pow =  (int)UD_MSG_2_EXP(MTU_SIZE(user_param.curr_mtu)) + 1;
+
+		for (i = 1; i < size_max_pow ; ++i) {
+
+			user_param.size = (uint64_t)1 << i;
+
+			/* Post receive recv_wqes fo current message size */
+			if (ctx_set_recv_wqes(&ctx,&user_param)) {
+				fprintf(stderr," Failed to post receive recv_wqes\n");
+				return FAILURE;
+			}
+
+			/* Sync between the client and server so the client won't send packets
+			 * Before the server has posted his receive wqes (in UC/UD it will result in a deadlock).
+			 */
+
+			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+				fprintf(stderr,"Failed to exchange data between server and clients\n");
+				return FAILURE;
+			}
+
+			if(run_iter_lat_send(&ctx, &user_param))
+				return 17;
+
+			user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
+		}
+
+	} else {
+
+		/* Post recevie recv_wqes fo current message size */
+		if (ctx_set_recv_wqes(&ctx,&user_param)) {
+			fprintf(stderr," Failed to post receive recv_wqes\n");
+			return FAILURE;
+		}
+
+		/* Sync between the client and server so the client won't send packets
+		 * Before the server has posted his receive wqes (in UC/UD it will result in a deadlock).
+		 */
+
+		if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		if(run_iter_lat_send(&ctx, &user_param))
+			return 17;
+
+		user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+	}
+
+	if (ctx_close_connection(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to close connection between server and client\n");
+		fprintf(stderr," Trying to close this side resources\n");
+	}
+
+	return send_destroy_ctx(&ctx,&user_param,&mcg_params);
+}

usr/benchmarks/ib/write_bw.c

@@ -0,0 +1,404 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "perftest_communication.h"
+
+
+int main(int argc, char *argv[])
+{
+	int				ret_parser,i = 0;
+	struct ibv_device		*ib_dev = NULL;
+	struct pingpong_context		ctx;
+	struct pingpong_dest		*my_dest,*rem_dest;
+	struct perftest_parameters	user_param;
+	struct perftest_comm		user_comm;
+	struct bw_report_data		my_bw_rep, rem_bw_rep;
+
+	/* init default values to user's parameters */
+	memset(&user_param,0,sizeof(struct perftest_parameters));
+	memset(&user_comm,0,sizeof(struct perftest_comm));
+	memset(&ctx,0,sizeof(struct pingpong_context));
+
+	user_param.verb    = WRITE;
+	user_param.tst     = BW;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+
+	/* Configure the parameters values according to user arguments or default values. */
+	ret_parser = parser(&user_param,argv,argc);
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		return FAILURE;
+	}
+
+	if((user_param.connection_type == DC || user_param.use_xrc) && user_param.duplex) {
+		user_param.num_of_qps *= 2;
+	}
+
+	/* Finding the IB device selected (or default if none is selected). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		return FAILURE;
+	}
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link(ctx.context,&user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
+	if (create_comm_struct(&user_comm,&user_param)) {
+		fprintf(stderr," Unable to create RDMA_CM resources\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
+		printf("\n************************************\n");
+		printf("* Waiting for client to connect... *\n");
+		printf("************************************\n");
+	}
+
+	/* Initialize the connection and print the local data. */
+	if (establish_connection(&user_comm)) {
+		fprintf(stderr," Unable to init the socket connection\n");
+		return FAILURE;
+	}
+	sleep(1);
+	exchange_versions(&user_comm, &user_param);
+
+	check_sys_data(&user_comm, &user_param);
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_mtu(ctx.context,&user_param, &user_comm)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx,&user_param);
+
+	/* Create (if necessary) the rdma_cm ids and channel. */
+	if (user_param.work_rdma_cm == ON) {
+
+		if (user_param.machine == CLIENT) {
+			if (retry_rdma_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+
+		} else {
+			if (create_rdma_resources(&ctx,&user_param)) {
+				fprintf(stderr," Unable to create the rdma_resources\n");
+				return FAILURE;
+			}
+			if (rdma_server_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+		}
+
+	} else {
+
+		/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
+		if (ctx_init(&ctx, &user_param)) {
+			fprintf(stderr, " Couldn't create IB resources\n");
+			return FAILURE;
+		}
+	}
+
+	/* Set up the Connection. */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+
+	/* Print this machine QP information */
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+
+	user_comm.rdma_params->side = REMOTE;
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
+			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
+			fprintf(stderr," Please Try to use a different IP version.\n\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
+			fprintf(stderr," Unable to Connect the HCA's through the link\n");
+			return FAILURE;
+		}
+	}
+
+	/* An additional handshake is required after moving qp to RTR. */
+	if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+		fprintf(stderr," Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		if (user_param.report_per_port) {
+			printf(RESULT_LINE_PER_PORT);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT_PER_PORT : RESULT_FMT_G_PER_PORT));
+		}
+		else {
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+		}
+
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	/* For half duplex tests, server just waits for client to exit */
+	if (user_param.machine == SERVER && !user_param.duplex) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+		print_full_bw_report(&user_param, &rem_bw_rep, NULL);
+
+		if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
+			fprintf(stderr,"Failed to close connection between server and client\n");
+			return FAILURE;
+		}
+
+		if (user_param.output == FULL_VERBOSITY) {
+			if (user_param.report_per_port)
+				printf(RESULT_LINE_PER_PORT);
+			else
+				printf(RESULT_LINE);
+		}
+
+		if (user_param.work_rdma_cm == ON) {
+			if (destroy_ctx(&ctx,&user_param)) {
+				fprintf(stderr, "Failed to destroy resources\n");
+				return FAILURE;
+			}
+			user_comm.rdma_params->work_rdma_cm = ON;
+			return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
+		}
+
+		return destroy_ctx(&ctx,&user_param);
+	}
+
+	if (user_param.test_method == RUN_ALL) {
+
+		for (i = 1; i < 24 ; ++i) {
+
+			user_param.size = (uint64_t)1 << i;
+			ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+			if (user_param.perform_warm_up) {
+				if(perform_warm_up(&ctx, &user_param)) {
+					fprintf(stderr, "Problems with warm up\n");
+					return FAILURE;
+				}
+			}
+
+			if(user_param.duplex) {
+				if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+					fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
+					return FAILURE;
+				}
+			}
+
+			if(run_iter_bw(&ctx,&user_param)) {
+				fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
+				return FAILURE;
+			}
+
+			if (user_param.duplex && (atof(user_param.version) >= 4.6)) {
+				if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+					fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
+					return FAILURE;
+				}
+			}
+
+			print_report_bw(&user_param,&my_bw_rep);
+
+			if (user_param.duplex) {
+				xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+				print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
+			}
+		}
+
+	} else if (user_param.test_method == RUN_REGULAR) {
+
+		ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+		if (user_param.verb != SEND) {
+
+			if (user_param.perform_warm_up) {
+				if(perform_warm_up(&ctx, &user_param)) {
+					fprintf(stderr, "Problems with warm up\n");
+					return FAILURE;
+				}
+			}
+		}
+
+		if(user_param.duplex) {
+			if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+				fprintf(stderr,"Failed to sync between server and client between different msg sizes\n");
+				return FAILURE;
+			}
+		}
+
+		if(run_iter_bw(&ctx,&user_param)) {
+			fprintf(stderr," Failed to complete run_iter_bw function successfully\n");
+			return FAILURE;
+		}
+
+		print_report_bw(&user_param,&my_bw_rep);
+
+		if (user_param.duplex) {
+			xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+			print_full_bw_report(&user_param, &my_bw_rep, &rem_bw_rep);
+		}
+
+		if (user_param.report_both && user_param.duplex) {
+			printf(RESULT_LINE);
+			printf("\n Local results: \n");
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+			print_full_bw_report(&user_param, &my_bw_rep, NULL);
+			printf(RESULT_LINE);
+
+			printf("\n Remote results: \n");
+			printf(RESULT_LINE);
+			printf((user_param.report_fmt == MBS ? RESULT_FMT : RESULT_FMT_G));
+			printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+			print_full_bw_report(&user_param, &rem_bw_rep, NULL);
+		}
+	} else if (user_param.test_method == RUN_INFINITELY) {
+
+		ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+		if(run_iter_bw_infinitely(&ctx,&user_param)) {
+			fprintf(stderr," Error occurred while running infinitely! aborting ...\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		if (user_param.report_per_port)
+			printf(RESULT_LINE_PER_PORT);
+		else
+			printf(RESULT_LINE);
+	}
+
+	/* For half duplex tests, server just waits for client to exit */
+	if (user_param.machine == CLIENT && !user_param.duplex) {
+
+		if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
+			fprintf(stderr," Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		xchg_bw_reports(&user_comm, &my_bw_rep,&rem_bw_rep,atof(user_param.rem_version));
+	}
+
+	/* Closing connection. */
+	if (ctx_close_connection(&user_comm,&my_dest[0],&rem_dest[0])) {
+		fprintf(stderr,"Failed to close connection between server and client\n");
+		return FAILURE;
+	}
+
+	if (!user_param.is_bw_limit_passed && (user_param.is_limit_bw == ON ) ) {
+		fprintf(stderr,"Error: BW result is below bw limit\n");
+		return FAILURE;
+	}
+
+	if (!user_param.is_msgrate_limit_passed && (user_param.is_limit_bw == ON )) {
+		fprintf(stderr,"Error: Msg rate  is below msg_rate limit\n");
+		return FAILURE;
+	}
+
+	free(my_dest);
+	free(rem_dest);
+
+	if (user_param.work_rdma_cm == ON) {
+		if (destroy_ctx(&ctx,&user_param)) {
+			fprintf(stderr, "Failed to destroy resources\n");
+			return FAILURE;
+		}
+		user_comm.rdma_params->work_rdma_cm = ON;
+		return destroy_ctx(user_comm.rdma_ctx,user_comm.rdma_params);
+	}
+
+	return destroy_ctx(&ctx,&user_param);
+}

usr/benchmarks/ib/write_lat.c

@@ -0,0 +1,256 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler)
+ * Copyright (c) 2009 HNR Consulting.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#if !defined(__FreeBSD__)
+#include <malloc.h>
+#endif
+
+#include "get_clock.h"
+#include "perftest_parameters.h"
+#include "perftest_resources.h"
+#include "perftest_communication.h"
+
+/******************************************************************************
+ *
+ ******************************************************************************/
+int main(int argc, char *argv[])
+{
+	int				ret_parser,i = 0;
+	struct report_options		report;
+	struct pingpong_context		ctx;
+	struct pingpong_dest		*my_dest  = NULL;
+	struct pingpong_dest		*rem_dest = NULL;
+	struct ibv_device		*ib_dev;
+	struct perftest_parameters	user_param;
+	struct perftest_comm		user_comm;
+
+	/* init default values to user's parameters */
+	memset(&ctx,0,sizeof(struct pingpong_context));
+	memset(&user_param, 0, sizeof(struct perftest_parameters));
+	memset(&user_comm,0,sizeof(struct perftest_comm));
+
+	user_param.verb    = WRITE;
+	user_param.tst     = LAT;
+	user_param.r_flag  = &report;
+	strncpy(user_param.version, VERSION, sizeof(user_param.version));
+
+	/* Configure the parameters values according to user arguments or defalut values. */
+	ret_parser = parser(&user_param,argv,argc);
+	if (ret_parser) {
+		if (ret_parser != VERSION_EXIT && ret_parser != HELP_EXIT)
+			fprintf(stderr," Parser function exited with Error\n");
+		return FAILURE;
+	}
+
+	if(user_param.use_xrc || user_param.connection_type == DC) {
+		user_param.num_of_qps *= 2;
+	}
+
+	/* Finding the IB device selected (or defalut if no selected). */
+	ib_dev = ctx_find_dev(user_param.ib_devname);
+	if (!ib_dev) {
+		fprintf(stderr," Unable to find the Infiniband/RoCE device\n");
+		return FAILURE;
+	}
+
+	/* Getting the relevant context from the device */
+	ctx.context = ibv_open_device(ib_dev);
+	if (!ctx.context) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_link(ctx.context,&user_param)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	/* copy the relevant user parameters to the comm struct + creating rdma_cm resources. */
+	if (create_comm_struct(&user_comm,&user_param)) {
+		fprintf(stderr," Unable to create RDMA_CM resources\n");
+		return FAILURE;
+	}
+
+	if (user_param.output == FULL_VERBOSITY && user_param.machine == SERVER) {
+		printf("\n************************************\n");
+		printf("* Waiting for client to connect... *\n");
+		printf("************************************\n");
+	}
+
+	/* Initialize the connection and print the local data. */
+	if (establish_connection(&user_comm)) {
+		fprintf(stderr," Unable to init the socket connection\n");
+		return FAILURE;
+	}
+
+	exchange_versions(&user_comm, &user_param);
+
+	check_sys_data(&user_comm, &user_param);
+
+	/* See if MTU and link type are valid and supported. */
+	if (check_mtu(ctx.context,&user_param, &user_comm)) {
+		fprintf(stderr, " Couldn't get context for the device\n");
+		return FAILURE;
+	}
+
+	ALLOCATE(my_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(my_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+	ALLOCATE(rem_dest , struct pingpong_dest , user_param.num_of_qps);
+	memset(rem_dest, 0, sizeof(struct pingpong_dest)*user_param.num_of_qps);
+
+	/* Allocating arrays needed for the test. */
+	alloc_ctx(&ctx,&user_param);
+
+	/* Create (if nessacery) the rdma_cm ids and channel. */
+	if (user_param.work_rdma_cm == ON) {
+
+		if (user_param.machine == CLIENT) {
+			if (retry_rdma_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+
+		} else {
+			if (create_rdma_resources(&ctx,&user_param)) {
+				fprintf(stderr," Unable to create the rdma_resources\n");
+				return FAILURE;
+			}
+			if (rdma_server_connect(&ctx,&user_param)) {
+				fprintf(stderr,"Unable to perform rdma_client function\n");
+				return FAILURE;
+			}
+		}
+
+	} else {
+
+		/* create all the basic IB resources (data buffer, PD, MR, CQ and events channel) */
+		if (ctx_init(&ctx,&user_param)) {
+			fprintf(stderr, " Couldn't create IB resources\n");
+			return FAILURE;
+		}
+	}
+
+	/* Set up the Connection. */
+	if (set_up_connection(&ctx,&user_param,my_dest)) {
+		fprintf(stderr," Unable to set up socket connection\n");
+		return FAILURE;
+	}
+
+	/* Print basic test information. */
+	ctx_print_test_info(&user_param);
+
+	for (i=0; i < user_param.num_of_qps; i++)
+		ctx_print_pingpong_data(&my_dest[i],&user_comm);
+
+	/* shaking hands and gather the other side info. */
+	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	user_comm.rdma_params->side = REMOTE;
+	for (i=0; i < user_param.num_of_qps; i++) {
+
+		/* shaking hands and gather the other side info. */
+		if (ctx_hand_shake(&user_comm,&my_dest[i],&rem_dest[i])) {
+			fprintf(stderr,"Failed to exchange data between server and clients\n");
+			return FAILURE;
+		}
+
+		ctx_print_pingpong_data(&rem_dest[i],&user_comm);
+	};
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_check_gid_compatibility(&my_dest[0], &rem_dest[0])) {
+			fprintf(stderr,"\n Found Incompatibility issue with GID types.\n");
+			fprintf(stderr," Please Try to use a different IP version.\n\n");
+			return FAILURE;
+		}
+	}
+
+	if (user_param.work_rdma_cm == OFF) {
+		if (ctx_connect(&ctx,rem_dest,&user_param,my_dest)) {
+			fprintf(stderr," Unable to Connect the HCA's through the link\n");
+			return FAILURE;
+		}
+	}
+
+	/* An additional handshake is required after moving qp to RTR. */
+	if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
+		fprintf(stderr,"Failed to exchange data between server and clients\n");
+		return FAILURE;
+	}
+
+	ctx_set_send_wqes(&ctx,&user_param,rem_dest);
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+		printf("%s",(user_param.test_type == ITERATIONS) ? RESULT_FMT_LAT : RESULT_FMT_LAT_DUR);
+		printf((user_param.cpu_util_data.enable ? RESULT_EXT_CPU_UTIL : RESULT_EXT));
+	}
+
+	if (user_param.test_method == RUN_ALL) {
+
+		for (i = 1; i < 24 ; ++i) {
+			user_param.size = (uint64_t)1 << i;
+			if(run_iter_lat_write(&ctx,&user_param)) {
+				fprintf(stderr,"Test exited with Error\n");
+				return FAILURE;
+			}
+
+			user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
+		}
+
+	} else {
+
+		if(run_iter_lat_write(&ctx,&user_param)) {
+			fprintf(stderr,"Test exited with Error\n");
+			return FAILURE;
+		}
+		user_param.test_type == ITERATIONS ? print_report_lat(&user_param) : print_report_lat_duration(&user_param);
+	}
+
+	if (user_param.output == FULL_VERBOSITY) {
+		printf(RESULT_LINE);
+	}
+	return 0;
+}