code cleanup and deleted some plot pdfs.

include/hermit/ibv_eth_cm.h

@@ -25,10 +25,19 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+/*
+ * This is a first implementation of a lightweight Ethernet Connection Manager
+ * that helps to exchange destination information before a data exchange via RDMA.
+ * LwIP socket-functions are used.
+ */
+
 #ifndef __IBV_ETH_CM__
 #define __IBV_ETH_CM__
 
-struct pingpong_dest {
+#include <hermit/verbs.h>
+
+
+struct eth_cm_dest {
 	int                lid;
 	int                out_reads;
 	int                qpn;
@@ -63,11 +72,19 @@ int eth_client_connect(const char *server_ip, int port);
 int eth_server_connect(int port);
 
 
-int eth_client_exch_dest(int sockfd, struct pingpong_dest *local_dest,
-                         struct pingpong_dest *rem_dest);
-int eth_server_exch_dest(int sockfd, struct pingpong_dest *local_dest,
-                         struct pingpong_dest *rem_dest);
+int eth_client_exch_dest(int sockfd, struct eth_cm_dest *local_dest,
+                         struct eth_cm_dest *rem_dest);
 
+int eth_server_exch_dest(int sockfd, struct eth_cm_dest *local_dest,
+                         struct eth_cm_dest *rem_dest);
+
+/*
+ * Close the connection to the remote end node. This function calls close.
+ *
+ * sockfd:    Socket file descriptor to close.
+ *
+ * Returns:   0 on success and -1 on failure.
+ */
 int eth_close(int sockfd);
 
 #endif // __IBV_ETH_CM__

include/hermit/verbs.h

@@ -1,9 +1,10 @@
 /*
- * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved.
- * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
- * Copyright (c) 2005 PathScale, Inc. All rights reserved.
- * Copyright (c) 2018 Annika Wierichs, RWTH Aachen. All rights reserved.
+ * Copyright (c) 2004, 2005 Topspin Communications.
+ * Copyright (c) 2004, 2011-2012 Intel Corporation.
+ * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc.
+ * Copyright (c) 2005, PathScale, Inc.
+ * Copyright (c) 2018, Annika Wierichs, RWTH Aachen.
+ * All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses. You may choose to be licensed under the terms of the GNU
@@ -42,7 +43,7 @@
  * original verbs.h:
  * https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/verbs.h
  *
- * API Version: 1.1.6
+ * Current API Version: 1.1.6
  */
 
 #ifndef __HERMIT_VERBS_H__

kernel/ibv.c

@@ -59,18 +59,13 @@
 
 #include <asm/processor.h>
 
-/* inline static unsigned long long rdtsc(void) { */
-	/* unsigned long lo, hi; */
-	/* asm volatile ("rdtsc" : "=a"(lo), "=d"(hi) :: "memory"); */
-
-	/* return ((unsigned long long) hi << 32ULL | (unsigned long long) lo); */
-/* } */
-
 
 /*
  * ibv_post_send
  */
 
+// TODO: Remove CPU cycle measurements (rdtsc()) and logs if not needed anymore.
+
 typedef struct {
 	// Parameters:
 	struct ibv_qp      * qp;
@@ -80,14 +75,14 @@ typedef struct {
 	int ret;
 } __attribute__((packed)) uhyve_ibv_post_send_t;
 
-int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_wr ** bad_wr) { // !!!
-	unsigned long long tick_start = rdtsc();
+int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_wr ** bad_wr) {
+	/* unsigned long long tick_start = rdtsc(); */
 
 	uhyve_ibv_post_send_t uhyve_args;
 	uhyve_args.qp     = qp;
-	unsigned long long tick_gh = rdtsc();
+	/* unsigned long long tick_gh = rdtsc(); */
 	uhyve_args.wr     = (struct ibv_send_wr *)  guest_to_host((size_t) wr);
-	tick_gh = rdtsc() - tick_gh;
+	/* tick_gh = rdtsc() - tick_gh; */
 	uhyve_args.bad_wr = (struct ibv_send_wr **) guest_to_host((size_t) bad_wr);
 
 	struct ibv_send_wr * curr_wr;
@@ -114,7 +109,6 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w
 
 	curr_wr = wr;
 	for (int w = 0; w < num_wrs; w++) {
-		/* LOG_INFO("ibv_post_send for wrs, w = %d\n", w); */
 		is_bind_mw = curr_wr->opcode == IBV_WR_BIND_MW;
 		is_tso     = curr_wr->opcode == IBV_WR_TSO;
 
@@ -132,7 +126,6 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w
 		curr_wr->next = (struct ibv_send_wr *) guest_to_host((size_t) curr_wr->next);
 
 		for (int s = 0; s < curr_wr->num_sge; s++) {
-			/* LOG_INFO("ibv_post_send for sges, s = %d\n", s); */
 			wr__sg_list__addr[w][s] = curr_wr->sg_list[s].addr;
 			curr_wr->sg_list[s].addr = (uint64_t) guest_to_host((size_t) curr_wr->sg_list[s].addr);
 		}
@@ -143,11 +136,9 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w
 		curr_wr = wr__next[w];
 	}
 
-	unsigned long long tick_pre_work = rdtsc() - tick_start;
-
+	/* unsigned long long tick_pre_work = rdtsc() - tick_start; */
 	uhyve_send(UHYVE_PORT_IBV_POST_SEND, (unsigned) virt_to_phys((size_t) &uhyve_args));
-
-	unsigned long long tick_call = rdtsc() - tick_start - tick_pre_work;
+	/* unsigned long long tick_call = rdtsc() - tick_start - tick_pre_work; */
 
 	if (*bad_wr && *bad_wr == uhyve_args.wr) {
 		*bad_wr = wr;
@@ -180,12 +171,13 @@ int ibv_post_send(struct ibv_qp * qp, struct ibv_send_wr * wr, struct ibv_send_w
 		curr_wr = curr_wr->next;
 	}
 
-	unsigned long long tick_post_work = rdtsc() - tick_start - tick_pre_work - tick_call;
+	/* unsigned long long tick_post_work = rdtsc() - tick_start - tick_pre_work - tick_call; */
 
-	LOG_INFO("tick_gh:        %llu\n", tick_gh);
-	LOG_INFO("tick_pre_work:  %llu\n", tick_pre_work);
-	LOG_INFO("tick_call:      %llu\n", tick_call);
-	LOG_INFO("tick_post_work: %llu\n", tick_post_work);
+	// TODO: Remove these prints.
+	/* LOG_INFO("tick_gh:        %llu\n", tick_gh); */
+	/* LOG_INFO("tick_pre_work:  %llu\n", tick_pre_work); */
+	/* LOG_INFO("tick_call:      %llu\n", tick_call); */
+	/* LOG_INFO("tick_post_work: %llu\n", tick_post_work); */
 	return uhyve_args.ret;
 }
 
@@ -203,7 +195,7 @@ typedef struct {
 	int ret;
 } __attribute__((packed)) uhyve_ibv_post_wq_recv_t;
 
-int ibv_post_wq_recv(struct ibv_wq * wq, struct ibv_recv_wr * recv_wr, struct ibv_recv_wr ** bad_recv_wr) { // !!!
+int ibv_post_wq_recv(struct ibv_wq * wq, struct ibv_recv_wr * recv_wr, struct ibv_recv_wr ** bad_recv_wr) {
 	uhyve_ibv_post_wq_recv_t uhyve_args;
 	uhyve_args.wq          = wq;
 	uhyve_args.recv_wr     = (struct ibv_recv_wr *)  guest_to_host((size_t) recv_wr);
@@ -284,7 +276,7 @@ typedef struct {
 	int ret;
 } __attribute__((packed)) uhyve_ibv_post_srq_recv_t;
 
-int ibv_post_srq_recv(struct ibv_srq * srq, struct ibv_recv_wr * recv_wr, struct ibv_recv_wr ** bad_recv_wr) { // !!!
+int ibv_post_srq_recv(struct ibv_srq * srq, struct ibv_recv_wr * recv_wr, struct ibv_recv_wr ** bad_recv_wr) {
 	uhyve_ibv_post_srq_recv_t uhyve_args;
 	uhyve_args.srq         = srq;
 	uhyve_args.recv_wr     = (struct ibv_recv_wr *)  guest_to_host((size_t) recv_wr);
@@ -445,7 +437,7 @@ typedef struct {
 	struct ibv_rwq_ind_table * ret;
 } __attribute__((packed)) uhyve_ibv_create_rwq_ind_table_t;
 
-struct ibv_rwq_ind_table * ibv_create_rwq_ind_table(struct ibv_context * context, struct ibv_rwq_ind_table_init_attr * init_attr) { // !!!
+struct ibv_rwq_ind_table * ibv_create_rwq_ind_table(struct ibv_context * context, struct ibv_rwq_ind_table_init_attr * init_attr) {
 	uhyve_ibv_create_rwq_ind_table_t uhyve_args;
 	uhyve_args.context = context;
 	uhyve_args.init_attr = (struct ibv_rwq_ind_table_init_attr *) guest_to_host((size_t) init_attr);
@@ -476,8 +468,9 @@ typedef struct {
 } __attribute__((packed)) uhyve_ibv_open_xrcd_t;
 
 struct ibv_xrcd * ibv_open_xrcd(struct ibv_context * context, struct ibv_xrcd_init_attr * xrcd_init_attr) {
-	// TODO: This will probably not work as xrcd_init_attr->fd is a file descriptor opened in HermitCore.
-	// Possibly comment this function out?
+	// If this function ever does weird things: xrcd_init_attr->fd is a file descriptor opened in
+	// HermitCore and therefore also opened in host space via uhyve_send(UHYVE_PORT_OPEN). This might
+	// become a problem if HermitCore's open() syscall is modified.
 	uhyve_ibv_open_xrcd_t uhyve_args;
 	uhyve_args.context = context;
 	uhyve_args.xrcd_init_attr = (struct ibv_xrcd_init_attr *) guest_to_host((size_t) xrcd_init_attr);
@@ -1449,7 +1442,7 @@ typedef struct {
 	struct ibv_mr * ret;
 } __attribute__((packed)) uhyve_ibv_reg_mr_t;
 
-struct ibv_mr * ibv_reg_mr(struct ibv_pd * pd, void * addr, int length, int access) { // !!!
+struct ibv_mr * ibv_reg_mr(struct ibv_pd * pd, void * addr, int length, int access) {
 	uhyve_ibv_reg_mr_t uhyve_args;
 	uhyve_args.pd = pd;
 	uhyve_args.addr = (void *) guest_to_host((size_t) addr);
@@ -1479,7 +1472,7 @@ typedef struct {
 } __attribute__((packed)) uhyve_ibv_rereg_mr_t;
 
 int ibv_rereg_mr(struct ibv_mr * mr, int flags, struct ibv_pd * pd, void * addr,
-	               int length, int access) { // !!!
+	               int length, int access) {
 	uhyve_ibv_rereg_mr_t uhyve_args;
 	uhyve_args.mr     = mr;
 	uhyve_args.flags  = flags;
@@ -1599,7 +1592,7 @@ int ibv_bind_mw(struct ibv_qp * qp, struct ibv_mw * mw, struct ibv_mw_bind * mw_
 	uhyve_args.mw = mw;
 	uhyve_args.mw_bind = (struct ibv_mw_bind *) guest_to_host((size_t) mw_bind);
 
-	uint64_t mw_bind__bind_info__addr = mw_bind->bind_info.addr; // !
+	uint64_t mw_bind__bind_info__addr = mw_bind->bind_info.addr;
 	mw_bind->bind_info.addr = (uint64_t) guest_to_host((size_t) mw_bind->bind_info.addr);
 
 	uhyve_send(UHYVE_PORT_IBV_BIND_MW, (unsigned) virt_to_phys((size_t) &uhyve_args));
@@ -2013,7 +2006,7 @@ typedef struct {
 	struct ibv_qp * ret;
 } __attribute__((packed)) uhyve_ibv_create_qp_ex_t;
 
-struct ibv_qp * ibv_create_qp_ex(struct ibv_context * context, struct ibv_qp_init_attr_ex * qp_init_attr_ex) { // !!!
+struct ibv_qp * ibv_create_qp_ex(struct ibv_context * context, struct ibv_qp_init_attr_ex * qp_init_attr_ex) {
 	uhyve_ibv_create_qp_ex_t uhyve_args;
 	uhyve_args.context = context;
 	uhyve_args.qp_init_attr_ex = (struct ibv_qp_init_attr_ex *) guest_to_host((size_t) qp_init_attr_ex);
@@ -2444,7 +2437,7 @@ int ibv_fork_init() {
 /* } __attribute__((packed)) uhyve_ibv_resolve_eth_l2_from_gid_t; */
 
 /* int ibv_resolve_eth_l2_from_gid(struct ibv_context * context, struct ibv_ah_attr * attr, */
-																/* uint8_t eth_mac[ETHERNET_LL_SIZE], uint16_t * vid) {	// !!! */
+																/* uint8_t eth_mac[ETHERNET_LL_SIZE], uint16_t * vid) { */
 	/* uhyve_ibv_resolve_eth_l2_from_gid_t uhyve_args; */
 	/* uhyve_args.context = context; */
 	/* uhyve_args.attr = (struct ibv_ah_attr *) guest_to_host((size_t) attr); */

kernel/ibv_eth_cm.c

@@ -25,13 +25,19 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <sys/socket.h>
-#include <netinet/in.h>
+#include <hermit/stdio.h>
+#include <hermit/stddef.h>
+#include <hermit/memory.h>
+#include <hermit/logging.h>
 
-#include <hermit/ibv_hermit_cm.h>
+#include <lwip/sockets.h>
+#include <lwip/netdb.h>
+
+#include <hermit/ibv_eth_cm.h>
 
 
 #define KEY_MSG_SIZE (59)
+#define KEY_PRINT_FMT "%04x:%04x:%06x:%06x:%08x:%016Lx:%08x"
 
 /*
  * Helper functions:
@@ -41,51 +47,53 @@ int check_add_port(char **service, int port, const char *server_ip, struct addri
                    struct addrinfo **res)
 {
   int str_size_max = 6;
-  *service = calloc(str_size_max, sizeof(char));
-	if (snprintf(*service, str_size_max, "%05d", port) < 0) {
+  *service = kmalloc(str_size_max);
+	memset(*service, 0, sizeof str_size_max);
+
+	if (ksnprintf(*service, str_size_max, "%05d", port) < 0) {
 		return 1;
 	}
 
-	if (getaddrinfo(server_ip, *service, hints, res) < 0) {
-    fprintf(stderr, "Error for %s:%d\n", server_ip, port);
+	if (lwip_getaddrinfo(server_ip, *service, hints, res) < 0) {
+    LOG_ERROR("Error for %s:%d\n", server_ip, port);
 		return 1;
 	}
 
 	return 0;
 }
 
-int eth_read(int sockfd, struct pingpong_dest *rem_dest)
+int eth_read(int sockfd, struct eth_cm_dest *rem_dest)
 {
   int parsed;
   char msg[KEY_MSG_SIZE];
 
-  if (read(sockfd, msg, sizeof msg) != sizeof msg) {
-    fprintf(stderr, "eth_read: Couldn't read remote address.\n");
+  if (lwip_read(sockfd, msg, sizeof msg) != sizeof msg) {
+    LOG_ERROR("eth_read: Couldn't read remote address.\n");
     return 1;
   }
 
+	// TODO: Replace sscanf (not available in HC) or implement the function.
   parsed = sscanf(msg, KEY_PRINT_FMT,
                   (unsigned int *) &rem_dest->lid, &rem_dest->out_reads, &rem_dest->qpn,
                   &rem_dest->psn, &rem_dest->rkey, &rem_dest->vaddr, &rem_dest->srqn);
 
   if (parsed != 7) {
-    fprintf(stderr, "Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);
+    LOG_ERROR("Couldn't parse line <%.*s>\n",(int)sizeof msg, msg);
     return 1;
   }
 
   return 0;
 }
 
-int eth_write(int sockfd, struct pingpong_dest *local_dest)
+int eth_write(int sockfd, struct eth_cm_dest *local_dest)
 {
   char msg[KEY_MSG_SIZE];
-  sprintf(msg, KEY_PRINT_FMT,
+  ksprintf(msg, KEY_PRINT_FMT,
           local_dest->lid, local_dest->out_reads, local_dest->qpn, local_dest->psn,
           local_dest->rkey, local_dest->vaddr, local_dest->srqn);
 
-  if (write(sockfd, msg, sizeof msg) != sizeof msg) {
-    perror("Client Write");
-    fprintf(stderr, "Couldn't send local address.\n");
+  if (lwip_write(sockfd, msg, sizeof msg) != sizeof msg) {
+    LOG_ERROR("Couldn't send local address.\n");
     return 1;
   }
 
@@ -109,26 +117,26 @@ int eth_client_connect(const char *server_ip, int port)
 	hints.ai_socktype = SOCK_STREAM;
 
 	if (check_add_port(&service, port, server_ip, &hints, &res)) {
-		fprintf(stderr, "Problem in resolving basic address and port\n");
+		LOG_ERROR("Problem in resolving basic address and port\n");
 		return -1;
 	}
 
 	for (t = res; t; t = t->ai_next) {
-		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
+		sockfd = lwip_socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 
 		if (sockfd >= 0) {
-			if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
+			if (!lwip_connect(sockfd, t->ai_addr, t->ai_addrlen))
 				break; // Success.
 
-			close(sockfd);
+			lwip_close(sockfd);
 			sockfd = -1;
 		}
 	}
 
-	freeaddrinfo(res);
+	lwip_freeaddrinfo(res);
 
 	if (sockfd < 0) {
-		fprintf(stderr, "Couldn't connect to %s:%d\n", server_ip, port);
+		LOG_ERROR("Couldn't connect to %s:%d\n", server_ip, port);
 		return -1;
 	}
 
@@ -149,71 +157,69 @@ int eth_server_connect(int port)
   hints.ai_socktype = SOCK_STREAM;
 
   if (check_add_port(&service, port, NULL, &hints, &res)) {
-    fprintf(stderr, "Problem in resolving basic address and port\n");
+    LOG_ERROR("Problem in resolving basic address and port\n");
     return -1;
   }
 
   for (t = res; t; t = t->ai_next) {
-    sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
+    sockfd = lwip_socket(t->ai_family, t->ai_socktype, t->ai_protocol);
 
     if (sockfd >= 0) {
       n = 1;
-      setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
+      lwip_setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
 
-      if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
+      if (!lwip_bind(sockfd, t->ai_addr, t->ai_addrlen))
         break; // Success
 
-      close(sockfd);
+      lwip_close(sockfd);
       sockfd = -1;
     }
   }
-  freeaddrinfo(res);
+  lwip_freeaddrinfo(res);
 
   if (sockfd < 0) {
-    fprintf(stderr, "Couldn't listen to port %d\n", port);
+    LOG_ERROR("Couldn't listen to port %d\n", port);
     return -1;
   }
 
-  listen(sockfd, 1);
-  connfd = accept(sockfd, NULL, 0);
+  lwip_listen(sockfd, 1);
+  connfd = lwip_accept(sockfd, NULL, 0);
 
   if (connfd < 0) {
-    perror("Server Accept");
-    fprintf(stderr, "accept() failed\n");
-    close(sockfd);
+    LOG_ERROR("accept() failed\n");
+    lwip_close(sockfd);
     return -1;
   }
 
-  close(sockfd);
   return connfd;
 }
 
-int eth_client_exch_dest(int sockfd, struct pingpong_dest *local_dest,
-                         struct pingpong_dest *rem_dest)
+int eth_client_exch_dest(int sockfd, struct eth_cm_dest *local_dest,
+                         struct eth_cm_dest *rem_dest)
 {
   if (eth_write(sockfd, local_dest)) {
-    fprintf(stderr, " Unable to write local destination information to socket.\n");
+    LOG_ERROR("Unable to write local destination information to socket.\n");
     return 1;
   }
 
   if (eth_read(sockfd, rem_dest)) {
-    fprintf(stderr, " Unable to read remote destination information from socket.\n");
+    LOG_ERROR("Unable to read remote destination information from socket.\n");
     return 1;
   }
 
   return 0;
 }
 
-int eth_server_exch_dest(int sockfd, struct pingpong_dest *local_dest,
-                         struct pingpong_dest *rem_dest)
+int eth_server_exch_dest(int sockfd, struct eth_cm_dest *local_dest,
+                         struct eth_cm_dest *rem_dest)
 {
   if (eth_read(sockfd, rem_dest)) {
-    fprintf(stderr, " Unable to read remote destination information from socket.\n");
+    LOG_ERROR("Unable to read remote destination information from socket.\n");
     return 1;
   }
 
   if (eth_write(sockfd, local_dest)) {
-    fprintf(stderr, " Unable to write local destination information to socket.\n");
+    LOG_ERROR("Unable to write local destination information to socket.\n");
     return 1;
   }
 
@@ -222,8 +228,8 @@ int eth_server_exch_dest(int sockfd, struct pingpong_dest *local_dest,
 
 int eth_close(int sockfd)
 {
-  if (close(sockfd)) {
-    fprintf(stderr, "Couldn't close socket.\n");
+  if (lwip_close(sockfd)) {
+    LOG_ERROR("Couldn't close socket.\n");
     return -1;
   }
 

tools/ibv_code_generator/custom_snippets.py

@@ -49,7 +49,10 @@ CONVERT = 2
 REVERT = 1
 
 #
-#  ibv_post_recv, ibv_post_srq_recv, ibv_post_wq_recv
+# ibv_post_recv, ibv_post_srq_recv, ibv_post_wq_recv
+#
+# (These functions may use the same backup/convert/revert routines in HermitCore since their
+#  prototypes are equal.)
 #
 
 post_recv_convert = \
@@ -114,7 +117,7 @@ post_recv_revert = \
 """
 
 #
-#  ibv_post_send
+# ibv_post_send
 #
 
 post_send_convert = \
@@ -206,7 +209,7 @@ post_send_revert = \
 """
 
 #
-#  ibv_create_rwq_ind_table
+# ibv_create_rwq_ind_table
 #
 
 create_rwq_ind_table_convert = \

tools/ibv_code_generator/generate-code.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 """Copyright (c) 2018, Annika Wierichs, RWTH Aachen University
-
 All rights reserved.
+
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
@@ -23,14 +23,14 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-TODO, docs
 """
 
 import custom_snippets
 
 # TODO: ibv_resolve_eth_l2_from_gid function does not work.
-# TODO: ibv_open_xrcd function might not work. Confirm.
+# TODO: A few trivial functions (those ending in '_str') are currently implemented in HermitCore and
+#       do not forward the function call via uhyve_send. The generator does not yet take this into
+#       account. Refer to kernel/ibv.c -> ibv_wc_status_str() as an example.
 
 
 # Path of the input file containing function prototypes.
@@ -127,9 +127,6 @@ app_owned_resources = restricted_resources + \
                        "struct ibv_async_event"]
 
 
-# int ibv_resolve_eth_l2_from_gid(uint8_t [6] eth_mac,uint16_t * vid)
-# char[NUM]
-
 # ----------------------------------------------------------------------------
 # CLASSES
 # ----------------------------------------------------------------------------
@@ -362,7 +359,7 @@ def generate_uhyve_function(fnc):
   """
   code  = generate_pretty_comment(fnc.name)
   code += "void call_{0}(struct kvm_run * run, uint8_t * guest_mem) {{\n".format(fnc.name)
-  code += "\tprintf(\"LOG: UHYVE - call_{0}\\n\");\n".format(fnc.name)
+  code += "\tprintf(\"LOG: UHYVE - call_{0}\\n\");\n".format(fnc.name)  # TODO: Delete later.
   code += "\tunsigned data = *((unsigned*) ((size_t) run + run->io.data_offset));\n"
   code += "\t{0} * args = ({0} *) (guest_mem + data);\n\n".format(fnc.args_struct_name)
   code += "\tuse_ib_mem_pool = true;\n"

tools/uhyve.c

@@ -30,6 +30,7 @@
  * 25.2.2017: add SMP support to enable more than one core
  * 24.4.2017: add checkpoint/restore support,
  *            remove memory limit
+ * 26.2.2018: add support for RDMA verbs API
  */
 
  #define _GNU_SOURCE

usr/benchmarks/CMakeLists.txt

@@ -8,9 +8,7 @@ target_link_libraries(basic pthread)
 
 add_executable(hg hg.c hist.c rdtsc.c run.c init.c opt.c report.c setup.c)
 
-# add_executable(ib-pingpong-ud ib/pingpong.c ib/pingpong-ud.c)
-# add_executable(ib_write_bw ib/write_bw.c ib/get_clock.c ib/perftest_parameters.c
-	 # ib/perftest_resources.c ib/perftest_communication.c)
+# RDMA verbs interface
 add_executable(ib_write_bw ib/write_bw.c ib/get_clock.c ib/perftest_parameters.c
 	ib/perftest_resources.c ib/perftest_communication.c)
 add_executable(ib_write_lat ib/write_lat.c ib/get_clock.c ib/perftest_parameters.c

usr/benchmarks/ib/evaluation/config.py

@@ -1,10 +1,10 @@
 import os
 
-BENCHMARKS = ['ib_write_bw',
-              'ib_write_lat',
-              'ib_read_bw',
-              'ib_read_lat']
-OPTIONS   = ['-a --post_list 1']
+#  BENCHMARKS = ['ib_write_bw',
+              #  'ib_write_lat',
+              #  'ib_read_bw',
+              #  'ib_read_lat']
+#  OPTIONS   = ['-a --post_list 1']
 
 #  BENCHMARKS = ['ib_write_lat',
               #  'ib_read_bw',
@@ -13,10 +13,10 @@ OPTIONS   = ['-a --post_list 1']
 #  OPTIONS   = ['-a --post_list 4',
              #  '-a --post_list 16']
 
-#  BENCHMARKS = ['ib_write_bw']
-#  OPTIONS   = ['-a --post_list 1',
-             #  '-a --post_list 4',
-             #  '-a --post_list 16']
+BENCHMARKS = ['ib_write_bw']
+OPTIONS   = ['-a --post_list 1',
+             '-a --post_list 4',
+             '-a --post_list 16']
 
 #  TIMESTAMP = '18-02-15-14-30' # TODO: temporary while testing plots
 TIMESTAMP = 'final2' # TODO: temporary while testing plots

usr/benchmarks/ib/evaluation/parse_results.py

@@ -128,12 +128,14 @@ def plot_results(results, directory):
       ax.set_xscale('log', basex=2)
       num_data_points = len(results['hermit'][bm][num_wrs_list[0]]['num_bytes'])
       ax.set_xticks(np.power(2, range(1, num_data_points + 1)))
+      ax.set_yscale('log', basey=2)
       if 'lat' in bm:
         ax.set_ylim(bottom = 0.5, top = 256)
+      elif 'bw' in bm:
+        ax.set_ylim(bottom = 0.25, top = 8192)
+      ax.get_yaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())
         #  y_ticks = np.power(2, range(-1, 8))
         #  ax.set_yticks(y_ticks, [(str(int(tick)) if tick >= 1 else str(tick)) for tick in y_ticks])
-        ax.set_yscale('log', basey=2)
-        ax.get_yaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())
 
       # Plot, save and then clear figure
       for num_wrs in num_wrs_list:
@@ -155,3 +157,48 @@ def plot_results(results, directory):
       plt.savefig(file_path, bbox_inches = 'tight')
       plt.gcf().clear()
 
+
+  #  file_name = 'plot_' + bm + '-' + metric + 'DIFF'
+  #  file_path = os.path.join(directory, file_name)
+
+  #  fig, ax = plt.subplots(figsize=(10, 5))
+
+  #  ax.grid()
+
+  #  # Axis labels
+  #  ax.set_xlabel('Message Size [B]')
+  #  ax.set_ylabel(labels[metric])
+
+  #  # Axis ticks and scale (x: log2 / y: log2 for latency)
+  #  ax.set_xscale('log', basex=2)
+  #  num_data_points = len(results['hermit'][bm][num_wrs_list[0]]['num_bytes'])
+  #  ax.set_xticks(np.power(2, range(1, num_data_points + 1)))
+  #  ax.set_yscale('log', basey=2)
+  #  if 'lat' in bm:
+    #  ax.set_ylim(bottom = 0.5, top = 256)
+  #  elif 'bw' in bm:
+    #  ax.set_ylim(bottom = 0.25, top = 8192)
+  #  ax.get_yaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())
+    #  #  y_ticks = np.power(2, range(-1, 8))
+    #  #  ax.set_yticks(y_ticks, [(str(int(tick)) if tick >= 1 else str(tick)) for tick in y_ticks])
+
+  #  # Plot, save and then clear figure
+  #  for num_wrs in num_wrs_list:
+    #  label_end = ' (' + num_wrs + ' WR' + \
+                #  (')' if num_wrs is '1' else 's)') if len(num_wrs_list) > 1 else ""
+    #  ax.plot(results['native'][bm][num_wrs]['num_bytes'][:20],
+            #  results['native'][bm][num_wrs][metric][:20],
+            #  'o-', color = line_color['native'][num_wrs], label = 'Native' + label_end,
+            #  linewidth = 2, markersize = 6)
+    #  label = 'HermitCore (' + num_wrs + ' work requests)'
+    #  ax.plot(results['hermit'][bm][num_wrs]['num_bytes'],
+            #  results['hermit'][bm][num_wrs][metric],
+            #  'o-', color = line_color['hermit'][num_wrs], label = 'HermitCore' + label_end,
+            #  linewidth = 2, markersize = 6)
+
+  #  plt.legend(fontsize = 12, ncol = 3, loc = 'lower right',
+             #  bbox_to_anchor = [1, 1.05])
+
+  #  plt.savefig(file_path, bbox_inches = 'tight')
+  #  plt.gcf().clear()
+

usr/benchmarks/ib/perftest_communication.h

@@ -1,5 +1,7 @@
 /*
- * Copyright (c) 2011 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2011 Mellanox Technologies Ltd.
+ * Copyright (c) 2018 Annika Wierichs, RWTH Aachen.
+ * All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU

usr/benchmarks/ib/perftest_parameters.h

@@ -1,5 +1,7 @@
 /*
- * Copyright (c) 2009 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 Mellanox Technologies Ltd.
+ * Copyright (c) 2018 Annika Wierichs, RWTH Aachen.
+ * All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -71,7 +73,6 @@
 // #endif
 
 /* ------------------------------------------------------------------- */
-/* Config TODO */
 
 // #define HAVE_VERBS_EXP
 

usr/benchmarks/ib/perftest_resources.c

@@ -9,7 +9,6 @@
 #include <signal.h>
 #include <string.h>
 #include <ctype.h>
-/* #include <sys/mman.h> */
 #include <sys/ipc.h>
 #include <sys/shm.h>
 #include <pthread.h>
@@ -1527,9 +1526,10 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p
 						is_sending_burst = 0;
 					}
 				}
-			}
-		}
+			} // Post WRs for loop
+		} // Queue Pair for loop
 
+		// Check for Work Completions
 		if (totccnt < tot_iters) {
 			if (user_param->use_event) {
 				if (ctx_notify_events(ctx->channel)) {
@@ -1575,8 +1575,8 @@ int run_iter_bw(struct pingpong_context *ctx, struct perftest_parameters *user_p
 				return_value = FAILURE;
 				goto cleaning;
 			}
-		}
-	}
+		} // Check for Work Completions if bloc
+	} // Iterations while loop
 
 	if (user_param->noPeak == ON && user_param->test_type == ITERATIONS)
 		user_param->tcompleted[0] = get_cycles();

usr/benchmarks/ib/perftest_resources.h

@@ -1,6 +1,8 @@
 /*
- * Copyright (c) 2009 Mellanox Technologies Ltd. All rights reserved.
- *
+ * Copyright (c) 2009 Mellanox Technologies Ltd.
+ * Copyright (c) 2018 Annika Wierichs, RWTH Aachen.
+ * All rights reserved.
+
  * This software is available to you under a choice of one of two
  * licenses. You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file

usr/benchmarks/ib/read_bw.c

@@ -1,7 +1,9 @@
 /*
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2006 Mellanox Technologies Ltd. All rights reserved.
- * Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2005 Topspin Communications.
+ * Copyright (c) 2006 Mellanox Technologies Ltd.
+ * Copyright (c) 2009 HNR Consulting.
+ * Copyright (c) 2018 Annika Wierichs, RWTH Aachen.
+ * All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses. You may choose to be licensed under the terms of the GNU

usr/benchmarks/ib/read_lat.c

@@ -1,8 +1,10 @@
 /*
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2005 Topspin Communications.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.
  * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler)
- * Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.
+ * Copyright (c) 2018 Annika Wierichs, RWTH Aachen.
+ * All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses. You may choose to be licensed under the terms of the GNU

usr/benchmarks/ib/write_bw.c

@@ -1,7 +1,9 @@
 /*
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
- * Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2005 Topspin Communications.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.
+ * Copyright (c) 2009 HNR Consulting.
+ * Copyright (c) 2018 Annika Wierichs, RWTH Aachen.
+ * All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses. You may choose to be licensed under the terms of the GNU

usr/benchmarks/ib/write_lat.c

@@ -1,8 +1,10 @@
 /*
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2005 Topspin Communications.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.
  * Copyright (c) 2005 Hewlett Packard, Inc (Grant Grundler)
- * Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2009 HNR Consulting.
+ * Copyright (c) 2018 Annika Wierichs, RWTH Aachen.
+ * All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses. You may choose to be licensed under the terms of the GNU

usr/tests/ib/rc_pingpong.c

@@ -39,12 +39,10 @@
 #include <unistd.h>
 #include <string.h>
 #include <sys/types.h>
-/* #include <sys/socket.h> */
 #include <sys/time.h>
 #include <netdb.h>
 #include <malloc.h>
 #include <getopt.h>
-/* #include <netinet/in.h> */
 #include <time.h>
 #include <inttypes.h>
 
@@ -86,6 +84,13 @@ static struct ibv_cq *pp_cq(struct pingpong_context *ctx)
 	return use_ts ? ibv_cq_ex_to_cq(ctx->cq_s.cq_ex) : ctx->cq_s.cq;
 }
 
+struct pingpong_dest {
+	int lid;
+	int qpn;
+	int psn;
+	union ibv_gid gid;
+};
+
 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, enum ibv_mtu mtu,
     int sl, struct pingpong_dest *dest, int sgid_idx)
 {
@@ -143,6 +148,175 @@ static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, en
 	return 0;
 }
 
+static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
+    const struct pingpong_dest *my_dest)
+{
+	struct addrinfo *res, *t;
+	struct addrinfo hints = {
+		.ai_family   = AF_UNSPEC,
+		.ai_socktype = SOCK_STREAM
+	};
+	char *service;
+	char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
+	int n;
+	int sockfd = -1;
+	struct pingpong_dest *rem_dest = NULL;
+	char gid[33];
+
+	if (asprintf(&service, "%d", port) < 0)
+		return NULL;
+
+	n = getaddrinfo(servername, service, &hints, &res);
+
+	if (n < 0) {
+		fprintf(stderr, "Error for %s:%d\n", servername, port);
+		free(service);
+		return NULL;
+	}
+
+	for (t = res; t; t = t->ai_next) {
+		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
+		if (sockfd >= 0) {
+			if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
+				break;
+			close(sockfd);
+			sockfd = -1;
+		}
+	}
+
+	freeaddrinfo(res);
+	free(service);
+
+	if (sockfd < 0) {
+		fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
+		return NULL;
+	}
+
+	gid_to_wire_gid(&my_dest->gid, gid);
+	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
+							my_dest->psn, gid);
+	if (write(sockfd, msg, sizeof msg) != sizeof msg) {
+		fprintf(stderr, "Couldn't send local address\n");
+		goto out;
+	}
+
+	if (read(sockfd, msg, sizeof msg) != sizeof msg ||
+	    write(sockfd, "done", sizeof "done") != sizeof "done") {
+		perror("client read/write");
+		fprintf(stderr, "Couldn't read/write remote address\n");
+		goto out;
+	}
+
+	rem_dest = malloc(sizeof *rem_dest);
+	if (!rem_dest)
+		goto out;
+
+	sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid);
+	wire_gid_to_gid(gid, &rem_dest->gid);
+
+out:
+	close(sockfd);
+	return rem_dest;
+}
+
+static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, int ib_port,
+    enum ibv_mtu mtu, int port, int sl, const struct pingpong_dest *my_dest, int sgid_idx)
+{
+	struct addrinfo *res, *t;
+	struct addrinfo hints = {
+		.ai_flags    = AI_PASSIVE,
+		.ai_family   = AF_UNSPEC,
+		.ai_socktype = SOCK_STREAM
+	};
+	char *service;
+	char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
+	int n;
+	int sockfd = -1, connfd;
+	struct pingpong_dest *rem_dest = NULL;
+	char gid[33];
+
+	if (asprintf(&service, "%d", port) < 0)
+		return NULL;
+
+	n = getaddrinfo(NULL, service, &hints, &res);
+
+	if (n < 0) {
+		fprintf(stderr, "Error for port %d\n", port);
+		free(service);
+		return NULL;
+	}
+
+	for (t = res; t; t = t->ai_next) {
+		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
+		if (sockfd >= 0) {
+			n = 1;
+
+			setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
+
+			if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
+				break;
+			close(sockfd);
+			sockfd = -1;
+		}
+	}
+
+	freeaddrinfo(res);
+	free(service);
+
+	if (sockfd < 0) {
+		fprintf(stderr, "Couldn't listen to port %d\n", port);
+		return NULL;
+	}
+
+	listen(sockfd, 1);
+	connfd = accept(sockfd, NULL, NULL);
+	close(sockfd);
+	if (connfd < 0) {
+		fprintf(stderr, "accept() failed\n");
+		return NULL;
+	}
+
+	n = read(connfd, msg, sizeof msg);
+	if (n != sizeof msg) {
+		perror("server read");
+		fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
+		goto out;
+	}
+
+	rem_dest = malloc(sizeof *rem_dest);
+	if (!rem_dest)
+		goto out;
+
+	sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
+							&rem_dest->psn, gid);
+	wire_gid_to_gid(gid, &rem_dest->gid);
+
+	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest,
+								sgid_idx)) {
+		fprintf(stderr, "Couldn't connect to remote QP\n");
+		free(rem_dest);
+		rem_dest = NULL;
+		goto out;
+	}
+
+
+	gid_to_wire_gid(&my_dest->gid, gid);
+	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
+							my_dest->psn, gid);
+	if (write(connfd, msg, sizeof msg) != sizeof msg ||
+	    read(connfd, msg, sizeof msg) != sizeof "done") {
+		fprintf(stderr, "Couldn't send/recv local address\n");
+		free(rem_dest);
+		rem_dest = NULL;
+		goto out;
+	}
+
+
+out:
+	close(connfd);
+	return rem_dest;
+}
+
 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, int rx_depth,
     int port, int use_event)
 {
@@ -712,9 +886,9 @@ int main(int argc, char *argv[])
 
 
 	if (servername)
-		rem_dest = client_exch_dest(servername, port, &my_dest);
+		rem_dest = pp_client_exch_dest(servername, port, &my_dest);
 	else
-		rem_dest = server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx);
+		rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx);
 
 	if (!rem_dest)
 		return 1;