diff --git a/etc/infiniband.conf b/etc/infiniband.conf index 8cfb2fb58..81107bdb3 100644 --- a/etc/infiniband.conf +++ b/etc/infiniband.conf @@ -4,10 +4,25 @@ logging = { } nodes = { - ib_node0 = { + results = { + type = "file", + + uri = "logs/ib_results-%Y%m%d_%H-%M-%S.log", + }, + + siggen = { + type = "signal", + + signal = "mixed", + values = 3, + frequency = 3, + rate = 100000, + }, + + ib_node_source = { type = "infiniband", - rdma_port_space = "RDMA_PS_TCP", + rdma_port_space = "RDMA_PS_UDP", in = { address = "10.0.0.2:1337", @@ -19,8 +34,11 @@ nodes = { poll_mode = "BUSY", buffer_subtraction = 128, - }, + hooks = ( + { type = "stats", verbose = true } + ) + }, out = { address = "10.0.0.1:1337", resolution_timeout = 1000, @@ -35,4 +53,27 @@ nodes = { } } + + ib_node_target = { + type = "infiniband", + + rdma_port_space = "RDMA_PS_UDP", + + in = { + address = "10.0.0.1:1337", + + max_wrs = 8192, + cq_size = 8192, + + vectorize = 1, + + poll_mode = "BUSY", + buffer_subtraction = 128, + + hooks = ( + { type = "stats", verbose = true } + ) + }, + } } + diff --git a/include/villas/nodes/infiniband.h b/include/villas/nodes/infiniband.h index 03a2ae7a3..476481296 100644 --- a/include/villas/nodes/infiniband.h +++ b/include/villas/nodes/infiniband.h @@ -90,9 +90,12 @@ struct infiniband { int buffer_subtraction; /* Unrealiable connectionless data */ - struct rdma_ud_param ud; - void *grh_ptr; - struct ibv_mr *grh_mr; + struct ud_s { + struct rdma_ud_param ud; + struct ibv_ah *ah; + void *grh_ptr; + struct ibv_mr *grh_mr; + } ud; } conn; diff --git a/lib/nodes/file.c b/lib/nodes/file.c index db891e201..96bf81fcf 100644 --- a/lib/nodes/file.c +++ b/lib/nodes/file.c @@ -90,7 +90,7 @@ int file_parse(struct node *n, json_t *cfg) f->epoch_mode = FILE_EPOCH_DIRECT; f->flush = 0; - ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: s, s: { s?: s, s?: F, s?: s, s?: F }, s: { s?: b } }", + ret = json_unpack_ex(cfg, &err, 0, "{ s: s, s?: s, s?: { s?: s, s?: F, s?: s, s?: F }, s?: { s?: b } }", "uri", &uri_tmpl, "format", &format, "in", @@ -263,20 +263,12 @@ int file_stop(struct node *n) if (ret) return ret; - free(f->uri); - - return 0; -} - -int file_destroy(struct node *n) -{ - int ret; - struct file *f = (struct file *) n->_vd; - ret = io_destroy(&f->io); if (ret) return ret; + free(f->uri); + return 0; } @@ -383,7 +375,6 @@ static struct plugin p = { .print = file_print, .start = file_start, .stop = file_stop, - .destroy = file_destroy, .read = file_read, .write = file_write, .fd = file_fd diff --git a/lib/nodes/infiniband.c b/lib/nodes/infiniband.c index dbba42abe..a24974dcb 100644 --- a/lib/nodes/infiniband.c +++ b/lib/nodes/infiniband.c @@ -547,8 +547,10 @@ void * ib_rdma_cm_event_thread(void *n) case RDMA_CM_EVENT_ESTABLISHED: // If the connection is unreliable connectionless, set appropriate variables - if (ib->conn.port_space == RDMA_PS_UDP) - ib->conn.ud = event->param.ud; + if (ib->conn.port_space == RDMA_PS_UDP) { + ib->conn.ud.ud = event->param.ud; + ib->conn.ud.ah = ibv_create_ah(ib->ctx.pd, &ib->conn.ud.ud.ah_attr); + } node->state = STATE_CONNECTED; @@ -630,8 +632,8 @@ int ib_start(struct node *n) // Allocate space for 40 Byte GHR. We don't use this. if (ib->conn.port_space == RDMA_PS_UDP) { - ib->conn.grh_ptr = alloc(40); - ib->conn.grh_mr = ibv_reg_mr(ib->ctx.pd, ib->conn.grh_ptr, 40, IBV_ACCESS_LOCAL_WRITE); + ib->conn.ud.grh_ptr = alloc(40); + ib->conn.ud.grh_mr = ibv_reg_mr(ib->ctx.pd, ib->conn.ud.grh_ptr, 40, IBV_ACCESS_LOCAL_WRITE); } // Several events should occur on the event channel, to make @@ -659,7 +661,7 @@ int ib_stop(struct node *n) // Call RDMA disconnect function // Will flush all outstanding WRs to the Completion Queue and // will call RDMA_CM_EVENT_DISCONNECTED if that is done. - if (n->state == STATE_CONNECTED) { + if (n->state == STATE_CONNECTED && ib->conn.port_space == RDMA_PS_TCP) { ret = rdma_disconnect(ib->ctx.id); if (ret) @@ -721,7 +723,9 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *relea // Poll Completion Queue // If we've already posted enough receive WRs, try to pull cnt if (ib->conn.available_recv_wrs >= (ib->qp_init.cap.max_recv_wr - ib->conn.buffer_subtraction) ) { - while(1) { + for (int i = 0;; i++) { + if (i % 2048 == 2047) pthread_testcancel(); + if (n->state != STATE_CONNECTED) return 0; wcs = ibv_poll_cq(ib->ctx.recv_cq, cnt, wc); @@ -758,9 +762,9 @@ int ib_read(struct node *n, struct sample *smps[], unsigned cnt, unsigned *relea // First 40 byte of UD data are GRH and unused in our case if (ib->conn.port_space == RDMA_PS_UDP) { - sge[i][j].addr = (uint64_t) ib->conn.grh_ptr; + sge[i][j].addr = (uint64_t) ib->conn.ud.grh_ptr; sge[i][j].length = 40; - sge[i][j].lkey = ib->conn.grh_mr->lkey; + sge[i][j].lkey = ib->conn.ud.grh_mr->lkey; j++; } @@ -894,9 +898,9 @@ int ib_write(struct node *n, struct sample *smps[], unsigned cnt, unsigned *rele // Check if connection is connected or unconnected and set appropriate values if (ib->conn.port_space == RDMA_PS_UDP) { - wr[sent].wr.ud.ah = ibv_create_ah(ib->ctx.pd, &ib->conn.ud.ah_attr); - wr[sent].wr.ud.remote_qkey = ib->conn.ud.qkey; - wr[sent].wr.ud.remote_qpn = ib->conn.ud.qp_num; + wr[sent].wr.ud.ah = ib->conn.ud.ah; + wr[sent].wr.ud.remote_qkey = ib->conn.ud.ud.qkey; + wr[sent].wr.ud.remote_qpn = ib->conn.ud.ud.qp_num; } // Check if data can be send inline diff --git a/tests/benchmarks/evaluate_logs.ipynb b/tests/benchmarks/evaluate_logs.ipynb new file mode 100644 index 000000000..7e6390723 --- /dev/null +++ b/tests/benchmarks/evaluate_logs.ipynb @@ -0,0 +1,396 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Benchmark" + ] + }, + { + "cell_type": "code", + "execution_count": 155, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Settings\n", + "root_dir = '/home/dennis/VILLASnode/tests/benchmarks'\n", + "benchmark_dir = 'benchmarks_20180801_00-11-11'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load all files\n", + "### Results\n", + "...\n", + "\n", + "### Source log\n", + "..." + ] + }, + { + "cell_type": "code", + "execution_count": 156, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded /home/dennis/VILLASnode/tests/benchmarks/benchmarks_20180801_00-11-11/0_TCP-3-100000-1000000.csv\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import os\n", + "import re\n", + "\n", + "# First, source log\n", + "\n", + "# Initialize arrays\n", + "result_files = []\n", + "result_paths = []\n", + "log_paths = []\n", + "result_array = []\n", + "\n", + "# Save complete path of files in an array\n", + "for subdir, dirs, files in os.walk(root_dir+'/'+benchmark_dir):\n", + " for file in files:\n", + " # Regex to match .csv files\n", + " if re.match(r'.*?csv', file, re.M|re.I):\n", + " result_paths.append(os.path.join(subdir, file))\n", + " # Regex to match .log files\n", + " elif re.match(r'.*?log', file, re.M|re.I):\n", + " log_paths.append(os.path.join(subdir, file))\n", + " \n", + " result_files = files\n", + "\n", + "# Loop through array with result files and save the comma separated data into a new array\n", + "for file in result_paths:\n", + " print(\"Loaded {}\".format(file))\n", + " \n", + " # Load file \n", + " result_array.append(np.genfromtxt(file, delimiter=','))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save characteristics of tests\n", + "All important settings are contained in the name of the file. We will save them in a separate array. The structure of the name is as follows:\n", + "\n", + "```bash\n", + "root_dir/benchmarks_${DATE}/${ID}_${MODE}-${VALUES IN SMP}-${RATE}-${SENT SMPS}\n", + "```\n", + "\n", + "Thus, we will structure it in the settings_array as follows:\n", + "\n", + "* `settings_array[*][0] = ID`\n", + "* `settings_array[*][1] = MODE`\n", + "* `settings_array[*][2] = VALUES IN SAMPLE`\n", + "* `settings_array[*][3] = RATE`\n", + "* `settings_array[*][4] = TOTAL NUMBER OF SAMPLES`" + ] + }, + { + "cell_type": "code", + "execution_count": 157, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Array with settings\n", + "settings_array = []\n", + "\n", + "for file in result_files:\n", + " settings = []\n", + " \n", + " matchObj = re.match(r'(\\d*)_(\\w*)-(\\d*)-(\\d*)-(\\d*).csv', file, re.M|re.I)\n", + "\n", + " # Fill values to array\n", + " if matchObj:\n", + " for i in range(0,5):\n", + " settings.append(matchObj.group(i+1))\n", + " \n", + " # Append array to big array\n", + " settings_array.append(settings)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get missed steps from source node\n", + "..." + ] + }, + { + "cell_type": "code", + "execution_count": 161, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "540\n" + ] + } + ], + "source": [ + "missing_send_arr = []\n", + "\n", + "# This line indicates if we passed the \"node connected\" line. Only after this line\n", + "# the script will start counting missed steps\n", + "connected = False\n", + "\n", + "for i, file in enumerate(log_paths):\n", + " F = open(file, \"r\")\n", + " line = F.readline()\n", + "\n", + " missing_send_arr.append(0)\n", + "\n", + " # Loop through file\n", + " while line:\n", + " #if re.match(r'.*Connection established in node', line):\n", + " connected = True\n", + " \n", + " if connected:\n", + " #matchObj = re.match(r'.*Missed steps: (\\d*)', line, re.M|re.I)\n", + " matchObj = re.match(r'.*written=0, expected=(\\d*)', line, re.M|re.I)\n", + " \n", + " if matchObj:\n", + " missing_send_arr[i] += int(matchObj.group(1))\n", + " \n", + " line = F.readline()\n", + " \n", + " print(missing_send_arr[i])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Process data\n", + "\n", + "### Number of samples\n", + "* First number which appears in `result_array[*][3]` will be the first sample in the benchmark. Ignore all missing samples before that entry.\n", + "* After that, check how many samples are missing.\n", + "* Then, calculate the percentage of missing samples." + ] + }, + { + "cell_type": "code", + "execution_count": 162, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Test 0 missed 68 (0.011%) of 995605 (1000000 before correction) samples\n" + ] + } + ], + "source": [ + "# Sequence number of first sample that occurs are receive side\n", + "start_sequence = []\n", + "# Real total number of sent messages, after subtraction of first sequence\n", + "real_total_arr = []\n", + "# Number of missing samples after first sample\n", + "missing_recv_arr = []\n", + "# Percentage of missed samples\n", + "perc_miss_arr = []\n", + "\n", + "# Generate real total and number of missing samples.\n", + "# Print percentage of missed samples\n", + "for (i, csv_vec) in enumerate(result_array):\n", + " start_sequence.append(csv_vec[0][3])\n", + " \n", + " # Real total number = total number - start_seqquence\n", + " # Number missing = real total number - length of data arary\n", + " real_total = int(settings_array[i][4]) - int(start_sequence[i])\n", + " missing_recv = real_total - len(csv_vec)\n", + " perc_miss = round(missing_recv / real_total * 100, 3)\n", + " \n", + " print(\"Test {} missed {} ({}%) of {} ({} before correction) samples\"\n", + " .format(i, number_miss, perc_miss, real_total, settings_array[i][4]))\n", + " \n", + " real_total_arr.append(real_total)\n", + " missing_recv_arr.append(number_miss)\n", + " perc_miss_arr.append(perc_miss)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "## Plot data\n", + "We want to plot the offset" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "from matplotlib.font_manager import FontProperties\n", + "import os\n", + "\n", + "# Start creating plots\n", + "for i, val in enumerate(result_array):\n", + " # Create figure\n", + " fig = plt.figure(num=None, figsize=(12, 8), dpi=90, facecolor='w', edgecolor='k')\n", + "\n", + " # Add plot and set title\n", + " ax = fig.add_subplot(111)\n", + " \n", + " # Set subtitle \n", + " title = \"Test {} — Mode: {}, Values in Sample: {}, Rate: {}, Total number of Samples: {}\".format(settings_array[i][0],\n", + " settings_array[i][1], \n", + " settings_array[i][2], \n", + " settings_array[i][3], \n", + " settings_array[i][4])\n", + " ax.set_title(title, fontsize=12, pad=12, loc='center')\n", + " \n", + " # Set grid\n", + " ax.set_axisbelow(True)\n", + " ax.grid(True, linestyle='--')\n", + "\n", + " bins = 6250\n", + " x_limit=0.0001\n", + " \n", + " # Data in plot\n", + " # http://www.color-hex.com/color-palette/33602\n", + " val_t = val.transpose()\n", + " \n", + " ax.hist(val_t[2], label='t_sent -> t_recv offset', edgecolor='black', bins=bins, color='#00549f')\n", + "\n", + " # Set axis and calculate values above limit\n", + " plt.xlim([0,x_limit])\n", + " \n", + "\n", + " plt.xticks(np.arange(0, x_limit + 0.000001, 0.00001), fontsize=11)\n", + " plt.yticks(fontsize=11)\n", + "\n", + " # Labels\n", + " ax.set_xlabel('time [s]', fontsize=13, labelpad=20)\n", + " ax.set_ylabel('frequency', fontsize=13, labelpad=20)\n", + " ax.set_yscale('log')\n", + "\n", + " # Create text for offset\n", + " off_smaller_4us = round((np.size(val_t[2][val_t[2] < 0.000004]) / np.size(val_t[2])) * 100, 3)\n", + " off_smaller_5us = round((np.size(val_t[2][val_t[2] < 0.000005]) / np.size(val_t[2])) * 100, 3)\n", + " off_smaller_10us = round((np.size(val_t[2][val_t[2] < 0.00001]) / np.size(val_t[2])) * 100, 3)\n", + " off_bigger_100us = round((np.size(val_t[2][val_t[2] > x_limit]) / np.size(val_t[2])) * 100, 4)\n", + " \n", + " offset_text = 'offset < {} for {}% of samples\\n'.format('4µs', off_smaller_4us)\n", + " offset_text += 'offset < {} for {}% of samples\\n'.format('5µs', off_smaller_5us)\n", + " offset_text += 'offset < {} for {}% of samples\\n'.format('10µs', off_smaller_10us)\n", + " offset_text += 'offset > {} for {}% of samples'.format('100µs', off_bigger_100us)\n", + " \n", + " # Create text for missed steps\n", + " #ToDo: Add percentage\n", + " missed_text = 'missed by villas-pipe: {0: <23}\\n'.format(missing_recv_arr[i])\n", + " missed_text += 'missed by villas-node: {0: <23}'.format(0)\n", + " \n", + " # bbox accepts FancyBboxPatch prop dict\n", + " font_header = FontProperties()\n", + " font_header.set_weight('bold')\n", + " font_header.set_size(10)\n", + " \n", + " # Offset boxes\n", + " ax.text(0.983, 0.88, \"Offset Ranges\",\n", + " verticalalignment='top', horizontalalignment='right',\n", + " transform=ax.transAxes,\n", + " color='black', fontproperties = font_header)\n", + " ax.text(0.712, 0.84, offset_text,\n", + " verticalalignment='top', horizontalalignment='left',\n", + " transform=ax.transAxes,\n", + " color='black', fontsize=9.25,\n", + " bbox={'facecolor':'white', 'alpha':0.85, 'pad':0.3, 'boxstyle':'round',\n", + " 'edgecolor':'#dbdbdb'})\n", + " \n", + " # Missed steps\n", + " ax.text(0.983, 0.70, \"Missed steps\",\n", + " verticalalignment='top', horizontalalignment='right',\n", + " transform=ax.transAxes,\n", + " color='black', fontproperties = font_header)\n", + " ax.text(0.712, 0.66, missed_text,\n", + " verticalalignment='top', horizontalalignment='left',\n", + " transform=ax.transAxes,\n", + " color='black', fontsize=9.25,\n", + " bbox={'facecolor':'white', 'alpha':0.85, 'pad':0.3, 'boxstyle':'round',\n", + " 'edgecolor':'#dbdbdb'})\n", + "\n", + " #Create legend\n", + " ax.legend(loc=1, fontsize=12)\n", + " \n", + "\n", + " #Show plot\n", + " plt.show()\n", + " \n", + " #Save plot\n", + " fig.savefig('./plots/test{}.png'.format(i))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/benchmarks/node-infiniband-benchmark.sh b/tests/benchmarks/node-infiniband-benchmark.sh new file mode 100755 index 000000000..aeb5cdf2b --- /dev/null +++ b/tests/benchmarks/node-infiniband-benchmark.sh @@ -0,0 +1,223 @@ +#!/bin/bash +# +# Integration Infiniband test using villas-node. +# +# @author Dennis Potter +# @copyright 2018, Institute for Automation of Complex Power Systems, EONERC +# @license GNU General Public License (version 3) +# +# VILLASnode +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +################################################################################## + +# Check if user is superuser. SU is used for namespace +if [[ "$EUID" -ne 0 ]]; then + echo "Please run as root" + exit 99 +fi + +# Check if Infiniband card is present +if [[ ! $(lspci | grep Infiniband) ]]; then + echo "Did not find any Infiniband cards in system" + exit 99 +fi + + +SCRIPT=$(realpath $0) +SCRIPTPATH=$(dirname ${SCRIPT}) +source ${SCRIPTPATH}/../../tools/integration-tests-helper.sh + + +CONFIG_FILE=$(mktemp /tmp/ib-configuration-XXXX.conf) +CONFIG_FILE_TARGET=$(mktemp /tmp/ib-configuration-target-XXXX.conf) +CONFIG_FILE_SOURCE=$(mktemp /tmp/ib-configuration-source-XXXX.conf) +INPUT_FILE=$(mktemp) +LOG_DIR=benchmarks_$(date +%Y%m%d_%H-%M-%S) +mkdir ${LOG_DIR} + +NUM_VALUES=(3 5 10 25 50) +RATE_SAMPLES=(10 100 1000 10000 100000 200000) +TIME_TO_RUN=10 + +# Declare modes +MODES=("TCP" "UDP") + +# Initialize counter +COUNT=0 + +# Set config file with a MODE flag +# NUM_VALUES, RATE_SAMPLES, NUM_SAMPLES, and MODE will be replaced in loop +cat > ${CONFIG_FILE} < ${CONFIG_FILE_TARGET} < ${CONFIG_FILE_SOURCE} <${LOG_DIR}/${COUNT}_source_node.log & + source_node_proc=$! + + sleep $((${TIME_TO_RUN} + 1)) + + # Stop node + kill $target_node_proc + + sleep 3 + #ToDo: Add checks + + echo "########################################################" + echo "## STOP $MODE"-${NUM_VALUE}-${RATE_SAMPLE}-${NUM_SAMPLE} + echo "########################################################" + echo "" + + sed -i -e 's/\/'${COUNT}'_/\/COUNT_/g' ${CONFIG_FILE} + sed -i -e 's/'${MODE}'/MODE/g' ${CONFIG_FILE} + sed -i -e 's/values\ =\ '${NUM_VALUE}'/values\ =\ NUM_VALUES/g' ${CONFIG_FILE} + sed -i -e 's/rate\ =\ '${RATE_SAMPLE}'/rate\ =\ RATE_SAMPLES/g' ${CONFIG_FILE} + sed -i -e 's/limit\ =\ '${NUM_SAMPLE}'/limit\ =\ NUM_SAMPLES/g' ${CONFIG_FILE} + sed -i -e 's/-'${NUM_VALUE}'/-NUM_VALUES/g' ${CONFIG_FILE} + sed -i -e 's/-'${RATE_SAMPLE}'/-RATE_SAMPLES/g' ${CONFIG_FILE} + sed -i -e 's/-'${NUM_SAMPLE}'/-NUM_SAMPLES/g' ${CONFIG_FILE} + + ((COUNT++)) + done + done +done + +# Since this script will be executed as sudo we should chmod the +# log dir 777. Otherwise too many unnecessary 'sudo rm -rf' will be called. +chmod -R 777 ${LOG_DIR} + +rm ${CONFIG_FILE} ${CONFIG_FILE_TARGET} ${CONFIG_FILE_SOURCE} ${INPUT_FILE} + +exit 0 diff --git a/tests/integration/node-infiniband.sh b/tests/integration/node-infiniband.sh index e357ebc34..d91d0615b 100755 --- a/tests/integration/node-infiniband.sh +++ b/tests/integration/node-infiniband.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Integration loopback test using villas-node. +# Integration Infiniband test using villas-node. # # @author Dennis Potter # @copyright 2018, Institute for Automation of Complex Power Systems, EONERC @@ -23,28 +23,46 @@ ################################################################################## # Check if user is superuser. SU is used for namespace -if [ "$EUID" -ne 0 ] - then echo "Please run as root" +if [[ "$EUID" -ne 0 ]]; then + echo "Please run as root" exit 99 fi +# Check if Infiniband card is present +if [[ ! $(lspci | grep Infiniband) ]]; then + echo "Did not find any Infiniband cards in system" + exit 99 +fi + + SCRIPT=$(realpath $0) SCRIPTPATH=$(dirname ${SCRIPT}) source ${SCRIPTPATH}/../../tools/integration-tests-helper.sh -CONFIG_FILE=$(mktemp) -CONFIG_FILE_TARGET=$(mktemp) +CONFIG_FILE=$(mktemp /tmp/ib-configuration-XXXX.conf) +CONFIG_FILE_TARGET=$(mktemp /tmp/ib-configuration-target-XXXX.conf) INPUT_FILE=$(mktemp) OUTPUT_FILE=$(mktemp) NUM_SAMPLES=${NUM_SAMPLES:-10} +RC=0 +# Generate test data for TCP and UDP test +VILLAS_LOG_PREFIX=$(colorize "[Signal]") \ +villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE} + +# Set config file with a MODE flag cat > ${CONFIG_FILE} < ${CONFIG_FILE_TARGET} < ${INPUT_FILE} +# Declare modes +MODES=("TCP" "UDP") -# Start receiving node -VILLAS_LOG_PREFIX=$(colorize "[Node] ") \ -villas-node ${CONFIG_FILE_TARGET} & -node_proc=$! +# Run through modes +for MODE in "${MODES[@]}" +do -# Wait for node to complete init -sleep 1 + echo "#############################" + echo "#############################" + echo "## START ${MODE} ##" + echo "#############################" + echo "#############################" -# Preprare fifo -DATAFIFO=/tmp/datafifo -if [[ ! -p ${DATAFIFO} ]]; then - mkfifo ${DATAFIFO} -fi + sed -i -e 's/RDMA_PS_MODE/RDMA_PS_'${MODE}'/g' ${CONFIG_FILE} -# Start sending pipe -VILLAS_LOG_PREFIX=$(colorize "[Pipe] ") \ -ip netns exec namespace0 villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} ib_node_source >${OUTPUT_FILE} <${DATAFIFO} & + # Start receiving node + VILLAS_LOG_PREFIX=$(colorize "[Node] ") \ + villas-node ${CONFIG_FILE_TARGET} & + node_proc=$! + + # Wait for node to complete init + sleep 1 + + # Preprare fifo + DATAFIFO=/tmp/datafifo + if [[ ! -p ${DATAFIFO} ]]; then + mkfifo ${DATAFIFO} + fi + + # Start sending pipe + VILLAS_LOG_PREFIX=$(colorize "[Pipe] ") \ + ip netns exec namespace0 villas-pipe -l ${NUM_SAMPLES} ${CONFIG_FILE} ib_node_source >${OUTPUT_FILE} <${DATAFIFO} & + node_pipe=$! + + # Keep pipe alive + sleep 5 >${DATAFIFO} & + + # Wait for pipe to connect to node + sleep 3 + + # Write data to pipe + cat ${INPUT_FILE} >${DATAFIFO} & + + # Wait for node to handle samples + sleep 2 + + # Stop node + kill $node_pipe + kill $node_proc + + # Compare data + villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} + RC=$? -# Keep pipe alive -sleep 5 >${DATAFIFO} & + # Exit, if an error occurs + if [[ $RC != 0 ]]; then + rm ${CONFIG_FILE} ${CONFIG_FILE_TARGET} ${INPUT_FILE} ${OUTPUT_FILE} ${DATAFIFO} + + exit $RC + fi -# Wait for pipe to connect to node -sleep 3 + echo "#############################" + echo "#############################" + echo "## STOP $MODE ##" + echo "#############################" + echo "#############################" + echo "" -# Write data to pipe -cat ${INPUT_FILE} >${DATAFIFO} & - -# Wait for node to handle samples -sleep 2 - -# Stop node -kill %1 -kill -9 $node_proc - -# Compare data -villas-test-cmp ${INPUT_FILE} ${OUTPUT_FILE} -RC=$? + sed -i -e 's/RDMA_PS_'${MODE}'/RDMA_PS_MODE/g' ${CONFIG_FILE} +done rm ${CONFIG_FILE} ${CONFIG_FILE_TARGET} ${INPUT_FILE} ${OUTPUT_FILE} ${DATAFIFO}