mirror of
https://git.rwth-aachen.de/acs/public/villas/node/
synced 2025-03-09 00:00:00 +01:00
Merge branch 'ib-uc' into develop. Closes #191
This commit is contained in:
Dennis Potter
commit
bb6cee70ab
4 changed files with 75 additions and 19 deletions
|
|
@ -23,7 +23,7 @@ nodes = {
|
|||
ib_node_source = {
|
||||
type = "infiniband",
|
||||
|
||||
rdma_port_space = "RDMA_PS_TCP",
|
||||
rdma_port_space = "RC",
|
||||
|
||||
in = {
|
||||
address = "10.0.0.2:1337",
|
||||
|
|
@ -56,7 +56,7 @@ nodes = {
|
|||
ib_node_target = {
|
||||
type = "infiniband",
|
||||
|
||||
rdma_port_space = "RDMA_PS_TCP",
|
||||
rdma_port_space = "RC",
|
||||
|
||||
in = {
|
||||
address = "10.0.0.1:1337",
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ int ib_parse(struct node *n, json_t *cfg)
|
|||
int ret;
|
||||
char *local = NULL;
|
||||
char *remote = NULL;
|
||||
const char *port_space = "RDMA_PS_TCP";
|
||||
const char *transport_mode = "RC";
|
||||
const char *poll_mode = "BUSY";
|
||||
int timeout = 1000;
|
||||
int recv_cq_size = 128;
|
||||
|
|
@ -207,7 +207,7 @@ int ib_parse(struct node *n, json_t *cfg)
|
|||
ret = json_unpack_ex(cfg, &err, 0, "{s?: o, s?: o, s?: s, s?: s}",
|
||||
"in", &json_in,
|
||||
"out", &json_out,
|
||||
"rdma_port_space", &port_space
|
||||
"rdma_transport_mode", &transport_mode
|
||||
);
|
||||
if (ret)
|
||||
jerror(&err, "Failed to parse in/out json blocks");
|
||||
|
|
@ -276,21 +276,28 @@ int ib_parse(struct node *n, json_t *cfg)
|
|||
debug(LOG_IB | 4, "Translated %s:%s to a struct addrinfo in node %s", ip_adr, port, node_name(n));
|
||||
|
||||
// Translate port space
|
||||
if (strcmp(port_space, "RDMA_PS_TCP") == 0) {
|
||||
if (strcmp(transport_mode, "RC") == 0) {
|
||||
ib->conn.port_space = RDMA_PS_TCP;
|
||||
ib->qp_init.qp_type = IBV_QPT_RC;
|
||||
}
|
||||
else if (strcmp(port_space, "RDMA_PS_UDP") == 0) {
|
||||
else if (strcmp(transport_mode, "UC") == 0) {
|
||||
#ifdef RDMA_CMA_H_CUSTOM
|
||||
ib->conn.port_space = RDMA_PS_IB;
|
||||
ib->qp_init.qp_type = IBV_QPT_UC;
|
||||
#else
|
||||
error("Unreliable Connected (UC) mode is only available with an adapted version of librdma. Please"
|
||||
"read the Infiniband node type Documentation for more information on UC!",
|
||||
#endif
|
||||
}
|
||||
else if (strcmp(transport_mode, "UD") == 0) {
|
||||
ib->conn.port_space = RDMA_PS_UDP;
|
||||
ib->qp_init.qp_type = IBV_QPT_UD;
|
||||
}
|
||||
else if (strcmp(port_space, "RDMA_PS_IB") == 0 || strcmp(port_space, "RDMA_PS_IPOIB"))
|
||||
error("Although RDMA_PS_IB and RDMA_PS_IPOIB are valid in rdma_cma.h, they are not supported by VILLAS!");
|
||||
else
|
||||
error("rdma_port_space (%s) is not a valid port space supported by rdma_cma.h in node %s!",
|
||||
port_space, node_name(n));
|
||||
error("transport_mode = %s is not a valid transport mode in node %s!",
|
||||
transport_mode, node_name(n));
|
||||
|
||||
debug(LOG_IB | 4, "Translated %s to enum rdma_port_space in node %s", port_space, node_name(n));
|
||||
debug(LOG_IB | 4, "Set transport mode to %s in node %s", transport_mode, node_name(n));
|
||||
|
||||
// Set timeout
|
||||
ib->conn.timeout = timeout;
|
||||
|
|
@ -421,7 +428,41 @@ static void ib_create_bind_id(struct node *n)
|
|||
int ret;
|
||||
|
||||
// Create rdma_cm_id
|
||||
/**
|
||||
* The unreliable connected mode is officially not supported by the rdma_cm library. Only the Reliable
|
||||
* Connected mode (RDMA_PS_TCP) and the Unreliable Datagram mode (RDMA_PS_UDP). Although it is not officially
|
||||
* supported, it is possible to use it with a few small adaptions to the sourcecode. To enable the
|
||||
* support for UC connections follow the steps below:
|
||||
*
|
||||
* 1. git clone https://github.com/linux-rdma/rdma-core
|
||||
* 2. cd rdma-core
|
||||
* 2. Edit librdmacm/cma.c and remove the keyword 'static' in front of:
|
||||
*
|
||||
* static int rdma_create_id2(struct rdma_event_channel *channel,
|
||||
* struct rdma_cm_id **id, void *context,
|
||||
* enum rdma_port_space ps, enum ibv_qp_type qp_type)
|
||||
*
|
||||
* 3. Edit librdmacm/rdma_cma.h and add the following two entries to the file:
|
||||
*
|
||||
* #define RDMA_CMA_H_CUSTOM
|
||||
*
|
||||
* int rdma_create_id2(struct rdma_event_channel *channel,
|
||||
* struct rdma_cm_id **id, void *context,
|
||||
* enum rdma_port_space ps, enum ibv_qp_type qp_type);
|
||||
*
|
||||
* 4. Edit librdmacm/librdmacm.map and add a new line with:
|
||||
*
|
||||
* rdma_create_id2
|
||||
*
|
||||
* 5. ./build.sh
|
||||
* 6. cd build && sudo make install
|
||||
*
|
||||
*/
|
||||
#ifdef RDMA_CMA_H_CUSTOM
|
||||
ret = rdma_create_id2(ib->ctx.ec, &ib->ctx.id, NULL, ib->conn.port_space, ib->qp_init.qp_type);
|
||||
#else
|
||||
ret = rdma_create_id(ib->ctx.ec, &ib->ctx.id, NULL, ib->conn.port_space);
|
||||
#endif
|
||||
if (ret)
|
||||
error("Failed to create rdma_cm_id of node %s: %s", node_name(n), gai_strerror(ret));
|
||||
|
||||
|
|
@ -670,7 +711,7 @@ int ib_stop(struct node *n)
|
|||
// Call RDMA disconnect function
|
||||
// Will flush all outstanding WRs to the Completion Queue and
|
||||
// will call RDMA_CM_EVENT_DISCONNECTED if that is done.
|
||||
if (n->state == STATE_CONNECTED && ib->conn.port_space == RDMA_PS_TCP) {
|
||||
if (n->state == STATE_CONNECTED && ib->conn.port_space != RDMA_PS_UDP) {
|
||||
ret = rdma_disconnect(ib->ctx.id);
|
||||
|
||||
if (ret)
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ RATE_SAMPLES=(10 100 1000 10000 50000)
|
|||
TIME_TO_RUN=30
|
||||
|
||||
# Declare modes
|
||||
MODES=("TCP" "UDP")
|
||||
MODES=("RC" "UC" "UD")
|
||||
|
||||
# Initialize counter
|
||||
COUNT=0
|
||||
|
|
@ -117,6 +117,8 @@ nodes = {
|
|||
frequency = 3,
|
||||
rate = ${RATE_SAMPLE},
|
||||
limit = ${NUM_SAMPLE},
|
||||
|
||||
monitor_missed = false,
|
||||
},
|
||||
|
||||
results_input = {
|
||||
|
|
@ -124,6 +126,8 @@ nodes = {
|
|||
|
||||
format = "csv",
|
||||
uri = "${LOG_DIR}/${COUNT}_${MODE}-${NUM_VALUE}-${RATE_SAMPLE}-${NUM_SAMPLE}_input.csv",
|
||||
|
||||
buffer_size = 2000000000,
|
||||
},
|
||||
|
||||
results_output = {
|
||||
|
|
@ -131,6 +135,8 @@ nodes = {
|
|||
|
||||
format = "csv",
|
||||
uri = "${LOG_DIR}/${COUNT}_${MODE}-${NUM_VALUE}-${RATE_SAMPLE}-${NUM_SAMPLE}_output.csv",
|
||||
|
||||
buffer_size = 2000000000,
|
||||
},
|
||||
|
||||
ib_node_source = {
|
||||
|
|
|
|||
|
|
@ -47,12 +47,21 @@ OUTPUT_FILE=$(mktemp)
|
|||
NUM_SAMPLES=${NUM_SAMPLES:-10}
|
||||
RC=0
|
||||
|
||||
# Generate test data for TCP and UDP test
|
||||
# Generate test data for RC, UC, and UD test
|
||||
VILLAS_LOG_PREFIX=$(colorize "[Signal]") \
|
||||
villas-signal random -l ${NUM_SAMPLES} -n > ${INPUT_FILE}
|
||||
|
||||
# Set config file with a MODE flag
|
||||
cat > ${CONFIG_FILE} <<EOF
|
||||
logging = {
|
||||
level = 0,
|
||||
facilities = "ib",
|
||||
},
|
||||
|
||||
http = {
|
||||
enabled = false,
|
||||
},
|
||||
|
||||
nodes = {
|
||||
results = {
|
||||
type = "file",
|
||||
|
|
@ -62,7 +71,7 @@ nodes = {
|
|||
ib_node_source = {
|
||||
type = "infiniband",
|
||||
|
||||
rdma_port_space = "RDMA_PS_MODE",
|
||||
rdma_port_space = "MODE",
|
||||
|
||||
in = {
|
||||
address = "10.0.0.2:1337",
|
||||
|
|
@ -91,7 +100,7 @@ nodes = {
|
|||
ib_node_target = {
|
||||
type = "infiniband",
|
||||
|
||||
rdma_port_space = "RDMA_PS_MODE",
|
||||
rdma_port_space = "MODE",
|
||||
|
||||
in = {
|
||||
address = "10.0.0.1:1337",
|
||||
|
|
@ -119,7 +128,7 @@ paths = (
|
|||
EOF
|
||||
|
||||
# Declare modes
|
||||
MODES=("TCP" "UDP")
|
||||
MODES=("RC" "UC" "UD")
|
||||
|
||||
# Run through modes
|
||||
for MODE in "${MODES[@]}"
|
||||
|
|
@ -131,7 +140,7 @@ do
|
|||
echo "#############################"
|
||||
echo "#############################"
|
||||
|
||||
sed -i -e 's/RDMA_PS_MODE/RDMA_PS_'${MODE}'/g' ${CONFIG_FILE}
|
||||
sed -i -e 's/MODE/'${MODE}'/g' ${CONFIG_FILE}
|
||||
|
||||
# Start receiving node
|
||||
VILLAS_LOG_PREFIX=$(colorize "[Node] ") \
|
||||
|
|
@ -186,7 +195,7 @@ do
|
|||
echo "#############################"
|
||||
echo ""
|
||||
|
||||
sed -i -e 's/RDMA_PS_'${MODE}'/RDMA_PS_MODE/g' ${CONFIG_FILE}
|
||||
sed -i -e 's/'${MODE}'/MODE/g' ${CONFIG_FILE}
|
||||
done
|
||||
|
||||
rm ${CONFIG_FILE} ${CONFIG_FILE_TARGET} ${INPUT_FILE} ${OUTPUT_FILE} ${DATAFIFO}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue