From a37d6c131b5afd6b344bd9592385d47b8de38a4b Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 13 Sep 2021 14:59:38 +0200 Subject: [PATCH] infiniband: fix compilation errors --- CMakeLists.txt | 2 +- lib/memory/ib.cpp | 8 ++++++-- lib/nodes/infiniband.cpp | 41 ++++++++++++++++++++++++---------------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d00ded104..98360c5a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,7 +184,7 @@ cmake_dependent_option(WITH_NODE_EXEC "Build with exec node-type" cmake_dependent_option(WITH_NODE_FILE "Build with file node-type" ON "" OFF) cmake_dependent_option(WITH_NODE_FPGA "Build with fpga node-type" ON "WITH_FPGA" OFF) cmake_dependent_option(WITH_NODE_IEC61850 "Build with iec61850 node-types" ON "LIBIEC61850_FOUND" OFF) -cmake_dependent_option(WITH_NODE_INFINIBAND "Build with infiniband node-type" ON "FALSE; IBVerbs_FOUND; RDMACM_FOUND" OFF) # Infiniband node-type is currenly broken +cmake_dependent_option(WITH_NODE_INFINIBAND "Build with infiniband node-type" ON "IBVerbs_FOUND; RDMACM_FOUND" OFF) # Infiniband node-type is currenly broken cmake_dependent_option(WITH_NODE_INFLUXDB "Build with influxdb node-type" ON "" OFF) cmake_dependent_option(WITH_NODE_KAFKA "Build with kafka node-type" ON "RDKAFKA_FOUND" OFF) cmake_dependent_option(WITH_NODE_LOOPBACK "Build with loopback node-type" ON "" OFF) diff --git a/lib/memory/ib.cpp b/lib/memory/ib.cpp index c8fe90cf4..54007c003 100644 --- a/lib/memory/ib.cpp +++ b/lib/memory/ib.cpp @@ -26,7 +26,9 @@ #include #include #include +#include +using namespace villas; using namespace villas::utils; struct ibv_mr * memory_ib_get_mr(void *ptr) @@ -55,8 +57,10 @@ static struct memory_allocation * memory_ib_alloc(size_t len, size_t alignment, ma->parent = mi->parent->alloc(len + sizeof(struct ibv_mr *), alignment, mi->parent); ma->address = ma->parent->address; - if (!mi->pd) - error("Protection domain is not registered!"); + if (!mi->pd) { + auto logger = logging.get("memory:ib"); + logger->error("Protection domain is not registered!"); + } ma->ib.mr = ibv_reg_mr(mi->pd, ma->address, ma->length, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!ma->ib.mr) { diff --git a/lib/nodes/infiniband.cpp b/lib/nodes/infiniband.cpp index 6cbe60ee2..39fa7beee 100644 --- a/lib/nodes/infiniband.cpp +++ b/lib/nodes/infiniband.cpp @@ -180,6 +180,8 @@ int ib_parse(struct vnode *n, json_t *json) { struct infiniband *ib = (struct infiniband *) n->_vd; + throw ConfigError(json, "The infiniband node-type is currently broken!"); + int ret; char *local = nullptr, *remote = nullptr, *lasts; const char *transport_mode = "RC"; @@ -777,7 +779,9 @@ int ib_read(struct vnode *n, struct sample * const smps[], unsigned cnt) } else { ib->conn.available_recv_wrs += max_wr_post; - *release = 0; /* While we fill the receive queue, we always use all samples */ + + // TODO: fix release logic + // *release = 0; /* While we fill the receive queue, we always use all samples */ } /* Get Memory Region */ @@ -855,7 +859,8 @@ int ib_read(struct vnode *n, struct sample * const smps[], unsigned cnt) */ int correction = (ib->conn.port_space == RDMA_PS_UDP) ? META_GRH_SIZE : META_SIZE; - smps[j] = (struct sample *) (wc[j].wr_id); + // TODO: fix release logic + // smps[j] = (struct sample *) (wc[j].wr_id); smps[j]->length = SAMPLE_NUMBER_OF_VALUES(wc[j].byte_len - correction); smps[j]->ts.received = ts_receive; @@ -881,7 +886,8 @@ int ib_write(struct vnode *n, struct sample * const smps[], unsigned cnt) n->logger->debug("ib_write is called"); if (n->state == State::CONNECTED) { - *release = 0; + // TODO: fix release logic + // *release = 0; /* First, write */ @@ -962,39 +968,42 @@ int ib_write(struct vnode *n, struct sample * const smps[], unsigned cnt) * and prepare them to be released */ n->logger->debug("Bad WR occured with ID: {:#x} and S/G address: {:#x}: {}", - bad_wr->wr_id, bad_wr->sg_list, ret); + bad_wr->wr_id, (void *) bad_wr->sg_list, ret); while (1) { - smps[*release] = smps[m]; - - (*release)++; /* Increment number of samples to be released */ - sent--; /* Decrement the number of succesfully posted elements */ + // TODO: fix release logic + // smps[*release] = smps[m]; + // (*release)++; /* Increment number of samples to be released */ + sent--; /* Decrement the number of successfully posted elements */ if (++m == cnt) break; } } else if (wr[m].send_flags & IBV_SEND_INLINE) { - smps[*release] = smps[m]; - - (*release)++; + // TODO: fix release logic + // smps[*release] = smps[m]; + // (*release)++; } } - n->logger->debug("{} samples will be released (before WC)", *release); + // TODO: fix release logic + // n->logger->debug("{} samples will be released (before WC)", *release); /* Try to grab as many CQEs from CQ as there is space in *smps[] */ - ret = ibv_poll_cq(ib->ctx.send_cq, cnt - *release, wc); + // ret = ibv_poll_cq(ib->ctx.send_cq, cnt - *release, wc); for (int i = 0; i < ret; i++) { if (wc[i].status != IBV_WC_SUCCESS && wc[i].status != IBV_WC_WR_FLUSH_ERR) n->logger->warn("Work Completion status was not IBV_WC_SUCCESS: {}", wc[i].status); - smps[*release] = (struct sample *) (wc[i].wr_id); - (*release)++; + // TODO: fix release logic + // smps[*release] = (struct sample *) (wc[i].wr_id); + // (*release)++; } - n->logger->debug("{} samples will be released (after WC)", *release); + // TODO: fix release logic + // n->logger->debug("{} samples will be released (after WC)", *release); } return sent;