mirror of
https://git.rwth-aachen.de/acs/public/villas/node/
synced 2025-03-09 00:00:00 +01:00
115 lines
2.5 KiB
C++
115 lines
2.5 KiB
C++
/* Node type: infiniband.
|
|
*
|
|
* Author: Dennis Potter <dennis@dennispotter.eu>
|
|
* SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <rdma/rdma_cma.h>
|
|
|
|
#include <villas/format.hpp>
|
|
#include <villas/pool.hpp>
|
|
#include <villas/queue_signalled.h>
|
|
|
|
namespace villas {
|
|
namespace node {
|
|
|
|
// Forward declarations
|
|
class NodeCompat;
|
|
|
|
// Constants
|
|
#define META_SIZE 24
|
|
#define GRH_SIZE 40
|
|
#define META_GRH_SIZE META_SIZE + GRH_SIZE
|
|
#define CHK_PER_ITER 2048
|
|
|
|
struct infiniband {
|
|
// IBV/RDMA CM structs
|
|
struct context_s {
|
|
struct rdma_cm_id *listen_id;
|
|
struct rdma_cm_id *id;
|
|
struct rdma_event_channel *ec;
|
|
|
|
struct ibv_pd *pd;
|
|
struct ibv_cq *recv_cq;
|
|
struct ibv_cq *send_cq;
|
|
struct ibv_comp_channel *comp_channel;
|
|
} ctx;
|
|
|
|
// Queue Pair init variables
|
|
struct ibv_qp_init_attr qp_init;
|
|
|
|
// Size of receive and send completion queue
|
|
int recv_cq_size;
|
|
int send_cq_size;
|
|
|
|
// Bool, set if threads should be aborted
|
|
int stopThreads;
|
|
|
|
// When most messages are sent inline, once every <X> cycles a signal must be sent.
|
|
unsigned signaling_counter;
|
|
unsigned periodic_signaling;
|
|
|
|
// Connection specific variables
|
|
struct connection_s {
|
|
struct addrinfo *src_addr;
|
|
struct addrinfo *dst_addr;
|
|
|
|
// RDMA_PS_TCP or RDMA_PS_UDP
|
|
enum rdma_port_space port_space;
|
|
|
|
// Timeout for rdma_resolve_route
|
|
int timeout;
|
|
|
|
// Thread to monitor RDMA CM Event threads
|
|
pthread_t rdma_cm_event_thread;
|
|
|
|
// Bool, should data be send inline if possible?
|
|
int send_inline;
|
|
|
|
// Bool, should node have a fallback if it can't connect to a remote host?
|
|
int use_fallback;
|
|
|
|
// Counter to keep track of available recv. WRs
|
|
unsigned available_recv_wrs;
|
|
|
|
/* Fixed number to substract from min. number available
|
|
* WRs in receive queue */
|
|
unsigned buffer_subtraction;
|
|
|
|
// Unrealiable connectionless data
|
|
struct ud_s {
|
|
struct rdma_ud_param ud;
|
|
struct ibv_ah *ah;
|
|
void *grh_ptr;
|
|
struct ibv_mr *grh_mr;
|
|
} ud;
|
|
|
|
} conn;
|
|
|
|
// Misc settings
|
|
int is_source;
|
|
};
|
|
|
|
int ib_reverse(NodeCompat *n);
|
|
|
|
char *ib_print(NodeCompat *n);
|
|
|
|
int ib_parse(NodeCompat *n, json_t *json);
|
|
|
|
int ib_check(NodeCompat *n);
|
|
|
|
int ib_start(NodeCompat *n);
|
|
|
|
int ib_destroy(NodeCompat *n);
|
|
|
|
int ib_stop(NodeCompat *n);
|
|
|
|
int ib_read(NodeCompat *n, struct Sample *const smps[], unsigned cnt);
|
|
|
|
int ib_write(NodeCompat *n, struct Sample *const smps[], unsigned cnt);
|
|
|
|
} // namespace node
|
|
} // namespace villas
|