libhermit/usr/rdma-core/providers/qedr/qelr.h

/*
 * Copyright (c) 2015-2016  QLogic Corporation
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and /or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifndef __QELR_H__
#define __QELR_H__

#include <inttypes.h>
#include <stddef.h>
#include <endian.h>
#include <stdio.h>
#include <endian.h>
#include <ccan/minmax.h>

#include <infiniband/driver.h>
#include <util/udma_barrier.h>

#define writel(b, p) (*(uint32_t *)(p) = (b))
#define writeq(b, p) (*(uint64_t *)(p) = (b))

#include "qelr_hsi.h"
#include "qelr_chain.h"

#define qelr_err(format, arg...) printf(format, ##arg)

extern uint32_t qelr_dp_level;
extern uint32_t qelr_dp_module;

enum DP_MODULE {
	QELR_MSG_CQ		= 0x10000,
	QELR_MSG_RQ		= 0x20000,
	QELR_MSG_SQ		= 0x40000,
	QELR_MSG_QP		= (QELR_MSG_SQ | QELR_MSG_RQ),
	QELR_MSG_MR		= 0x80000,
	QELR_MSG_INIT		= 0x100000,
	/* to be added...up to 0x8000000 */
};

enum DP_LEVEL {
	QELR_LEVEL_VERBOSE	= 0x0,
	QELR_LEVEL_INFO		= 0x1,
	QELR_LEVEL_NOTICE	= 0x2,
	QELR_LEVEL_ERR		= 0x3,
};

#define DP_ERR(fd, fmt, ...)					\
do {								\
	fprintf(fd, "[%s:%d]" fmt,				\
		__func__, __LINE__,				\
		##__VA_ARGS__);					\
	fflush(fd); \
} while (0)

#define DP_NOTICE(fd, fmt, ...)					\
do {								\
	if (qelr_dp_level <= QELR_LEVEL_NOTICE)	{\
		fprintf(fd, "[%s:%d]" fmt,			\
		      __func__, __LINE__,			\
		      ##__VA_ARGS__);				\
		      fflush(fd); }				\
} while (0)

#define DP_INFO(fd, fmt, ...)					\
do {								\
	if (qelr_dp_level <= QELR_LEVEL_INFO)	{		\
		fprintf(fd, "[%s:%d]" fmt,			\
		      __func__, __LINE__,			\
		      ##__VA_ARGS__); fflush(fd);		\
	}							\
} while (0)

#define DP_VERBOSE(fd, module, fmt, ...)			\
do {								\
	if ((qelr_dp_level <= QELR_LEVEL_VERBOSE) &&		\
		     (qelr_dp_module & (module))) {		\
		fprintf(fd, "[%s:%d]" fmt,			\
		      __func__, __LINE__,			\
		      ##__VA_ARGS__);	fflush(fd); }		\
} while (0)

struct qelr_buf {
	void		*addr;
	size_t		len;		/* a 64 uint is used as s preparation
					 * for double layer pbl.
					 */
};

struct qelr_device {
	struct verbs_device ibv_dev;
};

struct qelr_devctx {
	struct ibv_context	ibv_ctx;
	FILE			*dbg_fp;
	void			*db_addr;
	uint64_t		db_pa;
	uint32_t		db_size;
	uint8_t			disable_edpm;
	uint32_t		kernel_page_size;

	uint32_t		max_send_wr;
	uint32_t		max_recv_wr;
	uint32_t		sges_per_send_wr;
	uint32_t		sges_per_recv_wr;
	int			max_cqes;
};

struct qelr_pd {
	struct ibv_pd		ibv_pd;
	uint32_t		pd_id;
};

struct qelr_mr {
	struct ibv_mr		ibv_mr;
};

union db_prod64 {
	struct rdma_pwm_val32_data data;
	uint64_t raw;
};

struct qelr_cq {
	struct ibv_cq		ibv_cq;	/* must be first */

	struct qelr_chain	chain;

	void			*db_addr;
	union db_prod64		db;

	uint8_t			chain_toggle;
	union rdma_cqe		*latest_cqe;
	union rdma_cqe		*toggle_cqe;

	uint8_t			arm_flags;
};

enum qelr_qp_state {
	QELR_QPS_RST,
	QELR_QPS_INIT,
	QELR_QPS_RTR,
	QELR_QPS_RTS,
	QELR_QPS_SQD,
	QELR_QPS_ERR,
	QELR_QPS_SQE
};

union db_prod32 {
	struct rdma_pwm_val16_data	data;
	uint32_t			raw;
};

struct qelr_qp_hwq_info {
	/* WQE */
	struct qelr_chain			chain;
	uint8_t					max_sges;

	/* WQ */
	uint16_t				prod;
	uint16_t				wqe_cons;
	uint16_t				cons;
	uint16_t				max_wr;

	/* DB */
	void					*db;      /* Doorbell address */
	void					*edpm_db;
	union db_prod32				db_data;  /* Doorbell data */
	void					*iwarp_db2;
	union db_prod32				iwarp_db2_data;

	uint16_t				icid;
};

struct qelr_rdma_ext {
	__be64	remote_va;
	__be32	remote_key;
	__be32	dma_length;
};

/* rdma extension, invalidate / immediate data + padding, inline data... */
#define QELR_MAX_DPM_PAYLOAD (sizeof(struct qelr_rdma_ext) + sizeof(uint64_t) +\
			       ROCE_REQ_MAX_INLINE_DATA_SIZE)
struct qelr_dpm {
	uint8_t			is_edpm;
	union {
		struct db_roce_dpm_data	data;
		uint64_t raw;
	} msg;

	uint8_t			payload[QELR_MAX_DPM_PAYLOAD];
	uint32_t		payload_size;
	uint32_t		payload_offset;

	struct qelr_rdma_ext    *rdma_ext;
};

struct qelr_qp {
	struct ibv_qp				ibv_qp;
	pthread_spinlock_t			q_lock;
	enum qelr_qp_state			state;   /*  QP state */

	struct qelr_qp_hwq_info			sq;
	struct qelr_qp_hwq_info			rq;
	struct {
		uint64_t wr_id;
		enum ibv_wc_opcode opcode;
		uint32_t bytes_len;
		uint8_t wqe_size;
		uint8_t signaled;
	} *wqe_wr_id;

	struct {
		uint64_t wr_id;
		uint8_t wqe_size;
	} *rqe_wr_id;

	uint8_t					prev_wqe_size;
	uint32_t				max_inline_data;
	uint32_t				qp_id;
	int					sq_sig_all;
	int					atomic_supported;
	uint8_t					edpm_disabled;
};

static inline struct qelr_devctx *get_qelr_ctx(struct ibv_context *ibctx)
{
	return container_of(ibctx, struct qelr_devctx, ibv_ctx);
}

static inline struct qelr_device *get_qelr_dev(struct ibv_device *ibdev)
{
	return container_of(ibdev, struct qelr_device, ibv_dev);
}

static inline struct qelr_qp *get_qelr_qp(struct ibv_qp *ibqp)
{
	return container_of(ibqp, struct qelr_qp, ibv_qp);
}

static inline struct qelr_pd *get_qelr_pd(struct ibv_pd *ibpd)
{
	return container_of(ibpd, struct qelr_pd, ibv_pd);
}

static inline struct qelr_cq *get_qelr_cq(struct ibv_cq *ibcq)
{
	return container_of(ibcq, struct qelr_cq, ibv_cq);
}

#define SET_FIELD(value, name, flag)				\
	do {							\
		(value) &= ~(name ## _MASK << name ## _SHIFT);	\
		(value) |= ((flag) << (name ## _SHIFT));	\
	} while (0)

#define SET_FIELD2(value, name, flag)				\
		((value) |= ((flag) << (name ## _SHIFT)))

#define GET_FIELD(value, name) \
	(((value) >> (name ## _SHIFT)) & name ## _MASK)

#define ROCE_WQE_ELEM_SIZE	sizeof(struct rdma_sq_sge)
#define RDMA_WQE_BYTES		(16)

#define QELR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK <<	\
			RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
#define QELR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK <<	\
			RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
#define QELR_RESP_RDMA_IMM (QELR_RESP_IMM | QELR_RESP_RDMA)

#define TYPEPTR_ADDR_SET(type_ptr, field, vaddr)			\
	do {								\
		(type_ptr)->field.hi = htole32(U64_HI(vaddr));	\
		(type_ptr)->field.lo = htole32(U64_LO(vaddr));	\
	} while (0)

#define RQ_SGE_SET(sge, vaddr, vlength, vflags)			\
	do {							\
		TYPEPTR_ADDR_SET(sge, addr, vaddr);		\
		(sge)->length = htole32(vlength);		\
		(sge)->flags = htole32(vflags);		\
	} while (0)

#define U64_HI(val) ((uint32_t)(((uint64_t)(uintptr_t)(val)) >> 32))
#define U64_LO(val) ((uint32_t)(((uint64_t)(uintptr_t)(val)) & 0xffffffff))
#define HILO_U64(hi, lo) ((uintptr_t)((((uint64_t)(hi)) << 32) + (lo)))

#define QELR_MAX_RQ_WQE_SIZE (RDMA_MAX_SGE_PER_RQ_WQE)
#define QELR_MAX_SQ_WQE_SIZE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE /	\
			      ROCE_WQE_ELEM_SIZE)

#endif /* __QELR_H__ */