Mirrors/libhermit
1
0
Fork 0
mirror of https://github.com/hermitcore/libhermit.git synced 2025-03-09 00:00:03 +01:00
Code Issues Releases Wiki Activity

Added a code generator, work in progress. Added ping pong for testing later. Added some more API calls.

Browse source
This commit is contained in:
Annika Wierichs 2017-10-26 15:04:47 +02:00
parent 17f3ec78fc
commit 91ce18c6ec
12 changed files with 3187 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

126
kernel/ibv.c
View file

@ -23,6 +23,9 @@
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* TODO: Documentation
*
*/
@ -38,16 +41,20 @@
// allocate the right amount of memory?
#define MAX_NUM_OF_IBV_DEVICES 16
typedef struct {
int *num_devices;
// Parameters:
int , *num_devices;
// Return value:
struct ibv_device *dev_phys_ptr_list[MAX_NUM_OF_IBV_DEVICES];
// TODO: Can we make the return type struct ibv_device**?
} __attribute__((packed)) uhyve_ibv_get_device_list_t;
struct ibv_device** h_ibv_get_device_list(int *num_devices)
{
struct ibv_device** ibv_get_device_list(int *num_devices) {
// num_devices can be mapped to physical memory right away.
uhyve_ibv_get_device_list_t uhyve_args = {(int*) virt_to_phys((size_t) num_devices)};
uhyve_ibv_get_device_list_t uhyve_args = {
(int*) virt_to_phys((size_t) num_devices)
};
// Allocate memory for return value.
struct ibv_device *devs = kmalloc(MAX_NUM_OF_IBV_DEVICES * sizeof(struct ibv_device));
@ -65,6 +72,95 @@ struct ibv_device** h_ibv_get_device_list(int *num_devices)
return list_virt;
}
const char* ibv_get_device_name(struct ibv_device *device) {
// TODO: Also forward this to uhyve for consistency?
return device->name;
}
typedef struct {
// Parameters:
struct ibv_device *device;
// Return value:
struct ibv_context *ret;
} __attribute__((packed)) uhyve_ibv_open_device_t;
int ibv_open_device(struct ibv_device *device) {
uhyve_ibv_open_device_t uhyve_args = {
(struct ibv_device *) virt_to_phys((size_t) device),
};
uhyve_args->ret = kmalloc(sizeof(struct ibv_context));
uhyve_send(UHYVE_PORT_IBV_OPEN_DEVICE, (unsigned)virt_to_phys((size_t) &uhyve_args));
// Set all pointers contained in returned data structure to valid kernel space addresses.
uhyve_args->ret->device = device;
return uhyve_args.ret;
}
typedef struct {
// Parameters:
struct ibv_context *context;
// Return value:
struct ibv_comp_channel *ret;
} __attribute__((packed)) uhyve_ibv_create_comp_channel_t;
struct ibv_comp_channel* ibv_create_comp_channel(struct ibv_context *context) {
uhyve_ibv_create_comp_channel_t uhyve_args = {
(struct ibv_context *) virt_to_phys((size_t) context),
};
uhyve_args->ret = kmalloc(sizeof(struct ibv_comp_channel));
uhyve_send(UHYVE_PORT_IBV_CREATE_COMP_CHANNEL, (unsigned)virt_to_phys((size_t) &uhyve_args));
// Set all pointers contained in returned data structure to valid kernel space addresses.
uhyve_args->ret->device = device; // todo
return uhyve_args.ret;
}
typedef struct {
// Parameters:
struct ibv_qp *qp;
struct ibv_qp_attr *attr;
int attr_mask;
// Return value:
int ret;
} __attribute__((packed)) uhyve_ibv_modify_qp_t;
int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) {
uhyve_ibv_modify_qp_t uhyve_args = {
(struct ibv_qp *) virt_to_phys((size_t) qp),
(struct ibv_qp_attr *) virt_to_phys((size_t) attr),
attr_mask,
-1
};
uhyve_send(UHYVE_PORT_IBV_MODIFY_QP, (unsigned)virt_to_phys((size_t) &uhyve_args));
return uhyve_args.ret;
}
typedef struct {
// Parameters:
struct ibv_pd *pd;
struct ibv_ah_attr *attr;
// Return value:
struct ibv_ah *ret;
} __attribute__((packed)) uhyve_ibv_create_ah_t;
struct ibv_ah* ibv_create_ah(struct ibv_pd *pd,struct ibv_ah_attr *attr) {
uhyve_ibv_create_ah_t uhyve_args = {
(struct ibv_pd *) virt_to_phys((size_t) pd),
(struct ibv_ah_attr *) virt_to_phys((size_t) attr),
};
uhyve_send(UHYVE_PORT_IBV_CREATE_AH, (unsigned)virt_to_phys((size_t) &uhyve_args));
return uhyve_args.ret;
}
/*struct ibv_device** h_ibv_get_device_list(int *num_devices)*/
/*{*/
/*struct ibv_device *devs = calloc(MAX_NUM_OF_IBV_DEVICES, sizeof(struct ibv_device));*/
@ -85,25 +181,3 @@ struct ibv_device** h_ibv_get_device_list(int *num_devices)
/*uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned)virt_to_phys((size_t)&uhyve_args));*/
/*return list_virt;*/
/*}*/
/*void h_ibv_get_device_list(int *num_devices)*/
/*{*/
/*uhyve_ibv_get_device_list_t uhyve_args = {(int*) virt_to_phys((size_t) num_devices)};*/
/*uhyve_args.first_device = (struct ibv_device*) virt_to_phys((size_t) uhyve_args.devices);*/
/*[>&uhyve_args.devices = (struct ibv_device*) virt_to_phys((size_t) uhyve_args.devices);<]*/
/*uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned)virt_to_phys((size_t)&uhyve_args));*/
/*}*/
typedef struct {
struct ibv_device *device;
char *ret;
} __attribute__((packed)) uhyve_ibv_get_device_name_t;
const char* h_ibv_get_device_name(struct ibv_device *device)
{
uhyve_ibv_get_device_name_t uhyve_args = {(struct ibv_device*) virt_to_phys((size_t) device)};
uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_NAME, (unsigned)virt_to_phys((size_t)&uhyve_args));
return uhyve_args.ret;
}

95
tools/ibv_code_generator/function_prototypes.txt Normal file
View file

@ -0,0 +1,95 @@
const char * ibv_wc_status_str(enum ibv_wc_status status)
int ibv_rate_to_mult(enum ibv_rate rate)
enum ibv_rate mult_to_ibv_rate(int mult)
int ibv_rate_to_mbps(enum ibv_rate rate)
enum ibv_rate mbps_to_ibv_rate(int mbps)
struct ibv_cq * ibv_cq_ex_to_cq(struct ibv_cq_ex * cq)
int ibv_start_poll(struct ibv_cq_ex * cq,struct ibv_poll_cq_attr * attr)
int ibv_next_poll(struct ibv_cq_ex * cq)
void ibv_end_poll(struct ibv_cq_ex * cq)
enum ibv_wc_opcode ibv_wc_read_opcode(struct ibv_cq_ex * cq)
uint32_t ibv_wc_read_vendor_err(struct ibv_cq_ex * cq)
uint32_t ibv_wc_read_byte_len(struct ibv_cq_ex * cq)
__be32 ibv_wc_read_imm_data(struct ibv_cq_ex * cq)
uint32_t ibv_wc_read_invalidated_rkey(struct ibv_cq_ex * cq)
uint32_t ibv_wc_read_qp_num(struct ibv_cq_ex * cq)
uint32_t ibv_wc_read_src_qp(struct ibv_cq_ex * cq)
int ibv_wc_read_wc_flags(struct ibv_cq_ex * cq)
uint32_t ibv_wc_read_slid(struct ibv_cq_ex * cq)
uint8_t ibv_wc_read_sl(struct ibv_cq_ex * cq)
uint8_t ibv_wc_read_dlid_path_bits(struct ibv_cq_ex * cq)
uint64_t ibv_wc_read_completion_ts(struct ibv_cq_ex * cq)
uint16_t ibv_wc_read_cvlan(struct ibv_cq_ex * cq)
uint32_t ibv_wc_read_flow_tag(struct ibv_cq_ex * cq)
int ibv_post_wq_recv(struct ibv_wq * wq,struct ibv_recv_wr * recv_wr,struct ibv_recv_wr ** bad_recv_wr)
struct verbs_context * verbs_get_ctx(struct ibv_context * ctx)
struct ibv_device ** ibv_get_device_list(int * num_devices)
void ibv_free_device_list(struct ibv_device ** list)
const char * ibv_get_device_name(struct ibv_device * device)
__be64 ibv_get_device_guid(struct ibv_device * device)
struct ibv_context * ibv_open_device(struct ibv_device * device)
int ibv_close_device(struct ibv_context * context)
int ibv_get_async_event(struct ibv_context * context,struct ibv_async_event * event)
void ibv_ack_async_event(struct ibv_async_event * event)
int ibv_query_device(struct ibv_context * context,struct ibv_device_attr * device_attr)
int ibv_query_port(struct ibv_context * context,uint8_t port_num,struct ibv_port_attr * port_attr)
int ___ibv_query_port(struct ibv_context * context,uint8_t port_num,struct ibv_port_attr * port_attr)
int ibv_query_gid(struct ibv_context * context,uint8_t port_num,int index,union ibv_gid * gid)
int ibv_query_pkey(struct ibv_context * context,uint8_t port_num,int index,__be16 * pkey)
struct ibv_pd * ibv_alloc_pd(struct ibv_context * context)
int ibv_dealloc_pd(struct ibv_pd * pd)
struct ibv_flow * ibv_create_flow(struct ibv_qp * qp,struct ibv_flow_attr * flow)
int ibv_destroy_flow(struct ibv_flow * flow_id)
struct ibv_xrcd * ibv_open_xrcd(struct ibv_context * context,struct ibv_xrcd_init_attr * xrcd_init_attr)
int ibv_close_xrcd(struct ibv_xrcd * xrcd)
struct ibv_mr * ibv_reg_mr(struct ibv_pd * pd,void * addr,int length,int access)
int ibv_rereg_mr(struct ibv_mr * mr,int flags,struct ibv_pd * pd,void * addr,int length,int access)
int ibv_dereg_mr(struct ibv_mr * mr)
struct ibv_mw * ibv_alloc_mw(struct ibv_pd * pd,enum ibv_mw_type type)
int ibv_dealloc_mw(struct ibv_mw * mw)
uint32_t ibv_inc_rkey(uint32_t rkey)
int ibv_bind_mw(struct ibv_qp * qp,struct ibv_mw * mw,struct ibv_mw_bind * mw_bind)
struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context)
int ibv_destroy_comp_channel(struct ibv_comp_channel * channel)
struct ibv_cq * ibv_create_cq(struct ibv_context * context,int cqe,void * cq_context,struct ibv_comp_channel * channel,int comp_vector)
struct ibv_cq_ex * ibv_create_cq_ex(struct ibv_context * context,struct ibv_cq_init_attr_ex * cq_attr)
int ibv_resize_cq(struct ibv_cq * cq,int cqe)
int ibv_destroy_cq(struct ibv_cq * cq)
int ibv_get_cq_event(struct ibv_comp_channel * channel,struct ibv_cq ** cq,void ** cq_context)
void ibv_ack_cq_events(struct ibv_cq * cq,unsigned int nevents)
int ibv_poll_cq(struct ibv_cq * cq,int num_entries,struct ibv_wc * wc)
int ibv_req_notify_cq(struct ibv_cq * cq,int solicited_only)
struct ibv_srq * ibv_create_srq(struct ibv_pd * pd,struct ibv_srq_init_attr * srq_init_attr)
struct ibv_srq * ibv_create_srq_ex(struct ibv_context * context,struct ibv_srq_init_attr_ex * srq_init_attr_ex)
int ibv_modify_srq(struct ibv_srq * srq,struct ibv_srq_attr * srq_attr,int srq_attr_mask)
int ibv_query_srq(struct ibv_srq * srq,struct ibv_srq_attr * srq_attr)
int ibv_get_srq_num(struct ibv_srq * srq,uint32_t * srq_num)
int ibv_destroy_srq(struct ibv_srq * srq)
int ibv_post_srq_recv(struct ibv_srq * srq,struct ibv_recv_wr * recv_wr,struct ibv_recv_wr ** bad_recv_wr)
struct ibv_qp * ibv_create_qp(struct ibv_pd * pd,struct ibv_qp_init_attr * qp_init_attr)
struct ibv_qp * ibv_create_qp_ex(struct ibv_context * context,struct ibv_qp_init_attr_ex * qp_init_attr_ex)
int ibv_query_rt_values_ex(struct ibv_context * context,struct ibv_values_ex * values)
int ibv_query_device_ex(struct ibv_context * context,const struct ibv_query_device_ex_input * input,struct ibv_device_attr_ex * attr)
struct ibv_qp * ibv_open_qp(struct ibv_context * context,struct ibv_qp_open_attr * qp_open_attr)
int ibv_modify_qp(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask)
int ibv_query_qp(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_qp_init_attr * init_attr)
int ibv_destroy_qp(struct ibv_qp * qp)
struct ibv_wq * ibv_create_wq(struct ibv_context * context,struct ibv_wq_init_attr * wq_init_attr)
int ibv_modify_wq(struct ibv_wq * wq,struct ibv_wq_attr * wq_attr)
int ibv_destroy_wq(struct ibv_wq * wq)
struct ibv_rwq_ind_table * ibv_create_rwq_ind_table(struct ibv_context * context,struct ibv_rwq_ind_table_init_attr * init_attr)
int ibv_destroy_rwq_ind_table(struct ibv_rwq_ind_table * rwq_ind_table)
int ibv_post_send(struct ibv_qp * qp,struct ibv_send_wr * wr,struct ibv_send_wr ** bad_wr)
int ibv_post_recv(struct ibv_qp * qp,struct ibv_recv_wr * wr,struct ibv_recv_wr ** bad_wr)
struct ibv_ah * ibv_create_ah(struct ibv_pd * pd,struct ibv_ah_attr * attr)
int ibv_init_ah_from_wc(struct ibv_context * context,uint8_t port_num,struct ibv_wc * wc,struct ibv_grh * grh,struct ibv_ah_attr * ah_attr)
struct ibv_ah * ibv_create_ah_from_wc(struct ibv_pd * pd,struct ibv_wc * wc,struct ibv_grh * grh,uint8_t port_num)
int ibv_destroy_ah(struct ibv_ah * ah)
int ibv_attach_mcast(struct ibv_qp * qp,const union ibv_gid * gid,uint16_t lid)
int ibv_detach_mcast(struct ibv_qp * qp,const union ibv_gid * gid,uint16_t lid)
int ibv_fork_init()
const char * ibv_node_type_str(enum ibv_node_type node_type)
const char * ibv_port_state_str(enum ibv_port_state port_state)
const char * ibv_event_type_str(enum ibv_event_type event)
int ibv_resolve_eth_l2_from_gid(struct ibv_context * context,struct ibv_ah_attr * attr,uint8_t [6] eth_mac,uint16_t * vid)
int ibv_is_qpt_supported(uint32_t caps,enum ibv_qp_type qpt)

1805
tools/ibv_code_generator/ibv_gen.c Normal file
View file

File diff suppressed because it is too large Load diff

121
tools/ibv_code_generator/kernel_api.py Executable file
View file

@ -0,0 +1,121 @@
#!/usr/bin/env python
"""Copyright (c) 2017, Annika Wierichs, RWTH Aachen University
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the University nor the names of its contributors
may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
TODO: Documentation"""
src_file_path = 'function_prototypes.txt'
ibv_gen_path = 'ibv_gen.c'
uhyve_cases_gen_path = 'uhyve_gen.c'
with open(src_file_path, 'r') as f_src:
with open(ibv_gen_path, 'w') as f_ibv, open(uhyve_cases_gen_path, 'w') as f_uhyve:
# TODO: next: generate hyve stuff.
for line in f_src:
parens_split = line.split('(')
ret_and_name = parens_split[0].split(' ')
function_name = ret_and_name[-1]
ret = ' '.join(ret_and_name[:-1])
all_params = parens_split[-1][:-1]
params = all_params.split(',')
params[-1] = params[-1][:-1]
# Struct
struct_name = 'uhyve_{0}_t'.format(function_name)
f_ibv.write('typedef struct {\n')
if params:
f_ibv.write('\t// Parameters:\n')
for param in params:
f_ibv.write('\t{0};\n'.format(param))
if ret is not "void":
f_ibv.write('\t// Return value:\n')
f_ibv.write('\t{0} ret;\n'.format(ret))
f_ibv.write('}} __attribute__((packed)) {0};\n\n'.format(struct_name))
# Function
params_single_string = ', '.join(params)
f_ibv.write('{0} {1}({2}) {{\n'.format(ret, function_name,
params_single_string))
# Create uhyve_args and define parameters
f_ibv.write('\t{0} uhyve_args;\n'.format(struct_name))
for param in params:
param_split = param.split(' ')
param_type = ' '.join(param_split[:-1])
param_name = param_split[-1]
# Define struct members according to their type.
if '**' in param_type:
f_ibv.write('\t// TODO: Take care of ** parameter.\n')
elif '*' in param_type:
f_ibv.write('\tuhyve_args->{0} = ({1}) virt_to_phys((size_t) {2});\n'
.format(param_name, param_type, param_name))
else:
f_ibv.write('\tuhyve_args->{0} = {0};\n'.format(param_name))
# Allocate memory for return value if it is a pointer.
if '**' in ret:
f_ibv.write('\n\t// TODO: Take care of return value.\n')
elif '*' in ret:
f_ibv.write('\n\tuhyve_args->ret = kmalloc(sizeof({0}));\n'.format(ret[:-2]))
# call uhyve_send() using the respective port ID.
port_name = 'UHYVE_PORT_' + function_name.upper()
f_ibv.write('\n\tuhyve_send({0}, (unsigned) virt_to_phys((size_t) '
'&uhyve_args));\n\n'.format(port_name))
f_ibv.write('\t// TODO: Fix pointers in returned data structures.\n')
f_ibv.write('\treturn uhyve_args.ret;\n')
f_ibv.write('}\n\n\n')
with open(uhyve_cases_gen_path, 'w') as f_ibv:
# typedef struct {
# // Parameters:
# struct ibv_device *device;
# // Return value:
# struct ibv_context *ret;
# } __attribute__((packed)) uhyve_ibv_open_device_t;
# int ibv_open_device(struct ibv_device *device) {
# uhyve_ibv_open_device_t uhyve_args;
# uhyve_args.device = (struct ibv_device *) virt_to_phys((size_t) device);
# uhyve_args->ret = kmalloc(sizeof(struct ibv_context));
# uhyve_send(UHYVE_PORT_IBV_OPEN_DEVICE, (unsigned)virt_to_phys((size_t) &uhyve_args));
# // Set all pointers contained in returned data structure to valid kernel space addresses.
# uhyve_args->ret->device = device;
# return uhyve_args.ret;
# }

0
tools/ibv_code_generator/uhyve_handler.py Normal file
View file

32
tools/uhyve-ibv.h
View file

@ -26,6 +26,15 @@
typedef enum {
UHYVE_PORT_IBV_GET_DEVICE_LIST = 0x510,
UHYVE_PORT_IBV_GET_DEVICE_NAME = 0x511,
UHYVE_PORT_IBV_OPEN_DEVICE = 0x512,
UHYVE_PORT_IBV_CREATE_COMP_CHANNEL = 0x513,
UHYVE_PORT_IBV_ALLOC_PD = 0x514,
UHYVE_PORT_IBV_REG_MR = 0x515,
UHYVE_PORT_IBV_CREATE_CQ = 0x516,
UHYVE_PORT_IBV_CREATE_QP = 0x517,
UHYVE_PORT_IBV_QUERY_QP = 0x518,
UHYVE_PORT_IBV_MODIFY_QP = 0x519,
//UHYVE_PORT_IBV_CREATE_AH = 0x520,
} uhyve_ibv_t;
typedef struct {
@ -38,9 +47,26 @@ typedef struct {
} __attribute__((packed)) uhyve_ibv_get_device_list_t;
typedef struct {
struct ibv_device *device;
char *ret;
} __attribute__((packed)) uhyve_ibv_get_device_name_t;
// Parameters:
struct ibv_qp *qp;
struct ibv_qp_attr *attr;
int attr_mask;
// Return value:
int ret;
} __attribute__((packed)) uhyve_ibv_modify_qp_t;
typedef struct {
// Parameters:
struct ibv_pd *pd;
struct ibv_ah_attr *attr;
// Return value:
struct ibv_ah *ret;
} __attribute__((packed)) uhyve_ibv_create_ah_t;
//typedef struct {
//struct ibv_device *device;
//char *ret;
//} __attribute__((packed)) uhyve_ibv_get_device_name_t;
#endif // UHYVE_IBV_H

58
tools/uhyve.c
View file

@ -979,6 +979,37 @@ static int vcpu_loop(void)
// Copy number of devices to kernel memory
memcpy(guest_mem+(size_t)args->num_devices, &num_devices, sizeof(num_devices));
for (int d = 0; d < num_devices; d++) {
printf("uhyve.c: before memcpy list[d].\n");
// Copy array entry containing ibv_device struct to kernel memory
memcpy(guest_mem + (size_t)args->dev_phys_ptr_list[d], temp_dev_list[d], sizeof(struct ibv_device));
}
break;
}
case UHYVE_PORT_IBV_OPEN_DEVICE: {
unsigned data = *((unsigned*)((size_t)run+run->io.data_offset));
uhyve_ibv_open_device_t* args = (uhyve_ibv_open_device_t*) (guest_mem+data);
struct ibv_context* context = ibv_open_device(guest_mem+(size_t)args->device);
// Copy return value to memory location allocated in kernel space.
memcpy(guest_mem+(size_t)args->ret, context, sizeof(context));
break;
}
case UHYVE_PORT_IBV_MODIFY_QP: {
unsigned data = *((unsigned*)((size_t)run+run->io.data_offset));
uhyve_ibv_get_device_list_t* args = (uhyve_ibv_get_device_list_t*) (guest_mem+data);
// Call IBV function from hypervisor
int num_devices;
struct ibv_device **temp_dev_list = ibv_get_device_list(&num_devices);
// Copy number of devices to kernel memory
memcpy(guest_mem+(size_t)args->num_devices, &num_devices, sizeof(num_devices));
for (int d = 0; d < num_devices; d++) { // TODO switch to num devices
printf("uhyve.c: before memcpy list[d].\n");
// Copy array entry containing ibv_device struct to kernel memory
@ -987,15 +1018,34 @@ static int vcpu_loop(void)
break;
}
case UHYVE_PORT_IBV_GET_DEVICE_NAME: {
case UHYVE_PORT_IBV_CREATE_AH: {
unsigned data = *((unsigned*)((size_t)run+run->io.data_offset));
uhyve_ibv_get_device_name_t* args = (uhyve_ibv_get_device_name_t*) (guest_mem+data);
uhyve_ibv_get_device_list_t* args = (uhyve_ibv_get_device_list_t*) (guest_mem+data);
args->ret = ibv_get_device_name((struct ibv_device*)(guest_mem+(size_t)args->device));
printf("UHYVE_PORT_IBV_GET_DEVICE_NAME\n");
// Call IBV function from hypervisor
int num_devices;
struct ibv_device **temp_dev_list = ibv_get_device_list(&num_devices);
// Copy number of devices to kernel memory
memcpy(guest_mem+(size_t)args->num_devices, &num_devices, sizeof(num_devices));
for (int d = 0; d < num_devices; d++) { // TODO switch to num devices
printf("uhyve.c: before memcpy list[d].\n");
// Copy array entry containing ibv_device struct to kernel memory
memcpy(guest_mem + (size_t)args->dev_phys_ptr_list[d], temp_dev_list[d], sizeof(struct ibv_device));
}
break;
}
/*case UHYVE_PORT_IBV_GET_DEVICE_NAME: {*/
/*unsigned data = *((unsigned*)((size_t)run+run->io.data_offset));*/
/*uhyve_ibv_get_device_name_t* args = (uhyve_ibv_get_device_name_t*) (guest_mem+data);*/
/*args->ret = ibv_get_device_name((struct ibv_device*)(guest_mem+(size_t)args->device));*/
/*printf("UHYVE_PORT_IBV_GET_DEVICE_NAME\n");*/
/*break;*/
/*}*/
default:
err(1, "KVM: unhandled KVM_EXIT_IO at port 0x%x, direction %d\n", run->io.port, run->io.direction);
break;

1
usr/tests/CMakeLists.txt
View file

@ -10,6 +10,7 @@ add_executable(hellof hellof.f90)
add_executable(pi pi.go)
add_executable(ib-test ib_test.c)
#add_executable(ib-pingpong ib/pingpong.c ib/pingpong.h ib/pingpong_ud.c)
#target_link_libraries(ib-test ibverbs)
add_executable(test-malloc test-malloc.c)

81
usr/tests/ib/pingpong.c Normal file
View file

@ -0,0 +1,81 @@
/*
* Copyright (c) 2006 Cisco Systems. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "pingpong.h"
#include <endian.h> // TODO
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
enum ibv_mtu pp_mtu_to_enum(int mtu)
{
switch (mtu) {
case 256: return IBV_MTU_256;
case 512: return IBV_MTU_512;
case 1024: return IBV_MTU_1024;
case 2048: return IBV_MTU_2048;
case 4096: return IBV_MTU_4096;
default: return 0;
}
}
int pp_get_port_info(struct ibv_context *context, int port,
struct ibv_port_attr *attr)
{
return ibv_query_port(context, port, attr);
}
void wire_gid_to_gid(const char *wgid, union ibv_gid *gid)
{
char tmp[9];
__be32 v32;
int i;
uint32_t tmp_gid[4];
for (tmp[8] = 0, i = 0; i < 4; ++i) {
memcpy(tmp, wgid + i * 8, 8);
sscanf(tmp, "%x", &v32);
tmp_gid[i] = be32toh(v32);
}
memcpy(gid, tmp_gid, sizeof(*gid));
}
void gid_to_wire_gid(const union ibv_gid *gid, char wgid[])
{
uint32_t tmp_gid[4];
int i;
memcpy(tmp_gid, gid, sizeof(tmp_gid));
for (i = 0; i < 4; ++i)
sprintf(&wgid[i * 8], "%08x", htobe32(tmp_gid[i]));
}

38
usr/tests/ib/pingpong.h Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright (c) 2006 Cisco Systems. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <hermit/ibv.h>
enum ibv_mtu pp_mtu_to_enum(int mtu);
int pp_get_port_info(struct ibv_context *context, int port, struct ibv_port_attr *attr);
void wire_gid_to_gid(const char *wgid, union ibv_gid *gid);
void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]);

861
usr/tests/ib/pingpong_ud.c Normal file
View file

@ -0,0 +1,861 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#define _GNU_SOURCE
#include <config.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>
#include "pingpong.h"
enum {
PINGPONG_RECV_WRID = 1,
PINGPONG_SEND_WRID = 2,
};
static int page_size;
struct pingpong_context {
struct ibv_context *context;
struct ibv_comp_channel *channel;
struct ibv_pd *pd;
struct ibv_mr *mr;
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_ah *ah;
void *buf;
int size;
int send_flags;
int rx_depth;
int pending;
struct ibv_port_attr portinfo;
};
struct pingpong_dest {
int lid;
int qpn;
int psn;
union ibv_gid gid;
};
static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
int sl, struct pingpong_dest *dest, int sgid_idx)
{
struct ibv_ah_attr ah_attr = {
.is_global = 0,
.dlid = dest->lid,
.sl = sl,
.src_path_bits = 0,
.port_num = port
};
struct ibv_qp_attr attr = {
.qp_state = IBV_QPS_RTR
};
if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE)) {
fprintf(stderr, "Failed to modify QP to RTR\n");
return 1;
}
attr.qp_state = IBV_QPS_RTS;
attr.sq_psn = my_psn;
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_SQ_PSN)) {
fprintf(stderr, "Failed to modify QP to RTS\n");
return 1;
}
if (dest->gid.global.interface_id) {
ah_attr.is_global = 1;
ah_attr.grh.hop_limit = 1;
ah_attr.grh.dgid = dest->gid;
ah_attr.grh.sgid_index = sgid_idx;
}
ctx->ah = ibv_create_ah(ctx->pd, &ah_attr);
if (!ctx->ah) {
fprintf(stderr, "Failed to create AH\n");
return 1;
}
return 0;
}
static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
const struct pingpong_dest *my_dest)
{
struct addrinfo *res, *t;
struct addrinfo hints = {
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM
};
char *service;
char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int sockfd = -1;
struct pingpong_dest *rem_dest = NULL;
char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
n = getaddrinfo(servername, service, &hints, &res);
if (n < 0) {
fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
free(service);
return NULL;
}
for (t = res; t; t = t->ai_next) {
sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
if (sockfd >= 0) {
if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
break;
close(sockfd);
sockfd = -1;
}
}
freeaddrinfo(res);
free(service);
if (sockfd < 0) {
fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
return NULL;
}
gid_to_wire_gid(&my_dest->gid, gid);
sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
my_dest->psn, gid);
if (write(sockfd, msg, sizeof msg) != sizeof msg) {
fprintf(stderr, "Couldn't send local address\n");
goto out;
}
if (read(sockfd, msg, sizeof msg) != sizeof msg ||
write(sockfd, "done", sizeof "done") != sizeof "done") {
perror("client read/write");
fprintf(stderr, "Couldn't read/write remote address\n");
goto out;
}
rem_dest = malloc(sizeof *rem_dest);
if (!rem_dest)
goto out;
sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
&rem_dest->psn, gid);
wire_gid_to_gid(gid, &rem_dest->gid);
out:
close(sockfd);
return rem_dest;
}
static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
int ib_port, int port, int sl,
const struct pingpong_dest *my_dest,
int sgid_idx)
{
struct addrinfo *res, *t;
struct addrinfo hints = {
.ai_flags = AI_PASSIVE,
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM
};
char *service;
char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
int n;
int sockfd = -1, connfd;
struct pingpong_dest *rem_dest = NULL;
char gid[33];
if (asprintf(&service, "%d", port) < 0)
return NULL;
n = getaddrinfo(NULL, service, &hints, &res);
if (n < 0) {
fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
free(service);
return NULL;
}
for (t = res; t; t = t->ai_next) {
sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
if (sockfd >= 0) {
n = 1;
setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
break;
close(sockfd);
sockfd = -1;
}
}
freeaddrinfo(res);
free(service);
if (sockfd < 0) {
fprintf(stderr, "Couldn't listen to port %d\n", port);
return NULL;
}
listen(sockfd, 1);
connfd = accept(sockfd, NULL, NULL);
close(sockfd);
if (connfd < 0) {
fprintf(stderr, "accept() failed\n");
return NULL;
}
n = read(connfd, msg, sizeof msg);
if (n != sizeof msg) {
perror("server read");
fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
goto out;
}
rem_dest = malloc(sizeof *rem_dest);
if (!rem_dest)
goto out;
sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
&rem_dest->psn, gid);
wire_gid_to_gid(gid, &rem_dest->gid);
if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest,
sgid_idx)) {
fprintf(stderr, "Couldn't connect to remote QP\n");
free(rem_dest);
rem_dest = NULL;
goto out;
}
gid_to_wire_gid(&my_dest->gid, gid);
sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
my_dest->psn, gid);
if (write(connfd, msg, sizeof msg) != sizeof msg ||
read(connfd, msg, sizeof msg) != sizeof "done") {
fprintf(stderr, "Couldn't send/recv local address\n");
free(rem_dest);
rem_dest = NULL;
goto out;
}
out:
close(connfd);
return rem_dest;
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
int rx_depth, int port,
int use_event)
{
struct pingpong_context *ctx;
ctx = malloc(sizeof *ctx);
if (!ctx)
return NULL;
ctx->size = size;
ctx->send_flags = IBV_SEND_SIGNALED;
ctx->rx_depth = rx_depth;
ctx->buf = memalign(page_size, size + 40);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
goto clean_ctx;
}
/* FIXME memset(ctx->buf, 0, size + 40); */
memset(ctx->buf, 0x7b, size + 40);
ctx->context = ibv_open_device(ib_dev);
if (!ctx->context) {
fprintf(stderr, "Couldn't get context for %s\n",
ibv_get_device_name(ib_dev));
goto clean_buffer;
}
{
struct ibv_port_attr port_info = {};
int mtu;
if (ibv_query_port(ctx->context, port, &port_info)) {
fprintf(stderr, "Unable to query port info for port %d\n", port);
goto clean_device;
}
mtu = 1 << (port_info.active_mtu + 7);
if (size > mtu) {
fprintf(stderr, "Requested size larger than port MTU (%d)\n", mtu);
goto clean_device;
}
}
if (use_event) {
ctx->channel = ibv_create_comp_channel(ctx->context);
if (!ctx->channel) {
fprintf(stderr, "Couldn't create completion channel\n");
goto clean_device;
}
} else
ctx->channel = NULL;
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
goto clean_comp_channel;
}
ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size + 40, IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "Couldn't register MR\n");
goto clean_pd;
}
ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
ctx->channel, 0);
if (!ctx->cq) {
fprintf(stderr, "Couldn't create CQ\n");
goto clean_mr;
}
{
struct ibv_qp_attr attr;
struct ibv_qp_init_attr init_attr = {
.send_cq = ctx->cq,
.recv_cq = ctx->cq,
.cap = {
.max_send_wr = 1,
.max_recv_wr = rx_depth,
.max_send_sge = 1,
.max_recv_sge = 1
},
.qp_type = IBV_QPT_UD,
};
ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
if (!ctx->qp) {
fprintf(stderr, "Couldn't create QP\n");
goto clean_cq;
}
ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
if (init_attr.cap.max_inline_data >= size) {
ctx->send_flags |= IBV_SEND_INLINE;
}
}
{
struct ibv_qp_attr attr = {
.qp_state = IBV_QPS_INIT,
.pkey_index = 0,
.port_num = port,
.qkey = 0x11111111
};
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_QKEY)) {
fprintf(stderr, "Failed to modify QP to INIT\n");
goto clean_qp;
}
}
return ctx;
clean_qp:
ibv_destroy_qp(ctx->qp);
clean_cq:
ibv_destroy_cq(ctx->cq);
clean_mr:
ibv_dereg_mr(ctx->mr);
clean_pd:
ibv_dealloc_pd(ctx->pd);
clean_comp_channel:
if (ctx->channel)
ibv_destroy_comp_channel(ctx->channel);
clean_device:
ibv_close_device(ctx->context);
clean_buffer:
free(ctx->buf);
clean_ctx:
free(ctx);
return NULL;
}
static int pp_close_ctx(struct pingpong_context *ctx)
{
if (ibv_destroy_qp(ctx->qp)) {
fprintf(stderr, "Couldn't destroy QP\n");
return 1;
}
if (ibv_destroy_cq(ctx->cq)) {
fprintf(stderr, "Couldn't destroy CQ\n");
return 1;
}
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "Couldn't deregister MR\n");
return 1;
}
if (ibv_destroy_ah(ctx->ah)) {
fprintf(stderr, "Couldn't destroy AH\n");
return 1;
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "Couldn't deallocate PD\n");
return 1;
}
if (ctx->channel) {
if (ibv_destroy_comp_channel(ctx->channel)) {
fprintf(stderr, "Couldn't destroy completion channel\n");
return 1;
}
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "Couldn't release context\n");
return 1;
}
free(ctx->buf);
free(ctx);
return 0;
}
static int pp_post_recv(struct pingpong_context *ctx, int n)
{
struct ibv_sge list = {
.addr = (uintptr_t) ctx->buf,
.length = ctx->size + 40,
.lkey = ctx->mr->lkey
};
struct ibv_recv_wr wr = {
.wr_id = PINGPONG_RECV_WRID,
.sg_list = &list,
.num_sge = 1,
};
struct ibv_recv_wr *bad_wr;
int i;
for (i = 0; i < n; ++i)
if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
break;
return i;
}
static int pp_post_send(struct pingpong_context *ctx, uint32_t qpn)
{
struct ibv_sge list = {
.addr = (uintptr_t) ctx->buf + 40,
.length = ctx->size,
.lkey = ctx->mr->lkey
};
struct ibv_send_wr wr = {
.wr_id = PINGPONG_SEND_WRID,
.sg_list = &list,
.num_sge = 1,
.opcode = IBV_WR_SEND,
.send_flags = ctx->send_flags,
.wr = {
.ud = {
.ah = ctx->ah,
.remote_qpn = qpn,
.remote_qkey = 0x11111111
}
}
};
struct ibv_send_wr *bad_wr;
return ibv_post_send(ctx->qp, &wr, &bad_wr);
}
static void usage(const char *argv0)
{
printf("Usage:\n");
printf(" %s start a server and wait for connection\n", argv0);
printf(" %s <host> connect to server at <host>\n", argv0);
printf("\n");
printf("Options:\n");
printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
printf(" -s, --size=<size> size of message to exchange (default 2048)\n");
printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
printf(" -l, --sl=<SL> send messages with service level <SL> (default 0)\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
printf(" -g, --gid-idx=<gid index> local port gid index\n");
}
int main(int argc, char *argv[])
{
struct ibv_device **dev_list;
struct ibv_device *ib_dev;
struct pingpong_context *ctx;
struct pingpong_dest my_dest;
struct pingpong_dest *rem_dest;
struct timeval start, end;
char *ib_devname = NULL;
char *servername = NULL;
unsigned int port = 18515;
int ib_port = 1;
unsigned int size = 2048;
unsigned int rx_depth = 500;
unsigned int iters = 1000;
int use_event = 0;
int routs;
int rcnt, scnt;
int num_cq_events = 0;
int sl = 0;
int gidx = -1;
char gid[33];
srand48(getpid() * time(NULL));
while (1) {
int c;
static struct option long_options[] = {
{ .name = "port", .has_arg = 1, .val = 'p' },
{ .name = "ib-dev", .has_arg = 1, .val = 'd' },
{ .name = "ib-port", .has_arg = 1, .val = 'i' },
{ .name = "size", .has_arg = 1, .val = 's' },
{ .name = "rx-depth", .has_arg = 1, .val = 'r' },
{ .name = "iters", .has_arg = 1, .val = 'n' },
{ .name = "sl", .has_arg = 1, .val = 'l' },
{ .name = "events", .has_arg = 0, .val = 'e' },
{ .name = "gid-idx", .has_arg = 1, .val = 'g' },
{}
};
c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:",
long_options, NULL);
if (c == -1)
break;
switch (c) {
case 'p':
port = strtol(optarg, NULL, 0);
if (port > 65535) {
usage(argv[0]);
return 1;
}
break;
case 'd':
ib_devname = strdupa(optarg);
break;
case 'i':
ib_port = strtol(optarg, NULL, 0);
if (ib_port < 1) {
usage(argv[0]);
return 1;
}
break;
case 's':
size = strtoul(optarg, NULL, 0);
break;
case 'r':
rx_depth = strtoul(optarg, NULL, 0);
break;
case 'n':
iters = strtoul(optarg, NULL, 0);
break;
case 'l':
sl = strtol(optarg, NULL, 0);
break;
case 'e':
++use_event;
break;
case 'g':
gidx = strtol(optarg, NULL, 0);
break;
default:
usage(argv[0]);
return 1;
}
}
if (optind == argc - 1)
servername = strdupa(argv[optind]);
else if (optind < argc) {
usage(argv[0]);
return 1;
}
page_size = sysconf(_SC_PAGESIZE);
dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
perror("Failed to get IB devices list");
return 1;
}
if (!ib_devname) {
ib_dev = *dev_list;
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
} else {
int i;
for (i = 0; dev_list[i]; ++i)
if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
break;
ib_dev = dev_list[i];
if (!ib_dev) {
fprintf(stderr, "IB device %s not found\n", ib_devname);
return 1;
}
}
ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event);
if (!ctx)
return 1;
routs = pp_post_recv(ctx, ctx->rx_depth);
if (routs < ctx->rx_depth) {
fprintf(stderr, "Couldn't post receive (%d)\n", routs);
return 1;
}
if (use_event)
if (ibv_req_notify_cq(ctx->cq, 0)) {
fprintf(stderr, "Couldn't request CQ notification\n");
return 1;
}
if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
fprintf(stderr, "Couldn't get port info\n");
return 1;
}
my_dest.lid = ctx->portinfo.lid;
my_dest.qpn = ctx->qp->qp_num;
my_dest.psn = lrand48() & 0xffffff;
if (gidx >= 0) {
if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
fprintf(stderr, "Could not get local gid for gid index "
"%d\n", gidx);
return 1;
}
} else
memset(&my_dest.gid, 0, sizeof my_dest.gid);
inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid);
printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n",
my_dest.lid, my_dest.qpn, my_dest.psn, gid);
if (servername)
rem_dest = pp_client_exch_dest(servername, port, &my_dest);
else
rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl,
&my_dest, gidx);
if (!rem_dest)
return 1;
inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid);
printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid);
if (servername)
if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest,
gidx))
return 1;
ctx->pending = PINGPONG_RECV_WRID;
if (servername) {
if (pp_post_send(ctx, rem_dest->qpn)) {
fprintf(stderr, "Couldn't post send\n");
return 1;
}
ctx->pending |= PINGPONG_SEND_WRID;
}
if (gettimeofday(&start, NULL)) {
perror("gettimeofday");
return 1;
}
rcnt = scnt = 0;
while (rcnt < iters || scnt < iters) {
if (use_event) {
struct ibv_cq *ev_cq;
void *ev_ctx;
if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) {
fprintf(stderr, "Failed to get cq_event\n");
return 1;
}
++num_cq_events;
if (ev_cq != ctx->cq) {
fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
return 1;
}
if (ibv_req_notify_cq(ctx->cq, 0)) {
fprintf(stderr, "Couldn't request CQ notification\n");
return 1;
}
}
{
struct ibv_wc wc[2];
int ne, i;
do {
ne = ibv_poll_cq(ctx->cq, 2, wc);
if (ne < 0) {
fprintf(stderr, "poll CQ failed %d\n", ne);
return 1;
}
} while (!use_event && ne < 1);
for (i = 0; i < ne; ++i) {
if (wc[i].status != IBV_WC_SUCCESS) {
fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
ibv_wc_status_str(wc[i].status),
wc[i].status, (int) wc[i].wr_id);
return 1;
}
switch ((int) wc[i].wr_id) {
case PINGPONG_SEND_WRID:
++scnt;
break;
case PINGPONG_RECV_WRID:
if (--routs <= 1) {
routs += pp_post_recv(ctx, ctx->rx_depth - routs);
if (routs < ctx->rx_depth) {
fprintf(stderr,
"Couldn't post receive (%d)\n",
routs);
return 1;
}
}
++rcnt;
break;
default:
fprintf(stderr, "Completion for unknown wr_id %d\n",
(int) wc[i].wr_id);
return 1;
}
ctx->pending &= ~(int) wc[i].wr_id;
if (scnt < iters && !ctx->pending) {
if (pp_post_send(ctx, rem_dest->qpn)) {
fprintf(stderr, "Couldn't post send\n");
return 1;
}
ctx->pending = PINGPONG_RECV_WRID |
PINGPONG_SEND_WRID;
}
}
}
}
if (gettimeofday(&end, NULL)) {
perror("gettimeofday");
return 1;
}
{
float usec = (end.tv_sec - start.tv_sec) * 1000000 +
(end.tv_usec - start.tv_usec);
long long bytes = (long long) size * iters * 2;
printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
bytes, usec / 1000000., bytes * 8. / usec);
printf("%d iters in %.2f seconds = %.2f usec/iter\n",
iters, usec / 1000000., usec / iters);
}
ibv_ack_cq_events(ctx->cq, num_cq_events);
if (pp_close_ctx(ctx))
return 1;
ibv_free_device_list(dev_list);
free(rem_dest);
return 0;
}

4
usr/tests/ib_test.c
View file

@ -52,7 +52,7 @@ int main(int argc, char** argv)
int num_devices;
printf("ib_test.c: before get dev list.\n");
dev_list = h_ibv_get_device_list(&num_devices);
dev_list = ibv_get_device_list(&num_devices);
printf("ib_test.c: after get dev list.\n");
printf("ib_test.c: num devices: %d\n", num_devices);
@ -67,7 +67,7 @@ int main(int argc, char** argv)
/*}*/
/*printf("after dev list check.\n");*/
/*const char* dev_name = h_ibv_get_device_name(dev_list[0]);*/
/*const char* dev_name = ibv_get_device_name(dev_list[0]);*/
/*printf("after get device name.\n");*/
/*if (!dev_name) {*/