diff --git a/include/hermit/stddef.h b/include/hermit/stddef.h index 7f0893235..7e6c190b6 100644 --- a/include/hermit/stddef.h +++ b/include/hermit/stddef.h @@ -68,7 +68,7 @@ extern const size_t image_size; // InfiniBand uhyve port IDs #define UHYVE_PORT_IBV_OPEN_DEVICE 0x510 -//#define UHYVE_PORT_IBV_GET_DEVICE_LIST 0x511 +#define UHYVE_PORT_IBV_GET_DEVICE_LIST 0x511 #define UHYVE_PORT_IBV_GET_DEVICE_NAME 0x512 #define UHYVE_PORT_IBV_QUERY_PORT 0x513 #define UHYVE_PORT_IBV_CREATE_COMP_CHANNEL 0x514 diff --git a/kernel/ibv.c b/kernel/ibv.c index c08b06e73..da5a07b2a 100644 --- a/kernel/ibv.c +++ b/kernel/ibv.c @@ -156,33 +156,34 @@ struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context) } +/* + * ibv_get_device_list + */ -/*typedef struct { // CHECKED*/ - /*// Parameters:*/ - /*int , *num_devices;*/ - /*// Return value:*/ - /*struct ibv_device *dev_phys_ptr_list[MAX_NUM_OF_IBV_DEVICES];*/ - /*// TODO: Can we make the return type struct ibv_device**?*/ -/*} __attribute__((packed)) uhyve_ibv_get_device_list_t;*/ +typedef struct { + // Parameters: + int * num_devices; + // Return value: + struct ibv_device * ret[MAX_NUM_OF_IBV_DEVICES]; +} __attribute__((packed)) uhyve_ibv_get_device_list_t; -/*struct ibv_device** ibv_get_device_list(int *num_devices) {*/ - /*// num_devices can be mapped to physical memory right away.*/ - /*uhyve_ibv_get_device_list_t uhyve_args = {*/ - /*(int*) virt_to_phys((size_t) num_devices)*/ - /*};*/ +struct ibv_device ** ibv_get_device_list(int * num_devices) { + // num_devices can be mapped to physical memory right away. + uhyve_ibv_get_device_list_t uhyve_args; + uhyve_args.num_devices = (int *) guest_to_host((size_t) num_devices); - /*// Allocate memory for return value.*/ - /*struct ibv_device *devs = kmalloc(MAX_NUM_OF_IBV_DEVICES * sizeof(struct ibv_device));*/ - /*struct ibv_device **list_virt = kmalloc(MAX_NUM_OF_IBV_DEVICES * sizeof(struct ibv_device *));*/ + // Allocate memory for return value. + struct ibv_device * devs = kmalloc(MAX_NUM_OF_IBV_DEVICES * sizeof(struct ibv_device)); + struct ibv_device ** ret_guest_ptr = kmalloc(MAX_NUM_OF_IBV_DEVICES * sizeof(struct ibv_device *)); - /*// We keep a list of the virtual addresses, so we can return it later, and map*/ - /*// to physical addresses for the args struct passed to uhyve.*/ - /*for (int i = 0; i < MAX_NUM_OF_IBV_DEVICES; i++) {*/ - /*struct ibv_device* device_address = devs + i;*/ - /*list_virt[i] = device_address;*/ - /*uhyve_args.dev_phys_ptr_list[i] = (struct ibv_device*) virt_to_phys((size_t) device_address);*/ - /*}*/ + // We keep a list of the virtual addresses, so we can return it later, and map + // to physical addresses for the args struct passed to uhyve. + for (int i = 0; i < MAX_NUM_OF_IBV_DEVICES; i++) { + struct ibv_device* device_address = devs + i; + ret_guest_ptr[i] = device_address; + uhyve_args.ret[i] = (struct ibv_device *) guest_to_host((size_t) device_address); + } - /*uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned)virt_to_phys((size_t)&uhyve_args));*/ - /*return list_virt;*/ -/*}*/ + uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned) guest_to_host((size_t) &uhyve_args)); + return ret_guest_ptr; +} diff --git a/tools/uhyve-ibv-guest-host.c b/tools/uhyve-ibv-guest-host.c deleted file mode 100644 index 6350a9b23..000000000 --- a/tools/uhyve-ibv-guest-host.c +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2017, Annika Wierichs, RWTH Aachen University - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * TODO: Documentation - * - */ - - -#include "uhyve-ibv-guest-host.h" - - -struct ibv_context * guest_to_host_ibv_context(struct ibv_context * context) { - ibv_context_virt_ptrs.device = context->device, - ibv_context_virt_ptrs.abi_compat = context->abi_compat, - - context->device = guest_to_host_ibv_device(context->device); - context->abi_compat = guest_to_host_ibv_abi_compat_v2(context->abi_compat); - guest_to_host_ibv_context_ops(&context->ops); - /*guest_to_host_pthread_mutex_t(&context->mutex); // TODO*/ - - return (struct ibv_context *) guest_to_host((size_t) context); -} - -void phys_to_virt_ibv_context(struct ibv_context * context) { - context->device = ibv_context_virt_ptrs.device; - context->abi_compat = ibv_context_virt_ptrs.abi_compat; - - phys_to_virt_ibv_device(context->device); - phys_to_virt_ibv_abi_compat_v2(context->abi_compat); - phys_to_virt_ibv_context_ops(&context->ops); -} diff --git a/tools/uhyve-ibv-guest-host.h b/tools/uhyve-ibv-guest-host.h deleted file mode 100644 index 54da8ba9c..000000000 --- a/tools/uhyve-ibv-guest-host.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2017, Annika Wierichs, RWTH Aachen University - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * TODO: Documentation - * - */ - - -#ifndef UHYVE_IBV_GUEST_HOST_H -#define UHYVE_IBV_GUEST_HOST_H - -#include // Linux include - - - - - -#endif // UHYVE_IBV_GUEST_HOST_H diff --git a/tools/uhyve-ibv.c b/tools/uhyve-ibv.c index 0a2616f59..1544003c7 100644 --- a/tools/uhyve-ibv.c +++ b/tools/uhyve-ibv.c @@ -33,10 +33,10 @@ #include // Linux include -struct ibv_context * ibv_open_device(struct ibv_device * device); -const char* ibv_get_device_name(struct ibv_device *device); -int ibv_query_port(struct ibv_context * context, uint8_t port_num, struct ibv_port_attr * port_attr); -struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context); +// struct ibv_context * ibv_open_device(struct ibv_device * device); +// const char* ibv_get_device_name(struct ibv_device *device); +// int ibv_query_port(struct ibv_context * context, uint8_t port_num, struct ibv_port_attr * port_attr); +// struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context); /* @@ -45,10 +45,10 @@ struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context); void call_ibv_open_device(struct kvm_run * run) { unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_open_device_t * args = (uhyve_ibv_open_device_t *) (guest_mem + data); + uhyve_ibv_open_device_t * args = (uhyve_ibv_open_device_t *) data; - struct ibv_context * host_ret = ibv_open_device(guest_mem+(size_t)args->device); - memcpy(guest_mem+(size_t)args->ret, host_ret, sizeof(host_ret)); + struct ibv_context * host_ret = ibv_open_device(args->device); + memcpy(args->ret, host_ret, sizeof(host_ret)); // TODO: Convert ptrs contained in return value. free(host_ret); } @@ -60,11 +60,11 @@ void call_ibv_open_device(struct kvm_run * run) { void call_ibv_get_device_name(struct kvm_run * run) { unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_get_device_name_t * args = (uhyve_ibv_get_device_name_t *) (guest_mem + data); + uhyve_ibv_get_device_name_t * args = (uhyve_ibv_get_device_name_t *) data; // TODO: Tricky because char ptr isn't allocated in called function. - const char * host_ret = ibv_get_device_name(guest_mem+(size_t)args->device); - memcpy(guest_mem+(size_t)args->ret, host_ret, sizeof(host_ret)); + const char * host_ret = ibv_get_device_name(args->device); + memcpy(args->ret, host_ret, sizeof(host_ret)); // TODO: Convert ptrs contained in return value. // TODO: How to tell if ret needs to be deleted? } @@ -76,9 +76,9 @@ void call_ibv_get_device_name(struct kvm_run * run) { void call_ibv_query_port(struct kvm_run * run) { unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_query_port_t * args = (uhyve_ibv_query_port_t *) (guest_mem + data); + uhyve_ibv_query_port_t * args = (uhyve_ibv_query_port_t *) (data); - int host_ret = ibv_query_port(guest_mem+(size_t)args->context, port_num, guest_mem+(size_t)args->port_attr); + int host_ret = ibv_query_port(args->context, args->port_num, args->port_attr); args->ret = host_ret; } @@ -88,7 +88,9 @@ void call_ibv_query_port(struct kvm_run * run) { */ void call_ibv_create_comp_channel(struct kvm_run * run) { - uhyve_ibv_create_comp_channel_t * args = (uhyve_ibv_create_comp_channel_t *) get_data(run); + unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); + uhyve_ibv_create_comp_channel_t * args = (uhyve_ibv_create_comp_channel_t *) data; + /*uhyve_ibv_create_comp_channel_t * args = (uhyve_ibv_create_comp_channel_t *) get_data(run);*/ struct ibv_comp_channel * host_ret = ibv_create_comp_channel(args->context); memcpy(args->ret, host_ret, sizeof(host_ret)); // TODO: This will only work for ABI ver > 2. @@ -96,20 +98,22 @@ void call_ibv_create_comp_channel(struct kvm_run * run) { } -/*void call_ibv_get_device_list(struct kvm_run * run, uint8_t * guest_mem) {*/ - /*unsigned data = *((unsigned*)((size_t)run+run->io.data_offset));*/ - /*uhyve_ibv_get_device_list_t* args = (uhyve_ibv_get_device_list_t*) (guest_mem+data);*/ +void call_ibv_get_device_list(struct kvm_run * run) { + unsigned data = *((unsigned *)((size_t)run+run->io.data_offset)); + uhyve_ibv_get_device_list_t * args = (uhyve_ibv_get_device_list_t *) data; - /*// Call IBV function from hypervisor*/ - /*int num_devices;*/ - /*struct ibv_device **temp_dev_list = ibv_get_device_list(&num_devices);*/ + // Call IBV function from hypervisor + int num_devices; + struct ibv_device **host_ret = ibv_get_device_list(&num_devices); - /*// Copy number of devices to kernel memory */ - /*memcpy(guest_mem+(size_t)args->num_devices, &num_devices, sizeof(num_devices));*/ + // Copy number of devices to kernel memory + if (args->num_devices) { + memcpy(args->num_devices, &num_devices, sizeof(num_devices)); + } - /*for (int d = 0; d < num_devices; d++) {*/ + for (int d = 0; d < num_devices; d++) { /*printf("uhyve.c: before memcpy list[d].\n");*/ - /*// Copy array entry containing ibv_device struct to kernel memory*/ - /*memcpy(guest_mem + (size_t)args->dev_phys_ptr_list[d], temp_dev_list[d], sizeof(struct ibv_device));*/ - /*}*/ -/*}*/ + // Copy array entry containing ibv_device struct to kernel memory + memcpy(args->ret[d], host_ret[d], sizeof(struct ibv_device)); + } +} diff --git a/tools/uhyve-ibv.h b/tools/uhyve-ibv.h index ab49df036..0a6836221 100644 --- a/tools/uhyve-ibv.h +++ b/tools/uhyve-ibv.h @@ -20,23 +20,31 @@ #define UHYVE_IBV_H #include // Linux include +#include #define MAX_NUM_OF_IBV_DEVICES 16 typedef enum { UHYVE_PORT_IBV_OPEN_DEVICE = 0x510, - //UHYVE_PORT_IBV_GET_DEVICE_LIST = 0x511, + UHYVE_PORT_IBV_GET_DEVICE_LIST = 0x511, UHYVE_PORT_IBV_GET_DEVICE_NAME = 0x512, UHYVE_PORT_IBV_QUERY_PORT = 0x513, UHYVE_PORT_IBV_CREATE_COMP_CHANNEL = 0x514, } uhyve_ibv_t; -inline unsigned get_data(struct kvm_run * run) { - return *((unsigned*)((size_t)run+run->io.data_offset)); -} +//inline unsigned get_data(struct kvm_run * run) { + //return *((unsigned*)((size_t)run+run->io.data_offset)); +//} +typedef struct { + // Parameters: + int * num_devices; + // Return value: + struct ibv_device * ret[MAX_NUM_OF_IBV_DEVICES]; +} __attribute__((packed)) uhyve_ibv_get_device_list_t; + typedef struct { // Parameters: struct ibv_device * device; @@ -68,16 +76,6 @@ typedef struct { } __attribute__((packed)) uhyve_ibv_create_comp_channel_t; -//typedef struct { // CHECKED - //// In: - //int *num_devices; - //// Out: - ////struct ibv_device devices[MAX_NUM_OF_IBV_DEVICES]; - //struct ibv_device *dev_phys_ptr_list[MAX_NUM_OF_IBV_DEVICES]; - ////struct ibv_device **device_list; -//} __attribute__((packed)) uhyve_ibv_get_device_list_t; - - void call_ibv_open_device(struct kvm_run * run); void call_ibv_get_device_name(struct kvm_run * run); void call_ibv_query_port(struct kvm_run * run); diff --git a/usr/tests/CMakeLists.txt b/usr/tests/CMakeLists.txt index 3d3e50739..f3b523965 100644 --- a/usr/tests/CMakeLists.txt +++ b/usr/tests/CMakeLists.txt @@ -10,7 +10,8 @@ add_executable(hellof hellof.f90) add_executable(pi pi.go) #add_executable(ib-test ib_test.c) -add_executable(ib-pingpong ib/pingpong.c ib/pingpong.h ib/pingpong_ud.c) +#add_executable(ib-pingpong ib/pingpong.c ib/pingpong.h ib/pingpong_ud.c) +add_executable(ib-pingpong ib/pingpong.c ib/pingpong_ud.c) #target_link_libraries(ib-test ibverbs) add_executable(test-malloc test-malloc.c) diff --git a/usr/tests/ib/pingpong.c b/usr/tests/ib/pingpong.c index f47be7334..aab8e4c5d 100644 --- a/usr/tests/ib/pingpong.c +++ b/usr/tests/ib/pingpong.c @@ -32,7 +32,7 @@ #include "pingpong.h" -#include // TODO +/*#include // TODO*/ #include #include #include diff --git a/usr/tests/ib/pingpong.h b/usr/tests/ib/pingpong.h index 2f5b3eea9..1cdd81ff2 100644 --- a/usr/tests/ib/pingpong.h +++ b/usr/tests/ib/pingpong.h @@ -31,6 +31,7 @@ */ #include +#include enum ibv_mtu pp_mtu_to_enum(int mtu); int pp_get_port_info(struct ibv_context *context, int port, struct ibv_port_attr *attr); diff --git a/usr/tests/ib/pingpong_ud.c b/usr/tests/ib/pingpong_ud.c index 0002b59b6..f54e849a0 100644 --- a/usr/tests/ib/pingpong_ud.c +++ b/usr/tests/ib/pingpong_ud.c @@ -31,7 +31,20 @@ */ #define _GNU_SOURCE -#include +/*#include */ + +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include #include #include @@ -43,7 +56,8 @@ #include #include #include -#include +/*#include */ +#include #include #include "pingpong.h"