diff --git a/arch/x86/mm/memory.c b/arch/x86/mm/memory.c index 697c2b6ce..7e9f10d2a 100644 --- a/arch/x86/mm/memory.c +++ b/arch/x86/mm/memory.c @@ -54,6 +54,8 @@ typedef struct free_list { */ extern const void kernel_start; +uint8_t * host_kernel_start = NULL; + static spinlock_t list_lock = SPINLOCK_INIT; static free_list_t init_list = {0, 0, NULL, NULL}; @@ -248,6 +250,9 @@ int memory_init(void) { int ret = 0; + // guest_mem_workaround + uhyve_send(UHYVE_PORT_KERNEL_START, (unsigned) guest_to_host((size_t) &host_kernel_start)); + // enable paging and map Multiboot modules etc. ret = page_init(); if (BUILTIN_EXPECT(ret, 0)) { diff --git a/include/hermit/ibv_virt_phys.h b/include/hermit/ibv_virt_phys.h index 0bb05a59e..1b4cd0b1b 100644 --- a/include/hermit/ibv_virt_phys.h +++ b/include/hermit/ibv_virt_phys.h @@ -31,16 +31,23 @@ #include +extern uint8_t * host_kernel_start; -struct ibv_device * virt_to_phys_ibv_device(struct ibv_device * device); -struct ibv_context * virt_to_phys_ibv_context(struct ibv_contect * context); -struct ibv_context_ops * virt_to_phys_ibv_context_ops(struct ibv_context_ops * context_ops); -struct ibv_port_attr * virt_to_phys_ibv_port_attr(struct ibv_port_attr * port_attr); -struct ibv_abi_compat_v2 * virt_to_phys_ibv_abi_compat_v2(struct ibv_abi_compat_v2 * abi_compat); -void phys_to_virt_ibv_device(struct ibv_device * device); -void phys_to_virt_ibv_context(struct ibv_contect * context); -void phys_to_virt_ibv_context_ops(struct ibv_context_ops * context_ops); -void phys_to_virt_ibv_port_attr(struct ibv_port_attr * port_attr); -void phys_to_virt_ibv_abi_compat_v2(struct ibv_abi_compat_v2 * abi_compat); +inline size_t guest_to_host(size_t address) { + return virt_to_phys(address) + host_kernel_start; +} + + +struct ibv_device * guest_to_host_ibv_device(struct ibv_device * device); +struct ibv_context * guest_to_host_ibv_context(struct ibv_context * context); +struct ibv_context_ops * guest_to_host_ibv_context_ops(struct ibv_context_ops * context_ops); +struct ibv_port_attr * guest_to_host_ibv_port_attr(struct ibv_port_attr * port_attr); +struct ibv_abi_compat_v2 * guest_to_host_ibv_abi_compat_v2(struct ibv_abi_compat_v2 * abi_compat); + +void host_to_guest_ibv_device(struct ibv_device * device); +void host_to_guest_ibv_context(struct ibv_context * context); +void host_to_guest_ibv_context_ops(struct ibv_context_ops * context_ops); +void host_to_guest_ibv_port_attr(struct ibv_port_attr * port_attr); +void host_to_guest_ibv_abi_compat_v2(struct ibv_abi_compat_v2 * abi_compat); diff --git a/include/hermit/stddef.h b/include/hermit/stddef.h index 3567220d8..6d9076a9a 100644 --- a/include/hermit/stddef.h +++ b/include/hermit/stddef.h @@ -66,6 +66,8 @@ extern const size_t image_size; #define UHYVE_PORT_EXIT 0x503 #define UHYVE_PORT_LSEEK 0x504 +#define UHYVE_PORT_KERNEL_START 0x505 + // InfiniBand uhyve port IDs #define UHYVE_PORT_IBV_OPEN_DEVICE 0x510, //#define UHYVE_PORT_IBV_GET_DEVICE_LIST 0x511, diff --git a/kernel/ibv.c b/kernel/ibv.c index 76b8f902d..f82678c57 100644 --- a/kernel/ibv.c +++ b/kernel/ibv.c @@ -36,7 +36,7 @@ #include #include -#include +#include // TODO: Can/should we separate ibv_get_device_list into two KVM exit IOs to @@ -44,6 +44,119 @@ #define MAX_NUM_OF_IBV_DEVICES 16 +static void * ret_guest_ptr; + + +/* + * ibv_open_device + */ + +typedef struct { + // Parameters: + struct ibv_device * device; + // Return value: + struct ibv_context * ret; +} __attribute__((packed)) uhyve_ibv_open_device_t; + +struct ibv_context * ibv_open_device(struct ibv_device * device) { + uhyve_ibv_open_device_t uhyve_args; + uhyve_args.device = guest_to_host_ibv_device(device); + + ret_guest_ptr = kmalloc(sizeof(struct ibv_context)); + uhyve_args.ret = (struct ibv_context *) guest_to_host((size_t) ret_guest_ptr); + + uhyve_send(UHYVE_PORT_IBV_OPEN_DEVICE, (unsigned)guest_to_host((size_t)&uhyve_args)); + + host_to_guest_ibv_device(device); + uhyve_args.ret = (struct ibv_context *) ret_guest_ptr; + // TODO: Fix pointers in returned data structures. + + return uhyve_args.ret; +} + + +/* + * ibv_get_device_name + */ + +typedef struct { + // Parameters: + struct ibv_device * device; + // Return value: + const char * ret; +} __attribute__((packed)) uhyve_ibv_get_device_name_t; + +const char * ibv_get_device_name(struct ibv_device * device) { + uhyve_ibv_get_device_name_t uhyve_args; + uhyve_args.device = guest_to_host_ibv_device(device); + + uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_NAME, (unsigned) guest_to_host((size_t) &uhyve_args)); + + host_to_guest_ibv_device(device); + // Lookup return address in hash map created earlier. Found? convert back. + // Hack for testing: + uhyve_args.ret = uhyve_args.device->name; + + return uhyve_args.ret; +} + + +/* + * ibv_query_port + */ + +typedef struct { + // Parameters: + struct ibv_context * context; + uint8_t port_num; + struct ibv_port_attr * port_attr; + // Return value: + int ret; +} __attribute__((packed)) uhyve_ibv_query_port_t; + +int ibv_query_port(struct ibv_context * context, uint8_t port_num, struct ibv_port_attr * port_attr) { + uhyve_ibv_query_port_t uhyve_args; + uhyve_args.context = guest_to_host_ibv_context(context); + uhyve_args.port_num = port_num; + uhyve_args.port_attr = guest_to_host_ibv_port_attr(port_attr); + + uhyve_send(UHYVE_PORT_IBV_QUERY_PORT, (unsigned) guest_to_host((size_t) &uhyve_args)); + + host_to_guest_ibv_context(context); + host_to_guest_ibv_port_attr(port_attr); + + return uhyve_args.ret; +} + + +/* + * ibv_create_comp_channel + */ + +typedef struct { + // Parameters: + struct ibv_context * context; + // Return value: + struct ibv_comp_channel * ret; +} __attribute__((packed)) uhyve_ibv_create_comp_channel_t; + +struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context) { + uhyve_ibv_create_comp_channel_t uhyve_args; + uhyve_args.context = guest_to_host_ibv_context(context); + + ret_guest_ptr = kmalloc(sizeof(struct ibv_comp_channel)); + uhyve_args.ret = (struct ibv_comp_channel *) guest_to_host((size_t) ret_guest_ptr); + + uhyve_send(UHYVE_PORT_IBV_CREATE_COMP_CHANNEL, (unsigned) guest_to_host((size_t) &uhyve_args)); + + host_to_guest_ibv_context(context); + uhyve_args.ret = (struct ibv_comp_channel *) ret_guest_ptr; + + return uhyve_args.ret; +} + + + /*typedef struct { // CHECKED*/ /*// Parameters:*/ /*int , *num_devices;*/ @@ -73,93 +186,3 @@ /*uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned)virt_to_phys((size_t)&uhyve_args));*/ /*return list_virt;*/ /*}*/ - - -typedef struct { - // Parameters: - struct ibv_device * device; - // Return value: - struct ibv_context * ret; -} __attribute__((packed)) uhyve_ibv_open_device_t; - -struct ibv_context * ibv_open_device(struct ibv_device * device) { - uhyve_ibv_open_device_t uhyve_args; - uhyve_args->device = virt_to_phys_ibv_device(device); //TODO adapt code gen - - uhyve_args->ret = kmalloc(sizeof(struct ibv_context)); - - uhyve_send(UHYVE_PORT_IBV_OPEN_DEVICE, (unsigned) virt_to_phys((size_t) &uhyve_args)); - - phys_to_virt_ibv_device(device); //TODO adapt code gen - - // TODO: Fix pointers in returned data structures. - return uhyve_args.ret; -} - - -typedef struct { - // Parameters: - struct ibv_device * device; - // Return value: - const char * ret; -} __attribute__((packed)) uhyve_ibv_get_device_name_t; - -const char * ibv_get_device_name(struct ibv_device * device) { - uhyve_ibv_get_device_name_t uhyve_args; - uhyve_args->device = virt_to_phys_ibv_device(device); //TODO adapt code gen - - uhyve_args->ret = kmalloc(sizeof(const char)); - - uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_NAME, (unsigned) virt_to_phys((size_t) &uhyve_args)); - - phys_to_virt_ibv_device(device); //TODO adapt code gen - - // TODO: Fix pointers in returned data structures. - return uhyve_args.ret; -} - - -typedef struct { - // Parameters: - struct ibv_context * context; - uint8_t port_num; - struct ibv_port_attr * port_attr; - // Return value: - int ret; -} __attribute__((packed)) uhyve_ibv_query_port_t; - -int ibv_query_port(struct ibv_context * context, uint8_t port_num, struct ibv_port_attr * port_attr) { - uhyve_ibv_query_port_t uhyve_args; - uhyve_args->context = virt_to_phys_ibv_context(context); - uhyve_args->port_num = port_num; - uhyve_args->port_attr = virt_to_phys_ibv_port_attr(port_attr); - - uhyve_send(UHYVE_PORT_IBV_QUERY_PORT, (unsigned) virt_to_phys((size_t) &uhyve_args)); - - phys_to_virt_ibv_context(context); //TODO adapt code gen - phys_to_virt_ibv_port_attr(port_attr); //TODO adapt code gen - - return uhyve_args.ret; -} - - -typedef struct { - // Parameters: - struct ibv_context * context; - // Return value: - struct ibv_comp_channel * ret; -} __attribute__((packed)) uhyve_ibv_create_comp_channel_t; - -struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context) { - uhyve_ibv_create_comp_channel_t uhyve_args; - uhyve_args->context = virt_to_phys_ibv_context(context); - - uhyve_args->ret = kmalloc(sizeof(struct ibv_comp_channel)); - - uhyve_send(UHYVE_PORT_IBV_CREATE_COMP_CHANNEL, (unsigned) virt_to_phys((size_t) &uhyve_args)); - - phys_to_virt_ibv_context(context); //TODO adapt code gen - - // TODO: Fix pointers in returned data structures. - return uhyve_args.ret; -} diff --git a/kernel/ibv_virt_phys.c b/kernel/ibv_virt_phys.c index 0b0b5a3e6..c7962a4b8 100644 --- a/kernel/ibv_virt_phys.c +++ b/kernel/ibv_virt_phys.c @@ -31,7 +31,7 @@ #include -#include +#include /* @@ -43,24 +43,24 @@ static struct { char * dev_name; char * dev_path; char * ibdev_path; -} ibv_device_virt_ptrs; +} ibv_device_virt_ptrs; // TODO: Proper naming -struct ibv_device * virt_to_phys_ibv_device(struct ibv_device * device) { +struct ibv_device * guest_to_host_ibv_device(struct ibv_device * device) { ibv_device_virt_ptrs.name = device->name;; ibv_device_virt_ptrs.dev_name = device->dev_name; ibv_device_virt_ptrs.dev_path = device->dev_path; ibv_device_virt_ptrs.ibdev_path = device->ibdev_path; - device->name = (char *) virt_to_phys((size_t) device->name); - device->dev_name = (char *) virt_to_phys((size_t) device->dev_name); - device->dev_path = (char *) virt_to_phys((size_t) device->dev_path); - device->ibdev_path = (char *) virt_to_phys((size_t) device->ibdev_path); + device->name = (char *) guest_to_host((size_t) device->name); + device->dev_name = (char *) guest_to_host((size_t) device->dev_name); + device->dev_path = (char *) guest_to_host((size_t) device->dev_path); + device->ibdev_path = (char *) guest_to_host((size_t) device->ibdev_path); // _ops obsolete. - return (struct ibv_device *) virt_to_phys((size_t) device); + return (struct ibv_device *) guest_to_host((size_t) device); } -void phys_to_virt_ibv_device(struct ibv_device * device) { +void host_to_guest_ibv_device(struct ibv_device * device) { device->name = ibv_device_virt_ptrs.name; device->dev_name = ibv_device_virt_ptrs.dev_name; device->dev_path = ibv_device_virt_ptrs.dev_path; @@ -78,25 +78,25 @@ static struct { void * abi_compat; } ibv_context_virt_ptrs; -struct ibv_context * virt_to_phys_ibv_context(struct ibv_context * context) { - ibv_context_virt_ptrs.device = context->device, +struct ibv_context * guest_to_host_ibv_context(struct ibv_context * context) { + ibv_context_virt_ptrs.device = context->device, ibv_context_virt_ptrs.abi_compat = context->abi_compat, - context->device = virt_to_phys_ibv_device(context->device); - context->abi_compat = virt_to_phys_ibv_abi_compat_v2(context->abi_compat); - virt_to_phys_ibv_context_ops(&context->ops); - /*virt_to_phys_pthread_mutex_t(&context->mutex); // TODO*/ + context->device = guest_to_host_ibv_device(context->device); + context->abi_compat = guest_to_host_ibv_abi_compat_v2(context->abi_compat); + guest_to_host_ibv_context_ops(&context->ops); + /*guest_to_host_pthread_mutex_t(&context->mutex); // TODO*/ - return (struct ibv_context *) virt_to_phys((size_t) context); + return (struct ibv_context *) guest_to_host((size_t) context); } -void phys_to_virt_ibv_context(struct ibv_context * context) { - context->device = ibv_context_virt_ptrs.device; +void host_to_guest_ibv_context(struct ibv_context * context) { + context->device = ibv_context_virt_ptrs.device; context->abi_compat = ibv_context_virt_ptrs.abi_compat; - phys_to_virt_ibv_device(context->device); - phys_to_virt_ibv_abi_compat_v2(context->abi_compat); - phys_to_virt_ibv_context_ops(&context->ops); + host_to_guest_ibv_device(context->device); + host_to_guest_ibv_abi_compat_v2(context->abi_compat); + host_to_guest_ibv_context_ops(&context->ops); } @@ -139,7 +139,7 @@ static struct { void (*async_event)(struct ibv_async_event *); } ibv_context_ops_virt_ptrs; -struct ibv_context_ops * virt_to_phys_ibv_context_ops(struct ibv_context_ops * context_ops) { +struct ibv_context_ops * guest_to_host_ibv_context_ops(struct ibv_context_ops * context_ops) { ibv_context_virt_ptrs.query_device = context_ops->query_device; ibv_context_virt_ptrs.query_port = context_ops->query_port; ibv_context_virt_ptrs.alloc_pd = context_ops->alloc_pd; @@ -173,43 +173,44 @@ struct ibv_context_ops * virt_to_phys_ibv_context_ops(struct ibv_context_ops * c ibv_context_virt_ptrs.detach_mcast = context_ops->detach_mcast; ibv_context_virt_ptrs.async_event = context_ops->async_event; - context_ops->query_device = virt_to_phys((size_t) context_ops->query_device); - context_ops->query_port = virt_to_phys((size_t) context_ops->query_port); - context_ops->alloc_pd = virt_to_phys((size_t) context_ops->alloc_pd); - context_ops->dealloc_pd = virt_to_phys((size_t) context_ops->dealloc_pd); - context_ops->reg_mr = virt_to_phys((size_t) context_ops->reg_mr); - context_ops->rereg_mr = virt_to_phys((size_t) context_ops->rereg_mr); - context_ops->dereg_mr = virt_to_phys((size_t) context_ops->dereg_mr); - context_ops->alloc_mw = virt_to_phys((size_t) context_ops->alloc_mw); - context_ops->bind_mw = virt_to_phys((size_t) context_ops->bind_mw); - context_ops->dealloc_mw = virt_to_phys((size_t) context_ops->dealloc_mw); - context_ops->create_cq = virt_to_phys((size_t) context_ops->create_cq); - context_ops->poll_cq = virt_to_phys((size_t) context_ops->poll_cq); - context_ops->req_notify_cq = virt_to_phys((size_t) context_ops->req_notify_cq); - context_ops->cq_event = virt_to_phys((size_t) context_ops->cq_event); - context_ops->resize_cq = virt_to_phys((size_t) context_ops->resize_cq); - context_ops->destroy_cq = virt_to_phys((size_t) context_ops->destroy_cq); - context_ops->create_srq = virt_to_phys((size_t) context_ops->create_srq); - context_ops->modify_srq = virt_to_phys((size_t) context_ops->modify_srq); - context_ops->query_srq = virt_to_phys((size_t) context_ops->query_srq); - context_ops->destroy_srq = virt_to_phys((size_t) context_ops->destroy_srq); - context_ops->post_srq_recv = virt_to_phys((size_t) context_ops->post_srq_recv); - context_ops->create_qp = virt_to_phys((size_t) context_ops->create_qp); - context_ops->query_qp = virt_to_phys((size_t) context_ops->query_qp); - context_ops->modify_qp = virt_to_phys((size_t) context_ops->modify_qp); - context_ops->destroy_qp = virt_to_phys((size_t) context_ops->destroy_qp); - context_ops->post_send = virt_to_phys((size_t) context_ops->post_send); - context_ops->post_recv = virt_to_phys((size_t) context_ops->post_recv); - context_ops->create_ah = virt_to_phys((size_t) context_ops->create_ah); - context_ops->destroy_ah = virt_to_phys((size_t) context_ops->destroy_ah); - context_ops->attach_mcast = virt_to_phys((size_t) context_ops->attach_mcast); - context_ops->detach_mcast = virt_to_phys((size_t) context_ops->detach_mcast); - context_ops->async_event = virt_to_phys((size_t) context_ops->async_event); + // TODO: Does this work? Fcn returns size_t. Have to convert? + context_ops->query_device = guest_to_host((size_t) context_ops->query_device); + context_ops->query_port = guest_to_host((size_t) context_ops->query_port); + context_ops->alloc_pd = guest_to_host((size_t) context_ops->alloc_pd); + context_ops->dealloc_pd = guest_to_host((size_t) context_ops->dealloc_pd); + context_ops->reg_mr = guest_to_host((size_t) context_ops->reg_mr); + context_ops->rereg_mr = guest_to_host((size_t) context_ops->rereg_mr); + context_ops->dereg_mr = guest_to_host((size_t) context_ops->dereg_mr); + context_ops->alloc_mw = guest_to_host((size_t) context_ops->alloc_mw); + context_ops->bind_mw = guest_to_host((size_t) context_ops->bind_mw); + context_ops->dealloc_mw = guest_to_host((size_t) context_ops->dealloc_mw); + context_ops->create_cq = guest_to_host((size_t) context_ops->create_cq); + context_ops->poll_cq = guest_to_host((size_t) context_ops->poll_cq); + context_ops->req_notify_cq = guest_to_host((size_t) context_ops->req_notify_cq); + context_ops->cq_event = guest_to_host((size_t) context_ops->cq_event); + context_ops->resize_cq = guest_to_host((size_t) context_ops->resize_cq); + context_ops->destroy_cq = guest_to_host((size_t) context_ops->destroy_cq); + context_ops->create_srq = guest_to_host((size_t) context_ops->create_srq); + context_ops->modify_srq = guest_to_host((size_t) context_ops->modify_srq); + context_ops->query_srq = guest_to_host((size_t) context_ops->query_srq); + context_ops->destroy_srq = guest_to_host((size_t) context_ops->destroy_srq); + context_ops->post_srq_recv = guest_to_host((size_t) context_ops->post_srq_recv); + context_ops->create_qp = guest_to_host((size_t) context_ops->create_qp); + context_ops->query_qp = guest_to_host((size_t) context_ops->query_qp); + context_ops->modify_qp = guest_to_host((size_t) context_ops->modify_qp); + context_ops->destroy_qp = guest_to_host((size_t) context_ops->destroy_qp); + context_ops->post_send = guest_to_host((size_t) context_ops->post_send); + context_ops->post_recv = guest_to_host((size_t) context_ops->post_recv); + context_ops->create_ah = guest_to_host((size_t) context_ops->create_ah); + context_ops->destroy_ah = guest_to_host((size_t) context_ops->destroy_ah); + context_ops->attach_mcast = guest_to_host((size_t) context_ops->attach_mcast); + context_ops->detach_mcast = guest_to_host((size_t) context_ops->detach_mcast); + context_ops->async_event = guest_to_host((size_t) context_ops->async_event); - return (struct ibv_context_ops *) virt_to_phys((size_t) context_ops); + return (struct ibv_context_ops *) guest_to_host((size_t) context_ops); } -void phys_to_virt_ibv_context_ops(struct ibv_context_ops * context_ops) { +void host_to_guest_ibv_context_ops(struct ibv_context_ops * context_ops) { context_ops->query_device = ibv_context_ops_virt_ptrs.query_device; context_ops->query_port = ibv_context_ops_virt_ptrs.query_port; context_ops->alloc_pd = ibv_context_ops_virt_ptrs.alloc_pd; @@ -242,7 +243,6 @@ void phys_to_virt_ibv_context_ops(struct ibv_context_ops * context_ops) { context_ops->attach_mcast = ibv_context_ops_virt_ptrs.attach_mcast; context_ops->detach_mcast = ibv_context_ops_virt_ptrs.detach_mcast; context_ops->async_event = ibv_context_ops_virt_ptrs.async_event; - } @@ -250,27 +250,27 @@ void phys_to_virt_ibv_context_ops(struct ibv_context_ops * context_ops) { * struct ibv_port_attr */ -struct ibv_port_attr * virt_to_phys_ibv_port_attr(struct ibv_port_attr * port_attr) { - return (struct ibv_port_attr *) virt_to_phys((size_t) port_attr); +struct ibv_port_attr * guest_to_host_ibv_port_attr(struct ibv_port_attr * port_attr) { + return (struct ibv_port_attr *) guest_to_host((size_t) port_attr); } -void phys_to_virt_ibv_port_attr(struct ibv_port_attr * port_attr) {} +void host_to_guest_ibv_port_attr(struct ibv_port_attr * port_attr) {} /* * struct ibv_abi_compat_v2 */ -struct ibv_abi_compat_v2 * virt_to_phys_ibv_abi_compat_v2(struct ibv_abi_compat_v2 * abi_compat) { - virt_to_phys_ibv_comp_channel(&abi_compat->channel); - virt_to_phys_pthread_mutex_t(&abi_compat->in_use); +struct ibv_abi_compat_v2 * guest_to_host_ibv_abi_compat_v2(struct ibv_abi_compat_v2 * abi_compat) { + guest_to_host_ibv_comp_channel(&abi_compat->channel); + guest_to_host_pthread_mutex_t(&abi_compat->in_use); - return (struct ibv_abi_compat_v2 *) virt_to_phys((size_t) abi_compat); + return (struct ibv_abi_compat_v2 *) guest_to_host((size_t) abi_compat); } -void phys_to_virt_ibv_abi_compat_v2(struct ibv_abi_compat_v2 * abi_compat) { - phys_to_virt_ibv_comp_channel(&abi_compat->channel); - phys_to_virt_pthread_mutex_t(&abi_compat->in_use); +void host_to_guest_ibv_abi_compat_v2(struct ibv_abi_compat_v2 * abi_compat) { + host_to_guest_ibv_comp_channel(&abi_compat->channel); + host_to_guest_pthread_mutex_t(&abi_compat->in_use); } @@ -282,18 +282,18 @@ static struct { struct ibv_context * context; } ibv_comp_channel_virt_ptrs; -struct ibv_comp_channel * virt_to_phys_ibv_comp_channel(struct ibv_comp_channel * channel) { +struct ibv_comp_channel * guest_to_host_ibv_comp_channel(struct ibv_comp_channel * channel) { ibv_comp_channel_virt_ptrs.context = channel->context, - channel->context = virt_to_phys_ibv_device(channel->context); + channel->context = guest_to_host_ibv_device(channel->context); - return (struct ibv_comp_channel *) virt_to_phys((size_t) channel); + return (struct ibv_comp_channel *) guest_to_host((size_t) channel); } -void phys_to_virt_ibv_comp_channel(struct ibv_comp_channel * channel) { +void host_to_guest_ibv_comp_channel(struct ibv_comp_channel * channel) { channel->context = ibv_comp_channel_virt_ptrs.context; - phys_to_virt_ibv_device(channel->context); + host_to_guest_ibv_device(channel->context); } diff --git a/tools/uhyve-ibv.c b/tools/uhyve-ibv.c index 44a5e7986..0a2616f59 100644 --- a/tools/uhyve-ibv.c +++ b/tools/uhyve-ibv.c @@ -33,6 +33,69 @@ #include // Linux include +struct ibv_context * ibv_open_device(struct ibv_device * device); +const char* ibv_get_device_name(struct ibv_device *device); +int ibv_query_port(struct ibv_context * context, uint8_t port_num, struct ibv_port_attr * port_attr); +struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context); + + +/* + * ibv_open_device + */ + +void call_ibv_open_device(struct kvm_run * run) { + unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); + uhyve_ibv_open_device_t * args = (uhyve_ibv_open_device_t *) (guest_mem + data); + + struct ibv_context * host_ret = ibv_open_device(guest_mem+(size_t)args->device); + memcpy(guest_mem+(size_t)args->ret, host_ret, sizeof(host_ret)); + // TODO: Convert ptrs contained in return value. + free(host_ret); +} + + +/* + * ibv_get_device_name + */ + +void call_ibv_get_device_name(struct kvm_run * run) { + unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); + uhyve_ibv_get_device_name_t * args = (uhyve_ibv_get_device_name_t *) (guest_mem + data); + + // TODO: Tricky because char ptr isn't allocated in called function. + const char * host_ret = ibv_get_device_name(guest_mem+(size_t)args->device); + memcpy(guest_mem+(size_t)args->ret, host_ret, sizeof(host_ret)); + // TODO: Convert ptrs contained in return value. + // TODO: How to tell if ret needs to be deleted? +} + + +/* + * ibv_query_port + */ + +void call_ibv_query_port(struct kvm_run * run) { + unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); + uhyve_ibv_query_port_t * args = (uhyve_ibv_query_port_t *) (guest_mem + data); + + int host_ret = ibv_query_port(guest_mem+(size_t)args->context, port_num, guest_mem+(size_t)args->port_attr); + args->ret = host_ret; +} + + +/* + * ibv_create_comp_channel + */ + +void call_ibv_create_comp_channel(struct kvm_run * run) { + uhyve_ibv_create_comp_channel_t * args = (uhyve_ibv_create_comp_channel_t *) get_data(run); + + struct ibv_comp_channel * host_ret = ibv_create_comp_channel(args->context); + memcpy(args->ret, host_ret, sizeof(host_ret)); // TODO: This will only work for ABI ver > 2. + free(host_ret); +} + + /*void call_ibv_get_device_list(struct kvm_run * run, uint8_t * guest_mem) {*/ /*unsigned data = *((unsigned*)((size_t)run+run->io.data_offset));*/ /*uhyve_ibv_get_device_list_t* args = (uhyve_ibv_get_device_list_t*) (guest_mem+data);*/ @@ -50,86 +113,3 @@ /*memcpy(guest_mem + (size_t)args->dev_phys_ptr_list[d], temp_dev_list[d], sizeof(struct ibv_device));*/ /*}*/ /*}*/ - - -struct ibv_context * ibv_open_device(struct ibv_device * device); -const char* ibv_get_device_name(struct ibv_device *device); -int ibv_query_port(struct ibv_context * context, uint8_t port_num, struct ibv_port_attr * port_attr); -struct ibv_comp_channel * ibv_create_comp_channel(struct ibv_context * context); - - -/* - * ibv_open_device - */ - -void call_ibv_open_device(struct kvm_run * run, uint8_t * guest_mem) { - unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_open_device_t * args = (uhyve_ibv_open_device_t *) (guest_mem + data); - - struct ibv_context * host_ret = ibv_open_device(guest_mem+(size_t)args->device); - memcpy(guest_mem+(size_t)args->ret, host_ret, sizeof(host_ret)); - // TODO: Convert ptrs contained in return value. - free(host_ret); -} - - -/* - * ibv_get_device_name - */ - -void call_ibv_get_device_name(struct kvm_run * run, uint8_t * guest_mem) { - unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_get_device_name_t * args = (uhyve_ibv_get_device_name_t *) (guest_mem + data); - - // TODO: Tricky because char ptr isn't allocated in called function. - const char * host_ret = ibv_get_device_name(guest_mem+(size_t)args->device); - memcpy(guest_mem+(size_t)args->ret, host_ret, sizeof(host_ret)); - // TODO: Convert ptrs contained in return value. - // TODO: How to tell if ret needs to be deleted? -} - - -/* - * ibv_query_port - */ - -void call_ibv_query_port(struct kvm_run * run, uint8_t * guest_mem) { - unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_query_port_t * args = (uhyve_ibv_query_port_t *) (guest_mem + data); - - int host_ret = ibv_query_port(guest_mem+(size_t)args->context, port_num, guest_mem+(size_t)args->port_attr); - args->ret = host_ret; -} - - -/* - * ibv_create_comp_channel - */ - -/*void guest_to_host_ibv_create_comp_channel_t(struct uhyve_ibv_create_comp_channel_t * args, uint8_t * guest_mem) {*/ - /*[>args->context = (size_t)args->context + guest_mem;<]*/ - /*args->context = guest_to_host(args->context);*/ - /*args->ret = (size_t)args->ret + guest_mem;*/ -/*}*/ - -/*void host_to_guest_ibv_create_comp_channel_t(struct uhyve_ibv_create_comp_channel_t * args, uint8_t * guest_mem) {*/ - /*args->context = (size_t)args->context - guest_mem;*/ - /*args->ret = (size_t)args->ret - guest_mem;*/ -/*}*/ - -void call_ibv_create_comp_channel(struct kvm_run * run, uint8_t * guest_mem) { - uhyve_ibv_create_comp_channel_t * args = (uhyve_ibv_create_comp_channel_t *) get_data(run, guest_mem); - guest_to_host_ibv_create_comp_channel_t(args, guest_mem); - - /*args->context = (size_t)args->context + guest_mem;*/ - args->context = guest_to_host(args->context); - args->ret = (size_t)args->ret + guest_mem; - - struct ibv_comp_channel * host_ret = ibv_create_comp_channel(args->context); - memcpy(args->ret, host_ret, sizeof(host_ret)); // TODO: This will only work for ABI ver > 2. - free(host_ret); - - args->context = (size_t)args->context - guest_mem; - args->ret = (size_t)args->ret - guest_mem; - /*host_to_guest_ibv_create_comp_channel_t(args, guest_mem);*/ -} diff --git a/tools/uhyve-ibv.h b/tools/uhyve-ibv.h index b28776f78..b11a0bcf4 100644 --- a/tools/uhyve-ibv.h +++ b/tools/uhyve-ibv.h @@ -24,6 +24,8 @@ #define MAX_NUM_OF_IBV_DEVICES 16 typedef enum { + UHYVE_PORT_KERNEL_START = 0x505, + UHYVE_PORT_IBV_OPEN_DEVICE = 0x510, //UHYVE_PORT_IBV_GET_DEVICE_LIST = 0x511, UHYVE_PORT_IBV_GET_DEVICE_NAME = 0x512, @@ -32,21 +34,11 @@ typedef enum { } uhyve_ibv_t; -inline unsigned get_data(struct kvm_run * run, uint8_t guest_mem) { - unsigned data = guest_mem + *((unsigned*)((size_t)run+run->io.data_offset)); - return data; +inline unsigned get_data(struct kvm_run * run) { + return *((unsigned*)((size_t)run+run->io.data_offset)); } -//typedef struct { // CHECKED - //// In: - //int *num_devices; - //// Out: - ////struct ibv_device devices[MAX_NUM_OF_IBV_DEVICES]; - //struct ibv_device *dev_phys_ptr_list[MAX_NUM_OF_IBV_DEVICES]; - ////struct ibv_device **device_list; -//} __attribute__((packed)) uhyve_ibv_get_device_list_t; - typedef struct { // Parameters: struct ibv_device * device; @@ -76,3 +68,13 @@ typedef struct { // Return value: struct ibv_comp_channel * ret; } __attribute__((packed)) uhyve_ibv_create_comp_channel_t; + + +//typedef struct { // CHECKED + //// In: + //int *num_devices; + //// Out: + ////struct ibv_device devices[MAX_NUM_OF_IBV_DEVICES]; + //struct ibv_device *dev_phys_ptr_list[MAX_NUM_OF_IBV_DEVICES]; + ////struct ibv_device **device_list; +//} __attribute__((packed)) uhyve_ibv_get_device_list_t; diff --git a/tools/uhyve.c b/tools/uhyve.c index ae2ccbec6..a56935d0b 100644 --- a/tools/uhyve.c +++ b/tools/uhyve.c @@ -868,6 +868,14 @@ static int vcpu_loop(void) case KVM_EXIT_IO: //printf("port 0x%x\n", run->io.port); switch (run->io.port) { + case UHYVE_PORT_KERNEL_START: { + unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); + uint8_t ** ret = (uint8_t **) (guest_mem+data); + &ret = guest_mem; + printf("Guest mem in uhyve: %p", guest_mem); + break; + } + case UHYVE_PORT_WRITE: { unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); uhyve_write_t* uhyve_write = (uhyve_write_t*) (guest_mem+data); @@ -970,9 +978,6 @@ static int vcpu_loop(void) case UHYVE_PORT_IBV_OPEN_DEVICE: call_ibv_open_device(run, guest_mem); break; - /*case UHYVE_PORT_IBV_GET_DEVICE_LIST:*/ - /*call_ibv_get_device_list(run, guest_mem);*/ - /*break;*/ case UHYVE_PORT_IBV_GET_DEVICE_NAME: call_ibv_get_device_name(run, guest_mem); break; @@ -982,7 +987,9 @@ static int vcpu_loop(void) case UHYVE_PORT_IBV_CREATE_COMP_CHANNEL: call_ibv_create_comp_channel(run, guest_mem); break; - + /*case UHYVE_PORT_IBV_GET_DEVICE_LIST:*/ + /*call_ibv_get_device_list(run, guest_mem);*/ + /*break;*/ default: err(1, "KVM: unhandled KVM_EXIT_IO at port 0x%x, direction %d\n", run->io.port, run->io.direction); diff --git a/usr/tests/CMakeLists.txt b/usr/tests/CMakeLists.txt index 31aa3602b..3d3e50739 100644 --- a/usr/tests/CMakeLists.txt +++ b/usr/tests/CMakeLists.txt @@ -9,8 +9,8 @@ add_executable(hello++ hello++.cpp) add_executable(hellof hellof.f90) add_executable(pi pi.go) -add_executable(ib-test ib_test.c) -#add_executable(ib-pingpong ib/pingpong.c ib/pingpong.h ib/pingpong_ud.c) +#add_executable(ib-test ib_test.c) +add_executable(ib-pingpong ib/pingpong.c ib/pingpong.h ib/pingpong_ud.c) #target_link_libraries(ib-test ibverbs) add_executable(test-malloc test-malloc.c)