diff --git a/include/hermit/stddef.h b/include/hermit/stddef.h index 4bffe6278..abba048ed 100644 --- a/include/hermit/stddef.h +++ b/include/hermit/stddef.h @@ -73,7 +73,7 @@ extern const size_t image_size; #define UHYVE_PORT_IBV_QUERY_PORT 0x513 #define UHYVE_PORT_IBV_CREATE_COMP_CHANNEL 0x514 -#define UHYVE_PORT_KERNEL_IBV_LOG 0x515 +#define UHYVE_PORT_KERNEL_IBV_LOG 0x515 #define BUILTIN_EXPECT(exp, b) __builtin_expect((exp), (b)) diff --git a/kernel/ibv.c b/kernel/ibv.c index 64365134e..c47b3f404 100644 --- a/kernel/ibv.c +++ b/kernel/ibv.c @@ -42,7 +42,7 @@ // TODO: Can/should we separate ibv_get_device_list into two KVM exit IOs to // allocate the right amount of memory? -#define MAX_NUM_OF_IBV_DEVICES 16 +#define MAX_NUM_OF_IBV_DEVICES 16 static void * ret_guest_ptr; @@ -171,6 +171,38 @@ typedef struct { struct ibv_device * ret[MAX_NUM_OF_IBV_DEVICES]; } __attribute__((packed)) uhyve_ibv_get_device_list_t; +/* struct ibv_device ** ibv_get_device_list(int * num_devices) { */ + /* // num_devices can be mapped to physical memory right away. */ + /* uhyve_ibv_get_device_list_t uhyve_args; */ + /* uhyve_args.test = 42; */ + + /* uhyve_args.num_devices = (int *) virt_to_phys((size_t) num_devices); */ + + /* // Allocate memory for return value. */ + /* struct ibv_device * devs = kmalloc(MAX_NUM_OF_IBV_DEVICES * sizeof(struct ibv_device)); */ + /* struct ibv_device ** ret_guest_ptr = kmalloc(MAX_NUM_OF_IBV_DEVICES * sizeof(struct ibv_device *)); */ + + /* for (int i = 0; i < MAX_NUM_OF_IBV_DEVICES; i++) { */ + /* struct ibv_device * device_address = devs + i; */ + /* ret_guest_ptr[i] = device_address; */ + /* uhyve_args.ret[i] = (struct ibv_device *) virt_to_phys((size_t) device_address); */ + /* } */ + /* uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST_OLD, (unsigned) virt_to_phys((size_t) &uhyve_args)); */ + + /* // -------------- */ + + /* uhyve_args.num_devices = (int *) guest_to_host((size_t) num_devices); */ + + /* for (int i = 0; i < MAX_NUM_OF_IBV_DEVICES; i++) { */ + /* uhyve_args.ret[i] = (struct ibv_device *) guest_to_host((size_t) ret_guest_ptr[i]); */ + /* } */ + + /* uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned) guest_to_host((size_t) &uhyve_args)); */ + + /* return ret_guest_ptr; */ +/* } */ + + struct ibv_device ** ibv_get_device_list(int * num_devices) { // num_devices can be mapped to physical memory right away. uhyve_ibv_get_device_list_t uhyve_args; @@ -183,12 +215,12 @@ struct ibv_device ** ibv_get_device_list(int * num_devices) { // We keep a list of the virtual addresses, so we can return it later, and map // to physical addresses for the args struct passed to uhyve. for (int i = 0; i < MAX_NUM_OF_IBV_DEVICES; i++) { - struct ibv_device* device_address = devs + i; + struct ibv_device * device_address = devs + i; ret_guest_ptr[i] = device_address; uhyve_args.ret[i] = (struct ibv_device *) guest_to_host((size_t) device_address); } - uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned) guest_to_host((size_t) &uhyve_args)); + uhyve_send(UHYVE_PORT_IBV_GET_DEVICE_LIST, (unsigned) virt_to_phys((size_t) &uhyve_args)); return ret_guest_ptr; } diff --git a/tools/uhyve-ibv.c b/tools/uhyve-ibv.c index 4c106dbff..f71a5756f 100644 --- a/tools/uhyve-ibv.c +++ b/tools/uhyve-ibv.c @@ -43,10 +43,10 @@ * ibv_open_device */ -void call_ibv_open_device(struct kvm_run * run) { +void call_ibv_open_device(struct kvm_run * run, uint8_t * guest_mem) { printf("LOG: UHYVE - call_ibv_open_device"); unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_open_device_t * args = (uhyve_ibv_open_device_t *) data; + uhyve_ibv_open_device_t * args = (uhyve_ibv_open_device_t *) (guest_mem + data); struct ibv_context * host_ret = ibv_open_device(args->device); memcpy(args->ret, host_ret, sizeof(host_ret)); @@ -59,10 +59,10 @@ void call_ibv_open_device(struct kvm_run * run) { * ibv_get_device_name */ -void call_ibv_get_device_name(struct kvm_run * run) { +void call_ibv_get_device_name(struct kvm_run * run, uint8_t * guest_mem) { printf("LOG: UHYVE - call_ibv_get_device_name"); unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_get_device_name_t * args = (uhyve_ibv_get_device_name_t *) data; + uhyve_ibv_get_device_name_t * args = (uhyve_ibv_get_device_name_t *) (guest_mem + data); // TODO: Tricky because char ptr isn't allocated in called function. const char * host_ret = ibv_get_device_name(args->device); @@ -76,10 +76,10 @@ void call_ibv_get_device_name(struct kvm_run * run) { * ibv_query_port */ -void call_ibv_query_port(struct kvm_run * run) { +void call_ibv_query_port(struct kvm_run * run, uint8_t * guest_mem) { printf("LOG: UHYVE - call_ibv_query_port"); unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_query_port_t * args = (uhyve_ibv_query_port_t *) (data); + uhyve_ibv_query_port_t * args = (uhyve_ibv_query_port_t *) (guest_mem + data); int host_ret = ibv_query_port(args->context, args->port_num, args->port_attr); args->ret = host_ret; @@ -90,10 +90,10 @@ void call_ibv_query_port(struct kvm_run * run) { * ibv_create_comp_channel */ -void call_ibv_create_comp_channel(struct kvm_run * run) { +void call_ibv_create_comp_channel(struct kvm_run * run, uint8_t * guest_mem) { printf("LOG: UHYVE - call_ibv_create_comp_channel"); unsigned data = *((unsigned*)((size_t)run+run->io.data_offset)); - uhyve_ibv_create_comp_channel_t * args = (uhyve_ibv_create_comp_channel_t *) data; + uhyve_ibv_create_comp_channel_t * args = (uhyve_ibv_create_comp_channel_t *) (guest_mem + data); /*uhyve_ibv_create_comp_channel_t * args = (uhyve_ibv_create_comp_channel_t *) get_data(run);*/ struct ibv_comp_channel * host_ret = ibv_create_comp_channel(args->context); @@ -102,10 +102,10 @@ void call_ibv_create_comp_channel(struct kvm_run * run) { } -void call_ibv_get_device_list(struct kvm_run * run) { +void call_ibv_get_device_list(struct kvm_run * run, uint8_t * guest_mem) { printf("LOG: UHYVE - call_ibv_get_device_list"); unsigned data = *((unsigned *)((size_t)run+run->io.data_offset)); - uhyve_ibv_get_device_list_t * args = (uhyve_ibv_get_device_list_t *) data; + uhyve_ibv_get_device_list_t * args = (uhyve_ibv_get_device_list_t *) (guest_mem + data); printf("LOG: UHYVE - call_ibv_get_device_list"); // Call IBV function from hypervisor @@ -114,16 +114,14 @@ void call_ibv_get_device_list(struct kvm_run * run) { printf("LOG: UHYVE - call_ibv_get_device_list"); // Copy number of devices to kernel memory - memcpy(args->num_devices, &num_devices, sizeof(num_devices)); - /* if (args->num_devices) { */ - /* memcpy(args->num_devices, &num_devices, sizeof(num_devices)); */ - /* } */ + if (args->num_devices) { + memcpy(args->num_devices, &num_devices, sizeof(num_devices)); + } printf("LOG: UHYVE - call_ibv_get_device_list"); for (int d = 0; d < num_devices; d++) { - /*printf("uhyve.c: before memcpy list[d].\n");*/ // Copy array entry containing ibv_device struct to kernel memory - printf("LOG: UHYVE - call_ibv_get_device_list"); + printf("LOG: UHYVE FOR - call_ibv_get_device_list"); memcpy(args->ret[d], host_ret[d], sizeof(struct ibv_device)); } } diff --git a/tools/uhyve-ibv.h b/tools/uhyve-ibv.h index dae152c03..bbda5bc6b 100644 --- a/tools/uhyve-ibv.h +++ b/tools/uhyve-ibv.h @@ -39,13 +39,6 @@ typedef enum { //} -typedef struct { - // Parameters: - int * num_devices; - // Return value: - struct ibv_device * ret[MAX_NUM_OF_IBV_DEVICES]; -} __attribute__((packed)) uhyve_ibv_get_device_list_t; - typedef struct { // Parameters: struct ibv_device * device; @@ -76,11 +69,17 @@ typedef struct { struct ibv_comp_channel * ret; } __attribute__((packed)) uhyve_ibv_create_comp_channel_t; +typedef struct { + // Parameters: + int * num_devices; + // Return value: + struct ibv_device * ret[MAX_NUM_OF_IBV_DEVICES]; +} __attribute__((packed)) uhyve_ibv_get_device_list_t; -void call_ibv_open_device(struct kvm_run * run); -void call_ibv_get_device_name(struct kvm_run * run); -void call_ibv_query_port(struct kvm_run * run); -void call_ibv_create_comp_channel(struct kvm_run * run); -void call_ibv_get_device_list(struct kvm_run * run); +void call_ibv_open_device(struct kvm_run * run, uint8_t * guest_mem); +void call_ibv_get_device_name(struct kvm_run * run, uint8_t * guest_mem); +void call_ibv_query_port(struct kvm_run * run, uint8_t * guest_mem); +void call_ibv_create_comp_channel(struct kvm_run * run, uint8_t * guest_mem); +void call_ibv_get_device_list(struct kvm_run * run, uint8_t * guest_mem); #endif // UHYVE_IBV_H diff --git a/tools/uhyve.c b/tools/uhyve.c index d06eb9f1b..5f85b15b7 100644 --- a/tools/uhyve.c +++ b/tools/uhyve.c @@ -981,48 +981,43 @@ static int vcpu_loop(void) // InfiniBand case UHYVE_PORT_IBV_OPEN_DEVICE: - call_ibv_open_device(run); + call_ibv_open_device(run, guest_mem); break; case UHYVE_PORT_IBV_GET_DEVICE_NAME: - call_ibv_get_device_name(run); + call_ibv_get_device_name(run, guest_mem); break; case UHYVE_PORT_IBV_QUERY_PORT: - call_ibv_query_port(run); + call_ibv_query_port(run, guest_mem); break; case UHYVE_PORT_IBV_CREATE_COMP_CHANNEL: - call_ibv_create_comp_channel(run); + call_ibv_create_comp_channel(run, guest_mem); break; - case UHYVE_PORT_IBV_GET_DEVICE_LIST: - /* printf("LOG: UHYVE CASE"); */ - /* call_ibv_get_device_list(run); */ - printf("LOG: UHYVE CASE\n"); - unsigned data = *((unsigned *)((size_t)run+run->io.data_offset)); - printf("LOG: UHYVE CASE\n"); - uhyve_ibv_get_device_list_t * args = (uhyve_ibv_get_device_list_t *) (data); + case UHYVE_PORT_IBV_GET_DEVICE_LIST: { + printf("LOG: UHYVE CASE"); + call_ibv_get_device_list(run, guest_mem); - // Call IBV function from hypervisor - int num_devices; - printf("LOG: UHYVE CASE\n"); - struct ibv_device **host_ret = ibv_get_device_list(&num_devices); + /* unsigned data = *((unsigned *)((size_t)run+run->io.data_offset)); */ + /* uhyve_ibv_get_device_list_t * args = (uhyve_ibv_get_device_list_t *) (guest_mem + data); */ - // Copy number of devices to kernel memory - printf("LOG: UHYVE CASE\n"); - printf("LOG: UHYVE CASE - guest_mem correct: %p\n", guest_mem); - memcpy(args->num_devices, &num_devices, sizeof(num_devices)); + /* // Call IBV function from hypervisor */ + /* int num_devices; */ + /* struct ibv_device **host_ret = ibv_get_device_list(&num_devices); */ + + /* // Copy number of devices to kernel memory */ /* if (args->num_devices) { */ - /* printf("LOG: UHYVE CASE\n"); */ /* memcpy(args->num_devices, &num_devices, sizeof(num_devices)); */ /* } */ - for (int d = 0; d < num_devices; d++) { - // Copy array entry containing ibv_device struct to kernel memory - printf("LOG: UHYVE CASE\n"); - memcpy(args->ret[d], host_ret[d], sizeof(struct ibv_device)); - } + /* for (int d = 0; d < num_devices; d++) { */ + /* // Copy array entry containing ibv_device struct to kernel memory */ + /* printf("LOG: UHYVE CASE FOR\n"); */ + /* memcpy(args->ret[d], host_ret[d], sizeof(struct ibv_device)); */ + /* } */ - printf("LOG: UHYVE CASE\n"); + /* printf("LOG: UHYVE CASE\n"); */ break; + } default: err(1, "KVM: unhandled KVM_EXIT_IO at port 0x%x, direction %d\n", run->io.port, run->io.direction); diff --git a/usr/tests/ib-test.c b/usr/tests/ib-test.c index bb6b71ab2..63781c490 100644 --- a/usr/tests/ib-test.c +++ b/usr/tests/ib-test.c @@ -51,10 +51,11 @@ int main(int argc, char** argv) struct ibv_device **dev_list; int num_devices; - printf("ib_test.c: before get dev list.\n"); + printf("ib-test.c: before kernel ibv_log\n"); kernel_ibv_log(); + printf("ib-test.c: before get dev list.\n"); dev_list = ibv_get_device_list(&num_devices); - printf("ib_test.c: after get dev list.\n"); + printf("ib-test.c: after get dev list.\n"); printf("ib-test.c: num devices: %d\n", num_devices);