From e7c08c271c28e0241b119ee9aa11a08eb0e9088c Mon Sep 17 00:00:00 2001 From: bytesnake Date: Fri, 9 Jun 2017 13:36:10 +0200 Subject: [PATCH] Support memory and ncores conf, Extend MP support --- tools/hermit_proxy/src/hermit/error.rs | 8 +- tools/hermit_proxy/src/hermit/hermit_env.rs | 9 +++ tools/hermit_proxy/src/hermit/mod.rs | 1 + tools/hermit_proxy/src/hermit/uhyve/proto.rs | 36 ++++++--- tools/hermit_proxy/src/hermit/uhyve/uhyve.rs | 9 ++- tools/hermit_proxy/src/hermit/uhyve/vcpu.rs | 77 ++++++++++---------- tools/hermit_proxy/src/hermit/uhyve/vm.rs | 38 ++++++---- tools/hermit_proxy/src/hermit/utils.rs | 17 +++++ 8 files changed, 123 insertions(+), 72 deletions(-) diff --git a/tools/hermit_proxy/src/hermit/error.rs b/tools/hermit_proxy/src/hermit/error.rs index ad4f8c279..22329e8cb 100644 --- a/tools/hermit_proxy/src/hermit/error.rs +++ b/tools/hermit_proxy/src/hermit/error.rs @@ -14,7 +14,9 @@ pub enum Error { MultiIsleFailed, CannotCreateTmpFile(usize), QEmu((String, String)), - MissingBinary + MissingBinary, + Protocol(String), + ParseMemory } impl fmt::Debug for Error { @@ -33,7 +35,9 @@ impl fmt::Debug for Error { Error::MultiIsleFailed => write!(f, "The Multi isle was selected on a system without supported, please load the kernel driver."), Error::CannotCreateTmpFile(_) => write!(f, "Couldn't create a tmp file in /tmp."), Error::QEmu((_, ref stderr)) => write!(f, "The qemu binary has encountered an error: {}", stderr), - Error::MissingBinary => write!(f, "Please specify a binary.") + Error::MissingBinary => write!(f, "Please specify a binary."), + Error::Protocol(ref err) => write!(f, "{}", err), + Error::ParseMemory => write!(f, "Couldn't parse the guest memory size from the environment") } } } diff --git a/tools/hermit_proxy/src/hermit/hermit_env.rs b/tools/hermit_proxy/src/hermit/hermit_env.rs index 028ae3ce9..f4deb0bbe 100644 --- a/tools/hermit_proxy/src/hermit/hermit_env.rs +++ b/tools/hermit_proxy/src/hermit/hermit_env.rs @@ -1,4 +1,5 @@ use std::env; +use super::utils; pub fn isle_kind() -> String { env::var("HERMIT_ISLE").unwrap_or("qemu".into()) @@ -12,10 +13,18 @@ pub fn num_cpus() -> String { env::var("HERMIT_CPUS").unwrap_or("1".into()) } +pub fn num_cpus_parsed() -> u32 { + env::var("HERMIT_CPUS").map(|x| x.parse().unwrap_or(1)).unwrap_or(1) +} + pub fn mem_size() -> String { env::var("HERMIT_MEM").unwrap_or("2G".into()) } +pub fn mem_size_parsed() -> usize { + env::var("HERMIT_MEM").map(|x| utils::parse_mem(&x).unwrap_or(2*1000*1000)).unwrap_or(2*1000*1000) +} + pub fn use_kvm() -> String { env::var("HERMIT_KVM").unwrap_or("1".into()) } diff --git a/tools/hermit_proxy/src/hermit/mod.rs b/tools/hermit_proxy/src/hermit/mod.rs index 0e3dcb647..a830c3333 100644 --- a/tools/hermit_proxy/src/hermit/mod.rs +++ b/tools/hermit_proxy/src/hermit/mod.rs @@ -73,6 +73,7 @@ pub trait Isle { ino.add_watch(Path::new(&log_path), watch_mask::MODIFY | watch_mask::CREATE).unwrap(); + let mut buffer = [0; 1024]; loop { if let Some(_) = ino.read_events(&mut buffer).unwrap().next() { diff --git a/tools/hermit_proxy/src/hermit/uhyve/proto.rs b/tools/hermit_proxy/src/hermit/uhyve/proto.rs index e1d3fd346..e7e0c92b1 100644 --- a/tools/hermit_proxy/src/hermit/uhyve/proto.rs +++ b/tools/hermit_proxy/src/hermit/uhyve/proto.rs @@ -2,6 +2,7 @@ use libc::{write, read, lseek, exit, open, close, c_int, c_void}; use super::kvm_header::{kvm_run, KVM_EXIT_IO, KVM_EXIT_HLT, KVM_EXIT_MMIO,KVM_EXIT_FAIL_ENTRY, KVM_EXIT_INTERNAL_ERROR, KVM_EXIT_SHUTDOWN }; use std::ffi::CStr; +use super::{Error, Result}; const PORT_WRITE: u16 = 0x499; const PORT_OPEN: u16 = 0x500; @@ -57,6 +58,11 @@ pub enum Syscall { Other(*const kvm_run) } +pub enum Return { + Continue, + Exit(i32) +} + impl Syscall { pub fn from_mem(mem: *const u8, guest_mem: *const u8) -> Syscall { unsafe { @@ -84,7 +90,7 @@ impl Syscall { } - pub unsafe fn run(&self, guest_mem: *mut u8) { + pub unsafe fn run(&self, guest_mem: *mut u8) -> Result { debug!("{:?}", *self); match *self { @@ -95,7 +101,7 @@ impl Syscall { (*obj).ret = read((*obj).fd, guest_mem.offset((*obj).buf) as *mut c_void, (*obj).len); }, Syscall::Exit(obj) => { - exit(*((guest_mem as *mut i32).offset((*obj) as isize))); + return Ok(Return::Exit(*((guest_mem as *mut i32).offset((*obj) as isize)))); }, Syscall::Open(obj) => { (*obj).ret = open(guest_mem.offset((*obj).name) as *const i8, (*obj).flags, (*obj).mode); @@ -108,16 +114,22 @@ impl Syscall { Syscall::LSeek(obj) => { (*obj).offset = lseek((*obj).fd, (*obj).offset as i64, (*obj).whence); }, - Syscall::Other(id) => { match (*id).exit_reason { - KVM_EXIT_HLT => panic!("Guest has halted the CPU, this is considered as a normal exit."), - KVM_EXIT_MMIO =>panic!("KVM: unhandled KVM_EXIT_MMIO at 0x{}", (*id).__bindgen_anon_1.mmio.phys_addr ), - KVM_EXIT_FAIL_ENTRY => panic!("KVM: entry failure: hw_entry_failure_reason=0x{}", (*id).__bindgen_anon_1.fail_entry.hardware_entry_failure_reason), - KVM_EXIT_INTERNAL_ERROR => panic!("KVM: internal error exit: suberror = 0x{}", (*id).__bindgen_anon_1.internal.suberror), - KVM_EXIT_SHUTDOWN => panic!("KVM: receive shutdown command"), - _ => { - panic!("KVM: unhandled exit: exit_reason = 0x{}", (*id).exit_reason); - } - }} + Syscall::Other(id) => { + let err = match (*id).exit_reason { + KVM_EXIT_HLT => format!("Guest has halted the CPU, this is considered as a normal exit."), + KVM_EXIT_MMIO => panic!("KVM: unhandled KVM_EXIT_MMIO at 0x{}", (*id).__bindgen_anon_1.mmio.phys_addr ), + KVM_EXIT_FAIL_ENTRY => panic!("KVM: entry failure: hw_entry_failure_reason=0x{}", (*id).__bindgen_anon_1.fail_entry.hardware_entry_failure_reason), + KVM_EXIT_INTERNAL_ERROR => panic!("KVM: internal error exit: suberror = 0x{}", (*id).__bindgen_anon_1.internal.suberror), + KVM_EXIT_SHUTDOWN => format!("KVM: receive shutdown command"), + _ => { + panic!("KVM: unhandled exit: exit_reason = 0x{}", (*id).exit_reason) + } + }; + + return Err(Error::Protocol(err)); + } } + + return Ok(Return::Continue); } } diff --git a/tools/hermit_proxy/src/hermit/uhyve/uhyve.rs b/tools/hermit_proxy/src/hermit/uhyve/uhyve.rs index 0ada68f2f..54ecc60ed 100644 --- a/tools/hermit_proxy/src/hermit/uhyve/uhyve.rs +++ b/tools/hermit_proxy/src/hermit/uhyve/uhyve.rs @@ -10,6 +10,7 @@ use libc; use hermit::Isle; use super::{Error, Result, NameIOCTL}; use super::vm::VirtualMachine; +use hermit::hermit_env; /// The normal way of defining a IOCTL interface is provided by C macros. In Rust we have our own /// flawed macro system. The module below wraps a bunch of functions which are generated by the @@ -92,10 +93,10 @@ impl KVM { } // Creates a new virtual machine and forwards the new fd to an object - pub fn create_vm(&self, size: usize) -> Result { + pub fn create_vm(&self, size: usize,num_cpus: u32) -> Result { unsafe { match ioctl::create_vm(self.file.as_raw_fd(), ptr::null_mut()) { - Ok(vm_fd) => VirtualMachine::new(self.file.as_raw_fd(), vm_fd, size), + Ok(vm_fd) => VirtualMachine::new(self.file.as_raw_fd(), vm_fd, size,num_cpus), Err(_) => Err(Error::InternalError) } } @@ -112,7 +113,7 @@ pub struct Uhyve { impl Uhyve { pub fn new(path: &str) -> Result { let kvm = KVM::new(); - let mut vm = kvm.create_vm(0x20000000)?; + let mut vm = kvm.create_vm(hermit_env::mem_size_parsed(), hermit_env::num_cpus_parsed())?; vm.load_kernel(path)?; vm.init()?; @@ -141,7 +142,7 @@ impl Isle for Uhyve { } fn run(&mut self) -> Result<()> { - self.vm.run(); + self.vm.run()?; Ok(()) } diff --git a/tools/hermit_proxy/src/hermit/uhyve/vcpu.rs b/tools/hermit_proxy/src/hermit/uhyve/vcpu.rs index a1396b81b..ed857f49f 100644 --- a/tools/hermit_proxy/src/hermit/uhyve/vcpu.rs +++ b/tools/hermit_proxy/src/hermit/uhyve/vcpu.rs @@ -8,6 +8,7 @@ use std::thread; use std::thread::current; use std::ptr::Unique; use std::sync::Arc; +use std::thread::JoinHandle; use memmap::{Mmap, Protection}; use errno::errno; @@ -59,7 +60,7 @@ impl VirtualCPU { debug!("UHYVE - New virtual CPU with id {}", id); // create a new VCPU and save the file descriptor - let fd = VirtualCPU::create_vcpu(vm_fd)?; + let fd = VirtualCPU::create_vcpu(vm_fd, id as u32)?; debug!("Got fd {}", fd); @@ -87,32 +88,27 @@ impl VirtualCPU { } + debug!("Initialize the register of {} with start address {:?}", id, entry); + + + debug!("Set the CPUID"); + + cpu.setup_cpuid()?; + let mut regs = kvm_regs::empty(); regs.rip = entry; regs.rflags = 0x2; - regs.rax = 2; - regs.rbx = 2; - - debug!("Initialize the register of {} with start address {:?}", id, entry); - cpu.set_regs(regs)?; - debug!("Set the CPUID"); - - cpu.setup_cpuid()?; - - unsafe { - while volatile_load(mboot.offset(0x20)) < id {} - - volatile_store(mboot.offset(0x30), id); - } Ok(cpu) } - pub fn create_vcpu(fd: c_int) -> Result { + pub fn create_vcpu(fd: c_int, mut id: u32) -> Result { + let a = (&mut id) as *mut u32 as *mut u8; + unsafe { - uhyve::ioctl::create_vcpu(fd, ptr::null_mut()) + uhyve::ioctl::create_vcpu(fd, id as *mut u8) .map_err(|_| Error::IOCTL(NameIOCTL::CreateVcpu)) } } @@ -178,17 +174,17 @@ impl VirtualCPU { } } - pub fn single_run(obj: &VirtualCPU) -> Result { + pub fn single_run(obj: &VirtualCPU) -> Result { let ret = unsafe { uhyve::ioctl::run(obj.vcpu_fd, ptr::null_mut()) .map_err(|x| Error::IOCTL(NameIOCTL::Run)) }?; - debug!("Run with {}", ret); + debug!("Single Run CPU {}", obj.id); if ret == -1 { match errno().0 { - libc::EINTR => { return Ok(ret); }, + libc::EINTR => { return Ok(proto::Return::Continue); }, libc::EFAULT => { // TODO panic!("translation fault!"); @@ -200,32 +196,35 @@ impl VirtualCPU { } unsafe { - proto::Syscall::from_mem(obj.run_mem.ptr(), obj.guest_mem).run(obj.guest_mem); + proto::Syscall::from_mem(obj.run_mem.ptr(), obj.guest_mem).run(obj.guest_mem) } - - Ok(ret) } - pub fn run(self) -> Result { - - if self.id == 0 { - let wrapped = Arc::new(self); - loop { - VirtualCPU::single_run(&wrapped)?; - } - } else { - let wrapped = Arc::new(self); + pub fn run(self) -> JoinHandle> { + debug!("Run CPU {}", self.id); + + let wrapped = Arc::new(self); - let child = thread::spawn(move || { - loop { - VirtualCPU::single_run(&wrapped); + let child = thread::spawn(move || { + unsafe { + while volatile_load(wrapped.mboot.offset(0x20)) < wrapped.id { + thread::yield_now(); + debug!("{} - {}", wrapped.id, volatile_load(wrapped.mboot.offset(0x20))); } - }); - child.join(); - } + volatile_store(wrapped.mboot.offset(0x30), wrapped.id); + } - Ok(0) + loop { + match VirtualCPU::single_run(&wrapped) { + Ok(proto::Return::Exit(code)) => return Ok(code), + Err(err) => return Err(err), + _ => {} + } + } + }); + + child } pub fn setup_first(&self, mem: &mut [u8]) -> Result<()> { diff --git a/tools/hermit_proxy/src/hermit/uhyve/vm.rs b/tools/hermit_proxy/src/hermit/uhyve/vm.rs index 7726cc11b..c54c5d8db 100644 --- a/tools/hermit_proxy/src/hermit/uhyve/vm.rs +++ b/tools/hermit_proxy/src/hermit/uhyve/vm.rs @@ -10,6 +10,7 @@ use elf; use elf::types::{ELFCLASS64, OSABI, PT_LOAD}; use std::ffi::CStr; +use hermit::hermit_env; use hermit::utils; use hermit::uhyve; use super::kvm_header::{kvm_userspace_memory_region, KVM_CAP_SYNC_MMU, KVM_32BIT_GAP_START, KVM_32BIT_GAP_SIZE}; @@ -27,11 +28,12 @@ pub struct VirtualMachine { elf_header: Option, klog: Option<*const i8>, mboot: Option<*mut u8>, - vcpus: Vec + vcpus: Vec, + num_cpus: u32 } impl VirtualMachine { - pub fn new(kvm_fd: libc::c_int, fd: libc::c_int, size: usize) -> Result { + pub fn new(kvm_fd: libc::c_int, fd: libc::c_int, size: usize, num_cpus: u32) -> Result { debug!("New virtual machine with memory size {}", size); // create a new memory region to map the memory of our guest @@ -46,7 +48,7 @@ impl VirtualMachine { unsafe { libc::mprotect((mem.mut_ptr() as *mut libc::c_void).offset(KVM_32BIT_GAP_START as isize), KVM_32BIT_GAP_START, libc::PROT_NONE); } } - Ok(VirtualMachine { kvm_fd: kvm_fd, vm_fd: fd, mem: mem, elf_header: None, klog: None, vcpus: Vec::new(), mboot: None }) + Ok(VirtualMachine { kvm_fd: kvm_fd, vm_fd: fd, mem: mem, elf_header: None, klog: None, vcpus: Vec::new(), mboot: None, num_cpus: num_cpus}) } /// Loads a kernel from path and initialite mem and elf_entry @@ -147,7 +149,11 @@ impl VirtualMachine { self.create_irqchip()?; - self.create_vcpu(0) + for i in 0..self.num_cpus { + self.create_vcpu(i as u8)?; + } + + Ok(()) } pub fn set_user_memory_region(&self, mut region: kvm_userspace_memory_region) -> Result<()> { @@ -192,15 +198,6 @@ impl VirtualMachine { let id = id as usize; - /* - let len = self.vcpus.len(); - if len < id+1 { - debug!("Reserve one!"); - self.vcpus.reserve(id - len + 1); - } - - self.vcpus[id] = cpu; -*/ self.vcpus.insert(id, cpu); Ok(()) @@ -215,15 +212,24 @@ impl VirtualMachine { pub fn run(&mut self) -> Result<()> { let mut guest_mem = unsafe { self.mem.as_mut_slice() }; - + + unsafe { *(self.mboot.unwrap().offset(0x24) as *mut u32) = self.num_cpus; } + + let mut handles = vec![]; loop { if let Some(vcpu) = self.vcpus.pop() { - vcpu.run()?; + handles.push(vcpu.run()); } else { break; } } + for handle in handles { + let code = handle.join().unwrap()?; + + debug!("Exited with {}", code); + } + Ok(()) } @@ -234,7 +240,9 @@ impl VirtualMachine { impl Drop for VirtualMachine { fn drop(&mut self) { + debug!("Drop the Virtual Machine"); if let Ok(output) = self.output() { + debug!("-------- Output --------"); debug!("{}", output); } } diff --git a/tools/hermit_proxy/src/hermit/utils.rs b/tools/hermit_proxy/src/hermit/utils.rs index 650806e82..c2e35458d 100644 --- a/tools/hermit_proxy/src/hermit/utils.rs +++ b/tools/hermit_proxy/src/hermit/utils.rs @@ -31,6 +31,23 @@ pub fn cpufreq() -> Result { Err(Error::MissingFrequency) } +pub fn parse_mem(mem: &str) -> Result { + let (num, postfix): (String, String) = mem.chars().partition(|&x| x.is_numeric()); + let num = num.parse::().map_err(|_| Error::ParseMemory)?; + + let factor = match postfix.as_str() { + "E" | "e" => 1 << 60, + "P" | "p" => 1 << 50, + "T" | "t" => 1 << 40, + "G" | "g" => 1 << 30, + "M" | "m" => 1 << 20, + "K" | "k" => 1 << 10, + _ => return Err(Error::ParseMemory) + }; + + Ok(num*factor) +} + /// Creates a temporary file /tmp/-XXXX, the X will be replaced by an arbitrary sequence pub fn create_tmp_file(name: &str) -> Result { unsafe {