diff --git a/tools/hermit_proxy/Cargo.toml b/tools/hermit_proxy/Cargo.toml index d7fe374aa..5c1a8bdaa 100755 --- a/tools/hermit_proxy/Cargo.toml +++ b/tools/hermit_proxy/Cargo.toml @@ -13,6 +13,10 @@ inotify = "0.4" byteorder = "1" log = "0.3" clap = "2.24.2" +serde = "1.*.*" +chrono = { version = "0.4", features = ["serde"] } +bincode = "0.8" +serde_derive = "1.*.*" [dependencies.env_logger] version = "0.3" diff --git a/tools/hermit_proxy/src/daemon.rs b/tools/hermit_proxy/src/daemon.rs new file mode 100644 index 000000000..15dcb39d0 --- /dev/null +++ b/tools/hermit_proxy/src/daemon.rs @@ -0,0 +1,181 @@ +use std::os::unix::net::{UnixStream, UnixListener}; +use std::path::Path; +use std::thread; +use nix::unistd::{fork, ForkResult}; +use hermit::Isle; +use std::io::Write; +use std::io::Read; +use bincode::{serialize, deserialize, Infinite}; +use std::fs; +use hermit::IsleParameter; +use hermit::qemu::QEmu; +use hermit::multi::Multi; +use hermit::uhyve::Uhyve; +use hermit::error::Result; +use chrono::{DateTime,Local}; + +pub struct Connection { + pub socket: UnixStream +} + +impl Connection { + pub fn connect() -> Connection { + if !Path::new("/tmp/hermit_daemon").exists() { + match fork() { + Ok(ForkResult::Child) => daemon_handler(), + _ => {} + } + } + + loop { + if let Ok(socket) = UnixStream::connect("/tmp/hermit_daemon") { + return Connection { socket: socket }; + } + } + } + + pub fn send(&mut self,action: Action) -> ActionResult { + let encoded = serialize(&action, Infinite).unwrap(); + + self.socket.write(&encoded).unwrap(); + + let mut buf = Vec::new(); + + self.socket.read_to_end(&mut buf).unwrap(); + + deserialize(&buf).unwrap() + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum Action { + CreateIsle(String, IsleParameter), + StopIsle(u32), + RemoveIsle(u32), + Log(Option), + List, + KillDaemon +} + +#[derive(Serialize, Deserialize, Debug)] +pub enum ActionResult { + CreateIsle(Result), + StopIsle(Result), + RemoveIsle(Result), + Log(Vec), + IsleLog(Result), + List(Vec<(Result, IsleParameter)>), + KillDaemon(Result<()>) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Log { + pub time: DateTime, + pub action: Action +} + +struct State { + isles: Vec>, + specs: Vec, + logs: Vec +} + +impl State { + pub fn new() -> State { + State { isles: Vec::new(), logs: Vec::new(), specs: Vec::new() } + } + + pub fn create_isle(&mut self, path: String, specs: IsleParameter) -> Result { + self.specs.push(specs.clone()); + + let mut isle: Box = match specs { + IsleParameter::QEmu { mem_size, num_cpus, additional} => Box::new(QEmu::new(&path, mem_size, num_cpus, additional)?), + IsleParameter::UHyve{ mem_size, num_cpus } => Box::new(Uhyve::new(&path, mem_size, num_cpus)?), + IsleParameter::Multi{ mem_size, num_cpus } => Box::new(Multi::new(0, &path, mem_size, num_cpus)?) + }; + + isle.wait_until_available()?; + isle.run()?; + + self.isles.push(isle); + + Ok(self.isles.len() as u32 -1) + } + + pub fn list(&mut self) -> Vec<(Result, IsleParameter)> { + self.specs.iter().zip(self.isles.iter_mut()).map(|(x,y)| (y.is_running(), x.clone())).collect() + } + + pub fn log(&self) -> Vec { + self.logs.clone() + } + + pub fn add_log(&mut self, action: Action) { + self.logs.push(Log { time: Local::now(), action: action }); + } + + pub fn stop_isle(&mut self, id: u32) -> Result { + self.isles[id as usize].stop() + } + + pub fn remove_isle(&mut self, id: u32) -> Result { + if self.isles[id as usize].is_running()? { + self.isles[id as usize].stop()?; + } + + self.isles.remove(id as usize); + self.specs.remove(id as usize); + + Ok(0) + } +} + +pub fn daemon_handler() { + let mut state = State::new(); + let mut buf = vec![0; 256]; + + let listener = UnixListener::bind("/tmp/hermit_daemon").unwrap(); + //for stream in listener.incoming() { + // match stream { + // Ok(mut stream) => { + loop { match listener.accept() { + Ok((mut stream, addr)) => { +// loop { + if let Ok(nread) = stream.read(&mut buf) { + if nread > 0 { + let action:Action = deserialize(&buf).unwrap(); + + state.add_log(action.clone()); + + let ret = match action { + Action::KillDaemon => { + fs::remove_file("/tmp/hermit_daemon").unwrap(); + break; + }, + Action::CreateIsle(path, specs) => ActionResult::CreateIsle(state.create_isle(path,specs)), + Action::List => ActionResult::List(state.list()), + Action::Log(id) => { + match id { + Some(id) => ActionResult::IsleLog(state.isles[id as usize].output()), + None => ActionResult::Log(state.log()) + } + }, + Action::StopIsle(id) => ActionResult::StopIsle(state.stop_isle(id)), + Action::RemoveIsle(id) => ActionResult::RemoveIsle(state.remove_isle(id)), + _ => { panic!(""); } + }; + + let buf: Vec = serialize(&ret, Infinite).unwrap(); + stream.write(&buf); + } + } else { + + } + // } + }, + Err(err) => { + println!("ERR: {:?}", err); + } + } + } +} diff --git a/tools/hermit_proxy/src/hermit/error.rs b/tools/hermit_proxy/src/hermit/error.rs index 22329e8cb..d1dbb6837 100644 --- a/tools/hermit_proxy/src/hermit/error.rs +++ b/tools/hermit_proxy/src/hermit/error.rs @@ -4,6 +4,7 @@ use errno::errno; pub type Result = result::Result; +#[derive(Deserialize, Serialize)] pub enum Error { InternalError, NotEnoughMemory, @@ -42,7 +43,7 @@ impl fmt::Debug for Error { } } -#[derive(Debug)] +#[derive(Serialize, Deserialize,Debug)] pub enum NameIOCTL { GetVersion, CreateVM, diff --git a/tools/hermit_proxy/src/hermit/mod.rs b/tools/hermit_proxy/src/hermit/mod.rs index a830c3333..bf837f66c 100644 --- a/tools/hermit_proxy/src/hermit/mod.rs +++ b/tools/hermit_proxy/src/hermit/mod.rs @@ -1,16 +1,16 @@ pub mod error; mod utils; -mod hermit_env; -mod qemu; -mod multi; +pub mod qemu; +pub mod multi; mod proto; mod socket; -mod uhyve; +pub mod uhyve; use std::fs::File; use std::path::Path; use std::io::{Write, Read, BufReader, BufRead}; use inotify::{Inotify, watch_mask}; +use std::env; use hermit::qemu::QEmu; use hermit::multi::Multi; @@ -18,15 +18,78 @@ use hermit::uhyve::uhyve::Uhyve; use hermit::error::*; -pub fn new_isle(path: &str) -> Result> { - let isle = hermit_env::isle_kind(); - - debug!("Create a new {} isle", isle); - - match isle.as_str() { - "qemu"=> Ok(Box::new(QEmu::new(path)?)), - "uhyve" => Ok(Box::new(Uhyve::new(path)?)), - s => Ok(Box::new(Multi::new(s.parse::().unwrap_or(0), path)?)) +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct IsleParameterQEmu { + binary: String, + use_kvm: bool, + monitor: bool, + capture_net:bool, + port: u16, + should_debug: bool, + app_port: u16 +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub enum IsleParameter { + QEmu { + mem_size: u64, + num_cpus: u32, + additional: IsleParameterQEmu + }, + UHyve { + mem_size: u64, + num_cpus: u32 + }, + Multi { + mem_size: u64, + num_cpus: u32 + } +} + +impl IsleParameter { + pub fn from_env() -> IsleParameter { + let isle_kind = env::var("HERMIT_ISLE").unwrap_or("qemu".into()); + let mem_size: u64 = env::var("HERMIT_MEM").map(|x| utils::parse_mem(&x).unwrap_or(512*1024*1024)).unwrap_or(512*1024*1024); + let num_cpus: u32 = env::var("HERMIT_CPUS").map(|x| x.parse().unwrap_or(1)).unwrap_or(1); + + match isle_kind.as_str() { + "qemu" | "QEmu" | "QEMU" => { + let binary = env::var("HERMIT_BINARY").unwrap_or("qemu-system-x86_64".into()); + let kvm = env::var("HERMIT_KVM").map(|x| x.parse().unwrap_or(false)).unwrap_or(false); + let monitor = env::var("HERMIT_MONITOR").map(|x| x.parse().unwrap_or(false)).unwrap_or(false); + let capture_net = env::var("HERMIT_CAPTURE_NET").map(|x| x.parse().unwrap_or(false)).unwrap_or(false); + let port = env::var("HERMIT_PORT").map(|x| x.parse().unwrap_or(18766)).unwrap_or(18766); + let app_port = env::var("HERMIT_APP_PORT").map(|x| x.parse().unwrap_or(0)).unwrap_or(0); + let debug = env::var("HERMIT_DEBUG").map(|x| x.parse().unwrap_or(false)).unwrap_or(false); + + IsleParameter::QEmu { + mem_size: mem_size, + num_cpus: num_cpus, + additional: IsleParameterQEmu { + binary: binary, + use_kvm: kvm, + monitor: monitor, + capture_net: capture_net, + port:port, + app_port: app_port, + should_debug: debug + } + } + }, + "multi" | "MULTI" | "Multi" => { + IsleParameter::Multi { + mem_size: mem_size, + num_cpus: num_cpus + } + }, + "uhyve" | "UHyve" | "UHYVE" => { + IsleParameter::UHyve { + mem_size: mem_size, + num_cpus: num_cpus + } + }, + _ => { panic!(""); } + } } } @@ -36,20 +99,25 @@ pub trait Isle { fn log_path(&self) -> Option; fn cpu_path(&self) -> Option; + fn output(&self) -> Result; + + fn stop(&mut self) -> Result; fn run(&mut self) -> Result<()>; + fn is_running(&mut self) -> Result; + fn is_available(&self) -> Result { let log = match self.log_file() { Some(f) => f, None => return Ok(false) }; - + // open the log file let mut file = File::open(log) .map_err(|x| Error::InvalidFile(format!("{:?}",x)))?; let mut reader = BufReader::new(file); - + for line in reader.lines() { if line.unwrap().contains("TCP server is listening.") { debug!("Found key token, continue"); @@ -97,20 +165,4 @@ pub trait Isle { } } - fn stop(&self) -> Result<()> { - debug!("Stop the HermitIsle"); - - let cpu_path = match self.cpu_path() { - Some(f) => f, - None => return Ok(()) - }; - - let mut cpus_file = File::create(&cpu_path) - .map_err(|_| Error::InvalidFile(cpu_path.clone()))?; - - cpus_file.write("-1".as_bytes()) - .map_err(|_| Error::InvalidFile(cpu_path)); - - Ok(()) - } } diff --git a/tools/hermit_proxy/src/hermit/multi.rs b/tools/hermit_proxy/src/hermit/multi.rs index dc9d28c2a..f81f21850 100644 --- a/tools/hermit_proxy/src/hermit/multi.rs +++ b/tools/hermit_proxy/src/hermit/multi.rs @@ -4,7 +4,6 @@ use std::io::{Write, Read}; use hermit::Isle; use hermit::error::*; -use hermit::hermit_env; use hermit::socket::Socket; pub struct Multi { @@ -13,7 +12,7 @@ pub struct Multi { } impl Multi { - pub fn new(num: u8, path: &str) -> Result { + pub fn new(num: u8, path: &str, mem_size: u64, num_cpus: u32) -> Result { let cpu_path = format!("/sys/hermit/isle{}/path", num); let bin_path= format!("/sys/hermit/isle{}/cpus", num); @@ -25,7 +24,7 @@ impl Multi { let mut cpus_file = File::create(&cpu_path) .map_err(|_| Error::InvalidFile(cpu_path.clone()))?; - let cpus = hermit_env::num_cpus(); + let cpus = num_cpus.to_string(); path_file.write_all(path.as_bytes()) .map_err(|_| Error::InvalidFile(cpu_path.clone()))?; @@ -73,4 +72,28 @@ impl Isle for Multi { Ok(()) } + + fn output(&self) -> Result { + Ok("".into()) + } + fn stop(&mut self) -> Result { + debug!("Stop the HermitIsle"); + + let cpu_path = match self.cpu_path() { + Some(f) => f, + None => return Ok(0) + }; + + let mut cpus_file = File::create(&cpu_path) + .map_err(|_| Error::InvalidFile(cpu_path.clone()))?; + + cpus_file.write("-1".as_bytes()) + .map_err(|_| Error::InvalidFile(cpu_path)); + + Ok(0) + } + + fn is_running(&mut self) -> Result { + Ok(true) + } } diff --git a/tools/hermit_proxy/src/hermit/qemu.rs b/tools/hermit_proxy/src/hermit/qemu.rs index 59d0ec9fc..07262e6a2 100644 --- a/tools/hermit_proxy/src/hermit/qemu.rs +++ b/tools/hermit_proxy/src/hermit/qemu.rs @@ -4,11 +4,11 @@ use libc; use std::fs::File; use std::io::Read; use std::process::{ChildStdout, ChildStderr}; +use std::thread; -use hermit::Isle; +use hermit::{Isle, IsleParameterQEmu}; use hermit::utils; use hermit::error::*; -use hermit::hermit_env; use hermit::socket::Socket; const PIDNAME: &'static str = "/tmp/hpid-XXXXXX"; @@ -16,7 +16,7 @@ const TMPNAME: &'static str = "/tmp/hermit-XXXXXX"; #[derive(Debug)] pub struct QEmu { - socket: Socket, + socket: Option, child: Child, stdout: ChildStdout, stderr: ChildStderr, @@ -25,16 +25,16 @@ pub struct QEmu { } impl QEmu { - pub fn new(path: &str) -> Result { + pub fn new(path: &str, mem_size: u64, num_cpus: u32, additional: IsleParameterQEmu) -> Result { let tmpf = utils::create_tmp_file(TMPNAME)?; let pidf = utils::create_tmp_file(PIDNAME)?; - let mut child = QEmu::start_with(path, &tmpf, &pidf)?.spawn().expect("Couldn't find qemu binary!"); + let mut child = QEmu::start_with(path, mem_size, num_cpus, additional, &tmpf, &pidf)?.spawn().expect("Couldn't find qemu binary!"); let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); Ok(QEmu { - socket: Socket::new_qemu(), + socket: Some(Socket::new_qemu()), child: child, stdout: stdout, stderr: stderr, @@ -43,13 +43,13 @@ impl QEmu { }) } - pub fn start_with(path: &str, tmp_file: &str, pid_file: &str) -> Result { - let hostfwd = format!("user,hostfwd=tcp:127.0.0.1:{}-:{}", hermit_env::port(), hermit_env::port()); - let monitor_str = format!("telnet:127.0.0.1:{},server,nowait", (hermit_env::port().parse::().unwrap()+1).to_string()); + pub fn start_with(path: &str, mem_size: u64, num_cpus: u32, add: IsleParameterQEmu, tmp_file: &str, pid_file: &str) -> Result { + let hostfwd = format!("user,hostfwd=tcp:127.0.0.1:{}-:{}", add.port, add.port); + let monitor_str = format!("telnet:127.0.0.1:{},server,nowait", add.port+1); let chardev = format!("file,id=gnc0,path={}",&tmp_file); let freq = format!("\"-freq{} -proxy\"",(utils::cpufreq().unwrap()/1000).to_string()); - let cpus = hermit_env::num_cpus(); - let mem_size = hermit_env::mem_size(); + let num_cpus = num_cpus.to_string(); + let mem_size = mem_size.to_string(); let exe = env::current_exe().unwrap(); let name = exe.to_str().unwrap(); @@ -62,7 +62,7 @@ impl QEmu { let mut args: Vec<&str> = vec![ "-daemonize", "-display", "none", - "-smp", &cpus, + "-smp", &num_cpus, "-m", &mem_size, "-pidfile", pid_file, "-net", "nic,model=rtl8139", @@ -73,39 +73,40 @@ impl QEmu { "-initrd", path, "-append", &freq]; - let app_port = hermit_env::app_port(); - if app_port != "" { + let app_port = add.app_port; + if app_port != 0 { port_str = format!("{},hostfwd=tcp::{}-:{}", hostfwd,app_port, app_port); args[12] = &port_str; } - if hermit_env::use_kvm() != "0" { + if add.use_kvm { args.push("-machine"); args.push("accel=kvm"); args.push("-cpu"); args.push("host"); } - if hermit_env::monitor() != "0" { + //if add.monitor { args.push("-monitor"); args.push(&monitor_str); - } + //} - if hermit_env::should_debug() != "0" { + if add.should_debug { args.push("-s"); } - if hermit_env::capture_net() != "0" { + if add.capture_net { args.push("-net"); args.push("dump"); } + /* if hermit_env::verbose() != "0" { debug!("Execute {} with args {:#?}", hermit_env::qemu_binary(), args); - } + }*/ - let mut cmd = Command::new(hermit_env::qemu_binary()); + let mut cmd = Command::new(add.binary); cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); @@ -113,7 +114,7 @@ impl QEmu { Ok(cmd) } - pub fn output(&mut self) -> (String, String) { + pub fn qemu_log(&mut self) -> (String, String) { let mut stderr = String::new(); let mut stdout = String::new(); @@ -137,10 +138,32 @@ impl Isle for QEmu { } fn run(&mut self) -> Result<()> { - self.socket.connect().run(); + let socket = self.socket.take(); + + thread::spawn(|| { + socket.unwrap().connect().run(); + }); Ok(()) } + + fn output(&self) -> Result { + /*if let &Socket::Connected { ref stdout, ref stderr, .. } = &self.socket.unwrap() { + Ok(stdout.clone()) + } else { + Err(Error::InternalError) + }*/ + + Err(Error::InternalError) + } + + fn stop(&mut self) -> Result { + Ok(0) + } + + fn is_running(&mut self) -> Result { + Ok(true) + } } impl Drop for QEmu { diff --git a/tools/hermit_proxy/src/hermit/socket.rs b/tools/hermit_proxy/src/hermit/socket.rs index d0e9ddf03..29e3adb55 100644 --- a/tools/hermit_proxy/src/hermit/socket.rs +++ b/tools/hermit_proxy/src/hermit/socket.rs @@ -3,6 +3,7 @@ use std::env; use std::mem::transmute; use std::io::{Write, Read, Cursor}; use std::ffi::CString; +use std::ffi::CStr; use std::process; use byteorder::{ReadBytesExt, WriteBytesExt, LittleEndian}; @@ -17,7 +18,7 @@ const HERMIT_MAGIC: u32 = 0x7E317; pub enum Socket { QEmu, Multi(u8), - Connected { stream: TcpStream } + Connected { stream: TcpStream, stdout: String, stderr: String } } impl Socket { @@ -62,14 +63,14 @@ impl Socket { debug!("Transmitted environment and arguments with length {}", length); - Socket::Connected { stream: stream } + Socket::Connected { stream: stream, stdout: String::new(), stderr: String::new() } } pub fn run(&mut self) { debug!("Initializing protocol state machine"); let mut state = proto::State::Id; - let mut stream = match self { - &mut Socket::Connected { ref mut stream } => stream, + let (mut stream, mut stdout, mut stderr) = match self { + &mut Socket::Connected { ref mut stream, ref mut stdout, ref mut stderr } => (stream, stdout, stderr), _ => return }; @@ -100,7 +101,11 @@ impl Socket { Packet::Write { fd, buf } => { let buf_ret: [u8;8] = transmute(libc::write(fd as i32, buf.as_ptr() as *const libc::c_void, buf.len()).to_le()); - if fd > 2 { + if fd == 1 { + stdout.push_str(&String::from_utf8_unchecked(buf)); + } else if fd == 2 { + stderr.push_str(&String::from_utf8_unchecked(buf)); + } else { stream.write(&buf_ret); } }, @@ -109,7 +114,6 @@ impl Socket { debug!("got {:?}", buf); let written = stream.write(&buf).unwrap(); - //let written = stream.write(&[0,0,0,1]).unwrap(); debug!("Written {}", written); }, diff --git a/tools/hermit_proxy/src/hermit/uhyve/proto.rs b/tools/hermit_proxy/src/hermit/uhyve/proto.rs index c2a72ca92..523526760 100644 --- a/tools/hermit_proxy/src/hermit/uhyve/proto.rs +++ b/tools/hermit_proxy/src/hermit/uhyve/proto.rs @@ -96,6 +96,9 @@ impl Syscall { match *self { Syscall::Write(obj) => { + if (*obj).fd == 1 { + } else if (*obj).fd == 2 { + } (*obj).length = write((*obj).fd, guest_mem.offset((*obj).buf) as *const c_void, (*obj).length as usize); }, Syscall::Read(obj) => { diff --git a/tools/hermit_proxy/src/hermit/uhyve/uhyve.rs b/tools/hermit_proxy/src/hermit/uhyve/uhyve.rs index 54ecc60ed..82ad6acc0 100644 --- a/tools/hermit_proxy/src/hermit/uhyve/uhyve.rs +++ b/tools/hermit_proxy/src/hermit/uhyve/uhyve.rs @@ -10,7 +10,6 @@ use libc; use hermit::Isle; use super::{Error, Result, NameIOCTL}; use super::vm::VirtualMachine; -use hermit::hermit_env; /// The normal way of defining a IOCTL interface is provided by C macros. In Rust we have our own /// flawed macro system. The module below wraps a bunch of functions which are generated by the @@ -111,9 +110,9 @@ pub struct Uhyve { } impl Uhyve { - pub fn new(path: &str) -> Result { + pub fn new(path: &str, mem_size: u64, num_cpus: u32) -> Result { let kvm = KVM::new(); - let mut vm = kvm.create_vm(hermit_env::mem_size_parsed(), hermit_env::num_cpus_parsed())?; + let mut vm = kvm.create_vm(mem_size as usize, num_cpus)?; vm.load_kernel(path)?; vm.init()?; @@ -142,8 +141,18 @@ impl Isle for Uhyve { } fn run(&mut self) -> Result<()> { - self.vm.run()?; + self.vm.run().map(|_| ()) + } - Ok(()) + fn output(&self) -> Result { + self.vm.output() + } + + fn stop(&mut self) -> Result { + self.vm.stop() + } + + fn is_running(&mut self) -> Result { + self.vm.is_running() } } diff --git a/tools/hermit_proxy/src/hermit/uhyve/vcpu.rs b/tools/hermit_proxy/src/hermit/uhyve/vcpu.rs index ad94b9519..73da43975 100644 --- a/tools/hermit_proxy/src/hermit/uhyve/vcpu.rs +++ b/tools/hermit_proxy/src/hermit/uhyve/vcpu.rs @@ -1,14 +1,15 @@ use libc; -use libc::{c_void, c_int}; +use libc::c_void; use std::{mem, ptr}; use std::fs::File; -use std::os::unix::io::FromRawFd; +use std::os::unix::io::{FromRawFd, RawFd}; use std::intrinsics::{volatile_load,volatile_store}; use std::thread; use std::thread::current; use std::ptr::Unique; use std::sync::Arc; use std::thread::JoinHandle; +use std::sync::atomic::{AtomicBool, Ordering}; use memmap::{Mmap, Protection}; use errno::errno; @@ -44,25 +45,33 @@ pub const X86_PDPT_P: u64 = (1 << 0); pub const X86_PDPT_RW: u64 = (1 << 1); pub const X86_PDPT_PS: u64 = (1 << 7); -pub struct VirtualCPU { - kvm_fd: libc::c_int, - vm_fd: libc::c_int, - pub vcpu_fd: libc::c_int, - id: u8, - file: File, +pub enum ExitCode { + Cause(Result), + Innocent +} + +pub struct SharedState { run_mem: Mmap, mboot:*mut u8, - guest_mem: *mut u8 + guest_mem: *mut u8, + running_state: Arc +} + +pub struct VirtualCPU { + id: u32, + kvm_fd: RawFd, + vm_fd: RawFd, + vcpu_fd: RawFd, + state: Arc } impl VirtualCPU { - pub fn new(kvm_fd: libc::c_int, vm_fd: libc::c_int, id: u8, entry: u64, mem: &mut Mmap, mboot: *mut u8) -> Result { - debug!("UHYVE - New virtual CPU with id {}", id); + pub fn new(kvm_fd: RawFd, vm_fd: RawFd, id: u32, entry: u64, mem: &mut Mmap, mboot: *mut u8, running_state: Arc) -> Result { // create a new VCPU and save the file descriptor let fd = VirtualCPU::create_vcpu(vm_fd, id as u32)?; - debug!("Got fd {}", fd); + debug!("New virtual CPU with id {} and FD {}", id, fd); let file = unsafe { File::from_raw_fd(fd) }; @@ -71,8 +80,22 @@ impl VirtualCPU { .map_err(|x| panic!("{:?}", x) )//Error::InvalidFile) }?; + // forget the file, we don't want to close the file descriptor + mem::forget(file); + + let state = SharedState { + run_mem: run_mem, + mboot: mboot, + guest_mem: mem.mut_ptr(), + running_state: running_state + }; + let cpu = VirtualCPU { - kvm_fd: kvm_fd, vm_fd: vm_fd, vcpu_fd: fd, id: id, file: file, run_mem: run_mem, mboot: mboot, guest_mem: mem.mut_ptr() + kvm_fd: kvm_fd, + vm_fd: vm_fd, + vcpu_fd: fd, + id: id, + state: Arc::new(state) }; debug!("Set the CPUID"); @@ -93,7 +116,7 @@ impl VirtualCPU { Ok(cpu) } - pub fn create_vcpu(fd: c_int, mut id: u32) -> Result { + pub fn create_vcpu(fd: RawFd, mut id: u32) -> Result { unsafe { uhyve::ioctl::create_vcpu(fd, id as *mut u8) .map_err(|_| Error::IOCTL(NameIOCTL::CreateVcpu)) @@ -148,7 +171,7 @@ impl VirtualCPU { Ok(()) } - pub fn get_mmap_size(vcpu_fd: libc::c_int) -> Result { + pub fn get_mmap_size(vcpu_fd: RawFd) -> Result { unsafe { uhyve::ioctl::get_vcpu_mmap_size(vcpu_fd, ptr::null_mut()) .map_err(|x| Error::IOCTL(NameIOCTL::GetVCPUMMAPSize)).map(|x| { x as usize}) @@ -157,17 +180,18 @@ impl VirtualCPU { pub fn set_mp_state(&self, mp_state: kvm_mp_state) -> Result<()> { unsafe { - uhyve::ioctl::set_mp_state(self.vcpu_fd, (&mp_state) as *const kvm_mp_state).map_err(|x| Error::IOCTL(NameIOCTL::SetMPState)).map(|_| ()) + uhyve::ioctl::set_mp_state(self.vcpu_fd, (&mp_state) as *const kvm_mp_state) + .map_err(|x| Error::IOCTL(NameIOCTL::SetMPState)).map(|_| ()) } } - pub fn single_run(obj: &VirtualCPU) -> Result { + pub fn single_run(fd: RawFd, state: &Arc) -> Result { let ret = unsafe { - uhyve::ioctl::run(obj.vcpu_fd, ptr::null_mut()) + uhyve::ioctl::run(fd, ptr::null_mut()) .map_err(|x| Error::IOCTL(NameIOCTL::Run)) }?; - debug!("Single Run CPU {}", obj.id); + debug!("Single Run CPU {}", fd); if ret == -1 { match errno().0 { @@ -183,35 +207,45 @@ impl VirtualCPU { } unsafe { - let a = proto::Syscall::from_mem(obj.run_mem.ptr(), obj.guest_mem).run(obj.guest_mem); + let a = proto::Syscall::from_mem(state.run_mem.ptr(), state.guest_mem).run(state.guest_mem); - debug!("{:?}", a); a } } - pub fn run(self) -> JoinHandle> { + pub fn run(&self) -> JoinHandle { debug!("Run CPU {}", self.id); - let wrapped = Arc::new(self); - + let state = self.state.clone(); + let id = self.id; + let fd = self.vcpu_fd; + let child = thread::spawn(move || { unsafe { - while volatile_load(wrapped.mboot.offset(0x20)) < wrapped.id { + while volatile_load(state.mboot.offset(0x20)) < id as u8 { thread::yield_now(); - //debug!("{} - {}", wrapped.id, volatile_load(wrapped.mboot.offset(0x20))); } - volatile_store(wrapped.mboot.offset(0x30), wrapped.id); + volatile_store(state.mboot.offset(0x30), id as u8); } - loop { - match VirtualCPU::single_run(&wrapped) { - Ok(proto::Return::Exit(code)) => return Ok(code), - Err(err) => return Err(err), + while state.running_state.load(Ordering::Relaxed) { + match VirtualCPU::single_run(fd, &state) { + Ok(proto::Return::Exit(code)) => { + state.running_state.store(false, Ordering::Relaxed); + + return ExitCode::Cause(Ok(code)); + }, + Err(err) => { + state.running_state.store(false, Ordering::Relaxed); + + return ExitCode::Cause(Err(err)); + }, _ => {} } } + + ExitCode::Innocent }); child @@ -240,7 +274,7 @@ impl VirtualCPU { // apply the new GDTs to our guest memory unsafe { - let ptr = self.guest_mem.offset(offset as isize) as *mut u64; + let ptr = self.state.guest_mem.offset(offset as isize) as *mut u64; *(ptr.offset(gdt::BOOT_NULL)) = gdt_null.as_u64(); *(ptr.offset(gdt::BOOT_CODE)) = gdt_code.as_u64(); @@ -262,18 +296,10 @@ impl VirtualCPU { Ok(()) } pub fn setup_system_page_table(&self, sregs: &mut kvm_sregs) -> Result<()> { - /*let pml4 = mem[BOOT_PML4..].as_ptr() as *mut u64; - let pdpte = mem[BOOT_PDPTE..].as_ptr() as *mut u64; - let pde = mem[BOOT_PDE..].as_ptr() as *mut u64;*/ - - // TODO - //assert!((guest_size & (GUEST_PAGE_SIZE - 1)) == 0); - //assert!(guest_size <= (GUEST_PAGE_SIZE * 512)); - unsafe { - let pml4 = self.guest_mem.offset(BOOT_PML4 as isize) as *mut u64; - let pdpte = self.guest_mem.offset(BOOT_PDPTE as isize) as *mut u64; - let pde = self.guest_mem.offset(BOOT_PDE as isize) as *mut u64; + let pml4 = self.state.guest_mem.offset(BOOT_PML4 as isize) as *mut u64; + let pdpte = self.state.guest_mem.offset(BOOT_PDPTE as isize) as *mut u64; + let pde = self.state.guest_mem.offset(BOOT_PDE as isize) as *mut u64; libc::memset(pml4 as *mut c_void, 0x00, 4096); libc::memset(pdpte as *mut c_void, 0x00, 4096); @@ -325,5 +351,5 @@ impl VirtualCPU { } } -unsafe impl Sync for VirtualCPU { } -unsafe impl Send for VirtualCPU {} +unsafe impl Sync for SharedState { } +unsafe impl Send for SharedState {} diff --git a/tools/hermit_proxy/src/hermit/uhyve/vm.rs b/tools/hermit_proxy/src/hermit/uhyve/vm.rs index 0eb40edd7..dda68b353 100644 --- a/tools/hermit_proxy/src/hermit/uhyve/vm.rs +++ b/tools/hermit_proxy/src/hermit/uhyve/vm.rs @@ -9,13 +9,15 @@ use memmap::{Mmap, Protection}; use elf; use elf::types::{ELFCLASS64, OSABI, PT_LOAD}; use std::ffi::CStr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread::JoinHandle; -use hermit::hermit_env; use hermit::utils; use hermit::uhyve; use super::kvm_header::{kvm_userspace_memory_region, KVM_CAP_SYNC_MMU, KVM_32BIT_GAP_START, KVM_32BIT_GAP_SIZE, kvm_sregs}; use super::{Result, Error, NameIOCTL}; -use super::vcpu::VirtualCPU; +use super::vcpu::{ExitCode, VirtualCPU}; //use byteorder::ByteOrder; // guest offset? @@ -30,7 +32,9 @@ pub struct VirtualMachine { mboot: Option<*mut u8>, vcpus: Vec, num_cpus: u32, - sregs: kvm_sregs + sregs: kvm_sregs, + running_state: Arc, + thread_handles: Vec> } impl VirtualMachine { @@ -49,7 +53,7 @@ impl VirtualMachine { unsafe { libc::mprotect((mem.mut_ptr() as *mut libc::c_void).offset(KVM_32BIT_GAP_START as isize), KVM_32BIT_GAP_START, libc::PROT_NONE); } } - Ok(VirtualMachine { kvm_fd: kvm_fd, vm_fd: fd, mem: mem, elf_header: None, klog: None, vcpus: Vec::new(), mboot: None, num_cpus: num_cpus, sregs: kvm_sregs::default()}) + Ok(VirtualMachine { kvm_fd: kvm_fd, vm_fd: fd, mem: mem, elf_header: None, klog: None, vcpus: Vec::new(), mboot: None, num_cpus: num_cpus, sregs: kvm_sregs::default(), running_state: Arc::new(AtomicBool::new(false)), thread_handles: Vec::new() }) } /// Loads a kernel from path and initialite mem and elf_entry @@ -152,7 +156,7 @@ impl VirtualMachine { self.create_irqchip()?; for i in 0..self.num_cpus { - self.create_vcpu(i as u8)?; + self.create_vcpu(i as u32)?; } Ok(()) @@ -193,10 +197,10 @@ impl VirtualMachine { } } - pub fn create_vcpu(&mut self, id: u8) -> Result<()> { + pub fn create_vcpu(&mut self, id: u32) -> Result<()> { let entry = self.elf_header.ok_or(Error::KernelNotLoaded)?.entry; - let cpu = VirtualCPU::new(self.kvm_fd, self.vm_fd, id, entry, &mut self.mem,self.mboot.unwrap())?; + let cpu = VirtualCPU::new(self.kvm_fd, self.vm_fd, id, entry, &mut self.mem,self.mboot.unwrap(), self.running_state.clone())?; if id == 0 { self.sregs = cpu.init_sregs()?; @@ -222,25 +226,52 @@ impl VirtualMachine { let mut guest_mem = unsafe { self.mem.as_mut_slice() }; unsafe { *(self.mboot.unwrap().offset(0x24) as *mut u32) = self.num_cpus; } + self.running_state.store(true, Ordering::Relaxed); - let mut handles = vec![]; - loop { - if let Some(vcpu) = self.vcpus.pop() { - handles.push(vcpu.run()); - } else { - break; - } - } - - for handle in handles { - let code = handle.join().unwrap()?; - - debug!("Exited with {}", code); + for vcpu in &self.vcpus { + self.thread_handles.push(vcpu.run()); } Ok(()) } + pub fn stop(&mut self) -> Result { + self.running_state.store(false, Ordering::Relaxed); + + let mut reason = Ok(0); + while let Some(handle) = self.thread_handles.pop() { + if let Ok(ret) = handle.join() { + match ret { + ExitCode::Innocent => continue, + ExitCode::Cause(cause) => { + reason = cause; + } + } + } + } + + reason + } + + pub fn is_running(&mut self) -> Result { + if self.running_state.load(Ordering::Relaxed) { + return Ok(true); + } else { + /*while let Some(handle) = self.thread_handles.pop() { + if let Ok(ret) = handle.join() { + match ret { + ExitCode::Innocent => continue, + ExitCode::Cause(cause) => { + cause?; + } + } + } + }*/ + + return Ok(false); + } + } + pub fn mem_size(&self) -> usize { self.mem.len() } diff --git a/tools/hermit_proxy/src/hermit/utils.rs b/tools/hermit_proxy/src/hermit/utils.rs index c2e35458d..cf7e6084f 100644 --- a/tools/hermit_proxy/src/hermit/utils.rs +++ b/tools/hermit_proxy/src/hermit/utils.rs @@ -31,9 +31,9 @@ pub fn cpufreq() -> Result { Err(Error::MissingFrequency) } -pub fn parse_mem(mem: &str) -> Result { +pub fn parse_mem(mem: &str) -> Result { let (num, postfix): (String, String) = mem.chars().partition(|&x| x.is_numeric()); - let num = num.parse::().map_err(|_| Error::ParseMemory)?; + let num = num.parse::().map_err(|_| Error::ParseMemory)?; let factor = match postfix.as_str() { "E" | "e" => 1 << 60, diff --git a/tools/hermit_proxy/src/main.rs b/tools/hermit_proxy/src/main.rs index 217fcb06d..c75d697a0 100644 --- a/tools/hermit_proxy/src/main.rs +++ b/tools/hermit_proxy/src/main.rs @@ -12,22 +12,31 @@ extern crate env_logger; #[macro_use] extern crate clap; +#[macro_use] +extern crate serde_derive; +extern crate bincode; + extern crate libc; extern crate memmap; extern crate elf; extern crate errno; extern crate inotify; extern crate byteorder; +extern crate chrono; #[macro_use] extern crate nix; mod hermit; +mod daemon; use nix::sys::signal; use std::env; -use hermit::error; +use hermit::{IsleParameter, error}; +use daemon::{Action, ActionResult}; +use std::net::Shutdown; +use chrono::DateTime; extern fn exit(_:i32) { panic!("Aborting .."); @@ -35,11 +44,11 @@ extern fn exit(_:i32) { fn main() { // register a signal - let sig_action = signal::SigAction::new(signal::SigHandler::Handler(exit), signal::SaFlags::empty(), signal::SigSet::empty()); + /*let sig_action = signal::SigAction::new(signal::SigHandler::Handler(exit), signal::SaFlags::empty(), signal::SigSet::empty()); unsafe { signal::sigaction(signal::SIGINT, &sig_action).unwrap(); signal::sigaction(signal::SIGTERM, &sig_action).unwrap(); - } + }*/ let matches = clap_app!(HermitProxy => (version: "0.0.1") @@ -56,8 +65,108 @@ fn main() { (@arg port: --port +takes_value "Overrides the default port [qemu only]") (@arg app_port: --app_port +takes_value "Overrides the default app port [qemu only]") ) + (@subcommand list => + (about: "Lists all Hermit isles") + ) + (@subcommand kill_daemon => + (about: "Stops the daemon and deletes the unix socket") + ) + (@subcommand stop => + (about: "Stops an Hermit isle") + (@arg name: +required "The corresponding name") + ) + (@subcommand log => + (about: "Acquire logging information") + (@arg name: "Focus on a single instance") + ) + (@subcommand remove => + (about: "Remove a Isle") + (@arg name: "The number or name of the Isle") + ) ).get_matches(); + let mut daemon = daemon::Connection::connect(); + + // list all isles + if let Some(_) = matches.subcommand_matches("list") { + let isles = daemon.send(Action::List); + + if let ActionResult::List(isles) = isles { + println!("{0: <10} | {1: <10} | {2: <10} | {3: <10}", "number", "state", "CPUs", "memory size"); + + let mut id = 0; + for (running, infos) in isles.into_iter() { + let (cpus, mem_size) = match infos { + IsleParameter::QEmu { mem_size, num_cpus, .. } => (num_cpus, mem_size), + IsleParameter::Multi { mem_size, num_cpus } => (num_cpus, mem_size), + IsleParameter::UHyve { mem_size, num_cpus } => (num_cpus, mem_size) + }; + + let state = match running { + Ok(true) => "running", + Ok(false) => "stopped", + Err(_) => "error" + }; + + println!("{0: <10} | {1: <10} | {2: <10} | {3: <10}", id, state, cpus, mem_size); + + id += 1; + } + } + } + + if let Some(_) = matches.subcommand_matches("kill_daemon") { + daemon.send(Action::KillDaemon); + } + + if let Some(matches) = matches.subcommand_matches("log") { + let id = matches.value_of("name").and_then(|x| x.parse::().ok()); + let res = daemon.send(Action::Log(id)); + + if let ActionResult::Log(logs) = res { + let mut i = 0; + println!("{0: <10} | {1: <10} | {2: <10}", "number", "time", "action"); + for log in logs { + println!("{0: <10} | {1: <10} | {2: <10?}", i, log.time.to_string(), log.action); + i += 1; + } + } else if let ActionResult::IsleLog(content) = res { + match content { + Ok(log) => { + println!("Output of isle {}", id.unwrap()); + println!("{}", log); + }, + Err(_) => println!("An error occured!") + } + + } + } + + if let Some(matches) = matches.subcommand_matches("stop") { + match matches.value_of("name").unwrap().parse::() { + Ok(id) => { + let res = daemon.send(Action::StopIsle(id)); + + println!("Isle {} exited with code {:?}", id, res); + }, + Err(_) => { + println!("Invalid number!"); + } + } + } + + if let Some(matches) = matches.subcommand_matches("remove") { + match matches.value_of("name").unwrap().parse::() { + Ok(id) => { + let res = daemon.send(Action::RemoveIsle(id)); + + }, + Err(_) => { + println!("Invalid number!"); + } + } + } + // create the isle, wait to be available and start it if let Some(matches) = matches.subcommand_matches("run") { if let Some(isle) = matches.value_of("isle") { @@ -90,13 +199,19 @@ fn main() { } env_logger::init().unwrap(); - - hermit::new_isle(&matches.value_of("file").unwrap()) - .and_then(|mut isle| { - isle.wait_until_available()?; - isle.run()?; + let res = daemon.send(Action::CreateIsle( + matches.value_of("file").unwrap().into(), + IsleParameter::from_env() + )); - Ok(()) - }).unwrap(); + println!("Created Isle with number: {:?}", res); + +// hermit::new_isle(&matches.value_of("file").unwrap()) +// .and_then(|mut isle| { +// isle.wait_until_available()?; +// isle.run()?; +// +// Ok(()) +// }).unwrap(); } }