adapted c runtime and syscalls to x86-64 ABI calling convention

arch/x86/include/asm/tasks.h

@@ -97,17 +97,19 @@ static inline int register_task(void)
  *
  * @return 0 in any case
  */
-static inline int jump_to_user_code(uint32_t ep, uint32_t stack)
+static inline int jump_to_user_code(size_t ep, size_t stack)
 {
+
+	asm volatile ("push $0x23; push %0; push $0x1B; push %1" :: "r"(stack), "r"(ep)); // fake stack, see Intel Reference Manual, Vol 1, 6.3.6
+
 #ifdef CONFIG_X86_32
-	asm volatile ("mov %0, %%ds; mov %0, %%fs; mov %0, %%gs; mov %0, %%es" :: "r"(0x23));
-	asm volatile ("push $0x23; push %0; push $0x1B; push %1" :: "r"(stack), "r"(ep));
-	asm volatile ("lret" ::: "cc");
+	asm volatile ("mov %0, %%ds; mov %0, %%fs; mov %0, %%gs; mov %0, %%es" :: "r"(0x23)); // update segment registers
+	asm volatile ("lret" ::: "cc"); // far return to user level code
+#elif defined (CONFIG_X86_64)
+	asm volatile ("lretq" ::: "cc"); // far return to user level code
+#endif
 
 	return 0;
-#else
-	return -22;
-#endif
 }
 
 #ifdef __cplusplus

arch/x86/kernel/entry64.asm

@@ -729,41 +729,41 @@ extern syscall_handler
 
 ; used to realize system calls
 isrsyscall:
-    push r15
-    push r14
-    push r13
-    push r12
+    cli            ; disable interrupts during prologue
+
+    ; save caller saved registers
     push r11
     push r10
     push r9
     push r8
     push rdi
     push rsi
-    push rbp
-    push rsp
-    push rbx
     push rdx
     push rcx
-    push rax
 
-    mov rdi, rsp
+    ; set kernel data segmenets
+    mov ax, 0x10
+    mov ds, ax
+
+    ; x86-64 ABI calling convention
+    mov r8, rbx
+    mov r9, rax
+    mov rax, 0     ; we've not used vector registers for this va_arg call
+
+    sti            ; enable interrupts during syscall
     call syscall_handler
+    cli            ; disable interrupts during prologue
 
-    pop rax
+    ; restore caller saved registers
     pop rcx
     pop rdx
-    pop rbx
-    add rsp, 8
-    pop rbp
     pop rsi
     pop rdi
     pop r8
     pop r9
     pop r10
     pop r11
-    pop r12
-    pop r13
-    pop r14
+
     iretq
 
 global irq0

kernel/syscall.c

@@ -427,7 +427,7 @@ static int sys_sbrk(int incr)
 	return ret;
 }
 
-int syscall_handler(uint32_t sys_nr, ...)
+int syscall_handler(size_t sys_nr, ...)
 {
 	int ret = -EINVAL;
 	va_list vl;
@@ -607,7 +607,7 @@ int syscall_handler(uint32_t sys_nr, ...)
 	}
 #endif
 	default:
-		kputs("invalid system call\n");
+		kprintf("syscall_handler: invalid system call %u\n", sys_nr);
 		ret = -ENOSYS;
 		break;
 	};

newlib/src/libgloss/metalsvm/crt0_64.asm

@@ -56,9 +56,8 @@ L1:
    call rax
 L2:
    ; register a function to be called at normal process termination
-   push __do_global_dtors
+   mov rdi, __do_global_dtors
    call atexit
-   pop rax
 
    ; call init function
    call __do_global_ctors
@@ -76,13 +75,17 @@ L4:
 
    ; arguments are already on the stack
    ; call the user's function
+   pop rdi   ; argc
+   pop rsi   ; argv pointer
+   pop rdx   ; env pointer
+
    call main
 
    ; call exit from the C library so atexit gets called, and the
    ; C++ destructors get run. This calls our exit routine below    
    ; when it's done.
    ; call "exit"
-   push rax
+   mov rdi, rax
    call exit
 
    ; endless loop

newlib/src/libgloss/metalsvm/syscall.h

@@ -85,7 +85,7 @@ syscall(int nr, unsigned long arg0, unsigned long arg1, unsigned long arg2,
 
 	asm volatile (_SYSCALLSTR(INT_SYSCALL)
 	     : "=a" (res)
-	     : "0" (nr), "b" (arg0), "c" (arg1), "d" (arg2), "S" (arg3), "D" (arg4)
+	     : "D" (nr), "S" (arg0), "d" (arg1), "c" (arg2), "b" (arg3), "a" (arg4)
 	     : "memory", "cc");
 
 	return res;