[RFC 6/6] Catch unhandled page faults in vm86 mode.

* Add a "fault_callback" to the thread structure which is called when a
  unhandled page fault happens in user space. A SIGSEGV will only be sent if
  the callback return "true".
* Use fault_callback mechanism to return from vm86 mode when a unhandled page
  fault happens in vm86 mode.
---
 headers/private/kernel/thread_types.h |    4 ++++
 src/system/kernel/arch/x86/vm86.c     |   24 ++++++++++++++++++++----
 src/system/kernel/thread.cpp          |    1 +
 src/system/kernel/vm/vm.cpp           |    8 ++++++--
 4 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h
index ac3240e..c8b8424 100644
--- a/headers/private/kernel/thread_types.h
+++ b/headers/private/kernel/thread_types.h
@@ -214,6 +214,9 @@ struct team {
 
 typedef int32 (*thread_entry_func)(thread_func, void *);
 
+typedef bool (*page_fault_callback)(addr_t address, addr_t faultAddress,
+	bool isWrite);
+
 struct thread {
 	int32			flags;			// summary of events relevant in interrupt
 									// handlers (signals pending, user debugging
@@ -264,6 +267,7 @@ struct thread {
 	} msg;
 
 	addr_t			fault_handler;
+	page_fault_callback fault_callback;
 	int32			page_faults_allowed;
 		/* this field may only stay in debug builds in the future */
 
diff --git a/src/system/kernel/arch/x86/vm86.c b/src/system/kernel/arch/x86/vm86.c
index d8737e9..6cde8f4 100644
--- a/src/system/kernel/arch/x86/vm86.c
+++ b/src/system/kernel/arch/x86/vm86.c
@@ -511,6 +511,19 @@ emulate(struct vm86_state *state)
 }
 
 
+static bool
+vm86_fault_callback(addr_t address, addr_t faultAddress, bool isWrite)
+{
+	struct iframe *frame = i386_get_user_iframe();
+
+	// we shouldn't have unhandled page faults in vm86 mode
+	x86_vm86_return((struct vm86_iframe *)frame, B_BAD_ADDRESS);
+
+	// not reached
+	return false;
+}
+
+
 status_t
 vm86_prepare(struct vm86_state *state, unsigned int ram_size)
 {
@@ -586,6 +599,8 @@ vm86_do_int(struct vm86_state *state, uint8 vec)
 {
 	int8 *ip;
 	int emu_state;
+	struct thread *thread = thread_get_current_thread();
+	status_t ret;
 
 	// prepare environment
 	state->regs.ss  = 0x600 >> 4;
@@ -603,14 +618,15 @@ vm86_do_int(struct vm86_state *state, uint8 vec)
     *ip++ = RETURN_TO_32_INT;
 
 	// execute interrupt
+	thread->fault_callback = &vm86_fault_callback;
 	do {
-		if (x86_vm86_enter(&state->regs) != B_OK) {
-			emu_state = -1;
+		ret = x86_vm86_enter(&state->regs);
+		if (ret != B_OK)
 			break;
-		}
 		emu_state = emulate(state);
 	} while (emu_state == 0);
+	thread->fault_callback = NULL;
 
-	return (emu_state > 0) ? B_OK : B_BAD_DATA;
+	return (emu_state > 0) ? ret : B_BAD_DATA;
 }
 
diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp
index ad397db..466dedf 100644
--- a/src/system/kernel/thread.cpp
+++ b/src/system/kernel/thread.cpp
@@ -226,6 +226,7 @@ create_thread_struct(struct thread *inthread, const char *name,
 	thread->team = NULL;
 	thread->cpu = cpu;
 	thread->fault_handler = 0;
+	thread->fault_callback = NULL;
 	thread->page_faults_allowed = 1;
 	thread->kernel_stack_area = -1;
 	thread->kernel_stack_base = 0;
diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp
index e14f898..3ab7ad0 100644
--- a/src/system/kernel/vm/vm.cpp
+++ b/src/system/kernel/vm/vm.cpp
@@ -3840,8 +3840,12 @@ vm_page_fault(addr_t address, addr_t faultAddress, bool isWrite, bool isUser,
 			release_sem_etc(addressSpace->sem, READ_COUNT, 0);
 			vm_put_address_space(addressSpace);
 #endif
-			if (user_debug_exception_occurred(B_SEGMENT_VIOLATION, SIGSEGV))
-				send_signal(team_get_current_team_id(), SIGSEGV);
+			struct thread *thread = thread_get_current_thread();
+			if (thread->fault_callback == NULL || thread->fault_callback(
+				address, faultAddress, isWrite)) {
+				if (user_debug_exception_occurred(B_SEGMENT_VIOLATION, SIGSEGV))
+					send_signal(team_get_current_team_id(), SIGSEGV);
+			}
 		}
 	}
 
-- 
1.5.4.2


-------------------------------------------------------------------------
This SF.net email is sponsored by the 2008 JavaOne(SM) Conference 
Don't miss this year's exciting event. There's still time to save $100. 
Use priority code J8TL2D2. 
http://ad.doubleclick.net/clk;198757673;13503038;p?http://java.sun.com/javaone