c18n: [WIP] Do not store stack metadata at its bottom

Currently, each compartment's stack contains a struct stk_bottom at its
bottom. This struct keeps track of the top of that compartment's stack.
If a compartment corrupts this struct, it will behave in unexpected ways
and likely crash due to a corrupted stack pointer.

While this does not pose any security risk to other compartments or RTLD,
a better design is possible that does not rely on the compartment to
keep track of any information.

The trampoline and other parts of RTLD are refactored. A redzone in the
trusted stack is introduced to make trampolines reentrant. The trusted
frame is redesigned to allow trampolines to perform tail-calls that do
not push a trusted frame.

lib/libsysdecode/utrace.c

@@ -105,10 +105,9 @@ print_utrace_c18n(FILE *fp, void *p)
 	default:
 		return (0);
 	}
-	fprintf(fp, "\no_sp = %p, next = %p\nn_sp = %#p, csp = %#p\n"
-	    "fp = %#p, pc = %#p",
-	    (void *)(uintptr_t)ut->o_sp, (void *)(uintptr_t)ut->next,
-	    ut->n_sp, ut->csp, ut->fp, ut->pc);
+	fprintf(fp, "\nnsp = %#p\nosp = %#p, previous = %#p\n"
+	    "fp = %#p, pc = %#p", ut->sp, ut->osp, ut->previous,
+	    ut->fp, ut->pc);
 	return (1);
 }
 #endif

lib/libthr/thread/thr_create.c

@@ -210,14 +210,24 @@ _pthread_create(pthread_t * __restrict thread,
 		param.rtp = &rtp;
 	}
 
+	/*
+	 * c18n: Always block all signals when creating a new thread to allow
+	 * RTLD to set up the environment to handle signals.
+	 */
+#if defined(__CHERI_PURE_CAPABILITY__) && defined(RTLD_SANDBOX)
+	(void)create_suspended;
+#else
 	/* Schedule the new thread. */
 	if (create_suspended) {
+#endif
 		SIGFILLSET(set);
 		SIGDELSET(set, SIGTRAP);
 		__sys_sigprocmask(SIG_SETMASK, &set, &oset);
 		new_thread->sigmask = oset;
 		SIGDELSET(new_thread->sigmask, SIGCANCEL);
+#if !defined(__CHERI_PURE_CAPABILITY__) || !defined(RTLD_SANDBOX)
 	}
+#endif
 
 	ret = thr_new(&param, sizeof(param));
 
@@ -230,7 +240,9 @@ _pthread_create(pthread_t * __restrict thread,
 			ret = EAGAIN;
 	}
 
+#if !defined(__CHERI_PURE_CAPABILITY__) || !defined(RTLD_SANDBOX)
 	if (create_suspended)
+#endif
 		__sys_sigprocmask(SIG_SETMASK, &oset, NULL);
 
 	if (ret != 0) {
@@ -290,7 +302,9 @@ thread_start(struct pthread *curthread)
 {
 	sigset_t set;
 
+#if !defined(__CHERI_PURE_CAPABILITY__) || !defined(RTLD_SANDBOX)
 	if (curthread->attr.suspend == THR_CREATE_SUSPENDED)
+#endif
 		set = curthread->sigmask;
 	_thr_signal_block_setup(curthread);
 
@@ -305,7 +319,9 @@ thread_start(struct pthread *curthread)
 	if (curthread->force_exit)
 		_pthread_exit(PTHREAD_CANCELED);
 
+#if !defined(__CHERI_PURE_CAPABILITY__) || !defined(RTLD_SANDBOX)
 	if (curthread->attr.suspend == THR_CREATE_SUSPENDED) {
+#endif
 #if 0
 		/* Done in THR_UNLOCK() */
 		_thr_ast(curthread);
@@ -316,7 +332,9 @@ thread_start(struct pthread *curthread)
 		 * we should restore it now.
 		 */
 		__sys_sigprocmask(SIG_SETMASK, &set, NULL);
+#if !defined(__CHERI_PURE_CAPABILITY__) || !defined(RTLD_SANDBOX)
 	}
+#endif
 
 #ifdef _PTHREAD_FORCED_UNWIND
 	curthread->unwind_stackend = (char *)curthread->attr.stackaddr_attr +

lib/libthr/thread/thr_sig.c

@@ -73,19 +73,17 @@ static void check_cancel(struct pthread *curthread, ucontext_t *ucp);
 #if defined(__CHERI_PURE_CAPABILITY__) && defined(RTLD_SANDBOX)
 #pragma weak _thr_sighandler = thr_sighandler
 
-/*
- * XXX: Explicit function pointer used so that RTLD can wrap it in trampoline.
- */
-extern void (*_rtld_sighandler)(int, siginfo_t *, void *);
-
 /*
  * These weak symbols will always be resolved at runtime.
  */
+#pragma weak _rtld_sighandler
+void _rtld_sighandler(int, siginfo_t *, void *);
+
 #pragma weak _rtld_sigaction
 int _rtld_sigaction(int, const struct sigaction *, struct sigaction *);
 
 #pragma weak _rtld_siginvoke
-void _rtld_siginvoke(int, siginfo_t *, void *, __siginfohandler_t, sigset_t *);
+void _rtld_siginvoke(int, siginfo_t *, void *, const struct sigaction *);
 #endif
 
 int	_sigtimedwait(const sigset_t *set, siginfo_t *info,
@@ -299,12 +297,14 @@ handle_signal(struct sigaction *actp, int sig, siginfo_t *info, ucontext_t *ucp)
 	if (!cancel_async)
 		curthread->cancel_enable = 0;
 
-#if !defined(__CHERI_PURE_CAPABILITY__) || !defined(RTLD_SANDBOX)
+#if defined(__CHERI_PURE_CAPABILITY__) && defined(RTLD_SANDBOX)
+	(void)sigfunc;
+#else
 	/* restore correct mask before calling user handler */
 	__sys_sigprocmask(SIG_SETMASK, &actp->sa_mask, NULL);
-#endif
 
 	sigfunc = actp->sa_sigaction;
+#endif
 
 	/*
 	 * We have already reset cancellation point flags, so if user's code
@@ -315,7 +315,7 @@ handle_signal(struct sigaction *actp, int sig, siginfo_t *info, ucontext_t *ucp)
 	 * re-set cancel_enable flag by calling pthread_setcancelstate().
 	 */
 #if defined(__CHERI_PURE_CAPABILITY__) && defined(RTLD_SANDBOX)
-	_rtld_siginvoke(sig, info, ucp, sigfunc, &actp->sa_mask);
+	_rtld_siginvoke(sig, info, ucp, actp);
 #else
 	if ((actp->sa_flags & SA_SIGINFO) != 0) {
 		sigfunc(sig, info, ucp);

libexec/rtld-elf/Symbol-c18n.map

@@ -3,9 +3,9 @@ FBSDprivate_1.0 {
     _rtld_thread_start;
     _rtld_thr_exit;
     _rtld_sighandler_init;
-    _rtld_sigaction;
     _rtld_sighandler;
     _rtld_siginvoke;
+    _rtld_sigaction;
     _rtld_setjmp;
     _rtld_longjmp;
     _rtld_unw_getcontext;