caprevoke: stop and scan all procs sharing our vmspace

Maintain a list of processes sharing each vmspace so they can be scanned
efficently.

In addition to thread_single() on the revoking process, add
stop_vmspace_proc() which stops all other processes that share our
vmspace following the model of stop_all_proc().  For each process use a
new thread_single(SINGLE_VMSPACE) which acts like SINGLE_BOUNDRY except
that like SINGLE_ALLPROC it suspends all threads in the process and does
not expect proc to be curproc.

sys/kern/init_main.c

@@ -636,7 +636,10 @@ proc0_init(void *dummy __unused)
 
 	/* Allocate a prototype map so we have something to fork. */
 	p->p_vmspace = &vmspace0;
+	mtx_init(&vmspace0.vm_mtx, "vmspace", NULL, MTX_DEF);
 	refcount_init(&vmspace0.vm_refcnt, 1);
+	LIST_INIT(&vmspace0.vm_proclist);
+	LIST_INSERT_HEAD(&vmspace0.vm_proclist, p, p_vm_proclist);
 	pmap_pinit0(vmspace_pmap(&vmspace0));
 
 	/*

sys/kern/kern_cheri_revoke.c

@@ -359,6 +359,7 @@ kern_cheri_revoke(struct thread *td, int flags,
 #else
 	struct cheri_revoke_stats *crstp = NULL;
 #endif
+	struct proc *proc;
 	struct vmspace *vm;
 	vm_map_t vmm;
 	struct vm_cheri_revoke_cookie vmcrc;
@@ -613,6 +614,7 @@ kern_cheri_revoke(struct thread *td, int flags,
 				return (ERESTART);
 			}
 		}
+		stop_vmspace_proc(td->td_proc);
 
 		/*
 		 * Drop the process lock *then* iterate the threads in this
@@ -635,20 +637,27 @@ kern_cheri_revoke(struct thread *td, int flags,
 		 * This also risks the use of ptrace() to expose to userspace
 		 * the trap frame of a stalled thread that has not yet scanned
 		 * itself.  Yick.
+		 *
+		 * XXX-BD: is it safe to walk the proc list and run the
+		 * revokers or do we need to PHOLD them first?
 		 */
 
 		_PHOLD(td->td_proc);
 		PROC_UNLOCK(td->td_proc);
 
 		/* Per-thread kernel hoarders */
-		FOREACH_THREAD_IN_PROC (td->td_proc, ptd) {
-			cheri_revoke_td_frame(ptd, &vmcrc);
-			sig_thread_cheri_revoke(ptd, &vmcrc);
+		LIST_FOREACH(proc, &td->td_proc->p_vmspace->vm_proclist,
+		    p_vm_proclist) {
+			FOREACH_THREAD_IN_PROC (proc, ptd) {
+				cheri_revoke_td_frame(ptd, &vmcrc);
+				sig_thread_cheri_revoke(ptd, &vmcrc);
+			}
 		}
 	}
 
 	/* Per-process kernel hoarders */
-	cheri_revoke_hoarders(td->td_proc, &vmcrc);
+	LIST_FOREACH(proc, &td->td_proc->p_vmspace->vm_proclist, p_vm_proclist)
+		cheri_revoke_hoarders(proc, &vmcrc);
 
 	KASSERT(myst == CHERI_REVOKE_ST_INITING ||
 	    myst == CHERI_REVOKE_ST_CLOSING,
@@ -686,6 +695,7 @@ kern_cheri_revoke(struct thread *td, int flags,
 
 	PROC_LOCK(td->td_proc);
 	_PRELE(td->td_proc);
+	resume_vmspace_proc(td->td_proc);
 	if ((td->td_proc->p_flag & P_HADTHREADS) != 0) {
 		thread_single_end(td->td_proc, SINGLE_BOUNDARY);
 	}

sys/kern/kern_proc.c

@@ -3953,6 +3953,127 @@ resume_all_proc(void)
 	stop_all_proc_unblock();
 }
 
+/*
+ * stop_vmspace_proc() stops all proceses which share a vmspace with
+ * curproc.  It should be run after curproc has entered thread_single.
+ */
+void
+stop_vmspace_proc(struct proc* cp)
+{
+	struct proc *p;
+	struct vmspace *vm;
+	int r, gen;
+	bool restart, seen_stopped, seen_exiting, stopped_some;
+
+	/*
+	 * XXX-BD: Prevent multiple callers on a vmspace.
+	 * Maybe atomic CAS on a field in vmspace?
+	 */
+
+	vm = cp->p_vmspace;
+vmspace_loop:
+	VMSPACE_LOCK(vm);
+	/*
+	 * XXX: allproc_gen should work, but should vmspaces have a generation?
+	 */
+	gen = allproc_gen;
+	seen_exiting = seen_stopped = stopped_some = restart = false;
+	LIST_REMOVE(cp, p_vm_proclist);
+	LIST_INSERT_HEAD(&vm->vm_proclist, cp, p_vm_proclist);
+	for (;;) {
+		p = LIST_NEXT(cp, p_vm_proclist);
+		if (p == NULL)
+			break;
+		LIST_REMOVE(cp, p_vm_proclist);
+		LIST_INSERT_AFTER(p, cp, p_vm_proclist);
+		PROC_LOCK(p);
+		if ((p->p_flag & (P_KPROC | P_SYSTEM | P_TOTAL_STOP)) != 0) {
+			PROC_UNLOCK(p);
+			continue;
+		}
+		if ((p->p_flag2 & P2_WEXIT) != 0) {
+			seen_exiting = true;
+			PROC_UNLOCK(p);
+			continue;
+		}
+		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
+			/*
+			 * Stopped processes are tolerated when there
+			 * are no other processes which might continue
+			 * them.  P_STOPPED_SINGLE but not
+			 * P_TOTAL_STOP process still has at least one
+			 * thread running.
+			 */
+			seen_stopped = true;
+			PROC_UNLOCK(p);
+			continue;
+		}
+		VMSPACE_UNLOCK(vm);
+		_PHOLD(p);
+		r = thread_single(p, SINGLE_VMSPACE);
+		if (r != 0)
+			restart = true;
+		else
+			stopped_some = true;
+		_PRELE(p);
+		PROC_UNLOCK(p);
+		VMSPACE_LOCK(vm);
+	}
+	/* Catch forked children we did not see in iteration. */
+	if (gen != allproc_gen)
+		restart = true;
+	VMSPACE_UNLOCK(vm);
+	if (restart || stopped_some || seen_exiting || seen_stopped) {
+		kern_yield(PRI_USER);
+		goto vmspace_loop;
+	}
+}
+
+void
+resume_vmspace_proc(struct proc *cp)
+{
+	struct proc *p;
+	struct vmspace *vm;
+
+	vm = cp->p_vmspace;
+
+	VMSPACE_LOCK(vm);
+again:
+	LIST_REMOVE(cp, p_vm_proclist);
+	LIST_INSERT_HEAD(&vm->vm_proclist, cp, p_vm_proclist);
+	for (;;) {
+		p = LIST_NEXT(cp, p_vm_proclist);
+		if (p == NULL)
+			break;
+		LIST_REMOVE(cp, p_vm_proclist);
+		LIST_INSERT_AFTER(p, cp, p_vm_proclist);
+		PROC_LOCK(p);
+		if (p->p_vmspace != cp->p_vmspace) {
+			PROC_UNLOCK(p);
+			continue;
+		}
+		if ((p->p_flag & P_TOTAL_STOP) != 0) {
+			VMSPACE_UNLOCK(vm);
+			_PHOLD(p);
+			thread_single_end(p, SINGLE_VMSPACE);
+			_PRELE(p);
+			PROC_UNLOCK(p);
+			VMSPACE_LOCK(vm);
+		} else {
+			PROC_UNLOCK(p);
+		}
+	}
+	/*  Did the loop above missed any stopped process ? */
+	FOREACH_PROC_IN_SYSTEM(p) {
+		/* No need for proc lock. */
+		if ((p->p_flag & P_TOTAL_STOP) != 0)
+			goto again;
+	}
+	VMSPACE_UNLOCK(vm);
+
+	/* See "XXX-BD: Prevent multiple callers on a vmspace" above */
+}
+
 /* #define	TOTAL_STOP_DEBUG	1 */
 #ifdef TOTAL_STOP_DEBUG
 volatile static int ap_resume;

sys/kern/kern_thread.c

@@ -1117,7 +1117,7 @@ calc_remaining(struct proc *p, int mode)
 	PROC_SLOCK_ASSERT(p, MA_OWNED);
 	if (mode == SINGLE_EXIT)
 		remaining = p->p_numthreads;
-	else if (mode == SINGLE_BOUNDARY)
+	else if (mode == SINGLE_BOUNDARY || mode == SINGLE_VMSPACE)
 		remaining = p->p_numthreads - p->p_boundary_count;
 	else if (mode == SINGLE_NO_EXIT || mode == SINGLE_ALLPROC)
 		remaining = p->p_numthreads - p->p_suspcount;
@@ -1163,6 +1163,7 @@ weed_inhib(int mode, struct thread *td2, struct proc *p)
 		break;
 	case SINGLE_BOUNDARY:
 	case SINGLE_NO_EXIT:
+	case SINGLE_VMSPACE:
 		if (TD_IS_SUSPENDED(td2) &&
 		    (td2->td_flags & TDF_BOUNDARY) == 0) {
 			wakeup_swapper |= thread_unsuspend_one(td2, p, false);
@@ -1224,7 +1225,8 @@ thread_single(struct proc *p, int mode)
 
 	td = curthread;
 	KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY ||
-	    mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT,
+	    mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT ||
+	    mode == SINGLE_VMSPACE,
 	    ("invalid mode %d", mode));
 	/*
 	 * If allowing non-ALLPROC singlethreading for non-curproc
@@ -1233,7 +1235,8 @@ thread_single(struct proc *p, int mode)
 	 * this is not implemented because it is not used.
 	 */
 	KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) ||
-	    (mode != SINGLE_ALLPROC && td->td_proc == p),
+	    ((mode != SINGLE_ALLPROC || mode != SINGLE_VMSPACE) &&
+	    td->td_proc == p),
 	    ("mode %d proc %p curproc %p", mode, p, td->td_proc));
 	mtx_assert(&Giant, MA_NOTOWNED);
 	PROC_LOCK_ASSERT(p, MA_OWNED);
@@ -1258,7 +1261,7 @@ thread_single(struct proc *p, int mode)
 		p->p_flag &= ~P_SINGLE_BOUNDARY;
 	} else {
 		p->p_flag &= ~P_SINGLE_EXIT;
-		if (mode == SINGLE_BOUNDARY)
+		if (mode == SINGLE_BOUNDARY || mode == SINGLE_VMSPACE)
 			p->p_flag |= P_SINGLE_BOUNDARY;
 		else
 			p->p_flag &= ~P_SINGLE_BOUNDARY;
@@ -1326,7 +1329,7 @@ thread_single(struct proc *p, int mode)
 			PROC_LOCK(p);
 			PROC_SLOCK(p);
 		}
-	} else if (mode == SINGLE_BOUNDARY) {
+	} else if (mode == SINGLE_BOUNDARY || mode == SINGLE_VMSPACE) {
 		/*
 		 * Wait until all suspended threads are removed from
 		 * the processors.  The thread_suspend_check()

sys/sys/proc.h

@@ -686,6 +686,8 @@ struct proc {
 					       (if I am reaper). */
 	LIST_ENTRY(proc) p_reapsibling;	/* (e) List of siblings - descendants of
 					       the same reaper. */
+	LIST_ENTRY(proc) p_vm_proclist; /* (b) List of processes sharing
+						p_vmspace  */
 	struct mtx	p_mtx;		/* (n) Lock for this struct. */
 	struct mtx	p_statmtx;	/* Lock for the stats */
 	struct mtx	p_itimmtx;	/* Lock for the virt/prof timers */
@@ -958,6 +960,7 @@ struct proc {
 #define	SINGLE_EXIT	1
 #define	SINGLE_BOUNDARY	2
 #define	SINGLE_ALLPROC	3
+#define	SINGLE_VMSPACE	4
 
 #ifdef MALLOC_DECLARE
 MALLOC_DECLARE(M_PARGS);
@@ -1328,6 +1331,8 @@ bool	stop_all_proc_block(void);
 void	stop_all_proc_unblock(void);
 void	stop_all_proc(void);
 void	resume_all_proc(void);
+void	stop_vmspace_proc(struct proc *cp);
+void	resume_vmspace_proc(struct proc *cp);
 
 static __inline int
 curthread_pflags_set(int flags)

sys/vm/vm_glue.c

@@ -581,6 +581,7 @@ vm_forkproc(struct thread *td, struct proc *p2, struct thread *td2,
 		if (p1->p_vmspace->vm_shm)
 			shmfork(p1, p2);
 	}
+	vmspace_insert_proc(p2->p_vmspace, p2);
 
 	/*
 	 * cpu_fork will copy and update the pcb, set up the kernel stack,