caprevoke fork fixes

bin/cheribsdtest/cheribsdtest_vm.c

@@ -1801,6 +1801,8 @@ enum {
 	TCLR_MODE_STORE = 0,
 	TCLR_MODE_LOAD_ONCE = 1,
 	TCLR_MODE_LOAD_SPLIT = 2,
+	TCLR_MODE_LOAD_SPLIT_INIT = 3,
+	TCLR_MODE_LOAD_SPLIT_FINI = 4,
 };
 
 static void
@@ -1822,9 +1824,23 @@ cheribsdtest_cheri_revoke_lib_run(
 	while (bigblock_offset < bigblock_caps) {
 		struct cheri_revoke_syscall_info crsi;
 		uint32_t cyc_start, cyc_end;
-
-		size_t csz = rand() % 1024 + 1;
-		csz = MIN(csz, bigblock_caps - bigblock_offset);
+		size_t csz;
+
+		switch (mode) {
+			case TCLR_MODE_LOAD_SPLIT_INIT:
+			case TCLR_MODE_LOAD_SPLIT_FINI:
+				/*
+				 * Just do one big block so we can
+				 * call this function once to open the
+				 * epoch and once to close it.
+				 */
+				csz = bigblock_caps - bigblock_offset;
+				break;
+			default:
+				csz = rand() % 1024 + 1;
+				csz = MIN(csz, bigblock_caps - bigblock_offset);
+				break;
+		}
 
 		if (verbose > 1) {
 			fprintf(stderr, "left=%zd csz=%zd\n",
@@ -1843,6 +1859,9 @@ cheribsdtest_cheri_revoke_lib_run(
 		size_t chunk_offset = bigblock_offset;
 		bigblock_offset += csz;
 
+		if (mode == TCLR_MODE_LOAD_SPLIT_FINI)
+			goto load_split_fini;
+
 		if (verbose > 3) {
 			ptrdiff_t fwo, lwo;
 			uint64_t fwm, lwm;
@@ -1946,8 +1965,11 @@ cheribsdtest_cheri_revoke_lib_run(
 				}
 			}
 		}
+		if (mode == TCLR_MODE_LOAD_SPLIT_INIT)
+			return;
 
 		if (mode == TCLR_MODE_LOAD_SPLIT) {
+load_split_fini:
 			cyc_start = get_cyclecount();
 			CHERIBSDTEST_CHECK_SYSCALL(cheri_revoke(
 			    CHERI_REVOKE_LAST_PASS | CHERI_REVOKE_IGNORE_START |
@@ -2068,6 +2090,57 @@ CHERIBSDTEST(cheri_revoke_lib_fork,
 	cheribsdtest_success();
 }
 
+CHERIBSDTEST(cheri_revoke_lib_fork_split,
+    "Test libcheri_caprevoke split across fork")
+{
+	static const int verbose = 0;
+	static const int paranoia = 2;
+
+	static const size_t bigblock_caps = 4096;
+
+	void * __capability * __capability bigblock;
+	void * __capability shadow;
+	const volatile struct cheri_revoke_info * __capability cri;
+
+	int pid;
+
+	srand(1337);
+
+	cheribsdtest_cheri_revoke_lib_init(bigblock_caps, &bigblock, &shadow,
+	    &cri);
+
+	if (verbose > 0) {
+		fprintf(stderr, "bigblock: %#.16lp\n", bigblock);
+		fprintf(stderr, "shadow: %#.16lp\n", shadow);
+	}
+
+	/* Open the epoch and begin revocation */
+	cheribsdtest_cheri_revoke_lib_run(verbose, paranoia,
+	    TCLR_MODE_LOAD_SPLIT_INIT, bigblock_caps, bigblock, shadow, cri);
+
+	pid = fork();
+	if (pid == 0) {
+		/* Finish revocation */
+		cheribsdtest_cheri_revoke_lib_run(verbose, paranoia,
+		    TCLR_MODE_LOAD_SPLIT_FINI, bigblock_caps, bigblock,
+		    shadow, cri);
+	} else {
+		int res;
+
+		CHERIBSDTEST_VERIFY2(pid > 0, "fork failed");
+		waitpid(pid, &res, 0);
+		if (res == 0) {
+			cheribsdtest_success();
+		} else {
+			cheribsdtest_failure_errx("Bad child process exit");
+		}
+	}
+
+	munmap(bigblock, bigblock_caps * sizeof(void * __capability));
+
+	cheribsdtest_success();
+}
+
 CHERIBSDTEST(revoke_largest_quarantined_reservation,
     "Verify that the largest quarantined reservation is revoked",
     .ct_check_skip = skip_need_quarantine_unmapped_reservations)

lib/libsysdecode/flags.c

@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 #include <sys/thr.h>
 #include <sys/umtx.h>
+#include <cheri/revoke.h>
 #include <machine/sysarch.h>
 #include <netinet/in.h>
 #include <netinet/sctp.h>
@@ -243,6 +244,18 @@ sysdecode_cap_fcntlrights(FILE *fp, uint32_t rights, uint32_t *rem)
 	return (print_mask_int(fp, capfcntl, rights, rem));
 }
 
+bool
+sysdecode_cr_flags(FILE *fp, int flags, int *rem)
+{
+	return (print_mask_int(fp, cr_flags, flags, rem));
+}
+
+bool
+sysdecode_cr_get_shadow_flags(FILE *fp, int flags, int *rem)
+{
+	return (print_mask_int(fp, cr_get_shadow_flags, flags, rem));
+}
+
 bool
 sysdecode_close_range_flags(FILE *fp, int flags, int *rem)
 {

lib/libsysdecode/mktables

@@ -94,6 +94,8 @@ gen_table "accessmode"      "[A-Z]_OK[[:space:]]+0?x?[0-9A-Fa-f]+"         "sys/
 gen_table "acltype"         "ACL_TYPE_[A-Z4_]+[[:space:]]+0x[0-9]+"        "sys/acl.h"
 gen_table "atflags"         "AT_[A-Z_]+[[:space:]]+0x[0-9]+"               "sys/fcntl.h"
 gen_table "capfcntl"        "CAP_FCNTL_[A-Z]+[[:space:]]+\(1"              "sys/capsicum.h"
+gen_table "cr_flags" "CHERI_REVOKE_[A-Z_]+[[:space:]]+0x[0-9]+"   "cheri/revoke.h" "CHERI_REVOKE_SHADOW_"
+gen_table "cr_get_shadow_flags" "CHERI_REVOKE_SHADOW_[A-Z]+[[:space:]]+0x[0-9]+" "cheri/revoke.h"
 gen_table "closerangeflags" "CLOSE_RANGE_[A-Z]+[[:space:]]+\([0-9]+<<[0-9]+\)"       "sys/unistd.h"
 gen_table "extattrns"       "EXTATTR_NAMESPACE_[A-Z]+[[:space:]]+0x[0-9]+" "sys/extattr.h"
 gen_table "fadvisebehav"    "POSIX_FADV_[A-Z]+[[:space:]]+[0-9]+"          "sys/fcntl.h"

lib/libsysdecode/sysdecode.h

@@ -53,6 +53,8 @@ const char *sysdecode_atfd(int _fd);
 bool	sysdecode_atflags(FILE *_fp, int _flags, int *_rem);
 bool	sysdecode_cap_fcntlrights(FILE *_fp, uint32_t _rights, uint32_t *_rem);
 void	sysdecode_cap_rights(FILE *_fp, cap_rights_t *_rightsp);
+bool	sysdecode_cr_flags(FILE *_fp, int _flags, int *_rem);
+bool	sysdecode_cr_get_shadow_flags(FILE *_fp, int _flags, int *_rem);
 bool	sysdecode_close_range_flags(FILE *_fp, int _flags, int *_rem);
 const char *sysdecode_cmsg_type(int _cmsg_level, int _cmsg_type);
 const char *sysdecode_extattrnamespace(int _namespace);

lib/libsysdecode/sysdecode_mask.3

@@ -32,6 +32,8 @@
 .Nm sysdecode_accessmode ,
 .Nm sysdecode_atflags ,
 .Nm sysdecode_capfcntlrights ,
+.Nm sysdecode_cr_flags ,
+.Nm sysdecode_cr_get_shadow_flags ,
 .Nm sysdecode_close_range_flags ,
 .Nm sysdecode_fcntl_fileflags ,
 .Nm sysdecode_fileflags ,
@@ -72,6 +74,10 @@
 .Ft bool
 .Fn sysdecode_cap_fcntlrights "FILE *fp" "uint32_t rights" "uint32_t *rem"
 .Ft bool
+.Fn sysdecode_cr_flags "FILE *fp" "int flags" "int *rem"
+.Ft bool
+.Fn sysdecode_cr_get_shadow_flags "FILE *fp" "int flags" "int *rem"
+.Ft bool
 .Fn sysdecode_close_range_flags "FILE *fp" "int flags" "int *rem"
 .Ft bool
 .Fn sysdecode_fcntl_fileflags "FILE *fp" "int flags" "int *rem"
@@ -157,11 +163,13 @@ if any bit fields in the value were decoded and
 if no bit fields were decoded.
 .Pp
 Most of these functions decode an argument passed to a system call:
-.Bl -column "Fn sysdecode_flock_operation" "Xr cap_fcntls_limit 2"
+.Bl -column "Fn sysdecode_cr_get_shadow_flags" "Xr cheri_revoke_get_shadow 2"
 .It Sy Function Ta Sy System Call Ta Sy Argument
 .It Fn sysdecode_access_mode Ta Xr access 2 Ta Fa mode
 .It Fn sysdecode_atflags Ta Xr chflagsat 2 , Xr fstatat 2 Ta Fa atflag , Fa flag
 .It Fn sysdecode_cap_fcntlrights Ta Xr cap_fcntls_limit 2 Ta Fa fcntlrights
+.It Fn sysdecode_cr_flags Ta Xr cheri_revoke 2 Ta Fa flags
+.It Fn sysdecode_cr_get_shadow_flags Ta Xr cheri_revoke_shadow 2 Ta Fa flags
 .It Fn sysdecode_fileflags Ta Xr chflags 2 Ta Fa flags
 .It Fn sysdecode_filemode Ta Xr chmod 2 , Xr open 2 Ta mode
 .It Fn sysdecode_flock_operation Ta Xr flock 2 Ta Fa operation

libexec/rc/rc

@@ -94,7 +94,9 @@ fi
 # Do a first pass to get everything up to $early_late_divider so that
 # we can do a second pass that includes $local_startup directories
 #
-files=`rcorder ${skip} ${skip_firstboot} /etc/rc.d/* 2>/dev/null`
+unset system_rc
+find_system_scripts
+files=`rcorder ${skip} ${skip_firstboot} ${system_rc} 2>/dev/null`
 
 _rc_elem_done=' '
 for _rc_elem in ${files}; do
@@ -106,7 +108,7 @@ for _rc_elem in ${files}; do
 	esac
 done
 
-unset files local_rc
+unset files local_rc system_rc
 
 # Now that disks are mounted, for each dir in $local_startup
 # search for init scripts that use the new rc.d semantics.
@@ -122,7 +124,8 @@ if [ -e ${firstboot_sentinel} ]; then
 	skip_firstboot=""
 fi
 
-files=`rcorder ${skip} ${skip_firstboot} /etc/rc.d/* ${local_rc} 2>/dev/null`
+find_system_scripts
+files=`rcorder ${skip} ${skip_firstboot} ${system_rc} ${local_rc} 2>/dev/null`
 for _rc_elem in ${files}; do
 	case "$_rc_elem_done" in
 	*" $_rc_elem "*)	continue ;;

libexec/rc/rc.subr

@@ -2108,7 +2108,7 @@ find_local_scripts_new() {
 		if [ -d "${dir}" ]; then
 			for file in `grep -l '^# PROVIDE:' ${dir}/* 2>/dev/null`; do
 				case "$file" in
-				*.sample) ;;
+				*.sample|*.pkgsave) ;;
 				*)	if [ -x "$file" ]; then
 						_add_local_script $file
 					fi
@@ -2119,6 +2119,19 @@ find_local_scripts_new() {
 	done
 }
 
+find_system_scripts() {
+	system_rc=''
+	for file in /etc/rc.d/*; do
+		case "${file##*/}" in
+		*.pkgsave) ;;
+		*)	if [ -x "$file" ]; then
+				system_rc="${system_rc} ${file}"
+			fi
+			;;
+		esac
+	done
+}
+
 # check_required_{before|after} command
 #	Check for things required by the command before and after its precmd,
 #	respectively.  The two separate functions are needed because some

sys/arm64/arm64/pmap.c

@@ -5563,7 +5563,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 				rw_wunlock(lock);
 		}
 	}
-#endif
+#endif /* VM_NRESERVLEVEL > 0 */
 
 	PMAP_UNLOCK(pmap);
 out_unlocked:
@@ -5582,8 +5582,36 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 
 	return res;
 }
-#endif
-#endif
+
+void
+pmap_assert_consistent_clg(pmap_t pmap, vm_offset_t va)
+{
+	pt_entry_t *pte, tpte;
+	int level;
+
+	pte = pmap_pte(pmap, va, &level);
+	if (pte == NULL)
+		return;	/* XXX: why does this happen? */
+	tpte = pmap_load(pte);
+	if ((tpte & ATTR_SW_MANAGED) == 0 || !pmap_pte_capdirty(pmap, tpte))
+		return;
+
+	switch (tpte & ATTR_LC_MASK) {
+	case ATTR_LC_DISABLED:
+	case ATTR_LC_ENABLED:
+		panic("no clg");
+	case ATTR_LC_GEN0:
+		KASSERT(pmap->flags.uclg == 0, ("GEN0 LCLG with GEN1 GCLG"));
+		break;
+	case ATTR_LC_GEN1:
+		KASSERT(pmap->flags.uclg == 1, ("GEN1 LCLG with GEN0 GCLG"));
+		break;
+	default:
+		panic("impossible?");
+	}
+}
+#endif /* CHERI_CAPREVOKE */
+#endif /* __has_feature(capabilities) */
 
 /*
  * This code maps large physical mmap regions into the

sys/arm64/include/pmap.h

@@ -188,7 +188,9 @@ extern void (*pmap_invalidate_vpipt_icache)(void);
 static inline int
 pmap_vmspace_copy(pmap_t dst_pmap __unused, pmap_t src_pmap __unused)
 {
-
+#if __has_feature(capabilities)
+	dst_pmap->flags.uclg = src_pmap->flags.uclg;
+#endif
 	return (0);
 }
 

sys/cheri/revoke.h

@@ -34,7 +34,9 @@
 #ifndef __SYS_CHERI_REVOKE_H__
 #define	__SYS_CHERI_REVOKE_H__
 
+#if __has_feature(capabilities)
 #include <cheri/cherireg.h> /* For CHERI_OTYPE_BITS */
+#endif
 
 typedef uint64_t cheri_revoke_epoch_t;
 #define CHERI_REVOKE_ST_EPOCH_WIDTH	61
@@ -73,6 +75,7 @@ static inline int cheri_revoke_epoch_clears(cheri_revoke_epoch_t now,
 	return cheri_revoke_epoch_ge(now, then + (then & 1) + 2);
 }
 
+#if __has_feature(capabilities)
 /* Returns 1 if cap is revoked, 0 otherwise. */
 static inline int
 cheri_revoke_is_revoked(const void * __capability cap)
@@ -83,6 +86,7 @@ cheri_revoke_is_revoked(const void * __capability cap)
 	return (__builtin_cheri_tag_get(cap) == 0);
 #endif
 }
+#endif
 
 /*************************** SHADOW BITMAP LAYOUT ***************************/
 
@@ -133,8 +137,10 @@ cheri_revoke_is_revoked(const void * __capability cap)
 static const size_t VM_CHERI_REVOKE_GSZ_MEM_NOMAP = sizeof(void * __capability);
 static const size_t VM_CHERI_REVOKE_GSZ_OTYPE = 1;
 
+#if __has_feature(capabilities)
 static const size_t VM_CHERI_REVOKE_BSZ_OTYPE =
   ((1 << CHERI_OTYPE_BITS) / VM_CHERI_REVOKE_GSZ_OTYPE / 8);
+#endif
 
 /*************************** REVOKER CONTROL FLAGS ***************************/
 

sys/kern/kern_cheri_revoke.c

@@ -359,13 +359,13 @@ kern_cheri_revoke(struct thread *td, int flags,
 		    (flags & CHERI_REVOKE_LAST_NO_EARLY) == 0) {
 			int vmcflags = 0;
 
-			    /* Userspace can ask us to avoid an IPI here */
+			/* Userspace can ask us to avoid an IPI here */
 			vmcflags |= (flags & CHERI_REVOKE_EARLY_SYNC)
-					? VM_CHERI_REVOKE_SYNC_CD : 0;
+			    ? VM_CHERI_REVOKE_SYNC_CD : 0;
 
-			    /* If not first pass, only recently capdirty */
+			/* If not first pass, only recently capdirty */
 			vmcflags |= (entryst == CHERI_REVOKE_ST_SS_INITED)
-					? VM_CHERI_REVOKE_INCREMENTAL : 0;
+			    ? VM_CHERI_REVOKE_INCREMENTAL : 0;
 
 			res = vm_cheri_revoke_pass(&vmcrc, vmcflags);
 
@@ -579,7 +579,9 @@ kern_cheri_revoke(struct thread *td, int flags,
 	     ((myst == CHERI_REVOKE_ST_SS_LAST) ||
 	      (myst == CHERI_REVOKE_ST_LS_CLOSING))) {
 
-		// XXX Assert all capdirty PTEs have LCLG equal to GCLG
+#ifdef DIAGNOSTIC
+		vm_cheri_assert_consistent_clg(&vm->vm_map);
+#endif
 
 		/* Signal the end of this revocation epoch */
 		epoch++;

sys/kern/kern_exec.c

@@ -1201,6 +1201,10 @@ exec_new_vmspace(struct image_params *imgp, struct sysentvec *sv)
 		vmspace = p->p_vmspace;
 		map = &vmspace->vm_map;
 	}
+#ifdef CHERI_CAPREVOKE
+	KASSERT(map->vm_cheri_revoke_st == CHERI_REVOKE_ST_NONE,
+	    ("vm_map still revoking"));
+#endif
 	map->flags |= imgp->map_flags;
 	if (sv->sv_flags & SV_CHERI)
 		map->flags |= MAP_RESERVATIONS;