Merge branch 'bits/220-tso' into asahi-wip

arch/arm64/Kconfig

@@ -411,6 +411,9 @@ config KASAN_SHADOW_OFFSET
 config UNWIND_TABLES
 	bool
 
+config ARM64_ACTLR_STATE
+	bool
+
 source "arch/arm64/Kconfig.platforms"
 
 menu "Kernel Features"
@@ -2206,6 +2209,17 @@ config ARM64_DEBUG_PRIORITY_MASKING
 	  If unsure, say N
 endif # ARM64_PSEUDO_NMI
 
+config ARM64_MEMORY_MODEL_CONTROL
+	bool "Runtime memory model control"
+	default ARCH_APPLE
+	select ARM64_ACTLR_STATE
+	help
+	  Some ARM64 CPUs support runtime switching of the CPU memory
+	  model, which can be useful to emulate other CPU architectures
+	  which have different memory models. Say Y to enable support
+	  for the PR_SET_MEM_MODEL/PR_GET_MEM_MODEL prctl() calls on
+	  CPUs with this feature.
+
 config RELOCATABLE
 	bool "Build a relocatable kernel image" if EXPERT
 	select ARCH_HAS_RELR

arch/arm64/include/asm/apple_cpufeature.h

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#ifndef __ASM_APPLE_CPUFEATURES_H
+#define __ASM_APPLE_CPUFEATURES_H
+
+#include <linux/bits.h>
+#include <asm/sysreg.h>
+
+#define AIDR_APPLE_TSO_SHIFT	9
+#define AIDR_APPLE_TSO		BIT(9)
+
+#define ACTLR_APPLE_TSO_SHIFT	1
+#define ACTLR_APPLE_TSO		BIT(1)
+
+#endif

arch/arm64/include/asm/cpufeature.h

@@ -909,6 +909,12 @@ static inline unsigned int get_vmid_bits(u64 mmfr1)
 	return 8;
 }
 
+static __always_inline bool system_has_actlr_state(void)
+{
+	return IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) &&
+		alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE);
+}
+
 s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, s64 cur);
 struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id);
 
@@ -1032,6 +1038,10 @@ static inline bool cpu_has_lpa2(void)
 #endif
 }
 
+void __init init_cpucap_indirect_list_impdef(void);
+void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps);
+bool cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry);
+
 #endif /* __ASSEMBLY__ */
 
 #endif

arch/arm64/include/asm/kvm_emulate.h

@@ -80,6 +80,11 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 {
 	if (!vcpu_has_run_once(vcpu))
 		vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
+	if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) && (
+			alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT) ||
+			alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE)
+		))
+		vcpu->arch.hcr_el2 &= ~HCR_TACR;
 
 	/*
 	 * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C

arch/arm64/include/asm/processor.h

@@ -184,6 +184,9 @@ struct thread_struct {
 	u64			sctlr_user;
 	u64			svcr;
 	u64			tpidr2_el0;
+#ifdef CONFIG_ARM64_ACTLR_STATE
+	u64			actlr;
+#endif
 };
 
 static inline unsigned int thread_get_vl(struct thread_struct *thread,

arch/arm64/kernel/Makefile

@@ -33,7 +33,8 @@ obj-y			:= debug-monitors.o entry.o irq.o fpsimd.o		\
 			   return_address.o cpuinfo.o cpu_errata.o		\
 			   cpufeature.o alternative.o cacheinfo.o		\
 			   smp.o smp_spin_table.o topology.o smccc-call.o	\
-			   syscall.o proton-pack.o idle.o patching.o pi/
+			   syscall.o proton-pack.o idle.o patching.o		\
+			   cpufeature_impdef.o pi/
 
 obj-$(CONFIG_COMPAT)			+= sys32.o signal32.o			\
 					   sys_compat.o

arch/arm64/kernel/cpufeature.c

@@ -1029,7 +1029,7 @@ static void init_cpu_ftr_reg(u32 sys_reg, u64 new)
 extern const struct arm64_cpu_capabilities arm64_errata[];
 static const struct arm64_cpu_capabilities arm64_features[];
 
-static void __init
+void __init
 init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps)
 {
 	for (; caps->matches; caps++) {
@@ -1541,8 +1541,8 @@ has_always(const struct arm64_cpu_capabilities *entry, int scope)
 	return true;
 }
 
-static bool
-feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
+bool
+cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
 {
 	int val, min, max;
 	u64 tmp;
@@ -1595,14 +1595,14 @@ has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
 	if (!mask)
 		return false;
 
-	return feature_matches(val, entry);
+	return cpufeature_matches(val, entry);
 }
 
 static bool
 has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
 {
 	u64 val = read_scoped_sysreg(entry, scope);
-	return feature_matches(val, entry);
+	return cpufeature_matches(val, entry);
 }
 
 const struct cpumask *system_32bit_el0_cpumask(void)
@@ -3144,10 +3144,38 @@ static void update_cpu_capabilities(u16 scope_mask)
 
 	scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
 	for (i = 0; i < ARM64_NCAPS; i++) {
+		bool matches;
+
 		caps = cpucap_ptrs[i];
-		if (!caps || !(caps->type & scope_mask) ||
-		    cpus_have_cap(caps->capability) ||
-		    !caps->matches(caps, cpucap_default_scope(caps)))
+		if (!caps || !(caps->type & scope_mask))
+			continue;
+
+		if (!(scope_mask & SCOPE_LOCAL_CPU) && cpus_have_cap(caps->capability))
+			continue;
+
+		matches = caps->matches(caps, cpucap_default_scope(caps));
+
+		if (matches == cpus_have_cap(caps->capability))
+			continue;
+
+		if (!matches) {
+			/*
+			 * Cap detected on boot CPU but not this CPU,
+			 * disable it if not optional.
+			 */
+			if (!cpucap_late_cpu_optional(caps)) {
+				__clear_bit(caps->capability, system_cpucaps);
+				pr_info("missing on secondary: %s\n", caps->desc);
+			}
+			continue;
+		}
+
+		if (!(scope_mask & (SCOPE_BOOT_CPU | SCOPE_SYSTEM)) &&
+		    cpucap_late_cpu_permitted(caps))
+			/*
+			 * Cap detected on this CPU but not boot CPU,
+			 * skip it if permitted for late CPUs.
+			 */
 			continue;
 
 		if (caps->desc && !caps->cpus)
@@ -3495,6 +3523,7 @@ void __init setup_boot_cpu_features(void)
 	 * handle the boot CPU.
 	 */
 	init_cpucap_indirect_list();
+	init_cpucap_indirect_list_impdef();
 
 	/*
 	 * Detect broken pseudo-NMI. Must be called _before_ the call to

arch/arm64/kernel/cpufeature_impdef.c

@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Contains implementation-defined CPU feature definitions.
+ */
+
+#define pr_fmt(fmt) "CPU features: " fmt
+
+#include <asm/cpufeature.h>
+#include <asm/apple_cpufeature.h>
+#include <linux/irqflags.h>
+#include <linux/preempt.h>
+#include <linux/printk.h>
+
+#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL
+static bool has_apple_feature(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	u64 val;
+	WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
+
+	if (read_cpuid_implementor() != ARM_CPU_IMP_APPLE)
+		return false;
+
+	val = read_sysreg(aidr_el1);
+	return cpufeature_matches(val, entry);
+}
+
+static bool has_apple_tso(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	u64 val;
+
+	if (!has_apple_feature(entry, scope))
+		return false;
+
+	/*
+	 * KVM and old versions of the macOS hypervisor will advertise TSO in
+	 * AIDR_EL1, but then ignore writes to ACTLR_EL1. Test that the bit is
+	 * actually writable before enabling TSO.
+	 */
+
+	val = read_sysreg(actlr_el1);
+	write_sysreg(val ^ ACTLR_APPLE_TSO, actlr_el1);
+	if (!((val ^ read_sysreg(actlr_el1)) & ACTLR_APPLE_TSO)) {
+		pr_info_once("CPU advertises Apple TSO but it is broken, ignoring\n");
+		return false;
+	}
+
+	write_sysreg(val, actlr_el1);
+	return true;
+}
+
+static bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	/* List of CPUs that always use the TSO memory model */
+	static const struct midr_range fixed_tso_list[] = {
+		MIDR_ALL_VERSIONS(MIDR_NVIDIA_DENVER),
+		MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
+		MIDR_ALL_VERSIONS(MIDR_FUJITSU_A64FX),
+		{ /* sentinel */ }
+	};
+
+	return is_midr_in_range_list(read_cpuid_id(), fixed_tso_list);
+}
+#endif
+
+static bool has_apple_actlr_virt_impdef(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	u64 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK;
+
+	return midr >= MIDR_APPLE_M1_ICESTORM && midr <= MIDR_APPLE_M1_FIRESTORM_MAX;
+}
+
+static bool has_apple_actlr_virt(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	u64 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK;
+
+	return midr >= MIDR_APPLE_M2_BLIZZARD && midr <= MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0xfff);
+}
+
+static const struct arm64_cpu_capabilities arm64_impdef_features[] = {
+#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL
+	{
+		.desc = "TSO memory model (Apple)",
+		.capability = ARM64_HAS_TSO_APPLE,
+		.type = SCOPE_LOCAL_CPU | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU,
+		.matches = has_apple_tso,
+		.field_pos = AIDR_APPLE_TSO_SHIFT,
+		.field_width = 1,
+		.sign = FTR_UNSIGNED,
+		.min_field_value = 1,
+		.max_field_value = 1,
+	},
+	{
+		.desc = "TSO memory model (Fixed)",
+		.capability = ARM64_HAS_TSO_FIXED,
+		.type = SCOPE_LOCAL_CPU | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU,
+		.matches = has_tso_fixed,
+	},
+#endif
+	{
+		.desc = "ACTLR virtualization (IMPDEF, Apple)",
+		.capability = ARM64_HAS_ACTLR_VIRT_APPLE,
+		.type = SCOPE_LOCAL_CPU | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU,
+		.matches = has_apple_actlr_virt_impdef,
+	},
+	{
+		.desc = "ACTLR virtualization (architectural?)",
+		.capability = ARM64_HAS_ACTLR_VIRT,
+		.type = SCOPE_LOCAL_CPU | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU,
+		.matches = has_apple_actlr_virt,
+	},
+	{},
+};
+
+void __init init_cpucap_indirect_list_impdef(void)
+{
+	init_cpucap_indirect_list_from_array(arm64_impdef_features);
+}