arm64/mte: Add support for arm64 mte

For details, please refer to the kernel's introduction to this at "https://docs.kernel.org/arch/arm64/memory-tagging-extension.html" and Android's introduction to this at "https://source.android.com/docs/security/test/memory-safety/arm-mte"

Of course, there is also the following detailed principle introduction
https://developer.arm.com/-/media/Arm%20Developer%20Community/PDF/Arm_Memory_Tagging_Extension_Whitepaper.pdf

The modification of this patch is only to merge the simplest MTE function support. In the future, the MTE function will be integrated into the kernel to a greater extent, for example, hardware MTE Kasan will be supported in the future.

Signed-off-by: wangmingrong1 <[email protected]>

Documentation/guides/index.rst

@@ -45,6 +45,7 @@ Guides
   kernel_threads_with_custom_stacks.rst
   versioning_and_task_names.rst
   logging_rambuffer.rst
+  mte.rst
   ipv6.rst
   integrate_newlib.rst
   protected_build.rst

Documentation/guides/mte.rst

@@ -0,0 +1,95 @@
+====================================
+ATM64 MTE extension
+====================================
+
+Introduction
+------------
+
+Arm v8.5 introduced the Arm Memory Tagging Extension (MTE),
+a hardware implementation of tagged memory.
+
+Basically, MTE tags every memory allocation/deallocation
+with additional metadata. It assigns a tag to a memory location,
+which can then be associated with a pointer that references
+that memory location. At runtime, the CPU checks that the pointer
+and metadata tags match with every load and store.
+
+NX OS currently supports deploying MTE on ARM64 QEMU,
+which is supported at the EL1 level of NX OS.
+
+Principle
+---------
+
+The Arm Memory Tagging Extension implements lock and key access to memory.
+Locks can be set on memory and keys provided during memory access. If the key matches
+the lock, the access is permitted. If it does not match, an error is reported.
+
+Memory locations are tagged by adding four bits of metadata to each 16 bytes
+of physical memory. This is the Tag Granule. Tagging memory implements the lock.
+Pointers, and therefore virtual addresses, are modified to contain the key.
+In order to implement the key bits without requiring larger pointers MTE uses the Top Byte
+Ignore (TBI) feature of the Armv8-A Architecture. When TBI is enabled, the top byte of
+a virtual address is ignored when using it as an input for address translation. This allows the
+top byte to store metadata. In MTE four bits of the top byte are used to provide the key
+
+Architectural Details
+---------------------
+
+MTE adds instructions to the Armv8-A Architecture that are outlined below and grouped
+into three different categories [6]:
+Instructions for tag manipulation applicable to stack and heap tagging.
+
+IRG
+In order for the statistical basis of MTE to be valid, a source of random tags is required.
+IRG is defined to provide this in hardware and insert such a tag into a register for use
+by other instructions.
+
+GMI
+This instruction is for manipulating the excluded set of tags for use with the IRG instruction.
+This is intended for cases where software uses specific tag values for special purposes
+while retaining random tag behavior for normal allocations.
+
+LDG, STG, and STZG
+These instructions allow getting or setting tags in memory. They are intended for changing
+tags in memory either without modifying the data or zeroing the data.
+
+ST2G and STZ2G
+These are denser alternatives to STG and STZG which operate on two granules of memory
+when allocation size allows them to be used.
+
+STGP
+This instruction stores both tag and data to memory.
+Instructions Intended for pointer arithmetic and stack tagging:
+
+ADDG and SUBG
+These are variants of the ADD and SUB instructions, intended for arithmetic on addresses.
+They allow both the tag and address to be separately modified by an immediate value.
+These instructions are intended for creating the addresses of objects on the stack.
+
+SUBP(S)
+This instruction provides a 56-bit subtract with optional flag setting which is required
+for pointer arithmetic that ignores the tag in the top byte.
+
+Instructions intended for system use:
+LDGM, STGM, and STZGM
+These are bulk tag manipulation instructions which are UNDEFINED at EL0. These are
+intended for system software to manipulate tags for the purposes of initialization and
+serialization. For example, they can be used to implement swapping of tagged memory
+to a medium which is not tag-aware. The zeroing form can be used for efficient
+initialization of memory.
+
+Currently NX OS supports the execution of the above instructions,
+such as irg, ldg, stg instructions.
+Their test programs are stored in "apps/system/mte" to test whether the current system supports
+
+Usage
+-----
+
+If you want to experience the MTE function of NX OS, you can refer to the following：
+To enable ARM64_MTE, configure the kernel with::
+
+    CONFIG_ARM64_MTE=y
+
+Of course you can also run it with the existing configuration：
+
+    boards/arm64/qemu/qemu-armv8a/configs/mte

arch/arm64/Kconfig

@@ -108,7 +108,7 @@ config ARCH_CHIP_ZYNQ_MPSOC
 	select ARM64_HAVE_PSCI
 	---help---
 		XilinX ZYNQ MPSOC
-		
+
 config ARCH_CHIP_ARM64_CUSTOM
 	bool "Custom ARM64 chip"
 	select ARCH_CHIP_CUSTOM
@@ -127,6 +127,21 @@ config ARCH_ARMV8R
 	default n
 	select ARCH_SINGLE_SECURITY_STATE
 
+config ARCH_AS_HAS_ARMV8_5
+	bool "Support ARMv8.5 assembly"
+	depends on ARCH_ARMV8A
+	---help---
+		Support ARMv8.5 assembly instruction set
+
+menu "ARMv8.5 architectural features"
+	depends on ARCH_AS_HAS_ARMV8_5
+
+config ARM64_MTE
+	bool "Memory Tagging Extension support"
+	default y
+
+endmenu # "ARMv8.5 architectural features"
+
 config ARCH_SINGLE_SECURITY_STATE
 	bool "ARM Single Security State Support"
 	default n

arch/arm64/src/Toolchain.defs

@@ -30,7 +30,19 @@
 #                   reliable code generation.
 #
 
-ifeq ($(CONFIG_ARCH_CORTEX_A53),y)
+ifeq ($(CONFIG_ARCH_ARMV8A),y)
+  ifeq ($(CONFIG_ARCH_AS_HAS_ARMV8_5),y)
+    OPTION_MARCH = -march=armv8.5-a
+  else
+    OPTION_MARCH = -march=armv8-a
+  endif
+  ifeq ($(CONFIG_ARM64_MTE),y)
+    OPTION_MARCH_FEATURE = +memtag
+  endif
+  ARCHCPUFLAGS += $(OPTION_MARCH)$(OPTION_MARCH_FEATURE)
+else ifeq ($(CONFIG_ARCH_ARMV8R),y)
+  ARCHCPUFLAGS += -march=armv8-r
+else ifeq ($(CONFIG_ARCH_CORTEX_A53),y)
   ARCHCPUFLAGS += -mcpu=cortex-a53
 else ifeq ($(CONFIG_ARCH_CORTEX_A55),y)
   ARCHCPUFLAGS += -mcpu=cortex-a55
@@ -40,10 +52,6 @@ else ifeq ($(CONFIG_ARCH_CORTEX_A72),y)
   ARCHCPUFLAGS += -mcpu=cortex-a72
 else ifeq ($(CONFIG_ARCH_CORTEX_R82),y)
   ARCHCPUFLAGS += -mcpu=cortex-r82
-else ifeq ($(CONFIG_ARCH_ARMV8A),y)
-  ARCHCPUFLAGS += -march=armv8-a
-else ifeq ($(CONFIG_ARCH_ARMV8R),y)
-  ARCHCPUFLAGS += -march=armv8-r
 endif
 
 ifeq ($(CONFIG_DEBUG_CUSTOMOPT),y)

arch/arm64/src/common/CMakeLists.txt

@@ -61,6 +61,10 @@ if(CONFIG_ARCH_HAVE_MMU)
   list(APPEND SRCS arm64_mmu.c)
 endif()
 
+if(CONFIG_ARM64_MTE)
+  list(APPEND SRCS arm64_mte.c)
+endif()
+
 if(CONFIG_ARCH_HAVE_MPU)
   list(APPEND SRCS arm64_mpu.c)
 endif()

arch/arm64/src/common/Make.defs

@@ -74,6 +74,10 @@ ifeq ($(CONFIG_ARCH_HAVE_MMU),y)
 CMN_CSRCS += arm64_mmu.c
 endif
 
+ifeq ($(CONFIG_ARM64_MTE),y)
+CMN_CSRCS += arm64_mte.c
+endif
+
 ifeq ($(CONFIG_ARCH_HAVE_MPU),y)
 CMN_CSRCS += arm64_mpu.c
 common/arm64_mpu.c_CFLAGS += -fno-sanitize=kernel-address

arch/arm64/src/common/arm64_arch.h

@@ -96,6 +96,18 @@
 #define SCTLR_SA_BIT        BIT(3)
 #define SCTLR_I_BIT         BIT(12)
 
+/* Controls the impact of tag check faults
+ * due to loads and stores in EL0 EL1.
+ */
+
+#define SCTLR_TCF0_BIT       BIT(38)
+#define SCTLR_TCF1_BIT       BIT(40)
+
+/* Controlling EL0 EL1 access to assigned tags */
+
+#define SCTLR_ATA0_BIT       BIT(42)
+#define SCTLR_ATA_BIT        BIT(43)
+
 #define ACTLR_AUX_BIT        BIT(9)
 #define ACTLR_CLPORTS_BIT    BIT(8)
 #define ACTLR_CLPMU_BIT      BIT(7)
@@ -135,6 +147,12 @@
 #define SPSR_MODE_EL3H      (0xd)
 #define SPSR_MODE_MASK      (0xf)
 
+#define RGSR_EL1_TAG_MASK   0xfUL
+#define RGSR_EL1_SEED_SHIFT 8
+#define RGSR_EL1_SEED_MASK  0xffffUL
+
+#define TTBR_CNP_BIT        BIT(0)
+
 /* CurrentEL: Current Exception Level */
 
 #define MODE_EL_SHIFT       (0x2)
@@ -243,6 +261,7 @@
 #define HCR_IMO_BIT                 BIT(4)
 #define HCR_AMO_BIT                 BIT(5)
 #define HCR_RW_BIT                  BIT(31)
+#define HCR_ATA_BIT                 BIT(56)
 
 /* CNTHCTL_EL2 bits definitions */
 
@@ -485,6 +504,12 @@ uint64_t arm64_get_mpid(int cpu);
 int arm64_get_cpuid(uint64_t mpid);
 #endif
 
+#ifdef CONFIG_ARM64_MTE
+void arm64_enable_mte(void);
+#else
+#define arm64_enable_mte()
+#endif
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* ___ARCH_ARM64_SRC_COMMON_ARM64_ARCH_H */

arch/arm64/src/common/arm64_boot.c

@@ -137,6 +137,11 @@ void arm64_boot_el2_init(void)
 
   reg = read_sysreg(hcr_el2);
   reg |= HCR_RW_BIT;      /* EL1 Execution state is AArch64 */
+
+#ifdef CONFIG_ARM64_MTE
+  reg |= HCR_ATA_BIT;
+#endif
+
   write_sysreg(reg, hcr_el2);
 
   reg = 0U;                   /* RES0 */

arch/arm64/src/common/arm64_mmu.c

@@ -159,6 +159,12 @@
 #define TCR_KASAN_SW_FLAGS 0
 #endif
 
+#ifdef CONFIG_ARM64_MTE
+#define TCR_MTE_FLAGS (TCR_TCMA1 | TCR_TBI0 | TCR_TBI1 | TCR_ASID_8)
+#else
+#define TCR_MTE_FLAGS 0
+#endif
+
 /****************************************************************************
  * Private Data
  ****************************************************************************/
@@ -262,7 +268,7 @@ static uint64_t get_tcr(int el)
    */
 
   tcr |= TCR_TG0_4K | TCR_SHARED_INNER | TCR_ORGN_WBWA |
-         TCR_IRGN_WBWA | TCR_KASAN_SW_FLAGS;
+         TCR_IRGN_WBWA | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS;
 
   return tcr;
 }

arch/arm64/src/common/arm64_mmu.h

@@ -49,11 +49,17 @@
 #define MT_NORMAL_NC                3U
 #define MT_NORMAL                   4U
 
+#ifdef CONFIG_ARM64_MTE
+#define MT_NORMAL_VAL               0xf0UL
+#else
+#define MT_NORMAL_VAL               0xffUL
+#endif
+
 #define MEMORY_ATTRIBUTES           ((0x00 << (MT_DEVICE_NGNRNE * 8)) |  \
                                      (0x04 << (MT_DEVICE_NGNRE * 8))   | \
                                      (0x0c << (MT_DEVICE_GRE * 8))     | \
                                      (0x44 << (MT_NORMAL_NC * 8))      | \
-                                     (0xffUL << (MT_NORMAL * 8)))
+                                     (MT_NORMAL_VAL << (MT_NORMAL * 8)))
 
 /* More flags from user's perpective are supported using remaining bits
  * of "attrs" field, i.e. attrs[31:3], underlying code will take care
@@ -155,6 +161,16 @@
 #define TCR_TBI0                    (1ULL << 37)
 #define TCR_TBI1                    (1ULL << 38)
 
+/* TCMA1 (bit [58]) controls whether memory accesses
+ * in the address range [59:55] = 0b11111 are unchecked accesses.
+ *
+ * TCMA0 (bit [57]) controls whether memory accesses
+ * in the address range [59:55] = 0b00000 are unchecked accesses.
+ */
+
+#define TCR_TCMA0                   (1ULL << 57)
+#define TCR_TCMA1                   (1ULL << 58)
+
 #define TCR_PS_BITS_4GB             0x0ULL
 #define TCR_PS_BITS_64GB            0x1ULL
 #define TCR_PS_BITS_1TB             0x2ULL