Add initial memory versioning implementation.

It is incomplete and may be slow / buggy notably:

1) version checks are performed during capability permissions checks, using the iotlb to cache a pointer to version memory, which is allocated alongside tag memory in cheri_tagmem.c. It uses one byte per version instead of a nibble like the ARM MTE so a bit space inefficient.
2) Loads and stores via DDC will throw an exception if DDC is versioned.
3) No system register.

Note that this is the result of rebasing against upstream and simultaneously squashing the previous branch. There had been quite a lot of upstream changes with conflicts and some not very interesting history so it turned out to be easier that way.

accel/tcg/cputlb.c

@@ -2,6 +2,7 @@
  *  Common CPU TLB handling
  *
  *  Copyright (c) 2003 Fabrice Bellard
+ *  Copyright (c) 2021 Microsoft
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -1123,9 +1124,11 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
      * Getting tagmem can cause an invalidation, so best to do this before
      * any other entries are modified.
      */
+    uintptr_t vermem;
     uintptr_t tagmem = (uintptr_t)cheri_tagmem_for_addr(
-        env, vaddr, section->mr->ram_block, xlat, size, &prot, tag_setting);
+        env, vaddr, section->mr->ram_block, xlat, size, &prot, tag_setting, (void **) &vermem);
     assert((tagmem & TLBENTRYCAP_MASK) == 0);
+    assert((vermem & TLBENTRYVER_MASK) == 0);
 #endif
 
     address = vaddr_page;
@@ -1249,6 +1252,11 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
         desc->iotlb[index].tagmem_write |= TLBENTRYCAP_FLAG_TRAP;
     }
 
+    desc->iotlb[index].vermem = vermem;
+    if (prot & PAGE_SV_TRAP) {
+        desc->iotlb[index].vermem |= TLBENTRYVER_TRAP;
+    }
+
 #endif
     desc->iotlb[index].attrs = attrs;
 
@@ -1459,7 +1467,9 @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
 #ifdef TARGET_CHERI
         if (cap_write && ((env_tlb(env)->d[mmu_idx].viotlb[vidx].tagmem_write &
                            TLBENTRYCAP_INVALID_WRITE_MASK) ==
-                          TLBENTRYCAP_INVALID_WRITE_VALUE)) {
+                           TLBENTRYCAP_INVALID_WRITE_VALUE || 
+                           env_tlb(env)->d[mmu_idx].viotlb[vidx].vermem ==
+                           TLBENTRYVER_INVALID)) {
             continue;
         }
 #endif
@@ -1591,6 +1601,7 @@ probe_access_internal(CPUArchState *env, target_ulong addr, int fault_size,
          * done with a single probe.
          */
     case MMU_DATA_CAP_STORE:
+    case MMU_VERSION_STORE: // we don't currently need the address but treat the same wrt TLB_NOTDIRTY etc.
         elt_ofs = offsetof(CPUTLBEntry, addr_write);
         break;
     case MMU_INST_FETCH:
@@ -1613,11 +1624,18 @@ probe_access_internal(CPUArchState *env, target_ulong addr, int fault_size,
               .tagmem_write &
           TLBENTRYCAP_INVALID_WRITE_MASK) == TLBENTRYCAP_INVALID_WRITE_VALUE))
         tag_write_invalid = true;
+    if (access_type == MMU_VERSION_STORE &&
+        (env_tlb(env)
+             ->d[mmu_idx]
+             .iotlb[tlb_index(env, mmu_idx, addr)]
+             .vermem == TLBENTRYVER_INVALID))
+        tag_write_invalid = true;
 #endif
 
     if (!tlb_hit_page(tlb_addr, page_addr) || tag_write_invalid) {
         if (!victim_tlb_hit(env, mmu_idx, index, elt_ofs, page_addr,
-                            access_type == MMU_DATA_CAP_STORE)) {
+                            access_type == MMU_DATA_CAP_STORE ||
+                            access_type == MMU_VERSION_STORE)) {
             CPUState *cs = env_cpu(env);
             CPUClass *cc = CPU_GET_CLASS(cs);
 
@@ -1693,6 +1711,7 @@ probe_access_inlined(CPUArchState *env, target_ulong addr, int size,
             bool is_write =
 #ifdef TARGET_CHERI
                 access_type == MMU_DATA_CAP_STORE ||
+                access_type == MMU_VERSION_STORE ||
 #endif
                 access_type == MMU_DATA_STORE;
             int wp_access = is_write ? BP_MEM_WRITE : BP_MEM_READ;

accel/tcg/probe-access.inc.c

@@ -32,4 +32,11 @@ void *probe_cap_write(CPUArchState *env, target_ulong addr, int size,
     return probe_access_inlined(env, addr, size, MMU_DATA_CAP_STORE, mmu_idx,
                                 retaddr);
 }
+
+void *probe_ver_write(CPUArchState *env, target_ulong addr, int size,
+                      int mmu_idx, uintptr_t retaddr)
+{
+    return probe_access_inlined(env, addr, size, MMU_VERSION_STORE, mmu_idx,
+                                retaddr);
+}
 #endif

disas/riscv.c

@@ -3,6 +3,7 @@
  *
  * Copyright (c) 2016-2017 Michael Clark <michaeljclark@mac.com>
  * Copyright (c) 2017-2018 SiFive, Inc.
+ * Copyright (c) 2021 Microsoft <robert.norton@microsoft.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -517,6 +518,9 @@ typedef enum {
     rv_op_cgetaddr,
     rv_op_csealentry,
     rv_op_cloadtags,
+    rv_op_cgetversion,
+    rv_op_cloadversion,
+    rv_op_cstoreversion,
 
     // Three operand
     rv_op_cspecialrw,
@@ -537,6 +541,8 @@ typedef enum {
     rv_op_ccseal,
     rv_op_ctestsubset,
     rv_op_cseqx,
+    rv_op_csetversion,
+    rv_op_camocdecversion,
 
     // FP loads/store
     rv_op_cflw,
@@ -662,6 +668,7 @@ static const char rv_freg_name_sym[32][5] = {
 #define rv_fmt_cd_rs1                 "O\tC0,1"
 #define rv_fmt_rs1_offset             "O\t1,o"
 #define rv_fmt_rs2_offset             "O\t2,o"
+#define rv_fmt_cs1_rs2             "O\tC1,2"
 
 /* pseudo-instruction constraints */
 
@@ -1234,6 +1241,9 @@ const rv_opcode_data opcode_data[] = {
     [rv_op_cgetaddr] = { "cgetaddr", rv_codec_r, rv_fmt_rd_cs1, NULL, 0, 0, 0 },
     [rv_op_csealentry] = { "csealentry", rv_codec_r, rv_fmt_cd_cs1, NULL, 0, 0, 0 },
     [rv_op_cloadtags] = { "cloadtags", rv_codec_r, rv_fmt_rd_cs1, NULL, 0, 0, 0 },
+    [rv_op_cgetversion] = { "cgetversion", rv_codec_r, rv_fmt_rd_cs1, NULL, 0, 0, 0 },
+    [rv_op_cloadversion] = { "cloadversion", rv_codec_r, rv_fmt_rd_cs1, NULL, 0, 0, 0 },
+    [rv_op_cstoreversion] = { "cstoreversion", rv_codec_r, rv_fmt_cs1_rs2, NULL, 0, 0, 0 },
 
     // capmode loads:
     [rv_op_clb] = { "clb", rv_codec_i, rv_fmt_rd_offset_cs1, NULL, 0, 0, 0 },
@@ -1268,6 +1278,8 @@ const rv_opcode_data opcode_data[] = {
     [rv_op_ccseal] = { "ccseal", rv_codec_r, rv_fmt_cd_cs1_cs2, NULL, 0, 0, 0 },
     [rv_op_ctestsubset] = { "ctestsubset", rv_codec_r, rv_fmt_rd_cs1_cs2, NULL, 0, 0, 0 },
     [rv_op_cseqx] = { "cseqx", rv_codec_r, rv_fmt_rd_cs1_cs2, NULL, 0, 0, 0 },
+    [rv_op_csetversion] = { "csetversion", rv_codec_r, rv_fmt_cd_cs1_rs2, NULL, 0, 0, 0 },
+    [rv_op_camocdecversion] = { "camocdecversion", rv_codec_r, rv_fmt_rd_cs1_cs2, 0, 0, 0 },
 
     // FP load store
     [rv_op_cflw] = { "cflw", rv_codec_i, rv_fmt_frd_offset_cs1, NULL, 0, 0, 0 },
@@ -1503,6 +1515,17 @@ static rv_opcode decode_cheri_two_op(unsigned func) {
     case 0b01111: return rv_op_cgetaddr;
     case 0b10001: return rv_op_csealentry;
     case 0b10010: return rv_op_cloadtags;
+    case 0b10011: return rv_op_cgetversion;
+    // case 0b10101: return rv_op-cloadversions; // XXX not yet
+    case 0b10110: return rv_op_cloadversion;
+    default: return rv_op_illegal;
+    }
+}
+
+static rv_opcode decode_cheri_two_src_op(unsigned func) {
+    switch (func) {
+    //case 0b00001: return rv_op_cinvoke; TODO
+    case 0b00010: return rv_op_cstoreversion;
     default: return rv_op_illegal;
     }
 }
@@ -1517,7 +1540,9 @@ static rv_opcode decode_cheri_inst(rv_inst inst) {
     switch (func) {
     // 0000000, unused
     CHERI_THREEOP_CASE(cspecialrw,  0000001,  ..... ..... 000 ..... 1011011 @r)
-    // 0000010-0000111 unused
+    CHERI_THREEOP_CASE(csetversion, 0000010,  ..... ..... 000 ..... 1011011 @r)
+    CHERI_THREEOP_CASE(camocdecversion, 0000011,  ..... ..... 000 ..... 1011011 @r)
+    // 0000100-0000111 unused
     CHERI_THREEOP_CASE(csetbounds,  0001000,  ..... ..... 000 ..... 1011011 @r)
     CHERI_THREEOP_CASE(csetboundsexact, 0001001,  ..... ..... 000 ..... 1011011 @r)
     // 0001010 unused
@@ -1540,7 +1565,9 @@ static rv_opcode decode_cheri_inst(rv_inst inst) {
     // 1111011 unused
     // TODO: 1111100 Used for Stores (see below)
     // TODO: 1111101 Used for Loads (see below)
-    // TODO: 1111110 Used for two source ops
+    // 1111110 Used for two source ops
+    case 0b111110:
+        return decode_cheri_two_src_op((inst >> 7) & 0b11111);
     // 1111111 Used for Source & Dest ops (see above)
     case 0b111111:
         return decode_cheri_two_op((inst >> 20) & 0b11111);

include/exec/cpu-all.h

@@ -286,6 +286,8 @@ extern intptr_t qemu_host_page_mask;
 #define PAGE_C_BITS                                                            \
     (PAGE_LC_CLEAR | PAGE_LC_TRAP | PAGE_SC_TRAP | PAGE_SC_CLEAR |             \
      PAGE_LC_TRAP_ANY)
+/* Trap on store version */
+#define PAGE_SV_TRAP  0x80000
 #else
 #define PAGE_C_BITS 0
 #endif

include/exec/cpu-defs.h

@@ -2,6 +2,7 @@
  * common defines for all CPUs
  *
  * Copyright (c) 2003 Fabrice Bellard
+ * Copyright (c) 2021 Microsoft
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -178,6 +179,17 @@ typedef struct CPUIOTLBEntry {
     (TLBENTRYCAP_FLAG_TRAP | TLBENTRYCAP_FLAG_CLEAR)
 #define TLBENTRYCAP_INVALID_WRITE_VALUE (TLBENTRYCAP_FLAG_TRAP)
     uintptr_t tagmem_write;
+
+/* Similar to tagmem above but there is no clearing access */
+#define TLBENTRYVER_MASK    ((uintptr_t) 1)
+/* Trap if attempting to write version */
+#define TLBENTRYVER_TRAP    ((uintptr_t) 1)
+#define TLBENTRYVER_INVALID ((uintptr_t) (~0 & ~TLBENTRYVER_MASK))
+/* Entry that reads as all zeros, attempts allocation on write */
+#define ALL_ZERO_VERMEM     ((void *) TLBENTRYVER_INVALID)
+    /* Pointer to version memory for page. Will hopefully get away with single pointer
+       for both reads and writes. */
+    uintptr_t vermem;
 #endif
     MemTxAttrs attrs;
 } CPUIOTLBEntry;
@@ -189,6 +201,16 @@ typedef struct CPUIOTLBEntry {
     })
 #define IOTLB_GET_TAGMEM_FLAGS(iotlbentry, rw)                                 \
     ((uintptr_t)iotlbentry->tagmem_##rw & TLBENTRYCAP_MASK);
+
+#define IOTLB_GET_VERMEM(iotlbentry)                                       \
+    ({                                                                         \
+        cheri_debug_assert(iotlbentry->vermem != (uintptr_t)0);           \
+        (void *)((uintptr_t)iotlbentry->vermem & ~TLBENTRYVER_MASK);      \
+    })
+
+#define IOTLB_GET_VERMEM_FLAGS(iotlbentry)                                \
+    (iotlbentry->vermem & TLBENTRYVER_MASK)
+
 /*
  * Data elements that are per MMU mode, minus the bits accessed by
  * the TCG fast path.

include/exec/exec-all.h

@@ -402,6 +402,9 @@ void *probe_read(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
 #ifdef TARGET_CHERI
 void *probe_cap_write(CPUArchState *env, target_ulong addr, int size,
                       int mmu_idx, uintptr_t retaddr);
+
+void *probe_ver_write(CPUArchState *env, target_ulong addr, int size,
+                      int mmu_idx, uintptr_t retaddr);
 #endif
 
 /**

include/hw/core/cpu.h

@@ -73,6 +73,8 @@ typedef enum MMUAccessType {
     MMU_INST_FETCH = 2,
     MMU_DATA_CAP_LOAD = 3,
     MMU_DATA_CAP_STORE = 4,
+    MMU_VERSION_LOAD = 5, /* XXX not currently used because version loads use same perms as data (apparently MMU_DATA_CAP_LOAD is the same) */
+    MMU_VERSION_STORE = 6,
 } MMUAccessType;
 
 typedef struct CPUWatchpoint CPUWatchpoint;

target/cheri-common/cheri-compressed-cap/cheri_compressed_cap_128.h

@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2018-2020 Alex Richardson
+ * Copyright (c) 2021 Microsoft <robert.norton@microsoft.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -56,6 +57,8 @@
  * Morello embeds the user permissions in the middle of the hardware permissions.
  */
 
+#define CC_HAVE_VERSION 1
+
 // The following macros are expected to be defined
 #undef CC_BITS
 #define CC_BITS 128
@@ -125,7 +128,12 @@ enum {
     _CC_FIELD(HWPERMS, 123, 112),
     _CC_FIELD(RESERVED, 111, 110),
     _CC_FIELD(FLAGS, 109, 109),
+#ifdef CC_HAVE_VERSION
+    _CC_FIELD(VERSION, 108, 105),
+    _CC_FIELD(OTYPE, 104, 91),
+#else
     _CC_FIELD(OTYPE, 108, 91),
+#endif
     _CC_FIELD(EBT, 90, 64),
 
     _CC_FIELD(INTERNAL_EXPONENT, 90, 90),
@@ -149,6 +157,7 @@ enum {
 #define CC128_BOT_WIDTH CC128_FIELD_EXP_ZERO_BOTTOM_SIZE
 #define CC128_BOT_INTERNAL_EXP_WIDTH CC128_FIELD_EXP_NONZERO_BOTTOM_SIZE
 #define CC128_EXP_LOW_WIDTH CC128_FIELD_EXPONENT_LOW_PART_SIZE
+#define CC128_VERSION_BITS CC128_FIELD_VERSION_SIZE
 
 #ifdef CC_IS_MORELLO
 
@@ -244,14 +253,25 @@ enum _CC_N(OTypes) {
 
 _CC_STATIC_ASSERT_SAME(CC128_MANTISSA_WIDTH, CC128_FIELD_EXP_ZERO_BOTTOM_SIZE);
 
+#ifdef CC_HAVE_VERSION
+enum _CC_N(Versions) {
+    CC128_VERSION_UNVERSIONED = 0,
+    CC128_MAX_VERSION = ((1u << CC128_VERSION_BITS) - 1u),
+};
+#endif
+
 #include "cheri_compressed_cap_common.h"
 
 // Sanity-check mask is the expected NULL encoding
 #ifdef CC_IS_MORELLO
 _CC_STATIC_ASSERT_SAME(CC128_NULL_XOR_MASK, UINT64_C(0x0000000040070007));
 #else
+#ifdef CC_HAVE_VERSION
+_CC_STATIC_ASSERT_SAME(CC128_NULL_XOR_MASK, UINT64_C(0x000001fffc018004));
+#else
 _CC_STATIC_ASSERT_SAME(CC128_NULL_XOR_MASK, UINT64_C(0x00001ffffc018004));
 #endif
+#endif
 
 __attribute__((deprecated("Use cc128_compress_raw"))) static inline uint64_t
 compress_128cap_without_xor(const cc128_cap_t* csp) {

target/cheri-common/cheri-compressed-cap/cheri_compressed_cap_64.h

@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2018-2020 Alex Richardson
+ * Copyright (c) 2021 Microsoft <robert.norton@microsoft.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -84,6 +85,7 @@ enum {
     _CC_FIELD(EXPONENT_LOW_PART, 34, 32),
     _CC_FIELD(RESERVED, 31, 32), /* No reserved bits */
     _CC_FIELD(UPERMS, 31, 32), /* No uperms */
+    _CC_FIELD(VERSION, 31, 32), /* No version */
 };
 #pragma GCC diagnostic pop
 _CC_STATIC_ASSERT_SAME(CC64_FIELD_UPERMS_SIZE, 0);