[PATCH 1/3] accel/tcg: Add infrastructure for atomic 64-byte load/store

Signed-off-by: Richard Henderson <[email protected]>
---
 include/accel/tcg/cpu-ldst-common.h           |  6 ++++
 include/qemu/atomic512.h                      | 12 +++++++
 include/qemu/value64.h                        | 14 ++++++++
 accel/tcg/cputlb.c                            | 30 +++++++++++++++++
 accel/tcg/user-exec.c                         | 33 +++++++++++++++++++
 accel/tcg/ldst_common.c.inc                   | 31 +++++++++++++++++
 .../include/generic/host/atomic512-ldst.h.inc | 16 +++++++++
 7 files changed, 142 insertions(+)
 create mode 100644 include/qemu/atomic512.h
 create mode 100644 include/qemu/value64.h
 create mode 100644 host/include/generic/host/atomic512-ldst.h.inc

diff --git a/include/accel/tcg/cpu-ldst-common.h b/include/accel/tcg/cpu-ldst-common.h
index f12be8cfb7..d8fae3384e 100644
--- a/include/accel/tcg/cpu-ldst-common.h
+++ b/include/accel/tcg/cpu-ldst-common.h
@@ -15,6 +15,7 @@
 #include "exec/vaddr.h"
 #include "exec/mmu-access-type.h"
 #include "qemu/int128.h"
+#include "qemu/value64.h"
 
 uint8_t cpu_ldb_mmu(CPUArchState *env, vaddr ptr, MemOpIdx oi, uintptr_t ra);
 uint16_t cpu_ldw_mmu(CPUArchState *env, vaddr ptr, MemOpIdx oi, uintptr_t ra);
@@ -117,4 +118,9 @@ uint32_t cpu_ldl_code_mmu(CPUArchState *env, vaddr addr,
 uint64_t cpu_ldq_code_mmu(CPUArchState *env, vaddr addr,
                           MemOpIdx oi, uintptr_t ra);
 
+Value64 cpu_atomic_ld64_mmu(CPUArchState *env, vaddr ptr,
+                            MemOpIdx oi, uintptr_t ra);
+void cpu_atomic_st64_mmu(CPUArchState *env, vaddr ptr, const Value64 *val,
+                         MemOpIdx oi, uintptr_t ra);
+
 #endif /* ACCEL_TCG_CPU_LDST_COMMON_H */
diff --git a/include/qemu/atomic512.h b/include/qemu/atomic512.h
new file mode 100644
index 0000000000..8553a0feba
--- /dev/null
+++ b/include/qemu/atomic512.h
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Load/store for 512-bit atomic operations.
+ */
+
+#ifndef QEMU_ATOMIC512_H
+#define QEMU_ATOMIC512_H
+
+#include "qemu/value64.h"
+#include "host/atomic512-ldst.h.inc"
+
+#endif /* QEMU_ATOMIC512_H */
diff --git a/include/qemu/value64.h b/include/qemu/value64.h
new file mode 100644
index 0000000000..7de287a13d
--- /dev/null
+++ b/include/qemu/value64.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef QEMU_VALUE64_H
+#define QEMU_VALUE64_H
+
+/*
+ * An stream of 64 bytes; no endianness implied.
+ * The members are perforce host byte ordering.
+ */
+typedef union {
+    uint64_t l[8];
+} Value64;
+
+#endif /* QEMU_VALUE64_H */
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index 7f7c208ba1..1f192f1837 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -42,6 +42,7 @@
 #include "exec/tlb-flags.h"
 #include "qemu/atomic.h"
 #include "qemu/atomic128.h"
+#include "qemu/atomic512.h"
 #include "tb-internal.h"
 #include "trace.h"
 #include "tb-hash.h"
@@ -2429,6 +2430,20 @@ static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr,
     return ret;
 }
 
+static Value64 do_atomic_ld64_mmu(CPUState *cpu, vaddr addr,
+                                  MemOpIdx oi, uintptr_t ra)
+{
+    MMULookupLocals l;
+
+    cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
+    mmu_lookup(cpu, addr, oi, ra, MMU_DATA_LOAD, &l);
+
+    if (likely(HAVE_ATOMIC512_LD && !(l.page[0].flags & TLB_MMIO))) {
+        return atomic64_read(l.page[0].haddr);
+    }
+    cpu_loop_exit_atomic(cpu, ra);
+}
+
 /*
  * Store Helpers
  */
@@ -2825,6 +2840,21 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
     }
 }
 
+static void do_atomic_st64_mmu(CPUState *cpu, vaddr addr, const Value64 *val,
+                               MemOpIdx oi, uintptr_t ra)
+{
+    MMULookupLocals l;
+
+    cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
+    mmu_lookup(cpu, addr, oi, ra, MMU_DATA_LOAD, &l);
+
+    if (likely(HAVE_ATOMIC512_ST && !(l.page[0].flags & TLB_MMIO))) {
+        atomic64_set(l.page[0].haddr, val);
+    } else {
+        cpu_loop_exit_atomic(cpu, ra);
+    }
+}
+
 #include "ldst_common.c.inc"
 
 /*
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index a35aca7889..d6d2eec2e1 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -35,6 +35,7 @@
 #include "exec/page-protection.h"
 #include "exec/helper-proto-common.h"
 #include "qemu/atomic128.h"
+#include "qemu/atomic512.h"
 #include "qemu/bswap.h"
 #include "qemu/int128.h"
 #include "trace.h"
@@ -1116,6 +1117,22 @@ static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr,
     return ret;
 }
 
+static Value64 do_atomic_ld64_mmu(CPUState *cpu, vaddr addr,
+                                  MemOpIdx oi, uintptr_t ra)
+{
+    void *haddr;
+
+    cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
+    haddr = cpu_mmu_lookup(cpu, addr, get_memop(oi), ra, MMU_DATA_LOAD);
+
+    if (HAVE_ATOMIC512_LD) {
+        Value64 ret = atomic64_read(haddr);
+        clear_helper_retaddr();
+        return ret;
+    }
+    cpu_loop_exit_atomic(cpu, ra);
+}
+
 static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val,
                        MemOpIdx oi, uintptr_t ra)
 {
@@ -1191,6 +1208,22 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
     clear_helper_retaddr();
 }
 
+static void do_atomic_st64_mmu(CPUState *cpu, vaddr addr, const Value64 *val,
+                               MemOpIdx oi, uintptr_t ra)
+{
+    void *haddr;
+
+    cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
+    haddr = cpu_mmu_lookup(cpu, addr, get_memop(oi), ra, MMU_DATA_LOAD);
+
+    if (HAVE_ATOMIC512_ST) {
+        atomic64_set(haddr, val);
+        clear_helper_retaddr();
+    } else {
+        cpu_loop_exit_atomic(cpu, ra);
+    }
+}
+
 uint8_t cpu_ldb_code_mmu(CPUArchState *env, vaddr addr,
                          MemOpIdx oi, uintptr_t ra)
 {
diff --git a/accel/tcg/ldst_common.c.inc b/accel/tcg/ldst_common.c.inc
index 57f3e06192..3ce560a1f2 100644
--- a/accel/tcg/ldst_common.c.inc
+++ b/accel/tcg/ldst_common.c.inc
@@ -189,6 +189,23 @@ Int128 cpu_ld16_mmu(CPUArchState *env, vaddr addr,
     return ret;
 }
 
+Value64 cpu_atomic_ld64_mmu(CPUArchState *env, vaddr addr,
+                            MemOpIdx oi, uintptr_t ra)
+{
+    MemOp mop = get_memop(oi);
+
+    tcg_debug_assert((mop & MO_SIZE) == MO_512);
+    tcg_debug_assert(memop_alignment_bits(mop) == MO_512);
+    tcg_debug_assert((mop & MO_BSWAP) == 0);
+
+    {
+        Value64 ret = do_atomic_ld64_mmu(env_cpu(env), addr, oi, ra);
+        /* TODO: 48 other bytes unreported to plugins. */
+        plugin_load_cb(env, addr, ret.l[0], ret.l[1], oi);
+        return ret;
+    }
+}
+
 /*
  * Store helpers for cpu_ldst.h
  */
@@ -243,3 +260,17 @@ void cpu_st16_mmu(CPUArchState *env, vaddr addr, Int128 val,
     do_st16_mmu(env_cpu(env), addr, val, oi, retaddr);
     plugin_store_cb(env, addr, int128_getlo(val), int128_gethi(val), oi);
 }
+
+void cpu_atomic_st64_mmu(CPUArchState *env, vaddr addr, const Value64 *val,
+                         MemOpIdx oi, uintptr_t retaddr)
+{
+    MemOp mop = get_memop(oi);
+
+    tcg_debug_assert((mop & MO_SIZE) == MO_512);
+    tcg_debug_assert(memop_alignment_bits(mop) == MO_512);
+    tcg_debug_assert((mop & MO_BSWAP) == 0);
+
+    do_atomic_st64_mmu(env_cpu(env), addr, val, oi, retaddr);
+    /* TODO: 48 other bytes unreported to plugins. */
+    plugin_store_cb(env, addr, val->l[0], val->l[1], oi);
+}
diff --git a/host/include/generic/host/atomic512-ldst.h.inc b/host/include/generic/host/atomic512-ldst.h.inc
new file mode 100644
index 0000000000..8b79fcd13d
--- /dev/null
+++ b/host/include/generic/host/atomic512-ldst.h.inc
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Load/store for 512-bit atomic operations, generic version.
+ */
+
+#ifndef HOST_ATOMIC512_LDST_H
+#define HOST_ATOMIC512_LDST_H
+
+#define HAVE_ATOMIC512_LD 0
+#define HAVE_ATOMIC512_ST 0
+
+/* Fallback definitions that must be optimized away, or error.  */
+Value64 QEMU_ERROR("unsupported atomic") atomic64_read(const void *s);
+void QEMU_ERROR("unsupported atomic") atomic64_set(void *d, const Value64 *v);
+
+#endif /* HOST_ATOMIC512_LDST_H */
-- 
2.43.0