[PATCH v2] vhost-user: check memory slot availability for SHMEM_MAP

SHMEM_MAP creates a separate RAM MemoryRegion for every mapping. It can
therefore make a vhost-user memory table exceed the number of slots
negotiated with the backend.

Reject a mapping before changing the memory topology if KVM or a vhost
backend has no unreserved slot left. Also validate each memory table
against the negotiated vhost-user limit as a final defense for mappings
that become visible in a later memory transaction.

Fixes: b52e1896e764 ("vhost-user: Add VirtIO Shared Memory map request")
Signed-off-by: Feifan Qian <[email protected]>
---
Tested with an ASan/UBSan x86_64 build and a vhost-user backend that
filled the negotiated 512-slot memory table.  The next SHMEM_MAP request
was rejected and QEMU remained alive, with no sanitizer report.
qtest-x86_64/qos-test also passed all 134 subtests.

 hw/virtio/vhost-user.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 2881cec72d..3673653f67 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -10,6 +10,7 @@

 #include "qemu/osdep.h"
 #include "qapi/error.h"
+#include "hw/mem/memory-device.h"
 #include "hw/virtio/virtio-dmabuf.h"
 #include "hw/virtio/virtio-qmp.h"
 #include "hw/virtio/vhost.h"
@@ -1126,6 +1127,13 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,
             dev, VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
     int ret;

+    if (mem->nregions > u->user->memory_slots) {
+        error_report("vhost-user memory table has %u regions, "
+                     "but the backend supports only %d",
+                     mem->nregions, u->user->memory_slots);
+        return -ENOSPC;
+    }
+
     if (do_postcopy) {
         /*
          * Postcopy has enough differences that it's best done in it's own
@@ -1940,6 +1948,7 @@ vhost_user_backend_handle_shmem_map(struct vhost_dev *dev,
     VhostUserMMap *vu_mmap = &payload->mmap;
     VirtioSharedMemoryMapping *existing;
     Error *local_err = NULL;
+    unsigned int reserved_memslots;
     int ret = 0;

     if (fd < 0) {
@@ -1980,6 +1989,15 @@ vhost_user_backend_handle_shmem_map(struct vhost_dev *dev,
         }
     }

+    reserved_memslots = memory_devices_get_reserved_memslots();
+    if ((kvm_enabled() &&
+         kvm_get_free_memslots() <= reserved_memslots) ||
+        vhost_get_free_memslots() <= reserved_memslots) {
+        error_report("No free memory slots for shared memory mapping");
+        ret = -ENOSPC;
+        goto send_reply;
+    }
+
     memory_region_transaction_begin();

     /* Create VirtioSharedMemoryMapping object */
--
2.43.0