virtio: Add memory statistics reporting to the balloon driver

When using ballooning to manage overcommitted memory on a host, a system for
guests to communicate their memory usage to the host can provide information
that will minimize the impact of ballooning on the guests.  The current method
employs a daemon running in each guest that communicates memory statistics to a
host daemon at a specified time interval.  The host daemon aggregates this
information and inflates and/or deflates balloons according to the level of
host memory pressure.  This approach is effective but overly complex since a
daemon must be installed inside each guest and coordinated to communicate with
the host.  A simpler approach is to collect memory statistics in the virtio
balloon driver and communicate them directly to the hypervisor.

Signed-off-by: Adam Litke <agl@us.ibm.com>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

1 files changed, 123 insertions, 4 deletions

diff --git a/hw/virtio-balloon.c b/hw/virtio-balloon.c
index e17880ff7c..086d9d1ecd 100644
--- a/hw/virtio-balloon.c
+++ b/hw/virtio-balloon.c
@@ -16,9 +16,13 @@
 #include "pc.h"
 #include "sysemu.h"
 #include "cpu.h"
+#include "monitor.h"
 #include "balloon.h"
 #include "virtio-balloon.h"
 #include "kvm.h"
+#include "qlist.h"
+#include "qint.h"
+#include "qstring.h"
 
 #if defined(__linux__)
 #include <sys/mman.h>
@@ -27,9 +31,14 @@
 typedef struct VirtIOBalloon
 {
     VirtIODevice vdev;
-    VirtQueue *ivq, *dvq;
+    VirtQueue *ivq, *dvq, *svq;
     uint32_t num_pages;
     uint32_t actual;
+    uint64_t stats[VIRTIO_BALLOON_S_NR];
+    VirtQueueElement stats_vq_elem;
+    size_t stats_vq_offset;
+    MonitorCompletion *stats_callback;
+    void *stats_opaque_callback_data;
 } VirtIOBalloon;
 
 static VirtIOBalloon *to_virtio_balloon(VirtIODevice *vdev)
@@ -46,6 +55,42 @@ static void balloon_page(void *addr, int deflate)
 #endif
 }
 
+/*
+ * reset_stats - Mark all items in the stats array as unset
+ *
+ * This function needs to be called at device intialization and before
+ * before updating to a set of newly-generated stats.  This will ensure that no
+ * stale values stick around in case the guest reports a subset of the supported
+ * statistics.
+ */
+static inline void reset_stats(VirtIOBalloon *dev)
+{
+    int i;
+    for (i = 0; i < VIRTIO_BALLOON_S_NR; dev->stats[i++] = -1);
+}
+
+static void stat_put(QDict *dict, const char *label, uint64_t val)
+{
+    if (val != -1)
+        qdict_put(dict, label, qint_from_int(val));
+}
+
+static QObject *get_stats_qobject(VirtIOBalloon *dev)
+{
+    QDict *dict = qdict_new();
+    uint32_t actual = ram_size - (dev->actual << VIRTIO_BALLOON_PFN_SHIFT);
+
+    stat_put(dict, "actual", actual);
+    stat_put(dict, "mem_swapped_in", dev->stats[VIRTIO_BALLOON_S_SWAP_IN]);
+    stat_put(dict, "mem_swapped_out", dev->stats[VIRTIO_BALLOON_S_SWAP_OUT]);
+    stat_put(dict, "major_page_faults", dev->stats[VIRTIO_BALLOON_S_MAJFLT]);
+    stat_put(dict, "minor_page_faults", dev->stats[VIRTIO_BALLOON_S_MINFLT]);
+    stat_put(dict, "free_mem", dev->stats[VIRTIO_BALLOON_S_MEMFREE]);
+    stat_put(dict, "total_mem", dev->stats[VIRTIO_BALLOON_S_MEMTOT]);
+
+    return QOBJECT(dict);
+}
+
 /* FIXME: once we do a virtio refactoring, this will get subsumed into common
  * code */
 static size_t memcpy_from_iovector(void *data, size_t offset, size_t size,
@@ -104,6 +149,51 @@ static void virtio_balloon_handle_output(VirtIODevice *vdev, VirtQueue *vq)
     }
 }
 
+static void complete_stats_request(VirtIOBalloon *vb)
+{
+    QObject *stats;
+
+    if (!vb->stats_opaque_callback_data)
+        return;
+
+    stats = get_stats_qobject(vb);
+    vb->stats_callback(vb->stats_opaque_callback_data, stats);
+    qobject_decref(stats);
+    vb->stats_opaque_callback_data = NULL;
+    vb->stats_callback = NULL;
+}
+
+static void virtio_balloon_receive_stats(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOBalloon *s = DO_UPCAST(VirtIOBalloon, vdev, vdev);
+    VirtQueueElement *elem = &s->stats_vq_elem;
+    VirtIOBalloonStat stat;
+    size_t offset = 0;
+
+    if (!virtqueue_pop(vq, elem)) {
+        return;
+    }
+
+    /* Initialize the stats to get rid of any stale values.  This is only
+     * needed to handle the case where a guest supports fewer stats than it
+     * used to (ie. it has booted into an old kernel).
+     */
+    reset_stats(s);
+
+    while (memcpy_from_iovector(&stat, offset, sizeof(stat), elem->out_sg,
+                                elem->out_num) == sizeof(stat)) {
+        uint16_t tag = tswap16(stat.tag);
+        uint64_t val = tswap64(stat.val);
+
+        offset += sizeof(stat);
+        if (tag < VIRTIO_BALLOON_S_NR)
+            s->stats[tag] = val;
+    }
+    s->stats_vq_offset = offset;
+
+    complete_stats_request(s);
+}
+
 static void virtio_balloon_get_config(VirtIODevice *vdev, uint8_t *config_data)
 {
     VirtIOBalloon *dev = to_virtio_balloon(vdev);
@@ -126,10 +216,12 @@ static void virtio_balloon_set_config(VirtIODevice *vdev,
 
 static uint32_t virtio_balloon_get_features(VirtIODevice *vdev, uint32_t f)
 {
+    f |= (1 << VIRTIO_BALLOON_F_STATS_VQ);
     return f;
 }
 
-static ram_addr_t virtio_balloon_to_target(void *opaque, ram_addr_t target)
+static void virtio_balloon_to_target(void *opaque, ram_addr_t target,
+                                     MonitorCompletion cb, void *cb_data)
 {
     VirtIOBalloon *dev = opaque;
 
@@ -139,9 +231,26 @@ static ram_addr_t virtio_balloon_to_target(void *opaque, ram_addr_t target)
     if (target) {
         dev->num_pages = (ram_size - target) >> VIRTIO_BALLOON_PFN_SHIFT;
         virtio_notify_config(&dev->vdev);
+    } else {
+        /* For now, only allow one request at a time.  This restriction can be
+         * removed later by queueing callback and data pairs.
+         */
+        if (dev->stats_callback != NULL) {
+            return;
+        }
+        dev->stats_callback = cb;
+        dev->stats_opaque_callback_data = cb_data; 
+        if (dev->vdev.guest_features & (1 << VIRTIO_BALLOON_F_STATS_VQ)) {
+            virtqueue_push(dev->svq, &dev->stats_vq_elem, dev->stats_vq_offset);
+            virtio_notify(&dev->vdev, dev->svq);
+        } else {
+            /* Stats are not supported.  Clear out any stale values that might
+             * have been set by a more featureful guest kernel.
+             */
+            reset_stats(dev);
+            complete_stats_request(dev);
+        }
     }
-
-    return ram_size - (dev->actual << VIRTIO_BALLOON_PFN_SHIFT);
 }
 
 static void virtio_balloon_save(QEMUFile *f, void *opaque)
@@ -152,6 +261,10 @@ static void virtio_balloon_save(QEMUFile *f, void *opaque)
 
     qemu_put_be32(f, s->num_pages);
     qemu_put_be32(f, s->actual);
+    qemu_put_buffer(f, (uint8_t *)&s->stats_vq_elem, sizeof(VirtQueueElement));
+    qemu_put_buffer(f, (uint8_t *)&s->stats_vq_offset, sizeof(size_t));
+    qemu_put_buffer(f, (uint8_t *)&s->stats_callback, sizeof(MonitorCompletion));
+    qemu_put_buffer(f, (uint8_t *)&s->stats_opaque_callback_data, sizeof(void));
 }
 
 static int virtio_balloon_load(QEMUFile *f, void *opaque, int version_id)
@@ -165,6 +278,10 @@ static int virtio_balloon_load(QEMUFile *f, void *opaque, int version_id)
 
     s->num_pages = qemu_get_be32(f);
     s->actual = qemu_get_be32(f);
+    qemu_get_buffer(f, (uint8_t *)&s->stats_vq_elem, sizeof(VirtQueueElement));
+    qemu_get_buffer(f, (uint8_t *)&s->stats_vq_offset, sizeof(size_t));
+    qemu_get_buffer(f, (uint8_t *)&s->stats_callback, sizeof(MonitorCompletion));
+    qemu_get_buffer(f, (uint8_t *)&s->stats_opaque_callback_data, sizeof(void));
 
     return 0;
 }
@@ -183,7 +300,9 @@ VirtIODevice *virtio_balloon_init(DeviceState *dev)
 
     s->ivq = virtio_add_queue(&s->vdev, 128, virtio_balloon_handle_output);
     s->dvq = virtio_add_queue(&s->vdev, 128, virtio_balloon_handle_output);
+    s->svq = virtio_add_queue(&s->vdev, 128, virtio_balloon_receive_stats);
 
+    reset_stats(s);
     qemu_add_balloon_handler(virtio_balloon_to_target, s);
 
     register_savevm("virtio-balloon", -1, 1, virtio_balloon_save, virtio_balloon_load, s);