migration: Append JSON description of migration stream

One of the annoyances of the current migration format is the fact that
it's not self-describing. In fact, it's not properly describing at all.
Some code randomly scattered throughout QEMU elaborates roughly how to
read and write a stream of bytes.

We discussed an idea during KVM Forum 2013 to add a JSON description of
the migration protocol itself to the migration stream. This patch
adds a section after the VM_END migration end marker that contains
description data on what the device sections of the stream are composed of.

This approach is backwards compatible with any QEMU version reading the
stream, because QEMU just stops reading after the VM_END marker and ignores
any data following it.

With an additional external program this allows us to decipher the
contents of any migration stream and hopefully make migration bugs easier
to track down.

Signed-off-by: Alexander Graf <agraf@suse.de>
Signed-off-by: Amit Shah <amit.shah@redhat.com>
Signed-off-by: Juan Quintela <quintela@redhat.com>

1 files changed, 180 insertions, 6 deletions

diff --git a/migration/vmstate.c b/migration/vmstate.c
index dae5dd63af..e5388f0596 100644
--- a/migration/vmstate.c
+++ b/migration/vmstate.c
@@ -5,9 +5,10 @@
 #include "qemu/bitops.h"
 #include "qemu/error-report.h"
 #include "trace.h"
+#include "qjson.h"
 
 static void vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd,
-                                    void *opaque);
+                                    void *opaque, QJSON *vmdesc);
 static int vmstate_subsection_load(QEMUFile *f, const VMStateDescription *vmsd,
                                    void *opaque);
 
@@ -146,32 +147,181 @@ int vmstate_load_state(QEMUFile *f, const VMStateDescription *vmsd,
     return ret;
 }
 
+static int vmfield_name_num(VMStateField *start, VMStateField *search)
+{
+    VMStateField *field;
+    int found = 0;
+
+    for (field = start; field->name; field++) {
+        if (!strcmp(field->name, search->name)) {
+            if (field == search) {
+                return found;
+            }
+            found++;
+        }
+    }
+
+    return -1;
+}
+
+static bool vmfield_name_is_unique(VMStateField *start, VMStateField *search)
+{
+    VMStateField *field;
+    int found = 0;
+
+    for (field = start; field->name; field++) {
+        if (!strcmp(field->name, search->name)) {
+            found++;
+            /* name found more than once, so it's not unique */
+            if (found > 1) {
+                return false;
+            }
+        }
+    }
+
+    return true;
+}
+
+static const char *vmfield_get_type_name(VMStateField *field)
+{
+    const char *type = "unknown";
+
+    if (field->flags & VMS_STRUCT) {
+        type = "struct";
+    } else if (field->info->name) {
+        type = field->info->name;
+    }
+
+    return type;
+}
+
+static bool vmsd_can_compress(VMStateField *field)
+{
+    if (field->field_exists) {
+        /* Dynamically existing fields mess up compression */
+        return false;
+    }
+
+    if (field->flags & VMS_STRUCT) {
+        VMStateField *sfield = field->vmsd->fields;
+        while (sfield->name) {
+            if (!vmsd_can_compress(sfield)) {
+                /* Child elements can't compress, so can't we */
+                return false;
+            }
+            sfield++;
+        }
+
+        if (field->vmsd->subsections) {
+            /* Subsections may come and go, better don't compress */
+            return false;
+        }
+    }
+
+    return true;
+}
+
+static void vmsd_desc_field_start(const VMStateDescription *vmsd, QJSON *vmdesc,
+                                  VMStateField *field, int i, int max)
+{
+    char *name, *old_name;
+    bool is_array = max > 1;
+    bool can_compress = vmsd_can_compress(field);
+
+    if (!vmdesc) {
+        return;
+    }
+
+    name = g_strdup(field->name);
+
+    /* Field name is not unique, need to make it unique */
+    if (!vmfield_name_is_unique(vmsd->fields, field)) {
+        int num = vmfield_name_num(vmsd->fields, field);
+        old_name = name;
+        name = g_strdup_printf("%s[%d]", name, num);
+        g_free(old_name);
+    }
+
+    json_start_object(vmdesc, NULL);
+    json_prop_str(vmdesc, "name", name);
+    if (is_array) {
+        if (can_compress) {
+            json_prop_int(vmdesc, "array_len", max);
+        } else {
+            json_prop_int(vmdesc, "index", i);
+        }
+    }
+    json_prop_str(vmdesc, "type", vmfield_get_type_name(field));
+
+    if (field->flags & VMS_STRUCT) {
+        json_start_object(vmdesc, "struct");
+    }
+
+    g_free(name);
+}
+
+static void vmsd_desc_field_end(const VMStateDescription *vmsd, QJSON *vmdesc,
+                                VMStateField *field, size_t size, int i)
+{
+    if (!vmdesc) {
+        return;
+    }
+
+    if (field->flags & VMS_STRUCT) {
+        /* We printed a struct in between, close its child object */
+        json_end_object(vmdesc);
+    }
+
+    json_prop_int(vmdesc, "size", size);
+    json_end_object(vmdesc);
+}
+
 void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
-                        void *opaque)
+                        void *opaque, QJSON *vmdesc)
 {
     VMStateField *field = vmsd->fields;
 
     if (vmsd->pre_save) {
         vmsd->pre_save(opaque);
     }
+
+    if (vmdesc) {
+        json_prop_str(vmdesc, "vmsd_name", vmsd->name);
+        json_prop_int(vmdesc, "version", vmsd->version_id);
+        json_start_array(vmdesc, "fields");
+    }
+
     while (field->name) {
         if (!field->field_exists ||
             field->field_exists(opaque, vmsd->version_id)) {
             void *base_addr = vmstate_base_addr(opaque, field, false);
             int i, n_elems = vmstate_n_elems(opaque, field);
             int size = vmstate_size(opaque, field);
+            int64_t old_offset, written_bytes;
+            QJSON *vmdesc_loop = vmdesc;
 
             for (i = 0; i < n_elems; i++) {
                 void *addr = base_addr + size * i;
 
+                vmsd_desc_field_start(vmsd, vmdesc_loop, field, i, n_elems);
+                old_offset = qemu_ftell_fast(f);
+
                 if (field->flags & VMS_ARRAY_OF_POINTER) {
                     addr = *(void **)addr;
                 }
                 if (field->flags & VMS_STRUCT) {
-                    vmstate_save_state(f, field->vmsd, addr);
+                    vmstate_save_state(f, field->vmsd, addr, vmdesc_loop);
                 } else {
                     field->info->put(f, addr, size);
                 }
+
+                written_bytes = qemu_ftell_fast(f) - old_offset;
+                vmsd_desc_field_end(vmsd, vmdesc_loop, field, written_bytes, i);
+
+                /* Compressed arrays only care about the first element */
+                if (vmdesc_loop && vmsd_can_compress(field)) {
+                    vmdesc_loop = NULL;
+                }
             }
         } else {
             if (field->flags & VMS_MUST_EXIST) {
@@ -182,7 +332,12 @@ void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
         }
         field++;
     }
-    vmstate_subsection_save(f, vmsd, opaque);
+
+    if (vmdesc) {
+        json_end_array(vmdesc);
+    }
+
+    vmstate_subsection_save(f, vmsd, opaque, vmdesc);
 }
 
 static const VMStateDescription *
@@ -248,24 +403,43 @@ static int vmstate_subsection_load(QEMUFile *f, const VMStateDescription *vmsd,
 }
 
 static void vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd,
-                                    void *opaque)
+                                    void *opaque, QJSON *vmdesc)
 {
     const VMStateSubsection *sub = vmsd->subsections;
+    bool subsection_found = false;
 
     while (sub && sub->needed) {
         if (sub->needed(opaque)) {
             const VMStateDescription *vmsd = sub->vmsd;
             uint8_t len;
 
+            if (vmdesc) {
+                /* Only create subsection array when we have any */
+                if (!subsection_found) {
+                    json_start_array(vmdesc, "subsections");
+                    subsection_found = true;
+                }
+
+                json_start_object(vmdesc, NULL);
+            }
+
             qemu_put_byte(f, QEMU_VM_SUBSECTION);
             len = strlen(vmsd->name);
             qemu_put_byte(f, len);
             qemu_put_buffer(f, (uint8_t *)vmsd->name, len);
             qemu_put_be32(f, vmsd->version_id);
-            vmstate_save_state(f, vmsd, opaque);
+            vmstate_save_state(f, vmsd, opaque, vmdesc);
+
+            if (vmdesc) {
+                json_end_object(vmdesc);
+            }
         }
         sub++;
     }
+
+    if (vmdesc && subsection_found) {
+        json_end_array(vmdesc);
+    }
 }
 
 /* bool */