Merge remote-tracking branch 'remotes/dgibson/tags/ppc-for-2.7-20160617' into staging

ppc patch queue for 2016-06-17

Here's the current accumulated set of spapr, ppc and related patches.
  * The big thing in here is CPU hotplug for spapr
    - This includes a number of acked generic changes adding new
      infrastructure for hotplugging cpu cores
  * A number of TCG bug fixes are also included
  * This adds a new testcase to make it harder to accidentally break
    Macintosh (and other openbios) platforms

# gpg: Signature made Fri 17 Jun 2016 07:35:29 BST
# gpg:                using RSA key 0x6C38CACA20D9B392
# gpg: Good signature from "David Gibson <david@gibson.dropbear.id.au>"
# gpg:                 aka "David Gibson (Red Hat) <dgibson@redhat.com>"
# gpg:                 aka "David Gibson (ozlabs.org) <dgibson@ozlabs.org>"
# gpg: WARNING: This key is not certified with sufficiently trusted signatures!
# gpg:          It is not certain that the signature belongs to the owner.
# Primary key fingerprint: 75F4 6586 AE61 A66C C44E  87DC 6C38 CACA 20D9 B392

* remotes/dgibson/tags/ppc-for-2.7-20160617:
  spapr: implement query-hotpluggable-cpus callback
  hmp: Add 'info hotpluggable-cpus' HMP command
  QMP: Add query-hotpluggable-cpus
  spapr: CPU hot unplug support
  spapr: CPU hotplug support
  spapr: convert boot CPUs into CPU core devices
  spapr: Move spapr_cpu_init() to spapr_cpu_core.c
  spapr: Abstract CPU core device and type specific core devices
  qom: API to get instance_size of a type
  spapr_drc: Prevent detach racing against attach for CPU DR
  xics,xics_kvm: Handle CPU unplug correctly
  cpu: Abstract CPU core type
  qdev: hotplug: Introduce HotplugHandler.pre_plug() callback
  target-ppc: Fix rlwimi, rlwinm, rlwnm
  vfio: Fix broken EEH
  target-ppc: Bug in BookE wait instruction
  ppc / sparc: Add a tester for checking whether OpenBIOS runs successfully
  hw/ppc/spapr: Silence deprecation message in qtest mode

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

14 files changed, 735 insertions, 66 deletions

diff --git a/hw/core/hotplug.c b/hw/core/hotplug.c
index 645cfca1b9..17ac986685 100644
--- a/hw/core/hotplug.c
+++ b/hw/core/hotplug.c
@@ -13,6 +13,17 @@
 #include "hw/hotplug.h"
 #include "qemu/module.h"
 
+void hotplug_handler_pre_plug(HotplugHandler *plug_handler,
+                              DeviceState *plugged_dev,
+                              Error **errp)
+{
+    HotplugHandlerClass *hdc = HOTPLUG_HANDLER_GET_CLASS(plug_handler);
+
+    if (hdc->pre_plug) {
+        hdc->pre_plug(plug_handler, plugged_dev, errp);
+    }
+}
+
 void hotplug_handler_plug(HotplugHandler *plug_handler,
                           DeviceState *plugged_dev,
                           Error **errp)
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index dcc00f8c70..6680089154 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -902,6 +902,14 @@ static void device_set_realized(Object *obj, bool value, Error **errp)
             g_free(name);
         }
 
+        hotplug_ctrl = qdev_get_hotplug_handler(dev);
+        if (hotplug_ctrl) {
+            hotplug_handler_pre_plug(hotplug_ctrl, dev, &local_err);
+            if (local_err != NULL) {
+                goto fail;
+            }
+        }
+
         if (dc->realize) {
             dc->realize(dev, &local_err);
         }
@@ -912,7 +920,6 @@ static void device_set_realized(Object *obj, bool value, Error **errp)
 
         DEVICE_LISTENER_CALL(realize, Forward, dev);
 
-        hotplug_ctrl = qdev_get_hotplug_handler(dev);
         if (hotplug_ctrl) {
             hotplug_handler_plug(hotplug_ctrl, dev, &local_err);
         }
diff --git a/hw/cpu/Makefile.objs b/hw/cpu/Makefile.objs
index 0954a1872f..942a4bb82e 100644
--- a/hw/cpu/Makefile.objs
+++ b/hw/cpu/Makefile.objs
@@ -2,4 +2,5 @@ obj-$(CONFIG_ARM11MPCORE) += arm11mpcore.o
 obj-$(CONFIG_REALVIEW) += realview_mpcore.o
 obj-$(CONFIG_A9MPCORE) += a9mpcore.o
 obj-$(CONFIG_A15MPCORE) += a15mpcore.o
+obj-y += core.o
 
diff --git a/hw/cpu/core.c b/hw/cpu/core.c
new file mode 100644
index 0000000000..eff90c12be
--- /dev/null
+++ b/hw/cpu/core.c
@@ -0,0 +1,88 @@
+/*
+ * CPU core abstract device
+ *
+ * Copyright (C) 2016 Bharata B Rao <bharata@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "hw/cpu/core.h"
+#include "qapi/visitor.h"
+#include "qapi/error.h"
+#include "sysemu/cpus.h"
+
+static void core_prop_get_core_id(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
+{
+    CPUCore *core = CPU_CORE(obj);
+    int64_t value = core->core_id;
+
+    visit_type_int(v, name, &value, errp);
+}
+
+static void core_prop_set_core_id(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
+{
+    CPUCore *core = CPU_CORE(obj);
+    Error *local_err = NULL;
+    int64_t value;
+
+    visit_type_int(v, name, &value, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    core->core_id = value;
+}
+
+static void core_prop_get_nr_threads(Object *obj, Visitor *v, const char *name,
+                                     void *opaque, Error **errp)
+{
+    CPUCore *core = CPU_CORE(obj);
+    int64_t value = core->nr_threads;
+
+    visit_type_int(v, name, &value, errp);
+}
+
+static void core_prop_set_nr_threads(Object *obj, Visitor *v, const char *name,
+                                     void *opaque, Error **errp)
+{
+    CPUCore *core = CPU_CORE(obj);
+    Error *local_err = NULL;
+    int64_t value;
+
+    visit_type_int(v, name, &value, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    core->nr_threads = value;
+}
+
+static void cpu_core_instance_init(Object *obj)
+{
+    CPUCore *core = CPU_CORE(obj);
+
+    object_property_add(obj, "core-id", "int", core_prop_get_core_id,
+                        core_prop_set_core_id, NULL, NULL, NULL);
+    object_property_add(obj, "nr-threads", "int", core_prop_get_nr_threads,
+                        core_prop_set_nr_threads, NULL, NULL, NULL);
+    core->nr_threads = smp_threads;
+}
+
+static const TypeInfo cpu_core_type_info = {
+    .name = TYPE_CPU_CORE,
+    .parent = TYPE_DEVICE,
+    .abstract = true,
+    .instance_size = sizeof(CPUCore),
+    .instance_init = cpu_core_instance_init,
+};
+
+static void cpu_core_register_types(void)
+{
+    type_register_static(&cpu_core_type_info);
+}
+
+type_init(cpu_core_register_types)
diff --git a/hw/intc/xics.c b/hw/intc/xics.c
index 8659be0171..cce7f3d112 100644
--- a/hw/intc/xics.c
+++ b/hw/intc/xics.c
@@ -48,6 +48,18 @@ static int get_cpu_index_by_dt_id(int cpu_dt_id)
     return -1;
 }
 
+void xics_cpu_destroy(XICSState *icp, PowerPCCPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    ICPState *ss = &icp->ss[cs->cpu_index];
+
+    assert(cs->cpu_index < icp->nr_servers);
+    assert(cs == ss->cs);
+
+    ss->output = NULL;
+    ss->cs = NULL;
+}
+
 void xics_cpu_setup(XICSState *icp, PowerPCCPU *cpu)
 {
     CPUState *cs = CPU(cpu);
@@ -57,6 +69,8 @@ void xics_cpu_setup(XICSState *icp, PowerPCCPU *cpu)
 
     assert(cs->cpu_index < icp->nr_servers);
 
+    ss->cs = cs;
+
     if (info->cpu_setup) {
         info->cpu_setup(icp, cpu);
     }
diff --git a/hw/intc/xics_kvm.c b/hw/intc/xics_kvm.c
index 55fd801ffd..b17d6a9f43 100644
--- a/hw/intc/xics_kvm.c
+++ b/hw/intc/xics_kvm.c
@@ -114,8 +114,10 @@ static void icp_kvm_reset(DeviceState *dev)
     icp->pending_priority = 0xff;
     icp->mfrr = 0xff;
 
-    /* Make all outputs are deasserted */
-    qemu_set_irq(icp->output, 0);
+    /* Make all outputs as deasserted only if the CPU thread is in use */
+    if (icp->output) {
+        qemu_set_irq(icp->output, 0);
+    }
 
     icp_set_kvm_state(icp, 1);
 }
@@ -348,8 +350,6 @@ static void xics_kvm_cpu_setup(XICSState *icp, PowerPCCPU *cpu)
     if (icpkvm->kernel_xics_fd != -1) {
         int ret;
 
-        ss->cs = cs;
-
         ret = kvm_vcpu_enable_cap(cs, KVM_CAP_IRQ_XICS, 0,
                                   icpkvm->kernel_xics_fd, kvm_arch_vcpu_id(cs));
         if (ret < 0) {
diff --git a/hw/ppc/Makefile.objs b/hw/ppc/Makefile.objs
index c1ffc7771b..5cc6608e50 100644
--- a/hw/ppc/Makefile.objs
+++ b/hw/ppc/Makefile.objs
@@ -4,6 +4,7 @@ obj-y += ppc.o ppc_booke.o
 obj-$(CONFIG_PSERIES) += spapr.o spapr_vio.o spapr_events.o
 obj-$(CONFIG_PSERIES) += spapr_hcall.o spapr_iommu.o spapr_rtas.o
 obj-$(CONFIG_PSERIES) += spapr_pci.o spapr_rtc.o spapr_drc.o spapr_rng.o
+obj-$(CONFIG_PSERIES) += spapr_cpu_core.o
 ifeq ($(CONFIG_PCI)$(CONFIG_PSERIES)$(CONFIG_LINUX), yyy)
 obj-y += spapr_pci_vfio.o
 endif
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 9a4a803b17..778fa255a9 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -66,6 +66,8 @@
 
 #include "hw/compat.h"
 #include "qemu/cutils.h"
+#include "hw/ppc/spapr_cpu_core.h"
+#include "qmp-commands.h"
 
 #include <libfdt.h>
 
@@ -89,8 +91,6 @@
 
 #define MIN_RMA_SLOF            128UL
 
-#define TIMEBASE_FREQ           512000000ULL
-
 #define PHANDLE_XICP            0x00001111
 
 #define HTAB_SIZE(spapr)        (1ULL << ((spapr)->htab_shift))
@@ -599,12 +599,23 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
     int index = ppc_get_vcpu_dt_id(cpu);
     uint32_t segs[] = {cpu_to_be32(28), cpu_to_be32(40),
                        0xffffffff, 0xffffffff};
-    uint32_t tbfreq = kvm_enabled() ? kvmppc_get_tbfreq() : TIMEBASE_FREQ;
+    uint32_t tbfreq = kvm_enabled() ? kvmppc_get_tbfreq()
+        : SPAPR_TIMEBASE_FREQ;
     uint32_t cpufreq = kvm_enabled() ? kvmppc_get_clockfreq() : 1000000000;
     uint32_t page_sizes_prop[64];
     size_t page_sizes_prop_size;
     uint32_t vcpus_per_socket = smp_threads * smp_cores;
     uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    int drc_index;
+
+    drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index);
+    if (drc) {
+        drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+        drc_index = drck->get_index(drc);
+        _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index)));
+    }
 
     /* Note: we keep CI large pages off for now because a 64K capable guest
      * provisioned with large pages might otherwise try to map a qemu
@@ -1005,6 +1016,16 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr,
         _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB));
     }
 
+    if (smc->dr_cpu_enabled) {
+        int offset = fdt_path_offset(fdt, "/cpus");
+        ret = spapr_drc_populate_dt(fdt, offset, NULL,
+                                    SPAPR_DR_CONNECTOR_TYPE_CPU);
+        if (ret < 0) {
+            error_report("Couldn't set up CPU DR device tree properties");
+            exit(1);
+        }
+    }
+
     _FDT((fdt_pack(fdt)));
 
     if (fdt_totalsize(fdt) > FDT_MAX_SIZE) {
@@ -1198,26 +1219,6 @@ static void ppc_spapr_reset(void)
 
 }
 
-static void spapr_cpu_reset(void *opaque)
-{
-    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
-    PowerPCCPU *cpu = opaque;
-    CPUState *cs = CPU(cpu);
-    CPUPPCState *env = &cpu->env;
-
-    cpu_reset(cs);
-
-    /* All CPUs start halted.  CPU0 is unhalted from the machine level
-     * reset code and the rest are explicitly started up by the guest
-     * using an RTAS call */
-    cs->halted = 1;
-
-    env->spr[SPR_HIOR] = 0;
-
-    ppc_hash64_set_external_hpt(cpu, spapr->htab, spapr->htab_shift,
-                                &error_fatal);
-}
-
 static void spapr_create_nvram(sPAPRMachineState *spapr)
 {
     DeviceState *dev = qdev_create(&spapr->vio_bus->bus, "spapr-nvram");
@@ -1623,32 +1624,6 @@ static void spapr_boot_set(void *opaque, const char *boot_device,
     machine->boot_order = g_strdup(boot_device);
 }
 
-static void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu,
-                           Error **errp)
-{
-    CPUPPCState *env = &cpu->env;
-
-    /* Set time-base frequency to 512 MHz */
-    cpu_ppc_tb_init(env, TIMEBASE_FREQ);
-
-    /* Enable PAPR mode in TCG or KVM */
-    cpu_ppc_set_papr(cpu);
-
-    if (cpu->max_compat) {
-        Error *local_err = NULL;
-
-        ppc_set_compat(cpu, cpu->max_compat, &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err);
-            return;
-        }
-    }
-
-    xics_cpu_setup(spapr->icp, cpu);
-
-    qemu_register_reset(spapr_cpu_reset, cpu);
-}
-
 /*
  * Reset routine for LMB DR devices.
  *
@@ -1730,7 +1705,6 @@ static void ppc_spapr_init(MachineState *machine)
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
-    PowerPCCPU *cpu;
     PCIHostState *phb;
     int i;
     MemoryRegion *sysmem = get_system_memory();
@@ -1744,6 +1718,22 @@ static void ppc_spapr_init(MachineState *machine)
     long load_limit, fw_size;
     bool kernel_le = false;
     char *filename;
+    int smt = kvmppc_smt_threads();
+    int spapr_cores = smp_cpus / smp_threads;
+    int spapr_max_cores = max_cpus / smp_threads;
+
+    if (smc->dr_cpu_enabled) {
+        if (smp_cpus % smp_threads) {
+            error_report("smp_cpus (%u) must be multiple of threads (%u)",
+                         smp_cpus, smp_threads);
+            exit(1);
+        }
+        if (max_cpus % smp_threads) {
+            error_report("max_cpus (%u) must be multiple of threads (%u)",
+                         max_cpus, smp_threads);
+            exit(1);
+        }
+    }
 
     msi_nonbroken = true;
 
@@ -1790,8 +1780,7 @@ static void ppc_spapr_init(MachineState *machine)
 
     /* Set up Interrupt Controller before we create the VCPUs */
     spapr->icp = xics_system_init(machine,
-                                  DIV_ROUND_UP(max_cpus * kvmppc_smt_threads(),
-                                               smp_threads),
+                                  DIV_ROUND_UP(max_cpus * smt, smp_threads),
                                   XICS_IRQS, &error_fatal);
 
     if (smc->dr_lmb_enabled) {
@@ -1802,13 +1791,46 @@ static void ppc_spapr_init(MachineState *machine)
     if (machine->cpu_model == NULL) {
         machine->cpu_model = kvm_enabled() ? "host" : "POWER7";
     }
-    for (i = 0; i < smp_cpus; i++) {
-        cpu = cpu_ppc_init(machine->cpu_model);
-        if (cpu == NULL) {
-            error_report("Unable to find PowerPC CPU definition");
-            exit(1);
+
+    if (smc->dr_cpu_enabled) {
+        char *type = spapr_get_cpu_core_type(machine->cpu_model);
+
+        spapr->cores = g_new0(Object *, spapr_max_cores);
+        for (i = 0; i < spapr_max_cores; i++) {
+            int core_dt_id = i * smt;
+            sPAPRDRConnector *drc =
+                spapr_dr_connector_new(OBJECT(spapr),
+                                       SPAPR_DR_CONNECTOR_TYPE_CPU, core_dt_id);
+
+            qemu_register_reset(spapr_drc_reset, drc);
+
+            if (i < spapr_cores) {
+                char *type = spapr_get_cpu_core_type(machine->cpu_model);
+                Object *core;
+
+                if (!object_class_by_name(type)) {
+                    error_report("Unable to find sPAPR CPU Core definition");
+                    exit(1);
+                }
+
+                core  = object_new(type);
+                object_property_set_int(core, smp_threads, "nr-threads",
+                                        &error_fatal);
+                object_property_set_int(core, core_dt_id, CPU_CORE_PROP_CORE_ID,
+                                        &error_fatal);
+                object_property_set_bool(core, true, "realized", &error_fatal);
+            }
         }
-        spapr_cpu_init(spapr, cpu, &error_fatal);
+        g_free(type);
+    } else {
+        for (i = 0; i < smp_cpus; i++) {
+            PowerPCCPU *cpu = cpu_ppc_init(machine->cpu_model);
+            if (cpu == NULL) {
+                error_report("Unable to find PowerPC CPU definition");
+                exit(1);
+            }
+            spapr_cpu_init(spapr, cpu, &error_fatal);
+       }
     }
 
     if (kvm_enabled()) {
@@ -2219,6 +2241,27 @@ out:
     error_propagate(errp, local_err);
 }
 
+void *spapr_populate_hotplug_cpu_dt(CPUState *cs, int *fdt_offset,
+                                    sPAPRMachineState *spapr)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    DeviceClass *dc = DEVICE_GET_CLASS(cs);
+    int id = ppc_get_vcpu_dt_id(cpu);
+    void *fdt;
+    int offset, fdt_size;
+    char *nodename;
+
+    fdt = create_device_tree(&fdt_size);
+    nodename = g_strdup_printf("%s@%x", dc->fw_name, id);
+    offset = fdt_add_subnode(fdt, 0, nodename);
+
+    spapr_populate_cpu_dt(cs, fdt, offset, spapr);
+    g_free(nodename);
+
+    *fdt_offset = offset;
+    return fdt;
+}
+
 static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
                                       DeviceState *dev, Error **errp)
 {
@@ -2263,21 +2306,40 @@ static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
         }
 
         spapr_memory_plug(hotplug_dev, dev, node, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        spapr_core_plug(hotplug_dev, dev, errp);
     }
 }
 
 static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
                                       DeviceState *dev, Error **errp)
 {
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
+
     if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
         error_setg(errp, "Memory hot unplug not supported by sPAPR");
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        if (!smc->dr_cpu_enabled) {
+            error_setg(errp, "CPU hot unplug not supported on this machine");
+            return;
+        }
+        spapr_core_unplug(hotplug_dev, dev, errp);
+    }
+}
+
+static void spapr_machine_device_pre_plug(HotplugHandler *hotplug_dev,
+                                          DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        spapr_core_pre_plug(hotplug_dev, dev, errp);
     }
 }
 
 static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine,
                                              DeviceState *dev)
 {
-    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) ||
+        object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
         return HOTPLUG_HANDLER(machine);
     }
     return NULL;
@@ -2290,6 +2352,38 @@ static unsigned spapr_cpu_index_to_socket_id(unsigned cpu_index)
     return cpu_index / smp_threads / smp_cores;
 }
 
+static HotpluggableCPUList *spapr_query_hotpluggable_cpus(MachineState *machine)
+{
+    int i;
+    HotpluggableCPUList *head = NULL;
+    sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
+    int spapr_max_cores = max_cpus / smp_threads;
+    int smt = kvmppc_smt_threads();
+
+    for (i = 0; i < spapr_max_cores; i++) {
+        HotpluggableCPUList *list_item = g_new0(typeof(*list_item), 1);
+        HotpluggableCPU *cpu_item = g_new0(typeof(*cpu_item), 1);
+        CpuInstanceProperties *cpu_props = g_new0(typeof(*cpu_props), 1);
+
+        cpu_item->type = spapr_get_cpu_core_type(machine->cpu_model);
+        cpu_item->vcpus_count = smp_threads;
+        cpu_props->has_core = true;
+        cpu_props->core = i * smt;
+        /* TODO: add 'has_node/node' here to describe
+           to which node core belongs */
+
+        cpu_item->props = cpu_props;
+        if (spapr->cores[i]) {
+            cpu_item->has_qom_path = true;
+            cpu_item->qom_path = object_get_canonical_path(spapr->cores[i]);
+        }
+        list_item->value = cpu_item;
+        list_item->next = head;
+        head = list_item;
+    }
+    return head;
+}
+
 static void spapr_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
@@ -2316,11 +2410,14 @@ static void spapr_machine_class_init(ObjectClass *oc, void *data)
     mc->has_dynamic_sysbus = true;
     mc->pci_allow_0_address = true;
     mc->get_hotplug_handler = spapr_get_hotpug_handler;
+    hc->pre_plug = spapr_machine_device_pre_plug;
     hc->plug = spapr_machine_device_plug;
     hc->unplug = spapr_machine_device_unplug;
     mc->cpu_index_to_socket_id = spapr_cpu_index_to_socket_id;
+    mc->query_hotpluggable_cpus = spapr_query_hotpluggable_cpus;
 
     smc->dr_lmb_enabled = true;
+    smc->dr_cpu_enabled = true;
     fwc->get_dev_path = spapr_get_fw_dev_path;
     nc->nmi_monitor_handler = spapr_nmi;
 }
@@ -2396,7 +2493,10 @@ static void spapr_machine_2_6_instance_options(MachineState *machine)
 
 static void spapr_machine_2_6_class_options(MachineClass *mc)
 {
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
     spapr_machine_2_7_class_options(mc);
+    smc->dr_cpu_enabled = false;
     SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_6);
 }
 
diff --git a/hw/ppc/spapr_cpu_core.c b/hw/ppc/spapr_cpu_core.c
new file mode 100644
index 0000000000..3a5da09b99
--- /dev/null
+++ b/hw/ppc/spapr_cpu_core.c
@@ -0,0 +1,405 @@
+/*
+ * sPAPR CPU core device, acts as container of CPU thread devices.
+ *
+ * Copyright (C) 2016 Bharata B Rao <bharata@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "hw/cpu/core.h"
+#include "hw/ppc/spapr_cpu_core.h"
+#include "target-ppc/cpu.h"
+#include "hw/ppc/spapr.h"
+#include "hw/boards.h"
+#include "qapi/error.h"
+#include <sysemu/cpus.h>
+#include "target-ppc/kvm_ppc.h"
+#include "hw/ppc/ppc.h"
+#include "target-ppc/mmu-hash64.h"
+#include <sysemu/numa.h>
+
+static void spapr_cpu_reset(void *opaque)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    PowerPCCPU *cpu = opaque;
+    CPUState *cs = CPU(cpu);
+    CPUPPCState *env = &cpu->env;
+
+    cpu_reset(cs);
+
+    /* All CPUs start halted.  CPU0 is unhalted from the machine level
+     * reset code and the rest are explicitly started up by the guest
+     * using an RTAS call */
+    cs->halted = 1;
+
+    env->spr[SPR_HIOR] = 0;
+
+    ppc_hash64_set_external_hpt(cpu, spapr->htab, spapr->htab_shift,
+                                &error_fatal);
+}
+
+static void spapr_cpu_destroy(PowerPCCPU *cpu)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+
+    xics_cpu_destroy(spapr->icp, cpu);
+    qemu_unregister_reset(spapr_cpu_reset, cpu);
+}
+
+void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **errp)
+{
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+    int i;
+
+    /* Set time-base frequency to 512 MHz */
+    cpu_ppc_tb_init(env, SPAPR_TIMEBASE_FREQ);
+
+    /* Enable PAPR mode in TCG or KVM */
+    cpu_ppc_set_papr(cpu);
+
+    if (cpu->max_compat) {
+        Error *local_err = NULL;
+
+        ppc_set_compat(cpu, cpu->max_compat, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
+    /* Set NUMA node for the added CPUs  */
+    for (i = 0; i < nb_numa_nodes; i++) {
+        if (test_bit(cs->cpu_index, numa_info[i].node_cpu)) {
+            cs->numa_node = i;
+            break;
+        }
+    }
+
+    xics_cpu_setup(spapr->icp, cpu);
+
+    qemu_register_reset(spapr_cpu_reset, cpu);
+    spapr_cpu_reset(cpu);
+}
+
+/*
+ * Return the sPAPR CPU core type for @model which essentially is the CPU
+ * model specified with -cpu cmdline option.
+ */
+char *spapr_get_cpu_core_type(const char *model)
+{
+    char *core_type;
+    gchar **model_pieces = g_strsplit(model, ",", 2);
+
+    core_type = g_strdup_printf("%s-%s", model_pieces[0], TYPE_SPAPR_CPU_CORE);
+    g_strfreev(model_pieces);
+    return core_type;
+}
+
+static void spapr_core_release(DeviceState *dev, void *opaque)
+{
+    sPAPRCPUCore *sc = SPAPR_CPU_CORE(OBJECT(dev));
+    const char *typename = object_class_get_name(sc->cpu_class);
+    size_t size = object_type_get_instance_size(typename);
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
+    CPUCore *cc = CPU_CORE(dev);
+    int smt = kvmppc_smt_threads();
+    int i;
+
+    for (i = 0; i < cc->nr_threads; i++) {
+        void *obj = sc->threads + i * size;
+        DeviceState *dev = DEVICE(obj);
+        CPUState *cs = CPU(dev);
+        PowerPCCPU *cpu = POWERPC_CPU(cs);
+
+        spapr_cpu_destroy(cpu);
+        cpu_remove_sync(cs);
+        object_unparent(obj);
+    }
+
+    spapr->cores[cc->core_id / smt] = NULL;
+
+    g_free(core->threads);
+    object_unparent(OBJECT(dev));
+}
+
+void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                       Error **errp)
+{
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
+    PowerPCCPU *cpu = POWERPC_CPU(core->threads);
+    int id = ppc_get_vcpu_dt_id(cpu);
+    sPAPRDRConnector *drc =
+        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, id);
+    sPAPRDRConnectorClass *drck;
+    Error *local_err = NULL;
+
+    g_assert(drc);
+
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    drck->detach(drc, dev, spapr_core_release, NULL, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    spapr_hotplug_req_remove_by_index(drc);
+}
+
+void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                     Error **errp)
+{
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(OBJECT(hotplug_dev));
+    sPAPRMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
+    CPUCore *cc = CPU_CORE(dev);
+    CPUState *cs = CPU(core->threads);
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    Error *local_err = NULL;
+    void *fdt = NULL;
+    int fdt_offset = 0;
+    int index;
+    int smt = kvmppc_smt_threads();
+
+    drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, cc->core_id);
+    index = cc->core_id / smt;
+    spapr->cores[index] = OBJECT(dev);
+
+    if (!smc->dr_cpu_enabled) {
+        /*
+         * This is a cold plugged CPU core but the machine doesn't support
+         * DR. So skip the hotplug path ensuring that the core is brought
+         * up online with out an associated DR connector.
+         */
+        return;
+    }
+
+    g_assert(drc);
+
+    /*
+     * Setup CPU DT entries only for hotplugged CPUs. For boot time or
+     * coldplugged CPUs DT entries are setup in spapr_finalize_fdt().
+     */
+    if (dev->hotplugged) {
+        fdt = spapr_populate_hotplug_cpu_dt(cs, &fdt_offset, spapr);
+    }
+
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, &local_err);
+    if (local_err) {
+        g_free(fdt);
+        spapr->cores[index] = NULL;
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (dev->hotplugged) {
+        /*
+         * Send hotplug notification interrupt to the guest only in case
+         * of hotplugged CPUs.
+         */
+        spapr_hotplug_req_add_by_index(drc);
+    } else {
+        /*
+         * Set the right DRC states for cold plugged CPU.
+         */
+        drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_USABLE);
+        drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_UNISOLATED);
+    }
+}
+
+void spapr_core_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                         Error **errp)
+{
+    MachineState *machine = MACHINE(OBJECT(hotplug_dev));
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(OBJECT(hotplug_dev));
+    sPAPRMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
+    int spapr_max_cores = max_cpus / smp_threads;
+    int index;
+    int smt = kvmppc_smt_threads();
+    Error *local_err = NULL;
+    CPUCore *cc = CPU_CORE(dev);
+    char *base_core_type = spapr_get_cpu_core_type(machine->cpu_model);
+    const char *type = object_get_typename(OBJECT(dev));
+
+    if (strcmp(base_core_type, type)) {
+        error_setg(&local_err, "CPU core type should be %s", base_core_type);
+        goto out;
+    }
+
+    if (!smc->dr_cpu_enabled && dev->hotplugged) {
+        error_setg(&local_err, "CPU hotplug not supported for this machine");
+        goto out;
+    }
+
+    if (cc->nr_threads != smp_threads) {
+        error_setg(&local_err, "threads must be %d", smp_threads);
+        goto out;
+    }
+
+    if (cc->core_id % smt) {
+        error_setg(&local_err, "invalid core id %d\n", cc->core_id);
+        goto out;
+    }
+
+    index = cc->core_id / smt;
+    if (index < 0 || index >= spapr_max_cores) {
+        error_setg(&local_err, "core id %d out of range", cc->core_id);
+        goto out;
+    }
+
+    if (spapr->cores[index]) {
+        error_setg(&local_err, "core %d already populated", cc->core_id);
+        goto out;
+    }
+
+out:
+    g_free(base_core_type);
+    error_propagate(errp, local_err);
+}
+
+static int spapr_cpu_core_realize_child(Object *child, void *opaque)
+{
+    Error **errp = opaque;
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    CPUState *cs = CPU(child);
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+
+    object_property_set_bool(child, true, "realized", errp);
+    if (*errp) {
+        return 1;
+    }
+
+    spapr_cpu_init(spapr, cpu, errp);
+    if (*errp) {
+        return 1;
+    }
+    return 0;
+}
+
+static void spapr_cpu_core_realize(DeviceState *dev, Error **errp)
+{
+    sPAPRCPUCore *sc = SPAPR_CPU_CORE(OBJECT(dev));
+    CPUCore *cc = CPU_CORE(OBJECT(dev));
+    const char *typename = object_class_get_name(sc->cpu_class);
+    size_t size = object_type_get_instance_size(typename);
+    Error *local_err = NULL;
+    Object *obj;
+    int i;
+
+    sc->threads = g_malloc0(size * cc->nr_threads);
+    for (i = 0; i < cc->nr_threads; i++) {
+        char id[32];
+        void *obj = sc->threads + i * size;
+
+        object_initialize(obj, size, typename);
+        snprintf(id, sizeof(id), "thread[%d]", i);
+        object_property_add_child(OBJECT(sc), id, obj, &local_err);
+        if (local_err) {
+            goto err;
+        }
+    }
+    object_child_foreach(OBJECT(dev), spapr_cpu_core_realize_child, &local_err);
+    if (local_err) {
+        goto err;
+    } else {
+        return;
+    }
+
+err:
+    while (i >= 0) {
+        obj = sc->threads + i * size;
+        object_unparent(obj);
+        i--;
+    }
+    g_free(sc->threads);
+    error_propagate(errp, local_err);
+}
+
+static void spapr_cpu_core_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    dc->realize = spapr_cpu_core_realize;
+}
+
+/*
+ * instance_init routines from different flavours of sPAPR CPU cores.
+ * TODO: Add support for 'host' core type.
+ */
+#define SPAPR_CPU_CORE_INITFN(_type, _fname) \
+static void glue(glue(spapr_cpu_core_, _fname), _initfn(Object *obj)) \
+{ \
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(obj); \
+    char *name = g_strdup_printf("%s-" TYPE_POWERPC_CPU, stringify(_type)); \
+    ObjectClass *oc = object_class_by_name(name); \
+    g_assert(oc); \
+    g_free((void *)name); \
+    core->cpu_class = oc; \
+}
+
+SPAPR_CPU_CORE_INITFN(POWER7_v2.3, POWER7);
+SPAPR_CPU_CORE_INITFN(POWER7+_v2.1, POWER7plus);
+SPAPR_CPU_CORE_INITFN(POWER8_v2.0, POWER8);
+SPAPR_CPU_CORE_INITFN(POWER8E_v2.1, POWER8E);
+
+typedef struct SPAPRCoreInfo {
+    const char *name;
+    void (*initfn)(Object *obj);
+} SPAPRCoreInfo;
+
+static const SPAPRCoreInfo spapr_cores[] = {
+    /* POWER7 and aliases */
+    { .name = "POWER7_v2.3", .initfn = spapr_cpu_core_POWER7_initfn },
+    { .name = "POWER7", .initfn = spapr_cpu_core_POWER7_initfn },
+
+    /* POWER7+ and aliases */
+    { .name = "POWER7+_v2.1", .initfn = spapr_cpu_core_POWER7plus_initfn },
+    { .name = "POWER7+", .initfn = spapr_cpu_core_POWER7plus_initfn },
+
+    /* POWER8 and aliases */
+    { .name = "POWER8_v2.0", .initfn = spapr_cpu_core_POWER8_initfn },
+    { .name = "POWER8", .initfn = spapr_cpu_core_POWER8_initfn },
+    { .name = "power8", .initfn = spapr_cpu_core_POWER8_initfn },
+
+    /* POWER8E and aliases */
+    { .name = "POWER8E_v2.1", .initfn = spapr_cpu_core_POWER8E_initfn },
+    { .name = "POWER8E", .initfn = spapr_cpu_core_POWER8E_initfn },
+
+    { .name = NULL }
+};
+
+static void spapr_cpu_core_register(const SPAPRCoreInfo *info)
+{
+    TypeInfo type_info = {
+        .parent = TYPE_SPAPR_CPU_CORE,
+        .instance_size = sizeof(sPAPRCPUCore),
+        .instance_init = info->initfn,
+    };
+
+    type_info.name = g_strdup_printf("%s-" TYPE_SPAPR_CPU_CORE, info->name);
+    type_register(&type_info);
+    g_free((void *)type_info.name);
+}
+
+static const TypeInfo spapr_cpu_core_type_info = {
+    .name = TYPE_SPAPR_CPU_CORE,
+    .parent = TYPE_CPU_CORE,
+    .abstract = true,
+    .instance_size = sizeof(sPAPRCPUCore),
+    .class_init = spapr_cpu_core_class_init,
+};
+
+static void spapr_cpu_core_register_types(void)
+{
+    const SPAPRCoreInfo *info = spapr_cores;
+
+    type_register_static(&spapr_cpu_core_type_info);
+    while (info->name) {
+        spapr_cpu_core_register(info);
+        info++;
+    }
+}
+
+type_init(spapr_cpu_core_register_types)
diff --git a/hw/ppc/spapr_drc.c b/hw/ppc/spapr_drc.c
index 94c875d752..d276db3a72 100644
--- a/hw/ppc/spapr_drc.c
+++ b/hw/ppc/spapr_drc.c
@@ -140,6 +140,8 @@ static uint32_t set_allocation_state(sPAPRDRConnector *drc,
             DPRINTFN("finalizing device removal");
             drck->detach(drc, DEVICE(drc->dev), drc->detach_cb,
                          drc->detach_cb_opaque, NULL);
+        } else if (drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_USABLE) {
+            drc->awaiting_allocation = false;
         }
     }
     return RTAS_OUT_SUCCESS;
@@ -373,6 +375,10 @@ static void attach(sPAPRDRConnector *drc, DeviceState *d, void *fdt,
     drc->signalled = (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI)
                      ? true : coldplug;
 
+    if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI) {
+        drc->awaiting_allocation = true;
+    }
+
     object_property_add_link(OBJECT(drc), "device",
                              object_get_typename(OBJECT(drc->dev)),
                              (Object **)(&drc->dev),
@@ -421,6 +427,12 @@ static void detach(sPAPRDRConnector *drc, DeviceState *d,
         return;
     }
 
+    if (drc->awaiting_allocation) {
+        drc->awaiting_release = true;
+        DPRINTFN("awaiting allocation to complete before removal");
+        return;
+    }
+
     drc->indicator_state = SPAPR_DR_INDICATOR_STATE_INACTIVE;
 
     if (drc->detach_cb) {
diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c
index 049fb1b325..af8099220e 100644
--- a/hw/ppc/spapr_events.c
+++ b/hw/ppc/spapr_events.c
@@ -449,6 +449,9 @@ static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action,
     case SPAPR_DR_CONNECTOR_TYPE_LMB:
         hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_MEMORY;
         break;
+    case SPAPR_DR_CONNECTOR_TYPE_CPU:
+        hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_CPU;
+        break;
     default:
         /* we shouldn't be signaling hotplug events for resources
          * that don't support them
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index cbd3d23c91..f3cb141763 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -27,6 +27,7 @@
 #include "linux/vfio.h"
 #include "hw/vfio/vfio.h"
 #include "qemu/error-report.h"
+#include "sysemu/qtest.h"
 
 #define TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE "spapr-pci-vfio-host-bridge"
 
@@ -48,7 +49,9 @@ static Property spapr_phb_vfio_properties[] = {
 
 static void spapr_phb_vfio_instance_init(Object *obj)
 {
-    error_report("spapr-pci-vfio-host-bridge is deprecated");
+    if (!qtest_enabled()) {
+        error_report("spapr-pci-vfio-host-bridge is deprecated");
+    }
 }
 
 bool spapr_phb_eeh_available(sPAPRPHBState *sphb)
diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c
index 43e2c684fd..dc058e512b 100644
--- a/hw/ppc/spapr_rtas.c
+++ b/hw/ppc/spapr_rtas.c
@@ -36,6 +36,7 @@
 
 #include "hw/ppc/spapr.h"
 #include "hw/ppc/spapr_vio.h"
+#include "hw/ppc/ppc.h"
 #include "qapi-event.h"
 #include "hw/boards.h"
 
@@ -164,6 +165,27 @@ static void rtas_query_cpu_stopped_state(PowerPCCPU *cpu_,
     rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
 }
 
+/*
+ * Set the timebase offset of the CPU to that of first CPU.
+ * This helps hotplugged CPU to have the correct timebase offset.
+ */
+static void spapr_cpu_update_tb_offset(PowerPCCPU *cpu)
+{
+    PowerPCCPU *fcpu = POWERPC_CPU(first_cpu);
+
+    cpu->env.tb_env->tb_offset = fcpu->env.tb_env->tb_offset;
+}
+
+static void spapr_cpu_set_endianness(PowerPCCPU *cpu)
+{
+    PowerPCCPU *fcpu = POWERPC_CPU(first_cpu);
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(fcpu);
+
+    if (!pcc->interrupts_big_endian(fcpu)) {
+        cpu->env.spr[SPR_LPCR] |= LPCR_ILE;
+    }
+}
+
 static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMachineState *spapr,
                            uint32_t token, uint32_t nargs,
                            target_ulong args,
@@ -200,6 +222,8 @@ static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMachineState *spapr,
         env->nip = start;
         env->gpr[3] = r3;
         cs->halted = 0;
+        spapr_cpu_set_endianness(cpu);
+        spapr_cpu_update_tb_offset(cpu);
 
         qemu_cpu_kick(cs);
 
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 5ff5e9220a..1898f1f3e4 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -1257,7 +1257,7 @@ static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op)
         return -errno;
     }
 
-    return 0;
+    return ret;
 }
 
 static VFIOContainer *vfio_eeh_as_container(AddressSpace *as)