aboutsummaryrefslogtreecommitdiff
path: root/system/xen/xsa
diff options
context:
space:
mode:
authorMario Preksavec <mario@slackware.hr>2018-12-28 23:01:43 +0100
committerWilly Sudiarto Raharjo <willysr@slackbuilds.org>2018-12-29 07:03:42 +0700
commit7f4be9632b242e2cbb64abc70620ab1d38609225 (patch)
tree41d684ea2f2eb3a8441de46fabec14e22adc17c4 /system/xen/xsa
parentccd7927cfe455cf73f00a1a90577afd74d159774 (diff)
system/xen: Updated for version 4.11.1.
Signed-off-by: Mario Preksavec <mario@slackware.hr>
Diffstat (limited to 'system/xen/xsa')
-rw-r--r--system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch4115
1 files changed, 0 insertions, 4115 deletions
diff --git a/system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch b/system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch
deleted file mode 100644
index ab9794df5e883..0000000000000
--- a/system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch
+++ /dev/null
@@ -1,4115 +0,0 @@
-diff --git a/docs/man/xl.conf.pod.5 b/docs/man/xl.conf.pod.5
-index da91b8626c..37262a7ef8 100644
---- a/docs/man/xl.conf.pod.5
-+++ b/docs/man/xl.conf.pod.5
-@@ -185,6 +185,28 @@ massively huge guests).
-
- =back
-
-+=item B<vm.cpumask>="CPULIST"
-+
-+=item B<vm.hvm.cpumask>="CPULIST"
-+
-+=item B<vm.pv.cpumask>="CPULIST"
-+
-+Global masks that are applied when creating guests and pinning vcpus
-+to indicate which cpus they are allowed to run on. Specifically,
-+C<vm.cpumask> applies to all guest types, C<vm.hvm.cpumask> applies to
-+both HVM and PVH guests and C<vm.pv.cpumask> applies to PV guests.
-+
-+The hard affinity of guest's vcpus are logical-AND'ed with respective
-+masks. If the resulting affinity mask is empty, operation will fail.
-+
-+Use --ignore-global-affinity-masks to skip applying global masks.
-+
-+The default value for these masks are all 1's, i.e. all cpus are allowed.
-+
-+Due to bug(s), these options may not interact well with other options
-+concerning CPU affinity. One example is CPU pools. Users should always double
-+check that the required affinity has taken effect.
-+
- =back
-
- =head1 SEE ALSO
-diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown
-index 075e5ea159..0886706368 100644
---- a/docs/misc/xen-command-line.markdown
-+++ b/docs/misc/xen-command-line.markdown
-@@ -489,10 +489,10 @@ accounting for hardware capabilities as enumerated via CPUID.
-
- Currently accepted:
-
--The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`, `ssbd` are
--used by default if available and applicable. They can be ignored,
--e.g. `no-ibrsb`, at which point Xen won't use them itself, and won't offer
--them to guests.
-+The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`,
-+`l1d-flush` and `ssbd` are used by default if available and applicable. They can
-+be ignored, e.g. `no-ibrsb`, at which point Xen won't use them itself, and
-+won't offer them to guests.
-
- ### cpuid\_mask\_cpu (AMD only)
- > `= fam_0f_rev_c | fam_0f_rev_d | fam_0f_rev_e | fam_0f_rev_f | fam_0f_rev_g | fam_10_rev_b | fam_10_rev_c | fam_11_rev_b`
-@@ -936,6 +936,8 @@ version are 1 and 2.
- use of grant table v2 without transitive grants is an ABI breakage from the
- guests point of view.
-
-+The usage of gnttab v2 is not security supported on ARM platforms.
-+
- ### gnttab\_max\_frames
- > `= <integer>`
-
-@@ -1544,6 +1546,30 @@ do; there may be other custom operating systems which do. If you're
- certain you don't plan on having PV guests which use this feature,
- turning it off can reduce the attack surface.
-
-+### pv-l1tf (x86)
-+> `= List of [ <bool>, dom0=<bool>, domu=<bool> ]`
-+
-+> Default: `false` on believed-unaffected hardware, or in pv-shim mode.
-+> `domu` on believed-affected hardware.
-+
-+Mitigations for L1TF / XSA-273 / CVE-2018-3620 for PV guests.
-+
-+For backwards compatibility, we may not alter an architecturally-legitimate
-+pagetable entry a PV guest chooses to write. We can however force such a
-+guest into shadow mode so that Xen controls the PTEs which are reachable by
-+the CPU pagewalk.
-+
-+Shadowing is performed at the point where a PV guest first tries to write an
-+L1TF-vulnerable PTE. Therefore, a PV guest kernel which has been updated with
-+its own L1TF mitigations will not trigger shadow mode if it is well behaved.
-+
-+If CONFIG\_SHADOW\_PAGING is not compiled in, this mitigation instead crashes
-+the guest when an L1TF-vulnerable PTE is written, which still allows updated,
-+well-behaved PV guests to run, despite Shadow being compiled out.
-+
-+In the pv-shim case, Shadow is expected to be compiled out, and a malicious
-+guest kernel can only leak data from the shim Xen, rather than the host Xen.
-+
- ### pv-shim (x86)
- > `= <boolean>`
-
-@@ -1748,6 +1774,13 @@ Use `smap=hvm` to allow SMAP use by HVM guests only.
- Flag to enable Supervisor Mode Execution Protection
- Use `smep=hvm` to allow SMEP use by HVM guests only.
-
-+### smt (x86)
-+> `= <boolean>`
-+
-+Default: `true`
-+
-+Control bring up of multiple hyper-threads per CPU core.
-+
- ### snb\_igd\_quirk
- > `= <boolean> | cap | <integer>`
-
-@@ -1758,7 +1791,8 @@ false disable the quirk workaround, which is also the default.
-
- ### spec-ctrl (x86)
- > `= List of [ <bool>, xen=<bool>, {pv,hvm,msr-sc,rsb}=<bool>,
--> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu}=<bool> ]`
-+> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu,
-+> l1d-flush}=<bool> ]`
-
- Controls for speculative execution sidechannel mitigations. By default, Xen
- will pick the most appropriate mitigations based on compiled in support,
-@@ -1770,10 +1804,15 @@ extreme care.**
-
- An overall boolean value, `spec-ctrl=no`, can be specified to turn off all
- mitigations, including pieces of infrastructure used to virtualise certain
--mitigation features for guests. Alternatively, a slightly more restricted
--`spec-ctrl=no-xen` can be used to turn off all of Xen's mitigations, while
--leaving the virtualisation support in place for guests to use. Use of a
--positive boolean value for either of these options is invalid.
-+mitigation features for guests. This also includes settings which `xpti`,
-+`smt`, `pv-l1tf` control, unless the respective option(s) have been
-+specified earlier on the command line.
-+
-+Alternatively, a slightly more restricted `spec-ctrl=no-xen` can be used to
-+turn off all of Xen's mitigations, while leaving the virtualisation support
-+in place for guests to use.
-+
-+Use of a positive boolean value for either of these options is invalid.
-
- The booleans `pv=`, `hvm=`, `msr-sc=` and `rsb=` offer fine grained control
- over the alternative blocks used by Xen. These impact Xen's ability to
-@@ -1813,6 +1852,12 @@ from using fully eager FPU context switches. This is currently implemented as
- a global control. By default, Xen will choose to use fully eager context
- switches on hardware believed to speculate past #NM exceptions.
-
-+On hardware supporting L1D_FLUSH, the `l1d-flush=` option can be used to force
-+or prevent Xen from issuing an L1 data cache flush on each VMEntry.
-+Irrespective of Xen's setting, the feature is virtualised for HVM guests to
-+use. By default, Xen will enable this mitigation on hardware believed to be
-+vulnerable to L1TF.
-+
- ### sync\_console
- > `= <boolean>`
-
-diff --git a/tools/examples/xl.conf b/tools/examples/xl.conf
-index 374b6bbc2e..0446deb304 100644
---- a/tools/examples/xl.conf
-+++ b/tools/examples/xl.conf
-@@ -37,3 +37,8 @@
- # (which can take a long time to find out if launching huge guests).
- # see xl.conf(5) for details.
- #claim_mode=1
-+
-+# Specify global vcpu hard affinity masks. See xl.conf(5) for details.
-+#vm.cpumask="0-7"
-+#vm.pv.cpumask="0-3"
-+#vm.hvm.cpumask="3-7"
-diff --git a/tools/libxl/libxl_cpuid.c b/tools/libxl/libxl_cpuid.c
-index 7b0f594c3d..52e16c20ed 100644
---- a/tools/libxl/libxl_cpuid.c
-+++ b/tools/libxl/libxl_cpuid.c
-@@ -204,6 +204,7 @@ int libxl_cpuid_parse_config(libxl_cpuid_policy_list *cpuid, const char* str)
- {"avx512-4fmaps",0x00000007, 0, CPUID_REG_EDX, 3, 1},
- {"ibrsb", 0x00000007, 0, CPUID_REG_EDX, 26, 1},
- {"stibp", 0x00000007, 0, CPUID_REG_EDX, 27, 1},
-+ {"l1d-flush", 0x00000007, 0, CPUID_REG_EDX, 28, 1},
- {"arch-caps", 0x00000007, 0, CPUID_REG_EDX, 29, 1},
- {"ssbd", 0x00000007, 0, CPUID_REG_EDX, 31, 1},
-
-diff --git a/tools/misc/xen-cpuid.c b/tools/misc/xen-cpuid.c
-index e116339733..3888b4e158 100644
---- a/tools/misc/xen-cpuid.c
-+++ b/tools/misc/xen-cpuid.c
-@@ -143,7 +143,7 @@ static const char *str_7d0[32] =
- [ 2] = "avx512_4vnniw", [ 3] = "avx512_4fmaps",
-
- [26] = "ibrsb", [27] = "stibp",
-- /* 28 */ [29] = "arch_caps",
-+ [28] = "l1d_flush", [29] = "arch_caps",
- /* 30 */ [31] = "ssbd",
- };
-
-diff --git a/tools/ocaml/xenstored/store.ml b/tools/ocaml/xenstored/store.ml
-index 13cf3b5bf4..5a8c377603 100644
---- a/tools/ocaml/xenstored/store.ml
-+++ b/tools/ocaml/xenstored/store.ml
-@@ -262,7 +262,8 @@ let path_write store perm path value =
- Node.check_perm store.root perm Perms.WRITE;
- Node.set_value store.root value, false
- ) else
-- Path.apply_modify store.root path do_write, !node_created
-+ let root = Path.apply_modify store.root path do_write in
-+ root, !node_created
-
- let path_rm store perm path =
- let do_rm node name =
-diff --git a/tools/xl/xl.c b/tools/xl/xl.c
-index 179908b4f6..7d2142f16f 100644
---- a/tools/xl/xl.c
-+++ b/tools/xl/xl.c
-@@ -28,6 +28,9 @@
- #include <libxl_utils.h>
- #include <libxlutil.h>
- #include "xl.h"
-+#include "xl_parse.h"
-+
-+#include "xl_utils.h"
-
- xentoollog_logger_stdiostream *logger;
- int dryrun_only;
-@@ -42,6 +45,9 @@ char *default_gatewaydev = NULL;
- char *default_vifbackend = NULL;
- char *default_remus_netbufscript = NULL;
- char *default_colo_proxy_script = NULL;
-+libxl_bitmap global_vm_affinity_mask;
-+libxl_bitmap global_hvm_affinity_mask;
-+libxl_bitmap global_pv_affinity_mask;
- enum output_format default_output_format = OUTPUT_FORMAT_JSON;
- int claim_mode = 1;
- bool progress_use_cr = 0;
-@@ -203,6 +209,26 @@ static void parse_global_config(const char *configfile,
- if (!xlu_cfg_get_long (config, "max_maptrack_frames", &l, 0))
- max_maptrack_frames = l;
-
-+ libxl_bitmap_init(&global_vm_affinity_mask);
-+ libxl_cpu_bitmap_alloc(ctx, &global_vm_affinity_mask, 0);
-+ libxl_bitmap_init(&global_hvm_affinity_mask);
-+ libxl_cpu_bitmap_alloc(ctx, &global_hvm_affinity_mask, 0);
-+ libxl_bitmap_init(&global_pv_affinity_mask);
-+ libxl_cpu_bitmap_alloc(ctx, &global_pv_affinity_mask, 0);
-+
-+ if (!xlu_cfg_get_string (config, "vm.cpumask", &buf, 0))
-+ parse_cpurange(buf, &global_vm_affinity_mask);
-+ else
-+ libxl_bitmap_set_any(&global_vm_affinity_mask);
-+ if (!xlu_cfg_get_string (config, "vm.hvm.cpumask", &buf, 0))
-+ parse_cpurange(buf, &global_hvm_affinity_mask);
-+ else
-+ libxl_bitmap_set_any(&global_hvm_affinity_mask);
-+ if (!xlu_cfg_get_string (config, "vm.pv.cpumask", &buf, 0))
-+ parse_cpurange(buf, &global_pv_affinity_mask);
-+ else
-+ libxl_bitmap_set_any(&global_pv_affinity_mask);
-+
- xlu_cfg_destroy(config);
- }
-
-diff --git a/tools/xl/xl.h b/tools/xl/xl.h
-index 4e784ff402..7e97144b50 100644
---- a/tools/xl/xl.h
-+++ b/tools/xl/xl.h
-@@ -41,6 +41,7 @@ struct domain_create {
- int vncautopass;
- int console_autoconnect;
- int checkpointed_stream;
-+ int ignore_global_affinity_masks;
- const char *config_file;
- char *extra_config; /* extra config string */
- const char *restore_file;
-@@ -279,6 +280,9 @@ extern char *default_colo_proxy_script;
- extern char *blkdev_start;
- extern int max_grant_frames;
- extern int max_maptrack_frames;
-+extern libxl_bitmap global_vm_affinity_mask;
-+extern libxl_bitmap global_hvm_affinity_mask;
-+extern libxl_bitmap global_pv_affinity_mask;
-
- enum output_format {
- OUTPUT_FORMAT_JSON,
-@@ -294,6 +298,9 @@ typedef enum {
- } domain_restart_type;
-
- extern void printf_info_sexp(int domid, libxl_domain_config *d_config, FILE *fh);
-+extern void apply_global_affinity_masks(libxl_domain_type type,
-+ libxl_bitmap *vcpu_affinity_array,
-+ unsigned int size);
-
- #define XL_GLOBAL_CONFIG XEN_CONFIG_DIR "/xl.conf"
- #define XL_LOCK_FILE XEN_LOCK_DIR "/xl"
-diff --git a/tools/xl/xl_cmdtable.c b/tools/xl/xl_cmdtable.c
-index bf2ced8140..54c2db6022 100644
---- a/tools/xl/xl_cmdtable.c
-+++ b/tools/xl/xl_cmdtable.c
-@@ -34,7 +34,8 @@ struct cmd_spec cmd_table[] = {
- "-e Do not wait in the background for the death of the domain.\n"
- "-V, --vncviewer Connect to the VNC display after the domain is created.\n"
- "-A, --vncviewer-autopass\n"
-- " Pass VNC password to viewer via stdin."
-+ " Pass VNC password to viewer via stdin.\n"
-+ "--ignore-global-affinity-masks Ignore global masks in xl.conf."
- },
- { "config-update",
- &main_config_update, 1, 1,
-@@ -224,7 +225,8 @@ struct cmd_spec cmd_table[] = {
- &main_vcpupin, 1, 1,
- "Set which CPUs a VCPU can use",
- "[option] <Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>",
-- "-f, --force undo an override pinning done by the kernel",
-+ "-f, --force undo an override pinning done by the kernel\n"
-+ "--ignore-global-affinity-masks Ignore global masks in xl.conf",
- },
- { "vcpu-set",
- &main_vcpuset, 0, 1,
-diff --git a/tools/xl/xl_vcpu.c b/tools/xl/xl_vcpu.c
-index 8e735b38c1..3384eeed06 100644
---- a/tools/xl/xl_vcpu.c
-+++ b/tools/xl/xl_vcpu.c
-@@ -68,6 +68,61 @@ static void print_domain_vcpuinfo(uint32_t domid, uint32_t nr_cpus)
- libxl_vcpuinfo_list_free(vcpuinfo, nb_vcpu);
- }
-
-+void apply_global_affinity_masks(libxl_domain_type type,
-+ libxl_bitmap *vcpu_affinity_array,
-+ unsigned int size)
-+{
-+ libxl_bitmap *mask = &global_vm_affinity_mask;
-+ libxl_bitmap *type_mask;
-+ unsigned int i;
-+
-+ switch (type) {
-+ case LIBXL_DOMAIN_TYPE_HVM:
-+ case LIBXL_DOMAIN_TYPE_PVH:
-+ type_mask = &global_hvm_affinity_mask;
-+ break;
-+ case LIBXL_DOMAIN_TYPE_PV:
-+ type_mask = &global_pv_affinity_mask;
-+ break;
-+ default:
-+ fprintf(stderr, "Unknown guest type\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ for (i = 0; i < size; i++) {
-+ int rc;
-+ libxl_bitmap *t = &vcpu_affinity_array[i];
-+ libxl_bitmap b1, b2;
-+
-+ libxl_bitmap_init(&b1);
-+ libxl_bitmap_init(&b2);
-+
-+ rc = libxl_bitmap_and(ctx, &b1, t, mask);
-+ if (rc) {
-+ fprintf(stderr, "libxl_bitmap_and errored\n");
-+ exit(EXIT_FAILURE);
-+ }
-+ rc = libxl_bitmap_and(ctx, &b2, &b1, type_mask);
-+ if (rc) {
-+ fprintf(stderr, "libxl_bitmap_and errored\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ if (libxl_bitmap_is_empty(&b2)) {
-+ fprintf(stderr, "vcpu hard affinity map is empty\n");
-+ exit(EXIT_FAILURE);
-+ }
-+
-+ /* Replace target bitmap with the result */
-+ libxl_bitmap_dispose(t);
-+ libxl_bitmap_init(t);
-+ libxl_bitmap_copy_alloc(ctx, t, &b2);
-+
-+ libxl_bitmap_dispose(&b1);
-+ libxl_bitmap_dispose(&b2);
-+ }
-+}
-+
- static void vcpulist(int argc, char **argv)
- {
- libxl_dominfo *dominfo;
-@@ -118,6 +173,7 @@ int main_vcpupin(int argc, char **argv)
- {
- static struct option opts[] = {
- {"force", 0, 0, 'f'},
-+ {"ignore-global-affinity-masks", 0, 0, 'i'},
- COMMON_LONG_OPTS
- };
- libxl_vcpuinfo *vcpuinfo;
-@@ -132,15 +188,18 @@ int main_vcpupin(int argc, char **argv)
- const char *vcpu, *hard_str, *soft_str;
- char *endptr;
- int opt, nb_cpu, nb_vcpu, rc = EXIT_FAILURE;
-- bool force = false;
-+ bool force = false, ignore_masks = false;
-
- libxl_bitmap_init(&cpumap_hard);
- libxl_bitmap_init(&cpumap_soft);
-
-- SWITCH_FOREACH_OPT(opt, "f", opts, "vcpu-pin", 3) {
-+ SWITCH_FOREACH_OPT(opt, "fi", opts, "vcpu-pin", 3) {
- case 'f':
- force = true;
- break;
-+ case 'i':
-+ ignore_masks = true;
-+ break;
- default:
- break;
- }
-@@ -222,6 +281,23 @@ int main_vcpupin(int argc, char **argv)
- goto out;
- }
-
-+ /* Only hard affinity matters here */
-+ if (!ignore_masks) {
-+ libxl_domain_config d_config;
-+
-+ libxl_domain_config_init(&d_config);
-+ rc = libxl_retrieve_domain_configuration(ctx, domid, &d_config);
-+ if (rc) {
-+ fprintf(stderr, "Could not retrieve domain configuration\n");
-+ libxl_domain_config_dispose(&d_config);
-+ goto out;
-+ }
-+
-+ apply_global_affinity_masks(d_config.b_info.type, hard, 1);
-+
-+ libxl_domain_config_dispose(&d_config);
-+ }
-+
- if (force) {
- if (libxl_set_vcpuaffinity_force(ctx, domid, vcpuid, hard, soft)) {
- fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n",
-diff --git a/tools/xl/xl_vmcontrol.c b/tools/xl/xl_vmcontrol.c
-index 89c2b25ded..a1d633795c 100644
---- a/tools/xl/xl_vmcontrol.c
-+++ b/tools/xl/xl_vmcontrol.c
-@@ -804,6 +804,36 @@ int create_domain(struct domain_create *dom_info)
- parse_config_data(config_source, config_data, config_len, &d_config);
- }
-
-+ if (!dom_info->ignore_global_affinity_masks) {
-+ libxl_domain_build_info *b_info = &d_config.b_info;
-+
-+ /* It is possible that no hard affinity is specified in config file.
-+ * Generate hard affinity maps now if we care about those.
-+ */
-+ if (b_info->num_vcpu_hard_affinity == 0 &&
-+ (!libxl_bitmap_is_full(&global_vm_affinity_mask) ||
-+ (b_info->type == LIBXL_DOMAIN_TYPE_PV &&
-+ !libxl_bitmap_is_full(&global_pv_affinity_mask)) ||
-+ (b_info->type != LIBXL_DOMAIN_TYPE_PV &&
-+ !libxl_bitmap_is_full(&global_hvm_affinity_mask))
-+ )) {
-+ b_info->num_vcpu_hard_affinity = b_info->max_vcpus;
-+ b_info->vcpu_hard_affinity =
-+ xmalloc(b_info->max_vcpus * sizeof(libxl_bitmap));
-+
-+ for (i = 0; i < b_info->num_vcpu_hard_affinity; i++) {
-+ libxl_bitmap *m = &b_info->vcpu_hard_affinity[i];
-+ libxl_bitmap_init(m);
-+ libxl_cpu_bitmap_alloc(ctx, m, 0);
-+ libxl_bitmap_set_any(m);
-+ }
-+ }
-+
-+ apply_global_affinity_masks(b_info->type,
-+ b_info->vcpu_hard_affinity,
-+ b_info->num_vcpu_hard_affinity);
-+ }
-+
- if (migrate_fd >= 0) {
- if (d_config.c_info.name) {
- /* when we receive a domain we get its name from the config
-@@ -1124,7 +1154,7 @@ int main_create(int argc, char **argv)
- const char *filename = NULL;
- struct domain_create dom_info;
- int paused = 0, debug = 0, daemonize = 1, console_autoconnect = 0,
-- quiet = 0, monitor = 1, vnc = 0, vncautopass = 0;
-+ quiet = 0, monitor = 1, vnc = 0, vncautopass = 0, ignore_masks = 0;
- int opt, rc;
- static struct option opts[] = {
- {"dryrun", 0, 0, 'n'},
-@@ -1132,6 +1162,7 @@ int main_create(int argc, char **argv)
- {"defconfig", 1, 0, 'f'},
- {"vncviewer", 0, 0, 'V'},
- {"vncviewer-autopass", 0, 0, 'A'},
-+ {"ignore-global-affinity-masks", 0, 0, 'i'},
- COMMON_LONG_OPTS
- };
-
-@@ -1142,7 +1173,7 @@ int main_create(int argc, char **argv)
- argc--; argv++;
- }
-
-- SWITCH_FOREACH_OPT(opt, "Fnqf:pcdeVA", opts, "create", 0) {
-+ SWITCH_FOREACH_OPT(opt, "Fnqf:pcdeVAi", opts, "create", 0) {
- case 'f':
- filename = optarg;
- break;
-@@ -1174,6 +1205,9 @@ int main_create(int argc, char **argv)
- case 'A':
- vnc = vncautopass = 1;
- break;
-+ case 'i':
-+ ignore_masks = 1;
-+ break;
- }
-
- memset(&dom_info, 0, sizeof(dom_info));
-@@ -1203,6 +1237,7 @@ int main_create(int argc, char **argv)
- dom_info.vnc = vnc;
- dom_info.vncautopass = vncautopass;
- dom_info.console_autoconnect = console_autoconnect;
-+ dom_info.ignore_global_affinity_masks = ignore_masks;
-
- rc = create_domain(&dom_info);
- if (rc < 0) {
-#diff --git a/xen/Makefile b/xen/Makefile
-#index 4d075c381f..a922a1b7b5 100644
-#--- a/xen/Makefile
-#+++ b/xen/Makefile
-#@@ -2,7 +2,7 @@
-# # All other places this is stored (eg. compile.h) should be autogenerated.
-# export XEN_VERSION = 4
-# export XEN_SUBVERSION = 11
-#-export XEN_EXTRAVERSION ?= .0$(XEN_VENDORVERSION)
-#+export XEN_EXTRAVERSION ?= .1-pre$(XEN_VENDORVERSION)
-# export XEN_FULLVERSION = $(XEN_VERSION).$(XEN_SUBVERSION)$(XEN_EXTRAVERSION)
-# -include xen-version
-#
-diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig
-index f64fc56739..cfba4a708c 100644
---- a/xen/arch/x86/Kconfig
-+++ b/xen/arch/x86/Kconfig
-@@ -72,6 +72,7 @@ config SHADOW_PAGING
- * Running HVM guests on hardware lacking hardware paging support
- (First-generation Intel VT-x or AMD SVM).
- * Live migration of PV guests.
-+ * L1TF sidechannel mitigation for PV guests.
-
- Under a small number of specific workloads, shadow paging may be
- deliberately used as a performance optimisation.
-diff --git a/xen/arch/x86/cpu/amd.c b/xen/arch/x86/cpu/amd.c
-index 458a3fe60c..76078b55b2 100644
---- a/xen/arch/x86/cpu/amd.c
-+++ b/xen/arch/x86/cpu/amd.c
-@@ -505,17 +505,23 @@ static void amd_get_topology(struct cpuinfo_x86 *c)
- u32 eax, ebx, ecx, edx;
-
- cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
-- c->compute_unit_id = ebx & 0xFF;
- c->x86_num_siblings = ((ebx >> 8) & 0x3) + 1;
-+
-+ if (c->x86 < 0x17)
-+ c->compute_unit_id = ebx & 0xFF;
-+ else {
-+ c->cpu_core_id = ebx & 0xFF;
-+ c->x86_max_cores /= c->x86_num_siblings;
-+ }
- }
-
- if (opt_cpu_info)
- printk("CPU %d(%d) -> Processor %d, %s %d\n",
- cpu, c->x86_max_cores, c->phys_proc_id,
-- cpu_has(c, X86_FEATURE_TOPOEXT) ? "Compute Unit" :
-- "Core",
-- cpu_has(c, X86_FEATURE_TOPOEXT) ? c->compute_unit_id :
-- c->cpu_core_id);
-+ c->compute_unit_id != INVALID_CUID ? "Compute Unit"
-+ : "Core",
-+ c->compute_unit_id != INVALID_CUID ? c->compute_unit_id
-+ : c->cpu_core_id);
- }
-
- static void early_init_amd(struct cpuinfo_x86 *c)
-diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c
-index 528aff1811..fdb022875a 100644
---- a/xen/arch/x86/cpu/common.c
-+++ b/xen/arch/x86/cpu/common.c
-@@ -14,6 +14,7 @@
- #include <public/sysctl.h> /* for XEN_INVALID_{SOCKET,CORE}_ID */
-
- #include "cpu.h"
-+#include "mcheck/x86_mca.h"
-
- bool_t opt_arat = 1;
- boolean_param("arat", opt_arat);
-@@ -355,6 +356,9 @@ static void __init early_cpu_detect(void)
- hap_paddr_bits = PADDR_BITS;
- }
-
-+ if (c->x86_vendor != X86_VENDOR_AMD)
-+ park_offline_cpus = opt_mce;
-+
- initialize_cpu_data(0);
- }
-
-diff --git a/xen/arch/x86/cpu/mcheck/mce.c b/xen/arch/x86/cpu/mcheck/mce.c
-index a8c287d124..32273d9208 100644
---- a/xen/arch/x86/cpu/mcheck/mce.c
-+++ b/xen/arch/x86/cpu/mcheck/mce.c
-@@ -692,12 +692,15 @@ static void cpu_bank_free(unsigned int cpu)
-
- mcabanks_free(poll);
- mcabanks_free(clr);
-+
-+ per_cpu(poll_bankmask, cpu) = NULL;
-+ per_cpu(mce_clear_banks, cpu) = NULL;
- }
-
- static int cpu_bank_alloc(unsigned int cpu)
- {
-- struct mca_banks *poll = mcabanks_alloc();
-- struct mca_banks *clr = mcabanks_alloc();
-+ struct mca_banks *poll = per_cpu(poll_bankmask, cpu) ?: mcabanks_alloc();
-+ struct mca_banks *clr = per_cpu(mce_clear_banks, cpu) ?: mcabanks_alloc();
-
- if ( !poll || !clr )
- {
-@@ -725,7 +728,13 @@ static int cpu_callback(
-
- case CPU_UP_CANCELED:
- case CPU_DEAD:
-- cpu_bank_free(cpu);
-+ if ( !park_offline_cpus )
-+ cpu_bank_free(cpu);
-+ break;
-+
-+ case CPU_REMOVE:
-+ if ( park_offline_cpus )
-+ cpu_bank_free(cpu);
- break;
- }
-
-diff --git a/xen/arch/x86/cpu/mcheck/mce_intel.c b/xen/arch/x86/cpu/mcheck/mce_intel.c
-index e5dd956a24..4474a34e34 100644
---- a/xen/arch/x86/cpu/mcheck/mce_intel.c
-+++ b/xen/arch/x86/cpu/mcheck/mce_intel.c
-@@ -636,8 +636,6 @@ static void clear_cmci(void)
-
- static void cpu_mcheck_disable(void)
- {
-- clear_in_cr4(X86_CR4_MCE);
--
- if ( cmci_support && opt_mce )
- clear_cmci();
- }
-diff --git a/xen/arch/x86/cpu/vpmu_intel.c b/xen/arch/x86/cpu/vpmu_intel.c
-index 207e2e712c..6e27f6ec8e 100644
---- a/xen/arch/x86/cpu/vpmu_intel.c
-+++ b/xen/arch/x86/cpu/vpmu_intel.c
-@@ -454,13 +454,11 @@ static int core2_vpmu_alloc_resource(struct vcpu *v)
-
- if ( is_hvm_vcpu(v) )
- {
-- wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
-- if ( vmx_add_host_load_msr(MSR_CORE_PERF_GLOBAL_CTRL) )
-+ if ( vmx_add_host_load_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, 0) )
- goto out_err;
-
-- if ( vmx_add_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL) )
-+ if ( vmx_add_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, 0) )
- goto out_err;
-- vmx_write_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
- }
-
- core2_vpmu_cxt = xzalloc_bytes(sizeof(*core2_vpmu_cxt) +
-@@ -535,27 +533,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content,
- uint64_t *enabled_cntrs;
-
- if ( !core2_vpmu_msr_common_check(msr, &type, &index) )
-- {
-- /* Special handling for BTS */
-- if ( msr == MSR_IA32_DEBUGCTLMSR )
-- {
-- supported |= IA32_DEBUGCTLMSR_TR | IA32_DEBUGCTLMSR_BTS |
-- IA32_DEBUGCTLMSR_BTINT;
--
-- if ( cpu_has(&current_cpu_data, X86_FEATURE_DSCPL) )
-- supported |= IA32_DEBUGCTLMSR_BTS_OFF_OS |
-- IA32_DEBUGCTLMSR_BTS_OFF_USR;
-- if ( !(msr_content & ~supported) &&
-- vpmu_is_set(vpmu, VPMU_CPU_HAS_BTS) )
-- return 0;
-- if ( (msr_content & supported) &&
-- !vpmu_is_set(vpmu, VPMU_CPU_HAS_BTS) )
-- printk(XENLOG_G_WARNING
-- "%pv: Debug Store unsupported on this CPU\n",
-- current);
-- }
- return -EINVAL;
-- }
-
- ASSERT(!supported);
-
-@@ -613,7 +591,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content,
- return -EINVAL;
-
- if ( is_hvm_vcpu(v) )
-- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL,
-+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL,
- &core2_vpmu_cxt->global_ctrl);
- else
- rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, core2_vpmu_cxt->global_ctrl);
-@@ -682,7 +660,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content,
- return -EINVAL;
-
- if ( is_hvm_vcpu(v) )
-- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL,
-+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL,
- &core2_vpmu_cxt->global_ctrl);
- else
- rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, core2_vpmu_cxt->global_ctrl);
-@@ -701,7 +679,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content,
- else
- {
- if ( is_hvm_vcpu(v) )
-- vmx_write_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
-+ vmx_write_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
- else
- wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
- }
-@@ -735,7 +713,7 @@ static int core2_vpmu_do_rdmsr(unsigned int msr, uint64_t *msr_content)
- break;
- case MSR_CORE_PERF_GLOBAL_CTRL:
- if ( is_hvm_vcpu(v) )
-- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
-+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, msr_content);
- else
- rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, *msr_content);
- break;
-diff --git a/xen/arch/x86/cpuid.c b/xen/arch/x86/cpuid.c
-index beee47d0ed..5cc89e2b34 100644
---- a/xen/arch/x86/cpuid.c
-+++ b/xen/arch/x86/cpuid.c
-@@ -43,6 +43,11 @@ static int __init parse_xen_cpuid(const char *s)
- if ( !val )
- setup_clear_cpu_cap(X86_FEATURE_STIBP);
- }
-+ else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 )
-+ {
-+ if ( !val )
-+ setup_clear_cpu_cap(X86_FEATURE_L1D_FLUSH);
-+ }
- else if ( (val = parse_boolean("ssbd", s, ss)) >= 0 )
- {
- if ( !val )
-diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
-index 9850a782ec..c39cf2c6e5 100644
---- a/xen/arch/x86/domain.c
-+++ b/xen/arch/x86/domain.c
-@@ -107,10 +107,11 @@ static void play_dead(void)
- local_irq_disable();
-
- /*
-- * NOTE: After cpu_exit_clear, per-cpu variables are no longer accessible,
-- * as they may be freed at any time. In this case, heap corruption or
-- * #PF can occur (when heap debugging is enabled). For example, even
-- * printk() can involve tasklet scheduling, which touches per-cpu vars.
-+ * NOTE: After cpu_exit_clear, per-cpu variables may no longer accessible,
-+ * as they may be freed at any time if offline CPUs don't get parked. In
-+ * this case, heap corruption or #PF can occur (when heap debugging is
-+ * enabled). For example, even printk() can involve tasklet scheduling,
-+ * which touches per-cpu vars.
- *
- * Consider very carefully when adding code to *dead_idle. Most hypervisor
- * subsystems are unsafe to call.
-diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
-index 8fbbf3aeb3..dd91038a67 100644
---- a/xen/arch/x86/domctl.c
-+++ b/xen/arch/x86/domctl.c
-@@ -225,7 +225,8 @@ static int update_domain_cpuid_info(struct domain *d,
- */
- call_policy_changed = (is_hvm_domain(d) &&
- ((old_7d0 ^ p->feat.raw[0].d) &
-- cpufeat_mask(X86_FEATURE_IBRSB)));
-+ (cpufeat_mask(X86_FEATURE_IBRSB) |
-+ cpufeat_mask(X86_FEATURE_L1D_FLUSH))));
- break;
-
- case 0xa:
-@@ -1163,7 +1164,7 @@ long arch_do_domctl(
- if ( _xcr0_accum )
- {
- if ( evc->size >= PV_XSAVE_HDR_SIZE + XSTATE_AREA_MIN_SIZE )
-- ret = validate_xstate(_xcr0, _xcr0_accum,
-+ ret = validate_xstate(d, _xcr0, _xcr0_accum,
- &_xsave_area->xsave_hdr);
- }
- else if ( !_xcr0 )
-@@ -1187,8 +1188,7 @@ long arch_do_domctl(
- vcpu_pause(v);
- v->arch.xcr0 = _xcr0;
- v->arch.xcr0_accum = _xcr0_accum;
-- if ( _xcr0_accum & XSTATE_NONLAZY )
-- v->arch.nonlazy_xstate_used = 1;
-+ v->arch.nonlazy_xstate_used = _xcr0_accum & XSTATE_NONLAZY;
- compress_xsave_states(v, _xsave_area,
- evc->size - PV_XSAVE_HDR_SIZE);
- vcpu_unpause(v);
-diff --git a/xen/arch/x86/genapic/x2apic.c b/xen/arch/x86/genapic/x2apic.c
-index 4779b0d0d5..d997806272 100644
---- a/xen/arch/x86/genapic/x2apic.c
-+++ b/xen/arch/x86/genapic/x2apic.c
-@@ -201,18 +201,21 @@ static int update_clusterinfo(
- if ( !cluster_cpus_spare )
- cluster_cpus_spare = xzalloc(cpumask_t);
- if ( !cluster_cpus_spare ||
-- !alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) )
-+ !cond_alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) )
- err = -ENOMEM;
- break;
- case CPU_UP_CANCELED:
- case CPU_DEAD:
-+ case CPU_REMOVE:
-+ if ( park_offline_cpus == (action != CPU_REMOVE) )
-+ break;
- if ( per_cpu(cluster_cpus, cpu) )
- {
- cpumask_clear_cpu(cpu, per_cpu(cluster_cpus, cpu));
- if ( cpumask_empty(per_cpu(cluster_cpus, cpu)) )
-- xfree(per_cpu(cluster_cpus, cpu));
-+ XFREE(per_cpu(cluster_cpus, cpu));
- }
-- free_cpumask_var(per_cpu(scratch_mask, cpu));
-+ FREE_CPUMASK_VAR(per_cpu(scratch_mask, cpu));
- break;
- }
-
-diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
-index c23983cdff..4cbb688c05 100644
---- a/xen/arch/x86/hvm/hvm.c
-+++ b/xen/arch/x86/hvm/hvm.c
-@@ -907,6 +907,9 @@ const char *hvm_efer_valid(const struct vcpu *v, uint64_t value,
- else
- p = &host_cpuid_policy;
-
-+ if ( value & ~EFER_KNOWN_MASK )
-+ return "Unknown bits set";
-+
- if ( (value & EFER_SCE) && !p->extd.syscall )
- return "SCE without feature";
-
-@@ -1269,7 +1272,7 @@ static int hvm_load_cpu_xsave_states(struct domain *d, hvm_domain_context_t *h)
- ctxt = (struct hvm_hw_cpu_xsave *)&h->data[h->cur];
- h->cur += desc->length;
-
-- err = validate_xstate(ctxt->xcr0, ctxt->xcr0_accum,
-+ err = validate_xstate(d, ctxt->xcr0, ctxt->xcr0_accum,
- (const void *)&ctxt->save_area.xsave_hdr);
- if ( err )
- {
-@@ -1324,8 +1327,7 @@ static int hvm_load_cpu_xsave_states(struct domain *d, hvm_domain_context_t *h)
-
- v->arch.xcr0 = ctxt->xcr0;
- v->arch.xcr0_accum = ctxt->xcr0_accum;
-- if ( ctxt->xcr0_accum & XSTATE_NONLAZY )
-- v->arch.nonlazy_xstate_used = 1;
-+ v->arch.nonlazy_xstate_used = ctxt->xcr0_accum & XSTATE_NONLAZY;
- compress_xsave_states(v, &ctxt->save_area,
- size - offsetof(struct hvm_hw_cpu_xsave, save_area));
-
-diff --git a/xen/arch/x86/hvm/svm/svm.c b/xen/arch/x86/hvm/svm/svm.c
-index 165500e3f2..b964c59dad 100644
---- a/xen/arch/x86/hvm/svm/svm.c
-+++ b/xen/arch/x86/hvm/svm/svm.c
-@@ -1432,24 +1432,18 @@ static void svm_inject_event(const struct x86_event *event)
- * Xen must emulate enough of the event injection to be sure that a
- * further fault shouldn't occur during delivery. This covers the fact
- * that hardware doesn't perform DPL checking on injection.
-- *
-- * Also, it accounts for proper positioning of %rip for an event with trap
-- * semantics (where %rip should point after the instruction) which suffers
-- * a fault during injection (at which point %rip should point at the
-- * instruction).
- */
- if ( event->type == X86_EVENTTYPE_PRI_SW_EXCEPTION ||
-- (!cpu_has_svm_nrips && (event->type == X86_EVENTTYPE_SW_INTERRUPT ||
-- event->type == X86_EVENTTYPE_SW_EXCEPTION)) )
-+ (!cpu_has_svm_nrips && (event->type >= X86_EVENTTYPE_SW_INTERRUPT)) )
- svm_emul_swint_injection(&_event);
-
-- switch ( _event.vector )
-+ switch ( _event.vector | -(_event.type == X86_EVENTTYPE_SW_INTERRUPT) )
- {
- case TRAP_debug:
- if ( regs->eflags & X86_EFLAGS_TF )
- {
- __restore_debug_registers(vmcb, curr);
-- vmcb_set_dr6(vmcb, vmcb_get_dr6(vmcb) | 0x4000);
-+ vmcb_set_dr6(vmcb, vmcb_get_dr6(vmcb) | DR_STEP);
- }
- /* fall through */
- case TRAP_int3:
-@@ -1459,6 +1453,13 @@ static void svm_inject_event(const struct x86_event *event)
- domain_pause_for_debugger();
- return;
- }
-+ break;
-+
-+ case TRAP_page_fault:
-+ ASSERT(_event.type == X86_EVENTTYPE_HW_EXCEPTION);
-+ curr->arch.hvm_vcpu.guest_cr[2] = _event.cr2;
-+ vmcb_set_cr2(vmcb, _event.cr2);
-+ break;
- }
-
- if ( unlikely(eventinj.fields.v) &&
-@@ -1481,13 +1482,9 @@ static void svm_inject_event(const struct x86_event *event)
- * icebp, software events with trap semantics need emulating, so %rip in
- * the trap frame points after the instruction.
- *
-- * The x86 emulator (if requested by the x86_swint_emulate_* choice) will
-- * have performed checks such as presence/dpl/etc and believes that the
-- * event injection will succeed without faulting.
-- *
-- * The x86 emulator will always provide fault semantics for software
-- * events, with _trap.insn_len set appropriately. If the injection
-- * requires emulation, move %rip forwards at this point.
-+ * svm_emul_swint_injection() has already confirmed that events with trap
-+ * semantics won't fault on injection. Position %rip/NextRIP suitably,
-+ * and restrict the event type to what hardware will tolerate.
- */
- switch ( _event.type )
- {
-@@ -1544,16 +1541,12 @@ static void svm_inject_event(const struct x86_event *event)
- eventinj.fields.errorcode == (uint16_t)eventinj.fields.errorcode);
- vmcb->eventinj = eventinj;
-
-- if ( _event.vector == TRAP_page_fault )
-- {
-- curr->arch.hvm_vcpu.guest_cr[2] = _event.cr2;
-- vmcb_set_cr2(vmcb, _event.cr2);
-- HVMTRACE_LONG_2D(PF_INJECT, _event.error_code, TRC_PAR_LONG(_event.cr2));
-- }
-+ if ( _event.vector == TRAP_page_fault &&
-+ _event.type == X86_EVENTTYPE_HW_EXCEPTION )
-+ HVMTRACE_LONG_2D(PF_INJECT, _event.error_code,
-+ TRC_PAR_LONG(_event.cr2));
- else
-- {
- HVMTRACE_2D(INJ_EXC, _event.vector, _event.error_code);
-- }
- }
-
- static int svm_event_pending(struct vcpu *v)
-diff --git a/xen/arch/x86/hvm/vmx/entry.S b/xen/arch/x86/hvm/vmx/entry.S
-index aa2f103895..afd552f2b9 100644
---- a/xen/arch/x86/hvm/vmx/entry.S
-+++ b/xen/arch/x86/hvm/vmx/entry.S
-@@ -41,6 +41,15 @@ ENTRY(vmx_asm_vmexit_handler)
- SPEC_CTRL_ENTRY_FROM_HVM /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */
- /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */
-
-+ /* Hardware clears MSR_DEBUGCTL on VMExit. Reinstate it if debugging Xen. */
-+ .macro restore_lbr
-+ mov $IA32_DEBUGCTLMSR_LBR, %eax
-+ mov $MSR_IA32_DEBUGCTLMSR, %ecx
-+ xor %edx, %edx
-+ wrmsr
-+ .endm
-+ ALTERNATIVE "", restore_lbr, X86_FEATURE_XEN_LBR
-+
- mov %rsp,%rdi
- call vmx_vmexit_handler
-
-diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c
-index 258fc08f72..2ba0c40808 100644
---- a/xen/arch/x86/hvm/vmx/vmcs.c
-+++ b/xen/arch/x86/hvm/vmx/vmcs.c
-@@ -38,6 +38,7 @@
- #include <asm/flushtlb.h>
- #include <asm/monitor.h>
- #include <asm/shadow.h>
-+#include <asm/spec_ctrl.h>
- #include <asm/tboot.h>
- #include <asm/apic.h>
-
-@@ -996,6 +997,7 @@ static int construct_vmcs(struct vcpu *v)
- struct domain *d = v->domain;
- u32 vmexit_ctl = vmx_vmexit_control;
- u32 vmentry_ctl = vmx_vmentry_control;
-+ int rc = 0;
-
- vmx_vmcs_enter(v);
-
-@@ -1083,8 +1085,8 @@ static int construct_vmcs(struct vcpu *v)
-
- if ( msr_bitmap == NULL )
- {
-- vmx_vmcs_exit(v);
-- return -ENOMEM;
-+ rc = -ENOMEM;
-+ goto out;
- }
-
- memset(msr_bitmap, ~0, PAGE_SIZE);
-@@ -1268,141 +1270,197 @@ static int construct_vmcs(struct vcpu *v)
- if ( cpu_has_vmx_tsc_scaling )
- __vmwrite(TSC_MULTIPLIER, d->arch.hvm_domain.tsc_scaling_ratio);
-
-- vmx_vmcs_exit(v);
--
- /* will update HOST & GUEST_CR3 as reqd */
- paging_update_paging_modes(v);
-
- vmx_vlapic_msr_changed(v);
-
-- return 0;
-+ if ( opt_l1d_flush && paging_mode_hap(d) )
-+ rc = vmx_add_msr(v, MSR_FLUSH_CMD, FLUSH_CMD_L1D,
-+ VMX_MSR_GUEST_LOADONLY);
-+
-+ out:
-+ vmx_vmcs_exit(v);
-+
-+ return rc;
- }
-
--static int vmx_msr_entry_key_cmp(const void *key, const void *elt)
-+/*
-+ * Search an MSR list looking for an MSR entry, or the slot in which it should
-+ * live (to keep the data sorted) if an entry is not found.
-+ *
-+ * The return pointer is guaranteed to be bounded by start and end. However,
-+ * it may point at end, and may be invalid for the caller to dereference.
-+ */
-+static struct vmx_msr_entry *locate_msr_entry(
-+ struct vmx_msr_entry *start, struct vmx_msr_entry *end, uint32_t msr)
- {
-- const u32 *msr = key;
-- const struct vmx_msr_entry *entry = elt;
-+ while ( start < end )
-+ {
-+ struct vmx_msr_entry *mid = start + (end - start) / 2;
-
-- if ( *msr > entry->index )
-- return 1;
-- if ( *msr < entry->index )
-- return -1;
-+ if ( msr < mid->index )
-+ end = mid;
-+ else if ( msr > mid->index )
-+ start = mid + 1;
-+ else
-+ return mid;
-+ }
-
-- return 0;
-+ return start;
- }
-
--struct vmx_msr_entry *vmx_find_msr(u32 msr, int type)
-+struct vmx_msr_entry *vmx_find_msr(const struct vcpu *v, uint32_t msr,
-+ enum vmx_msr_list_type type)
- {
-- struct vcpu *curr = current;
-- unsigned int msr_count;
-- struct vmx_msr_entry *msr_area;
-+ const struct arch_vmx_struct *vmx = &v->arch.hvm_vmx;
-+ struct vmx_msr_entry *start = NULL, *ent, *end;
-+ unsigned int substart = 0, subend = vmx->msr_save_count;
-+ unsigned int total = vmx->msr_load_count;
-
-- if ( type == VMX_GUEST_MSR )
-- {
-- msr_count = curr->arch.hvm_vmx.msr_count;
-- msr_area = curr->arch.hvm_vmx.msr_area;
-- }
-- else
-+ ASSERT(v == current || !vcpu_runnable(v));
-+
-+ switch ( type )
- {
-- ASSERT(type == VMX_HOST_MSR);
-- msr_count = curr->arch.hvm_vmx.host_msr_count;
-- msr_area = curr->arch.hvm_vmx.host_msr_area;
-+ case VMX_MSR_HOST:
-+ start = vmx->host_msr_area;
-+ subend = vmx->host_msr_count;
-+ total = subend;
-+ break;
-+
-+ case VMX_MSR_GUEST:
-+ start = vmx->msr_area;
-+ break;
-+
-+ case VMX_MSR_GUEST_LOADONLY:
-+ start = vmx->msr_area;
-+ substart = subend;
-+ subend = total;
-+ break;
-+
-+ default:
-+ ASSERT_UNREACHABLE();
- }
-
-- if ( msr_area == NULL )
-+ if ( !start )
- return NULL;
-
-- return bsearch(&msr, msr_area, msr_count, sizeof(struct vmx_msr_entry),
-- vmx_msr_entry_key_cmp);
-+ end = start + total;
-+ ent = locate_msr_entry(start + substart, start + subend, msr);
-+
-+ return ((ent < end) && (ent->index == msr)) ? ent : NULL;
- }
-
--int vmx_read_guest_msr(u32 msr, u64 *val)
-+int vmx_add_msr(struct vcpu *v, uint32_t msr, uint64_t val,
-+ enum vmx_msr_list_type type)
- {
-- struct vmx_msr_entry *ent;
-+ struct arch_vmx_struct *vmx = &v->arch.hvm_vmx;
-+ struct vmx_msr_entry **ptr, *start = NULL, *ent, *end;
-+ unsigned int substart, subend, total;
-+ int rc;
-
-- if ( (ent = vmx_find_msr(msr, VMX_GUEST_MSR)) != NULL )
-+ ASSERT(v == current || !vcpu_runnable(v));
-+
-+ switch ( type )
- {
-- *val = ent->data;
-- return 0;
-- }
-+ case VMX_MSR_HOST:
-+ ptr = &vmx->host_msr_area;
-+ substart = 0;
-+ subend = vmx->host_msr_count;
-+ total = subend;
-+ break;
-
-- return -ESRCH;
--}
-+ case VMX_MSR_GUEST:
-+ ptr = &vmx->msr_area;
-+ substart = 0;
-+ subend = vmx->msr_save_count;
-+ total = vmx->msr_load_count;
-+ break;
-
--int vmx_write_guest_msr(u32 msr, u64 val)
--{
-- struct vmx_msr_entry *ent;
-+ case VMX_MSR_GUEST_LOADONLY:
-+ ptr = &vmx->msr_area;
-+ substart = vmx->msr_save_count;
-+ subend = vmx->msr_load_count;
-+ total = subend;
-+ break;
-
-- if ( (ent = vmx_find_msr(msr, VMX_GUEST_MSR)) != NULL )
-- {
-- ent->data = val;
-- return 0;
-+ default:
-+ ASSERT_UNREACHABLE();
-+ return -EINVAL;
- }
-
-- return -ESRCH;
--}
--
--int vmx_add_msr(u32 msr, int type)
--{
-- struct vcpu *curr = current;
-- unsigned int idx, *msr_count;
-- struct vmx_msr_entry **msr_area, *msr_area_elem;
-+ vmx_vmcs_enter(v);
-
-- if ( type == VMX_GUEST_MSR )
-- {
-- msr_count = &curr->arch.hvm_vmx.msr_count;
-- msr_area = &curr->arch.hvm_vmx.msr_area;
-- }
-- else
-+ /* Allocate memory on first use. */
-+ if ( unlikely(!*ptr) )
- {
-- ASSERT(type == VMX_HOST_MSR);
-- msr_count = &curr->arch.hvm_vmx.host_msr_count;
-- msr_area = &curr->arch.hvm_vmx.host_msr_area;
-- }
-+ paddr_t addr;
-
-- if ( *msr_area == NULL )
-- {
-- if ( (*msr_area = alloc_xenheap_page()) == NULL )
-- return -ENOMEM;
-+ if ( (*ptr = alloc_xenheap_page()) == NULL )
-+ {
-+ rc = -ENOMEM;
-+ goto out;
-+ }
-
-- if ( type == VMX_GUEST_MSR )
-+ addr = virt_to_maddr(*ptr);
-+
-+ switch ( type )
- {
-- __vmwrite(VM_EXIT_MSR_STORE_ADDR, virt_to_maddr(*msr_area));
-- __vmwrite(VM_ENTRY_MSR_LOAD_ADDR, virt_to_maddr(*msr_area));
-+ case VMX_MSR_HOST:
-+ __vmwrite(VM_EXIT_MSR_LOAD_ADDR, addr);
-+ break;
-+
-+ case VMX_MSR_GUEST:
-+ case VMX_MSR_GUEST_LOADONLY:
-+ __vmwrite(VM_EXIT_MSR_STORE_ADDR, addr);
-+ __vmwrite(VM_ENTRY_MSR_LOAD_ADDR, addr);
-+ break;
- }
-- else
-- __vmwrite(VM_EXIT_MSR_LOAD_ADDR, virt_to_maddr(*msr_area));
- }
-
-- for ( idx = 0; idx < *msr_count && (*msr_area)[idx].index <= msr; idx++ )
-- if ( (*msr_area)[idx].index == msr )
-- return 0;
-+ start = *ptr;
-+ end = start + total;
-+ ent = locate_msr_entry(start + substart, start + subend, msr);
-
-- if ( *msr_count == (PAGE_SIZE / sizeof(struct vmx_msr_entry)) )
-- return -ENOSPC;
-+ if ( (ent < end) && (ent->index == msr) )
-+ goto found;
-
-- memmove(*msr_area + idx + 1, *msr_area + idx,
-- sizeof(*msr_area_elem) * (*msr_count - idx));
-+ /* If there isn't an existing entry for msr, insert room for one. */
-+ if ( total == (PAGE_SIZE / sizeof(*ent)) )
-+ {
-+ rc = -ENOSPC;
-+ goto out;
-+ }
-
-- msr_area_elem = *msr_area + idx;
-- msr_area_elem->index = msr;
-- msr_area_elem->mbz = 0;
-+ memmove(ent + 1, ent, sizeof(*ent) * (end - ent));
-
-- ++*msr_count;
-+ ent->index = msr;
-+ ent->mbz = 0;
-
-- if ( type == VMX_GUEST_MSR )
-+ switch ( type )
- {
-- msr_area_elem->data = 0;
-- __vmwrite(VM_EXIT_MSR_STORE_COUNT, *msr_count);
-- __vmwrite(VM_ENTRY_MSR_LOAD_COUNT, *msr_count);
-- }
-- else
-- {
-- rdmsrl(msr, msr_area_elem->data);
-- __vmwrite(VM_EXIT_MSR_LOAD_COUNT, *msr_count);
-+ case VMX_MSR_HOST:
-+ __vmwrite(VM_EXIT_MSR_LOAD_COUNT, ++vmx->host_msr_count);
-+ break;
-+
-+ case VMX_MSR_GUEST:
-+ __vmwrite(VM_EXIT_MSR_STORE_COUNT, ++vmx->msr_save_count);
-+
-+ /* Fallthrough */
-+ case VMX_MSR_GUEST_LOADONLY:
-+ __vmwrite(VM_ENTRY_MSR_LOAD_COUNT, ++vmx->msr_load_count);
-+ break;
- }
-
-- return 0;
-+ /* Set the msr's value. */
-+ found:
-+ ent->data = val;
-+ rc = 0;
-+
-+ out:
-+ vmx_vmcs_exit(v);
-+
-+ return rc;
- }
-
- void vmx_set_eoi_exit_bitmap(struct vcpu *v, u8 vector)
-diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
-index 610c8d6eb9..b0fababede 100644
---- a/xen/arch/x86/hvm/vmx/vmx.c
-+++ b/xen/arch/x86/hvm/vmx/vmx.c
-@@ -583,6 +583,12 @@ static void vmx_cpuid_policy_changed(struct vcpu *v)
- vmx_clear_msr_intercept(v, MSR_PRED_CMD, VMX_MSR_RW);
- else
- vmx_set_msr_intercept(v, MSR_PRED_CMD, VMX_MSR_RW);
-+
-+ /* MSR_FLUSH_CMD is safe to pass through if the guest knows about it. */
-+ if ( cp->feat.l1d_flush )
-+ vmx_clear_msr_intercept(v, MSR_FLUSH_CMD, VMX_MSR_RW);
-+ else
-+ vmx_set_msr_intercept(v, MSR_FLUSH_CMD, VMX_MSR_RW);
- }
-
- int vmx_guest_x86_mode(struct vcpu *v)
-@@ -2758,8 +2764,10 @@ enum
-
- #define LBR_FROM_SIGNEXT_2MSB ((1ULL << 59) | (1ULL << 60))
-
--#define FIXUP_LBR_TSX (1u << 0)
--#define FIXUP_BDW_ERRATUM_BDF14 (1u << 1)
-+#define LBR_MSRS_INSERTED (1u << 0)
-+#define LBR_FIXUP_TSX (1u << 1)
-+#define LBR_FIXUP_BDF14 (1u << 2)
-+#define LBR_FIXUP_MASK (LBR_FIXUP_TSX | LBR_FIXUP_BDF14)
-
- static bool __read_mostly lbr_tsx_fixup_needed;
- static bool __read_mostly bdw_erratum_bdf14_fixup_needed;
-@@ -2822,7 +2830,7 @@ static int is_last_branch_msr(u32 ecx)
-
- static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content)
- {
-- const struct vcpu *curr = current;
-+ struct vcpu *curr = current;
-
- HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x", msr);
-
-@@ -2901,7 +2909,7 @@ static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content)
- if ( passive_domain_do_rdmsr(msr, msr_content) )
- goto done;
-
-- if ( vmx_read_guest_msr(msr, msr_content) == 0 )
-+ if ( vmx_read_guest_msr(curr, msr, msr_content) == 0 )
- break;
-
- if ( is_last_branch_msr(msr) )
-@@ -3036,11 +3044,14 @@ void vmx_vlapic_msr_changed(struct vcpu *v)
- static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content)
- {
- struct vcpu *v = current;
-+ const struct cpuid_policy *cp = v->domain->arch.cpuid;
-
- HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x, msr_value=%#"PRIx64, msr, msr_content);
-
- switch ( msr )
- {
-+ uint64_t rsvd;
-+
- case MSR_IA32_SYSENTER_CS:
- __vmwrite(GUEST_SYSENTER_CS, msr_content);
- break;
-@@ -3093,45 +3104,85 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content)
- wrmsrl(MSR_SYSCALL_MASK, msr_content);
- break;
-
-- case MSR_IA32_DEBUGCTLMSR: {
-- int i, rc = 0;
-- uint64_t supported = IA32_DEBUGCTLMSR_LBR | IA32_DEBUGCTLMSR_BTF;
-+ case MSR_IA32_DEBUGCTLMSR:
-+ rsvd = ~(IA32_DEBUGCTLMSR_LBR | IA32_DEBUGCTLMSR_BTF);
-
-- if ( boot_cpu_has(X86_FEATURE_RTM) )
-- supported |= IA32_DEBUGCTLMSR_RTM;
-- if ( msr_content & ~supported )
-+ /* TODO: Wire vPMU settings properly through the CPUID policy */
-+ if ( vpmu_is_set(vcpu_vpmu(v), VPMU_CPU_HAS_BTS) )
- {
-- /* Perhaps some other bits are supported in vpmu. */
-- if ( vpmu_do_wrmsr(msr, msr_content, supported) )
-- break;
-+ rsvd &= ~(IA32_DEBUGCTLMSR_TR | IA32_DEBUGCTLMSR_BTS |
-+ IA32_DEBUGCTLMSR_BTINT);
-+
-+ if ( cpu_has(&current_cpu_data, X86_FEATURE_DSCPL) )
-+ rsvd &= ~(IA32_DEBUGCTLMSR_BTS_OFF_OS |
-+ IA32_DEBUGCTLMSR_BTS_OFF_USR);
- }
-- if ( msr_content & IA32_DEBUGCTLMSR_LBR )
-+
-+ if ( cp->feat.rtm )
-+ rsvd &= ~IA32_DEBUGCTLMSR_RTM;
-+
-+ if ( msr_content & rsvd )
-+ goto gp_fault;
-+
-+ /*
-+ * When a guest first enables LBR, arrange to save and restore the LBR
-+ * MSRs and allow the guest direct access.
-+ *
-+ * MSR_DEBUGCTL and LBR has existed almost as long as MSRs have
-+ * existed, and there is no architectural way to hide the feature, or
-+ * fail the attempt to enable LBR.
-+ *
-+ * Unknown host LBR MSRs or hitting -ENOSPC with the guest load/save
-+ * list are definitely hypervisor bugs, whereas -ENOMEM for allocating
-+ * the load/save list is simply unlucky (and shouldn't occur with
-+ * sensible management by the toolstack).
-+ *
-+ * Either way, there is nothing we can do right now to recover, and
-+ * the guest won't execute correctly either. Simply crash the domain
-+ * to make the failure obvious.
-+ */
-+ if ( !(v->arch.hvm_vmx.lbr_flags & LBR_MSRS_INSERTED) &&
-+ (msr_content & IA32_DEBUGCTLMSR_LBR) )
- {
- const struct lbr_info *lbr = last_branch_msr_get();
-- if ( lbr == NULL )
-- break;
-
-- for ( ; (rc == 0) && lbr->count; lbr++ )
-- for ( i = 0; (rc == 0) && (i < lbr->count); i++ )
-- if ( (rc = vmx_add_guest_msr(lbr->base + i)) == 0 )
-+ if ( unlikely(!lbr) )
-+ {
-+ gprintk(XENLOG_ERR, "Unknown Host LBR MSRs\n");
-+ domain_crash(v->domain);
-+ return X86EMUL_OKAY;
-+ }
-+
-+ for ( ; lbr->count; lbr++ )
-+ {
-+ unsigned int i;
-+
-+ for ( i = 0; i < lbr->count; i++ )
-+ {
-+ int rc = vmx_add_guest_msr(v, lbr->base + i, 0);
-+
-+ if ( unlikely(rc) )
- {
-- vmx_clear_msr_intercept(v, lbr->base + i, VMX_MSR_RW);
-- if ( lbr_tsx_fixup_needed )
-- v->arch.hvm_vmx.lbr_fixup_enabled |= FIXUP_LBR_TSX;
-- if ( bdw_erratum_bdf14_fixup_needed )
-- v->arch.hvm_vmx.lbr_fixup_enabled |=
-- FIXUP_BDW_ERRATUM_BDF14;
-+ gprintk(XENLOG_ERR,
-+ "Guest load/save list error %d\n", rc);
-+ domain_crash(v->domain);
-+ return X86EMUL_OKAY;
- }
-- }
-
-- if ( (rc < 0) ||
-- (msr_content && (vmx_add_host_load_msr(msr) < 0)) )
-- hvm_inject_hw_exception(TRAP_machine_check, X86_EVENT_NO_EC);
-- else
-- __vmwrite(GUEST_IA32_DEBUGCTL, msr_content);
-+ vmx_clear_msr_intercept(v, lbr->base + i, VMX_MSR_RW);
-+ }
-+ }
-+
-+ v->arch.hvm_vmx.lbr_flags |= LBR_MSRS_INSERTED;
-+ if ( lbr_tsx_fixup_needed )
-+ v->arch.hvm_vmx.lbr_flags |= LBR_FIXUP_TSX;
-+ if ( bdw_erratum_bdf14_fixup_needed )
-+ v->arch.hvm_vmx.lbr_flags |= LBR_FIXUP_BDF14;
-+ }
-
-+ __vmwrite(GUEST_IA32_DEBUGCTL, msr_content);
- break;
-- }
-+
- case MSR_IA32_FEATURE_CONTROL:
- case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
- /* None of these MSRs are writeable. */
-@@ -3154,7 +3205,7 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content)
- if ( wrmsr_viridian_regs(msr, msr_content) )
- break;
-
-- if ( vmx_write_guest_msr(msr, msr_content) == 0 ||
-+ if ( vmx_write_guest_msr(v, msr, msr_content) == 0 ||
- is_last_branch_msr(msr) )
- break;
-
-@@ -3701,6 +3752,7 @@ void vmx_vmexit_handler(struct cpu_user_regs *regs)
- */
- __vmread(EXIT_QUALIFICATION, &exit_qualification);
- HVMTRACE_1D(TRAP_DEBUG, exit_qualification);
-+ __restore_debug_registers(v);
- write_debugreg(6, exit_qualification | DR_STATUS_RESERVED_ONE);
- if ( !v->domain->debugger_attached )
- {
-@@ -4165,11 +4217,11 @@ out:
- static void lbr_tsx_fixup(void)
- {
- struct vcpu *curr = current;
-- unsigned int msr_count = curr->arch.hvm_vmx.msr_count;
-+ unsigned int msr_count = curr->arch.hvm_vmx.msr_save_count;
- struct vmx_msr_entry *msr_area = curr->arch.hvm_vmx.msr_area;
- struct vmx_msr_entry *msr;
-
-- if ( (msr = vmx_find_msr(lbr_from_start, VMX_GUEST_MSR)) != NULL )
-+ if ( (msr = vmx_find_msr(curr, lbr_from_start, VMX_MSR_GUEST)) != NULL )
- {
- /*
- * Sign extend into bits 61:62 while preserving bit 63
-@@ -4179,15 +4231,15 @@ static void lbr_tsx_fixup(void)
- msr->data |= ((LBR_FROM_SIGNEXT_2MSB & msr->data) << 2);
- }
-
-- if ( (msr = vmx_find_msr(lbr_lastint_from, VMX_GUEST_MSR)) != NULL )
-+ if ( (msr = vmx_find_msr(curr, lbr_lastint_from, VMX_MSR_GUEST)) != NULL )
- msr->data |= ((LBR_FROM_SIGNEXT_2MSB & msr->data) << 2);
- }
-
--static void sign_extend_msr(u32 msr, int type)
-+static void sign_extend_msr(struct vcpu *v, u32 msr, int type)
- {
- struct vmx_msr_entry *entry;
-
-- if ( (entry = vmx_find_msr(msr, type)) != NULL )
-+ if ( (entry = vmx_find_msr(v, msr, type)) != NULL )
- {
- if ( entry->data & VADDR_TOP_BIT )
- entry->data |= CANONICAL_MASK;
-@@ -4198,6 +4250,8 @@ static void sign_extend_msr(u32 msr, int type)
-
- static void bdw_erratum_bdf14_fixup(void)
- {
-+ struct vcpu *curr = current;
-+
- /*
- * Occasionally, on certain Broadwell CPUs MSR_IA32_LASTINTTOIP has
- * been observed to have the top three bits corrupted as though the
-@@ -4207,17 +4261,17 @@ static void bdw_erratum_bdf14_fixup(void)
- * erratum BDF14. Fix up MSR_IA32_LASTINT{FROM,TO}IP by
- * sign-extending into bits 48:63.
- */
-- sign_extend_msr(MSR_IA32_LASTINTFROMIP, VMX_GUEST_MSR);
-- sign_extend_msr(MSR_IA32_LASTINTTOIP, VMX_GUEST_MSR);
-+ sign_extend_msr(curr, MSR_IA32_LASTINTFROMIP, VMX_MSR_GUEST);
-+ sign_extend_msr(curr, MSR_IA32_LASTINTTOIP, VMX_MSR_GUEST);
- }
-
- static void lbr_fixup(void)
- {
- struct vcpu *curr = current;
-
-- if ( curr->arch.hvm_vmx.lbr_fixup_enabled & FIXUP_LBR_TSX )
-+ if ( curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_TSX )
- lbr_tsx_fixup();
-- if ( curr->arch.hvm_vmx.lbr_fixup_enabled & FIXUP_BDW_ERRATUM_BDF14 )
-+ if ( curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_BDF14 )
- bdw_erratum_bdf14_fixup();
- }
-
-@@ -4285,7 +4339,7 @@ bool vmx_vmenter_helper(const struct cpu_user_regs *regs)
- }
-
- out:
-- if ( unlikely(curr->arch.hvm_vmx.lbr_fixup_enabled) )
-+ if ( unlikely(curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_MASK) )
- lbr_fixup();
-
- HVMTRACE_ND(VMENTRY, 0, 1/*cycles*/, 0, 0, 0, 0, 0, 0, 0);
-diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
-index bcf46c0743..7d4871b791 100644
---- a/xen/arch/x86/mm.c
-+++ b/xen/arch/x86/mm.c
-@@ -613,6 +613,9 @@ static int alloc_segdesc_page(struct page_info *page)
- return i == 512 ? 0 : -EINVAL;
- }
-
-+static int _get_page_type(struct page_info *page, unsigned long type,
-+ bool preemptible);
-+
- static int get_page_and_type_from_mfn(
- mfn_t mfn, unsigned long type, struct domain *d,
- int partial, int preemptible)
-@@ -624,9 +627,7 @@ static int get_page_and_type_from_mfn(
- unlikely(!get_page_from_mfn(mfn, d)) )
- return -EINVAL;
-
-- rc = (preemptible ?
-- get_page_type_preemptible(page, type) :
-- (get_page_type(page, type) ? 0 : -EINVAL));
-+ rc = _get_page_type(page, type, preemptible);
-
- if ( unlikely(rc) && partial >= 0 &&
- (!preemptible || page != current->arch.old_guest_table) )
-@@ -1115,7 +1116,7 @@ get_page_from_l2e(
- int rc;
-
- if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
-- return 1;
-+ return pv_l1tf_check_l2e(d, l2e) ? -ERESTART : 1;
-
- if ( unlikely((l2e_get_flags(l2e) & L2_DISALLOW_MASK)) )
- {
-@@ -1146,7 +1147,7 @@ get_page_from_l3e(
- int rc;
-
- if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
-- return 1;
-+ return pv_l1tf_check_l3e(d, l3e) ? -ERESTART : 1;
-
- if ( unlikely((l3e_get_flags(l3e) & l3_disallow_mask(d))) )
- {
-@@ -1179,7 +1180,7 @@ get_page_from_l4e(
- int rc;
-
- if ( !(l4e_get_flags(l4e) & _PAGE_PRESENT) )
-- return 1;
-+ return pv_l1tf_check_l4e(d, l4e) ? -ERESTART : 1;
-
- if ( unlikely((l4e_get_flags(l4e) & L4_DISALLOW_MASK)) )
- {
-@@ -1389,6 +1390,13 @@ static int alloc_l1_table(struct page_info *page)
-
- for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ )
- {
-+ if ( !(l1e_get_flags(pl1e[i]) & _PAGE_PRESENT) )
-+ {
-+ ret = pv_l1tf_check_l1e(d, pl1e[i]) ? -ERESTART : 0;
-+ if ( ret )
-+ goto out;
-+ }
-+
- switch ( ret = get_page_from_l1e(pl1e[i], d, d) )
- {
- default:
-@@ -1409,6 +1417,7 @@ static int alloc_l1_table(struct page_info *page)
-
- fail:
- gdprintk(XENLOG_WARNING, "Failure in alloc_l1_table: slot %#x\n", i);
-+ out:
- while ( i-- > 0 )
- put_page_from_l1e(pl1e[i], d);
-
-@@ -1456,8 +1465,7 @@ static int create_pae_xen_mappings(struct domain *d, l3_pgentry_t *pl3e)
- return 1;
- }
-
--static int alloc_l2_table(struct page_info *page, unsigned long type,
-- int preemptible)
-+static int alloc_l2_table(struct page_info *page, unsigned long type)
- {
- struct domain *d = page_get_owner(page);
- unsigned long pfn = mfn_x(page_to_mfn(page));
-@@ -1469,8 +1477,7 @@ static int alloc_l2_table(struct page_info *page, unsigned long type,
-
- for ( i = page->nr_validated_ptes; i < L2_PAGETABLE_ENTRIES; i++ )
- {
-- if ( preemptible && i > page->nr_validated_ptes
-- && hypercall_preempt_check() )
-+ if ( i > page->nr_validated_ptes && hypercall_preempt_check() )
- {
- page->nr_validated_ptes = i;
- rc = -ERESTART;
-@@ -1481,6 +1488,12 @@ static int alloc_l2_table(struct page_info *page, unsigned long type,
- (rc = get_page_from_l2e(pl2e[i], pfn, d)) > 0 )
- continue;
-
-+ if ( unlikely(rc == -ERESTART) )
-+ {
-+ page->nr_validated_ptes = i;
-+ break;
-+ }
-+
- if ( rc < 0 )
- {
- gdprintk(XENLOG_WARNING, "Failure in alloc_l2_table: slot %#x\n", i);
-@@ -1763,7 +1776,7 @@ static void free_l1_table(struct page_info *page)
- }
-
-
--static int free_l2_table(struct page_info *page, int preemptible)
-+static int free_l2_table(struct page_info *page)
- {
- struct domain *d = page_get_owner(page);
- unsigned long pfn = mfn_x(page_to_mfn(page));
-@@ -1777,7 +1790,7 @@ static int free_l2_table(struct page_info *page, int preemptible)
- do {
- if ( is_guest_l2_slot(d, page->u.inuse.type_info, i) &&
- put_page_from_l2e(pl2e[i], pfn) == 0 &&
-- preemptible && i && hypercall_preempt_check() )
-+ i && hypercall_preempt_check() )
- {
- page->nr_validated_ptes = i;
- err = -ERESTART;
-@@ -2055,6 +2068,8 @@ static int mod_l1_entry(l1_pgentry_t *pl1e, l1_pgentry_t nl1e,
- rc = -EBUSY;
- }
- }
-+ else if ( pv_l1tf_check_l1e(pt_dom, nl1e) )
-+ return -ERESTART;
- else if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, pt_vcpu,
- preserve_ad)) )
- {
-@@ -2118,6 +2133,8 @@ static int mod_l2_entry(l2_pgentry_t *pl2e,
- rc = -EBUSY;
- }
- }
-+ else if ( pv_l1tf_check_l2e(d, nl2e) )
-+ return -ERESTART;
- else if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn, vcpu,
- preserve_ad)) )
- {
-@@ -2179,6 +2196,8 @@ static int mod_l3_entry(l3_pgentry_t *pl3e,
- rc = -EFAULT;
- }
- }
-+ else if ( pv_l1tf_check_l3e(d, nl3e) )
-+ return -ERESTART;
- else if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, vcpu,
- preserve_ad)) )
- {
-@@ -2244,6 +2263,8 @@ static int mod_l4_entry(l4_pgentry_t *pl4e,
- rc = -EFAULT;
- }
- }
-+ else if ( pv_l1tf_check_l4e(d, nl4e) )
-+ return -ERESTART;
- else if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, vcpu,
- preserve_ad)) )
- {
-@@ -2373,7 +2394,8 @@ static int alloc_page_type(struct page_info *page, unsigned long type,
- rc = alloc_l1_table(page);
- break;
- case PGT_l2_page_table:
-- rc = alloc_l2_table(page, type, preemptible);
-+ ASSERT(preemptible);
-+ rc = alloc_l2_table(page, type);
- break;
- case PGT_l3_page_table:
- ASSERT(preemptible);
-@@ -2463,7 +2485,8 @@ int free_page_type(struct page_info *page, unsigned long type,
- rc = 0;
- break;
- case PGT_l2_page_table:
-- rc = free_l2_table(page, preemptible);
-+ ASSERT(preemptible);
-+ rc = free_l2_table(page);
- break;
- case PGT_l3_page_table:
- ASSERT(preemptible);
-@@ -3550,12 +3573,9 @@ long do_mmuext_op(
- }
-
- if ( rc == -ERESTART )
-- {
-- ASSERT(i < count);
- rc = hypercall_create_continuation(
- __HYPERVISOR_mmuext_op, "hihi",
- uops, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom);
-- }
- else if ( curr->arch.old_guest_table )
- {
- XEN_GUEST_HANDLE_PARAM(void) null;
-@@ -3861,12 +3881,9 @@ long do_mmu_update(
- }
-
- if ( rc == -ERESTART )
-- {
-- ASSERT(i < count);
- rc = hypercall_create_continuation(
- __HYPERVISOR_mmu_update, "hihi",
- ureqs, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom);
-- }
- else if ( curr->arch.old_guest_table )
- {
- XEN_GUEST_HANDLE_PARAM(void) null;
-@@ -4121,7 +4138,13 @@ static int __do_update_va_mapping(
- long do_update_va_mapping(unsigned long va, u64 val64,
- unsigned long flags)
- {
-- return __do_update_va_mapping(va, val64, flags, current->domain);
-+ int rc = __do_update_va_mapping(va, val64, flags, current->domain);
-+
-+ if ( rc == -ERESTART )
-+ rc = hypercall_create_continuation(
-+ __HYPERVISOR_update_va_mapping, "lll", va, val64, flags);
-+
-+ return rc;
- }
-
- long do_update_va_mapping_otherdomain(unsigned long va, u64 val64,
-@@ -4138,6 +4161,46 @@ long do_update_va_mapping_otherdomain(unsigned long va, u64 val64,
-
- put_pg_owner(pg_owner);
-
-+ if ( rc == -ERESTART )
-+ rc = hypercall_create_continuation(
-+ __HYPERVISOR_update_va_mapping_otherdomain,
-+ "llli", va, val64, flags, domid);
-+
-+ return rc;
-+}
-+
-+int compat_update_va_mapping(unsigned int va, uint32_t lo, uint32_t hi,
-+ unsigned int flags)
-+{
-+ int rc = __do_update_va_mapping(va, ((uint64_t)hi << 32) | lo,
-+ flags, current->domain);
-+
-+ if ( rc == -ERESTART )
-+ rc = hypercall_create_continuation(
-+ __HYPERVISOR_update_va_mapping, "iiii", va, lo, hi, flags);
-+
-+ return rc;
-+}
-+
-+int compat_update_va_mapping_otherdomain(unsigned int va,
-+ uint32_t lo, uint32_t hi,
-+ unsigned int flags, domid_t domid)
-+{
-+ struct domain *pg_owner;
-+ int rc;
-+
-+ if ( (pg_owner = get_pg_owner(domid)) == NULL )
-+ return -ESRCH;
-+
-+ rc = __do_update_va_mapping(va, ((uint64_t)hi << 32) | lo, flags, pg_owner);
-+
-+ put_pg_owner(pg_owner);
-+
-+ if ( rc == -ERESTART )
-+ rc = hypercall_create_continuation(
-+ __HYPERVISOR_update_va_mapping_otherdomain,
-+ "iiiii", va, lo, hi, flags, domid);
-+
- return rc;
- }
-
-diff --git a/xen/arch/x86/mm/paging.c b/xen/arch/x86/mm/paging.c
-index 2b0445ffe9..dcee496eb0 100644
---- a/xen/arch/x86/mm/paging.c
-+++ b/xen/arch/x86/mm/paging.c
-@@ -873,6 +873,8 @@ void paging_dump_domain_info(struct domain *d)
- printk(" paging assistance: ");
- if ( paging_mode_shadow(d) )
- printk("shadow ");
-+ if ( paging_mode_sh_forced(d) )
-+ printk("forced ");
- if ( paging_mode_hap(d) )
- printk("hap ");
- if ( paging_mode_refcounts(d) )
-diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c
-index dd61b50eb7..fd42d734e7 100644
---- a/xen/arch/x86/mm/shadow/common.c
-+++ b/xen/arch/x86/mm/shadow/common.c
-@@ -3177,6 +3177,15 @@ static void sh_new_mode(struct domain *d, u32 new_mode)
- ASSERT(paging_locked_by_me(d));
- ASSERT(d != current->domain);
-
-+ /*
-+ * If PG_SH_forced has previously been activated because of writing an
-+ * L1TF-vulnerable PTE, it must remain active for the remaining lifetime
-+ * of the domain, even if the logdirty mode needs to be controlled for
-+ * migration purposes.
-+ */
-+ if ( paging_mode_sh_forced(d) )
-+ new_mode |= PG_SH_forced | PG_SH_enable;
-+
- d->arch.paging.mode = new_mode;
- for_each_vcpu(d, v)
- sh_update_paging_modes(v);
-@@ -4057,6 +4066,33 @@ void shadow_audit_tables(struct vcpu *v)
-
- #endif /* Shadow audit */
-
-+#ifdef CONFIG_PV
-+
-+void pv_l1tf_tasklet(unsigned long data)
-+{
-+ struct domain *d = (void *)data;
-+
-+ domain_pause(d);
-+ paging_lock(d);
-+
-+ if ( !paging_mode_sh_forced(d) && !d->is_dying )
-+ {
-+ int ret = shadow_one_bit_enable(d, PG_SH_forced);
-+
-+ if ( ret )
-+ {
-+ printk(XENLOG_G_ERR "d%d Failed to enable PG_SH_forced: %d\n",
-+ d->domain_id, ret);
-+ domain_crash(d);
-+ }
-+ }
-+
-+ paging_unlock(d);
-+ domain_unpause(d);
-+}
-+
-+#endif /* CONFIG_PV */
-+
- /*
- * Local variables:
- * mode: C
-diff --git a/xen/arch/x86/mpparse.c b/xen/arch/x86/mpparse.c
-index 49140e46f0..f3f6d48668 100644
---- a/xen/arch/x86/mpparse.c
-+++ b/xen/arch/x86/mpparse.c
-@@ -68,19 +68,26 @@ physid_mask_t phys_cpu_present_map;
-
- void __init set_nr_cpu_ids(unsigned int max_cpus)
- {
-+ unsigned int tot_cpus = num_processors + disabled_cpus;
-+
- if (!max_cpus)
-- max_cpus = num_processors + disabled_cpus;
-+ max_cpus = tot_cpus;
- if (max_cpus > NR_CPUS)
- max_cpus = NR_CPUS;
- else if (!max_cpus)
- max_cpus = 1;
- printk(XENLOG_INFO "SMP: Allowing %u CPUs (%d hotplug CPUs)\n",
- max_cpus, max_t(int, max_cpus - num_processors, 0));
-- nr_cpu_ids = max_cpus;
-+
-+ if (!park_offline_cpus)
-+ tot_cpus = max_cpus;
-+ nr_cpu_ids = min(tot_cpus, NR_CPUS + 0u);
-+ if (park_offline_cpus && nr_cpu_ids < num_processors)
-+ printk(XENLOG_WARNING "SMP: Cannot bring up %u further CPUs\n",
-+ num_processors - nr_cpu_ids);
-
- #ifndef nr_cpumask_bits
-- nr_cpumask_bits = (max_cpus + (BITS_PER_LONG - 1)) &
-- ~(BITS_PER_LONG - 1);
-+ nr_cpumask_bits = ROUNDUP(nr_cpu_ids, BITS_PER_LONG);
- printk(XENLOG_DEBUG "NR_CPUS:%u nr_cpumask_bits:%u\n",
- NR_CPUS, nr_cpumask_bits);
- #endif
-diff --git a/xen/arch/x86/msr.c b/xen/arch/x86/msr.c
-index 1e12ccb729..1a591dd2b5 100644
---- a/xen/arch/x86/msr.c
-+++ b/xen/arch/x86/msr.c
-@@ -150,6 +150,7 @@ int guest_rdmsr(const struct vcpu *v, uint32_t msr, uint64_t *val)
- case MSR_AMD_PATCHLOADER:
- case MSR_IA32_UCODE_WRITE:
- case MSR_PRED_CMD:
-+ case MSR_FLUSH_CMD:
- /* Write-only */
- goto gp_fault;
-
-@@ -254,6 +255,17 @@ int guest_wrmsr(struct vcpu *v, uint32_t msr, uint64_t val)
- wrmsrl(MSR_PRED_CMD, val);
- break;
-
-+ case MSR_FLUSH_CMD:
-+ if ( !cp->feat.l1d_flush )
-+ goto gp_fault; /* MSR available? */
-+
-+ if ( val & ~FLUSH_CMD_L1D )
-+ goto gp_fault; /* Rsvd bit set? */
-+
-+ if ( v == curr )
-+ wrmsrl(MSR_FLUSH_CMD, val);
-+ break;
-+
- case MSR_INTEL_MISC_FEATURES_ENABLES:
- {
- bool old_cpuid_faulting = vp->misc_features_enables.cpuid_faulting;
-diff --git a/xen/arch/x86/oprofile/nmi_int.c b/xen/arch/x86/oprofile/nmi_int.c
-index d8f5230906..3dfb8fef93 100644
---- a/xen/arch/x86/oprofile/nmi_int.c
-+++ b/xen/arch/x86/oprofile/nmi_int.c
-@@ -182,7 +182,7 @@ int nmi_reserve_counters(void)
- if (!allocate_msrs())
- return -ENOMEM;
-
-- /* We walk a thin line between law and rape here.
-+ /*
- * We need to be careful to install our NMI handler
- * without actually triggering any NMIs as this will
- * break the core code horrifically.
-diff --git a/xen/arch/x86/percpu.c b/xen/arch/x86/percpu.c
-index c9997b7937..8be4ebddf4 100644
---- a/xen/arch/x86/percpu.c
-+++ b/xen/arch/x86/percpu.c
-@@ -28,7 +28,7 @@ static int init_percpu_area(unsigned int cpu)
- char *p;
-
- if ( __per_cpu_offset[cpu] != INVALID_PERCPU_AREA )
-- return -EBUSY;
-+ return 0;
-
- if ( (p = alloc_xenheap_pages(PERCPU_ORDER, 0)) == NULL )
- return -ENOMEM;
-@@ -76,9 +76,12 @@ static int cpu_percpu_callback(
- break;
- case CPU_UP_CANCELED:
- case CPU_DEAD:
-- free_percpu_area(cpu);
-+ if ( !park_offline_cpus )
-+ free_percpu_area(cpu);
- break;
-- default:
-+ case CPU_REMOVE:
-+ if ( park_offline_cpus )
-+ free_percpu_area(cpu);
- break;
- }
-
-diff --git a/xen/arch/x86/pv/domain.c b/xen/arch/x86/pv/domain.c
-index a4f0bd239d..3230ac6a22 100644
---- a/xen/arch/x86/pv/domain.c
-+++ b/xen/arch/x86/pv/domain.c
-@@ -13,6 +13,7 @@
- #include <asm/invpcid.h>
- #include <asm/spec_ctrl.h>
- #include <asm/pv/domain.h>
-+#include <asm/shadow.h>
-
- static __read_mostly enum {
- PCID_OFF,
-@@ -209,6 +210,8 @@ int pv_vcpu_initialise(struct vcpu *v)
-
- void pv_domain_destroy(struct domain *d)
- {
-+ pv_l1tf_domain_destroy(d);
-+
- destroy_perdomain_mapping(d, GDT_LDT_VIRT_START,
- GDT_LDT_MBYTES << (20 - PAGE_SHIFT));
-
-@@ -229,6 +232,8 @@ int pv_domain_initialise(struct domain *d)
- };
- int rc = -ENOMEM;
-
-+ pv_l1tf_domain_init(d);
-+
- d->arch.pv_domain.gdt_ldt_l1tab =
- alloc_xenheap_pages(0, MEMF_node(domain_to_node(d)));
- if ( !d->arch.pv_domain.gdt_ldt_l1tab )
-diff --git a/xen/arch/x86/pv/ro-page-fault.c b/xen/arch/x86/pv/ro-page-fault.c
-index aa8d5a7556..a3c0c2dd19 100644
---- a/xen/arch/x86/pv/ro-page-fault.c
-+++ b/xen/arch/x86/pv/ro-page-fault.c
-@@ -29,6 +29,7 @@
- #include <asm/mm.h>
- #include <asm/pci.h>
- #include <asm/pv/mm.h>
-+#include <asm/shadow.h>
-
- #include "emulate.h"
- #include "mm.h"
-@@ -129,6 +130,10 @@ static int ptwr_emulated_update(unsigned long addr, intpte_t *p_old,
-
- /* Check the new PTE. */
- nl1e = l1e_from_intpte(val);
-+
-+ if ( !(l1e_get_flags(nl1e) & _PAGE_PRESENT) && pv_l1tf_check_l1e(d, nl1e) )
-+ return X86EMUL_RETRY;
-+
- switch ( ret = get_page_from_l1e(nl1e, d, d) )
- {
- default:
-diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
-index a3172ca92c..3cd3e81b30 100644
---- a/xen/arch/x86/setup.c
-+++ b/xen/arch/x86/setup.c
-@@ -62,6 +62,9 @@ boolean_param("nosmp", opt_nosmp);
- static unsigned int __initdata max_cpus;
- integer_param("maxcpus", max_cpus);
-
-+int8_t __read_mostly opt_smt = -1;
-+boolean_param("smt", opt_smt);
-+
- /* opt_invpcid: If false, don't use INVPCID instruction even if available. */
- static bool __initdata opt_invpcid = true;
- boolean_param("invpcid", opt_invpcid);
-@@ -665,7 +668,7 @@ void __init noreturn __start_xen(unsigned long mbi_p)
- {
- char *memmap_type = NULL;
- char *cmdline, *kextra, *loader;
-- unsigned int initrdidx;
-+ unsigned int initrdidx, num_parked = 0;
- multiboot_info_t *mbi;
- module_t *mod;
- unsigned long nr_pages, raw_max_page, modules_headroom, *module_map;
-@@ -909,6 +912,18 @@ void __init noreturn __start_xen(unsigned long mbi_p)
- /* Sanitise the raw E820 map to produce a final clean version. */
- max_page = raw_max_page = init_e820(memmap_type, &e820_raw);
-
-+ if ( !efi_enabled(EFI_BOOT) )
-+ {
-+ /*
-+ * Supplement the heuristics in l1tf_calculations() by assuming that
-+ * anything referenced in the E820 may be cacheable.
-+ */
-+ l1tf_safe_maddr =
-+ max(l1tf_safe_maddr,
-+ ROUNDUP(e820_raw.map[e820_raw.nr_map - 1].addr +
-+ e820_raw.map[e820_raw.nr_map - 1].size, PAGE_SIZE));
-+ }
-+
- /* Create a temporary copy of the E820 map. */
- memcpy(&boot_e820, &e820, sizeof(e820));
-
-@@ -1494,7 +1509,8 @@ void __init noreturn __start_xen(unsigned long mbi_p)
- else
- {
- set_nr_cpu_ids(max_cpus);
-- max_cpus = nr_cpu_ids;
-+ if ( !max_cpus )
-+ max_cpus = nr_cpu_ids;
- }
-
- if ( xen_guest )
-@@ -1617,16 +1633,30 @@ void __init noreturn __start_xen(unsigned long mbi_p)
- /* Set up node_to_cpumask based on cpu_to_node[]. */
- numa_add_cpu(i);
-
-- if ( (num_online_cpus() < max_cpus) && !cpu_online(i) )
-+ if ( (park_offline_cpus || num_online_cpus() < max_cpus) &&
-+ !cpu_online(i) )
- {
- int ret = cpu_up(i);
- if ( ret != 0 )
- printk("Failed to bring up CPU %u (error %d)\n", i, ret);
-+ else if ( num_online_cpus() > max_cpus ||
-+ (!opt_smt &&
-+ cpu_data[i].compute_unit_id == INVALID_CUID &&
-+ cpumask_weight(per_cpu(cpu_sibling_mask, i)) > 1) )
-+ {
-+ ret = cpu_down(i);
-+ if ( !ret )
-+ ++num_parked;
-+ else
-+ printk("Could not re-offline CPU%u (%d)\n", i, ret);
-+ }
- }
- }
- }
-
- printk("Brought up %ld CPUs\n", (long)num_online_cpus());
-+ if ( num_parked )
-+ printk(XENLOG_INFO "Parked %u CPUs\n", num_parked);
- smp_cpus_done();
-
- do_initcalls();
-diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
-index d4478e6132..7e76cc3d68 100644
---- a/xen/arch/x86/smpboot.c
-+++ b/xen/arch/x86/smpboot.c
-@@ -63,6 +63,8 @@ static cpumask_t scratch_cpu0mask;
- cpumask_t cpu_online_map __read_mostly;
- EXPORT_SYMBOL(cpu_online_map);
-
-+bool __read_mostly park_offline_cpus;
-+
- unsigned int __read_mostly nr_sockets;
- cpumask_t **__read_mostly socket_cpumask;
- static cpumask_t *secondary_socket_cpumask;
-@@ -234,33 +236,41 @@ static void link_thread_siblings(int cpu1, int cpu2)
- cpumask_set_cpu(cpu2, per_cpu(cpu_core_mask, cpu1));
- }
-
--static void set_cpu_sibling_map(int cpu)
-+static void set_cpu_sibling_map(unsigned int cpu)
- {
-- int i;
-+ unsigned int i;
- struct cpuinfo_x86 *c = cpu_data;
-
- cpumask_set_cpu(cpu, &cpu_sibling_setup_map);
-
- cpumask_set_cpu(cpu, socket_cpumask[cpu_to_socket(cpu)]);
-+ cpumask_set_cpu(cpu, per_cpu(cpu_core_mask, cpu));
-+ cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu));
-
- if ( c[cpu].x86_num_siblings > 1 )
- {
- for_each_cpu ( i, &cpu_sibling_setup_map )
- {
-- if ( cpu_has(c, X86_FEATURE_TOPOEXT) ) {
-- if ( (c[cpu].phys_proc_id == c[i].phys_proc_id) &&
-- (c[cpu].compute_unit_id == c[i].compute_unit_id) )
-+ if ( cpu == i || c[cpu].phys_proc_id != c[i].phys_proc_id )
-+ continue;
-+ if ( c[cpu].compute_unit_id != INVALID_CUID &&
-+ c[i].compute_unit_id != INVALID_CUID )
-+ {
-+ if ( c[cpu].compute_unit_id == c[i].compute_unit_id )
- link_thread_siblings(cpu, i);
-- } else if ( (c[cpu].phys_proc_id == c[i].phys_proc_id) &&
-- (c[cpu].cpu_core_id == c[i].cpu_core_id) ) {
-- link_thread_siblings(cpu, i);
- }
-+ else if ( c[cpu].cpu_core_id != XEN_INVALID_CORE_ID &&
-+ c[i].cpu_core_id != XEN_INVALID_CORE_ID )
-+ {
-+ if ( c[cpu].cpu_core_id == c[i].cpu_core_id )
-+ link_thread_siblings(cpu, i);
-+ }
-+ else
-+ printk(XENLOG_WARNING
-+ "CPU%u: unclear relationship with CPU%u\n",
-+ cpu, i);
- }
- }
-- else
-- {
-- cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu));
-- }
-
- if ( c[cpu].x86_max_cores == 1 )
- {
-@@ -887,7 +897,14 @@ static void cleanup_cpu_root_pgt(unsigned int cpu)
- }
- }
-
--static void cpu_smpboot_free(unsigned int cpu)
-+/*
-+ * The 'remove' boolean controls whether a CPU is just getting offlined (and
-+ * parked), or outright removed / offlined without parking. Parked CPUs need
-+ * things like their stack, GDT, IDT, TSS, and per-CPU data still available.
-+ * A few other items, in particular CPU masks, are also retained, as it's
-+ * difficult to prove that they're entirely unreferenced from parked CPUs.
-+ */
-+static void cpu_smpboot_free(unsigned int cpu, bool remove)
- {
- unsigned int order, socket = cpu_to_socket(cpu);
- struct cpuinfo_x86 *c = cpu_data;
-@@ -898,15 +915,19 @@ static void cpu_smpboot_free(unsigned int cpu)
- socket_cpumask[socket] = NULL;
- }
-
-- c[cpu].phys_proc_id = XEN_INVALID_SOCKET_ID;
-- c[cpu].cpu_core_id = XEN_INVALID_CORE_ID;
-- c[cpu].compute_unit_id = INVALID_CUID;
- cpumask_clear_cpu(cpu, &cpu_sibling_setup_map);
-
-- free_cpumask_var(per_cpu(cpu_sibling_mask, cpu));
-- free_cpumask_var(per_cpu(cpu_core_mask, cpu));
-- if ( per_cpu(scratch_cpumask, cpu) != &scratch_cpu0mask )
-- free_cpumask_var(per_cpu(scratch_cpumask, cpu));
-+ if ( remove )
-+ {
-+ c[cpu].phys_proc_id = XEN_INVALID_SOCKET_ID;
-+ c[cpu].cpu_core_id = XEN_INVALID_CORE_ID;
-+ c[cpu].compute_unit_id = INVALID_CUID;
-+
-+ FREE_CPUMASK_VAR(per_cpu(cpu_sibling_mask, cpu));
-+ FREE_CPUMASK_VAR(per_cpu(cpu_core_mask, cpu));
-+ if ( per_cpu(scratch_cpumask, cpu) != &scratch_cpu0mask )
-+ FREE_CPUMASK_VAR(per_cpu(scratch_cpumask, cpu));
-+ }
-
- cleanup_cpu_root_pgt(cpu);
-
-@@ -928,19 +949,21 @@ static void cpu_smpboot_free(unsigned int cpu)
- }
-
- order = get_order_from_pages(NR_RESERVED_GDT_PAGES);
-- free_xenheap_pages(per_cpu(gdt_table, cpu), order);
-+ if ( remove )
-+ FREE_XENHEAP_PAGES(per_cpu(gdt_table, cpu), order);
-
- free_xenheap_pages(per_cpu(compat_gdt_table, cpu), order);
-
-- order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t));
-- free_xenheap_pages(idt_tables[cpu], order);
-- idt_tables[cpu] = NULL;
--
-- if ( stack_base[cpu] != NULL )
-+ if ( remove )
- {
-- memguard_unguard_stack(stack_base[cpu]);
-- free_xenheap_pages(stack_base[cpu], STACK_ORDER);
-- stack_base[cpu] = NULL;
-+ order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t));
-+ FREE_XENHEAP_PAGES(idt_tables[cpu], order);
-+
-+ if ( stack_base[cpu] )
-+ {
-+ memguard_unguard_stack(stack_base[cpu]);
-+ FREE_XENHEAP_PAGES(stack_base[cpu], STACK_ORDER);
-+ }
- }
- }
-
-@@ -955,15 +978,17 @@ static int cpu_smpboot_alloc(unsigned int cpu)
- if ( node != NUMA_NO_NODE )
- memflags = MEMF_node(node);
-
-- stack_base[cpu] = alloc_xenheap_pages(STACK_ORDER, memflags);
-+ if ( stack_base[cpu] == NULL )
-+ stack_base[cpu] = alloc_xenheap_pages(STACK_ORDER, memflags);
- if ( stack_base[cpu] == NULL )
- goto out;
- memguard_guard_stack(stack_base[cpu]);
-
- order = get_order_from_pages(NR_RESERVED_GDT_PAGES);
-- per_cpu(gdt_table, cpu) = gdt = alloc_xenheap_pages(order, memflags);
-+ gdt = per_cpu(gdt_table, cpu) ?: alloc_xenheap_pages(order, memflags);
- if ( gdt == NULL )
- goto out;
-+ per_cpu(gdt_table, cpu) = gdt;
- memcpy(gdt, boot_cpu_gdt_table, NR_RESERVED_GDT_PAGES * PAGE_SIZE);
- BUILD_BUG_ON(NR_CPUS > 0x10000);
- gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu;
-@@ -975,7 +1000,8 @@ static int cpu_smpboot_alloc(unsigned int cpu)
- gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu;
-
- order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t));
-- idt_tables[cpu] = alloc_xenheap_pages(order, memflags);
-+ if ( idt_tables[cpu] == NULL )
-+ idt_tables[cpu] = alloc_xenheap_pages(order, memflags);
- if ( idt_tables[cpu] == NULL )
- goto out;
- memcpy(idt_tables[cpu], idt_table, IDT_ENTRIES * sizeof(idt_entry_t));
-@@ -1003,16 +1029,16 @@ static int cpu_smpboot_alloc(unsigned int cpu)
- (secondary_socket_cpumask = xzalloc(cpumask_t)) == NULL )
- goto out;
-
-- if ( !(zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) &&
-- zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) &&
-- alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu))) )
-+ if ( !(cond_zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) &&
-+ cond_zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) &&
-+ cond_alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu))) )
- goto out;
-
- rc = 0;
-
- out:
- if ( rc )
-- cpu_smpboot_free(cpu);
-+ cpu_smpboot_free(cpu, true);
-
- return rc;
- }
-@@ -1030,9 +1056,10 @@ static int cpu_smpboot_callback(
- break;
- case CPU_UP_CANCELED:
- case CPU_DEAD:
-- cpu_smpboot_free(cpu);
-+ cpu_smpboot_free(cpu, !park_offline_cpus);
- break;
-- default:
-+ case CPU_REMOVE:
-+ cpu_smpboot_free(cpu, true);
- break;
- }
-
-diff --git a/xen/arch/x86/spec_ctrl.c b/xen/arch/x86/spec_ctrl.c
-index 08e6784c4c..f0c50d6703 100644
---- a/xen/arch/x86/spec_ctrl.c
-+++ b/xen/arch/x86/spec_ctrl.c
-@@ -19,10 +19,13 @@
- #include <xen/errno.h>
- #include <xen/init.h>
- #include <xen/lib.h>
-+#include <xen/warning.h>
-
- #include <asm/microcode.h>
- #include <asm/msr.h>
- #include <asm/processor.h>
-+#include <asm/pv/shim.h>
-+#include <asm/setup.h>
- #include <asm/spec_ctrl.h>
- #include <asm/spec_ctrl_asm.h>
-
-@@ -45,11 +48,16 @@ static int8_t __initdata opt_ibrs = -1;
- bool __read_mostly opt_ibpb = true;
- bool __read_mostly opt_ssbd = false;
- int8_t __read_mostly opt_eager_fpu = -1;
-+int8_t __read_mostly opt_l1d_flush = -1;
-
- bool __initdata bsp_delay_spec_ctrl;
- uint8_t __read_mostly default_xen_spec_ctrl;
- uint8_t __read_mostly default_spec_ctrl_flags;
-
-+paddr_t __read_mostly l1tf_addr_mask, __read_mostly l1tf_safe_maddr;
-+static bool __initdata cpu_has_bug_l1tf;
-+static unsigned int __initdata l1d_maxphysaddr;
-+
- static int __init parse_bti(const char *s)
- {
- const char *ss;
-@@ -124,6 +132,17 @@ static int __init parse_spec_ctrl(const char *s)
- opt_msr_sc_pv = false;
- opt_msr_sc_hvm = false;
-
-+ opt_eager_fpu = 0;
-+
-+ if ( opt_xpti < 0 )
-+ opt_xpti = 0;
-+
-+ if ( opt_smt < 0 )
-+ opt_smt = 1;
-+
-+ if ( opt_pv_l1tf < 0 )
-+ opt_pv_l1tf = 0;
-+
- disable_common:
- opt_rsb_pv = false;
- opt_rsb_hvm = false;
-@@ -131,7 +150,8 @@ static int __init parse_spec_ctrl(const char *s)
- opt_thunk = THUNK_JMP;
- opt_ibrs = 0;
- opt_ibpb = false;
-- opt_eager_fpu = 0;
-+ opt_ssbd = false;
-+ opt_l1d_flush = 0;
- }
- else if ( val > 0 )
- rc = -EINVAL;
-@@ -187,6 +207,8 @@ static int __init parse_spec_ctrl(const char *s)
- opt_ssbd = val;
- else if ( (val = parse_boolean("eager-fpu", s, ss)) >= 0 )
- opt_eager_fpu = val;
-+ else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 )
-+ opt_l1d_flush = val;
- else
- rc = -EINVAL;
-
-@@ -197,6 +219,55 @@ static int __init parse_spec_ctrl(const char *s)
- }
- custom_param("spec-ctrl", parse_spec_ctrl);
-
-+int8_t __read_mostly opt_pv_l1tf = -1;
-+
-+static __init int parse_pv_l1tf(const char *s)
-+{
-+ const char *ss;
-+ int val, rc = 0;
-+
-+ /* Inhibit the defaults as an explicit choice has been given. */
-+ if ( opt_pv_l1tf == -1 )
-+ opt_pv_l1tf = 0;
-+
-+ /* Interpret 'pv-l1tf' alone in its positive boolean form. */
-+ if ( *s == '\0' )
-+ opt_xpti = OPT_PV_L1TF_DOM0 | OPT_PV_L1TF_DOMU;
-+
-+ do {
-+ ss = strchr(s, ',');
-+ if ( !ss )
-+ ss = strchr(s, '\0');
-+
-+ switch ( parse_bool(s, ss) )
-+ {
-+ case 0:
-+ opt_pv_l1tf = 0;
-+ break;
-+
-+ case 1:
-+ opt_pv_l1tf = OPT_PV_L1TF_DOM0 | OPT_PV_L1TF_DOMU;
-+ break;
-+
-+ default:
-+ if ( (val = parse_boolean("dom0", s, ss)) >= 0 )
-+ opt_pv_l1tf = ((opt_pv_l1tf & ~OPT_PV_L1TF_DOM0) |
-+ (val ? OPT_PV_L1TF_DOM0 : 0));
-+ else if ( (val = parse_boolean("domu", s, ss)) >= 0 )
-+ opt_pv_l1tf = ((opt_pv_l1tf & ~OPT_PV_L1TF_DOMU) |
-+ (val ? OPT_PV_L1TF_DOMU : 0));
-+ else
-+ rc = -EINVAL;
-+ break;
-+ }
-+
-+ s = ss + 1;
-+ } while ( *ss );
-+
-+ return rc;
-+}
-+custom_param("pv-l1tf", parse_pv_l1tf);
-+
- static void __init print_details(enum ind_thunk thunk, uint64_t caps)
- {
- unsigned int _7d0 = 0, e8b = 0, tmp;
-@@ -210,22 +281,31 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps)
- printk("Speculative mitigation facilities:\n");
-
- /* Hardware features which pertain to speculative mitigations. */
-- printk(" Hardware features:%s%s%s%s%s%s%s%s\n",
-+ printk(" Hardware features:%s%s%s%s%s%s%s%s%s%s\n",
- (_7d0 & cpufeat_mask(X86_FEATURE_IBRSB)) ? " IBRS/IBPB" : "",
- (_7d0 & cpufeat_mask(X86_FEATURE_STIBP)) ? " STIBP" : "",
-+ (_7d0 & cpufeat_mask(X86_FEATURE_L1D_FLUSH)) ? " L1D_FLUSH" : "",
- (_7d0 & cpufeat_mask(X86_FEATURE_SSBD)) ? " SSBD" : "",
- (e8b & cpufeat_mask(X86_FEATURE_IBPB)) ? " IBPB" : "",
- (caps & ARCH_CAPABILITIES_IBRS_ALL) ? " IBRS_ALL" : "",
- (caps & ARCH_CAPABILITIES_RDCL_NO) ? " RDCL_NO" : "",
- (caps & ARCH_CAPS_RSBA) ? " RSBA" : "",
-+ (caps & ARCH_CAPS_SKIP_L1DFL) ? " SKIP_L1DFL": "",
- (caps & ARCH_CAPS_SSB_NO) ? " SSB_NO" : "");
-
-- /* Compiled-in support which pertains to BTI mitigations. */
-- if ( IS_ENABLED(CONFIG_INDIRECT_THUNK) )
-- printk(" Compiled-in support: INDIRECT_THUNK\n");
-+ /* Compiled-in support which pertains to mitigations. */
-+ if ( IS_ENABLED(CONFIG_INDIRECT_THUNK) || IS_ENABLED(CONFIG_SHADOW_PAGING) )
-+ printk(" Compiled-in support:"
-+#ifdef CONFIG_INDIRECT_THUNK
-+ " INDIRECT_THUNK"
-+#endif
-+#ifdef CONFIG_SHADOW_PAGING
-+ " SHADOW_PAGING"
-+#endif
-+ "\n");
-
- /* Settings for Xen's protection, irrespective of guests. */
-- printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s\n",
-+ printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s%s\n",
- thunk == THUNK_NONE ? "N/A" :
- thunk == THUNK_RETPOLINE ? "RETPOLINE" :
- thunk == THUNK_LFENCE ? "LFENCE" :
-@@ -234,7 +314,15 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps)
- (default_xen_spec_ctrl & SPEC_CTRL_IBRS) ? "IBRS+" : "IBRS-",
- !boot_cpu_has(X86_FEATURE_SSBD) ? "" :
- (default_xen_spec_ctrl & SPEC_CTRL_SSBD) ? " SSBD+" : " SSBD-",
-- opt_ibpb ? " IBPB" : "");
-+ opt_ibpb ? " IBPB" : "",
-+ opt_l1d_flush ? " L1D_FLUSH" : "");
-+
-+ /* L1TF diagnostics, printed if vulnerable or PV shadowing is in use. */
-+ if ( cpu_has_bug_l1tf || opt_pv_l1tf )
-+ printk(" L1TF: believed%s vulnerable, maxphysaddr L1D %u, CPUID %u"
-+ ", Safe address %"PRIx64"\n",
-+ cpu_has_bug_l1tf ? "" : " not",
-+ l1d_maxphysaddr, paddr_bits, l1tf_safe_maddr);
-
- /*
- * Alternatives blocks for protecting against and/or virtualising
-@@ -257,6 +345,10 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps)
- printk(" XPTI (64-bit PV only): Dom0 %s, DomU %s\n",
- opt_xpti & OPT_XPTI_DOM0 ? "enabled" : "disabled",
- opt_xpti & OPT_XPTI_DOMU ? "enabled" : "disabled");
-+
-+ printk(" PV L1TF shadowing: Dom0 %s, DomU %s\n",
-+ opt_pv_l1tf & OPT_PV_L1TF_DOM0 ? "enabled" : "disabled",
-+ opt_pv_l1tf & OPT_PV_L1TF_DOMU ? "enabled" : "disabled");
- }
-
- /* Calculate whether Retpoline is known-safe on this CPU. */
-@@ -418,20 +510,159 @@ static bool __init should_use_eager_fpu(void)
- }
- }
-
--#define OPT_XPTI_DEFAULT 0xff
--uint8_t __read_mostly opt_xpti = OPT_XPTI_DEFAULT;
--
--static __init void xpti_init_default(bool force)
-+/* Calculate whether this CPU is vulnerable to L1TF. */
-+static __init void l1tf_calculations(uint64_t caps)
- {
-- uint64_t caps = 0;
-+ bool hit_default = false;
-+
-+ l1d_maxphysaddr = paddr_bits;
-+
-+ /* L1TF is only known to affect Intel Family 6 processors at this time. */
-+ if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
-+ boot_cpu_data.x86 == 6 )
-+ {
-+ switch ( boot_cpu_data.x86_model )
-+ {
-+ /*
-+ * Core processors since at least Penryn are vulnerable.
-+ */
-+ case 0x17: /* Penryn */
-+ case 0x1d: /* Dunnington */
-+ cpu_has_bug_l1tf = true;
-+ break;
-+
-+ case 0x1f: /* Auburndale / Havendale */
-+ case 0x1e: /* Nehalem */
-+ case 0x1a: /* Nehalem EP */
-+ case 0x2e: /* Nehalem EX */
-+ case 0x25: /* Westmere */
-+ case 0x2c: /* Westmere EP */
-+ case 0x2f: /* Westmere EX */
-+ cpu_has_bug_l1tf = true;
-+ l1d_maxphysaddr = 44;
-+ break;
-+
-+ case 0x2a: /* SandyBridge */
-+ case 0x2d: /* SandyBridge EP/EX */
-+ case 0x3a: /* IvyBridge */
-+ case 0x3e: /* IvyBridge EP/EX */
-+ case 0x3c: /* Haswell */
-+ case 0x3f: /* Haswell EX/EP */
-+ case 0x45: /* Haswell D */
-+ case 0x46: /* Haswell H */
-+ case 0x3d: /* Broadwell */
-+ case 0x47: /* Broadwell H */
-+ case 0x4f: /* Broadwell EP/EX */
-+ case 0x56: /* Broadwell D */
-+ case 0x4e: /* Skylake M */
-+ case 0x55: /* Skylake X */
-+ case 0x5e: /* Skylake D */
-+ case 0x66: /* Cannonlake */
-+ case 0x67: /* Cannonlake? */
-+ case 0x8e: /* Kabylake M */
-+ case 0x9e: /* Kabylake D */
-+ cpu_has_bug_l1tf = true;
-+ l1d_maxphysaddr = 46;
-+ break;
-+
-+ /*
-+ * Atom processors are not vulnerable.
-+ */
-+ case 0x1c: /* Pineview */
-+ case 0x26: /* Lincroft */
-+ case 0x27: /* Penwell */
-+ case 0x35: /* Cloverview */
-+ case 0x36: /* Cedarview */
-+ case 0x37: /* Baytrail / Valleyview (Silvermont) */
-+ case 0x4d: /* Avaton / Rangely (Silvermont) */
-+ case 0x4c: /* Cherrytrail / Brasswell */
-+ case 0x4a: /* Merrifield */
-+ case 0x5a: /* Moorefield */
-+ case 0x5c: /* Goldmont */
-+ case 0x5f: /* Denverton */
-+ case 0x7a: /* Gemini Lake */
-+ break;
-+
-+ /*
-+ * Knights processors are not vulnerable.
-+ */
-+ case 0x57: /* Knights Landing */
-+ case 0x85: /* Knights Mill */
-+ break;
-+
-+ default:
-+ /* Defer printk() until we've accounted for RDCL_NO. */
-+ hit_default = true;
-+ cpu_has_bug_l1tf = true;
-+ break;
-+ }
-+ }
-+
-+ /* Any processor advertising RDCL_NO should be not vulnerable to L1TF. */
-+ if ( caps & ARCH_CAPABILITIES_RDCL_NO )
-+ cpu_has_bug_l1tf = false;
-+
-+ if ( cpu_has_bug_l1tf && hit_default )
-+ printk("Unrecognised CPU model %#x - assuming vulnerable to L1TF\n",
-+ boot_cpu_data.x86_model);
-+
-+ /*
-+ * L1TF safe address heuristics. These apply to the real hardware we are
-+ * running on, and are best-effort-only if Xen is virtualised.
-+ *
-+ * The address mask which the L1D cache uses, which might be wider than
-+ * the CPUID-reported maxphysaddr.
-+ */
-+ l1tf_addr_mask = ((1ul << l1d_maxphysaddr) - 1) & PAGE_MASK;
-+
-+ /*
-+ * To be safe, l1tf_safe_maddr must be above the highest cacheable entity
-+ * in system physical address space. However, to preserve space for
-+ * paged-out metadata, it should be as low as possible above the highest
-+ * cacheable address, so as to require fewer high-order bits being set.
-+ *
-+ * These heuristics are based on some guesswork to improve the likelihood
-+ * of safety in the common case, including Linux's L1TF mitigation of
-+ * inverting all address bits in a non-present PTE.
-+ *
-+ * - If L1D is wider than CPUID (Nehalem and later mobile/desktop/low end
-+ * server), setting any address bit beyond CPUID maxphysaddr guarantees
-+ * to make the PTE safe. This case doesn't require all the high-order
-+ * bits being set, and doesn't require any other source of information
-+ * for safety.
-+ *
-+ * - If L1D is the same as CPUID (Pre-Nehalem, or high end server), we
-+ * must sacrifice high order bits from the real address space for
-+ * safety. Therefore, make a blind guess that there is nothing
-+ * cacheable in the top quarter of physical address space.
-+ *
-+ * It is exceedingly unlikely for machines to be populated with this
-+ * much RAM (likely 512G on pre-Nehalem, 16T on Nehalem/Westmere, 64T on
-+ * Sandybridge and later) due to the sheer volume of DIMMs this would
-+ * actually take.
-+ *
-+ * However, it is possible to find machines this large, so the "top
-+ * quarter" guess is supplemented to push the limit higher if references
-+ * to cacheable mappings (E820/SRAT/EFI/etc) are found above the top
-+ * quarter boundary.
-+ *
-+ * Finally, this top quarter guess gives us a good chance of being safe
-+ * when running virtualised (and the CPUID maxphysaddr hasn't been
-+ * levelled for heterogeneous migration safety), where the safety
-+ * consideration is still in terms of host details, but all E820/etc
-+ * information is in terms of guest physical layout.
-+ */
-+ l1tf_safe_maddr = max(l1tf_safe_maddr, ((l1d_maxphysaddr > paddr_bits)
-+ ? (1ul << paddr_bits)
-+ : (3ul << (paddr_bits - 2))));
-+}
-
-- if ( !force && (opt_xpti != OPT_XPTI_DEFAULT) )
-- return;
-+int8_t __read_mostly opt_xpti = -1;
-
-+static __init void xpti_init_default(uint64_t caps)
-+{
- if ( boot_cpu_data.x86_vendor == X86_VENDOR_AMD )
- caps = ARCH_CAPABILITIES_RDCL_NO;
-- else if ( boot_cpu_has(X86_FEATURE_ARCH_CAPS) )
-- rdmsrl(MSR_ARCH_CAPABILITIES, caps);
-
- if ( caps & ARCH_CAPABILITIES_RDCL_NO )
- opt_xpti = 0;
-@@ -444,7 +675,13 @@ static __init int parse_xpti(const char *s)
- const char *ss;
- int val, rc = 0;
-
-- xpti_init_default(false);
-+ /* Inhibit the defaults as an explicit choice has been given. */
-+ if ( opt_xpti == -1 )
-+ opt_xpti = 0;
-+
-+ /* Interpret 'xpti' alone in its positive boolean form. */
-+ if ( *s == '\0' )
-+ opt_xpti = OPT_XPTI_DOM0 | OPT_XPTI_DOMU;
-
- do {
- ss = strchr(s, ',');
-@@ -463,7 +700,7 @@ static __init int parse_xpti(const char *s)
-
- default:
- if ( !strcmp(s, "default") )
-- xpti_init_default(true);
-+ opt_xpti = -1;
- else if ( (val = parse_boolean("dom0", s, ss)) >= 0 )
- opt_xpti = (opt_xpti & ~OPT_XPTI_DOM0) |
- (val ? OPT_XPTI_DOM0 : 0);
-@@ -625,12 +862,58 @@ void __init init_speculation_mitigations(void)
- if ( default_xen_spec_ctrl )
- setup_force_cpu_cap(X86_FEATURE_SC_MSR_IDLE);
-
-- xpti_init_default(false);
-+ if ( opt_xpti == -1 )
-+ xpti_init_default(caps);
-+
- if ( opt_xpti == 0 )
- setup_force_cpu_cap(X86_FEATURE_NO_XPTI);
- else
- setup_clear_cpu_cap(X86_FEATURE_NO_XPTI);
-
-+ l1tf_calculations(caps);
-+
-+ /*
-+ * By default, enable PV domU L1TF mitigations on all L1TF-vulnerable
-+ * hardware, except when running in shim mode.
-+ *
-+ * In shim mode, SHADOW is expected to be compiled out, and a malicious
-+ * guest kernel can only attack the shim Xen, not the host Xen.
-+ */
-+ if ( opt_pv_l1tf == -1 )
-+ {
-+ if ( pv_shim || !cpu_has_bug_l1tf )
-+ opt_pv_l1tf = 0;
-+ else
-+ opt_pv_l1tf = OPT_PV_L1TF_DOMU;
-+ }
-+
-+ /*
-+ * By default, enable L1D_FLUSH on L1TF-vulnerable hardware, unless
-+ * instructed to skip the flush on vmentry by our outer hypervisor.
-+ */
-+ if ( !boot_cpu_has(X86_FEATURE_L1D_FLUSH) )
-+ opt_l1d_flush = 0;
-+ else if ( opt_l1d_flush == -1 )
-+ opt_l1d_flush = cpu_has_bug_l1tf && !(caps & ARCH_CAPS_SKIP_L1DFL);
-+
-+ /*
-+ * We do not disable HT by default on affected hardware.
-+ *
-+ * Firstly, if the user intends to use exclusively PV, or HVM shadow
-+ * guests, HT isn't a concern and should remain fully enabled. Secondly,
-+ * safety for HVM HAP guests can be arranged by the toolstack with core
-+ * parking, pinning or cpupool configurations, including mixed setups.
-+ *
-+ * However, if we are on affected hardware, with HT enabled, and the user
-+ * hasn't explicitly chosen whether to use HT or not, nag them to do so.
-+ */
-+ if ( opt_smt == -1 && cpu_has_bug_l1tf && !pv_shim &&
-+ boot_cpu_data.x86_num_siblings > 1 )
-+ warning_add(
-+ "Booted on L1TF-vulnerable hardware with SMT/Hyperthreading\n"
-+ "enabled. Please assess your configuration and choose an\n"
-+ "explicit 'smt=<bool>' setting. See XSA-273.\n");
-+
- print_details(thunk, caps);
-
- /*
-diff --git a/xen/arch/x86/srat.c b/xen/arch/x86/srat.c
-index 166eb44fe2..2d70b45909 100644
---- a/xen/arch/x86/srat.c
-+++ b/xen/arch/x86/srat.c
-@@ -20,6 +20,7 @@
- #include <xen/pfn.h>
- #include <asm/e820.h>
- #include <asm/page.h>
-+#include <asm/spec_ctrl.h>
-
- static struct acpi_table_slit *__read_mostly acpi_slit;
-
-@@ -284,6 +285,11 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma)
- if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
- return;
-
-+ start = ma->base_address;
-+ end = start + ma->length;
-+ /* Supplement the heuristics in l1tf_calculations(). */
-+ l1tf_safe_maddr = max(l1tf_safe_maddr, ROUNDUP(end, PAGE_SIZE));
-+
- if (num_node_memblks >= NR_NODE_MEMBLKS)
- {
- dprintk(XENLOG_WARNING,
-@@ -292,8 +298,6 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma)
- return;
- }
-
-- start = ma->base_address;
-- end = start + ma->length;
- pxm = ma->proximity_domain;
- if (srat_rev < 2)
- pxm &= 0xff;
-diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
-index 4d372db12b..e704ed7f1c 100644
---- a/xen/arch/x86/sysctl.c
-+++ b/xen/arch/x86/sysctl.c
-@@ -23,6 +23,7 @@
- #include <asm/hvm/hvm.h>
- #include <asm/hvm/support.h>
- #include <asm/processor.h>
-+#include <asm/setup.h>
- #include <asm/smp.h>
- #include <asm/numa.h>
- #include <xen/nodemask.h>
-@@ -48,14 +49,27 @@ static void l3_cache_get(void *arg)
-
- long cpu_up_helper(void *data)
- {
-- int cpu = (unsigned long)data;
-+ unsigned int cpu = (unsigned long)data;
- int ret = cpu_up(cpu);
-+
- if ( ret == -EBUSY )
- {
- /* On EBUSY, flush RCU work and have one more go. */
- rcu_barrier();
- ret = cpu_up(cpu);
- }
-+
-+ if ( !ret && !opt_smt &&
-+ cpu_data[cpu].compute_unit_id == INVALID_CUID &&
-+ cpumask_weight(per_cpu(cpu_sibling_mask, cpu)) > 1 )
-+ {
-+ ret = cpu_down_helper(data);
-+ if ( ret )
-+ printk("Could not re-offline CPU%u (%d)\n", cpu, ret);
-+ else
-+ ret = -EPERM;
-+ }
-+
- return ret;
- }
-
-diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
-index 9f045a2045..789d7ff8cd 100644
---- a/xen/arch/x86/traps.c
-+++ b/xen/arch/x86/traps.c
-@@ -96,8 +96,6 @@ string_param("nmi", opt_nmi);
- DEFINE_PER_CPU(uint64_t, efer);
- static DEFINE_PER_CPU(unsigned long, last_extable_addr);
-
--DEFINE_PER_CPU_READ_MOSTLY(u32, ler_msr);
--
- DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, gdt_table);
- DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, compat_gdt_table);
-
-@@ -117,6 +115,9 @@ integer_param("debug_stack_lines", debug_stack_lines);
- static bool opt_ler;
- boolean_param("ler", opt_ler);
-
-+/* LastExceptionFromIP on this hardware. Zero if LER is not in use. */
-+unsigned int __read_mostly ler_msr;
-+
- #define stack_words_per_line 4
- #define ESP_BEFORE_EXCEPTION(regs) ((unsigned long *)regs->rsp)
-
-@@ -1778,17 +1779,6 @@ void do_device_not_available(struct cpu_user_regs *regs)
- return;
- }
-
--static void ler_enable(void)
--{
-- u64 debugctl;
--
-- if ( !this_cpu(ler_msr) )
-- return;
--
-- rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
-- wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl | IA32_DEBUGCTLMSR_LBR);
--}
--
- void do_debug(struct cpu_user_regs *regs)
- {
- unsigned long dr6;
-@@ -1821,6 +1811,10 @@ void do_debug(struct cpu_user_regs *regs)
- */
- write_debugreg(6, X86_DR6_DEFAULT);
-
-+ /* #DB automatically disabled LBR. Reinstate it if debugging Xen. */
-+ if ( cpu_has_xen_lbr )
-+ wrmsrl(MSR_IA32_DEBUGCTLMSR, IA32_DEBUGCTLMSR_LBR);
-+
- if ( !guest_mode(regs) )
- {
- /*
-@@ -1838,7 +1832,7 @@ void do_debug(struct cpu_user_regs *regs)
- {
- if ( regs->rip == (unsigned long)sysenter_eflags_saved )
- regs->eflags &= ~X86_EFLAGS_TF;
-- goto out;
-+ return;
- }
- if ( !debugger_trap_fatal(TRAP_debug, regs) )
- {
-@@ -1895,20 +1889,14 @@ void do_debug(struct cpu_user_regs *regs)
- regs->cs, _p(regs->rip), _p(regs->rip),
- regs->ss, _p(regs->rsp), dr6);
-
-- goto out;
-+ return;
- }
-
- /* Save debug status register where guest OS can peek at it */
- v->arch.debugreg[6] |= (dr6 & ~X86_DR6_DEFAULT);
- v->arch.debugreg[6] &= (dr6 | ~X86_DR6_DEFAULT);
-
-- ler_enable();
- pv_inject_hw_exception(TRAP_debug, X86_EVENT_NO_EC);
-- return;
--
-- out:
-- ler_enable();
-- return;
- }
-
- static void __init noinline __set_intr_gate(unsigned int n,
-@@ -1952,38 +1940,46 @@ void load_TR(void)
- : "=m" (old_gdt) : "rm" (TSS_ENTRY << 3), "m" (tss_gdt) : "memory" );
- }
-
--void percpu_traps_init(void)
-+static unsigned int calc_ler_msr(void)
- {
-- subarch_percpu_traps_init();
--
-- if ( !opt_ler )
-- return;
--
- switch ( boot_cpu_data.x86_vendor )
- {
- case X86_VENDOR_INTEL:
- switch ( boot_cpu_data.x86 )
- {
- case 6:
-- this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP;
-- break;
-+ return MSR_IA32_LASTINTFROMIP;
-+
- case 15:
-- this_cpu(ler_msr) = MSR_P4_LER_FROM_LIP;
-- break;
-+ return MSR_P4_LER_FROM_LIP;
- }
- break;
-+
- case X86_VENDOR_AMD:
- switch ( boot_cpu_data.x86 )
- {
- case 6:
- case 0xf ... 0x17:
-- this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP;
-- break;
-+ return MSR_IA32_LASTINTFROMIP;
- }
- break;
- }
-
-- ler_enable();
-+ return 0;
-+}
-+
-+void percpu_traps_init(void)
-+{
-+ subarch_percpu_traps_init();
-+
-+ if ( !opt_ler )
-+ return;
-+
-+ if ( !ler_msr && (ler_msr = calc_ler_msr()) )
-+ setup_force_cpu_cap(X86_FEATURE_XEN_LBR);
-+
-+ if ( cpu_has_xen_lbr )
-+ wrmsrl(MSR_IA32_DEBUGCTLMSR, IA32_DEBUGCTLMSR_LBR);
- }
-
- void __init init_idt_traps(void)
-diff --git a/xen/arch/x86/x86_64/compat/mm.c b/xen/arch/x86/x86_64/compat/mm.c
-index c2aa6f2fdb..02bc75b91e 100644
---- a/xen/arch/x86/x86_64/compat/mm.c
-+++ b/xen/arch/x86/x86_64/compat/mm.c
-@@ -163,19 +163,6 @@ int compat_arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg)
- return rc;
- }
-
--int compat_update_va_mapping(unsigned int va, u32 lo, u32 hi,
-- unsigned int flags)
--{
-- return do_update_va_mapping(va, lo | ((u64)hi << 32), flags);
--}
--
--int compat_update_va_mapping_otherdomain(unsigned long va, u32 lo, u32 hi,
-- unsigned long flags,
-- domid_t domid)
--{
-- return do_update_va_mapping_otherdomain(va, lo | ((u64)hi << 32), flags, domid);
--}
--
- DEFINE_XEN_GUEST_HANDLE(mmuext_op_compat_t);
-
- int compat_mmuext_op(XEN_GUEST_HANDLE_PARAM(void) arg,
-diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c
-index f7f6928d70..b0401850ef 100644
---- a/xen/arch/x86/x86_64/traps.c
-+++ b/xen/arch/x86/x86_64/traps.c
-@@ -144,11 +144,12 @@ void show_registers(const struct cpu_user_regs *regs)
- printk("CPU: %d\n", smp_processor_id());
- _show_registers(&fault_regs, fault_crs, context, v);
-
-- if ( this_cpu(ler_msr) && !guest_mode(regs) )
-+ if ( ler_msr && !guest_mode(regs) )
- {
- u64 from, to;
-- rdmsrl(this_cpu(ler_msr), from);
-- rdmsrl(this_cpu(ler_msr) + 1, to);
-+
-+ rdmsrl(ler_msr, from);
-+ rdmsrl(ler_msr + 1, to);
- printk("ler: %016lx -> %016lx\n", from, to);
- }
- }
-diff --git a/xen/arch/x86/xstate.c b/xen/arch/x86/xstate.c
-index b4aea4b50a..15edd5df96 100644
---- a/xen/arch/x86/xstate.c
-+++ b/xen/arch/x86/xstate.c
-@@ -670,12 +670,17 @@ static bool valid_xcr0(u64 xcr0)
- return !(xcr0 & X86_XCR0_BNDREGS) == !(xcr0 & X86_XCR0_BNDCSR);
- }
-
--int validate_xstate(u64 xcr0, u64 xcr0_accum, const struct xsave_hdr *hdr)
-+int validate_xstate(const struct domain *d, uint64_t xcr0, uint64_t xcr0_accum,
-+ const struct xsave_hdr *hdr)
- {
-+ const struct cpuid_policy *cp = d->arch.cpuid;
-+ uint64_t xcr0_max =
-+ ((uint64_t)cp->xstate.xcr0_high << 32) | cp->xstate.xcr0_low;
- unsigned int i;
-
- if ( (hdr->xstate_bv & ~xcr0_accum) ||
- (xcr0 & ~xcr0_accum) ||
-+ (xcr0_accum & ~xcr0_max) ||
- !valid_xcr0(xcr0) ||
- !valid_xcr0(xcr0_accum) )
- return -EINVAL;
-@@ -694,20 +699,40 @@ int validate_xstate(u64 xcr0, u64 xcr0_accum, const struct xsave_hdr *hdr)
- int handle_xsetbv(u32 index, u64 new_bv)
- {
- struct vcpu *curr = current;
-+ const struct cpuid_policy *cp = curr->domain->arch.cpuid;
-+ uint64_t xcr0_max =
-+ ((uint64_t)cp->xstate.xcr0_high << 32) | cp->xstate.xcr0_low;
- u64 mask;
-
- if ( index != XCR_XFEATURE_ENABLED_MASK )
- return -EOPNOTSUPP;
-
-- if ( (new_bv & ~xfeature_mask) || !valid_xcr0(new_bv) )
-+ /*
-+ * The CPUID logic shouldn't be able to hand out an XCR0 exceeding Xen's
-+ * maximum features, but keep the check for robustness.
-+ */
-+ if ( unlikely(xcr0_max & ~xfeature_mask) )
-+ {
-+ gprintk(XENLOG_ERR,
-+ "xcr0_max %016" PRIx64 " exceeds hardware max %016" PRIx64 "\n",
-+ xcr0_max, xfeature_mask);
-+ domain_crash(curr->domain);
-+
-+ return -EINVAL;
-+ }
-+
-+ if ( (new_bv & ~xcr0_max) || !valid_xcr0(new_bv) )
- return -EINVAL;
-
-- /* XCR0.PKRU is disabled on PV mode. */
-- if ( is_pv_vcpu(curr) && (new_bv & X86_XCR0_PKRU) )
-- return -EOPNOTSUPP;
-+ /* By this point, new_bv really should be accepted by hardware. */
-+ if ( unlikely(!set_xcr0(new_bv)) )
-+ {
-+ gprintk(XENLOG_ERR, "new_bv %016" PRIx64 " rejected by hardware\n",
-+ new_bv);
-+ domain_crash(curr->domain);
-
-- if ( !set_xcr0(new_bv) )
- return -EFAULT;
-+ }
-
- mask = new_bv & ~curr->arch.xcr0_accum;
- curr->arch.xcr0 = new_bv;
-diff --git a/xen/common/cpu.c b/xen/common/cpu.c
-index 6350f150bd..653a56b840 100644
---- a/xen/common/cpu.c
-+++ b/xen/common/cpu.c
-@@ -67,12 +67,17 @@ void __init register_cpu_notifier(struct notifier_block *nb)
- spin_unlock(&cpu_add_remove_lock);
- }
-
--static int take_cpu_down(void *unused)
-+static void _take_cpu_down(void *unused)
- {
- void *hcpu = (void *)(long)smp_processor_id();
- int notifier_rc = notifier_call_chain(&cpu_chain, CPU_DYING, hcpu, NULL);
- BUG_ON(notifier_rc != NOTIFY_DONE);
- __cpu_disable();
-+}
-+
-+static int take_cpu_down(void *arg)
-+{
-+ _take_cpu_down(arg);
- return 0;
- }
-
-@@ -98,7 +103,9 @@ int cpu_down(unsigned int cpu)
- goto fail;
- }
-
-- if ( (err = stop_machine_run(take_cpu_down, NULL, cpu)) < 0 )
-+ if ( unlikely(system_state < SYS_STATE_active) )
-+ on_selected_cpus(cpumask_of(cpu), _take_cpu_down, NULL, true);
-+ else if ( (err = stop_machine_run(take_cpu_down, NULL, cpu)) < 0 )
- goto fail;
-
- __cpu_die(cpu);
-diff --git a/xen/common/cpupool.c b/xen/common/cpupool.c
-index 999839444e..1e8edcbd57 100644
---- a/xen/common/cpupool.c
-+++ b/xen/common/cpupool.c
-@@ -490,7 +490,7 @@ static int cpupool_cpu_add(unsigned int cpu)
- cpumask_clear_cpu(cpu, &cpupool_locked_cpus);
- cpumask_set_cpu(cpu, &cpupool_free_cpus);
-
-- if ( system_state == SYS_STATE_resume )
-+ if ( system_state == SYS_STATE_suspend || system_state == SYS_STATE_resume )
- {
- struct cpupool **c;
-
-@@ -522,6 +522,7 @@ static int cpupool_cpu_add(unsigned int cpu)
- * (or unplugging would have failed) and that is the default behavior
- * anyway.
- */
-+ per_cpu(cpupool, cpu) = NULL;
- ret = cpupool_assign_cpu_locked(cpupool0, cpu);
- }
- out:
-diff --git a/xen/common/efi/boot.c b/xen/common/efi/boot.c
-index 64d12685d3..6be0b3986f 100644
---- a/xen/common/efi/boot.c
-+++ b/xen/common/efi/boot.c
-@@ -1304,6 +1304,8 @@ efi_start(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable)
-
- #ifndef CONFIG_ARM /* TODO - runtime service support */
-
-+#include <asm/spec_ctrl.h>
-+
- static bool __initdata efi_map_uc;
-
- static int __init parse_efi_param(const char *s)
-@@ -1419,6 +1421,16 @@ void __init efi_init_memory(void)
- desc->PhysicalStart, desc->PhysicalStart + len - 1,
- desc->Type, desc->Attribute);
-
-+ if ( (desc->Attribute & (EFI_MEMORY_WB | EFI_MEMORY_WT)) ||
-+ (efi_bs_revision >= EFI_REVISION(2, 5) &&
-+ (desc->Attribute & EFI_MEMORY_WP)) )
-+ {
-+ /* Supplement the heuristics in l1tf_calculations(). */
-+ l1tf_safe_maddr =
-+ max(l1tf_safe_maddr,
-+ ROUNDUP(desc->PhysicalStart + len, PAGE_SIZE));
-+ }
-+
- if ( !efi_enabled(EFI_RS) ||
- (!(desc->Attribute & EFI_MEMORY_RUNTIME) &&
- (!map_bs ||
-diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c
-index c757b7f6f5..231ecf509a 100644
---- a/xen/common/grant_table.c
-+++ b/xen/common/grant_table.c
-@@ -97,7 +97,11 @@ static unsigned int __read_mostly max_maptrack_frames =
- DEFAULT_MAX_MAPTRACK_FRAMES;
- integer_runtime_param("gnttab_max_maptrack_frames", max_maptrack_frames);
-
--static unsigned int __read_mostly opt_gnttab_max_version = 2;
-+#ifndef GNTTAB_MAX_VERSION
-+#define GNTTAB_MAX_VERSION 2
-+#endif
-+
-+static unsigned int __read_mostly opt_gnttab_max_version = GNTTAB_MAX_VERSION;
- static bool __read_mostly opt_transitive_grants = true;
-
- static int __init parse_gnttab(const char *s)
-diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c
-index 20ee1e4897..02aeed7c47 100644
---- a/xen/common/page_alloc.c
-+++ b/xen/common/page_alloc.c
-@@ -1426,7 +1426,7 @@ static void free_heap_pages(
-
- page_list_del(predecessor, &heap(node, zone, order));
-
-- /* Keep predecessor's first_dirty if it is already set. */
-+ /* Update predecessor's first_dirty if necessary. */
- if ( predecessor->u.free.first_dirty == INVALID_DIRTY_IDX &&
- pg->u.free.first_dirty != INVALID_DIRTY_IDX )
- predecessor->u.free.first_dirty = (1U << order) +
-@@ -1447,6 +1447,12 @@ static void free_heap_pages(
-
- check_and_stop_scrub(successor);
-
-+ /* Update pg's first_dirty if necessary. */
-+ if ( pg->u.free.first_dirty == INVALID_DIRTY_IDX &&
-+ successor->u.free.first_dirty != INVALID_DIRTY_IDX )
-+ pg->u.free.first_dirty = (1U << order) +
-+ successor->u.free.first_dirty;
-+
- page_list_del(successor, &heap(node, zone, order));
- }
-
-diff --git a/xen/common/tasklet.c b/xen/common/tasklet.c
-index 0f0a6f8365..d4fea3151c 100644
---- a/xen/common/tasklet.c
-+++ b/xen/common/tasklet.c
-@@ -156,6 +156,10 @@ void tasklet_kill(struct tasklet *t)
-
- spin_lock_irqsave(&tasklet_lock, flags);
-
-+ /* Cope with uninitialised tasklets. */
-+ if ( list_head_is_null(&t->list) )
-+ goto unlock;
-+
- if ( !list_empty(&t->list) )
- {
- BUG_ON(t->is_dead || t->is_running || (t->scheduled_on < 0));
-@@ -172,6 +176,7 @@ void tasklet_kill(struct tasklet *t)
- spin_lock_irqsave(&tasklet_lock, flags);
- }
-
-+ unlock:
- spin_unlock_irqrestore(&tasklet_lock, flags);
- }
-
-diff --git a/xen/include/asm-arm/arm32/system.h b/xen/include/asm-arm/arm32/system.h
-index c617b40438..ab57abfbc5 100644
---- a/xen/include/asm-arm/arm32/system.h
-+++ b/xen/include/asm-arm/arm32/system.h
-@@ -48,6 +48,24 @@ static inline int local_fiq_is_enabled(void)
- return !(flags & PSR_FIQ_MASK);
- }
-
-+#define CSDB ".inst 0xe320f014"
-+
-+static inline unsigned long array_index_mask_nospec(unsigned long idx,
-+ unsigned long sz)
-+{
-+ unsigned long mask;
-+
-+ asm volatile( "cmp %1, %2\n"
-+ "sbc %0, %1, %1\n"
-+ CSDB
-+ : "=r" (mask)
-+ : "r" (idx), "Ir" (sz)
-+ : "cc" );
-+
-+ return mask;
-+}
-+#define array_index_mask_nospec array_index_mask_nospec
-+
- #endif
- /*
- * Local variables:
-diff --git a/xen/include/asm-arm/arm64/system.h b/xen/include/asm-arm/arm64/system.h
-index 2e2ee212a1..2e36573ac6 100644
---- a/xen/include/asm-arm/arm64/system.h
-+++ b/xen/include/asm-arm/arm64/system.h
-@@ -58,6 +58,28 @@ static inline int local_fiq_is_enabled(void)
- return !(flags & PSR_FIQ_MASK);
- }
-
-+#define csdb() asm volatile ( "hint #20" : : : "memory" )
-+
-+/*
-+ * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz
-+ * and 0 otherwise.
-+ */
-+static inline unsigned long array_index_mask_nospec(unsigned long idx,
-+ unsigned long sz)
-+{
-+ unsigned long mask;
-+
-+ asm volatile ( "cmp %1, %2\n"
-+ "sbc %0, xzr, xzr\n"
-+ : "=r" (mask)
-+ : "r" (idx), "Ir" (sz)
-+ : "cc" );
-+ csdb();
-+
-+ return mask;
-+}
-+#define array_index_mask_nospec array_index_mask_nospec
-+
- #endif
- /*
- * Local variables:
-diff --git a/xen/include/asm-arm/grant_table.h b/xen/include/asm-arm/grant_table.h
-index e52936c79f..24958e4670 100644
---- a/xen/include/asm-arm/grant_table.h
-+++ b/xen/include/asm-arm/grant_table.h
-@@ -7,6 +7,7 @@
- #include <xen/sched.h>
-
- #define INITIAL_NR_GRANT_FRAMES 1U
-+#define GNTTAB_MAX_VERSION 1
-
- struct grant_table_arch {
- gfn_t *shared_gfn;
-diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
-index 2cf8f7ea2a..b237da165c 100644
---- a/xen/include/asm-x86/cpufeature.h
-+++ b/xen/include/asm-x86/cpufeature.h
-@@ -113,6 +113,7 @@
- #define cpu_has_aperfmperf boot_cpu_has(X86_FEATURE_APERFMPERF)
- #define cpu_has_lfence_dispatch boot_cpu_has(X86_FEATURE_LFENCE_DISPATCH)
- #define cpu_has_no_xpti boot_cpu_has(X86_FEATURE_NO_XPTI)
-+#define cpu_has_xen_lbr boot_cpu_has(X86_FEATURE_XEN_LBR)
-
- enum _cache_type {
- CACHE_TYPE_NULL = 0,
-diff --git a/xen/include/asm-x86/cpufeatures.h b/xen/include/asm-x86/cpufeatures.h
-index b90aa2d046..8e5cc53dde 100644
---- a/xen/include/asm-x86/cpufeatures.h
-+++ b/xen/include/asm-x86/cpufeatures.h
-@@ -32,3 +32,4 @@ XEN_CPUFEATURE(SC_RSB_PV, (FSCAPINTS+0)*32+18) /* RSB overwrite needed for
- XEN_CPUFEATURE(SC_RSB_HVM, (FSCAPINTS+0)*32+19) /* RSB overwrite needed for HVM */
- XEN_CPUFEATURE(NO_XPTI, (FSCAPINTS+0)*32+20) /* XPTI mitigation not in use */
- XEN_CPUFEATURE(SC_MSR_IDLE, (FSCAPINTS+0)*32+21) /* (SC_MSR_PV || SC_MSR_HVM) && default_xen_spec_ctrl */
-+XEN_CPUFEATURE(XEN_LBR, (FSCAPINTS+0)*32+22) /* Xen uses MSR_DEBUGCTL.LBR */
-diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
-index e0d413c7de..61e6900465 100644
---- a/xen/include/asm-x86/domain.h
-+++ b/xen/include/asm-x86/domain.h
-@@ -121,6 +121,11 @@ struct shadow_domain {
-
- /* Has this domain ever used HVMOP_pagetable_dying? */
- bool_t pagetable_dying_op;
-+
-+#ifdef CONFIG_PV
-+ /* PV L1 Terminal Fault mitigation. */
-+ struct tasklet pv_l1tf_tasklet;
-+#endif /* CONFIG_PV */
- #endif
- };
-
-@@ -257,6 +262,8 @@ struct pv_domain
- bool xpti;
- /* Use PCID feature? */
- bool pcid;
-+ /* Mitigate L1TF with shadow/crashing? */
-+ bool check_l1tf;
-
- /* map_domain_page() mapping cache. */
- struct mapcache_domain mapcache;
-diff --git a/xen/include/asm-x86/hvm/vmx/vmcs.h b/xen/include/asm-x86/hvm/vmx/vmcs.h
-index 06c3179cec..57e5098b99 100644
---- a/xen/include/asm-x86/hvm/vmx/vmcs.h
-+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h
-@@ -130,10 +130,18 @@ struct arch_vmx_struct {
- uint64_t sfmask;
-
- struct vmx_msr_bitmap *msr_bitmap;
-- unsigned int msr_count;
-+
-+ /*
-+ * Most accesses to the MSR host/guest load/save lists are in current
-+ * context. However, the data can be modified by toolstack/migration
-+ * actions. Remote access is only permitted for paused vcpus, and is
-+ * protected under the domctl lock.
-+ */
- struct vmx_msr_entry *msr_area;
-- unsigned int host_msr_count;
- struct vmx_msr_entry *host_msr_area;
-+ unsigned int msr_load_count;
-+ unsigned int msr_save_count;
-+ unsigned int host_msr_count;
-
- unsigned long eoi_exitmap_changed;
- DECLARE_BITMAP(eoi_exit_bitmap, NR_VECTORS);
-@@ -149,7 +157,7 @@ struct arch_vmx_struct {
- /* Are we emulating rather than VMENTERing? */
- uint8_t vmx_emulate;
-
-- uint8_t lbr_fixup_enabled;
-+ uint8_t lbr_flags;
-
- /* Bitmask of segments that we can't safely use in virtual 8086 mode */
- uint16_t vm86_segment_mask;
-@@ -514,9 +522,6 @@ enum vmcs_field {
-
- #define VMCS_VPID_WIDTH 16
-
--#define VMX_GUEST_MSR 0
--#define VMX_HOST_MSR 1
--
- /* VM Instruction error numbers */
- enum vmx_insn_errno
- {
-@@ -534,6 +539,67 @@ enum vmx_insn_errno
- VMX_INSN_FAIL_INVALID = ~0,
- };
-
-+/* MSR load/save list infrastructure. */
-+enum vmx_msr_list_type {
-+ VMX_MSR_HOST, /* MSRs loaded on VMExit. */
-+ VMX_MSR_GUEST, /* MSRs saved on VMExit, loaded on VMEntry. */
-+ VMX_MSR_GUEST_LOADONLY, /* MSRs loaded on VMEntry only. */
-+};
-+
-+/**
-+ * Add an MSR to an MSR list (inserting space for the entry if necessary), and
-+ * set the MSRs value.
-+ *
-+ * It is undefined behaviour to try and insert the same MSR into both the
-+ * GUEST and GUEST_LOADONLY list.
-+ *
-+ * May fail if unable to allocate memory for the list, or the total number of
-+ * entries exceeds the memory allocated.
-+ */
-+int vmx_add_msr(struct vcpu *v, uint32_t msr, uint64_t val,
-+ enum vmx_msr_list_type type);
-+
-+static inline int vmx_add_guest_msr(struct vcpu *v, uint32_t msr, uint64_t val)
-+{
-+ return vmx_add_msr(v, msr, val, VMX_MSR_GUEST);
-+}
-+static inline int vmx_add_host_load_msr(struct vcpu *v, uint32_t msr,
-+ uint64_t val)
-+{
-+ return vmx_add_msr(v, msr, val, VMX_MSR_HOST);
-+}
-+
-+struct vmx_msr_entry *vmx_find_msr(const struct vcpu *v, uint32_t msr,
-+ enum vmx_msr_list_type type);
-+
-+static inline int vmx_read_guest_msr(const struct vcpu *v, uint32_t msr,
-+ uint64_t *val)
-+{
-+ const struct vmx_msr_entry *ent = vmx_find_msr(v, msr, VMX_MSR_GUEST);
-+
-+ if ( !ent )
-+ return -ESRCH;
-+
-+ *val = ent->data;
-+
-+ return 0;
-+}
-+
-+static inline int vmx_write_guest_msr(struct vcpu *v, uint32_t msr,
-+ uint64_t val)
-+{
-+ struct vmx_msr_entry *ent = vmx_find_msr(v, msr, VMX_MSR_GUEST);
-+
-+ if ( !ent )
-+ return -ESRCH;
-+
-+ ent->data = val;
-+
-+ return 0;
-+}
-+
-+
-+/* MSR intercept bitmap infrastructure. */
- enum vmx_msr_intercept_type {
- VMX_MSR_R = 1,
- VMX_MSR_W = 2,
-@@ -544,10 +610,6 @@ void vmx_clear_msr_intercept(struct vcpu *v, unsigned int msr,
- enum vmx_msr_intercept_type type);
- void vmx_set_msr_intercept(struct vcpu *v, unsigned int msr,
- enum vmx_msr_intercept_type type);
--int vmx_read_guest_msr(u32 msr, u64 *val);
--int vmx_write_guest_msr(u32 msr, u64 val);
--struct vmx_msr_entry *vmx_find_msr(u32 msr, int type);
--int vmx_add_msr(u32 msr, int type);
- void vmx_vmcs_switch(paddr_t from, paddr_t to);
- void vmx_set_eoi_exit_bitmap(struct vcpu *v, u8 vector);
- void vmx_clear_eoi_exit_bitmap(struct vcpu *v, u8 vector);
-@@ -562,15 +624,6 @@ void virtual_vmcs_vmwrite(const struct vcpu *, u32 encoding, u64 val);
- enum vmx_insn_errno virtual_vmcs_vmwrite_safe(const struct vcpu *v,
- u32 vmcs_encoding, u64 val);
-
--static inline int vmx_add_guest_msr(u32 msr)
--{
-- return vmx_add_msr(msr, VMX_GUEST_MSR);
--}
--static inline int vmx_add_host_load_msr(u32 msr)
--{
-- return vmx_add_msr(msr, VMX_HOST_MSR);
--}
--
- DECLARE_PER_CPU(bool_t, vmxon);
-
- bool_t vmx_vcpu_pml_enabled(const struct vcpu *v);
-diff --git a/xen/include/asm-x86/hypercall.h b/xen/include/asm-x86/hypercall.h
-index 1cc2e37d5c..da38b7991c 100644
---- a/xen/include/asm-x86/hypercall.h
-+++ b/xen/include/asm-x86/hypercall.h
-@@ -165,7 +165,7 @@ extern int compat_update_va_mapping(
- unsigned int va, u32 lo, u32 hi, unsigned int flags);
-
- extern int compat_update_va_mapping_otherdomain(
-- unsigned long va, u32 lo, u32 hi, unsigned long flags, domid_t domid);
-+ unsigned int va, u32 lo, u32 hi, unsigned int flags, domid_t domid);
-
- DEFINE_XEN_GUEST_HANDLE(trap_info_compat_t);
- extern int compat_set_trap_table(XEN_GUEST_HANDLE(trap_info_compat_t) traps);
-diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
-index 8fbccc88a7..7235623c86 100644
---- a/xen/include/asm-x86/msr-index.h
-+++ b/xen/include/asm-x86/msr-index.h
-@@ -47,8 +47,12 @@
- #define ARCH_CAPABILITIES_RDCL_NO (_AC(1, ULL) << 0)
- #define ARCH_CAPABILITIES_IBRS_ALL (_AC(1, ULL) << 1)
- #define ARCH_CAPS_RSBA (_AC(1, ULL) << 2)
-+#define ARCH_CAPS_SKIP_L1DFL (_AC(1, ULL) << 3)
- #define ARCH_CAPS_SSB_NO (_AC(1, ULL) << 4)
-
-+#define MSR_FLUSH_CMD 0x0000010b
-+#define FLUSH_CMD_L1D (_AC(1, ULL) << 0)
-+
- /* Intel MSRs. Some also available on other CPUs */
- #define MSR_IA32_PERFCTR0 0x000000c1
- #define MSR_IA32_A_PERFCTR0 0x000004c1
-diff --git a/xen/include/asm-x86/msr.h b/xen/include/asm-x86/msr.h
-index f14f265aa5..afbeb7f155 100644
---- a/xen/include/asm-x86/msr.h
-+++ b/xen/include/asm-x86/msr.h
-@@ -241,7 +241,7 @@ static inline void write_efer(uint64_t val)
- wrmsrl(MSR_EFER, val);
- }
-
--DECLARE_PER_CPU(u32, ler_msr);
-+extern unsigned int ler_msr;
-
- DECLARE_PER_CPU(uint32_t, tsc_aux);
-
-diff --git a/xen/include/asm-x86/paging.h b/xen/include/asm-x86/paging.h
-index f0085511c7..f440e3e53c 100644
---- a/xen/include/asm-x86/paging.h
-+++ b/xen/include/asm-x86/paging.h
-@@ -37,11 +37,14 @@
-
- #define PG_SH_shift 20
- #define PG_HAP_shift 21
-+#define PG_SHF_shift 22
- /* We're in one of the shadow modes */
- #ifdef CONFIG_SHADOW_PAGING
- #define PG_SH_enable (1U << PG_SH_shift)
-+#define PG_SH_forced (1U << PG_SHF_shift)
- #else
- #define PG_SH_enable 0
-+#define PG_SH_forced 0
- #endif
- #define PG_HAP_enable (1U << PG_HAP_shift)
-
-@@ -62,6 +65,7 @@
-
- #define paging_mode_enabled(_d) (!!(_d)->arch.paging.mode)
- #define paging_mode_shadow(_d) (!!((_d)->arch.paging.mode & PG_SH_enable))
-+#define paging_mode_sh_forced(_d) (!!((_d)->arch.paging.mode & PG_SH_forced))
- #define paging_mode_hap(_d) (!!((_d)->arch.paging.mode & PG_HAP_enable))
-
- #define paging_mode_refcounts(_d) (!!((_d)->arch.paging.mode & PG_refcounts))
-diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h
-index 9924cdf1f3..2bd9e69684 100644
---- a/xen/include/asm-x86/processor.h
-+++ b/xen/include/asm-x86/processor.h
-@@ -337,12 +337,6 @@ static always_inline void set_in_cr4 (unsigned long mask)
- write_cr4(read_cr4() | mask);
- }
-
--static always_inline void clear_in_cr4 (unsigned long mask)
--{
-- mmu_cr4_features &= ~mask;
-- write_cr4(read_cr4() & ~mask);
--}
--
- static inline unsigned int read_pkru(void)
- {
- unsigned int pkru;
-diff --git a/xen/include/asm-x86/setup.h b/xen/include/asm-x86/setup.h
-index 19232afa01..c09a5ff381 100644
---- a/xen/include/asm-x86/setup.h
-+++ b/xen/include/asm-x86/setup.h
-@@ -66,6 +66,8 @@ extern uint8_t kbd_shift_flags;
- extern unsigned long highmem_start;
- #endif
-
-+extern int8_t opt_smt;
-+
- #ifdef CONFIG_SHADOW_PAGING
- extern bool opt_dom0_shadow;
- #else
-diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h
-index 94a34fd16a..f40f411871 100644
---- a/xen/include/asm-x86/shadow.h
-+++ b/xen/include/asm-x86/shadow.h
-@@ -29,6 +29,7 @@
- #include <asm/flushtlb.h>
- #include <asm/paging.h>
- #include <asm/p2m.h>
-+#include <asm/spec_ctrl.h>
-
- /*****************************************************************************
- * Macros to tell which shadow paging mode a domain is in*/
-@@ -115,6 +116,131 @@ static inline int shadow_domctl(struct domain *d,
-
- #endif /* CONFIG_SHADOW_PAGING */
-
-+/*
-+ * Mitigations for L1TF / CVE-2018-3620 for PV guests.
-+ *
-+ * We cannot alter an architecturally-legitimate PTE which a PV guest has
-+ * chosen to write, as traditional paged-out metadata is L1TF-vulnerable.
-+ * What we can do is force a PV guest which writes a vulnerable PTE into
-+ * shadow mode, so Xen controls the pagetables which are reachable by the CPU
-+ * pagewalk.
-+ *
-+ * The core of the L1TF vulnerability is that the address bits of the PTE
-+ * (accounting for PSE and factoring in the level-relevant part of the linear
-+ * access) are sent for an L1D lookup (to retrieve the next-level PTE, or
-+ * eventual memory address) before the Present or reserved bits (which would
-+ * cause a terminal fault) are accounted for. If an L1D hit occurs, the
-+ * resulting data is available for potentially dependent instructions.
-+ *
-+ * For Present PTEs, the PV type-count safety logic ensures that the address
-+ * bits always point at a guest-accessible frame, which is safe WRT L1TF from
-+ * Xen's point of view. In practice, a PV guest should be unable to set any
-+ * reserved bits, so should be unable to create any present L1TF-vulnerable
-+ * PTEs at all.
-+ *
-+ * Therefore, these safety checks apply to Not-Present PTEs only, where
-+ * traditionally, Xen would have let the guest write any value it chose.
-+ *
-+ * The all-zero PTE potentially leaks mfn 0. All software on the system is
-+ * expected to cooperate and not put any secrets there. In a Xen system,
-+ * neither Xen nor dom0 are expected to touch mfn 0, as it typically contains
-+ * the real mode IVT and Bios Data Area. Therefore, mfn 0 is considered safe.
-+ *
-+ * Any PTE whose address is higher than the maximum cacheable address is safe,
-+ * as it won't get an L1D hit.
-+ *
-+ * Speculative superpages also need accounting for, as PSE is considered
-+ * irrespective of Present. We disallow PSE being set, as it allows an
-+ * attacker to leak 2M or 1G of data starting from mfn 0. Also, because of
-+ * recursive/linear pagetables, we must consider PSE even at L4, as hardware
-+ * will interpret an L4e as an L3e during a recursive walk.
-+ */
-+
-+static inline bool is_l1tf_safe_maddr(intpte_t pte)
-+{
-+ paddr_t maddr = pte & l1tf_addr_mask;
-+
-+ return maddr == 0 || maddr >= l1tf_safe_maddr;
-+}
-+
-+static inline bool pv_l1tf_check_pte(struct domain *d, unsigned int level,
-+ intpte_t pte)
-+{
-+ ASSERT(is_pv_domain(d));
-+ ASSERT(!(pte & _PAGE_PRESENT));
-+
-+ if ( d->arch.pv_domain.check_l1tf && !paging_mode_sh_forced(d) &&
-+ (((level > 1) && (pte & _PAGE_PSE)) || !is_l1tf_safe_maddr(pte)) )
-+ {
-+#ifdef CONFIG_SHADOW_PAGING
-+ struct tasklet *t = &d->arch.paging.shadow.pv_l1tf_tasklet;
-+
-+ printk(XENLOG_G_WARNING
-+ "d%d L1TF-vulnerable L%ue %016"PRIx64" - Shadowing\n",
-+ d->domain_id, level, pte);
-+ /*
-+ * Safety consideration for accessing tasklet.scheduled_on without the
-+ * tasklet lock. This is a singleshot tasklet with the side effect of
-+ * setting PG_SH_forced (checked just above). Multiple vcpus can race
-+ * to schedule the tasklet, but if we observe it scheduled anywhere,
-+ * that is good enough.
-+ */
-+ smp_rmb();
-+ if ( !tasklet_is_scheduled(t) )
-+ tasklet_schedule(t);
-+#else
-+ printk(XENLOG_G_ERR
-+ "d%d L1TF-vulnerable L%ue %016"PRIx64" - Crashing\n",
-+ d->domain_id, level, pte);
-+ domain_crash(d);
-+#endif
-+ return true;
-+ }
-+
-+ return false;
-+}
-+
-+static inline bool pv_l1tf_check_l1e(struct domain *d, l1_pgentry_t l1e)
-+{
-+ return pv_l1tf_check_pte(d, 1, l1e.l1);
-+}
-+
-+static inline bool pv_l1tf_check_l2e(struct domain *d, l2_pgentry_t l2e)
-+{
-+ return pv_l1tf_check_pte(d, 2, l2e.l2);
-+}
-+
-+static inline bool pv_l1tf_check_l3e(struct domain *d, l3_pgentry_t l3e)
-+{
-+ return pv_l1tf_check_pte(d, 3, l3e.l3);
-+}
-+
-+static inline bool pv_l1tf_check_l4e(struct domain *d, l4_pgentry_t l4e)
-+{
-+ return pv_l1tf_check_pte(d, 4, l4e.l4);
-+}
-+
-+void pv_l1tf_tasklet(unsigned long data);
-+
-+static inline void pv_l1tf_domain_init(struct domain *d)
-+{
-+ d->arch.pv_domain.check_l1tf =
-+ opt_pv_l1tf & (is_hardware_domain(d)
-+ ? OPT_PV_L1TF_DOM0 : OPT_PV_L1TF_DOMU);
-+
-+#if defined(CONFIG_SHADOW_PAGING) && defined(CONFIG_PV)
-+ tasklet_init(&d->arch.paging.shadow.pv_l1tf_tasklet,
-+ pv_l1tf_tasklet, (unsigned long)d);
-+#endif
-+}
-+
-+static inline void pv_l1tf_domain_destroy(struct domain *d)
-+{
-+#if defined(CONFIG_SHADOW_PAGING) && defined(CONFIG_PV)
-+ tasklet_kill(&d->arch.paging.shadow.pv_l1tf_tasklet);
-+#endif
-+}
-+
- /* Remove all shadows of the guest mfn. */
- static inline void shadow_remove_all_shadows(struct domain *d, mfn_t gmfn)
- {
-diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h
-index 4e5f673fec..09c55458df 100644
---- a/xen/include/asm-x86/smp.h
-+++ b/xen/include/asm-x86/smp.h
-@@ -26,6 +26,8 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_mask);
- DECLARE_PER_CPU(cpumask_var_t, cpu_core_mask);
- DECLARE_PER_CPU(cpumask_var_t, scratch_cpumask);
-
-+extern bool park_offline_cpus;
-+
- void smp_send_nmi_allbutself(void);
-
- void send_IPI_mask(const cpumask_t *, int vector);
-diff --git a/xen/include/asm-x86/spec_ctrl.h b/xen/include/asm-x86/spec_ctrl.h
-index 5b40afbab0..8f8aad40bb 100644
---- a/xen/include/asm-x86/spec_ctrl.h
-+++ b/xen/include/asm-x86/spec_ctrl.h
-@@ -29,15 +29,27 @@ void init_speculation_mitigations(void);
- extern bool opt_ibpb;
- extern bool opt_ssbd;
- extern int8_t opt_eager_fpu;
-+extern int8_t opt_l1d_flush;
-
- extern bool bsp_delay_spec_ctrl;
- extern uint8_t default_xen_spec_ctrl;
- extern uint8_t default_spec_ctrl_flags;
-
--extern uint8_t opt_xpti;
-+extern int8_t opt_xpti;
- #define OPT_XPTI_DOM0 0x01
- #define OPT_XPTI_DOMU 0x02
-
-+extern int8_t opt_pv_l1tf;
-+#define OPT_PV_L1TF_DOM0 0x01
-+#define OPT_PV_L1TF_DOMU 0x02
-+
-+/*
-+ * The L1D address mask, which might be wider than reported in CPUID, and the
-+ * system physical address above which there are believed to be no cacheable
-+ * memory regions, thus unable to leak data via the L1TF vulnerability.
-+ */
-+extern paddr_t l1tf_addr_mask, l1tf_safe_maddr;
-+
- static inline void init_shadow_spec_ctrl_state(void)
- {
- struct cpu_info *info = get_cpu_info();
-diff --git a/xen/include/asm-x86/system.h b/xen/include/asm-x86/system.h
-index 43fb6fe489..483cd20afd 100644
---- a/xen/include/asm-x86/system.h
-+++ b/xen/include/asm-x86/system.h
-@@ -221,6 +221,30 @@ static always_inline unsigned long __xadd(
- #define set_mb(var, value) do { xchg(&var, value); } while (0)
- #define set_wmb(var, value) do { var = value; smp_wmb(); } while (0)
-
-+/**
-+ * array_index_mask_nospec() - generate a mask that is ~0UL when the
-+ * bounds check succeeds and 0 otherwise
-+ * @index: array element index
-+ * @size: number of elements in array
-+ *
-+ * Returns:
-+ * 0 - (index < size)
-+ */
-+static inline unsigned long array_index_mask_nospec(unsigned long index,
-+ unsigned long size)
-+{
-+ unsigned long mask;
-+
-+ asm volatile ( "cmp %[size], %[index]; sbb %[mask], %[mask];"
-+ : [mask] "=r" (mask)
-+ : [size] "g" (size), [index] "r" (index) );
-+
-+ return mask;
-+}
-+
-+/* Override default implementation in nospec.h. */
-+#define array_index_mask_nospec array_index_mask_nospec
-+
- #define local_irq_disable() asm volatile ( "cli" : : : "memory" )
- #define local_irq_enable() asm volatile ( "sti" : : : "memory" )
-
-diff --git a/xen/include/asm-x86/xstate.h b/xen/include/asm-x86/xstate.h
-index 86a4a1f75c..47f602b855 100644
---- a/xen/include/asm-x86/xstate.h
-+++ b/xen/include/asm-x86/xstate.h
-@@ -97,8 +97,9 @@ void xsave(struct vcpu *v, uint64_t mask);
- void xrstor(struct vcpu *v, uint64_t mask);
- void xstate_set_init(uint64_t mask);
- bool xsave_enabled(const struct vcpu *v);
--int __must_check validate_xstate(u64 xcr0, u64 xcr0_accum,
-- const struct xsave_hdr *);
-+int __must_check validate_xstate(const struct domain *d,
-+ uint64_t xcr0, uint64_t xcr0_accum,
-+ const struct xsave_hdr *hdr);
- int __must_check handle_xsetbv(u32 index, u64 new_bv);
- void expand_xsave_states(struct vcpu *v, void *dest, unsigned int size);
- void compress_xsave_states(struct vcpu *v, const void *src, unsigned int size);
-diff --git a/xen/include/public/arch-x86/cpufeatureset.h b/xen/include/public/arch-x86/cpufeatureset.h
-index f1a5ed93e0..6c82816fd3 100644
---- a/xen/include/public/arch-x86/cpufeatureset.h
-+++ b/xen/include/public/arch-x86/cpufeatureset.h
-@@ -244,6 +244,7 @@ XEN_CPUFEATURE(AVX512_4VNNIW, 9*32+ 2) /*A AVX512 Neural Network Instructions *
- XEN_CPUFEATURE(AVX512_4FMAPS, 9*32+ 3) /*A AVX512 Multiply Accumulation Single Precision */
- XEN_CPUFEATURE(IBRSB, 9*32+26) /*A IBRS and IBPB support (used by Intel) */
- XEN_CPUFEATURE(STIBP, 9*32+27) /*A STIBP */
-+XEN_CPUFEATURE(L1D_FLUSH, 9*32+28) /*S MSR_FLUSH_CMD and L1D flush. */
- XEN_CPUFEATURE(ARCH_CAPS, 9*32+29) /* IA32_ARCH_CAPABILITIES MSR */
- XEN_CPUFEATURE(SSBD, 9*32+31) /*A MSR_SPEC_CTRL.SSBD available */
-
-diff --git a/xen/include/xen/compiler.h b/xen/include/xen/compiler.h
-index 533a8ea0f3..a7e05681c9 100644
---- a/xen/include/xen/compiler.h
-+++ b/xen/include/xen/compiler.h
-@@ -81,6 +81,9 @@
- #pragma GCC visibility push(hidden)
- #endif
-
-+/* Make the optimizer believe the variable can be manipulated arbitrarily. */
-+#define OPTIMIZER_HIDE_VAR(var) __asm__ ( "" : "+g" (var) )
-+
- /* This macro obfuscates arithmetic on a variable address so that gcc
- shouldn't recognize the original var, and make assumptions about it */
- /*
-diff --git a/xen/include/xen/cpu.h b/xen/include/xen/cpu.h
-index ffefc09f8e..2fe3ec05d8 100644
---- a/xen/include/xen/cpu.h
-+++ b/xen/include/xen/cpu.h
-@@ -47,6 +47,8 @@ void register_cpu_notifier(struct notifier_block *nb);
- #define CPU_DYING (0x0007 | NOTIFY_REVERSE)
- /* CPU_DEAD: CPU is dead. */
- #define CPU_DEAD (0x0008 | NOTIFY_REVERSE)
-+/* CPU_REMOVE: CPU was removed. */
-+#define CPU_REMOVE (0x0009 | NOTIFY_REVERSE)
-
- /* Perform CPU hotplug. May return -EAGAIN. */
- int cpu_down(unsigned int cpu);
-diff --git a/xen/include/xen/cpumask.h b/xen/include/xen/cpumask.h
-index 42340a098e..4a11bcc3f3 100644
---- a/xen/include/xen/cpumask.h
-+++ b/xen/include/xen/cpumask.h
-@@ -351,16 +351,35 @@ static inline bool_t alloc_cpumask_var(cpumask_var_t *mask)
- return *mask != NULL;
- }
-
-+static inline bool cond_alloc_cpumask_var(cpumask_var_t *mask)
-+{
-+ if (*mask == NULL)
-+ *mask = _xmalloc(nr_cpumask_bits / 8, sizeof(long));
-+ return *mask != NULL;
-+}
-+
- static inline bool_t zalloc_cpumask_var(cpumask_var_t *mask)
- {
- *(void **)mask = _xzalloc(nr_cpumask_bits / 8, sizeof(long));
- return *mask != NULL;
- }
-
-+static inline bool cond_zalloc_cpumask_var(cpumask_var_t *mask)
-+{
-+ if (*mask == NULL)
-+ *mask = _xzalloc(nr_cpumask_bits / 8, sizeof(long));
-+ else
-+ cpumask_clear(*mask);
-+ return *mask != NULL;
-+}
-+
- static inline void free_cpumask_var(cpumask_var_t mask)
- {
- xfree(mask);
- }
-+
-+/* Free an allocated mask, and zero the pointer to it. */
-+#define FREE_CPUMASK_VAR(m) XFREE(m)
- #else
- typedef cpumask_t cpumask_var_t[1];
-
-@@ -368,16 +387,20 @@ static inline bool_t alloc_cpumask_var(cpumask_var_t *mask)
- {
- return 1;
- }
-+#define cond_alloc_cpumask_var alloc_cpumask_var
-
- static inline bool_t zalloc_cpumask_var(cpumask_var_t *mask)
- {
- cpumask_clear(*mask);
- return 1;
- }
-+#define cond_zalloc_cpumask_var zalloc_cpumask_var
-
- static inline void free_cpumask_var(cpumask_var_t mask)
- {
- }
-+
-+#define FREE_CPUMASK_VAR(m) free_cpumask_var(m)
- #endif
-
- #if NR_CPUS > 1
-diff --git a/xen/include/xen/list.h b/xen/include/xen/list.h
-index fa07d720ee..1387abb211 100644
---- a/xen/include/xen/list.h
-+++ b/xen/include/xen/list.h
-@@ -51,6 +51,11 @@ static inline void INIT_LIST_HEAD(struct list_head *list)
- list->prev = list;
- }
-
-+static inline bool list_head_is_null(const struct list_head *list)
-+{
-+ return !list->next && !list->prev;
-+}
-+
- /*
- * Insert a new entry between two known consecutive entries.
- *
-diff --git a/xen/include/xen/mm.h b/xen/include/xen/mm.h
-index e928551c91..24654e8e22 100644
---- a/xen/include/xen/mm.h
-+++ b/xen/include/xen/mm.h
-@@ -162,6 +162,14 @@ void free_xenheap_pages(void *v, unsigned int order);
- bool scrub_free_pages(void);
- #define alloc_xenheap_page() (alloc_xenheap_pages(0,0))
- #define free_xenheap_page(v) (free_xenheap_pages(v,0))
-+
-+/* Free an allocation, and zero the pointer to it. */
-+#define FREE_XENHEAP_PAGES(p, o) do { \
-+ free_xenheap_pages(p, o); \
-+ (p) = NULL; \
-+} while ( false )
-+#define FREE_XENHEAP_PAGE(p) FREE_XENHEAP_PAGES(p, 0)
-+
- /* Map machine page range in Xen virtual address space. */
- int map_pages_to_xen(
- unsigned long virt,
-diff --git a/xen/include/xen/nospec.h b/xen/include/xen/nospec.h
-new file mode 100644
-index 0000000000..48793996e8
---- /dev/null
-+++ b/xen/include/xen/nospec.h
-@@ -0,0 +1,70 @@
-+/* SPDX-License-Identifier: GPL-2.0 */
-+/* Copyright(c) 2018 Linus Torvalds. All rights reserved. */
-+/* Copyright(c) 2018 Alexei Starovoitov. All rights reserved. */
-+/* Copyright(c) 2018 Intel Corporation. All rights reserved. */
-+/* Copyright(c) 2018 Citrix Systems R&D Ltd. All rights reserved. */
-+
-+#ifndef XEN_NOSPEC_H
-+#define XEN_NOSPEC_H
-+
-+#include <asm/system.h>
-+
-+/**
-+ * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
-+ * @index: array element index
-+ * @size: number of elements in array
-+ *
-+ * When @index is out of bounds (@index >= @size), the sign bit will be
-+ * set. Extend the sign bit to all bits and invert, giving a result of
-+ * zero for an out of bounds index, or ~0 if within bounds [0, @size).
-+ */
-+#ifndef array_index_mask_nospec
-+static inline unsigned long array_index_mask_nospec(unsigned long index,
-+ unsigned long size)
-+{
-+ /*
-+ * Always calculate and emit the mask even if the compiler
-+ * thinks the mask is not needed. The compiler does not take
-+ * into account the value of @index under speculation.
-+ */
-+ OPTIMIZER_HIDE_VAR(index);
-+ return ~(long)(index | (size - 1UL - index)) >> (BITS_PER_LONG - 1);
-+}
-+#endif
-+
-+/*
-+ * array_index_nospec - sanitize an array index after a bounds check
-+ *
-+ * For a code sequence like:
-+ *
-+ * if (index < size) {
-+ * index = array_index_nospec(index, size);
-+ * val = array[index];
-+ * }
-+ *
-+ * ...if the CPU speculates past the bounds check then
-+ * array_index_nospec() will clamp the index within the range of [0,
-+ * size).
-+ */
-+#define array_index_nospec(index, size) \
-+({ \
-+ typeof(index) _i = (index); \
-+ typeof(size) _s = (size); \
-+ unsigned long _mask = array_index_mask_nospec(_i, _s); \
-+ \
-+ BUILD_BUG_ON(sizeof(_i) > sizeof(long)); \
-+ BUILD_BUG_ON(sizeof(_s) > sizeof(long)); \
-+ \
-+ (typeof(_i)) (_i & _mask); \
-+})
-+
-+#endif /* XEN_NOSPEC_H */
-+
-+/*
-+ * Local variables:
-+ * mode: C
-+ * c-file-style: "BSD"
-+ * c-basic-offset: 4
-+ * indent-tabs-mode: nil
-+ * End:
-+ */
-diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
-index 99d2af2e1f..e79d5a36ca 100644
---- a/xen/include/xen/sched.h
-+++ b/xen/include/xen/sched.h
-@@ -788,7 +788,7 @@ static inline struct domain *next_domain_in_cpupool(
- #define _VPF_parked 8
- #define VPF_parked (1UL<<_VPF_parked)
-
--static inline int vcpu_runnable(struct vcpu *v)
-+static inline bool vcpu_runnable(const struct vcpu *v)
- {
- return !(v->pause_flags |
- atomic_read(&v->pause_count) |
-diff --git a/xen/include/xen/tasklet.h b/xen/include/xen/tasklet.h
-index 23d69c738e..bc9ddace6d 100644
---- a/xen/include/xen/tasklet.h
-+++ b/xen/include/xen/tasklet.h
-@@ -50,6 +50,11 @@ static inline bool tasklet_work_to_do(unsigned int cpu)
- TASKLET_scheduled);
- }
-
-+static inline bool tasklet_is_scheduled(const struct tasklet *t)
-+{
-+ return t->scheduled_on != -1;
-+}
-+
- void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu);
- void tasklet_schedule(struct tasklet *t);
- void do_tasklet(void);
-diff --git a/xen/include/xen/xmalloc.h b/xen/include/xen/xmalloc.h
-index cc2673d8ae..9aa5edf593 100644
---- a/xen/include/xen/xmalloc.h
-+++ b/xen/include/xen/xmalloc.h
-@@ -26,6 +26,12 @@
- /* Free any of the above. */
- extern void xfree(void *);
-
-+/* Free an allocation, and zero the pointer to it. */
-+#define XFREE(p) do { \
-+ xfree(p); \
-+ (p) = NULL; \
-+} while ( false )
-+
- /* Underlying functions */
- extern void *_xmalloc(unsigned long size, unsigned long align);
- extern void *_xzalloc(unsigned long size, unsigned long align);
generated by cgit v1.2.3 (git 2.26.2) at 2024-12-26 22:00:56 +0000