diff options
| -rw-r--r-- | system/xen/dom0/README.dom0 | 2 | ||||
| -rw-r--r-- | system/xen/dom0/config-4.4.157-xen.i686 (renamed from system/xen/dom0/config-4.4.118-xen.i686) | 10 | ||||
| -rw-r--r-- | system/xen/dom0/config-4.4.157-xen.x86_64 (renamed from system/xen/dom0/config-4.4.118-xen.x86_64) | 10 | ||||
| -rw-r--r-- | system/xen/dom0/kernel-xen.sh | 4 | ||||
| -rw-r--r-- | system/xen/domU/domU.sh | 2 | ||||
| -rw-r--r-- | system/xen/xen.SlackBuild | 2 | ||||
| -rw-r--r-- | system/xen/xen.info | 6 | ||||
| -rw-r--r-- | system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch | 4115 |
8 files changed, 18 insertions, 4133 deletions
diff --git a/system/xen/dom0/README.dom0 b/system/xen/dom0/README.dom0 index 025189788bfe0..9047c983be383 100644 --- a/system/xen/dom0/README.dom0 +++ b/system/xen/dom0/README.dom0 @@ -46,7 +46,7 @@ Xen EFI binary. To make things a bit easier, a copy of Xen EFI binary can be found here: - http://slackware.hr/~mario/xen/xen-4.11.0.efi.gz + http://slackware.hr/~mario/xen/xen-4.11.1.efi.gz If an automatic boot to Xen kernel is desired, the binary should be renamed and copied to the following location: /boot/efi/EFI/BOOT/bootx64.efi diff --git a/system/xen/dom0/config-4.4.118-xen.i686 b/system/xen/dom0/config-4.4.157-xen.i686 index 2ba089e19fefb..0f409db35c866 100644 --- a/system/xen/dom0/config-4.4.118-xen.i686 +++ b/system/xen/dom0/config-4.4.157-xen.i686 @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86 4.4.118 Kernel Configuration +# Linux/x86 4.4.157 Kernel Configuration # # CONFIG_64BIT is not set CONFIG_X86_32=y @@ -34,7 +34,6 @@ CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y CONFIG_HAVE_INTEL_TXT=y CONFIG_X86_32_SMP=y -CONFIG_ARCH_HWEIGHT_CFLAGS="-fcall-saved-ecx -fcall-saved-edx" CONFIG_ARCH_SUPPORTS_UPROBES=y CONFIG_FIX_EARLYCON_MEM=y CONFIG_PGTABLE_LEVELS=3 @@ -178,6 +177,7 @@ CONFIG_RD_LZMA=y CONFIG_RD_XZ=y CONFIG_RD_LZO=y CONFIG_RD_LZ4=y +CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set CONFIG_SYSCTL=y CONFIG_ANON_INODES=y @@ -366,6 +366,7 @@ CONFIG_FREEZER=y CONFIG_ZONE_DMA=y CONFIG_SMP=y CONFIG_X86_FEATURE_NAMES=y +CONFIG_X86_FAST_FEATURE_TESTS=y CONFIG_X86_MPPARSE=y CONFIG_X86_BIGSMP=y CONFIG_RETPOLINE=y @@ -1549,7 +1550,6 @@ CONFIG_BT_HCIUART_H4=y CONFIG_BT_HCIUART_BCSP=y CONFIG_BT_HCIUART_ATH3K=y CONFIG_BT_HCIUART_LL=y -CONFIG_BT_HCIUART_3WIRE=y CONFIG_BT_HCIUART_INTEL=y CONFIG_BT_HCIUART_BCM=y CONFIG_BT_HCIUART_QCA=y @@ -6838,7 +6838,8 @@ CONFIG_CIFS_WEAK_PW_HASH=y CONFIG_CIFS_DEBUG=y # CONFIG_CIFS_DEBUG2 is not set CONFIG_CIFS_DFS_UPCALL=y -# CONFIG_CIFS_SMB2 is not set +CONFIG_CIFS_SMB2=y +# CONFIG_CIFS_SMB311 is not set CONFIG_CIFS_FSCACHE=y CONFIG_NCP_FS=m CONFIG_NCPFS_PACKET_SIGNING=y @@ -7123,7 +7124,6 @@ CONFIG_DEFAULT_IO_DELAY_TYPE=0 CONFIG_OPTIMIZE_INLINING=y # CONFIG_DEBUG_ENTRY is not set # CONFIG_DEBUG_NMI_SELFTEST is not set -# CONFIG_X86_DEBUG_STATIC_CPU_HAS is not set CONFIG_X86_DEBUG_FPU=y # CONFIG_PUNIT_ATOM_DEBUG is not set diff --git a/system/xen/dom0/config-4.4.118-xen.x86_64 b/system/xen/dom0/config-4.4.157-xen.x86_64 index b690312fe3add..99b7205209c42 100644 --- a/system/xen/dom0/config-4.4.118-xen.x86_64 +++ b/system/xen/dom0/config-4.4.157-xen.x86_64 @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86 4.4.118 Kernel Configuration +# Linux/x86 4.4.157 Kernel Configuration # CONFIG_64BIT=y CONFIG_X86_64=y @@ -37,7 +37,6 @@ CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y CONFIG_HAVE_INTEL_TXT=y CONFIG_X86_64_SMP=y -CONFIG_ARCH_HWEIGHT_CFLAGS="-fcall-saved-rdi -fcall-saved-rsi -fcall-saved-rdx -fcall-saved-rcx -fcall-saved-r8 -fcall-saved-r9 -fcall-saved-r10 -fcall-saved-r11" CONFIG_ARCH_SUPPORTS_UPROBES=y CONFIG_FIX_EARLYCON_MEM=y CONFIG_PGTABLE_LEVELS=4 @@ -185,6 +184,7 @@ CONFIG_RD_LZMA=y CONFIG_RD_XZ=y CONFIG_RD_LZO=y CONFIG_RD_LZ4=y +CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set CONFIG_SYSCTL=y CONFIG_ANON_INODES=y @@ -378,6 +378,7 @@ CONFIG_FREEZER=y CONFIG_ZONE_DMA=y CONFIG_SMP=y CONFIG_X86_FEATURE_NAMES=y +CONFIG_X86_FAST_FEATURE_TESTS=y CONFIG_X86_X2APIC=y CONFIG_X86_MPPARSE=y CONFIG_RETPOLINE=y @@ -1513,7 +1514,6 @@ CONFIG_BT_HCIUART_H4=y CONFIG_BT_HCIUART_BCSP=y CONFIG_BT_HCIUART_ATH3K=y CONFIG_BT_HCIUART_LL=y -CONFIG_BT_HCIUART_3WIRE=y CONFIG_BT_HCIUART_INTEL=y CONFIG_BT_HCIUART_BCM=y CONFIG_BT_HCIUART_QCA=y @@ -6647,7 +6647,8 @@ CONFIG_CIFS_WEAK_PW_HASH=y CONFIG_CIFS_DEBUG=y # CONFIG_CIFS_DEBUG2 is not set CONFIG_CIFS_DFS_UPCALL=y -# CONFIG_CIFS_SMB2 is not set +CONFIG_CIFS_SMB2=y +# CONFIG_CIFS_SMB311 is not set CONFIG_CIFS_FSCACHE=y CONFIG_NCP_FS=m CONFIG_NCPFS_PACKET_SIGNING=y @@ -6935,7 +6936,6 @@ CONFIG_DEFAULT_IO_DELAY_TYPE=0 CONFIG_OPTIMIZE_INLINING=y # CONFIG_DEBUG_ENTRY is not set # CONFIG_DEBUG_NMI_SELFTEST is not set -# CONFIG_X86_DEBUG_STATIC_CPU_HAS is not set CONFIG_X86_DEBUG_FPU=y # CONFIG_PUNIT_ATOM_DEBUG is not set diff --git a/system/xen/dom0/kernel-xen.sh b/system/xen/dom0/kernel-xen.sh index 0829676da6347..5c4bf8b4593fc 100644 --- a/system/xen/dom0/kernel-xen.sh +++ b/system/xen/dom0/kernel-xen.sh @@ -5,8 +5,8 @@ # Written by Chris Abela <chris.abela@maltats.com>, 20100515 # Modified by Mario Preksavec <mario@slackware.hr> -KERNEL=${KERNEL:-4.4.118} -XEN=${XEN:-4.11.0} +KERNEL=${KERNEL:-4.4.157} +XEN=${XEN:-4.11.1} BOOTLOADER=${BOOTLOADER:-lilo} ROOTMOD=${ROOTMOD:-ext4} diff --git a/system/xen/domU/domU.sh b/system/xen/domU/domU.sh index 87165495360a6..b9037f4f18efa 100644 --- a/system/xen/domU/domU.sh +++ b/system/xen/domU/domU.sh @@ -7,7 +7,7 @@ set -e -KERNEL=${KERNEL:-4.4.88} +KERNEL=${KERNEL:-4.4.157} # Build an image for the root file system and another for the swap # Default values : 8GB and 500MB resepectively. diff --git a/system/xen/xen.SlackBuild b/system/xen/xen.SlackBuild index 29c2ddf3cd2ff..14d683d841b6a 100644 --- a/system/xen/xen.SlackBuild +++ b/system/xen/xen.SlackBuild @@ -23,7 +23,7 @@ # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PRGNAM=xen -VERSION=${VERSION:-4.11.0} +VERSION=${VERSION:-4.11.1} BUILD=${BUILD:-1} TAG=${TAG:-_SBo} diff --git a/system/xen/xen.info b/system/xen/xen.info index 2e6b41b5828f6..07fd91f17620b 100644 --- a/system/xen/xen.info +++ b/system/xen/xen.info @@ -1,7 +1,7 @@ PRGNAM="xen" -VERSION="4.11.0" +VERSION="4.11.1" HOMEPAGE="http://www.xenproject.org/" -DOWNLOAD="http://mirror.slackware.hr/sources/xen/xen-4.11.0.tar.gz \ +DOWNLOAD="http://mirror.slackware.hr/sources/xen/xen-4.11.1.tar.gz \ http://mirror.slackware.hr/sources/xen-extfiles/ipxe-git-356f6c1b64d7a97746d1816cef8ca22bdd8d0b5d.tar.gz \ http://mirror.slackware.hr/sources/xen-extfiles/lwip-1.3.0.tar.gz \ http://mirror.slackware.hr/sources/xen-extfiles/zlib-1.2.3.tar.gz \ @@ -13,7 +13,7 @@ DOWNLOAD="http://mirror.slackware.hr/sources/xen/xen-4.11.0.tar.gz \ http://mirror.slackware.hr/sources/xen-extfiles/tpm_emulator-0.7.4.tar.gz \ http://mirror.slackware.hr/sources/xen-seabios/seabios-1.11.1.tar.xz \ http://mirror.slackware.hr/sources/xen-ovmf/xen-ovmf-20170920_947f3737a.tar.bz2" -MD5SUM="cbec0600284921744bc14119f4ed3fff \ +MD5SUM="0c72b97432465e7f81113630fcd6e460 \ 0061f103c84b25c2e6ac47649b909bde \ 36cc57650cffda9a0269493be2a169bb \ debc62758716a169df9f62e6ab2bc634 \ diff --git a/system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch b/system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch deleted file mode 100644 index ab9794df5e883..0000000000000 --- a/system/xen/xsa/xsa273-d757c29ffe2e31b15397e43cd58da88b6318b654.patch +++ /dev/null @@ -1,4115 +0,0 @@ -diff --git a/docs/man/xl.conf.pod.5 b/docs/man/xl.conf.pod.5 -index da91b8626c..37262a7ef8 100644 ---- a/docs/man/xl.conf.pod.5 -+++ b/docs/man/xl.conf.pod.5 -@@ -185,6 +185,28 @@ massively huge guests). - - =back - -+=item B<vm.cpumask>="CPULIST" -+ -+=item B<vm.hvm.cpumask>="CPULIST" -+ -+=item B<vm.pv.cpumask>="CPULIST" -+ -+Global masks that are applied when creating guests and pinning vcpus -+to indicate which cpus they are allowed to run on. Specifically, -+C<vm.cpumask> applies to all guest types, C<vm.hvm.cpumask> applies to -+both HVM and PVH guests and C<vm.pv.cpumask> applies to PV guests. -+ -+The hard affinity of guest's vcpus are logical-AND'ed with respective -+masks. If the resulting affinity mask is empty, operation will fail. -+ -+Use --ignore-global-affinity-masks to skip applying global masks. -+ -+The default value for these masks are all 1's, i.e. all cpus are allowed. -+ -+Due to bug(s), these options may not interact well with other options -+concerning CPU affinity. One example is CPU pools. Users should always double -+check that the required affinity has taken effect. -+ - =back - - =head1 SEE ALSO -diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown -index 075e5ea159..0886706368 100644 ---- a/docs/misc/xen-command-line.markdown -+++ b/docs/misc/xen-command-line.markdown -@@ -489,10 +489,10 @@ accounting for hardware capabilities as enumerated via CPUID. - - Currently accepted: - --The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`, `ssbd` are --used by default if available and applicable. They can be ignored, --e.g. `no-ibrsb`, at which point Xen won't use them itself, and won't offer --them to guests. -+The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`, -+`l1d-flush` and `ssbd` are used by default if available and applicable. They can -+be ignored, e.g. `no-ibrsb`, at which point Xen won't use them itself, and -+won't offer them to guests. - - ### cpuid\_mask\_cpu (AMD only) - > `= fam_0f_rev_c | fam_0f_rev_d | fam_0f_rev_e | fam_0f_rev_f | fam_0f_rev_g | fam_10_rev_b | fam_10_rev_c | fam_11_rev_b` -@@ -936,6 +936,8 @@ version are 1 and 2. - use of grant table v2 without transitive grants is an ABI breakage from the - guests point of view. - -+The usage of gnttab v2 is not security supported on ARM platforms. -+ - ### gnttab\_max\_frames - > `= <integer>` - -@@ -1544,6 +1546,30 @@ do; there may be other custom operating systems which do. If you're - certain you don't plan on having PV guests which use this feature, - turning it off can reduce the attack surface. - -+### pv-l1tf (x86) -+> `= List of [ <bool>, dom0=<bool>, domu=<bool> ]` -+ -+> Default: `false` on believed-unaffected hardware, or in pv-shim mode. -+> `domu` on believed-affected hardware. -+ -+Mitigations for L1TF / XSA-273 / CVE-2018-3620 for PV guests. -+ -+For backwards compatibility, we may not alter an architecturally-legitimate -+pagetable entry a PV guest chooses to write. We can however force such a -+guest into shadow mode so that Xen controls the PTEs which are reachable by -+the CPU pagewalk. -+ -+Shadowing is performed at the point where a PV guest first tries to write an -+L1TF-vulnerable PTE. Therefore, a PV guest kernel which has been updated with -+its own L1TF mitigations will not trigger shadow mode if it is well behaved. -+ -+If CONFIG\_SHADOW\_PAGING is not compiled in, this mitigation instead crashes -+the guest when an L1TF-vulnerable PTE is written, which still allows updated, -+well-behaved PV guests to run, despite Shadow being compiled out. -+ -+In the pv-shim case, Shadow is expected to be compiled out, and a malicious -+guest kernel can only leak data from the shim Xen, rather than the host Xen. -+ - ### pv-shim (x86) - > `= <boolean>` - -@@ -1748,6 +1774,13 @@ Use `smap=hvm` to allow SMAP use by HVM guests only. - Flag to enable Supervisor Mode Execution Protection - Use `smep=hvm` to allow SMEP use by HVM guests only. - -+### smt (x86) -+> `= <boolean>` -+ -+Default: `true` -+ -+Control bring up of multiple hyper-threads per CPU core. -+ - ### snb\_igd\_quirk - > `= <boolean> | cap | <integer>` - -@@ -1758,7 +1791,8 @@ false disable the quirk workaround, which is also the default. - - ### spec-ctrl (x86) - > `= List of [ <bool>, xen=<bool>, {pv,hvm,msr-sc,rsb}=<bool>, --> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu}=<bool> ]` -+> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu, -+> l1d-flush}=<bool> ]` - - Controls for speculative execution sidechannel mitigations. By default, Xen - will pick the most appropriate mitigations based on compiled in support, -@@ -1770,10 +1804,15 @@ extreme care.** - - An overall boolean value, `spec-ctrl=no`, can be specified to turn off all - mitigations, including pieces of infrastructure used to virtualise certain --mitigation features for guests. Alternatively, a slightly more restricted --`spec-ctrl=no-xen` can be used to turn off all of Xen's mitigations, while --leaving the virtualisation support in place for guests to use. Use of a --positive boolean value for either of these options is invalid. -+mitigation features for guests. This also includes settings which `xpti`, -+`smt`, `pv-l1tf` control, unless the respective option(s) have been -+specified earlier on the command line. -+ -+Alternatively, a slightly more restricted `spec-ctrl=no-xen` can be used to -+turn off all of Xen's mitigations, while leaving the virtualisation support -+in place for guests to use. -+ -+Use of a positive boolean value for either of these options is invalid. - - The booleans `pv=`, `hvm=`, `msr-sc=` and `rsb=` offer fine grained control - over the alternative blocks used by Xen. These impact Xen's ability to -@@ -1813,6 +1852,12 @@ from using fully eager FPU context switches. This is currently implemented as - a global control. By default, Xen will choose to use fully eager context - switches on hardware believed to speculate past #NM exceptions. - -+On hardware supporting L1D_FLUSH, the `l1d-flush=` option can be used to force -+or prevent Xen from issuing an L1 data cache flush on each VMEntry. -+Irrespective of Xen's setting, the feature is virtualised for HVM guests to -+use. By default, Xen will enable this mitigation on hardware believed to be -+vulnerable to L1TF. -+ - ### sync\_console - > `= <boolean>` - -diff --git a/tools/examples/xl.conf b/tools/examples/xl.conf -index 374b6bbc2e..0446deb304 100644 ---- a/tools/examples/xl.conf -+++ b/tools/examples/xl.conf -@@ -37,3 +37,8 @@ - # (which can take a long time to find out if launching huge guests). - # see xl.conf(5) for details. - #claim_mode=1 -+ -+# Specify global vcpu hard affinity masks. See xl.conf(5) for details. -+#vm.cpumask="0-7" -+#vm.pv.cpumask="0-3" -+#vm.hvm.cpumask="3-7" -diff --git a/tools/libxl/libxl_cpuid.c b/tools/libxl/libxl_cpuid.c -index 7b0f594c3d..52e16c20ed 100644 ---- a/tools/libxl/libxl_cpuid.c -+++ b/tools/libxl/libxl_cpuid.c -@@ -204,6 +204,7 @@ int libxl_cpuid_parse_config(libxl_cpuid_policy_list *cpuid, const char* str) - {"avx512-4fmaps",0x00000007, 0, CPUID_REG_EDX, 3, 1}, - {"ibrsb", 0x00000007, 0, CPUID_REG_EDX, 26, 1}, - {"stibp", 0x00000007, 0, CPUID_REG_EDX, 27, 1}, -+ {"l1d-flush", 0x00000007, 0, CPUID_REG_EDX, 28, 1}, - {"arch-caps", 0x00000007, 0, CPUID_REG_EDX, 29, 1}, - {"ssbd", 0x00000007, 0, CPUID_REG_EDX, 31, 1}, - -diff --git a/tools/misc/xen-cpuid.c b/tools/misc/xen-cpuid.c -index e116339733..3888b4e158 100644 ---- a/tools/misc/xen-cpuid.c -+++ b/tools/misc/xen-cpuid.c -@@ -143,7 +143,7 @@ static const char *str_7d0[32] = - [ 2] = "avx512_4vnniw", [ 3] = "avx512_4fmaps", - - [26] = "ibrsb", [27] = "stibp", -- /* 28 */ [29] = "arch_caps", -+ [28] = "l1d_flush", [29] = "arch_caps", - /* 30 */ [31] = "ssbd", - }; - -diff --git a/tools/ocaml/xenstored/store.ml b/tools/ocaml/xenstored/store.ml -index 13cf3b5bf4..5a8c377603 100644 ---- a/tools/ocaml/xenstored/store.ml -+++ b/tools/ocaml/xenstored/store.ml -@@ -262,7 +262,8 @@ let path_write store perm path value = - Node.check_perm store.root perm Perms.WRITE; - Node.set_value store.root value, false - ) else -- Path.apply_modify store.root path do_write, !node_created -+ let root = Path.apply_modify store.root path do_write in -+ root, !node_created - - let path_rm store perm path = - let do_rm node name = -diff --git a/tools/xl/xl.c b/tools/xl/xl.c -index 179908b4f6..7d2142f16f 100644 ---- a/tools/xl/xl.c -+++ b/tools/xl/xl.c -@@ -28,6 +28,9 @@ - #include <libxl_utils.h> - #include <libxlutil.h> - #include "xl.h" -+#include "xl_parse.h" -+ -+#include "xl_utils.h" - - xentoollog_logger_stdiostream *logger; - int dryrun_only; -@@ -42,6 +45,9 @@ char *default_gatewaydev = NULL; - char *default_vifbackend = NULL; - char *default_remus_netbufscript = NULL; - char *default_colo_proxy_script = NULL; -+libxl_bitmap global_vm_affinity_mask; -+libxl_bitmap global_hvm_affinity_mask; -+libxl_bitmap global_pv_affinity_mask; - enum output_format default_output_format = OUTPUT_FORMAT_JSON; - int claim_mode = 1; - bool progress_use_cr = 0; -@@ -203,6 +209,26 @@ static void parse_global_config(const char *configfile, - if (!xlu_cfg_get_long (config, "max_maptrack_frames", &l, 0)) - max_maptrack_frames = l; - -+ libxl_bitmap_init(&global_vm_affinity_mask); -+ libxl_cpu_bitmap_alloc(ctx, &global_vm_affinity_mask, 0); -+ libxl_bitmap_init(&global_hvm_affinity_mask); -+ libxl_cpu_bitmap_alloc(ctx, &global_hvm_affinity_mask, 0); -+ libxl_bitmap_init(&global_pv_affinity_mask); -+ libxl_cpu_bitmap_alloc(ctx, &global_pv_affinity_mask, 0); -+ -+ if (!xlu_cfg_get_string (config, "vm.cpumask", &buf, 0)) -+ parse_cpurange(buf, &global_vm_affinity_mask); -+ else -+ libxl_bitmap_set_any(&global_vm_affinity_mask); -+ if (!xlu_cfg_get_string (config, "vm.hvm.cpumask", &buf, 0)) -+ parse_cpurange(buf, &global_hvm_affinity_mask); -+ else -+ libxl_bitmap_set_any(&global_hvm_affinity_mask); -+ if (!xlu_cfg_get_string (config, "vm.pv.cpumask", &buf, 0)) -+ parse_cpurange(buf, &global_pv_affinity_mask); -+ else -+ libxl_bitmap_set_any(&global_pv_affinity_mask); -+ - xlu_cfg_destroy(config); - } - -diff --git a/tools/xl/xl.h b/tools/xl/xl.h -index 4e784ff402..7e97144b50 100644 ---- a/tools/xl/xl.h -+++ b/tools/xl/xl.h -@@ -41,6 +41,7 @@ struct domain_create { - int vncautopass; - int console_autoconnect; - int checkpointed_stream; -+ int ignore_global_affinity_masks; - const char *config_file; - char *extra_config; /* extra config string */ - const char *restore_file; -@@ -279,6 +280,9 @@ extern char *default_colo_proxy_script; - extern char *blkdev_start; - extern int max_grant_frames; - extern int max_maptrack_frames; -+extern libxl_bitmap global_vm_affinity_mask; -+extern libxl_bitmap global_hvm_affinity_mask; -+extern libxl_bitmap global_pv_affinity_mask; - - enum output_format { - OUTPUT_FORMAT_JSON, -@@ -294,6 +298,9 @@ typedef enum { - } domain_restart_type; - - extern void printf_info_sexp(int domid, libxl_domain_config *d_config, FILE *fh); -+extern void apply_global_affinity_masks(libxl_domain_type type, -+ libxl_bitmap *vcpu_affinity_array, -+ unsigned int size); - - #define XL_GLOBAL_CONFIG XEN_CONFIG_DIR "/xl.conf" - #define XL_LOCK_FILE XEN_LOCK_DIR "/xl" -diff --git a/tools/xl/xl_cmdtable.c b/tools/xl/xl_cmdtable.c -index bf2ced8140..54c2db6022 100644 ---- a/tools/xl/xl_cmdtable.c -+++ b/tools/xl/xl_cmdtable.c -@@ -34,7 +34,8 @@ struct cmd_spec cmd_table[] = { - "-e Do not wait in the background for the death of the domain.\n" - "-V, --vncviewer Connect to the VNC display after the domain is created.\n" - "-A, --vncviewer-autopass\n" -- " Pass VNC password to viewer via stdin." -+ " Pass VNC password to viewer via stdin.\n" -+ "--ignore-global-affinity-masks Ignore global masks in xl.conf." - }, - { "config-update", - &main_config_update, 1, 1, -@@ -224,7 +225,8 @@ struct cmd_spec cmd_table[] = { - &main_vcpupin, 1, 1, - "Set which CPUs a VCPU can use", - "[option] <Domain> <VCPU|all> <Hard affinity|-|all> <Soft affinity|-|all>", -- "-f, --force undo an override pinning done by the kernel", -+ "-f, --force undo an override pinning done by the kernel\n" -+ "--ignore-global-affinity-masks Ignore global masks in xl.conf", - }, - { "vcpu-set", - &main_vcpuset, 0, 1, -diff --git a/tools/xl/xl_vcpu.c b/tools/xl/xl_vcpu.c -index 8e735b38c1..3384eeed06 100644 ---- a/tools/xl/xl_vcpu.c -+++ b/tools/xl/xl_vcpu.c -@@ -68,6 +68,61 @@ static void print_domain_vcpuinfo(uint32_t domid, uint32_t nr_cpus) - libxl_vcpuinfo_list_free(vcpuinfo, nb_vcpu); - } - -+void apply_global_affinity_masks(libxl_domain_type type, -+ libxl_bitmap *vcpu_affinity_array, -+ unsigned int size) -+{ -+ libxl_bitmap *mask = &global_vm_affinity_mask; -+ libxl_bitmap *type_mask; -+ unsigned int i; -+ -+ switch (type) { -+ case LIBXL_DOMAIN_TYPE_HVM: -+ case LIBXL_DOMAIN_TYPE_PVH: -+ type_mask = &global_hvm_affinity_mask; -+ break; -+ case LIBXL_DOMAIN_TYPE_PV: -+ type_mask = &global_pv_affinity_mask; -+ break; -+ default: -+ fprintf(stderr, "Unknown guest type\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ for (i = 0; i < size; i++) { -+ int rc; -+ libxl_bitmap *t = &vcpu_affinity_array[i]; -+ libxl_bitmap b1, b2; -+ -+ libxl_bitmap_init(&b1); -+ libxl_bitmap_init(&b2); -+ -+ rc = libxl_bitmap_and(ctx, &b1, t, mask); -+ if (rc) { -+ fprintf(stderr, "libxl_bitmap_and errored\n"); -+ exit(EXIT_FAILURE); -+ } -+ rc = libxl_bitmap_and(ctx, &b2, &b1, type_mask); -+ if (rc) { -+ fprintf(stderr, "libxl_bitmap_and errored\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ if (libxl_bitmap_is_empty(&b2)) { -+ fprintf(stderr, "vcpu hard affinity map is empty\n"); -+ exit(EXIT_FAILURE); -+ } -+ -+ /* Replace target bitmap with the result */ -+ libxl_bitmap_dispose(t); -+ libxl_bitmap_init(t); -+ libxl_bitmap_copy_alloc(ctx, t, &b2); -+ -+ libxl_bitmap_dispose(&b1); -+ libxl_bitmap_dispose(&b2); -+ } -+} -+ - static void vcpulist(int argc, char **argv) - { - libxl_dominfo *dominfo; -@@ -118,6 +173,7 @@ int main_vcpupin(int argc, char **argv) - { - static struct option opts[] = { - {"force", 0, 0, 'f'}, -+ {"ignore-global-affinity-masks", 0, 0, 'i'}, - COMMON_LONG_OPTS - }; - libxl_vcpuinfo *vcpuinfo; -@@ -132,15 +188,18 @@ int main_vcpupin(int argc, char **argv) - const char *vcpu, *hard_str, *soft_str; - char *endptr; - int opt, nb_cpu, nb_vcpu, rc = EXIT_FAILURE; -- bool force = false; -+ bool force = false, ignore_masks = false; - - libxl_bitmap_init(&cpumap_hard); - libxl_bitmap_init(&cpumap_soft); - -- SWITCH_FOREACH_OPT(opt, "f", opts, "vcpu-pin", 3) { -+ SWITCH_FOREACH_OPT(opt, "fi", opts, "vcpu-pin", 3) { - case 'f': - force = true; - break; -+ case 'i': -+ ignore_masks = true; -+ break; - default: - break; - } -@@ -222,6 +281,23 @@ int main_vcpupin(int argc, char **argv) - goto out; - } - -+ /* Only hard affinity matters here */ -+ if (!ignore_masks) { -+ libxl_domain_config d_config; -+ -+ libxl_domain_config_init(&d_config); -+ rc = libxl_retrieve_domain_configuration(ctx, domid, &d_config); -+ if (rc) { -+ fprintf(stderr, "Could not retrieve domain configuration\n"); -+ libxl_domain_config_dispose(&d_config); -+ goto out; -+ } -+ -+ apply_global_affinity_masks(d_config.b_info.type, hard, 1); -+ -+ libxl_domain_config_dispose(&d_config); -+ } -+ - if (force) { - if (libxl_set_vcpuaffinity_force(ctx, domid, vcpuid, hard, soft)) { - fprintf(stderr, "Could not set affinity for vcpu `%ld'.\n", -diff --git a/tools/xl/xl_vmcontrol.c b/tools/xl/xl_vmcontrol.c -index 89c2b25ded..a1d633795c 100644 ---- a/tools/xl/xl_vmcontrol.c -+++ b/tools/xl/xl_vmcontrol.c -@@ -804,6 +804,36 @@ int create_domain(struct domain_create *dom_info) - parse_config_data(config_source, config_data, config_len, &d_config); - } - -+ if (!dom_info->ignore_global_affinity_masks) { -+ libxl_domain_build_info *b_info = &d_config.b_info; -+ -+ /* It is possible that no hard affinity is specified in config file. -+ * Generate hard affinity maps now if we care about those. -+ */ -+ if (b_info->num_vcpu_hard_affinity == 0 && -+ (!libxl_bitmap_is_full(&global_vm_affinity_mask) || -+ (b_info->type == LIBXL_DOMAIN_TYPE_PV && -+ !libxl_bitmap_is_full(&global_pv_affinity_mask)) || -+ (b_info->type != LIBXL_DOMAIN_TYPE_PV && -+ !libxl_bitmap_is_full(&global_hvm_affinity_mask)) -+ )) { -+ b_info->num_vcpu_hard_affinity = b_info->max_vcpus; -+ b_info->vcpu_hard_affinity = -+ xmalloc(b_info->max_vcpus * sizeof(libxl_bitmap)); -+ -+ for (i = 0; i < b_info->num_vcpu_hard_affinity; i++) { -+ libxl_bitmap *m = &b_info->vcpu_hard_affinity[i]; -+ libxl_bitmap_init(m); -+ libxl_cpu_bitmap_alloc(ctx, m, 0); -+ libxl_bitmap_set_any(m); -+ } -+ } -+ -+ apply_global_affinity_masks(b_info->type, -+ b_info->vcpu_hard_affinity, -+ b_info->num_vcpu_hard_affinity); -+ } -+ - if (migrate_fd >= 0) { - if (d_config.c_info.name) { - /* when we receive a domain we get its name from the config -@@ -1124,7 +1154,7 @@ int main_create(int argc, char **argv) - const char *filename = NULL; - struct domain_create dom_info; - int paused = 0, debug = 0, daemonize = 1, console_autoconnect = 0, -- quiet = 0, monitor = 1, vnc = 0, vncautopass = 0; -+ quiet = 0, monitor = 1, vnc = 0, vncautopass = 0, ignore_masks = 0; - int opt, rc; - static struct option opts[] = { - {"dryrun", 0, 0, 'n'}, -@@ -1132,6 +1162,7 @@ int main_create(int argc, char **argv) - {"defconfig", 1, 0, 'f'}, - {"vncviewer", 0, 0, 'V'}, - {"vncviewer-autopass", 0, 0, 'A'}, -+ {"ignore-global-affinity-masks", 0, 0, 'i'}, - COMMON_LONG_OPTS - }; - -@@ -1142,7 +1173,7 @@ int main_create(int argc, char **argv) - argc--; argv++; - } - -- SWITCH_FOREACH_OPT(opt, "Fnqf:pcdeVA", opts, "create", 0) { -+ SWITCH_FOREACH_OPT(opt, "Fnqf:pcdeVAi", opts, "create", 0) { - case 'f': - filename = optarg; - break; -@@ -1174,6 +1205,9 @@ int main_create(int argc, char **argv) - case 'A': - vnc = vncautopass = 1; - break; -+ case 'i': -+ ignore_masks = 1; -+ break; - } - - memset(&dom_info, 0, sizeof(dom_info)); -@@ -1203,6 +1237,7 @@ int main_create(int argc, char **argv) - dom_info.vnc = vnc; - dom_info.vncautopass = vncautopass; - dom_info.console_autoconnect = console_autoconnect; -+ dom_info.ignore_global_affinity_masks = ignore_masks; - - rc = create_domain(&dom_info); - if (rc < 0) { -#diff --git a/xen/Makefile b/xen/Makefile -#index 4d075c381f..a922a1b7b5 100644 -#--- a/xen/Makefile -#+++ b/xen/Makefile -#@@ -2,7 +2,7 @@ -# # All other places this is stored (eg. compile.h) should be autogenerated. -# export XEN_VERSION = 4 -# export XEN_SUBVERSION = 11 -#-export XEN_EXTRAVERSION ?= .0$(XEN_VENDORVERSION) -#+export XEN_EXTRAVERSION ?= .1-pre$(XEN_VENDORVERSION) -# export XEN_FULLVERSION = $(XEN_VERSION).$(XEN_SUBVERSION)$(XEN_EXTRAVERSION) -# -include xen-version -# -diff --git a/xen/arch/x86/Kconfig b/xen/arch/x86/Kconfig -index f64fc56739..cfba4a708c 100644 ---- a/xen/arch/x86/Kconfig -+++ b/xen/arch/x86/Kconfig -@@ -72,6 +72,7 @@ config SHADOW_PAGING - * Running HVM guests on hardware lacking hardware paging support - (First-generation Intel VT-x or AMD SVM). - * Live migration of PV guests. -+ * L1TF sidechannel mitigation for PV guests. - - Under a small number of specific workloads, shadow paging may be - deliberately used as a performance optimisation. -diff --git a/xen/arch/x86/cpu/amd.c b/xen/arch/x86/cpu/amd.c -index 458a3fe60c..76078b55b2 100644 ---- a/xen/arch/x86/cpu/amd.c -+++ b/xen/arch/x86/cpu/amd.c -@@ -505,17 +505,23 @@ static void amd_get_topology(struct cpuinfo_x86 *c) - u32 eax, ebx, ecx, edx; - - cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); -- c->compute_unit_id = ebx & 0xFF; - c->x86_num_siblings = ((ebx >> 8) & 0x3) + 1; -+ -+ if (c->x86 < 0x17) -+ c->compute_unit_id = ebx & 0xFF; -+ else { -+ c->cpu_core_id = ebx & 0xFF; -+ c->x86_max_cores /= c->x86_num_siblings; -+ } - } - - if (opt_cpu_info) - printk("CPU %d(%d) -> Processor %d, %s %d\n", - cpu, c->x86_max_cores, c->phys_proc_id, -- cpu_has(c, X86_FEATURE_TOPOEXT) ? "Compute Unit" : -- "Core", -- cpu_has(c, X86_FEATURE_TOPOEXT) ? c->compute_unit_id : -- c->cpu_core_id); -+ c->compute_unit_id != INVALID_CUID ? "Compute Unit" -+ : "Core", -+ c->compute_unit_id != INVALID_CUID ? c->compute_unit_id -+ : c->cpu_core_id); - } - - static void early_init_amd(struct cpuinfo_x86 *c) -diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c -index 528aff1811..fdb022875a 100644 ---- a/xen/arch/x86/cpu/common.c -+++ b/xen/arch/x86/cpu/common.c -@@ -14,6 +14,7 @@ - #include <public/sysctl.h> /* for XEN_INVALID_{SOCKET,CORE}_ID */ - - #include "cpu.h" -+#include "mcheck/x86_mca.h" - - bool_t opt_arat = 1; - boolean_param("arat", opt_arat); -@@ -355,6 +356,9 @@ static void __init early_cpu_detect(void) - hap_paddr_bits = PADDR_BITS; - } - -+ if (c->x86_vendor != X86_VENDOR_AMD) -+ park_offline_cpus = opt_mce; -+ - initialize_cpu_data(0); - } - -diff --git a/xen/arch/x86/cpu/mcheck/mce.c b/xen/arch/x86/cpu/mcheck/mce.c -index a8c287d124..32273d9208 100644 ---- a/xen/arch/x86/cpu/mcheck/mce.c -+++ b/xen/arch/x86/cpu/mcheck/mce.c -@@ -692,12 +692,15 @@ static void cpu_bank_free(unsigned int cpu) - - mcabanks_free(poll); - mcabanks_free(clr); -+ -+ per_cpu(poll_bankmask, cpu) = NULL; -+ per_cpu(mce_clear_banks, cpu) = NULL; - } - - static int cpu_bank_alloc(unsigned int cpu) - { -- struct mca_banks *poll = mcabanks_alloc(); -- struct mca_banks *clr = mcabanks_alloc(); -+ struct mca_banks *poll = per_cpu(poll_bankmask, cpu) ?: mcabanks_alloc(); -+ struct mca_banks *clr = per_cpu(mce_clear_banks, cpu) ?: mcabanks_alloc(); - - if ( !poll || !clr ) - { -@@ -725,7 +728,13 @@ static int cpu_callback( - - case CPU_UP_CANCELED: - case CPU_DEAD: -- cpu_bank_free(cpu); -+ if ( !park_offline_cpus ) -+ cpu_bank_free(cpu); -+ break; -+ -+ case CPU_REMOVE: -+ if ( park_offline_cpus ) -+ cpu_bank_free(cpu); - break; - } - -diff --git a/xen/arch/x86/cpu/mcheck/mce_intel.c b/xen/arch/x86/cpu/mcheck/mce_intel.c -index e5dd956a24..4474a34e34 100644 ---- a/xen/arch/x86/cpu/mcheck/mce_intel.c -+++ b/xen/arch/x86/cpu/mcheck/mce_intel.c -@@ -636,8 +636,6 @@ static void clear_cmci(void) - - static void cpu_mcheck_disable(void) - { -- clear_in_cr4(X86_CR4_MCE); -- - if ( cmci_support && opt_mce ) - clear_cmci(); - } -diff --git a/xen/arch/x86/cpu/vpmu_intel.c b/xen/arch/x86/cpu/vpmu_intel.c -index 207e2e712c..6e27f6ec8e 100644 ---- a/xen/arch/x86/cpu/vpmu_intel.c -+++ b/xen/arch/x86/cpu/vpmu_intel.c -@@ -454,13 +454,11 @@ static int core2_vpmu_alloc_resource(struct vcpu *v) - - if ( is_hvm_vcpu(v) ) - { -- wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0); -- if ( vmx_add_host_load_msr(MSR_CORE_PERF_GLOBAL_CTRL) ) -+ if ( vmx_add_host_load_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, 0) ) - goto out_err; - -- if ( vmx_add_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL) ) -+ if ( vmx_add_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, 0) ) - goto out_err; -- vmx_write_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, 0); - } - - core2_vpmu_cxt = xzalloc_bytes(sizeof(*core2_vpmu_cxt) + -@@ -535,27 +533,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content, - uint64_t *enabled_cntrs; - - if ( !core2_vpmu_msr_common_check(msr, &type, &index) ) -- { -- /* Special handling for BTS */ -- if ( msr == MSR_IA32_DEBUGCTLMSR ) -- { -- supported |= IA32_DEBUGCTLMSR_TR | IA32_DEBUGCTLMSR_BTS | -- IA32_DEBUGCTLMSR_BTINT; -- -- if ( cpu_has(¤t_cpu_data, X86_FEATURE_DSCPL) ) -- supported |= IA32_DEBUGCTLMSR_BTS_OFF_OS | -- IA32_DEBUGCTLMSR_BTS_OFF_USR; -- if ( !(msr_content & ~supported) && -- vpmu_is_set(vpmu, VPMU_CPU_HAS_BTS) ) -- return 0; -- if ( (msr_content & supported) && -- !vpmu_is_set(vpmu, VPMU_CPU_HAS_BTS) ) -- printk(XENLOG_G_WARNING -- "%pv: Debug Store unsupported on this CPU\n", -- current); -- } - return -EINVAL; -- } - - ASSERT(!supported); - -@@ -613,7 +591,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content, - return -EINVAL; - - if ( is_hvm_vcpu(v) ) -- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, -+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, - &core2_vpmu_cxt->global_ctrl); - else - rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, core2_vpmu_cxt->global_ctrl); -@@ -682,7 +660,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content, - return -EINVAL; - - if ( is_hvm_vcpu(v) ) -- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, -+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, - &core2_vpmu_cxt->global_ctrl); - else - rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, core2_vpmu_cxt->global_ctrl); -@@ -701,7 +679,7 @@ static int core2_vpmu_do_wrmsr(unsigned int msr, uint64_t msr_content, - else - { - if ( is_hvm_vcpu(v) ) -- vmx_write_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, msr_content); -+ vmx_write_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, msr_content); - else - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, msr_content); - } -@@ -735,7 +713,7 @@ static int core2_vpmu_do_rdmsr(unsigned int msr, uint64_t *msr_content) - break; - case MSR_CORE_PERF_GLOBAL_CTRL: - if ( is_hvm_vcpu(v) ) -- vmx_read_guest_msr(MSR_CORE_PERF_GLOBAL_CTRL, msr_content); -+ vmx_read_guest_msr(v, MSR_CORE_PERF_GLOBAL_CTRL, msr_content); - else - rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, *msr_content); - break; -diff --git a/xen/arch/x86/cpuid.c b/xen/arch/x86/cpuid.c -index beee47d0ed..5cc89e2b34 100644 ---- a/xen/arch/x86/cpuid.c -+++ b/xen/arch/x86/cpuid.c -@@ -43,6 +43,11 @@ static int __init parse_xen_cpuid(const char *s) - if ( !val ) - setup_clear_cpu_cap(X86_FEATURE_STIBP); - } -+ else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 ) -+ { -+ if ( !val ) -+ setup_clear_cpu_cap(X86_FEATURE_L1D_FLUSH); -+ } - else if ( (val = parse_boolean("ssbd", s, ss)) >= 0 ) - { - if ( !val ) -diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c -index 9850a782ec..c39cf2c6e5 100644 ---- a/xen/arch/x86/domain.c -+++ b/xen/arch/x86/domain.c -@@ -107,10 +107,11 @@ static void play_dead(void) - local_irq_disable(); - - /* -- * NOTE: After cpu_exit_clear, per-cpu variables are no longer accessible, -- * as they may be freed at any time. In this case, heap corruption or -- * #PF can occur (when heap debugging is enabled). For example, even -- * printk() can involve tasklet scheduling, which touches per-cpu vars. -+ * NOTE: After cpu_exit_clear, per-cpu variables may no longer accessible, -+ * as they may be freed at any time if offline CPUs don't get parked. In -+ * this case, heap corruption or #PF can occur (when heap debugging is -+ * enabled). For example, even printk() can involve tasklet scheduling, -+ * which touches per-cpu vars. - * - * Consider very carefully when adding code to *dead_idle. Most hypervisor - * subsystems are unsafe to call. -diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c -index 8fbbf3aeb3..dd91038a67 100644 ---- a/xen/arch/x86/domctl.c -+++ b/xen/arch/x86/domctl.c -@@ -225,7 +225,8 @@ static int update_domain_cpuid_info(struct domain *d, - */ - call_policy_changed = (is_hvm_domain(d) && - ((old_7d0 ^ p->feat.raw[0].d) & -- cpufeat_mask(X86_FEATURE_IBRSB))); -+ (cpufeat_mask(X86_FEATURE_IBRSB) | -+ cpufeat_mask(X86_FEATURE_L1D_FLUSH)))); - break; - - case 0xa: -@@ -1163,7 +1164,7 @@ long arch_do_domctl( - if ( _xcr0_accum ) - { - if ( evc->size >= PV_XSAVE_HDR_SIZE + XSTATE_AREA_MIN_SIZE ) -- ret = validate_xstate(_xcr0, _xcr0_accum, -+ ret = validate_xstate(d, _xcr0, _xcr0_accum, - &_xsave_area->xsave_hdr); - } - else if ( !_xcr0 ) -@@ -1187,8 +1188,7 @@ long arch_do_domctl( - vcpu_pause(v); - v->arch.xcr0 = _xcr0; - v->arch.xcr0_accum = _xcr0_accum; -- if ( _xcr0_accum & XSTATE_NONLAZY ) -- v->arch.nonlazy_xstate_used = 1; -+ v->arch.nonlazy_xstate_used = _xcr0_accum & XSTATE_NONLAZY; - compress_xsave_states(v, _xsave_area, - evc->size - PV_XSAVE_HDR_SIZE); - vcpu_unpause(v); -diff --git a/xen/arch/x86/genapic/x2apic.c b/xen/arch/x86/genapic/x2apic.c -index 4779b0d0d5..d997806272 100644 ---- a/xen/arch/x86/genapic/x2apic.c -+++ b/xen/arch/x86/genapic/x2apic.c -@@ -201,18 +201,21 @@ static int update_clusterinfo( - if ( !cluster_cpus_spare ) - cluster_cpus_spare = xzalloc(cpumask_t); - if ( !cluster_cpus_spare || -- !alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) ) -+ !cond_alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) ) - err = -ENOMEM; - break; - case CPU_UP_CANCELED: - case CPU_DEAD: -+ case CPU_REMOVE: -+ if ( park_offline_cpus == (action != CPU_REMOVE) ) -+ break; - if ( per_cpu(cluster_cpus, cpu) ) - { - cpumask_clear_cpu(cpu, per_cpu(cluster_cpus, cpu)); - if ( cpumask_empty(per_cpu(cluster_cpus, cpu)) ) -- xfree(per_cpu(cluster_cpus, cpu)); -+ XFREE(per_cpu(cluster_cpus, cpu)); - } -- free_cpumask_var(per_cpu(scratch_mask, cpu)); -+ FREE_CPUMASK_VAR(per_cpu(scratch_mask, cpu)); - break; - } - -diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c -index c23983cdff..4cbb688c05 100644 ---- a/xen/arch/x86/hvm/hvm.c -+++ b/xen/arch/x86/hvm/hvm.c -@@ -907,6 +907,9 @@ const char *hvm_efer_valid(const struct vcpu *v, uint64_t value, - else - p = &host_cpuid_policy; - -+ if ( value & ~EFER_KNOWN_MASK ) -+ return "Unknown bits set"; -+ - if ( (value & EFER_SCE) && !p->extd.syscall ) - return "SCE without feature"; - -@@ -1269,7 +1272,7 @@ static int hvm_load_cpu_xsave_states(struct domain *d, hvm_domain_context_t *h) - ctxt = (struct hvm_hw_cpu_xsave *)&h->data[h->cur]; - h->cur += desc->length; - -- err = validate_xstate(ctxt->xcr0, ctxt->xcr0_accum, -+ err = validate_xstate(d, ctxt->xcr0, ctxt->xcr0_accum, - (const void *)&ctxt->save_area.xsave_hdr); - if ( err ) - { -@@ -1324,8 +1327,7 @@ static int hvm_load_cpu_xsave_states(struct domain *d, hvm_domain_context_t *h) - - v->arch.xcr0 = ctxt->xcr0; - v->arch.xcr0_accum = ctxt->xcr0_accum; -- if ( ctxt->xcr0_accum & XSTATE_NONLAZY ) -- v->arch.nonlazy_xstate_used = 1; -+ v->arch.nonlazy_xstate_used = ctxt->xcr0_accum & XSTATE_NONLAZY; - compress_xsave_states(v, &ctxt->save_area, - size - offsetof(struct hvm_hw_cpu_xsave, save_area)); - -diff --git a/xen/arch/x86/hvm/svm/svm.c b/xen/arch/x86/hvm/svm/svm.c -index 165500e3f2..b964c59dad 100644 ---- a/xen/arch/x86/hvm/svm/svm.c -+++ b/xen/arch/x86/hvm/svm/svm.c -@@ -1432,24 +1432,18 @@ static void svm_inject_event(const struct x86_event *event) - * Xen must emulate enough of the event injection to be sure that a - * further fault shouldn't occur during delivery. This covers the fact - * that hardware doesn't perform DPL checking on injection. -- * -- * Also, it accounts for proper positioning of %rip for an event with trap -- * semantics (where %rip should point after the instruction) which suffers -- * a fault during injection (at which point %rip should point at the -- * instruction). - */ - if ( event->type == X86_EVENTTYPE_PRI_SW_EXCEPTION || -- (!cpu_has_svm_nrips && (event->type == X86_EVENTTYPE_SW_INTERRUPT || -- event->type == X86_EVENTTYPE_SW_EXCEPTION)) ) -+ (!cpu_has_svm_nrips && (event->type >= X86_EVENTTYPE_SW_INTERRUPT)) ) - svm_emul_swint_injection(&_event); - -- switch ( _event.vector ) -+ switch ( _event.vector | -(_event.type == X86_EVENTTYPE_SW_INTERRUPT) ) - { - case TRAP_debug: - if ( regs->eflags & X86_EFLAGS_TF ) - { - __restore_debug_registers(vmcb, curr); -- vmcb_set_dr6(vmcb, vmcb_get_dr6(vmcb) | 0x4000); -+ vmcb_set_dr6(vmcb, vmcb_get_dr6(vmcb) | DR_STEP); - } - /* fall through */ - case TRAP_int3: -@@ -1459,6 +1453,13 @@ static void svm_inject_event(const struct x86_event *event) - domain_pause_for_debugger(); - return; - } -+ break; -+ -+ case TRAP_page_fault: -+ ASSERT(_event.type == X86_EVENTTYPE_HW_EXCEPTION); -+ curr->arch.hvm_vcpu.guest_cr[2] = _event.cr2; -+ vmcb_set_cr2(vmcb, _event.cr2); -+ break; - } - - if ( unlikely(eventinj.fields.v) && -@@ -1481,13 +1482,9 @@ static void svm_inject_event(const struct x86_event *event) - * icebp, software events with trap semantics need emulating, so %rip in - * the trap frame points after the instruction. - * -- * The x86 emulator (if requested by the x86_swint_emulate_* choice) will -- * have performed checks such as presence/dpl/etc and believes that the -- * event injection will succeed without faulting. -- * -- * The x86 emulator will always provide fault semantics for software -- * events, with _trap.insn_len set appropriately. If the injection -- * requires emulation, move %rip forwards at this point. -+ * svm_emul_swint_injection() has already confirmed that events with trap -+ * semantics won't fault on injection. Position %rip/NextRIP suitably, -+ * and restrict the event type to what hardware will tolerate. - */ - switch ( _event.type ) - { -@@ -1544,16 +1541,12 @@ static void svm_inject_event(const struct x86_event *event) - eventinj.fields.errorcode == (uint16_t)eventinj.fields.errorcode); - vmcb->eventinj = eventinj; - -- if ( _event.vector == TRAP_page_fault ) -- { -- curr->arch.hvm_vcpu.guest_cr[2] = _event.cr2; -- vmcb_set_cr2(vmcb, _event.cr2); -- HVMTRACE_LONG_2D(PF_INJECT, _event.error_code, TRC_PAR_LONG(_event.cr2)); -- } -+ if ( _event.vector == TRAP_page_fault && -+ _event.type == X86_EVENTTYPE_HW_EXCEPTION ) -+ HVMTRACE_LONG_2D(PF_INJECT, _event.error_code, -+ TRC_PAR_LONG(_event.cr2)); - else -- { - HVMTRACE_2D(INJ_EXC, _event.vector, _event.error_code); -- } - } - - static int svm_event_pending(struct vcpu *v) -diff --git a/xen/arch/x86/hvm/vmx/entry.S b/xen/arch/x86/hvm/vmx/entry.S -index aa2f103895..afd552f2b9 100644 ---- a/xen/arch/x86/hvm/vmx/entry.S -+++ b/xen/arch/x86/hvm/vmx/entry.S -@@ -41,6 +41,15 @@ ENTRY(vmx_asm_vmexit_handler) - SPEC_CTRL_ENTRY_FROM_HVM /* Req: b=curr %rsp=regs/cpuinfo, Clob: acd */ - /* WARNING! `ret`, `call *`, `jmp *` not safe before this point. */ - -+ /* Hardware clears MSR_DEBUGCTL on VMExit. Reinstate it if debugging Xen. */ -+ .macro restore_lbr -+ mov $IA32_DEBUGCTLMSR_LBR, %eax -+ mov $MSR_IA32_DEBUGCTLMSR, %ecx -+ xor %edx, %edx -+ wrmsr -+ .endm -+ ALTERNATIVE "", restore_lbr, X86_FEATURE_XEN_LBR -+ - mov %rsp,%rdi - call vmx_vmexit_handler - -diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c -index 258fc08f72..2ba0c40808 100644 ---- a/xen/arch/x86/hvm/vmx/vmcs.c -+++ b/xen/arch/x86/hvm/vmx/vmcs.c -@@ -38,6 +38,7 @@ - #include <asm/flushtlb.h> - #include <asm/monitor.h> - #include <asm/shadow.h> -+#include <asm/spec_ctrl.h> - #include <asm/tboot.h> - #include <asm/apic.h> - -@@ -996,6 +997,7 @@ static int construct_vmcs(struct vcpu *v) - struct domain *d = v->domain; - u32 vmexit_ctl = vmx_vmexit_control; - u32 vmentry_ctl = vmx_vmentry_control; -+ int rc = 0; - - vmx_vmcs_enter(v); - -@@ -1083,8 +1085,8 @@ static int construct_vmcs(struct vcpu *v) - - if ( msr_bitmap == NULL ) - { -- vmx_vmcs_exit(v); -- return -ENOMEM; -+ rc = -ENOMEM; -+ goto out; - } - - memset(msr_bitmap, ~0, PAGE_SIZE); -@@ -1268,141 +1270,197 @@ static int construct_vmcs(struct vcpu *v) - if ( cpu_has_vmx_tsc_scaling ) - __vmwrite(TSC_MULTIPLIER, d->arch.hvm_domain.tsc_scaling_ratio); - -- vmx_vmcs_exit(v); -- - /* will update HOST & GUEST_CR3 as reqd */ - paging_update_paging_modes(v); - - vmx_vlapic_msr_changed(v); - -- return 0; -+ if ( opt_l1d_flush && paging_mode_hap(d) ) -+ rc = vmx_add_msr(v, MSR_FLUSH_CMD, FLUSH_CMD_L1D, -+ VMX_MSR_GUEST_LOADONLY); -+ -+ out: -+ vmx_vmcs_exit(v); -+ -+ return rc; - } - --static int vmx_msr_entry_key_cmp(const void *key, const void *elt) -+/* -+ * Search an MSR list looking for an MSR entry, or the slot in which it should -+ * live (to keep the data sorted) if an entry is not found. -+ * -+ * The return pointer is guaranteed to be bounded by start and end. However, -+ * it may point at end, and may be invalid for the caller to dereference. -+ */ -+static struct vmx_msr_entry *locate_msr_entry( -+ struct vmx_msr_entry *start, struct vmx_msr_entry *end, uint32_t msr) - { -- const u32 *msr = key; -- const struct vmx_msr_entry *entry = elt; -+ while ( start < end ) -+ { -+ struct vmx_msr_entry *mid = start + (end - start) / 2; - -- if ( *msr > entry->index ) -- return 1; -- if ( *msr < entry->index ) -- return -1; -+ if ( msr < mid->index ) -+ end = mid; -+ else if ( msr > mid->index ) -+ start = mid + 1; -+ else -+ return mid; -+ } - -- return 0; -+ return start; - } - --struct vmx_msr_entry *vmx_find_msr(u32 msr, int type) -+struct vmx_msr_entry *vmx_find_msr(const struct vcpu *v, uint32_t msr, -+ enum vmx_msr_list_type type) - { -- struct vcpu *curr = current; -- unsigned int msr_count; -- struct vmx_msr_entry *msr_area; -+ const struct arch_vmx_struct *vmx = &v->arch.hvm_vmx; -+ struct vmx_msr_entry *start = NULL, *ent, *end; -+ unsigned int substart = 0, subend = vmx->msr_save_count; -+ unsigned int total = vmx->msr_load_count; - -- if ( type == VMX_GUEST_MSR ) -- { -- msr_count = curr->arch.hvm_vmx.msr_count; -- msr_area = curr->arch.hvm_vmx.msr_area; -- } -- else -+ ASSERT(v == current || !vcpu_runnable(v)); -+ -+ switch ( type ) - { -- ASSERT(type == VMX_HOST_MSR); -- msr_count = curr->arch.hvm_vmx.host_msr_count; -- msr_area = curr->arch.hvm_vmx.host_msr_area; -+ case VMX_MSR_HOST: -+ start = vmx->host_msr_area; -+ subend = vmx->host_msr_count; -+ total = subend; -+ break; -+ -+ case VMX_MSR_GUEST: -+ start = vmx->msr_area; -+ break; -+ -+ case VMX_MSR_GUEST_LOADONLY: -+ start = vmx->msr_area; -+ substart = subend; -+ subend = total; -+ break; -+ -+ default: -+ ASSERT_UNREACHABLE(); - } - -- if ( msr_area == NULL ) -+ if ( !start ) - return NULL; - -- return bsearch(&msr, msr_area, msr_count, sizeof(struct vmx_msr_entry), -- vmx_msr_entry_key_cmp); -+ end = start + total; -+ ent = locate_msr_entry(start + substart, start + subend, msr); -+ -+ return ((ent < end) && (ent->index == msr)) ? ent : NULL; - } - --int vmx_read_guest_msr(u32 msr, u64 *val) -+int vmx_add_msr(struct vcpu *v, uint32_t msr, uint64_t val, -+ enum vmx_msr_list_type type) - { -- struct vmx_msr_entry *ent; -+ struct arch_vmx_struct *vmx = &v->arch.hvm_vmx; -+ struct vmx_msr_entry **ptr, *start = NULL, *ent, *end; -+ unsigned int substart, subend, total; -+ int rc; - -- if ( (ent = vmx_find_msr(msr, VMX_GUEST_MSR)) != NULL ) -+ ASSERT(v == current || !vcpu_runnable(v)); -+ -+ switch ( type ) - { -- *val = ent->data; -- return 0; -- } -+ case VMX_MSR_HOST: -+ ptr = &vmx->host_msr_area; -+ substart = 0; -+ subend = vmx->host_msr_count; -+ total = subend; -+ break; - -- return -ESRCH; --} -+ case VMX_MSR_GUEST: -+ ptr = &vmx->msr_area; -+ substart = 0; -+ subend = vmx->msr_save_count; -+ total = vmx->msr_load_count; -+ break; - --int vmx_write_guest_msr(u32 msr, u64 val) --{ -- struct vmx_msr_entry *ent; -+ case VMX_MSR_GUEST_LOADONLY: -+ ptr = &vmx->msr_area; -+ substart = vmx->msr_save_count; -+ subend = vmx->msr_load_count; -+ total = subend; -+ break; - -- if ( (ent = vmx_find_msr(msr, VMX_GUEST_MSR)) != NULL ) -- { -- ent->data = val; -- return 0; -+ default: -+ ASSERT_UNREACHABLE(); -+ return -EINVAL; - } - -- return -ESRCH; --} -- --int vmx_add_msr(u32 msr, int type) --{ -- struct vcpu *curr = current; -- unsigned int idx, *msr_count; -- struct vmx_msr_entry **msr_area, *msr_area_elem; -+ vmx_vmcs_enter(v); - -- if ( type == VMX_GUEST_MSR ) -- { -- msr_count = &curr->arch.hvm_vmx.msr_count; -- msr_area = &curr->arch.hvm_vmx.msr_area; -- } -- else -+ /* Allocate memory on first use. */ -+ if ( unlikely(!*ptr) ) - { -- ASSERT(type == VMX_HOST_MSR); -- msr_count = &curr->arch.hvm_vmx.host_msr_count; -- msr_area = &curr->arch.hvm_vmx.host_msr_area; -- } -+ paddr_t addr; - -- if ( *msr_area == NULL ) -- { -- if ( (*msr_area = alloc_xenheap_page()) == NULL ) -- return -ENOMEM; -+ if ( (*ptr = alloc_xenheap_page()) == NULL ) -+ { -+ rc = -ENOMEM; -+ goto out; -+ } - -- if ( type == VMX_GUEST_MSR ) -+ addr = virt_to_maddr(*ptr); -+ -+ switch ( type ) - { -- __vmwrite(VM_EXIT_MSR_STORE_ADDR, virt_to_maddr(*msr_area)); -- __vmwrite(VM_ENTRY_MSR_LOAD_ADDR, virt_to_maddr(*msr_area)); -+ case VMX_MSR_HOST: -+ __vmwrite(VM_EXIT_MSR_LOAD_ADDR, addr); -+ break; -+ -+ case VMX_MSR_GUEST: -+ case VMX_MSR_GUEST_LOADONLY: -+ __vmwrite(VM_EXIT_MSR_STORE_ADDR, addr); -+ __vmwrite(VM_ENTRY_MSR_LOAD_ADDR, addr); -+ break; - } -- else -- __vmwrite(VM_EXIT_MSR_LOAD_ADDR, virt_to_maddr(*msr_area)); - } - -- for ( idx = 0; idx < *msr_count && (*msr_area)[idx].index <= msr; idx++ ) -- if ( (*msr_area)[idx].index == msr ) -- return 0; -+ start = *ptr; -+ end = start + total; -+ ent = locate_msr_entry(start + substart, start + subend, msr); - -- if ( *msr_count == (PAGE_SIZE / sizeof(struct vmx_msr_entry)) ) -- return -ENOSPC; -+ if ( (ent < end) && (ent->index == msr) ) -+ goto found; - -- memmove(*msr_area + idx + 1, *msr_area + idx, -- sizeof(*msr_area_elem) * (*msr_count - idx)); -+ /* If there isn't an existing entry for msr, insert room for one. */ -+ if ( total == (PAGE_SIZE / sizeof(*ent)) ) -+ { -+ rc = -ENOSPC; -+ goto out; -+ } - -- msr_area_elem = *msr_area + idx; -- msr_area_elem->index = msr; -- msr_area_elem->mbz = 0; -+ memmove(ent + 1, ent, sizeof(*ent) * (end - ent)); - -- ++*msr_count; -+ ent->index = msr; -+ ent->mbz = 0; - -- if ( type == VMX_GUEST_MSR ) -+ switch ( type ) - { -- msr_area_elem->data = 0; -- __vmwrite(VM_EXIT_MSR_STORE_COUNT, *msr_count); -- __vmwrite(VM_ENTRY_MSR_LOAD_COUNT, *msr_count); -- } -- else -- { -- rdmsrl(msr, msr_area_elem->data); -- __vmwrite(VM_EXIT_MSR_LOAD_COUNT, *msr_count); -+ case VMX_MSR_HOST: -+ __vmwrite(VM_EXIT_MSR_LOAD_COUNT, ++vmx->host_msr_count); -+ break; -+ -+ case VMX_MSR_GUEST: -+ __vmwrite(VM_EXIT_MSR_STORE_COUNT, ++vmx->msr_save_count); -+ -+ /* Fallthrough */ -+ case VMX_MSR_GUEST_LOADONLY: -+ __vmwrite(VM_ENTRY_MSR_LOAD_COUNT, ++vmx->msr_load_count); -+ break; - } - -- return 0; -+ /* Set the msr's value. */ -+ found: -+ ent->data = val; -+ rc = 0; -+ -+ out: -+ vmx_vmcs_exit(v); -+ -+ return rc; - } - - void vmx_set_eoi_exit_bitmap(struct vcpu *v, u8 vector) -diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c -index 610c8d6eb9..b0fababede 100644 ---- a/xen/arch/x86/hvm/vmx/vmx.c -+++ b/xen/arch/x86/hvm/vmx/vmx.c -@@ -583,6 +583,12 @@ static void vmx_cpuid_policy_changed(struct vcpu *v) - vmx_clear_msr_intercept(v, MSR_PRED_CMD, VMX_MSR_RW); - else - vmx_set_msr_intercept(v, MSR_PRED_CMD, VMX_MSR_RW); -+ -+ /* MSR_FLUSH_CMD is safe to pass through if the guest knows about it. */ -+ if ( cp->feat.l1d_flush ) -+ vmx_clear_msr_intercept(v, MSR_FLUSH_CMD, VMX_MSR_RW); -+ else -+ vmx_set_msr_intercept(v, MSR_FLUSH_CMD, VMX_MSR_RW); - } - - int vmx_guest_x86_mode(struct vcpu *v) -@@ -2758,8 +2764,10 @@ enum - - #define LBR_FROM_SIGNEXT_2MSB ((1ULL << 59) | (1ULL << 60)) - --#define FIXUP_LBR_TSX (1u << 0) --#define FIXUP_BDW_ERRATUM_BDF14 (1u << 1) -+#define LBR_MSRS_INSERTED (1u << 0) -+#define LBR_FIXUP_TSX (1u << 1) -+#define LBR_FIXUP_BDF14 (1u << 2) -+#define LBR_FIXUP_MASK (LBR_FIXUP_TSX | LBR_FIXUP_BDF14) - - static bool __read_mostly lbr_tsx_fixup_needed; - static bool __read_mostly bdw_erratum_bdf14_fixup_needed; -@@ -2822,7 +2830,7 @@ static int is_last_branch_msr(u32 ecx) - - static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content) - { -- const struct vcpu *curr = current; -+ struct vcpu *curr = current; - - HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x", msr); - -@@ -2901,7 +2909,7 @@ static int vmx_msr_read_intercept(unsigned int msr, uint64_t *msr_content) - if ( passive_domain_do_rdmsr(msr, msr_content) ) - goto done; - -- if ( vmx_read_guest_msr(msr, msr_content) == 0 ) -+ if ( vmx_read_guest_msr(curr, msr, msr_content) == 0 ) - break; - - if ( is_last_branch_msr(msr) ) -@@ -3036,11 +3044,14 @@ void vmx_vlapic_msr_changed(struct vcpu *v) - static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content) - { - struct vcpu *v = current; -+ const struct cpuid_policy *cp = v->domain->arch.cpuid; - - HVM_DBG_LOG(DBG_LEVEL_MSR, "ecx=%#x, msr_value=%#"PRIx64, msr, msr_content); - - switch ( msr ) - { -+ uint64_t rsvd; -+ - case MSR_IA32_SYSENTER_CS: - __vmwrite(GUEST_SYSENTER_CS, msr_content); - break; -@@ -3093,45 +3104,85 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content) - wrmsrl(MSR_SYSCALL_MASK, msr_content); - break; - -- case MSR_IA32_DEBUGCTLMSR: { -- int i, rc = 0; -- uint64_t supported = IA32_DEBUGCTLMSR_LBR | IA32_DEBUGCTLMSR_BTF; -+ case MSR_IA32_DEBUGCTLMSR: -+ rsvd = ~(IA32_DEBUGCTLMSR_LBR | IA32_DEBUGCTLMSR_BTF); - -- if ( boot_cpu_has(X86_FEATURE_RTM) ) -- supported |= IA32_DEBUGCTLMSR_RTM; -- if ( msr_content & ~supported ) -+ /* TODO: Wire vPMU settings properly through the CPUID policy */ -+ if ( vpmu_is_set(vcpu_vpmu(v), VPMU_CPU_HAS_BTS) ) - { -- /* Perhaps some other bits are supported in vpmu. */ -- if ( vpmu_do_wrmsr(msr, msr_content, supported) ) -- break; -+ rsvd &= ~(IA32_DEBUGCTLMSR_TR | IA32_DEBUGCTLMSR_BTS | -+ IA32_DEBUGCTLMSR_BTINT); -+ -+ if ( cpu_has(¤t_cpu_data, X86_FEATURE_DSCPL) ) -+ rsvd &= ~(IA32_DEBUGCTLMSR_BTS_OFF_OS | -+ IA32_DEBUGCTLMSR_BTS_OFF_USR); - } -- if ( msr_content & IA32_DEBUGCTLMSR_LBR ) -+ -+ if ( cp->feat.rtm ) -+ rsvd &= ~IA32_DEBUGCTLMSR_RTM; -+ -+ if ( msr_content & rsvd ) -+ goto gp_fault; -+ -+ /* -+ * When a guest first enables LBR, arrange to save and restore the LBR -+ * MSRs and allow the guest direct access. -+ * -+ * MSR_DEBUGCTL and LBR has existed almost as long as MSRs have -+ * existed, and there is no architectural way to hide the feature, or -+ * fail the attempt to enable LBR. -+ * -+ * Unknown host LBR MSRs or hitting -ENOSPC with the guest load/save -+ * list are definitely hypervisor bugs, whereas -ENOMEM for allocating -+ * the load/save list is simply unlucky (and shouldn't occur with -+ * sensible management by the toolstack). -+ * -+ * Either way, there is nothing we can do right now to recover, and -+ * the guest won't execute correctly either. Simply crash the domain -+ * to make the failure obvious. -+ */ -+ if ( !(v->arch.hvm_vmx.lbr_flags & LBR_MSRS_INSERTED) && -+ (msr_content & IA32_DEBUGCTLMSR_LBR) ) - { - const struct lbr_info *lbr = last_branch_msr_get(); -- if ( lbr == NULL ) -- break; - -- for ( ; (rc == 0) && lbr->count; lbr++ ) -- for ( i = 0; (rc == 0) && (i < lbr->count); i++ ) -- if ( (rc = vmx_add_guest_msr(lbr->base + i)) == 0 ) -+ if ( unlikely(!lbr) ) -+ { -+ gprintk(XENLOG_ERR, "Unknown Host LBR MSRs\n"); -+ domain_crash(v->domain); -+ return X86EMUL_OKAY; -+ } -+ -+ for ( ; lbr->count; lbr++ ) -+ { -+ unsigned int i; -+ -+ for ( i = 0; i < lbr->count; i++ ) -+ { -+ int rc = vmx_add_guest_msr(v, lbr->base + i, 0); -+ -+ if ( unlikely(rc) ) - { -- vmx_clear_msr_intercept(v, lbr->base + i, VMX_MSR_RW); -- if ( lbr_tsx_fixup_needed ) -- v->arch.hvm_vmx.lbr_fixup_enabled |= FIXUP_LBR_TSX; -- if ( bdw_erratum_bdf14_fixup_needed ) -- v->arch.hvm_vmx.lbr_fixup_enabled |= -- FIXUP_BDW_ERRATUM_BDF14; -+ gprintk(XENLOG_ERR, -+ "Guest load/save list error %d\n", rc); -+ domain_crash(v->domain); -+ return X86EMUL_OKAY; - } -- } - -- if ( (rc < 0) || -- (msr_content && (vmx_add_host_load_msr(msr) < 0)) ) -- hvm_inject_hw_exception(TRAP_machine_check, X86_EVENT_NO_EC); -- else -- __vmwrite(GUEST_IA32_DEBUGCTL, msr_content); -+ vmx_clear_msr_intercept(v, lbr->base + i, VMX_MSR_RW); -+ } -+ } -+ -+ v->arch.hvm_vmx.lbr_flags |= LBR_MSRS_INSERTED; -+ if ( lbr_tsx_fixup_needed ) -+ v->arch.hvm_vmx.lbr_flags |= LBR_FIXUP_TSX; -+ if ( bdw_erratum_bdf14_fixup_needed ) -+ v->arch.hvm_vmx.lbr_flags |= LBR_FIXUP_BDF14; -+ } - -+ __vmwrite(GUEST_IA32_DEBUGCTL, msr_content); - break; -- } -+ - case MSR_IA32_FEATURE_CONTROL: - case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC: - /* None of these MSRs are writeable. */ -@@ -3154,7 +3205,7 @@ static int vmx_msr_write_intercept(unsigned int msr, uint64_t msr_content) - if ( wrmsr_viridian_regs(msr, msr_content) ) - break; - -- if ( vmx_write_guest_msr(msr, msr_content) == 0 || -+ if ( vmx_write_guest_msr(v, msr, msr_content) == 0 || - is_last_branch_msr(msr) ) - break; - -@@ -3701,6 +3752,7 @@ void vmx_vmexit_handler(struct cpu_user_regs *regs) - */ - __vmread(EXIT_QUALIFICATION, &exit_qualification); - HVMTRACE_1D(TRAP_DEBUG, exit_qualification); -+ __restore_debug_registers(v); - write_debugreg(6, exit_qualification | DR_STATUS_RESERVED_ONE); - if ( !v->domain->debugger_attached ) - { -@@ -4165,11 +4217,11 @@ out: - static void lbr_tsx_fixup(void) - { - struct vcpu *curr = current; -- unsigned int msr_count = curr->arch.hvm_vmx.msr_count; -+ unsigned int msr_count = curr->arch.hvm_vmx.msr_save_count; - struct vmx_msr_entry *msr_area = curr->arch.hvm_vmx.msr_area; - struct vmx_msr_entry *msr; - -- if ( (msr = vmx_find_msr(lbr_from_start, VMX_GUEST_MSR)) != NULL ) -+ if ( (msr = vmx_find_msr(curr, lbr_from_start, VMX_MSR_GUEST)) != NULL ) - { - /* - * Sign extend into bits 61:62 while preserving bit 63 -@@ -4179,15 +4231,15 @@ static void lbr_tsx_fixup(void) - msr->data |= ((LBR_FROM_SIGNEXT_2MSB & msr->data) << 2); - } - -- if ( (msr = vmx_find_msr(lbr_lastint_from, VMX_GUEST_MSR)) != NULL ) -+ if ( (msr = vmx_find_msr(curr, lbr_lastint_from, VMX_MSR_GUEST)) != NULL ) - msr->data |= ((LBR_FROM_SIGNEXT_2MSB & msr->data) << 2); - } - --static void sign_extend_msr(u32 msr, int type) -+static void sign_extend_msr(struct vcpu *v, u32 msr, int type) - { - struct vmx_msr_entry *entry; - -- if ( (entry = vmx_find_msr(msr, type)) != NULL ) -+ if ( (entry = vmx_find_msr(v, msr, type)) != NULL ) - { - if ( entry->data & VADDR_TOP_BIT ) - entry->data |= CANONICAL_MASK; -@@ -4198,6 +4250,8 @@ static void sign_extend_msr(u32 msr, int type) - - static void bdw_erratum_bdf14_fixup(void) - { -+ struct vcpu *curr = current; -+ - /* - * Occasionally, on certain Broadwell CPUs MSR_IA32_LASTINTTOIP has - * been observed to have the top three bits corrupted as though the -@@ -4207,17 +4261,17 @@ static void bdw_erratum_bdf14_fixup(void) - * erratum BDF14. Fix up MSR_IA32_LASTINT{FROM,TO}IP by - * sign-extending into bits 48:63. - */ -- sign_extend_msr(MSR_IA32_LASTINTFROMIP, VMX_GUEST_MSR); -- sign_extend_msr(MSR_IA32_LASTINTTOIP, VMX_GUEST_MSR); -+ sign_extend_msr(curr, MSR_IA32_LASTINTFROMIP, VMX_MSR_GUEST); -+ sign_extend_msr(curr, MSR_IA32_LASTINTTOIP, VMX_MSR_GUEST); - } - - static void lbr_fixup(void) - { - struct vcpu *curr = current; - -- if ( curr->arch.hvm_vmx.lbr_fixup_enabled & FIXUP_LBR_TSX ) -+ if ( curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_TSX ) - lbr_tsx_fixup(); -- if ( curr->arch.hvm_vmx.lbr_fixup_enabled & FIXUP_BDW_ERRATUM_BDF14 ) -+ if ( curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_BDF14 ) - bdw_erratum_bdf14_fixup(); - } - -@@ -4285,7 +4339,7 @@ bool vmx_vmenter_helper(const struct cpu_user_regs *regs) - } - - out: -- if ( unlikely(curr->arch.hvm_vmx.lbr_fixup_enabled) ) -+ if ( unlikely(curr->arch.hvm_vmx.lbr_flags & LBR_FIXUP_MASK) ) - lbr_fixup(); - - HVMTRACE_ND(VMENTRY, 0, 1/*cycles*/, 0, 0, 0, 0, 0, 0, 0); -diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c -index bcf46c0743..7d4871b791 100644 ---- a/xen/arch/x86/mm.c -+++ b/xen/arch/x86/mm.c -@@ -613,6 +613,9 @@ static int alloc_segdesc_page(struct page_info *page) - return i == 512 ? 0 : -EINVAL; - } - -+static int _get_page_type(struct page_info *page, unsigned long type, -+ bool preemptible); -+ - static int get_page_and_type_from_mfn( - mfn_t mfn, unsigned long type, struct domain *d, - int partial, int preemptible) -@@ -624,9 +627,7 @@ static int get_page_and_type_from_mfn( - unlikely(!get_page_from_mfn(mfn, d)) ) - return -EINVAL; - -- rc = (preemptible ? -- get_page_type_preemptible(page, type) : -- (get_page_type(page, type) ? 0 : -EINVAL)); -+ rc = _get_page_type(page, type, preemptible); - - if ( unlikely(rc) && partial >= 0 && - (!preemptible || page != current->arch.old_guest_table) ) -@@ -1115,7 +1116,7 @@ get_page_from_l2e( - int rc; - - if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) ) -- return 1; -+ return pv_l1tf_check_l2e(d, l2e) ? -ERESTART : 1; - - if ( unlikely((l2e_get_flags(l2e) & L2_DISALLOW_MASK)) ) - { -@@ -1146,7 +1147,7 @@ get_page_from_l3e( - int rc; - - if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) ) -- return 1; -+ return pv_l1tf_check_l3e(d, l3e) ? -ERESTART : 1; - - if ( unlikely((l3e_get_flags(l3e) & l3_disallow_mask(d))) ) - { -@@ -1179,7 +1180,7 @@ get_page_from_l4e( - int rc; - - if ( !(l4e_get_flags(l4e) & _PAGE_PRESENT) ) -- return 1; -+ return pv_l1tf_check_l4e(d, l4e) ? -ERESTART : 1; - - if ( unlikely((l4e_get_flags(l4e) & L4_DISALLOW_MASK)) ) - { -@@ -1389,6 +1390,13 @@ static int alloc_l1_table(struct page_info *page) - - for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ ) - { -+ if ( !(l1e_get_flags(pl1e[i]) & _PAGE_PRESENT) ) -+ { -+ ret = pv_l1tf_check_l1e(d, pl1e[i]) ? -ERESTART : 0; -+ if ( ret ) -+ goto out; -+ } -+ - switch ( ret = get_page_from_l1e(pl1e[i], d, d) ) - { - default: -@@ -1409,6 +1417,7 @@ static int alloc_l1_table(struct page_info *page) - - fail: - gdprintk(XENLOG_WARNING, "Failure in alloc_l1_table: slot %#x\n", i); -+ out: - while ( i-- > 0 ) - put_page_from_l1e(pl1e[i], d); - -@@ -1456,8 +1465,7 @@ static int create_pae_xen_mappings(struct domain *d, l3_pgentry_t *pl3e) - return 1; - } - --static int alloc_l2_table(struct page_info *page, unsigned long type, -- int preemptible) -+static int alloc_l2_table(struct page_info *page, unsigned long type) - { - struct domain *d = page_get_owner(page); - unsigned long pfn = mfn_x(page_to_mfn(page)); -@@ -1469,8 +1477,7 @@ static int alloc_l2_table(struct page_info *page, unsigned long type, - - for ( i = page->nr_validated_ptes; i < L2_PAGETABLE_ENTRIES; i++ ) - { -- if ( preemptible && i > page->nr_validated_ptes -- && hypercall_preempt_check() ) -+ if ( i > page->nr_validated_ptes && hypercall_preempt_check() ) - { - page->nr_validated_ptes = i; - rc = -ERESTART; -@@ -1481,6 +1488,12 @@ static int alloc_l2_table(struct page_info *page, unsigned long type, - (rc = get_page_from_l2e(pl2e[i], pfn, d)) > 0 ) - continue; - -+ if ( unlikely(rc == -ERESTART) ) -+ { -+ page->nr_validated_ptes = i; -+ break; -+ } -+ - if ( rc < 0 ) - { - gdprintk(XENLOG_WARNING, "Failure in alloc_l2_table: slot %#x\n", i); -@@ -1763,7 +1776,7 @@ static void free_l1_table(struct page_info *page) - } - - --static int free_l2_table(struct page_info *page, int preemptible) -+static int free_l2_table(struct page_info *page) - { - struct domain *d = page_get_owner(page); - unsigned long pfn = mfn_x(page_to_mfn(page)); -@@ -1777,7 +1790,7 @@ static int free_l2_table(struct page_info *page, int preemptible) - do { - if ( is_guest_l2_slot(d, page->u.inuse.type_info, i) && - put_page_from_l2e(pl2e[i], pfn) == 0 && -- preemptible && i && hypercall_preempt_check() ) -+ i && hypercall_preempt_check() ) - { - page->nr_validated_ptes = i; - err = -ERESTART; -@@ -2055,6 +2068,8 @@ static int mod_l1_entry(l1_pgentry_t *pl1e, l1_pgentry_t nl1e, - rc = -EBUSY; - } - } -+ else if ( pv_l1tf_check_l1e(pt_dom, nl1e) ) -+ return -ERESTART; - else if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, pt_vcpu, - preserve_ad)) ) - { -@@ -2118,6 +2133,8 @@ static int mod_l2_entry(l2_pgentry_t *pl2e, - rc = -EBUSY; - } - } -+ else if ( pv_l1tf_check_l2e(d, nl2e) ) -+ return -ERESTART; - else if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn, vcpu, - preserve_ad)) ) - { -@@ -2179,6 +2196,8 @@ static int mod_l3_entry(l3_pgentry_t *pl3e, - rc = -EFAULT; - } - } -+ else if ( pv_l1tf_check_l3e(d, nl3e) ) -+ return -ERESTART; - else if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, vcpu, - preserve_ad)) ) - { -@@ -2244,6 +2263,8 @@ static int mod_l4_entry(l4_pgentry_t *pl4e, - rc = -EFAULT; - } - } -+ else if ( pv_l1tf_check_l4e(d, nl4e) ) -+ return -ERESTART; - else if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, vcpu, - preserve_ad)) ) - { -@@ -2373,7 +2394,8 @@ static int alloc_page_type(struct page_info *page, unsigned long type, - rc = alloc_l1_table(page); - break; - case PGT_l2_page_table: -- rc = alloc_l2_table(page, type, preemptible); -+ ASSERT(preemptible); -+ rc = alloc_l2_table(page, type); - break; - case PGT_l3_page_table: - ASSERT(preemptible); -@@ -2463,7 +2485,8 @@ int free_page_type(struct page_info *page, unsigned long type, - rc = 0; - break; - case PGT_l2_page_table: -- rc = free_l2_table(page, preemptible); -+ ASSERT(preemptible); -+ rc = free_l2_table(page); - break; - case PGT_l3_page_table: - ASSERT(preemptible); -@@ -3550,12 +3573,9 @@ long do_mmuext_op( - } - - if ( rc == -ERESTART ) -- { -- ASSERT(i < count); - rc = hypercall_create_continuation( - __HYPERVISOR_mmuext_op, "hihi", - uops, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom); -- } - else if ( curr->arch.old_guest_table ) - { - XEN_GUEST_HANDLE_PARAM(void) null; -@@ -3861,12 +3881,9 @@ long do_mmu_update( - } - - if ( rc == -ERESTART ) -- { -- ASSERT(i < count); - rc = hypercall_create_continuation( - __HYPERVISOR_mmu_update, "hihi", - ureqs, (count - i) | MMU_UPDATE_PREEMPTED, pdone, foreigndom); -- } - else if ( curr->arch.old_guest_table ) - { - XEN_GUEST_HANDLE_PARAM(void) null; -@@ -4121,7 +4138,13 @@ static int __do_update_va_mapping( - long do_update_va_mapping(unsigned long va, u64 val64, - unsigned long flags) - { -- return __do_update_va_mapping(va, val64, flags, current->domain); -+ int rc = __do_update_va_mapping(va, val64, flags, current->domain); -+ -+ if ( rc == -ERESTART ) -+ rc = hypercall_create_continuation( -+ __HYPERVISOR_update_va_mapping, "lll", va, val64, flags); -+ -+ return rc; - } - - long do_update_va_mapping_otherdomain(unsigned long va, u64 val64, -@@ -4138,6 +4161,46 @@ long do_update_va_mapping_otherdomain(unsigned long va, u64 val64, - - put_pg_owner(pg_owner); - -+ if ( rc == -ERESTART ) -+ rc = hypercall_create_continuation( -+ __HYPERVISOR_update_va_mapping_otherdomain, -+ "llli", va, val64, flags, domid); -+ -+ return rc; -+} -+ -+int compat_update_va_mapping(unsigned int va, uint32_t lo, uint32_t hi, -+ unsigned int flags) -+{ -+ int rc = __do_update_va_mapping(va, ((uint64_t)hi << 32) | lo, -+ flags, current->domain); -+ -+ if ( rc == -ERESTART ) -+ rc = hypercall_create_continuation( -+ __HYPERVISOR_update_va_mapping, "iiii", va, lo, hi, flags); -+ -+ return rc; -+} -+ -+int compat_update_va_mapping_otherdomain(unsigned int va, -+ uint32_t lo, uint32_t hi, -+ unsigned int flags, domid_t domid) -+{ -+ struct domain *pg_owner; -+ int rc; -+ -+ if ( (pg_owner = get_pg_owner(domid)) == NULL ) -+ return -ESRCH; -+ -+ rc = __do_update_va_mapping(va, ((uint64_t)hi << 32) | lo, flags, pg_owner); -+ -+ put_pg_owner(pg_owner); -+ -+ if ( rc == -ERESTART ) -+ rc = hypercall_create_continuation( -+ __HYPERVISOR_update_va_mapping_otherdomain, -+ "iiiii", va, lo, hi, flags, domid); -+ - return rc; - } - -diff --git a/xen/arch/x86/mm/paging.c b/xen/arch/x86/mm/paging.c -index 2b0445ffe9..dcee496eb0 100644 ---- a/xen/arch/x86/mm/paging.c -+++ b/xen/arch/x86/mm/paging.c -@@ -873,6 +873,8 @@ void paging_dump_domain_info(struct domain *d) - printk(" paging assistance: "); - if ( paging_mode_shadow(d) ) - printk("shadow "); -+ if ( paging_mode_sh_forced(d) ) -+ printk("forced "); - if ( paging_mode_hap(d) ) - printk("hap "); - if ( paging_mode_refcounts(d) ) -diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c -index dd61b50eb7..fd42d734e7 100644 ---- a/xen/arch/x86/mm/shadow/common.c -+++ b/xen/arch/x86/mm/shadow/common.c -@@ -3177,6 +3177,15 @@ static void sh_new_mode(struct domain *d, u32 new_mode) - ASSERT(paging_locked_by_me(d)); - ASSERT(d != current->domain); - -+ /* -+ * If PG_SH_forced has previously been activated because of writing an -+ * L1TF-vulnerable PTE, it must remain active for the remaining lifetime -+ * of the domain, even if the logdirty mode needs to be controlled for -+ * migration purposes. -+ */ -+ if ( paging_mode_sh_forced(d) ) -+ new_mode |= PG_SH_forced | PG_SH_enable; -+ - d->arch.paging.mode = new_mode; - for_each_vcpu(d, v) - sh_update_paging_modes(v); -@@ -4057,6 +4066,33 @@ void shadow_audit_tables(struct vcpu *v) - - #endif /* Shadow audit */ - -+#ifdef CONFIG_PV -+ -+void pv_l1tf_tasklet(unsigned long data) -+{ -+ struct domain *d = (void *)data; -+ -+ domain_pause(d); -+ paging_lock(d); -+ -+ if ( !paging_mode_sh_forced(d) && !d->is_dying ) -+ { -+ int ret = shadow_one_bit_enable(d, PG_SH_forced); -+ -+ if ( ret ) -+ { -+ printk(XENLOG_G_ERR "d%d Failed to enable PG_SH_forced: %d\n", -+ d->domain_id, ret); -+ domain_crash(d); -+ } -+ } -+ -+ paging_unlock(d); -+ domain_unpause(d); -+} -+ -+#endif /* CONFIG_PV */ -+ - /* - * Local variables: - * mode: C -diff --git a/xen/arch/x86/mpparse.c b/xen/arch/x86/mpparse.c -index 49140e46f0..f3f6d48668 100644 ---- a/xen/arch/x86/mpparse.c -+++ b/xen/arch/x86/mpparse.c -@@ -68,19 +68,26 @@ physid_mask_t phys_cpu_present_map; - - void __init set_nr_cpu_ids(unsigned int max_cpus) - { -+ unsigned int tot_cpus = num_processors + disabled_cpus; -+ - if (!max_cpus) -- max_cpus = num_processors + disabled_cpus; -+ max_cpus = tot_cpus; - if (max_cpus > NR_CPUS) - max_cpus = NR_CPUS; - else if (!max_cpus) - max_cpus = 1; - printk(XENLOG_INFO "SMP: Allowing %u CPUs (%d hotplug CPUs)\n", - max_cpus, max_t(int, max_cpus - num_processors, 0)); -- nr_cpu_ids = max_cpus; -+ -+ if (!park_offline_cpus) -+ tot_cpus = max_cpus; -+ nr_cpu_ids = min(tot_cpus, NR_CPUS + 0u); -+ if (park_offline_cpus && nr_cpu_ids < num_processors) -+ printk(XENLOG_WARNING "SMP: Cannot bring up %u further CPUs\n", -+ num_processors - nr_cpu_ids); - - #ifndef nr_cpumask_bits -- nr_cpumask_bits = (max_cpus + (BITS_PER_LONG - 1)) & -- ~(BITS_PER_LONG - 1); -+ nr_cpumask_bits = ROUNDUP(nr_cpu_ids, BITS_PER_LONG); - printk(XENLOG_DEBUG "NR_CPUS:%u nr_cpumask_bits:%u\n", - NR_CPUS, nr_cpumask_bits); - #endif -diff --git a/xen/arch/x86/msr.c b/xen/arch/x86/msr.c -index 1e12ccb729..1a591dd2b5 100644 ---- a/xen/arch/x86/msr.c -+++ b/xen/arch/x86/msr.c -@@ -150,6 +150,7 @@ int guest_rdmsr(const struct vcpu *v, uint32_t msr, uint64_t *val) - case MSR_AMD_PATCHLOADER: - case MSR_IA32_UCODE_WRITE: - case MSR_PRED_CMD: -+ case MSR_FLUSH_CMD: - /* Write-only */ - goto gp_fault; - -@@ -254,6 +255,17 @@ int guest_wrmsr(struct vcpu *v, uint32_t msr, uint64_t val) - wrmsrl(MSR_PRED_CMD, val); - break; - -+ case MSR_FLUSH_CMD: -+ if ( !cp->feat.l1d_flush ) -+ goto gp_fault; /* MSR available? */ -+ -+ if ( val & ~FLUSH_CMD_L1D ) -+ goto gp_fault; /* Rsvd bit set? */ -+ -+ if ( v == curr ) -+ wrmsrl(MSR_FLUSH_CMD, val); -+ break; -+ - case MSR_INTEL_MISC_FEATURES_ENABLES: - { - bool old_cpuid_faulting = vp->misc_features_enables.cpuid_faulting; -diff --git a/xen/arch/x86/oprofile/nmi_int.c b/xen/arch/x86/oprofile/nmi_int.c -index d8f5230906..3dfb8fef93 100644 ---- a/xen/arch/x86/oprofile/nmi_int.c -+++ b/xen/arch/x86/oprofile/nmi_int.c -@@ -182,7 +182,7 @@ int nmi_reserve_counters(void) - if (!allocate_msrs()) - return -ENOMEM; - -- /* We walk a thin line between law and rape here. -+ /* - * We need to be careful to install our NMI handler - * without actually triggering any NMIs as this will - * break the core code horrifically. -diff --git a/xen/arch/x86/percpu.c b/xen/arch/x86/percpu.c -index c9997b7937..8be4ebddf4 100644 ---- a/xen/arch/x86/percpu.c -+++ b/xen/arch/x86/percpu.c -@@ -28,7 +28,7 @@ static int init_percpu_area(unsigned int cpu) - char *p; - - if ( __per_cpu_offset[cpu] != INVALID_PERCPU_AREA ) -- return -EBUSY; -+ return 0; - - if ( (p = alloc_xenheap_pages(PERCPU_ORDER, 0)) == NULL ) - return -ENOMEM; -@@ -76,9 +76,12 @@ static int cpu_percpu_callback( - break; - case CPU_UP_CANCELED: - case CPU_DEAD: -- free_percpu_area(cpu); -+ if ( !park_offline_cpus ) -+ free_percpu_area(cpu); - break; -- default: -+ case CPU_REMOVE: -+ if ( park_offline_cpus ) -+ free_percpu_area(cpu); - break; - } - -diff --git a/xen/arch/x86/pv/domain.c b/xen/arch/x86/pv/domain.c -index a4f0bd239d..3230ac6a22 100644 ---- a/xen/arch/x86/pv/domain.c -+++ b/xen/arch/x86/pv/domain.c -@@ -13,6 +13,7 @@ - #include <asm/invpcid.h> - #include <asm/spec_ctrl.h> - #include <asm/pv/domain.h> -+#include <asm/shadow.h> - - static __read_mostly enum { - PCID_OFF, -@@ -209,6 +210,8 @@ int pv_vcpu_initialise(struct vcpu *v) - - void pv_domain_destroy(struct domain *d) - { -+ pv_l1tf_domain_destroy(d); -+ - destroy_perdomain_mapping(d, GDT_LDT_VIRT_START, - GDT_LDT_MBYTES << (20 - PAGE_SHIFT)); - -@@ -229,6 +232,8 @@ int pv_domain_initialise(struct domain *d) - }; - int rc = -ENOMEM; - -+ pv_l1tf_domain_init(d); -+ - d->arch.pv_domain.gdt_ldt_l1tab = - alloc_xenheap_pages(0, MEMF_node(domain_to_node(d))); - if ( !d->arch.pv_domain.gdt_ldt_l1tab ) -diff --git a/xen/arch/x86/pv/ro-page-fault.c b/xen/arch/x86/pv/ro-page-fault.c -index aa8d5a7556..a3c0c2dd19 100644 ---- a/xen/arch/x86/pv/ro-page-fault.c -+++ b/xen/arch/x86/pv/ro-page-fault.c -@@ -29,6 +29,7 @@ - #include <asm/mm.h> - #include <asm/pci.h> - #include <asm/pv/mm.h> -+#include <asm/shadow.h> - - #include "emulate.h" - #include "mm.h" -@@ -129,6 +130,10 @@ static int ptwr_emulated_update(unsigned long addr, intpte_t *p_old, - - /* Check the new PTE. */ - nl1e = l1e_from_intpte(val); -+ -+ if ( !(l1e_get_flags(nl1e) & _PAGE_PRESENT) && pv_l1tf_check_l1e(d, nl1e) ) -+ return X86EMUL_RETRY; -+ - switch ( ret = get_page_from_l1e(nl1e, d, d) ) - { - default: -diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c -index a3172ca92c..3cd3e81b30 100644 ---- a/xen/arch/x86/setup.c -+++ b/xen/arch/x86/setup.c -@@ -62,6 +62,9 @@ boolean_param("nosmp", opt_nosmp); - static unsigned int __initdata max_cpus; - integer_param("maxcpus", max_cpus); - -+int8_t __read_mostly opt_smt = -1; -+boolean_param("smt", opt_smt); -+ - /* opt_invpcid: If false, don't use INVPCID instruction even if available. */ - static bool __initdata opt_invpcid = true; - boolean_param("invpcid", opt_invpcid); -@@ -665,7 +668,7 @@ void __init noreturn __start_xen(unsigned long mbi_p) - { - char *memmap_type = NULL; - char *cmdline, *kextra, *loader; -- unsigned int initrdidx; -+ unsigned int initrdidx, num_parked = 0; - multiboot_info_t *mbi; - module_t *mod; - unsigned long nr_pages, raw_max_page, modules_headroom, *module_map; -@@ -909,6 +912,18 @@ void __init noreturn __start_xen(unsigned long mbi_p) - /* Sanitise the raw E820 map to produce a final clean version. */ - max_page = raw_max_page = init_e820(memmap_type, &e820_raw); - -+ if ( !efi_enabled(EFI_BOOT) ) -+ { -+ /* -+ * Supplement the heuristics in l1tf_calculations() by assuming that -+ * anything referenced in the E820 may be cacheable. -+ */ -+ l1tf_safe_maddr = -+ max(l1tf_safe_maddr, -+ ROUNDUP(e820_raw.map[e820_raw.nr_map - 1].addr + -+ e820_raw.map[e820_raw.nr_map - 1].size, PAGE_SIZE)); -+ } -+ - /* Create a temporary copy of the E820 map. */ - memcpy(&boot_e820, &e820, sizeof(e820)); - -@@ -1494,7 +1509,8 @@ void __init noreturn __start_xen(unsigned long mbi_p) - else - { - set_nr_cpu_ids(max_cpus); -- max_cpus = nr_cpu_ids; -+ if ( !max_cpus ) -+ max_cpus = nr_cpu_ids; - } - - if ( xen_guest ) -@@ -1617,16 +1633,30 @@ void __init noreturn __start_xen(unsigned long mbi_p) - /* Set up node_to_cpumask based on cpu_to_node[]. */ - numa_add_cpu(i); - -- if ( (num_online_cpus() < max_cpus) && !cpu_online(i) ) -+ if ( (park_offline_cpus || num_online_cpus() < max_cpus) && -+ !cpu_online(i) ) - { - int ret = cpu_up(i); - if ( ret != 0 ) - printk("Failed to bring up CPU %u (error %d)\n", i, ret); -+ else if ( num_online_cpus() > max_cpus || -+ (!opt_smt && -+ cpu_data[i].compute_unit_id == INVALID_CUID && -+ cpumask_weight(per_cpu(cpu_sibling_mask, i)) > 1) ) -+ { -+ ret = cpu_down(i); -+ if ( !ret ) -+ ++num_parked; -+ else -+ printk("Could not re-offline CPU%u (%d)\n", i, ret); -+ } - } - } - } - - printk("Brought up %ld CPUs\n", (long)num_online_cpus()); -+ if ( num_parked ) -+ printk(XENLOG_INFO "Parked %u CPUs\n", num_parked); - smp_cpus_done(); - - do_initcalls(); -diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c -index d4478e6132..7e76cc3d68 100644 ---- a/xen/arch/x86/smpboot.c -+++ b/xen/arch/x86/smpboot.c -@@ -63,6 +63,8 @@ static cpumask_t scratch_cpu0mask; - cpumask_t cpu_online_map __read_mostly; - EXPORT_SYMBOL(cpu_online_map); - -+bool __read_mostly park_offline_cpus; -+ - unsigned int __read_mostly nr_sockets; - cpumask_t **__read_mostly socket_cpumask; - static cpumask_t *secondary_socket_cpumask; -@@ -234,33 +236,41 @@ static void link_thread_siblings(int cpu1, int cpu2) - cpumask_set_cpu(cpu2, per_cpu(cpu_core_mask, cpu1)); - } - --static void set_cpu_sibling_map(int cpu) -+static void set_cpu_sibling_map(unsigned int cpu) - { -- int i; -+ unsigned int i; - struct cpuinfo_x86 *c = cpu_data; - - cpumask_set_cpu(cpu, &cpu_sibling_setup_map); - - cpumask_set_cpu(cpu, socket_cpumask[cpu_to_socket(cpu)]); -+ cpumask_set_cpu(cpu, per_cpu(cpu_core_mask, cpu)); -+ cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu)); - - if ( c[cpu].x86_num_siblings > 1 ) - { - for_each_cpu ( i, &cpu_sibling_setup_map ) - { -- if ( cpu_has(c, X86_FEATURE_TOPOEXT) ) { -- if ( (c[cpu].phys_proc_id == c[i].phys_proc_id) && -- (c[cpu].compute_unit_id == c[i].compute_unit_id) ) -+ if ( cpu == i || c[cpu].phys_proc_id != c[i].phys_proc_id ) -+ continue; -+ if ( c[cpu].compute_unit_id != INVALID_CUID && -+ c[i].compute_unit_id != INVALID_CUID ) -+ { -+ if ( c[cpu].compute_unit_id == c[i].compute_unit_id ) - link_thread_siblings(cpu, i); -- } else if ( (c[cpu].phys_proc_id == c[i].phys_proc_id) && -- (c[cpu].cpu_core_id == c[i].cpu_core_id) ) { -- link_thread_siblings(cpu, i); - } -+ else if ( c[cpu].cpu_core_id != XEN_INVALID_CORE_ID && -+ c[i].cpu_core_id != XEN_INVALID_CORE_ID ) -+ { -+ if ( c[cpu].cpu_core_id == c[i].cpu_core_id ) -+ link_thread_siblings(cpu, i); -+ } -+ else -+ printk(XENLOG_WARNING -+ "CPU%u: unclear relationship with CPU%u\n", -+ cpu, i); - } - } -- else -- { -- cpumask_set_cpu(cpu, per_cpu(cpu_sibling_mask, cpu)); -- } - - if ( c[cpu].x86_max_cores == 1 ) - { -@@ -887,7 +897,14 @@ static void cleanup_cpu_root_pgt(unsigned int cpu) - } - } - --static void cpu_smpboot_free(unsigned int cpu) -+/* -+ * The 'remove' boolean controls whether a CPU is just getting offlined (and -+ * parked), or outright removed / offlined without parking. Parked CPUs need -+ * things like their stack, GDT, IDT, TSS, and per-CPU data still available. -+ * A few other items, in particular CPU masks, are also retained, as it's -+ * difficult to prove that they're entirely unreferenced from parked CPUs. -+ */ -+static void cpu_smpboot_free(unsigned int cpu, bool remove) - { - unsigned int order, socket = cpu_to_socket(cpu); - struct cpuinfo_x86 *c = cpu_data; -@@ -898,15 +915,19 @@ static void cpu_smpboot_free(unsigned int cpu) - socket_cpumask[socket] = NULL; - } - -- c[cpu].phys_proc_id = XEN_INVALID_SOCKET_ID; -- c[cpu].cpu_core_id = XEN_INVALID_CORE_ID; -- c[cpu].compute_unit_id = INVALID_CUID; - cpumask_clear_cpu(cpu, &cpu_sibling_setup_map); - -- free_cpumask_var(per_cpu(cpu_sibling_mask, cpu)); -- free_cpumask_var(per_cpu(cpu_core_mask, cpu)); -- if ( per_cpu(scratch_cpumask, cpu) != &scratch_cpu0mask ) -- free_cpumask_var(per_cpu(scratch_cpumask, cpu)); -+ if ( remove ) -+ { -+ c[cpu].phys_proc_id = XEN_INVALID_SOCKET_ID; -+ c[cpu].cpu_core_id = XEN_INVALID_CORE_ID; -+ c[cpu].compute_unit_id = INVALID_CUID; -+ -+ FREE_CPUMASK_VAR(per_cpu(cpu_sibling_mask, cpu)); -+ FREE_CPUMASK_VAR(per_cpu(cpu_core_mask, cpu)); -+ if ( per_cpu(scratch_cpumask, cpu) != &scratch_cpu0mask ) -+ FREE_CPUMASK_VAR(per_cpu(scratch_cpumask, cpu)); -+ } - - cleanup_cpu_root_pgt(cpu); - -@@ -928,19 +949,21 @@ static void cpu_smpboot_free(unsigned int cpu) - } - - order = get_order_from_pages(NR_RESERVED_GDT_PAGES); -- free_xenheap_pages(per_cpu(gdt_table, cpu), order); -+ if ( remove ) -+ FREE_XENHEAP_PAGES(per_cpu(gdt_table, cpu), order); - - free_xenheap_pages(per_cpu(compat_gdt_table, cpu), order); - -- order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t)); -- free_xenheap_pages(idt_tables[cpu], order); -- idt_tables[cpu] = NULL; -- -- if ( stack_base[cpu] != NULL ) -+ if ( remove ) - { -- memguard_unguard_stack(stack_base[cpu]); -- free_xenheap_pages(stack_base[cpu], STACK_ORDER); -- stack_base[cpu] = NULL; -+ order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t)); -+ FREE_XENHEAP_PAGES(idt_tables[cpu], order); -+ -+ if ( stack_base[cpu] ) -+ { -+ memguard_unguard_stack(stack_base[cpu]); -+ FREE_XENHEAP_PAGES(stack_base[cpu], STACK_ORDER); -+ } - } - } - -@@ -955,15 +978,17 @@ static int cpu_smpboot_alloc(unsigned int cpu) - if ( node != NUMA_NO_NODE ) - memflags = MEMF_node(node); - -- stack_base[cpu] = alloc_xenheap_pages(STACK_ORDER, memflags); -+ if ( stack_base[cpu] == NULL ) -+ stack_base[cpu] = alloc_xenheap_pages(STACK_ORDER, memflags); - if ( stack_base[cpu] == NULL ) - goto out; - memguard_guard_stack(stack_base[cpu]); - - order = get_order_from_pages(NR_RESERVED_GDT_PAGES); -- per_cpu(gdt_table, cpu) = gdt = alloc_xenheap_pages(order, memflags); -+ gdt = per_cpu(gdt_table, cpu) ?: alloc_xenheap_pages(order, memflags); - if ( gdt == NULL ) - goto out; -+ per_cpu(gdt_table, cpu) = gdt; - memcpy(gdt, boot_cpu_gdt_table, NR_RESERVED_GDT_PAGES * PAGE_SIZE); - BUILD_BUG_ON(NR_CPUS > 0x10000); - gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu; -@@ -975,7 +1000,8 @@ static int cpu_smpboot_alloc(unsigned int cpu) - gdt[PER_CPU_GDT_ENTRY - FIRST_RESERVED_GDT_ENTRY].a = cpu; - - order = get_order_from_bytes(IDT_ENTRIES * sizeof(idt_entry_t)); -- idt_tables[cpu] = alloc_xenheap_pages(order, memflags); -+ if ( idt_tables[cpu] == NULL ) -+ idt_tables[cpu] = alloc_xenheap_pages(order, memflags); - if ( idt_tables[cpu] == NULL ) - goto out; - memcpy(idt_tables[cpu], idt_table, IDT_ENTRIES * sizeof(idt_entry_t)); -@@ -1003,16 +1029,16 @@ static int cpu_smpboot_alloc(unsigned int cpu) - (secondary_socket_cpumask = xzalloc(cpumask_t)) == NULL ) - goto out; - -- if ( !(zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) && -- zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) && -- alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu))) ) -+ if ( !(cond_zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, cpu)) && -+ cond_zalloc_cpumask_var(&per_cpu(cpu_core_mask, cpu)) && -+ cond_alloc_cpumask_var(&per_cpu(scratch_cpumask, cpu))) ) - goto out; - - rc = 0; - - out: - if ( rc ) -- cpu_smpboot_free(cpu); -+ cpu_smpboot_free(cpu, true); - - return rc; - } -@@ -1030,9 +1056,10 @@ static int cpu_smpboot_callback( - break; - case CPU_UP_CANCELED: - case CPU_DEAD: -- cpu_smpboot_free(cpu); -+ cpu_smpboot_free(cpu, !park_offline_cpus); - break; -- default: -+ case CPU_REMOVE: -+ cpu_smpboot_free(cpu, true); - break; - } - -diff --git a/xen/arch/x86/spec_ctrl.c b/xen/arch/x86/spec_ctrl.c -index 08e6784c4c..f0c50d6703 100644 ---- a/xen/arch/x86/spec_ctrl.c -+++ b/xen/arch/x86/spec_ctrl.c -@@ -19,10 +19,13 @@ - #include <xen/errno.h> - #include <xen/init.h> - #include <xen/lib.h> -+#include <xen/warning.h> - - #include <asm/microcode.h> - #include <asm/msr.h> - #include <asm/processor.h> -+#include <asm/pv/shim.h> -+#include <asm/setup.h> - #include <asm/spec_ctrl.h> - #include <asm/spec_ctrl_asm.h> - -@@ -45,11 +48,16 @@ static int8_t __initdata opt_ibrs = -1; - bool __read_mostly opt_ibpb = true; - bool __read_mostly opt_ssbd = false; - int8_t __read_mostly opt_eager_fpu = -1; -+int8_t __read_mostly opt_l1d_flush = -1; - - bool __initdata bsp_delay_spec_ctrl; - uint8_t __read_mostly default_xen_spec_ctrl; - uint8_t __read_mostly default_spec_ctrl_flags; - -+paddr_t __read_mostly l1tf_addr_mask, __read_mostly l1tf_safe_maddr; -+static bool __initdata cpu_has_bug_l1tf; -+static unsigned int __initdata l1d_maxphysaddr; -+ - static int __init parse_bti(const char *s) - { - const char *ss; -@@ -124,6 +132,17 @@ static int __init parse_spec_ctrl(const char *s) - opt_msr_sc_pv = false; - opt_msr_sc_hvm = false; - -+ opt_eager_fpu = 0; -+ -+ if ( opt_xpti < 0 ) -+ opt_xpti = 0; -+ -+ if ( opt_smt < 0 ) -+ opt_smt = 1; -+ -+ if ( opt_pv_l1tf < 0 ) -+ opt_pv_l1tf = 0; -+ - disable_common: - opt_rsb_pv = false; - opt_rsb_hvm = false; -@@ -131,7 +150,8 @@ static int __init parse_spec_ctrl(const char *s) - opt_thunk = THUNK_JMP; - opt_ibrs = 0; - opt_ibpb = false; -- opt_eager_fpu = 0; -+ opt_ssbd = false; -+ opt_l1d_flush = 0; - } - else if ( val > 0 ) - rc = -EINVAL; -@@ -187,6 +207,8 @@ static int __init parse_spec_ctrl(const char *s) - opt_ssbd = val; - else if ( (val = parse_boolean("eager-fpu", s, ss)) >= 0 ) - opt_eager_fpu = val; -+ else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 ) -+ opt_l1d_flush = val; - else - rc = -EINVAL; - -@@ -197,6 +219,55 @@ static int __init parse_spec_ctrl(const char *s) - } - custom_param("spec-ctrl", parse_spec_ctrl); - -+int8_t __read_mostly opt_pv_l1tf = -1; -+ -+static __init int parse_pv_l1tf(const char *s) -+{ -+ const char *ss; -+ int val, rc = 0; -+ -+ /* Inhibit the defaults as an explicit choice has been given. */ -+ if ( opt_pv_l1tf == -1 ) -+ opt_pv_l1tf = 0; -+ -+ /* Interpret 'pv-l1tf' alone in its positive boolean form. */ -+ if ( *s == '\0' ) -+ opt_xpti = OPT_PV_L1TF_DOM0 | OPT_PV_L1TF_DOMU; -+ -+ do { -+ ss = strchr(s, ','); -+ if ( !ss ) -+ ss = strchr(s, '\0'); -+ -+ switch ( parse_bool(s, ss) ) -+ { -+ case 0: -+ opt_pv_l1tf = 0; -+ break; -+ -+ case 1: -+ opt_pv_l1tf = OPT_PV_L1TF_DOM0 | OPT_PV_L1TF_DOMU; -+ break; -+ -+ default: -+ if ( (val = parse_boolean("dom0", s, ss)) >= 0 ) -+ opt_pv_l1tf = ((opt_pv_l1tf & ~OPT_PV_L1TF_DOM0) | -+ (val ? OPT_PV_L1TF_DOM0 : 0)); -+ else if ( (val = parse_boolean("domu", s, ss)) >= 0 ) -+ opt_pv_l1tf = ((opt_pv_l1tf & ~OPT_PV_L1TF_DOMU) | -+ (val ? OPT_PV_L1TF_DOMU : 0)); -+ else -+ rc = -EINVAL; -+ break; -+ } -+ -+ s = ss + 1; -+ } while ( *ss ); -+ -+ return rc; -+} -+custom_param("pv-l1tf", parse_pv_l1tf); -+ - static void __init print_details(enum ind_thunk thunk, uint64_t caps) - { - unsigned int _7d0 = 0, e8b = 0, tmp; -@@ -210,22 +281,31 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps) - printk("Speculative mitigation facilities:\n"); - - /* Hardware features which pertain to speculative mitigations. */ -- printk(" Hardware features:%s%s%s%s%s%s%s%s\n", -+ printk(" Hardware features:%s%s%s%s%s%s%s%s%s%s\n", - (_7d0 & cpufeat_mask(X86_FEATURE_IBRSB)) ? " IBRS/IBPB" : "", - (_7d0 & cpufeat_mask(X86_FEATURE_STIBP)) ? " STIBP" : "", -+ (_7d0 & cpufeat_mask(X86_FEATURE_L1D_FLUSH)) ? " L1D_FLUSH" : "", - (_7d0 & cpufeat_mask(X86_FEATURE_SSBD)) ? " SSBD" : "", - (e8b & cpufeat_mask(X86_FEATURE_IBPB)) ? " IBPB" : "", - (caps & ARCH_CAPABILITIES_IBRS_ALL) ? " IBRS_ALL" : "", - (caps & ARCH_CAPABILITIES_RDCL_NO) ? " RDCL_NO" : "", - (caps & ARCH_CAPS_RSBA) ? " RSBA" : "", -+ (caps & ARCH_CAPS_SKIP_L1DFL) ? " SKIP_L1DFL": "", - (caps & ARCH_CAPS_SSB_NO) ? " SSB_NO" : ""); - -- /* Compiled-in support which pertains to BTI mitigations. */ -- if ( IS_ENABLED(CONFIG_INDIRECT_THUNK) ) -- printk(" Compiled-in support: INDIRECT_THUNK\n"); -+ /* Compiled-in support which pertains to mitigations. */ -+ if ( IS_ENABLED(CONFIG_INDIRECT_THUNK) || IS_ENABLED(CONFIG_SHADOW_PAGING) ) -+ printk(" Compiled-in support:" -+#ifdef CONFIG_INDIRECT_THUNK -+ " INDIRECT_THUNK" -+#endif -+#ifdef CONFIG_SHADOW_PAGING -+ " SHADOW_PAGING" -+#endif -+ "\n"); - - /* Settings for Xen's protection, irrespective of guests. */ -- printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s\n", -+ printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s%s\n", - thunk == THUNK_NONE ? "N/A" : - thunk == THUNK_RETPOLINE ? "RETPOLINE" : - thunk == THUNK_LFENCE ? "LFENCE" : -@@ -234,7 +314,15 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps) - (default_xen_spec_ctrl & SPEC_CTRL_IBRS) ? "IBRS+" : "IBRS-", - !boot_cpu_has(X86_FEATURE_SSBD) ? "" : - (default_xen_spec_ctrl & SPEC_CTRL_SSBD) ? " SSBD+" : " SSBD-", -- opt_ibpb ? " IBPB" : ""); -+ opt_ibpb ? " IBPB" : "", -+ opt_l1d_flush ? " L1D_FLUSH" : ""); -+ -+ /* L1TF diagnostics, printed if vulnerable or PV shadowing is in use. */ -+ if ( cpu_has_bug_l1tf || opt_pv_l1tf ) -+ printk(" L1TF: believed%s vulnerable, maxphysaddr L1D %u, CPUID %u" -+ ", Safe address %"PRIx64"\n", -+ cpu_has_bug_l1tf ? "" : " not", -+ l1d_maxphysaddr, paddr_bits, l1tf_safe_maddr); - - /* - * Alternatives blocks for protecting against and/or virtualising -@@ -257,6 +345,10 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps) - printk(" XPTI (64-bit PV only): Dom0 %s, DomU %s\n", - opt_xpti & OPT_XPTI_DOM0 ? "enabled" : "disabled", - opt_xpti & OPT_XPTI_DOMU ? "enabled" : "disabled"); -+ -+ printk(" PV L1TF shadowing: Dom0 %s, DomU %s\n", -+ opt_pv_l1tf & OPT_PV_L1TF_DOM0 ? "enabled" : "disabled", -+ opt_pv_l1tf & OPT_PV_L1TF_DOMU ? "enabled" : "disabled"); - } - - /* Calculate whether Retpoline is known-safe on this CPU. */ -@@ -418,20 +510,159 @@ static bool __init should_use_eager_fpu(void) - } - } - --#define OPT_XPTI_DEFAULT 0xff --uint8_t __read_mostly opt_xpti = OPT_XPTI_DEFAULT; -- --static __init void xpti_init_default(bool force) -+/* Calculate whether this CPU is vulnerable to L1TF. */ -+static __init void l1tf_calculations(uint64_t caps) - { -- uint64_t caps = 0; -+ bool hit_default = false; -+ -+ l1d_maxphysaddr = paddr_bits; -+ -+ /* L1TF is only known to affect Intel Family 6 processors at this time. */ -+ if ( boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && -+ boot_cpu_data.x86 == 6 ) -+ { -+ switch ( boot_cpu_data.x86_model ) -+ { -+ /* -+ * Core processors since at least Penryn are vulnerable. -+ */ -+ case 0x17: /* Penryn */ -+ case 0x1d: /* Dunnington */ -+ cpu_has_bug_l1tf = true; -+ break; -+ -+ case 0x1f: /* Auburndale / Havendale */ -+ case 0x1e: /* Nehalem */ -+ case 0x1a: /* Nehalem EP */ -+ case 0x2e: /* Nehalem EX */ -+ case 0x25: /* Westmere */ -+ case 0x2c: /* Westmere EP */ -+ case 0x2f: /* Westmere EX */ -+ cpu_has_bug_l1tf = true; -+ l1d_maxphysaddr = 44; -+ break; -+ -+ case 0x2a: /* SandyBridge */ -+ case 0x2d: /* SandyBridge EP/EX */ -+ case 0x3a: /* IvyBridge */ -+ case 0x3e: /* IvyBridge EP/EX */ -+ case 0x3c: /* Haswell */ -+ case 0x3f: /* Haswell EX/EP */ -+ case 0x45: /* Haswell D */ -+ case 0x46: /* Haswell H */ -+ case 0x3d: /* Broadwell */ -+ case 0x47: /* Broadwell H */ -+ case 0x4f: /* Broadwell EP/EX */ -+ case 0x56: /* Broadwell D */ -+ case 0x4e: /* Skylake M */ -+ case 0x55: /* Skylake X */ -+ case 0x5e: /* Skylake D */ -+ case 0x66: /* Cannonlake */ -+ case 0x67: /* Cannonlake? */ -+ case 0x8e: /* Kabylake M */ -+ case 0x9e: /* Kabylake D */ -+ cpu_has_bug_l1tf = true; -+ l1d_maxphysaddr = 46; -+ break; -+ -+ /* -+ * Atom processors are not vulnerable. -+ */ -+ case 0x1c: /* Pineview */ -+ case 0x26: /* Lincroft */ -+ case 0x27: /* Penwell */ -+ case 0x35: /* Cloverview */ -+ case 0x36: /* Cedarview */ -+ case 0x37: /* Baytrail / Valleyview (Silvermont) */ -+ case 0x4d: /* Avaton / Rangely (Silvermont) */ -+ case 0x4c: /* Cherrytrail / Brasswell */ -+ case 0x4a: /* Merrifield */ -+ case 0x5a: /* Moorefield */ -+ case 0x5c: /* Goldmont */ -+ case 0x5f: /* Denverton */ -+ case 0x7a: /* Gemini Lake */ -+ break; -+ -+ /* -+ * Knights processors are not vulnerable. -+ */ -+ case 0x57: /* Knights Landing */ -+ case 0x85: /* Knights Mill */ -+ break; -+ -+ default: -+ /* Defer printk() until we've accounted for RDCL_NO. */ -+ hit_default = true; -+ cpu_has_bug_l1tf = true; -+ break; -+ } -+ } -+ -+ /* Any processor advertising RDCL_NO should be not vulnerable to L1TF. */ -+ if ( caps & ARCH_CAPABILITIES_RDCL_NO ) -+ cpu_has_bug_l1tf = false; -+ -+ if ( cpu_has_bug_l1tf && hit_default ) -+ printk("Unrecognised CPU model %#x - assuming vulnerable to L1TF\n", -+ boot_cpu_data.x86_model); -+ -+ /* -+ * L1TF safe address heuristics. These apply to the real hardware we are -+ * running on, and are best-effort-only if Xen is virtualised. -+ * -+ * The address mask which the L1D cache uses, which might be wider than -+ * the CPUID-reported maxphysaddr. -+ */ -+ l1tf_addr_mask = ((1ul << l1d_maxphysaddr) - 1) & PAGE_MASK; -+ -+ /* -+ * To be safe, l1tf_safe_maddr must be above the highest cacheable entity -+ * in system physical address space. However, to preserve space for -+ * paged-out metadata, it should be as low as possible above the highest -+ * cacheable address, so as to require fewer high-order bits being set. -+ * -+ * These heuristics are based on some guesswork to improve the likelihood -+ * of safety in the common case, including Linux's L1TF mitigation of -+ * inverting all address bits in a non-present PTE. -+ * -+ * - If L1D is wider than CPUID (Nehalem and later mobile/desktop/low end -+ * server), setting any address bit beyond CPUID maxphysaddr guarantees -+ * to make the PTE safe. This case doesn't require all the high-order -+ * bits being set, and doesn't require any other source of information -+ * for safety. -+ * -+ * - If L1D is the same as CPUID (Pre-Nehalem, or high end server), we -+ * must sacrifice high order bits from the real address space for -+ * safety. Therefore, make a blind guess that there is nothing -+ * cacheable in the top quarter of physical address space. -+ * -+ * It is exceedingly unlikely for machines to be populated with this -+ * much RAM (likely 512G on pre-Nehalem, 16T on Nehalem/Westmere, 64T on -+ * Sandybridge and later) due to the sheer volume of DIMMs this would -+ * actually take. -+ * -+ * However, it is possible to find machines this large, so the "top -+ * quarter" guess is supplemented to push the limit higher if references -+ * to cacheable mappings (E820/SRAT/EFI/etc) are found above the top -+ * quarter boundary. -+ * -+ * Finally, this top quarter guess gives us a good chance of being safe -+ * when running virtualised (and the CPUID maxphysaddr hasn't been -+ * levelled for heterogeneous migration safety), where the safety -+ * consideration is still in terms of host details, but all E820/etc -+ * information is in terms of guest physical layout. -+ */ -+ l1tf_safe_maddr = max(l1tf_safe_maddr, ((l1d_maxphysaddr > paddr_bits) -+ ? (1ul << paddr_bits) -+ : (3ul << (paddr_bits - 2)))); -+} - -- if ( !force && (opt_xpti != OPT_XPTI_DEFAULT) ) -- return; -+int8_t __read_mostly opt_xpti = -1; - -+static __init void xpti_init_default(uint64_t caps) -+{ - if ( boot_cpu_data.x86_vendor == X86_VENDOR_AMD ) - caps = ARCH_CAPABILITIES_RDCL_NO; -- else if ( boot_cpu_has(X86_FEATURE_ARCH_CAPS) ) -- rdmsrl(MSR_ARCH_CAPABILITIES, caps); - - if ( caps & ARCH_CAPABILITIES_RDCL_NO ) - opt_xpti = 0; -@@ -444,7 +675,13 @@ static __init int parse_xpti(const char *s) - const char *ss; - int val, rc = 0; - -- xpti_init_default(false); -+ /* Inhibit the defaults as an explicit choice has been given. */ -+ if ( opt_xpti == -1 ) -+ opt_xpti = 0; -+ -+ /* Interpret 'xpti' alone in its positive boolean form. */ -+ if ( *s == '\0' ) -+ opt_xpti = OPT_XPTI_DOM0 | OPT_XPTI_DOMU; - - do { - ss = strchr(s, ','); -@@ -463,7 +700,7 @@ static __init int parse_xpti(const char *s) - - default: - if ( !strcmp(s, "default") ) -- xpti_init_default(true); -+ opt_xpti = -1; - else if ( (val = parse_boolean("dom0", s, ss)) >= 0 ) - opt_xpti = (opt_xpti & ~OPT_XPTI_DOM0) | - (val ? OPT_XPTI_DOM0 : 0); -@@ -625,12 +862,58 @@ void __init init_speculation_mitigations(void) - if ( default_xen_spec_ctrl ) - setup_force_cpu_cap(X86_FEATURE_SC_MSR_IDLE); - -- xpti_init_default(false); -+ if ( opt_xpti == -1 ) -+ xpti_init_default(caps); -+ - if ( opt_xpti == 0 ) - setup_force_cpu_cap(X86_FEATURE_NO_XPTI); - else - setup_clear_cpu_cap(X86_FEATURE_NO_XPTI); - -+ l1tf_calculations(caps); -+ -+ /* -+ * By default, enable PV domU L1TF mitigations on all L1TF-vulnerable -+ * hardware, except when running in shim mode. -+ * -+ * In shim mode, SHADOW is expected to be compiled out, and a malicious -+ * guest kernel can only attack the shim Xen, not the host Xen. -+ */ -+ if ( opt_pv_l1tf == -1 ) -+ { -+ if ( pv_shim || !cpu_has_bug_l1tf ) -+ opt_pv_l1tf = 0; -+ else -+ opt_pv_l1tf = OPT_PV_L1TF_DOMU; -+ } -+ -+ /* -+ * By default, enable L1D_FLUSH on L1TF-vulnerable hardware, unless -+ * instructed to skip the flush on vmentry by our outer hypervisor. -+ */ -+ if ( !boot_cpu_has(X86_FEATURE_L1D_FLUSH) ) -+ opt_l1d_flush = 0; -+ else if ( opt_l1d_flush == -1 ) -+ opt_l1d_flush = cpu_has_bug_l1tf && !(caps & ARCH_CAPS_SKIP_L1DFL); -+ -+ /* -+ * We do not disable HT by default on affected hardware. -+ * -+ * Firstly, if the user intends to use exclusively PV, or HVM shadow -+ * guests, HT isn't a concern and should remain fully enabled. Secondly, -+ * safety for HVM HAP guests can be arranged by the toolstack with core -+ * parking, pinning or cpupool configurations, including mixed setups. -+ * -+ * However, if we are on affected hardware, with HT enabled, and the user -+ * hasn't explicitly chosen whether to use HT or not, nag them to do so. -+ */ -+ if ( opt_smt == -1 && cpu_has_bug_l1tf && !pv_shim && -+ boot_cpu_data.x86_num_siblings > 1 ) -+ warning_add( -+ "Booted on L1TF-vulnerable hardware with SMT/Hyperthreading\n" -+ "enabled. Please assess your configuration and choose an\n" -+ "explicit 'smt=<bool>' setting. See XSA-273.\n"); -+ - print_details(thunk, caps); - - /* -diff --git a/xen/arch/x86/srat.c b/xen/arch/x86/srat.c -index 166eb44fe2..2d70b45909 100644 ---- a/xen/arch/x86/srat.c -+++ b/xen/arch/x86/srat.c -@@ -20,6 +20,7 @@ - #include <xen/pfn.h> - #include <asm/e820.h> - #include <asm/page.h> -+#include <asm/spec_ctrl.h> - - static struct acpi_table_slit *__read_mostly acpi_slit; - -@@ -284,6 +285,11 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma) - if (!(ma->flags & ACPI_SRAT_MEM_ENABLED)) - return; - -+ start = ma->base_address; -+ end = start + ma->length; -+ /* Supplement the heuristics in l1tf_calculations(). */ -+ l1tf_safe_maddr = max(l1tf_safe_maddr, ROUNDUP(end, PAGE_SIZE)); -+ - if (num_node_memblks >= NR_NODE_MEMBLKS) - { - dprintk(XENLOG_WARNING, -@@ -292,8 +298,6 @@ acpi_numa_memory_affinity_init(const struct acpi_srat_mem_affinity *ma) - return; - } - -- start = ma->base_address; -- end = start + ma->length; - pxm = ma->proximity_domain; - if (srat_rev < 2) - pxm &= 0xff; -diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c -index 4d372db12b..e704ed7f1c 100644 ---- a/xen/arch/x86/sysctl.c -+++ b/xen/arch/x86/sysctl.c -@@ -23,6 +23,7 @@ - #include <asm/hvm/hvm.h> - #include <asm/hvm/support.h> - #include <asm/processor.h> -+#include <asm/setup.h> - #include <asm/smp.h> - #include <asm/numa.h> - #include <xen/nodemask.h> -@@ -48,14 +49,27 @@ static void l3_cache_get(void *arg) - - long cpu_up_helper(void *data) - { -- int cpu = (unsigned long)data; -+ unsigned int cpu = (unsigned long)data; - int ret = cpu_up(cpu); -+ - if ( ret == -EBUSY ) - { - /* On EBUSY, flush RCU work and have one more go. */ - rcu_barrier(); - ret = cpu_up(cpu); - } -+ -+ if ( !ret && !opt_smt && -+ cpu_data[cpu].compute_unit_id == INVALID_CUID && -+ cpumask_weight(per_cpu(cpu_sibling_mask, cpu)) > 1 ) -+ { -+ ret = cpu_down_helper(data); -+ if ( ret ) -+ printk("Could not re-offline CPU%u (%d)\n", cpu, ret); -+ else -+ ret = -EPERM; -+ } -+ - return ret; - } - -diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c -index 9f045a2045..789d7ff8cd 100644 ---- a/xen/arch/x86/traps.c -+++ b/xen/arch/x86/traps.c -@@ -96,8 +96,6 @@ string_param("nmi", opt_nmi); - DEFINE_PER_CPU(uint64_t, efer); - static DEFINE_PER_CPU(unsigned long, last_extable_addr); - --DEFINE_PER_CPU_READ_MOSTLY(u32, ler_msr); -- - DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, gdt_table); - DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, compat_gdt_table); - -@@ -117,6 +115,9 @@ integer_param("debug_stack_lines", debug_stack_lines); - static bool opt_ler; - boolean_param("ler", opt_ler); - -+/* LastExceptionFromIP on this hardware. Zero if LER is not in use. */ -+unsigned int __read_mostly ler_msr; -+ - #define stack_words_per_line 4 - #define ESP_BEFORE_EXCEPTION(regs) ((unsigned long *)regs->rsp) - -@@ -1778,17 +1779,6 @@ void do_device_not_available(struct cpu_user_regs *regs) - return; - } - --static void ler_enable(void) --{ -- u64 debugctl; -- -- if ( !this_cpu(ler_msr) ) -- return; -- -- rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); -- wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl | IA32_DEBUGCTLMSR_LBR); --} -- - void do_debug(struct cpu_user_regs *regs) - { - unsigned long dr6; -@@ -1821,6 +1811,10 @@ void do_debug(struct cpu_user_regs *regs) - */ - write_debugreg(6, X86_DR6_DEFAULT); - -+ /* #DB automatically disabled LBR. Reinstate it if debugging Xen. */ -+ if ( cpu_has_xen_lbr ) -+ wrmsrl(MSR_IA32_DEBUGCTLMSR, IA32_DEBUGCTLMSR_LBR); -+ - if ( !guest_mode(regs) ) - { - /* -@@ -1838,7 +1832,7 @@ void do_debug(struct cpu_user_regs *regs) - { - if ( regs->rip == (unsigned long)sysenter_eflags_saved ) - regs->eflags &= ~X86_EFLAGS_TF; -- goto out; -+ return; - } - if ( !debugger_trap_fatal(TRAP_debug, regs) ) - { -@@ -1895,20 +1889,14 @@ void do_debug(struct cpu_user_regs *regs) - regs->cs, _p(regs->rip), _p(regs->rip), - regs->ss, _p(regs->rsp), dr6); - -- goto out; -+ return; - } - - /* Save debug status register where guest OS can peek at it */ - v->arch.debugreg[6] |= (dr6 & ~X86_DR6_DEFAULT); - v->arch.debugreg[6] &= (dr6 | ~X86_DR6_DEFAULT); - -- ler_enable(); - pv_inject_hw_exception(TRAP_debug, X86_EVENT_NO_EC); -- return; -- -- out: -- ler_enable(); -- return; - } - - static void __init noinline __set_intr_gate(unsigned int n, -@@ -1952,38 +1940,46 @@ void load_TR(void) - : "=m" (old_gdt) : "rm" (TSS_ENTRY << 3), "m" (tss_gdt) : "memory" ); - } - --void percpu_traps_init(void) -+static unsigned int calc_ler_msr(void) - { -- subarch_percpu_traps_init(); -- -- if ( !opt_ler ) -- return; -- - switch ( boot_cpu_data.x86_vendor ) - { - case X86_VENDOR_INTEL: - switch ( boot_cpu_data.x86 ) - { - case 6: -- this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP; -- break; -+ return MSR_IA32_LASTINTFROMIP; -+ - case 15: -- this_cpu(ler_msr) = MSR_P4_LER_FROM_LIP; -- break; -+ return MSR_P4_LER_FROM_LIP; - } - break; -+ - case X86_VENDOR_AMD: - switch ( boot_cpu_data.x86 ) - { - case 6: - case 0xf ... 0x17: -- this_cpu(ler_msr) = MSR_IA32_LASTINTFROMIP; -- break; -+ return MSR_IA32_LASTINTFROMIP; - } - break; - } - -- ler_enable(); -+ return 0; -+} -+ -+void percpu_traps_init(void) -+{ -+ subarch_percpu_traps_init(); -+ -+ if ( !opt_ler ) -+ return; -+ -+ if ( !ler_msr && (ler_msr = calc_ler_msr()) ) -+ setup_force_cpu_cap(X86_FEATURE_XEN_LBR); -+ -+ if ( cpu_has_xen_lbr ) -+ wrmsrl(MSR_IA32_DEBUGCTLMSR, IA32_DEBUGCTLMSR_LBR); - } - - void __init init_idt_traps(void) -diff --git a/xen/arch/x86/x86_64/compat/mm.c b/xen/arch/x86/x86_64/compat/mm.c -index c2aa6f2fdb..02bc75b91e 100644 ---- a/xen/arch/x86/x86_64/compat/mm.c -+++ b/xen/arch/x86/x86_64/compat/mm.c -@@ -163,19 +163,6 @@ int compat_arch_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) - return rc; - } - --int compat_update_va_mapping(unsigned int va, u32 lo, u32 hi, -- unsigned int flags) --{ -- return do_update_va_mapping(va, lo | ((u64)hi << 32), flags); --} -- --int compat_update_va_mapping_otherdomain(unsigned long va, u32 lo, u32 hi, -- unsigned long flags, -- domid_t domid) --{ -- return do_update_va_mapping_otherdomain(va, lo | ((u64)hi << 32), flags, domid); --} -- - DEFINE_XEN_GUEST_HANDLE(mmuext_op_compat_t); - - int compat_mmuext_op(XEN_GUEST_HANDLE_PARAM(void) arg, -diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c -index f7f6928d70..b0401850ef 100644 ---- a/xen/arch/x86/x86_64/traps.c -+++ b/xen/arch/x86/x86_64/traps.c -@@ -144,11 +144,12 @@ void show_registers(const struct cpu_user_regs *regs) - printk("CPU: %d\n", smp_processor_id()); - _show_registers(&fault_regs, fault_crs, context, v); - -- if ( this_cpu(ler_msr) && !guest_mode(regs) ) -+ if ( ler_msr && !guest_mode(regs) ) - { - u64 from, to; -- rdmsrl(this_cpu(ler_msr), from); -- rdmsrl(this_cpu(ler_msr) + 1, to); -+ -+ rdmsrl(ler_msr, from); -+ rdmsrl(ler_msr + 1, to); - printk("ler: %016lx -> %016lx\n", from, to); - } - } -diff --git a/xen/arch/x86/xstate.c b/xen/arch/x86/xstate.c -index b4aea4b50a..15edd5df96 100644 ---- a/xen/arch/x86/xstate.c -+++ b/xen/arch/x86/xstate.c -@@ -670,12 +670,17 @@ static bool valid_xcr0(u64 xcr0) - return !(xcr0 & X86_XCR0_BNDREGS) == !(xcr0 & X86_XCR0_BNDCSR); - } - --int validate_xstate(u64 xcr0, u64 xcr0_accum, const struct xsave_hdr *hdr) -+int validate_xstate(const struct domain *d, uint64_t xcr0, uint64_t xcr0_accum, -+ const struct xsave_hdr *hdr) - { -+ const struct cpuid_policy *cp = d->arch.cpuid; -+ uint64_t xcr0_max = -+ ((uint64_t)cp->xstate.xcr0_high << 32) | cp->xstate.xcr0_low; - unsigned int i; - - if ( (hdr->xstate_bv & ~xcr0_accum) || - (xcr0 & ~xcr0_accum) || -+ (xcr0_accum & ~xcr0_max) || - !valid_xcr0(xcr0) || - !valid_xcr0(xcr0_accum) ) - return -EINVAL; -@@ -694,20 +699,40 @@ int validate_xstate(u64 xcr0, u64 xcr0_accum, const struct xsave_hdr *hdr) - int handle_xsetbv(u32 index, u64 new_bv) - { - struct vcpu *curr = current; -+ const struct cpuid_policy *cp = curr->domain->arch.cpuid; -+ uint64_t xcr0_max = -+ ((uint64_t)cp->xstate.xcr0_high << 32) | cp->xstate.xcr0_low; - u64 mask; - - if ( index != XCR_XFEATURE_ENABLED_MASK ) - return -EOPNOTSUPP; - -- if ( (new_bv & ~xfeature_mask) || !valid_xcr0(new_bv) ) -+ /* -+ * The CPUID logic shouldn't be able to hand out an XCR0 exceeding Xen's -+ * maximum features, but keep the check for robustness. -+ */ -+ if ( unlikely(xcr0_max & ~xfeature_mask) ) -+ { -+ gprintk(XENLOG_ERR, -+ "xcr0_max %016" PRIx64 " exceeds hardware max %016" PRIx64 "\n", -+ xcr0_max, xfeature_mask); -+ domain_crash(curr->domain); -+ -+ return -EINVAL; -+ } -+ -+ if ( (new_bv & ~xcr0_max) || !valid_xcr0(new_bv) ) - return -EINVAL; - -- /* XCR0.PKRU is disabled on PV mode. */ -- if ( is_pv_vcpu(curr) && (new_bv & X86_XCR0_PKRU) ) -- return -EOPNOTSUPP; -+ /* By this point, new_bv really should be accepted by hardware. */ -+ if ( unlikely(!set_xcr0(new_bv)) ) -+ { -+ gprintk(XENLOG_ERR, "new_bv %016" PRIx64 " rejected by hardware\n", -+ new_bv); -+ domain_crash(curr->domain); - -- if ( !set_xcr0(new_bv) ) - return -EFAULT; -+ } - - mask = new_bv & ~curr->arch.xcr0_accum; - curr->arch.xcr0 = new_bv; -diff --git a/xen/common/cpu.c b/xen/common/cpu.c -index 6350f150bd..653a56b840 100644 ---- a/xen/common/cpu.c -+++ b/xen/common/cpu.c -@@ -67,12 +67,17 @@ void __init register_cpu_notifier(struct notifier_block *nb) - spin_unlock(&cpu_add_remove_lock); - } - --static int take_cpu_down(void *unused) -+static void _take_cpu_down(void *unused) - { - void *hcpu = (void *)(long)smp_processor_id(); - int notifier_rc = notifier_call_chain(&cpu_chain, CPU_DYING, hcpu, NULL); - BUG_ON(notifier_rc != NOTIFY_DONE); - __cpu_disable(); -+} -+ -+static int take_cpu_down(void *arg) -+{ -+ _take_cpu_down(arg); - return 0; - } - -@@ -98,7 +103,9 @@ int cpu_down(unsigned int cpu) - goto fail; - } - -- if ( (err = stop_machine_run(take_cpu_down, NULL, cpu)) < 0 ) -+ if ( unlikely(system_state < SYS_STATE_active) ) -+ on_selected_cpus(cpumask_of(cpu), _take_cpu_down, NULL, true); -+ else if ( (err = stop_machine_run(take_cpu_down, NULL, cpu)) < 0 ) - goto fail; - - __cpu_die(cpu); -diff --git a/xen/common/cpupool.c b/xen/common/cpupool.c -index 999839444e..1e8edcbd57 100644 ---- a/xen/common/cpupool.c -+++ b/xen/common/cpupool.c -@@ -490,7 +490,7 @@ static int cpupool_cpu_add(unsigned int cpu) - cpumask_clear_cpu(cpu, &cpupool_locked_cpus); - cpumask_set_cpu(cpu, &cpupool_free_cpus); - -- if ( system_state == SYS_STATE_resume ) -+ if ( system_state == SYS_STATE_suspend || system_state == SYS_STATE_resume ) - { - struct cpupool **c; - -@@ -522,6 +522,7 @@ static int cpupool_cpu_add(unsigned int cpu) - * (or unplugging would have failed) and that is the default behavior - * anyway. - */ -+ per_cpu(cpupool, cpu) = NULL; - ret = cpupool_assign_cpu_locked(cpupool0, cpu); - } - out: -diff --git a/xen/common/efi/boot.c b/xen/common/efi/boot.c -index 64d12685d3..6be0b3986f 100644 ---- a/xen/common/efi/boot.c -+++ b/xen/common/efi/boot.c -@@ -1304,6 +1304,8 @@ efi_start(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable) - - #ifndef CONFIG_ARM /* TODO - runtime service support */ - -+#include <asm/spec_ctrl.h> -+ - static bool __initdata efi_map_uc; - - static int __init parse_efi_param(const char *s) -@@ -1419,6 +1421,16 @@ void __init efi_init_memory(void) - desc->PhysicalStart, desc->PhysicalStart + len - 1, - desc->Type, desc->Attribute); - -+ if ( (desc->Attribute & (EFI_MEMORY_WB | EFI_MEMORY_WT)) || -+ (efi_bs_revision >= EFI_REVISION(2, 5) && -+ (desc->Attribute & EFI_MEMORY_WP)) ) -+ { -+ /* Supplement the heuristics in l1tf_calculations(). */ -+ l1tf_safe_maddr = -+ max(l1tf_safe_maddr, -+ ROUNDUP(desc->PhysicalStart + len, PAGE_SIZE)); -+ } -+ - if ( !efi_enabled(EFI_RS) || - (!(desc->Attribute & EFI_MEMORY_RUNTIME) && - (!map_bs || -diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c -index c757b7f6f5..231ecf509a 100644 ---- a/xen/common/grant_table.c -+++ b/xen/common/grant_table.c -@@ -97,7 +97,11 @@ static unsigned int __read_mostly max_maptrack_frames = - DEFAULT_MAX_MAPTRACK_FRAMES; - integer_runtime_param("gnttab_max_maptrack_frames", max_maptrack_frames); - --static unsigned int __read_mostly opt_gnttab_max_version = 2; -+#ifndef GNTTAB_MAX_VERSION -+#define GNTTAB_MAX_VERSION 2 -+#endif -+ -+static unsigned int __read_mostly opt_gnttab_max_version = GNTTAB_MAX_VERSION; - static bool __read_mostly opt_transitive_grants = true; - - static int __init parse_gnttab(const char *s) -diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c -index 20ee1e4897..02aeed7c47 100644 ---- a/xen/common/page_alloc.c -+++ b/xen/common/page_alloc.c -@@ -1426,7 +1426,7 @@ static void free_heap_pages( - - page_list_del(predecessor, &heap(node, zone, order)); - -- /* Keep predecessor's first_dirty if it is already set. */ -+ /* Update predecessor's first_dirty if necessary. */ - if ( predecessor->u.free.first_dirty == INVALID_DIRTY_IDX && - pg->u.free.first_dirty != INVALID_DIRTY_IDX ) - predecessor->u.free.first_dirty = (1U << order) + -@@ -1447,6 +1447,12 @@ static void free_heap_pages( - - check_and_stop_scrub(successor); - -+ /* Update pg's first_dirty if necessary. */ -+ if ( pg->u.free.first_dirty == INVALID_DIRTY_IDX && -+ successor->u.free.first_dirty != INVALID_DIRTY_IDX ) -+ pg->u.free.first_dirty = (1U << order) + -+ successor->u.free.first_dirty; -+ - page_list_del(successor, &heap(node, zone, order)); - } - -diff --git a/xen/common/tasklet.c b/xen/common/tasklet.c -index 0f0a6f8365..d4fea3151c 100644 ---- a/xen/common/tasklet.c -+++ b/xen/common/tasklet.c -@@ -156,6 +156,10 @@ void tasklet_kill(struct tasklet *t) - - spin_lock_irqsave(&tasklet_lock, flags); - -+ /* Cope with uninitialised tasklets. */ -+ if ( list_head_is_null(&t->list) ) -+ goto unlock; -+ - if ( !list_empty(&t->list) ) - { - BUG_ON(t->is_dead || t->is_running || (t->scheduled_on < 0)); -@@ -172,6 +176,7 @@ void tasklet_kill(struct tasklet *t) - spin_lock_irqsave(&tasklet_lock, flags); - } - -+ unlock: - spin_unlock_irqrestore(&tasklet_lock, flags); - } - -diff --git a/xen/include/asm-arm/arm32/system.h b/xen/include/asm-arm/arm32/system.h -index c617b40438..ab57abfbc5 100644 ---- a/xen/include/asm-arm/arm32/system.h -+++ b/xen/include/asm-arm/arm32/system.h -@@ -48,6 +48,24 @@ static inline int local_fiq_is_enabled(void) - return !(flags & PSR_FIQ_MASK); - } - -+#define CSDB ".inst 0xe320f014" -+ -+static inline unsigned long array_index_mask_nospec(unsigned long idx, -+ unsigned long sz) -+{ -+ unsigned long mask; -+ -+ asm volatile( "cmp %1, %2\n" -+ "sbc %0, %1, %1\n" -+ CSDB -+ : "=r" (mask) -+ : "r" (idx), "Ir" (sz) -+ : "cc" ); -+ -+ return mask; -+} -+#define array_index_mask_nospec array_index_mask_nospec -+ - #endif - /* - * Local variables: -diff --git a/xen/include/asm-arm/arm64/system.h b/xen/include/asm-arm/arm64/system.h -index 2e2ee212a1..2e36573ac6 100644 ---- a/xen/include/asm-arm/arm64/system.h -+++ b/xen/include/asm-arm/arm64/system.h -@@ -58,6 +58,28 @@ static inline int local_fiq_is_enabled(void) - return !(flags & PSR_FIQ_MASK); - } - -+#define csdb() asm volatile ( "hint #20" : : : "memory" ) -+ -+/* -+ * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz -+ * and 0 otherwise. -+ */ -+static inline unsigned long array_index_mask_nospec(unsigned long idx, -+ unsigned long sz) -+{ -+ unsigned long mask; -+ -+ asm volatile ( "cmp %1, %2\n" -+ "sbc %0, xzr, xzr\n" -+ : "=r" (mask) -+ : "r" (idx), "Ir" (sz) -+ : "cc" ); -+ csdb(); -+ -+ return mask; -+} -+#define array_index_mask_nospec array_index_mask_nospec -+ - #endif - /* - * Local variables: -diff --git a/xen/include/asm-arm/grant_table.h b/xen/include/asm-arm/grant_table.h -index e52936c79f..24958e4670 100644 ---- a/xen/include/asm-arm/grant_table.h -+++ b/xen/include/asm-arm/grant_table.h -@@ -7,6 +7,7 @@ - #include <xen/sched.h> - - #define INITIAL_NR_GRANT_FRAMES 1U -+#define GNTTAB_MAX_VERSION 1 - - struct grant_table_arch { - gfn_t *shared_gfn; -diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h -index 2cf8f7ea2a..b237da165c 100644 ---- a/xen/include/asm-x86/cpufeature.h -+++ b/xen/include/asm-x86/cpufeature.h -@@ -113,6 +113,7 @@ - #define cpu_has_aperfmperf boot_cpu_has(X86_FEATURE_APERFMPERF) - #define cpu_has_lfence_dispatch boot_cpu_has(X86_FEATURE_LFENCE_DISPATCH) - #define cpu_has_no_xpti boot_cpu_has(X86_FEATURE_NO_XPTI) -+#define cpu_has_xen_lbr boot_cpu_has(X86_FEATURE_XEN_LBR) - - enum _cache_type { - CACHE_TYPE_NULL = 0, -diff --git a/xen/include/asm-x86/cpufeatures.h b/xen/include/asm-x86/cpufeatures.h -index b90aa2d046..8e5cc53dde 100644 ---- a/xen/include/asm-x86/cpufeatures.h -+++ b/xen/include/asm-x86/cpufeatures.h -@@ -32,3 +32,4 @@ XEN_CPUFEATURE(SC_RSB_PV, (FSCAPINTS+0)*32+18) /* RSB overwrite needed for - XEN_CPUFEATURE(SC_RSB_HVM, (FSCAPINTS+0)*32+19) /* RSB overwrite needed for HVM */ - XEN_CPUFEATURE(NO_XPTI, (FSCAPINTS+0)*32+20) /* XPTI mitigation not in use */ - XEN_CPUFEATURE(SC_MSR_IDLE, (FSCAPINTS+0)*32+21) /* (SC_MSR_PV || SC_MSR_HVM) && default_xen_spec_ctrl */ -+XEN_CPUFEATURE(XEN_LBR, (FSCAPINTS+0)*32+22) /* Xen uses MSR_DEBUGCTL.LBR */ -diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h -index e0d413c7de..61e6900465 100644 ---- a/xen/include/asm-x86/domain.h -+++ b/xen/include/asm-x86/domain.h -@@ -121,6 +121,11 @@ struct shadow_domain { - - /* Has this domain ever used HVMOP_pagetable_dying? */ - bool_t pagetable_dying_op; -+ -+#ifdef CONFIG_PV -+ /* PV L1 Terminal Fault mitigation. */ -+ struct tasklet pv_l1tf_tasklet; -+#endif /* CONFIG_PV */ - #endif - }; - -@@ -257,6 +262,8 @@ struct pv_domain - bool xpti; - /* Use PCID feature? */ - bool pcid; -+ /* Mitigate L1TF with shadow/crashing? */ -+ bool check_l1tf; - - /* map_domain_page() mapping cache. */ - struct mapcache_domain mapcache; -diff --git a/xen/include/asm-x86/hvm/vmx/vmcs.h b/xen/include/asm-x86/hvm/vmx/vmcs.h -index 06c3179cec..57e5098b99 100644 ---- a/xen/include/asm-x86/hvm/vmx/vmcs.h -+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h -@@ -130,10 +130,18 @@ struct arch_vmx_struct { - uint64_t sfmask; - - struct vmx_msr_bitmap *msr_bitmap; -- unsigned int msr_count; -+ -+ /* -+ * Most accesses to the MSR host/guest load/save lists are in current -+ * context. However, the data can be modified by toolstack/migration -+ * actions. Remote access is only permitted for paused vcpus, and is -+ * protected under the domctl lock. -+ */ - struct vmx_msr_entry *msr_area; -- unsigned int host_msr_count; - struct vmx_msr_entry *host_msr_area; -+ unsigned int msr_load_count; -+ unsigned int msr_save_count; -+ unsigned int host_msr_count; - - unsigned long eoi_exitmap_changed; - DECLARE_BITMAP(eoi_exit_bitmap, NR_VECTORS); -@@ -149,7 +157,7 @@ struct arch_vmx_struct { - /* Are we emulating rather than VMENTERing? */ - uint8_t vmx_emulate; - -- uint8_t lbr_fixup_enabled; -+ uint8_t lbr_flags; - - /* Bitmask of segments that we can't safely use in virtual 8086 mode */ - uint16_t vm86_segment_mask; -@@ -514,9 +522,6 @@ enum vmcs_field { - - #define VMCS_VPID_WIDTH 16 - --#define VMX_GUEST_MSR 0 --#define VMX_HOST_MSR 1 -- - /* VM Instruction error numbers */ - enum vmx_insn_errno - { -@@ -534,6 +539,67 @@ enum vmx_insn_errno - VMX_INSN_FAIL_INVALID = ~0, - }; - -+/* MSR load/save list infrastructure. */ -+enum vmx_msr_list_type { -+ VMX_MSR_HOST, /* MSRs loaded on VMExit. */ -+ VMX_MSR_GUEST, /* MSRs saved on VMExit, loaded on VMEntry. */ -+ VMX_MSR_GUEST_LOADONLY, /* MSRs loaded on VMEntry only. */ -+}; -+ -+/** -+ * Add an MSR to an MSR list (inserting space for the entry if necessary), and -+ * set the MSRs value. -+ * -+ * It is undefined behaviour to try and insert the same MSR into both the -+ * GUEST and GUEST_LOADONLY list. -+ * -+ * May fail if unable to allocate memory for the list, or the total number of -+ * entries exceeds the memory allocated. -+ */ -+int vmx_add_msr(struct vcpu *v, uint32_t msr, uint64_t val, -+ enum vmx_msr_list_type type); -+ -+static inline int vmx_add_guest_msr(struct vcpu *v, uint32_t msr, uint64_t val) -+{ -+ return vmx_add_msr(v, msr, val, VMX_MSR_GUEST); -+} -+static inline int vmx_add_host_load_msr(struct vcpu *v, uint32_t msr, -+ uint64_t val) -+{ -+ return vmx_add_msr(v, msr, val, VMX_MSR_HOST); -+} -+ -+struct vmx_msr_entry *vmx_find_msr(const struct vcpu *v, uint32_t msr, -+ enum vmx_msr_list_type type); -+ -+static inline int vmx_read_guest_msr(const struct vcpu *v, uint32_t msr, -+ uint64_t *val) -+{ -+ const struct vmx_msr_entry *ent = vmx_find_msr(v, msr, VMX_MSR_GUEST); -+ -+ if ( !ent ) -+ return -ESRCH; -+ -+ *val = ent->data; -+ -+ return 0; -+} -+ -+static inline int vmx_write_guest_msr(struct vcpu *v, uint32_t msr, -+ uint64_t val) -+{ -+ struct vmx_msr_entry *ent = vmx_find_msr(v, msr, VMX_MSR_GUEST); -+ -+ if ( !ent ) -+ return -ESRCH; -+ -+ ent->data = val; -+ -+ return 0; -+} -+ -+ -+/* MSR intercept bitmap infrastructure. */ - enum vmx_msr_intercept_type { - VMX_MSR_R = 1, - VMX_MSR_W = 2, -@@ -544,10 +610,6 @@ void vmx_clear_msr_intercept(struct vcpu *v, unsigned int msr, - enum vmx_msr_intercept_type type); - void vmx_set_msr_intercept(struct vcpu *v, unsigned int msr, - enum vmx_msr_intercept_type type); --int vmx_read_guest_msr(u32 msr, u64 *val); --int vmx_write_guest_msr(u32 msr, u64 val); --struct vmx_msr_entry *vmx_find_msr(u32 msr, int type); --int vmx_add_msr(u32 msr, int type); - void vmx_vmcs_switch(paddr_t from, paddr_t to); - void vmx_set_eoi_exit_bitmap(struct vcpu *v, u8 vector); - void vmx_clear_eoi_exit_bitmap(struct vcpu *v, u8 vector); -@@ -562,15 +624,6 @@ void virtual_vmcs_vmwrite(const struct vcpu *, u32 encoding, u64 val); - enum vmx_insn_errno virtual_vmcs_vmwrite_safe(const struct vcpu *v, - u32 vmcs_encoding, u64 val); - --static inline int vmx_add_guest_msr(u32 msr) --{ -- return vmx_add_msr(msr, VMX_GUEST_MSR); --} --static inline int vmx_add_host_load_msr(u32 msr) --{ -- return vmx_add_msr(msr, VMX_HOST_MSR); --} -- - DECLARE_PER_CPU(bool_t, vmxon); - - bool_t vmx_vcpu_pml_enabled(const struct vcpu *v); -diff --git a/xen/include/asm-x86/hypercall.h b/xen/include/asm-x86/hypercall.h -index 1cc2e37d5c..da38b7991c 100644 ---- a/xen/include/asm-x86/hypercall.h -+++ b/xen/include/asm-x86/hypercall.h -@@ -165,7 +165,7 @@ extern int compat_update_va_mapping( - unsigned int va, u32 lo, u32 hi, unsigned int flags); - - extern int compat_update_va_mapping_otherdomain( -- unsigned long va, u32 lo, u32 hi, unsigned long flags, domid_t domid); -+ unsigned int va, u32 lo, u32 hi, unsigned int flags, domid_t domid); - - DEFINE_XEN_GUEST_HANDLE(trap_info_compat_t); - extern int compat_set_trap_table(XEN_GUEST_HANDLE(trap_info_compat_t) traps); -diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h -index 8fbccc88a7..7235623c86 100644 ---- a/xen/include/asm-x86/msr-index.h -+++ b/xen/include/asm-x86/msr-index.h -@@ -47,8 +47,12 @@ - #define ARCH_CAPABILITIES_RDCL_NO (_AC(1, ULL) << 0) - #define ARCH_CAPABILITIES_IBRS_ALL (_AC(1, ULL) << 1) - #define ARCH_CAPS_RSBA (_AC(1, ULL) << 2) -+#define ARCH_CAPS_SKIP_L1DFL (_AC(1, ULL) << 3) - #define ARCH_CAPS_SSB_NO (_AC(1, ULL) << 4) - -+#define MSR_FLUSH_CMD 0x0000010b -+#define FLUSH_CMD_L1D (_AC(1, ULL) << 0) -+ - /* Intel MSRs. Some also available on other CPUs */ - #define MSR_IA32_PERFCTR0 0x000000c1 - #define MSR_IA32_A_PERFCTR0 0x000004c1 -diff --git a/xen/include/asm-x86/msr.h b/xen/include/asm-x86/msr.h -index f14f265aa5..afbeb7f155 100644 ---- a/xen/include/asm-x86/msr.h -+++ b/xen/include/asm-x86/msr.h -@@ -241,7 +241,7 @@ static inline void write_efer(uint64_t val) - wrmsrl(MSR_EFER, val); - } - --DECLARE_PER_CPU(u32, ler_msr); -+extern unsigned int ler_msr; - - DECLARE_PER_CPU(uint32_t, tsc_aux); - -diff --git a/xen/include/asm-x86/paging.h b/xen/include/asm-x86/paging.h -index f0085511c7..f440e3e53c 100644 ---- a/xen/include/asm-x86/paging.h -+++ b/xen/include/asm-x86/paging.h -@@ -37,11 +37,14 @@ - - #define PG_SH_shift 20 - #define PG_HAP_shift 21 -+#define PG_SHF_shift 22 - /* We're in one of the shadow modes */ - #ifdef CONFIG_SHADOW_PAGING - #define PG_SH_enable (1U << PG_SH_shift) -+#define PG_SH_forced (1U << PG_SHF_shift) - #else - #define PG_SH_enable 0 -+#define PG_SH_forced 0 - #endif - #define PG_HAP_enable (1U << PG_HAP_shift) - -@@ -62,6 +65,7 @@ - - #define paging_mode_enabled(_d) (!!(_d)->arch.paging.mode) - #define paging_mode_shadow(_d) (!!((_d)->arch.paging.mode & PG_SH_enable)) -+#define paging_mode_sh_forced(_d) (!!((_d)->arch.paging.mode & PG_SH_forced)) - #define paging_mode_hap(_d) (!!((_d)->arch.paging.mode & PG_HAP_enable)) - - #define paging_mode_refcounts(_d) (!!((_d)->arch.paging.mode & PG_refcounts)) -diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h -index 9924cdf1f3..2bd9e69684 100644 ---- a/xen/include/asm-x86/processor.h -+++ b/xen/include/asm-x86/processor.h -@@ -337,12 +337,6 @@ static always_inline void set_in_cr4 (unsigned long mask) - write_cr4(read_cr4() | mask); - } - --static always_inline void clear_in_cr4 (unsigned long mask) --{ -- mmu_cr4_features &= ~mask; -- write_cr4(read_cr4() & ~mask); --} -- - static inline unsigned int read_pkru(void) - { - unsigned int pkru; -diff --git a/xen/include/asm-x86/setup.h b/xen/include/asm-x86/setup.h -index 19232afa01..c09a5ff381 100644 ---- a/xen/include/asm-x86/setup.h -+++ b/xen/include/asm-x86/setup.h -@@ -66,6 +66,8 @@ extern uint8_t kbd_shift_flags; - extern unsigned long highmem_start; - #endif - -+extern int8_t opt_smt; -+ - #ifdef CONFIG_SHADOW_PAGING - extern bool opt_dom0_shadow; - #else -diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h -index 94a34fd16a..f40f411871 100644 ---- a/xen/include/asm-x86/shadow.h -+++ b/xen/include/asm-x86/shadow.h -@@ -29,6 +29,7 @@ - #include <asm/flushtlb.h> - #include <asm/paging.h> - #include <asm/p2m.h> -+#include <asm/spec_ctrl.h> - - /***************************************************************************** - * Macros to tell which shadow paging mode a domain is in*/ -@@ -115,6 +116,131 @@ static inline int shadow_domctl(struct domain *d, - - #endif /* CONFIG_SHADOW_PAGING */ - -+/* -+ * Mitigations for L1TF / CVE-2018-3620 for PV guests. -+ * -+ * We cannot alter an architecturally-legitimate PTE which a PV guest has -+ * chosen to write, as traditional paged-out metadata is L1TF-vulnerable. -+ * What we can do is force a PV guest which writes a vulnerable PTE into -+ * shadow mode, so Xen controls the pagetables which are reachable by the CPU -+ * pagewalk. -+ * -+ * The core of the L1TF vulnerability is that the address bits of the PTE -+ * (accounting for PSE and factoring in the level-relevant part of the linear -+ * access) are sent for an L1D lookup (to retrieve the next-level PTE, or -+ * eventual memory address) before the Present or reserved bits (which would -+ * cause a terminal fault) are accounted for. If an L1D hit occurs, the -+ * resulting data is available for potentially dependent instructions. -+ * -+ * For Present PTEs, the PV type-count safety logic ensures that the address -+ * bits always point at a guest-accessible frame, which is safe WRT L1TF from -+ * Xen's point of view. In practice, a PV guest should be unable to set any -+ * reserved bits, so should be unable to create any present L1TF-vulnerable -+ * PTEs at all. -+ * -+ * Therefore, these safety checks apply to Not-Present PTEs only, where -+ * traditionally, Xen would have let the guest write any value it chose. -+ * -+ * The all-zero PTE potentially leaks mfn 0. All software on the system is -+ * expected to cooperate and not put any secrets there. In a Xen system, -+ * neither Xen nor dom0 are expected to touch mfn 0, as it typically contains -+ * the real mode IVT and Bios Data Area. Therefore, mfn 0 is considered safe. -+ * -+ * Any PTE whose address is higher than the maximum cacheable address is safe, -+ * as it won't get an L1D hit. -+ * -+ * Speculative superpages also need accounting for, as PSE is considered -+ * irrespective of Present. We disallow PSE being set, as it allows an -+ * attacker to leak 2M or 1G of data starting from mfn 0. Also, because of -+ * recursive/linear pagetables, we must consider PSE even at L4, as hardware -+ * will interpret an L4e as an L3e during a recursive walk. -+ */ -+ -+static inline bool is_l1tf_safe_maddr(intpte_t pte) -+{ -+ paddr_t maddr = pte & l1tf_addr_mask; -+ -+ return maddr == 0 || maddr >= l1tf_safe_maddr; -+} -+ -+static inline bool pv_l1tf_check_pte(struct domain *d, unsigned int level, -+ intpte_t pte) -+{ -+ ASSERT(is_pv_domain(d)); -+ ASSERT(!(pte & _PAGE_PRESENT)); -+ -+ if ( d->arch.pv_domain.check_l1tf && !paging_mode_sh_forced(d) && -+ (((level > 1) && (pte & _PAGE_PSE)) || !is_l1tf_safe_maddr(pte)) ) -+ { -+#ifdef CONFIG_SHADOW_PAGING -+ struct tasklet *t = &d->arch.paging.shadow.pv_l1tf_tasklet; -+ -+ printk(XENLOG_G_WARNING -+ "d%d L1TF-vulnerable L%ue %016"PRIx64" - Shadowing\n", -+ d->domain_id, level, pte); -+ /* -+ * Safety consideration for accessing tasklet.scheduled_on without the -+ * tasklet lock. This is a singleshot tasklet with the side effect of -+ * setting PG_SH_forced (checked just above). Multiple vcpus can race -+ * to schedule the tasklet, but if we observe it scheduled anywhere, -+ * that is good enough. -+ */ -+ smp_rmb(); -+ if ( !tasklet_is_scheduled(t) ) -+ tasklet_schedule(t); -+#else -+ printk(XENLOG_G_ERR -+ "d%d L1TF-vulnerable L%ue %016"PRIx64" - Crashing\n", -+ d->domain_id, level, pte); -+ domain_crash(d); -+#endif -+ return true; -+ } -+ -+ return false; -+} -+ -+static inline bool pv_l1tf_check_l1e(struct domain *d, l1_pgentry_t l1e) -+{ -+ return pv_l1tf_check_pte(d, 1, l1e.l1); -+} -+ -+static inline bool pv_l1tf_check_l2e(struct domain *d, l2_pgentry_t l2e) -+{ -+ return pv_l1tf_check_pte(d, 2, l2e.l2); -+} -+ -+static inline bool pv_l1tf_check_l3e(struct domain *d, l3_pgentry_t l3e) -+{ -+ return pv_l1tf_check_pte(d, 3, l3e.l3); -+} -+ -+static inline bool pv_l1tf_check_l4e(struct domain *d, l4_pgentry_t l4e) -+{ -+ return pv_l1tf_check_pte(d, 4, l4e.l4); -+} -+ -+void pv_l1tf_tasklet(unsigned long data); -+ -+static inline void pv_l1tf_domain_init(struct domain *d) -+{ -+ d->arch.pv_domain.check_l1tf = -+ opt_pv_l1tf & (is_hardware_domain(d) -+ ? OPT_PV_L1TF_DOM0 : OPT_PV_L1TF_DOMU); -+ -+#if defined(CONFIG_SHADOW_PAGING) && defined(CONFIG_PV) -+ tasklet_init(&d->arch.paging.shadow.pv_l1tf_tasklet, -+ pv_l1tf_tasklet, (unsigned long)d); -+#endif -+} -+ -+static inline void pv_l1tf_domain_destroy(struct domain *d) -+{ -+#if defined(CONFIG_SHADOW_PAGING) && defined(CONFIG_PV) -+ tasklet_kill(&d->arch.paging.shadow.pv_l1tf_tasklet); -+#endif -+} -+ - /* Remove all shadows of the guest mfn. */ - static inline void shadow_remove_all_shadows(struct domain *d, mfn_t gmfn) - { -diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h -index 4e5f673fec..09c55458df 100644 ---- a/xen/include/asm-x86/smp.h -+++ b/xen/include/asm-x86/smp.h -@@ -26,6 +26,8 @@ DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_mask); - DECLARE_PER_CPU(cpumask_var_t, cpu_core_mask); - DECLARE_PER_CPU(cpumask_var_t, scratch_cpumask); - -+extern bool park_offline_cpus; -+ - void smp_send_nmi_allbutself(void); - - void send_IPI_mask(const cpumask_t *, int vector); -diff --git a/xen/include/asm-x86/spec_ctrl.h b/xen/include/asm-x86/spec_ctrl.h -index 5b40afbab0..8f8aad40bb 100644 ---- a/xen/include/asm-x86/spec_ctrl.h -+++ b/xen/include/asm-x86/spec_ctrl.h -@@ -29,15 +29,27 @@ void init_speculation_mitigations(void); - extern bool opt_ibpb; - extern bool opt_ssbd; - extern int8_t opt_eager_fpu; -+extern int8_t opt_l1d_flush; - - extern bool bsp_delay_spec_ctrl; - extern uint8_t default_xen_spec_ctrl; - extern uint8_t default_spec_ctrl_flags; - --extern uint8_t opt_xpti; -+extern int8_t opt_xpti; - #define OPT_XPTI_DOM0 0x01 - #define OPT_XPTI_DOMU 0x02 - -+extern int8_t opt_pv_l1tf; -+#define OPT_PV_L1TF_DOM0 0x01 -+#define OPT_PV_L1TF_DOMU 0x02 -+ -+/* -+ * The L1D address mask, which might be wider than reported in CPUID, and the -+ * system physical address above which there are believed to be no cacheable -+ * memory regions, thus unable to leak data via the L1TF vulnerability. -+ */ -+extern paddr_t l1tf_addr_mask, l1tf_safe_maddr; -+ - static inline void init_shadow_spec_ctrl_state(void) - { - struct cpu_info *info = get_cpu_info(); -diff --git a/xen/include/asm-x86/system.h b/xen/include/asm-x86/system.h -index 43fb6fe489..483cd20afd 100644 ---- a/xen/include/asm-x86/system.h -+++ b/xen/include/asm-x86/system.h -@@ -221,6 +221,30 @@ static always_inline unsigned long __xadd( - #define set_mb(var, value) do { xchg(&var, value); } while (0) - #define set_wmb(var, value) do { var = value; smp_wmb(); } while (0) - -+/** -+ * array_index_mask_nospec() - generate a mask that is ~0UL when the -+ * bounds check succeeds and 0 otherwise -+ * @index: array element index -+ * @size: number of elements in array -+ * -+ * Returns: -+ * 0 - (index < size) -+ */ -+static inline unsigned long array_index_mask_nospec(unsigned long index, -+ unsigned long size) -+{ -+ unsigned long mask; -+ -+ asm volatile ( "cmp %[size], %[index]; sbb %[mask], %[mask];" -+ : [mask] "=r" (mask) -+ : [size] "g" (size), [index] "r" (index) ); -+ -+ return mask; -+} -+ -+/* Override default implementation in nospec.h. */ -+#define array_index_mask_nospec array_index_mask_nospec -+ - #define local_irq_disable() asm volatile ( "cli" : : : "memory" ) - #define local_irq_enable() asm volatile ( "sti" : : : "memory" ) - -diff --git a/xen/include/asm-x86/xstate.h b/xen/include/asm-x86/xstate.h -index 86a4a1f75c..47f602b855 100644 ---- a/xen/include/asm-x86/xstate.h -+++ b/xen/include/asm-x86/xstate.h -@@ -97,8 +97,9 @@ void xsave(struct vcpu *v, uint64_t mask); - void xrstor(struct vcpu *v, uint64_t mask); - void xstate_set_init(uint64_t mask); - bool xsave_enabled(const struct vcpu *v); --int __must_check validate_xstate(u64 xcr0, u64 xcr0_accum, -- const struct xsave_hdr *); -+int __must_check validate_xstate(const struct domain *d, -+ uint64_t xcr0, uint64_t xcr0_accum, -+ const struct xsave_hdr *hdr); - int __must_check handle_xsetbv(u32 index, u64 new_bv); - void expand_xsave_states(struct vcpu *v, void *dest, unsigned int size); - void compress_xsave_states(struct vcpu *v, const void *src, unsigned int size); -diff --git a/xen/include/public/arch-x86/cpufeatureset.h b/xen/include/public/arch-x86/cpufeatureset.h -index f1a5ed93e0..6c82816fd3 100644 ---- a/xen/include/public/arch-x86/cpufeatureset.h -+++ b/xen/include/public/arch-x86/cpufeatureset.h -@@ -244,6 +244,7 @@ XEN_CPUFEATURE(AVX512_4VNNIW, 9*32+ 2) /*A AVX512 Neural Network Instructions * - XEN_CPUFEATURE(AVX512_4FMAPS, 9*32+ 3) /*A AVX512 Multiply Accumulation Single Precision */ - XEN_CPUFEATURE(IBRSB, 9*32+26) /*A IBRS and IBPB support (used by Intel) */ - XEN_CPUFEATURE(STIBP, 9*32+27) /*A STIBP */ -+XEN_CPUFEATURE(L1D_FLUSH, 9*32+28) /*S MSR_FLUSH_CMD and L1D flush. */ - XEN_CPUFEATURE(ARCH_CAPS, 9*32+29) /* IA32_ARCH_CAPABILITIES MSR */ - XEN_CPUFEATURE(SSBD, 9*32+31) /*A MSR_SPEC_CTRL.SSBD available */ - -diff --git a/xen/include/xen/compiler.h b/xen/include/xen/compiler.h -index 533a8ea0f3..a7e05681c9 100644 ---- a/xen/include/xen/compiler.h -+++ b/xen/include/xen/compiler.h -@@ -81,6 +81,9 @@ - #pragma GCC visibility push(hidden) - #endif - -+/* Make the optimizer believe the variable can be manipulated arbitrarily. */ -+#define OPTIMIZER_HIDE_VAR(var) __asm__ ( "" : "+g" (var) ) -+ - /* This macro obfuscates arithmetic on a variable address so that gcc - shouldn't recognize the original var, and make assumptions about it */ - /* -diff --git a/xen/include/xen/cpu.h b/xen/include/xen/cpu.h -index ffefc09f8e..2fe3ec05d8 100644 ---- a/xen/include/xen/cpu.h -+++ b/xen/include/xen/cpu.h -@@ -47,6 +47,8 @@ void register_cpu_notifier(struct notifier_block *nb); - #define CPU_DYING (0x0007 | NOTIFY_REVERSE) - /* CPU_DEAD: CPU is dead. */ - #define CPU_DEAD (0x0008 | NOTIFY_REVERSE) -+/* CPU_REMOVE: CPU was removed. */ -+#define CPU_REMOVE (0x0009 | NOTIFY_REVERSE) - - /* Perform CPU hotplug. May return -EAGAIN. */ - int cpu_down(unsigned int cpu); -diff --git a/xen/include/xen/cpumask.h b/xen/include/xen/cpumask.h -index 42340a098e..4a11bcc3f3 100644 ---- a/xen/include/xen/cpumask.h -+++ b/xen/include/xen/cpumask.h -@@ -351,16 +351,35 @@ static inline bool_t alloc_cpumask_var(cpumask_var_t *mask) - return *mask != NULL; - } - -+static inline bool cond_alloc_cpumask_var(cpumask_var_t *mask) -+{ -+ if (*mask == NULL) -+ *mask = _xmalloc(nr_cpumask_bits / 8, sizeof(long)); -+ return *mask != NULL; -+} -+ - static inline bool_t zalloc_cpumask_var(cpumask_var_t *mask) - { - *(void **)mask = _xzalloc(nr_cpumask_bits / 8, sizeof(long)); - return *mask != NULL; - } - -+static inline bool cond_zalloc_cpumask_var(cpumask_var_t *mask) -+{ -+ if (*mask == NULL) -+ *mask = _xzalloc(nr_cpumask_bits / 8, sizeof(long)); -+ else -+ cpumask_clear(*mask); -+ return *mask != NULL; -+} -+ - static inline void free_cpumask_var(cpumask_var_t mask) - { - xfree(mask); - } -+ -+/* Free an allocated mask, and zero the pointer to it. */ -+#define FREE_CPUMASK_VAR(m) XFREE(m) - #else - typedef cpumask_t cpumask_var_t[1]; - -@@ -368,16 +387,20 @@ static inline bool_t alloc_cpumask_var(cpumask_var_t *mask) - { - return 1; - } -+#define cond_alloc_cpumask_var alloc_cpumask_var - - static inline bool_t zalloc_cpumask_var(cpumask_var_t *mask) - { - cpumask_clear(*mask); - return 1; - } -+#define cond_zalloc_cpumask_var zalloc_cpumask_var - - static inline void free_cpumask_var(cpumask_var_t mask) - { - } -+ -+#define FREE_CPUMASK_VAR(m) free_cpumask_var(m) - #endif - - #if NR_CPUS > 1 -diff --git a/xen/include/xen/list.h b/xen/include/xen/list.h -index fa07d720ee..1387abb211 100644 ---- a/xen/include/xen/list.h -+++ b/xen/include/xen/list.h -@@ -51,6 +51,11 @@ static inline void INIT_LIST_HEAD(struct list_head *list) - list->prev = list; - } - -+static inline bool list_head_is_null(const struct list_head *list) -+{ -+ return !list->next && !list->prev; -+} -+ - /* - * Insert a new entry between two known consecutive entries. - * -diff --git a/xen/include/xen/mm.h b/xen/include/xen/mm.h -index e928551c91..24654e8e22 100644 ---- a/xen/include/xen/mm.h -+++ b/xen/include/xen/mm.h -@@ -162,6 +162,14 @@ void free_xenheap_pages(void *v, unsigned int order); - bool scrub_free_pages(void); - #define alloc_xenheap_page() (alloc_xenheap_pages(0,0)) - #define free_xenheap_page(v) (free_xenheap_pages(v,0)) -+ -+/* Free an allocation, and zero the pointer to it. */ -+#define FREE_XENHEAP_PAGES(p, o) do { \ -+ free_xenheap_pages(p, o); \ -+ (p) = NULL; \ -+} while ( false ) -+#define FREE_XENHEAP_PAGE(p) FREE_XENHEAP_PAGES(p, 0) -+ - /* Map machine page range in Xen virtual address space. */ - int map_pages_to_xen( - unsigned long virt, -diff --git a/xen/include/xen/nospec.h b/xen/include/xen/nospec.h -new file mode 100644 -index 0000000000..48793996e8 ---- /dev/null -+++ b/xen/include/xen/nospec.h -@@ -0,0 +1,70 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* Copyright(c) 2018 Linus Torvalds. All rights reserved. */ -+/* Copyright(c) 2018 Alexei Starovoitov. All rights reserved. */ -+/* Copyright(c) 2018 Intel Corporation. All rights reserved. */ -+/* Copyright(c) 2018 Citrix Systems R&D Ltd. All rights reserved. */ -+ -+#ifndef XEN_NOSPEC_H -+#define XEN_NOSPEC_H -+ -+#include <asm/system.h> -+ -+/** -+ * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise -+ * @index: array element index -+ * @size: number of elements in array -+ * -+ * When @index is out of bounds (@index >= @size), the sign bit will be -+ * set. Extend the sign bit to all bits and invert, giving a result of -+ * zero for an out of bounds index, or ~0 if within bounds [0, @size). -+ */ -+#ifndef array_index_mask_nospec -+static inline unsigned long array_index_mask_nospec(unsigned long index, -+ unsigned long size) -+{ -+ /* -+ * Always calculate and emit the mask even if the compiler -+ * thinks the mask is not needed. The compiler does not take -+ * into account the value of @index under speculation. -+ */ -+ OPTIMIZER_HIDE_VAR(index); -+ return ~(long)(index | (size - 1UL - index)) >> (BITS_PER_LONG - 1); -+} -+#endif -+ -+/* -+ * array_index_nospec - sanitize an array index after a bounds check -+ * -+ * For a code sequence like: -+ * -+ * if (index < size) { -+ * index = array_index_nospec(index, size); -+ * val = array[index]; -+ * } -+ * -+ * ...if the CPU speculates past the bounds check then -+ * array_index_nospec() will clamp the index within the range of [0, -+ * size). -+ */ -+#define array_index_nospec(index, size) \ -+({ \ -+ typeof(index) _i = (index); \ -+ typeof(size) _s = (size); \ -+ unsigned long _mask = array_index_mask_nospec(_i, _s); \ -+ \ -+ BUILD_BUG_ON(sizeof(_i) > sizeof(long)); \ -+ BUILD_BUG_ON(sizeof(_s) > sizeof(long)); \ -+ \ -+ (typeof(_i)) (_i & _mask); \ -+}) -+ -+#endif /* XEN_NOSPEC_H */ -+ -+/* -+ * Local variables: -+ * mode: C -+ * c-file-style: "BSD" -+ * c-basic-offset: 4 -+ * indent-tabs-mode: nil -+ * End: -+ */ -diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h -index 99d2af2e1f..e79d5a36ca 100644 ---- a/xen/include/xen/sched.h -+++ b/xen/include/xen/sched.h -@@ -788,7 +788,7 @@ static inline struct domain *next_domain_in_cpupool( - #define _VPF_parked 8 - #define VPF_parked (1UL<<_VPF_parked) - --static inline int vcpu_runnable(struct vcpu *v) -+static inline bool vcpu_runnable(const struct vcpu *v) - { - return !(v->pause_flags | - atomic_read(&v->pause_count) | -diff --git a/xen/include/xen/tasklet.h b/xen/include/xen/tasklet.h -index 23d69c738e..bc9ddace6d 100644 ---- a/xen/include/xen/tasklet.h -+++ b/xen/include/xen/tasklet.h -@@ -50,6 +50,11 @@ static inline bool tasklet_work_to_do(unsigned int cpu) - TASKLET_scheduled); - } - -+static inline bool tasklet_is_scheduled(const struct tasklet *t) -+{ -+ return t->scheduled_on != -1; -+} -+ - void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu); - void tasklet_schedule(struct tasklet *t); - void do_tasklet(void); -diff --git a/xen/include/xen/xmalloc.h b/xen/include/xen/xmalloc.h -index cc2673d8ae..9aa5edf593 100644 ---- a/xen/include/xen/xmalloc.h -+++ b/xen/include/xen/xmalloc.h -@@ -26,6 +26,12 @@ - /* Free any of the above. */ - extern void xfree(void *); - -+/* Free an allocation, and zero the pointer to it. */ -+#define XFREE(p) do { \ -+ xfree(p); \ -+ (p) = NULL; \ -+} while ( false ) -+ - /* Underlying functions */ - extern void *_xmalloc(unsigned long size, unsigned long align); - extern void *_xzalloc(unsigned long size, unsigned long align); |