From aadfe320919b15e24ac070e9b25085e07599a613 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Fri, 29 Apr 2022 15:40:52 +0100 Subject: hw/cxl/host: Add support for CXL Fixed Memory Windows. The concept of these is introduced in [1] in terms of the description the CEDT ACPI table. The principal is more general. Unlike once traffic hits the CXL root bridges, the host system memory address routing is implementation defined and effectively static once observable by standard / generic system software. Each CXL Fixed Memory Windows (CFMW) is a region of PA space which has fixed system dependent routing configured so that accesses can be routed to the CXL devices below a set of target root bridges. The accesses may be interleaved across multiple root bridges. For QEMU we could have fully specified these regions in terms of a base PA + size, but as the absolute address does not matter it is simpler to let individual platforms place the memory regions. ExampleS: -cxl-fixed-memory-window targets.0=cxl.0,size=128G -cxl-fixed-memory-window targets.0=cxl.1,size=128G -cxl-fixed-memory-window targets.0=cxl0,targets.1=cxl.1,size=256G,interleave-granularity=2k Specifies * 2x 128G regions not interleaved across root bridges, one for each of the root bridges with ids cxl.0 and cxl.1 * 256G region interleaved across root bridges with ids cxl.0 and cxl.1 with a 2k interleave granularity. When system software enumerates the devices below a given root bridge it can then decide which CFMW to use. If non interleave is desired (or possible) it can use the appropriate CFMW for the root bridge in question. If there are suitable devices to interleave across the two root bridges then it may use the 3rd CFMS. A number of other designs were considered but the following constraints made it hard to adapt existing QEMU approaches to this particular problem. 1) The size must be known before a specific architecture / board brings up it's PA memory map. We need to set up an appropriate region. 2) Using links to the host bridges provides a clean command line interface but these links cannot be established until command line devices have been added. Hence the two step process used here of first establishing the size, interleave-ways and granularity + caching the ids of the host bridges and then, once available finding the actual host bridges so they can be used later to support interleave decoding. [1] CXL 2.0 ECN: CEDT CFMWS & QTG DSM (computeexpresslink.org / specifications) Signed-off-by: Jonathan Cameron Acked-by: Markus Armbruster # QAPI Schema Message-Id: <20220429144110.25167-28-Jonathan.Cameron@huawei.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- softmmu/vl.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) (limited to 'softmmu') diff --git a/softmmu/vl.c b/softmmu/vl.c index 817d515783..2390c13fb6 100644 --- a/softmmu/vl.c +++ b/softmmu/vl.c @@ -93,6 +93,7 @@ #include "qemu/config-file.h" #include "qemu/qemu-options.h" #include "qemu/main-loop.h" +#include "hw/cxl/cxl.h" #ifdef CONFIG_VIRTFS #include "fsdev/qemu-fsdev.h" #endif @@ -118,6 +119,7 @@ #include "qapi/qapi-events-run-state.h" #include "qapi/qapi-visit-block-core.h" #include "qapi/qapi-visit-compat.h" +#include "qapi/qapi-visit-machine.h" #include "qapi/qapi-visit-ui.h" #include "qapi/qapi-commands-block-core.h" #include "qapi/qapi-commands-migration.h" @@ -143,6 +145,12 @@ typedef struct BlockdevOptionsQueueEntry { typedef QSIMPLEQ_HEAD(, BlockdevOptionsQueueEntry) BlockdevOptionsQueue; +typedef struct CXLFMWOptionQueueEntry { + CXLFixedMemoryWindowOptions *opts; + Location loc; + QSIMPLEQ_ENTRY(CXLFMWOptionQueueEntry) entry; +} CXLFMWOptionQueueEntry; + typedef struct ObjectOption { ObjectOptions *opts; QTAILQ_ENTRY(ObjectOption) next; @@ -169,6 +177,8 @@ static int snapshot; static bool preconfig_requested; static QemuPluginList plugin_list = QTAILQ_HEAD_INITIALIZER(plugin_list); static BlockdevOptionsQueue bdo_queue = QSIMPLEQ_HEAD_INITIALIZER(bdo_queue); +static QSIMPLEQ_HEAD(, CXLFMWOptionQueueEntry) CXLFMW_opts = + QSIMPLEQ_HEAD_INITIALIZER(CXLFMW_opts); static bool nographic = false; static int mem_prealloc; /* force preallocation of physical target memory */ static const char *vga_model = NULL; @@ -1153,6 +1163,24 @@ static void parse_display(const char *p) } } +static void parse_cxl_fixed_memory_window(const char *optarg) +{ + CXLFMWOptionQueueEntry *cfmws_entry; + Visitor *v; + + v = qobject_input_visitor_new_str(optarg, "cxl-fixed-memory-window", + &error_fatal); + cfmws_entry = g_new(CXLFMWOptionQueueEntry, 1); + visit_type_CXLFixedMemoryWindowOptions(v, NULL, &cfmws_entry->opts, + &error_fatal); + if (!cfmws_entry->opts) { + exit(1); + } + visit_free(v); + loc_save(&cfmws_entry->loc); + QSIMPLEQ_INSERT_TAIL(&CXLFMW_opts, cfmws_entry, entry); +} + static inline bool nonempty_str(const char *str) { return str && *str; @@ -2015,6 +2043,20 @@ static void qemu_create_late_backends(void) qemu_semihosting_console_init(); } +static void cxl_set_opts(void) +{ + while (!QSIMPLEQ_EMPTY(&CXLFMW_opts)) { + CXLFMWOptionQueueEntry *cfmws_entry = QSIMPLEQ_FIRST(&CXLFMW_opts); + + loc_restore(&cfmws_entry->loc); + QSIMPLEQ_REMOVE_HEAD(&CXLFMW_opts, entry); + cxl_fixed_memory_window_config(current_machine, cfmws_entry->opts, + &error_fatal); + qapi_free_CXLFixedMemoryWindowOptions(cfmws_entry->opts); + g_free(cfmws_entry); + } +} + static void qemu_resolve_machine_memdev(void) { if (ram_memdev_id) { @@ -2661,6 +2703,7 @@ void qmp_x_exit_preconfig(Error **errp) qemu_init_board(); qemu_create_cli_devices(); + cxl_fixed_memory_window_link_targets(errp); qemu_machine_creation_done(); if (loadvm) { @@ -2841,6 +2884,9 @@ void qemu_init(int argc, char **argv, char **envp) exit(1); } break; + case QEMU_OPTION_cxl_fixed_memory_window: + parse_cxl_fixed_memory_window(optarg); + break; case QEMU_OPTION_display: parse_display(optarg); break; @@ -3652,6 +3698,7 @@ void qemu_init(int argc, char **argv, char **envp) qemu_resolve_machine_memdev(); parse_numa_opts(current_machine); + cxl_set_opts(); if (vmstate_dump_file) { /* dump and exit */ -- cgit v1.2.3