1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
/*
* QEMU SPAPR PCI BUS definitions
*
* Copyright (c) 2011 Alexey Kardashevskiy <aik@au1.ibm.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#ifndef PCI_HOST_SPAPR_H
#define PCI_HOST_SPAPR_H
#include "hw/ppc/spapr.h"
#include "hw/pci/pci.h"
#include "hw/pci/pci_host.h"
#include "hw/ppc/xics.h"
#define TYPE_SPAPR_PCI_HOST_BRIDGE "spapr-pci-host-bridge"
#define SPAPR_PCI_HOST_BRIDGE(obj) \
OBJECT_CHECK(SpaprPhbState, (obj), TYPE_SPAPR_PCI_HOST_BRIDGE)
#define SPAPR_PCI_DMA_MAX_WINDOWS 2
typedef struct SpaprPhbState SpaprPhbState;
typedef struct SpaprPciMsi {
uint32_t first_irq;
uint32_t num;
} SpaprPciMsi;
typedef struct SpaprPciMsiMig {
uint32_t key;
SpaprPciMsi value;
} SpaprPciMsiMig;
typedef struct SpaprPciLsi {
uint32_t irq;
} SpaprPciLsi;
typedef struct SpaprPhbPciNvGpuConfig SpaprPhbPciNvGpuConfig;
struct SpaprPhbState {
PCIHostState parent_obj;
uint32_t index;
uint64_t buid;
char *dtbusname;
bool dr_enabled;
MemoryRegion memspace, iospace;
hwaddr mem_win_addr, mem_win_size, mem64_win_addr, mem64_win_size;
uint64_t mem64_win_pciaddr;
hwaddr io_win_addr, io_win_size;
MemoryRegion mem32window, mem64window, iowindow, msiwindow;
uint32_t dma_liobn[SPAPR_PCI_DMA_MAX_WINDOWS];
hwaddr dma_win_addr, dma_win_size;
AddressSpace iommu_as;
MemoryRegion iommu_root;
SpaprPciLsi lsi_table[PCI_NUM_PINS];
GHashTable *msi;
/* Temporary cache for migration purposes */
int32_t msi_devs_num;
SpaprPciMsiMig *msi_devs;
QLIST_ENTRY(SpaprPhbState) list;
bool ddw_enabled;
uint64_t page_size_mask;
uint64_t dma64_win_addr;
uint32_t numa_node;
bool pcie_ecs; /* Allow access to PCIe extended config space? */
/* Fields for migration compatibility hacks */
bool pre_2_8_migration;
uint32_t mig_liobn;
hwaddr mig_mem_win_addr, mig_mem_win_size;
hwaddr mig_io_win_addr, mig_io_win_size;
hwaddr nv2_gpa_win_addr;
hwaddr nv2_atsd_win_addr;
SpaprPhbPciNvGpuConfig *nvgpus;
bool pre_5_1_assoc;
};
#define SPAPR_PCI_MEM_WIN_BUS_OFFSET 0x80000000ULL
#define SPAPR_PCI_MEM32_WIN_SIZE \
((1ULL << 32) - SPAPR_PCI_MEM_WIN_BUS_OFFSET)
#define SPAPR_PCI_MEM64_WIN_SIZE 0x10000000000ULL /* 1 TiB */
/* All PCI outbound windows will be within this range */
#define SPAPR_PCI_BASE (1ULL << 45) /* 32 TiB */
#define SPAPR_PCI_LIMIT (1ULL << 46) /* 64 TiB */
#define SPAPR_MAX_PHBS ((SPAPR_PCI_LIMIT - SPAPR_PCI_BASE) / \
SPAPR_PCI_MEM64_WIN_SIZE - 1)
#define SPAPR_PCI_IO_WIN_SIZE 0x10000
#define SPAPR_PCI_MSI_WINDOW 0x40000000000ULL
#define SPAPR_PCI_NV2RAM64_WIN_BASE SPAPR_PCI_LIMIT
#define SPAPR_PCI_NV2RAM64_WIN_SIZE (2 * TiB) /* For up to 6 GPUs 256GB each */
/* Max number of these GPUsper a physical box */
#define NVGPU_MAX_NUM 6
/* Max number of NVLinks per GPU in any physical box */
#define NVGPU_MAX_LINKS 3
/*
* GPU RAM starts at 64TiB so huge DMA window to cover it all ends at 128TiB
* which is enough. We do not need DMA for ATSD so we put them at 128TiB.
*/
#define SPAPR_PCI_NV2ATSD_WIN_BASE (128 * TiB)
#define SPAPR_PCI_NV2ATSD_WIN_SIZE (NVGPU_MAX_NUM * NVGPU_MAX_LINKS * \
64 * KiB)
int spapr_dt_phb(SpaprMachineState *spapr, SpaprPhbState *phb,
uint32_t intc_phandle, void *fdt, int *node_offset);
void spapr_pci_rtas_init(void);
SpaprPhbState *spapr_pci_find_phb(SpaprMachineState *spapr, uint64_t buid);
PCIDevice *spapr_pci_find_dev(SpaprMachineState *spapr, uint64_t buid,
uint32_t config_addr);
/* DRC callbacks */
void spapr_phb_remove_pci_device_cb(DeviceState *dev);
int spapr_pci_dt_populate(SpaprDrc *drc, SpaprMachineState *spapr,
void *fdt, int *fdt_start_offset, Error **errp);
/* VFIO EEH hooks */
#ifdef CONFIG_LINUX
bool spapr_phb_eeh_available(SpaprPhbState *sphb);
int spapr_phb_vfio_eeh_set_option(SpaprPhbState *sphb,
unsigned int addr, int option);
int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb, int *state);
int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option);
int spapr_phb_vfio_eeh_configure(SpaprPhbState *sphb);
void spapr_phb_vfio_reset(DeviceState *qdev);
void spapr_phb_nvgpu_setup(SpaprPhbState *sphb, Error **errp);
void spapr_phb_nvgpu_free(SpaprPhbState *sphb);
void spapr_phb_nvgpu_populate_dt(SpaprPhbState *sphb, void *fdt, int bus_off,
Error **errp);
void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt);
void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
SpaprPhbState *sphb);
#else
static inline bool spapr_phb_eeh_available(SpaprPhbState *sphb)
{
return false;
}
static inline int spapr_phb_vfio_eeh_set_option(SpaprPhbState *sphb,
unsigned int addr, int option)
{
return RTAS_OUT_HW_ERROR;
}
static inline int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb,
int *state)
{
return RTAS_OUT_HW_ERROR;
}
static inline int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
{
return RTAS_OUT_HW_ERROR;
}
static inline int spapr_phb_vfio_eeh_configure(SpaprPhbState *sphb)
{
return RTAS_OUT_HW_ERROR;
}
static inline void spapr_phb_vfio_reset(DeviceState *qdev)
{
}
static inline void spapr_phb_nvgpu_setup(SpaprPhbState *sphb, Error **errp)
{
}
static inline void spapr_phb_nvgpu_free(SpaprPhbState *sphb)
{
}
static inline void spapr_phb_nvgpu_populate_dt(SpaprPhbState *sphb, void *fdt,
int bus_off, Error **errp)
{
}
static inline void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb,
void *fdt)
{
}
static inline void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt,
int offset,
SpaprPhbState *sphb)
{
}
#endif
void spapr_phb_dma_reset(SpaprPhbState *sphb);
static inline unsigned spapr_phb_windows_supported(SpaprPhbState *sphb)
{
return sphb->ddw_enabled ? SPAPR_PCI_DMA_MAX_WINDOWS : 1;
}
#endif /* PCI_HOST_SPAPR_H */
|
