Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * PCI Message Signaled Interrupt (MSI) - irqdomain support
4 : */
5 : #include <linux/acpi_iort.h>
6 : #include <linux/irqdomain.h>
7 : #include <linux/of_irq.h>
8 :
9 : #include "msi.h"
10 :
11 0 : int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
12 : {
13 : struct irq_domain *domain;
14 :
15 0 : domain = dev_get_msi_domain(&dev->dev);
16 0 : if (domain && irq_domain_is_hierarchy(domain))
17 0 : return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec);
18 :
19 0 : return pci_msi_legacy_setup_msi_irqs(dev, nvec, type);
20 : }
21 :
22 0 : void pci_msi_teardown_msi_irqs(struct pci_dev *dev)
23 : {
24 : struct irq_domain *domain;
25 :
26 0 : domain = dev_get_msi_domain(&dev->dev);
27 0 : if (domain && irq_domain_is_hierarchy(domain)) {
28 0 : msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN);
29 : } else {
30 0 : pci_msi_legacy_teardown_msi_irqs(dev);
31 0 : msi_free_msi_descs(&dev->dev);
32 : }
33 0 : }
34 :
35 : /**
36 : * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space
37 : * @irq_data: Pointer to interrupt data of the MSI interrupt
38 : * @msg: Pointer to the message
39 : */
40 0 : static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg)
41 : {
42 0 : struct msi_desc *desc = irq_data_get_msi_desc(irq_data);
43 :
44 : /*
45 : * For MSI-X desc->irq is always equal to irq_data->irq. For
46 : * MSI only the first interrupt of MULTI MSI passes the test.
47 : */
48 0 : if (desc->irq == irq_data->irq)
49 0 : __pci_write_msi_msg(desc, msg);
50 0 : }
51 :
52 : /**
53 : * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
54 : * @desc: Pointer to the MSI descriptor
55 : *
56 : * The ID number is only used within the irqdomain.
57 : */
58 0 : static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc)
59 : {
60 0 : struct pci_dev *dev = msi_desc_to_pci_dev(desc);
61 :
62 0 : return (irq_hw_number_t)desc->msi_index |
63 0 : pci_dev_id(dev) << 11 |
64 0 : (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27;
65 : }
66 :
67 0 : static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,
68 : struct msi_desc *desc)
69 : {
70 0 : arg->desc = desc;
71 0 : arg->hwirq = pci_msi_domain_calc_hwirq(desc);
72 0 : }
73 :
74 : static struct msi_domain_ops pci_msi_domain_ops_default = {
75 : .set_desc = pci_msi_domain_set_desc,
76 : };
77 :
78 : static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info)
79 : {
80 0 : struct msi_domain_ops *ops = info->ops;
81 :
82 0 : if (ops == NULL) {
83 0 : info->ops = &pci_msi_domain_ops_default;
84 : } else {
85 0 : if (ops->set_desc == NULL)
86 0 : ops->set_desc = pci_msi_domain_set_desc;
87 : }
88 : }
89 :
90 0 : static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info)
91 : {
92 0 : struct irq_chip *chip = info->chip;
93 :
94 0 : BUG_ON(!chip);
95 0 : if (!chip->irq_write_msi_msg)
96 0 : chip->irq_write_msi_msg = pci_msi_domain_write_msg;
97 0 : if (!chip->irq_mask)
98 0 : chip->irq_mask = pci_msi_mask_irq;
99 0 : if (!chip->irq_unmask)
100 0 : chip->irq_unmask = pci_msi_unmask_irq;
101 0 : }
102 :
103 : /**
104 : * pci_msi_create_irq_domain - Create a MSI interrupt domain
105 : * @fwnode: Optional fwnode of the interrupt controller
106 : * @info: MSI domain info
107 : * @parent: Parent irq domain
108 : *
109 : * Updates the domain and chip ops and creates a MSI interrupt domain.
110 : *
111 : * Returns:
112 : * A domain pointer or NULL in case of failure.
113 : */
114 0 : struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
115 : struct msi_domain_info *info,
116 : struct irq_domain *parent)
117 : {
118 0 : if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE))
119 0 : info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
120 :
121 0 : if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
122 0 : pci_msi_domain_update_dom_ops(info);
123 0 : if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
124 0 : pci_msi_domain_update_chip_ops(info);
125 :
126 : /* Let the core code free MSI descriptors when freeing interrupts */
127 0 : info->flags |= MSI_FLAG_FREE_MSI_DESCS;
128 :
129 0 : info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS;
130 : if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))
131 : info->flags |= MSI_FLAG_MUST_REACTIVATE;
132 :
133 : /* PCI-MSI is oneshot-safe */
134 0 : info->chip->flags |= IRQCHIP_ONESHOT_SAFE;
135 : /* Let the core update the bus token */
136 0 : info->bus_token = DOMAIN_BUS_PCI_MSI;
137 :
138 0 : return msi_create_irq_domain(fwnode, info, parent);
139 : }
140 : EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain);
141 :
142 : /*
143 : * Per device MSI[-X] domain functionality
144 : */
145 0 : static void pci_device_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
146 : {
147 0 : arg->desc = desc;
148 0 : arg->hwirq = desc->msi_index;
149 0 : }
150 :
151 0 : static void pci_irq_mask_msi(struct irq_data *data)
152 : {
153 0 : struct msi_desc *desc = irq_data_get_msi_desc(data);
154 :
155 0 : pci_msi_mask(desc, BIT(data->irq - desc->irq));
156 0 : }
157 :
158 0 : static void pci_irq_unmask_msi(struct irq_data *data)
159 : {
160 0 : struct msi_desc *desc = irq_data_get_msi_desc(data);
161 :
162 0 : pci_msi_unmask(desc, BIT(data->irq - desc->irq));
163 0 : }
164 :
165 : #ifdef CONFIG_GENERIC_IRQ_RESERVATION_MODE
166 : # define MSI_REACTIVATE MSI_FLAG_MUST_REACTIVATE
167 : #else
168 : # define MSI_REACTIVATE 0
169 : #endif
170 :
171 : #define MSI_COMMON_FLAGS (MSI_FLAG_FREE_MSI_DESCS | \
172 : MSI_FLAG_ACTIVATE_EARLY | \
173 : MSI_FLAG_DEV_SYSFS | \
174 : MSI_REACTIVATE)
175 :
176 : static const struct msi_domain_template pci_msi_template = {
177 : .chip = {
178 : .name = "PCI-MSI",
179 : .irq_mask = pci_irq_mask_msi,
180 : .irq_unmask = pci_irq_unmask_msi,
181 : .irq_write_msi_msg = pci_msi_domain_write_msg,
182 : .flags = IRQCHIP_ONESHOT_SAFE,
183 : },
184 :
185 : .ops = {
186 : .set_desc = pci_device_domain_set_desc,
187 : },
188 :
189 : .info = {
190 : .flags = MSI_COMMON_FLAGS | MSI_FLAG_MULTI_PCI_MSI,
191 : .bus_token = DOMAIN_BUS_PCI_DEVICE_MSI,
192 : },
193 : };
194 :
195 0 : static void pci_irq_mask_msix(struct irq_data *data)
196 : {
197 0 : pci_msix_mask(irq_data_get_msi_desc(data));
198 0 : }
199 :
200 0 : static void pci_irq_unmask_msix(struct irq_data *data)
201 : {
202 0 : pci_msix_unmask(irq_data_get_msi_desc(data));
203 0 : }
204 :
205 0 : static void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *arg,
206 : struct msi_desc *desc)
207 : {
208 : /* Don't fiddle with preallocated MSI descriptors */
209 0 : if (!desc->pci.mask_base)
210 0 : msix_prepare_msi_desc(to_pci_dev(desc->dev), desc);
211 0 : }
212 :
213 : static const struct msi_domain_template pci_msix_template = {
214 : .chip = {
215 : .name = "PCI-MSIX",
216 : .irq_mask = pci_irq_mask_msix,
217 : .irq_unmask = pci_irq_unmask_msix,
218 : .irq_write_msi_msg = pci_msi_domain_write_msg,
219 : .flags = IRQCHIP_ONESHOT_SAFE,
220 : },
221 :
222 : .ops = {
223 : .prepare_desc = pci_msix_prepare_desc,
224 : .set_desc = pci_device_domain_set_desc,
225 : },
226 :
227 : .info = {
228 : .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX |
229 : MSI_FLAG_PCI_MSIX_ALLOC_DYN,
230 : .bus_token = DOMAIN_BUS_PCI_DEVICE_MSIX,
231 : },
232 : };
233 :
234 : static bool pci_match_device_domain(struct pci_dev *pdev, enum irq_domain_bus_token bus_token)
235 : {
236 0 : return msi_match_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, bus_token);
237 : }
238 :
239 0 : static bool pci_create_device_domain(struct pci_dev *pdev, const struct msi_domain_template *tmpl,
240 : unsigned int hwsize)
241 : {
242 0 : struct irq_domain *domain = dev_get_msi_domain(&pdev->dev);
243 :
244 0 : if (!domain || !irq_domain_is_msi_parent(domain))
245 : return true;
246 :
247 0 : return msi_create_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, tmpl,
248 : hwsize, NULL, NULL);
249 : }
250 :
251 : /**
252 : * pci_setup_msi_device_domain - Setup a device MSI interrupt domain
253 : * @pdev: The PCI device to create the domain on
254 : *
255 : * Return:
256 : * True when:
257 : * - The device does not have a MSI parent irq domain associated,
258 : * which keeps the legacy architecture specific and the global
259 : * PCI/MSI domain models working
260 : * - The MSI domain exists already
261 : * - The MSI domain was successfully allocated
262 : * False when:
263 : * - MSI-X is enabled
264 : * - The domain creation fails.
265 : *
266 : * The created MSI domain is preserved until:
267 : * - The device is removed
268 : * - MSI is disabled and a MSI-X domain is created
269 : */
270 0 : bool pci_setup_msi_device_domain(struct pci_dev *pdev)
271 : {
272 0 : if (WARN_ON_ONCE(pdev->msix_enabled))
273 : return false;
274 :
275 0 : if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI))
276 : return true;
277 0 : if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX))
278 0 : msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN);
279 :
280 0 : return pci_create_device_domain(pdev, &pci_msi_template, 1);
281 : }
282 :
283 : /**
284 : * pci_setup_msix_device_domain - Setup a device MSI-X interrupt domain
285 : * @pdev: The PCI device to create the domain on
286 : * @hwsize: The size of the MSI-X vector table
287 : *
288 : * Return:
289 : * True when:
290 : * - The device does not have a MSI parent irq domain associated,
291 : * which keeps the legacy architecture specific and the global
292 : * PCI/MSI domain models working
293 : * - The MSI-X domain exists already
294 : * - The MSI-X domain was successfully allocated
295 : * False when:
296 : * - MSI is enabled
297 : * - The domain creation fails.
298 : *
299 : * The created MSI-X domain is preserved until:
300 : * - The device is removed
301 : * - MSI-X is disabled and a MSI domain is created
302 : */
303 0 : bool pci_setup_msix_device_domain(struct pci_dev *pdev, unsigned int hwsize)
304 : {
305 0 : if (WARN_ON_ONCE(pdev->msi_enabled))
306 : return false;
307 :
308 0 : if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX))
309 : return true;
310 0 : if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI))
311 0 : msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN);
312 :
313 0 : return pci_create_device_domain(pdev, &pci_msix_template, hwsize);
314 : }
315 :
316 : /**
317 : * pci_msi_domain_supports - Check for support of a particular feature flag
318 : * @pdev: The PCI device to operate on
319 : * @feature_mask: The feature mask to check for (full match)
320 : * @mode: If ALLOW_LEGACY this grants the feature when there is no irq domain
321 : * associated to the device. If DENY_LEGACY the lack of an irq domain
322 : * makes the feature unsupported
323 : */
324 0 : bool pci_msi_domain_supports(struct pci_dev *pdev, unsigned int feature_mask,
325 : enum support_mode mode)
326 : {
327 : struct msi_domain_info *info;
328 : struct irq_domain *domain;
329 : unsigned int supported;
330 :
331 0 : domain = dev_get_msi_domain(&pdev->dev);
332 :
333 0 : if (!domain || !irq_domain_is_hierarchy(domain))
334 0 : return mode == ALLOW_LEGACY;
335 :
336 0 : if (!irq_domain_is_msi_parent(domain)) {
337 : /*
338 : * For "global" PCI/MSI interrupt domains the associated
339 : * msi_domain_info::flags is the authoritive source of
340 : * information.
341 : */
342 0 : info = domain->host_data;
343 0 : supported = info->flags;
344 : } else {
345 : /*
346 : * For MSI parent domains the supported feature set
347 : * is avaliable in the parent ops. This makes checks
348 : * possible before actually instantiating the
349 : * per device domain because the parent is never
350 : * expanding the PCI/MSI functionality.
351 : */
352 0 : supported = domain->msi_parent_ops->supported_flags;
353 : }
354 :
355 0 : return (supported & feature_mask) == feature_mask;
356 : }
357 :
358 : /**
359 : * pci_create_ims_domain - Create a secondary IMS domain for a PCI device
360 : * @pdev: The PCI device to operate on
361 : * @template: The MSI info template which describes the domain
362 : * @hwsize: The size of the hardware entry table or 0 if the domain
363 : * is purely software managed
364 : * @data: Optional pointer to domain specific data to be stored
365 : * in msi_domain_info::data
366 : *
367 : * Return: True on success, false otherwise
368 : *
369 : * An IMS domain is expected to have the following constraints:
370 : * - The index space is managed by the core code
371 : *
372 : * - There is no requirement for consecutive index ranges
373 : *
374 : * - The interrupt chip must provide the following callbacks:
375 : * - irq_mask()
376 : * - irq_unmask()
377 : * - irq_write_msi_msg()
378 : *
379 : * - The interrupt chip must provide the following optional callbacks
380 : * when the irq_mask(), irq_unmask() and irq_write_msi_msg() callbacks
381 : * cannot operate directly on hardware, e.g. in the case that the
382 : * interrupt message store is in queue memory:
383 : * - irq_bus_lock()
384 : * - irq_bus_unlock()
385 : *
386 : * These callbacks are invoked from preemptible task context and are
387 : * allowed to sleep. In this case the mandatory callbacks above just
388 : * store the information. The irq_bus_unlock() callback is supposed
389 : * to make the change effective before returning.
390 : *
391 : * - Interrupt affinity setting is handled by the underlying parent
392 : * interrupt domain and communicated to the IMS domain via
393 : * irq_write_msi_msg().
394 : *
395 : * The domain is automatically destroyed when the PCI device is removed.
396 : */
397 0 : bool pci_create_ims_domain(struct pci_dev *pdev, const struct msi_domain_template *template,
398 : unsigned int hwsize, void *data)
399 : {
400 0 : struct irq_domain *domain = dev_get_msi_domain(&pdev->dev);
401 :
402 0 : if (!domain || !irq_domain_is_msi_parent(domain))
403 : return false;
404 :
405 0 : if (template->info.bus_token != DOMAIN_BUS_PCI_DEVICE_IMS ||
406 0 : !(template->info.flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS) ||
407 0 : !(template->info.flags & MSI_FLAG_FREE_MSI_DESCS) ||
408 0 : !template->chip.irq_mask || !template->chip.irq_unmask ||
409 0 : !template->chip.irq_write_msi_msg || template->chip.irq_set_affinity)
410 : return false;
411 :
412 0 : return msi_create_device_irq_domain(&pdev->dev, MSI_SECONDARY_DOMAIN, template,
413 : hwsize, data, NULL);
414 : }
415 : EXPORT_SYMBOL_GPL(pci_create_ims_domain);
416 :
417 : /*
418 : * Users of the generic MSI infrastructure expect a device to have a single ID,
419 : * so with DMA aliases we have to pick the least-worst compromise. Devices with
420 : * DMA phantom functions tend to still emit MSIs from the real function number,
421 : * so we ignore those and only consider topological aliases where either the
422 : * alias device or RID appears on a different bus number. We also make the
423 : * reasonable assumption that bridges are walked in an upstream direction (so
424 : * the last one seen wins), and the much braver assumption that the most likely
425 : * case is that of PCI->PCIe so we should always use the alias RID. This echoes
426 : * the logic from intel_irq_remapping's set_msi_sid(), which presumably works
427 : * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions
428 : * for taking ownership all we can really do is close our eyes and hope...
429 : */
430 0 : static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data)
431 : {
432 0 : u32 *pa = data;
433 0 : u8 bus = PCI_BUS_NUM(*pa);
434 :
435 0 : if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus)
436 0 : *pa = alias;
437 :
438 0 : return 0;
439 : }
440 :
441 : /**
442 : * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID)
443 : * @domain: The interrupt domain
444 : * @pdev: The PCI device.
445 : *
446 : * The RID for a device is formed from the alias, with a firmware
447 : * supplied mapping applied
448 : *
449 : * Returns: The RID.
450 : */
451 0 : u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
452 : {
453 : struct device_node *of_node;
454 0 : u32 rid = pci_dev_id(pdev);
455 :
456 0 : pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
457 :
458 0 : of_node = irq_domain_get_of_node(domain);
459 0 : rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) :
460 0 : iort_msi_map_id(&pdev->dev, rid);
461 :
462 0 : return rid;
463 : }
464 :
465 : /**
466 : * pci_msi_get_device_domain - Get the MSI domain for a given PCI device
467 : * @pdev: The PCI device
468 : *
469 : * Use the firmware data to find a device-specific MSI domain
470 : * (i.e. not one that is set as a default).
471 : *
472 : * Returns: The corresponding MSI domain or NULL if none has been found.
473 : */
474 0 : struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
475 : {
476 : struct irq_domain *dom;
477 0 : u32 rid = pci_dev_id(pdev);
478 :
479 0 : pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
480 0 : dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI);
481 : if (!dom)
482 0 : dom = iort_get_device_domain(&pdev->dev, rid,
483 : DOMAIN_BUS_PCI_MSI);
484 0 : return dom;
485 : }
|
