Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * PCI MSI/MSI-X — Exported APIs for device drivers
4 : *
5 : * Copyright (C) 2003-2004 Intel
6 : * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 : * Copyright (C) 2016 Christoph Hellwig.
8 : * Copyright (C) 2022 Linutronix GmbH
9 : */
10 :
11 : #include <linux/export.h>
12 : #include <linux/irq.h>
13 :
14 : #include "msi.h"
15 :
16 : /**
17 : * pci_enable_msi() - Enable MSI interrupt mode on device
18 : * @dev: the PCI device to operate on
19 : *
20 : * Legacy device driver API to enable MSI interrupts mode on device and
21 : * allocate a single interrupt vector. On success, the allocated vector
22 : * Linux IRQ will be saved at @dev->irq. The driver must invoke
23 : * pci_disable_msi() on cleanup.
24 : *
25 : * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
26 : * pair should, in general, be used instead.
27 : *
28 : * Return: 0 on success, errno otherwise
29 : */
30 0 : int pci_enable_msi(struct pci_dev *dev)
31 : {
32 0 : int rc = __pci_enable_msi_range(dev, 1, 1, NULL);
33 0 : if (rc < 0)
34 : return rc;
35 0 : return 0;
36 : }
37 : EXPORT_SYMBOL(pci_enable_msi);
38 :
39 : /**
40 : * pci_disable_msi() - Disable MSI interrupt mode on device
41 : * @dev: the PCI device to operate on
42 : *
43 : * Legacy device driver API to disable MSI interrupt mode on device,
44 : * free earlier allocated interrupt vectors, and restore INTx emulation.
45 : * The PCI device Linux IRQ (@dev->irq) is restored to its default
46 : * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi().
47 : *
48 : * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
49 : * pair should, in general, be used instead.
50 : */
51 0 : void pci_disable_msi(struct pci_dev *dev)
52 : {
53 0 : if (!pci_msi_enabled() || !dev || !dev->msi_enabled)
54 : return;
55 :
56 0 : msi_lock_descs(&dev->dev);
57 0 : pci_msi_shutdown(dev);
58 0 : pci_free_msi_irqs(dev);
59 0 : msi_unlock_descs(&dev->dev);
60 : }
61 : EXPORT_SYMBOL(pci_disable_msi);
62 :
63 : /**
64 : * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device
65 : * @dev: the PCI device to operate on
66 : *
67 : * Return: number of MSI-X interrupt vectors available on this device
68 : * (i.e., the device's MSI-X capability structure "table size"), -EINVAL
69 : * if the device is not MSI-X capable, other errnos otherwise.
70 : */
71 0 : int pci_msix_vec_count(struct pci_dev *dev)
72 : {
73 : u16 control;
74 :
75 0 : if (!dev->msix_cap)
76 : return -EINVAL;
77 :
78 0 : pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
79 0 : return msix_table_size(control);
80 : }
81 : EXPORT_SYMBOL(pci_msix_vec_count);
82 :
83 : /**
84 : * pci_enable_msix_range() - Enable MSI-X interrupt mode on device
85 : * @dev: the PCI device to operate on
86 : * @entries: input/output parameter, array of MSI-X configuration entries
87 : * @minvec: minimum required number of MSI-X vectors
88 : * @maxvec: maximum desired number of MSI-X vectors
89 : *
90 : * Legacy device driver API to enable MSI-X interrupt mode on device and
91 : * configure its MSI-X capability structure as appropriate. The passed
92 : * @entries array must have each of its members "entry" field set to a
93 : * desired (valid) MSI-X vector number, where the range of valid MSI-X
94 : * vector numbers can be queried through pci_msix_vec_count(). If
95 : * successful, the driver must invoke pci_disable_msix() on cleanup.
96 : *
97 : * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
98 : * pair should, in general, be used instead.
99 : *
100 : * Return: number of MSI-X vectors allocated (which might be smaller
101 : * than @maxvecs), where Linux IRQ numbers for such allocated vectors
102 : * are saved back in the @entries array elements' "vector" field. Return
103 : * -ENOSPC if less than @minvecs interrupt vectors are available.
104 : * Return -EINVAL if one of the passed @entries members "entry" field
105 : * was invalid or a duplicate, or if plain MSI interrupts mode was
106 : * earlier enabled on device. Return other errnos otherwise.
107 : */
108 0 : int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
109 : int minvec, int maxvec)
110 : {
111 0 : return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0);
112 : }
113 : EXPORT_SYMBOL(pci_enable_msix_range);
114 :
115 : /**
116 : * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling
117 : * MSI-X is supported
118 : *
119 : * @dev: PCI device to operate on
120 : *
121 : * Return: True if supported, false otherwise
122 : */
123 0 : bool pci_msix_can_alloc_dyn(struct pci_dev *dev)
124 : {
125 0 : if (!dev->msix_cap)
126 : return false;
127 :
128 0 : return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY);
129 : }
130 : EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn);
131 :
132 : /**
133 : * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X
134 : * at a given MSI-X vector index or any free vector index
135 : *
136 : * @dev: PCI device to operate on
137 : * @index: Index to allocate. If @index == MSI_ANY_INDEX this allocates
138 : * the next free index in the MSI-X table
139 : * @affdesc: Optional pointer to an affinity descriptor structure. NULL otherwise
140 : *
141 : * Return: A struct msi_map
142 : *
143 : * On success msi_map::index contains the allocated index (>= 0) and
144 : * msi_map::virq contains the allocated Linux interrupt number (> 0).
145 : *
146 : * On fail msi_map::index contains the error code and msi_map::virq
147 : * is set to 0.
148 : */
149 0 : struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index,
150 : const struct irq_affinity_desc *affdesc)
151 : {
152 0 : struct msi_map map = { .index = -ENOTSUPP };
153 :
154 0 : if (!dev->msix_enabled)
155 0 : return map;
156 :
157 0 : if (!pci_msix_can_alloc_dyn(dev))
158 0 : return map;
159 :
160 0 : return msi_domain_alloc_irq_at(&dev->dev, MSI_DEFAULT_DOMAIN, index, affdesc, NULL);
161 : }
162 : EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at);
163 :
164 : /**
165 : * pci_msix_free_irq - Free an interrupt on a PCI/MSIX interrupt domain
166 : *
167 : * @dev: The PCI device to operate on
168 : * @map: A struct msi_map describing the interrupt to free
169 : *
170 : * Undo an interrupt vector allocation. Does not disable MSI-X.
171 : */
172 0 : void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map)
173 : {
174 0 : if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
175 : return;
176 0 : if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev)))
177 : return;
178 0 : msi_domain_free_irqs_range(&dev->dev, MSI_DEFAULT_DOMAIN, map.index, map.index);
179 : }
180 : EXPORT_SYMBOL_GPL(pci_msix_free_irq);
181 :
182 : /**
183 : * pci_disable_msix() - Disable MSI-X interrupt mode on device
184 : * @dev: the PCI device to operate on
185 : *
186 : * Legacy device driver API to disable MSI-X interrupt mode on device,
187 : * free earlier-allocated interrupt vectors, and restore INTx.
188 : * The PCI device Linux IRQ (@dev->irq) is restored to its default pin
189 : * assertion IRQ. This is the cleanup pair of pci_enable_msix_range().
190 : *
191 : * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
192 : * pair should, in general, be used instead.
193 : */
194 0 : void pci_disable_msix(struct pci_dev *dev)
195 : {
196 0 : if (!pci_msi_enabled() || !dev || !dev->msix_enabled)
197 : return;
198 :
199 0 : msi_lock_descs(&dev->dev);
200 0 : pci_msix_shutdown(dev);
201 0 : pci_free_msi_irqs(dev);
202 0 : msi_unlock_descs(&dev->dev);
203 : }
204 : EXPORT_SYMBOL(pci_disable_msix);
205 :
206 : /**
207 : * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors
208 : * @dev: the PCI device to operate on
209 : * @min_vecs: minimum required number of vectors (must be >= 1)
210 : * @max_vecs: maximum desired number of vectors
211 : * @flags: One or more of:
212 : *
213 : * * %PCI_IRQ_MSIX Allow trying MSI-X vector allocations
214 : * * %PCI_IRQ_MSI Allow trying MSI vector allocations
215 : *
216 : * * %PCI_IRQ_LEGACY Allow trying legacy INTx interrupts, if
217 : * and only if @min_vecs == 1
218 : *
219 : * * %PCI_IRQ_AFFINITY Auto-manage IRQs affinity by spreading
220 : * the vectors around available CPUs
221 : *
222 : * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq
223 : * vector allocation has a higher precedence over plain MSI, which has a
224 : * higher precedence over legacy INTx emulation.
225 : *
226 : * Upon a successful allocation, the caller should use pci_irq_vector()
227 : * to get the Linux IRQ number to be passed to request_threaded_irq().
228 : * The driver must call pci_free_irq_vectors() on cleanup.
229 : *
230 : * Return: number of allocated vectors (which might be smaller than
231 : * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are
232 : * available, other errnos otherwise.
233 : */
234 0 : int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,
235 : unsigned int max_vecs, unsigned int flags)
236 : {
237 0 : return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs,
238 : flags, NULL);
239 : }
240 : EXPORT_SYMBOL(pci_alloc_irq_vectors);
241 :
242 : /**
243 : * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt
244 : * vectors with affinity requirements
245 : * @dev: the PCI device to operate on
246 : * @min_vecs: minimum required number of vectors (must be >= 1)
247 : * @max_vecs: maximum desired number of vectors
248 : * @flags: allocation flags, as in pci_alloc_irq_vectors()
249 : * @affd: affinity requirements (can be %NULL).
250 : *
251 : * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter.
252 : * Check that function docs, and &struct irq_affinity, for more details.
253 : */
254 0 : int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
255 : unsigned int max_vecs, unsigned int flags,
256 : struct irq_affinity *affd)
257 : {
258 0 : struct irq_affinity msi_default_affd = {0};
259 0 : int nvecs = -ENOSPC;
260 :
261 0 : if (flags & PCI_IRQ_AFFINITY) {
262 0 : if (!affd)
263 0 : affd = &msi_default_affd;
264 : } else {
265 0 : if (WARN_ON(affd))
266 0 : affd = NULL;
267 : }
268 :
269 0 : if (flags & PCI_IRQ_MSIX) {
270 0 : nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,
271 : affd, flags);
272 0 : if (nvecs > 0)
273 : return nvecs;
274 : }
275 :
276 0 : if (flags & PCI_IRQ_MSI) {
277 0 : nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd);
278 0 : if (nvecs > 0)
279 : return nvecs;
280 : }
281 :
282 : /* use legacy IRQ if allowed */
283 0 : if (flags & PCI_IRQ_LEGACY) {
284 0 : if (min_vecs == 1 && dev->irq) {
285 : /*
286 : * Invoke the affinity spreading logic to ensure that
287 : * the device driver can adjust queue configuration
288 : * for the single interrupt case.
289 : */
290 : if (affd)
291 : irq_create_affinity_masks(1, affd);
292 0 : pci_intx(dev, 1);
293 0 : return 1;
294 : }
295 : }
296 :
297 : return nvecs;
298 : }
299 : EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
300 :
301 : /**
302 : * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector
303 : * @dev: the PCI device to operate on
304 : * @nr: device-relative interrupt vector index (0-based); has different
305 : * meanings, depending on interrupt mode:
306 : *
307 : * * MSI-X the index in the MSI-X vector table
308 : * * MSI the index of the enabled MSI vectors
309 : * * INTx must be 0
310 : *
311 : * Return: the Linux IRQ number, or -EINVAL if @nr is out of range
312 : */
313 0 : int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
314 : {
315 : unsigned int irq;
316 :
317 0 : if (!dev->msi_enabled && !dev->msix_enabled)
318 0 : return !nr ? dev->irq : -EINVAL;
319 :
320 0 : irq = msi_get_virq(&dev->dev, nr);
321 0 : return irq ? irq : -EINVAL;
322 : }
323 : EXPORT_SYMBOL(pci_irq_vector);
324 :
325 : /**
326 : * pci_irq_get_affinity() - Get a device interrupt vector affinity
327 : * @dev: the PCI device to operate on
328 : * @nr: device-relative interrupt vector index (0-based); has different
329 : * meanings, depending on interrupt mode:
330 : *
331 : * * MSI-X the index in the MSI-X vector table
332 : * * MSI the index of the enabled MSI vectors
333 : * * INTx must be 0
334 : *
335 : * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if
336 : * the MSI(-X) vector was allocated without explicit affinity
337 : * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or
338 : * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a
339 : * generic set of CPU IDs representing all possible CPUs available
340 : * during system boot if the device is in legacy INTx mode.
341 : */
342 0 : const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
343 : {
344 0 : int idx, irq = pci_irq_vector(dev, nr);
345 : struct msi_desc *desc;
346 :
347 0 : if (WARN_ON_ONCE(irq <= 0))
348 : return NULL;
349 :
350 0 : desc = irq_get_msi_desc(irq);
351 : /* Non-MSI does not have the information handy */
352 0 : if (!desc)
353 : return cpu_possible_mask;
354 :
355 : /* MSI[X] interrupts can be allocated without affinity descriptor */
356 0 : if (!desc->affinity)
357 : return NULL;
358 :
359 : /*
360 : * MSI has a mask array in the descriptor.
361 : * MSI-X has a single mask.
362 : */
363 0 : idx = dev->msi_enabled ? nr : 0;
364 0 : return &desc->affinity[idx].mask;
365 : }
366 : EXPORT_SYMBOL(pci_irq_get_affinity);
367 :
368 : /**
369 : * pci_ims_alloc_irq - Allocate an interrupt on a PCI/IMS interrupt domain
370 : * @dev: The PCI device to operate on
371 : * @icookie: Pointer to an IMS implementation specific cookie for this
372 : * IMS instance (PASID, queue ID, pointer...).
373 : * The cookie content is copied into the MSI descriptor for the
374 : * interrupt chip callbacks or domain specific setup functions.
375 : * @affdesc: Optional pointer to an interrupt affinity descriptor
376 : *
377 : * There is no index for IMS allocations as IMS is an implementation
378 : * specific storage and does not have any direct associations between
379 : * index, which might be a pure software construct, and device
380 : * functionality. This association is established by the driver either via
381 : * the index - if there is a hardware table - or in case of purely software
382 : * managed IMS implementation the association happens via the
383 : * irq_write_msi_msg() callback of the implementation specific interrupt
384 : * chip, which utilizes the provided @icookie to store the MSI message in
385 : * the appropriate place.
386 : *
387 : * Return: A struct msi_map
388 : *
389 : * On success msi_map::index contains the allocated index (>= 0) and
390 : * msi_map::virq the allocated Linux interrupt number (> 0).
391 : *
392 : * On fail msi_map::index contains the error code and msi_map::virq
393 : * is set to 0.
394 : */
395 0 : struct msi_map pci_ims_alloc_irq(struct pci_dev *dev, union msi_instance_cookie *icookie,
396 : const struct irq_affinity_desc *affdesc)
397 : {
398 0 : return msi_domain_alloc_irq_at(&dev->dev, MSI_SECONDARY_DOMAIN, MSI_ANY_INDEX,
399 : affdesc, icookie);
400 : }
401 : EXPORT_SYMBOL_GPL(pci_ims_alloc_irq);
402 :
403 : /**
404 : * pci_ims_free_irq - Allocate an interrupt on a PCI/IMS interrupt domain
405 : * which was allocated via pci_ims_alloc_irq()
406 : * @dev: The PCI device to operate on
407 : * @map: A struct msi_map describing the interrupt to free as
408 : * returned from pci_ims_alloc_irq()
409 : */
410 0 : void pci_ims_free_irq(struct pci_dev *dev, struct msi_map map)
411 : {
412 0 : if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
413 : return;
414 0 : msi_domain_free_irqs_range(&dev->dev, MSI_SECONDARY_DOMAIN, map.index, map.index);
415 : }
416 : EXPORT_SYMBOL_GPL(pci_ims_free_irq);
417 :
418 : /**
419 : * pci_free_irq_vectors() - Free previously allocated IRQs for a device
420 : * @dev: the PCI device to operate on
421 : *
422 : * Undo the interrupt vector allocations and possible device MSI/MSI-X
423 : * enablement earlier done through pci_alloc_irq_vectors_affinity() or
424 : * pci_alloc_irq_vectors().
425 : */
426 0 : void pci_free_irq_vectors(struct pci_dev *dev)
427 : {
428 0 : pci_disable_msix(dev);
429 0 : pci_disable_msi(dev);
430 0 : }
431 : EXPORT_SYMBOL(pci_free_irq_vectors);
432 :
433 : /**
434 : * pci_restore_msi_state() - Restore cached MSI(-X) state on device
435 : * @dev: the PCI device to operate on
436 : *
437 : * Write the Linux-cached MSI(-X) state back on device. This is
438 : * typically useful upon system resume, or after an error-recovery PCI
439 : * adapter reset.
440 : */
441 0 : void pci_restore_msi_state(struct pci_dev *dev)
442 : {
443 0 : __pci_restore_msi_state(dev);
444 0 : __pci_restore_msix_state(dev);
445 0 : }
446 : EXPORT_SYMBOL_GPL(pci_restore_msi_state);
447 :
448 : /**
449 : * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide?
450 : *
451 : * Return: true if MSI has not been globally disabled through ACPI FADT,
452 : * PCI bridge quirks, or the "pci=nomsi" kernel command-line option.
453 : */
454 0 : int pci_msi_enabled(void)
455 : {
456 0 : return pci_msi_enable;
457 : }
458 : EXPORT_SYMBOL(pci_msi_enabled);
|
