Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * io_uring opcode handling table
4 : */
5 : #include <linux/kernel.h>
6 : #include <linux/errno.h>
7 : #include <linux/fs.h>
8 : #include <linux/file.h>
9 : #include <linux/io_uring.h>
10 :
11 : #include "io_uring.h"
12 : #include "opdef.h"
13 : #include "refs.h"
14 : #include "tctx.h"
15 : #include "sqpoll.h"
16 : #include "fdinfo.h"
17 : #include "kbuf.h"
18 : #include "rsrc.h"
19 :
20 : #include "xattr.h"
21 : #include "nop.h"
22 : #include "fs.h"
23 : #include "splice.h"
24 : #include "sync.h"
25 : #include "advise.h"
26 : #include "openclose.h"
27 : #include "uring_cmd.h"
28 : #include "epoll.h"
29 : #include "statx.h"
30 : #include "net.h"
31 : #include "msg_ring.h"
32 : #include "timeout.h"
33 : #include "poll.h"
34 : #include "cancel.h"
35 : #include "rw.h"
36 :
37 0 : static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38 : {
39 0 : WARN_ON_ONCE(1);
40 0 : return -ECANCELED;
41 : }
42 :
43 0 : static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
44 : const struct io_uring_sqe *sqe)
45 : {
46 0 : return -EOPNOTSUPP;
47 : }
48 :
49 : const struct io_issue_def io_issue_defs[] = {
50 : [IORING_OP_NOP] = {
51 : .audit_skip = 1,
52 : .iopoll = 1,
53 : .prep = io_nop_prep,
54 : .issue = io_nop,
55 : },
56 : [IORING_OP_READV] = {
57 : .needs_file = 1,
58 : .unbound_nonreg_file = 1,
59 : .pollin = 1,
60 : .buffer_select = 1,
61 : .plug = 1,
62 : .audit_skip = 1,
63 : .ioprio = 1,
64 : .iopoll = 1,
65 : .iopoll_queue = 1,
66 : .prep = io_prep_rw,
67 : .issue = io_read,
68 : },
69 : [IORING_OP_WRITEV] = {
70 : .needs_file = 1,
71 : .hash_reg_file = 1,
72 : .unbound_nonreg_file = 1,
73 : .pollout = 1,
74 : .plug = 1,
75 : .audit_skip = 1,
76 : .ioprio = 1,
77 : .iopoll = 1,
78 : .iopoll_queue = 1,
79 : .prep = io_prep_rw,
80 : .issue = io_write,
81 : },
82 : [IORING_OP_FSYNC] = {
83 : .needs_file = 1,
84 : .audit_skip = 1,
85 : .prep = io_fsync_prep,
86 : .issue = io_fsync,
87 : },
88 : [IORING_OP_READ_FIXED] = {
89 : .needs_file = 1,
90 : .unbound_nonreg_file = 1,
91 : .pollin = 1,
92 : .plug = 1,
93 : .audit_skip = 1,
94 : .ioprio = 1,
95 : .iopoll = 1,
96 : .iopoll_queue = 1,
97 : .prep = io_prep_rw,
98 : .issue = io_read,
99 : },
100 : [IORING_OP_WRITE_FIXED] = {
101 : .needs_file = 1,
102 : .hash_reg_file = 1,
103 : .unbound_nonreg_file = 1,
104 : .pollout = 1,
105 : .plug = 1,
106 : .audit_skip = 1,
107 : .ioprio = 1,
108 : .iopoll = 1,
109 : .iopoll_queue = 1,
110 : .prep = io_prep_rw,
111 : .issue = io_write,
112 : },
113 : [IORING_OP_POLL_ADD] = {
114 : .needs_file = 1,
115 : .unbound_nonreg_file = 1,
116 : .audit_skip = 1,
117 : .prep = io_poll_add_prep,
118 : .issue = io_poll_add,
119 : },
120 : [IORING_OP_POLL_REMOVE] = {
121 : .audit_skip = 1,
122 : .prep = io_poll_remove_prep,
123 : .issue = io_poll_remove,
124 : },
125 : [IORING_OP_SYNC_FILE_RANGE] = {
126 : .needs_file = 1,
127 : .audit_skip = 1,
128 : .prep = io_sfr_prep,
129 : .issue = io_sync_file_range,
130 : },
131 : [IORING_OP_SENDMSG] = {
132 : .needs_file = 1,
133 : .unbound_nonreg_file = 1,
134 : .pollout = 1,
135 : .ioprio = 1,
136 : .manual_alloc = 1,
137 : #if defined(CONFIG_NET)
138 : .prep = io_sendmsg_prep,
139 : .issue = io_sendmsg,
140 : #else
141 : .prep = io_eopnotsupp_prep,
142 : #endif
143 : },
144 : [IORING_OP_RECVMSG] = {
145 : .needs_file = 1,
146 : .unbound_nonreg_file = 1,
147 : .pollin = 1,
148 : .buffer_select = 1,
149 : .ioprio = 1,
150 : .manual_alloc = 1,
151 : #if defined(CONFIG_NET)
152 : .prep = io_recvmsg_prep,
153 : .issue = io_recvmsg,
154 : #else
155 : .prep = io_eopnotsupp_prep,
156 : #endif
157 : },
158 : [IORING_OP_TIMEOUT] = {
159 : .audit_skip = 1,
160 : .prep = io_timeout_prep,
161 : .issue = io_timeout,
162 : },
163 : [IORING_OP_TIMEOUT_REMOVE] = {
164 : /* used by timeout updates' prep() */
165 : .audit_skip = 1,
166 : .prep = io_timeout_remove_prep,
167 : .issue = io_timeout_remove,
168 : },
169 : [IORING_OP_ACCEPT] = {
170 : .needs_file = 1,
171 : .unbound_nonreg_file = 1,
172 : .pollin = 1,
173 : .poll_exclusive = 1,
174 : .ioprio = 1, /* used for flags */
175 : #if defined(CONFIG_NET)
176 : .prep = io_accept_prep,
177 : .issue = io_accept,
178 : #else
179 : .prep = io_eopnotsupp_prep,
180 : #endif
181 : },
182 : [IORING_OP_ASYNC_CANCEL] = {
183 : .audit_skip = 1,
184 : .prep = io_async_cancel_prep,
185 : .issue = io_async_cancel,
186 : },
187 : [IORING_OP_LINK_TIMEOUT] = {
188 : .audit_skip = 1,
189 : .prep = io_link_timeout_prep,
190 : .issue = io_no_issue,
191 : },
192 : [IORING_OP_CONNECT] = {
193 : .needs_file = 1,
194 : .unbound_nonreg_file = 1,
195 : .pollout = 1,
196 : #if defined(CONFIG_NET)
197 : .prep = io_connect_prep,
198 : .issue = io_connect,
199 : #else
200 : .prep = io_eopnotsupp_prep,
201 : #endif
202 : },
203 : [IORING_OP_FALLOCATE] = {
204 : .needs_file = 1,
205 : .prep = io_fallocate_prep,
206 : .issue = io_fallocate,
207 : },
208 : [IORING_OP_OPENAT] = {
209 : .prep = io_openat_prep,
210 : .issue = io_openat,
211 : },
212 : [IORING_OP_CLOSE] = {
213 : .prep = io_close_prep,
214 : .issue = io_close,
215 : },
216 : [IORING_OP_FILES_UPDATE] = {
217 : .audit_skip = 1,
218 : .iopoll = 1,
219 : .prep = io_files_update_prep,
220 : .issue = io_files_update,
221 : },
222 : [IORING_OP_STATX] = {
223 : .audit_skip = 1,
224 : .prep = io_statx_prep,
225 : .issue = io_statx,
226 : },
227 : [IORING_OP_READ] = {
228 : .needs_file = 1,
229 : .unbound_nonreg_file = 1,
230 : .pollin = 1,
231 : .buffer_select = 1,
232 : .plug = 1,
233 : .audit_skip = 1,
234 : .ioprio = 1,
235 : .iopoll = 1,
236 : .iopoll_queue = 1,
237 : .prep = io_prep_rw,
238 : .issue = io_read,
239 : },
240 : [IORING_OP_WRITE] = {
241 : .needs_file = 1,
242 : .hash_reg_file = 1,
243 : .unbound_nonreg_file = 1,
244 : .pollout = 1,
245 : .plug = 1,
246 : .audit_skip = 1,
247 : .ioprio = 1,
248 : .iopoll = 1,
249 : .iopoll_queue = 1,
250 : .prep = io_prep_rw,
251 : .issue = io_write,
252 : },
253 : [IORING_OP_FADVISE] = {
254 : .needs_file = 1,
255 : .audit_skip = 1,
256 : .prep = io_fadvise_prep,
257 : .issue = io_fadvise,
258 : },
259 : [IORING_OP_MADVISE] = {
260 : .audit_skip = 1,
261 : .prep = io_madvise_prep,
262 : .issue = io_madvise,
263 : },
264 : [IORING_OP_SEND] = {
265 : .needs_file = 1,
266 : .unbound_nonreg_file = 1,
267 : .pollout = 1,
268 : .audit_skip = 1,
269 : .ioprio = 1,
270 : .manual_alloc = 1,
271 : #if defined(CONFIG_NET)
272 : .prep = io_sendmsg_prep,
273 : .issue = io_send,
274 : #else
275 : .prep = io_eopnotsupp_prep,
276 : #endif
277 : },
278 : [IORING_OP_RECV] = {
279 : .needs_file = 1,
280 : .unbound_nonreg_file = 1,
281 : .pollin = 1,
282 : .buffer_select = 1,
283 : .audit_skip = 1,
284 : .ioprio = 1,
285 : #if defined(CONFIG_NET)
286 : .prep = io_recvmsg_prep,
287 : .issue = io_recv,
288 : #else
289 : .prep = io_eopnotsupp_prep,
290 : #endif
291 : },
292 : [IORING_OP_OPENAT2] = {
293 : .prep = io_openat2_prep,
294 : .issue = io_openat2,
295 : },
296 : [IORING_OP_EPOLL_CTL] = {
297 : .unbound_nonreg_file = 1,
298 : .audit_skip = 1,
299 : #if defined(CONFIG_EPOLL)
300 : .prep = io_epoll_ctl_prep,
301 : .issue = io_epoll_ctl,
302 : #else
303 : .prep = io_eopnotsupp_prep,
304 : #endif
305 : },
306 : [IORING_OP_SPLICE] = {
307 : .needs_file = 1,
308 : .hash_reg_file = 1,
309 : .unbound_nonreg_file = 1,
310 : .audit_skip = 1,
311 : .prep = io_splice_prep,
312 : .issue = io_splice,
313 : },
314 : [IORING_OP_PROVIDE_BUFFERS] = {
315 : .audit_skip = 1,
316 : .iopoll = 1,
317 : .prep = io_provide_buffers_prep,
318 : .issue = io_provide_buffers,
319 : },
320 : [IORING_OP_REMOVE_BUFFERS] = {
321 : .audit_skip = 1,
322 : .iopoll = 1,
323 : .prep = io_remove_buffers_prep,
324 : .issue = io_remove_buffers,
325 : },
326 : [IORING_OP_TEE] = {
327 : .needs_file = 1,
328 : .hash_reg_file = 1,
329 : .unbound_nonreg_file = 1,
330 : .audit_skip = 1,
331 : .prep = io_tee_prep,
332 : .issue = io_tee,
333 : },
334 : [IORING_OP_SHUTDOWN] = {
335 : .needs_file = 1,
336 : #if defined(CONFIG_NET)
337 : .prep = io_shutdown_prep,
338 : .issue = io_shutdown,
339 : #else
340 : .prep = io_eopnotsupp_prep,
341 : #endif
342 : },
343 : [IORING_OP_RENAMEAT] = {
344 : .prep = io_renameat_prep,
345 : .issue = io_renameat,
346 : },
347 : [IORING_OP_UNLINKAT] = {
348 : .prep = io_unlinkat_prep,
349 : .issue = io_unlinkat,
350 : },
351 : [IORING_OP_MKDIRAT] = {
352 : .prep = io_mkdirat_prep,
353 : .issue = io_mkdirat,
354 : },
355 : [IORING_OP_SYMLINKAT] = {
356 : .prep = io_symlinkat_prep,
357 : .issue = io_symlinkat,
358 : },
359 : [IORING_OP_LINKAT] = {
360 : .prep = io_linkat_prep,
361 : .issue = io_linkat,
362 : },
363 : [IORING_OP_MSG_RING] = {
364 : .needs_file = 1,
365 : .iopoll = 1,
366 : .prep = io_msg_ring_prep,
367 : .issue = io_msg_ring,
368 : },
369 : [IORING_OP_FSETXATTR] = {
370 : .needs_file = 1,
371 : .prep = io_fsetxattr_prep,
372 : .issue = io_fsetxattr,
373 : },
374 : [IORING_OP_SETXATTR] = {
375 : .prep = io_setxattr_prep,
376 : .issue = io_setxattr,
377 : },
378 : [IORING_OP_FGETXATTR] = {
379 : .needs_file = 1,
380 : .prep = io_fgetxattr_prep,
381 : .issue = io_fgetxattr,
382 : },
383 : [IORING_OP_GETXATTR] = {
384 : .prep = io_getxattr_prep,
385 : .issue = io_getxattr,
386 : },
387 : [IORING_OP_SOCKET] = {
388 : .audit_skip = 1,
389 : #if defined(CONFIG_NET)
390 : .prep = io_socket_prep,
391 : .issue = io_socket,
392 : #else
393 : .prep = io_eopnotsupp_prep,
394 : #endif
395 : },
396 : [IORING_OP_URING_CMD] = {
397 : .needs_file = 1,
398 : .plug = 1,
399 : .iopoll = 1,
400 : .iopoll_queue = 1,
401 : .prep = io_uring_cmd_prep,
402 : .issue = io_uring_cmd,
403 : },
404 : [IORING_OP_SEND_ZC] = {
405 : .needs_file = 1,
406 : .unbound_nonreg_file = 1,
407 : .pollout = 1,
408 : .audit_skip = 1,
409 : .ioprio = 1,
410 : .manual_alloc = 1,
411 : #if defined(CONFIG_NET)
412 : .prep = io_send_zc_prep,
413 : .issue = io_send_zc,
414 : #else
415 : .prep = io_eopnotsupp_prep,
416 : #endif
417 : },
418 : [IORING_OP_SENDMSG_ZC] = {
419 : .needs_file = 1,
420 : .unbound_nonreg_file = 1,
421 : .pollout = 1,
422 : .ioprio = 1,
423 : .manual_alloc = 1,
424 : #if defined(CONFIG_NET)
425 : .prep = io_send_zc_prep,
426 : .issue = io_sendmsg_zc,
427 : #else
428 : .prep = io_eopnotsupp_prep,
429 : #endif
430 : },
431 : };
432 :
433 :
434 : const struct io_cold_def io_cold_defs[] = {
435 : [IORING_OP_NOP] = {
436 : .name = "NOP",
437 : },
438 : [IORING_OP_READV] = {
439 : .async_size = sizeof(struct io_async_rw),
440 : .name = "READV",
441 : .prep_async = io_readv_prep_async,
442 : .cleanup = io_readv_writev_cleanup,
443 : .fail = io_rw_fail,
444 : },
445 : [IORING_OP_WRITEV] = {
446 : .async_size = sizeof(struct io_async_rw),
447 : .name = "WRITEV",
448 : .prep_async = io_writev_prep_async,
449 : .cleanup = io_readv_writev_cleanup,
450 : .fail = io_rw_fail,
451 : },
452 : [IORING_OP_FSYNC] = {
453 : .name = "FSYNC",
454 : },
455 : [IORING_OP_READ_FIXED] = {
456 : .async_size = sizeof(struct io_async_rw),
457 : .name = "READ_FIXED",
458 : .fail = io_rw_fail,
459 : },
460 : [IORING_OP_WRITE_FIXED] = {
461 : .async_size = sizeof(struct io_async_rw),
462 : .name = "WRITE_FIXED",
463 : .fail = io_rw_fail,
464 : },
465 : [IORING_OP_POLL_ADD] = {
466 : .name = "POLL_ADD",
467 : },
468 : [IORING_OP_POLL_REMOVE] = {
469 : .name = "POLL_REMOVE",
470 : },
471 : [IORING_OP_SYNC_FILE_RANGE] = {
472 : .name = "SYNC_FILE_RANGE",
473 : },
474 : [IORING_OP_SENDMSG] = {
475 : .name = "SENDMSG",
476 : #if defined(CONFIG_NET)
477 : .async_size = sizeof(struct io_async_msghdr),
478 : .prep_async = io_sendmsg_prep_async,
479 : .cleanup = io_sendmsg_recvmsg_cleanup,
480 : .fail = io_sendrecv_fail,
481 : #endif
482 : },
483 : [IORING_OP_RECVMSG] = {
484 : .name = "RECVMSG",
485 : #if defined(CONFIG_NET)
486 : .async_size = sizeof(struct io_async_msghdr),
487 : .prep_async = io_recvmsg_prep_async,
488 : .cleanup = io_sendmsg_recvmsg_cleanup,
489 : .fail = io_sendrecv_fail,
490 : #endif
491 : },
492 : [IORING_OP_TIMEOUT] = {
493 : .async_size = sizeof(struct io_timeout_data),
494 : .name = "TIMEOUT",
495 : },
496 : [IORING_OP_TIMEOUT_REMOVE] = {
497 : .name = "TIMEOUT_REMOVE",
498 : },
499 : [IORING_OP_ACCEPT] = {
500 : .name = "ACCEPT",
501 : },
502 : [IORING_OP_ASYNC_CANCEL] = {
503 : .name = "ASYNC_CANCEL",
504 : },
505 : [IORING_OP_LINK_TIMEOUT] = {
506 : .async_size = sizeof(struct io_timeout_data),
507 : .name = "LINK_TIMEOUT",
508 : },
509 : [IORING_OP_CONNECT] = {
510 : .name = "CONNECT",
511 : #if defined(CONFIG_NET)
512 : .async_size = sizeof(struct io_async_connect),
513 : .prep_async = io_connect_prep_async,
514 : #endif
515 : },
516 : [IORING_OP_FALLOCATE] = {
517 : .name = "FALLOCATE",
518 : },
519 : [IORING_OP_OPENAT] = {
520 : .name = "OPENAT",
521 : .cleanup = io_open_cleanup,
522 : },
523 : [IORING_OP_CLOSE] = {
524 : .name = "CLOSE",
525 : },
526 : [IORING_OP_FILES_UPDATE] = {
527 : .name = "FILES_UPDATE",
528 : },
529 : [IORING_OP_STATX] = {
530 : .name = "STATX",
531 : .cleanup = io_statx_cleanup,
532 : },
533 : [IORING_OP_READ] = {
534 : .async_size = sizeof(struct io_async_rw),
535 : .name = "READ",
536 : .fail = io_rw_fail,
537 : },
538 : [IORING_OP_WRITE] = {
539 : .async_size = sizeof(struct io_async_rw),
540 : .name = "WRITE",
541 : .fail = io_rw_fail,
542 : },
543 : [IORING_OP_FADVISE] = {
544 : .name = "FADVISE",
545 : },
546 : [IORING_OP_MADVISE] = {
547 : .name = "MADVISE",
548 : },
549 : [IORING_OP_SEND] = {
550 : .name = "SEND",
551 : #if defined(CONFIG_NET)
552 : .async_size = sizeof(struct io_async_msghdr),
553 : .fail = io_sendrecv_fail,
554 : .prep_async = io_send_prep_async,
555 : #endif
556 : },
557 : [IORING_OP_RECV] = {
558 : .name = "RECV",
559 : #if defined(CONFIG_NET)
560 : .fail = io_sendrecv_fail,
561 : #endif
562 : },
563 : [IORING_OP_OPENAT2] = {
564 : .name = "OPENAT2",
565 : .cleanup = io_open_cleanup,
566 : },
567 : [IORING_OP_EPOLL_CTL] = {
568 : .name = "EPOLL",
569 : },
570 : [IORING_OP_SPLICE] = {
571 : .name = "SPLICE",
572 : },
573 : [IORING_OP_PROVIDE_BUFFERS] = {
574 : .name = "PROVIDE_BUFFERS",
575 : },
576 : [IORING_OP_REMOVE_BUFFERS] = {
577 : .name = "REMOVE_BUFFERS",
578 : },
579 : [IORING_OP_TEE] = {
580 : .name = "TEE",
581 : },
582 : [IORING_OP_SHUTDOWN] = {
583 : .name = "SHUTDOWN",
584 : },
585 : [IORING_OP_RENAMEAT] = {
586 : .name = "RENAMEAT",
587 : .cleanup = io_renameat_cleanup,
588 : },
589 : [IORING_OP_UNLINKAT] = {
590 : .name = "UNLINKAT",
591 : .cleanup = io_unlinkat_cleanup,
592 : },
593 : [IORING_OP_MKDIRAT] = {
594 : .name = "MKDIRAT",
595 : .cleanup = io_mkdirat_cleanup,
596 : },
597 : [IORING_OP_SYMLINKAT] = {
598 : .name = "SYMLINKAT",
599 : .cleanup = io_link_cleanup,
600 : },
601 : [IORING_OP_LINKAT] = {
602 : .name = "LINKAT",
603 : .cleanup = io_link_cleanup,
604 : },
605 : [IORING_OP_MSG_RING] = {
606 : .name = "MSG_RING",
607 : .cleanup = io_msg_ring_cleanup,
608 : },
609 : [IORING_OP_FSETXATTR] = {
610 : .name = "FSETXATTR",
611 : .cleanup = io_xattr_cleanup,
612 : },
613 : [IORING_OP_SETXATTR] = {
614 : .name = "SETXATTR",
615 : .cleanup = io_xattr_cleanup,
616 : },
617 : [IORING_OP_FGETXATTR] = {
618 : .name = "FGETXATTR",
619 : .cleanup = io_xattr_cleanup,
620 : },
621 : [IORING_OP_GETXATTR] = {
622 : .name = "GETXATTR",
623 : .cleanup = io_xattr_cleanup,
624 : },
625 : [IORING_OP_SOCKET] = {
626 : .name = "SOCKET",
627 : },
628 : [IORING_OP_URING_CMD] = {
629 : .name = "URING_CMD",
630 : .async_size = uring_cmd_pdu_size(1),
631 : .prep_async = io_uring_cmd_prep_async,
632 : },
633 : [IORING_OP_SEND_ZC] = {
634 : .name = "SEND_ZC",
635 : #if defined(CONFIG_NET)
636 : .async_size = sizeof(struct io_async_msghdr),
637 : .prep_async = io_send_prep_async,
638 : .cleanup = io_send_zc_cleanup,
639 : .fail = io_sendrecv_fail,
640 : #endif
641 : },
642 : [IORING_OP_SENDMSG_ZC] = {
643 : .name = "SENDMSG_ZC",
644 : #if defined(CONFIG_NET)
645 : .async_size = sizeof(struct io_async_msghdr),
646 : .prep_async = io_sendmsg_prep_async,
647 : .cleanup = io_send_zc_cleanup,
648 : .fail = io_sendrecv_fail,
649 : #endif
650 : },
651 : };
652 :
653 0 : const char *io_uring_get_opcode(u8 opcode)
654 : {
655 0 : if (opcode < IORING_OP_LAST)
656 0 : return io_cold_defs[opcode].name;
657 : return "INVALID";
658 : }
659 :
660 1 : void __init io_uring_optable_init(void)
661 : {
662 : int i;
663 :
664 : BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
665 : BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
666 :
667 50 : for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
668 49 : BUG_ON(!io_issue_defs[i].prep);
669 49 : if (io_issue_defs[i].prep != io_eopnotsupp_prep)
670 39 : BUG_ON(!io_issue_defs[i].issue);
671 49 : WARN_ON_ONCE(!io_cold_defs[i].name);
672 : }
673 1 : }
|
