1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
|
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (c) 2020 ARM. All rights reserved.
* Copyright (c) 2020 Petr Vorel <pvorel@suse.cz>
*
* Mostly copied/adapted from <linux/io_uring.h>
*/
#ifndef IO_URING_H__
#define IO_URING_H__
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <stdlib.h>
#include <linux/fs.h>
#include "lapi/syscalls.h"
#ifndef IOSQE_FIXED_FILE
#ifndef __kernel_rwf_t
typedef int __kernel_rwf_t;
#endif
/*
* IO submission data structure (Submission Queue Entry)
*/
struct io_uring_sqe {
uint8_t opcode; /* type of operation for this sqe */
uint8_t flags; /* IOSQE_ flags */
uint16_t ioprio; /* ioprio for the request */
int32_t fd; /* file descriptor to do IO on */
union {
uint64_t off; /* offset into file */
uint64_t addr2;
};
uint64_t addr; /* pointer to buffer or iovecs */
uint32_t len; /* buffer size or number of iovecs */
union {
__kernel_rwf_t rw_flags;
uint32_t fsync_flags;
uint16_t poll_events;
uint32_t sync_range_flags;
uint32_t msg_flags;
uint32_t timeout_flags;
uint32_t accept_flags;
uint32_t cancel_flags;
uint32_t open_flags;
uint32_t statx_flags;
uint32_t fadvise_advice;
};
uint64_t user_data; /* data to be passed back at completion time */
union {
struct {
/* index into fixed buffers, if used */
uint16_t buf_index;
/* personality to use, if used */
uint16_t personality;
};
uint64_t __pad2[3];
};
};
enum {
IOSQE_FIXED_FILE_BIT,
IOSQE_IO_DRAIN_BIT,
IOSQE_IO_LINK_BIT,
IOSQE_IO_HARDLINK_BIT,
IOSQE_ASYNC_BIT,
};
/*
* sqe->flags
*/
/* use fixed fileset */
#define IOSQE_FIXED_FILE (1U << IOSQE_FIXED_FILE_BIT)
/* issue after inflight IO */
#define IOSQE_IO_DRAIN (1U << IOSQE_IO_DRAIN_BIT)
/* links next sqe */
#define IOSQE_IO_LINK (1U << IOSQE_IO_LINK_BIT)
/* like LINK, but stronger */
#define IOSQE_IO_HARDLINK (1U << IOSQE_IO_HARDLINK_BIT)
/* always go async */
#define IOSQE_ASYNC (1U << IOSQE_ASYNC_BIT)
/*
* io_uring_setup() flags
*/
#define IORING_SETUP_IOPOLL (1U << 0) /* io_context is polled */
#define IORING_SETUP_SQPOLL (1U << 1) /* SQ poll thread */
#define IORING_SETUP_SQ_AFF (1U << 2) /* sq_thread_cpu is valid */
#define IORING_SETUP_CQSIZE (1U << 3) /* app defines CQ size */
#define IORING_SETUP_CLAMP (1U << 4) /* clamp SQ/CQ ring sizes */
#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */
enum {
IORING_OP_NOP,
IORING_OP_READV,
IORING_OP_WRITEV,
IORING_OP_FSYNC,
IORING_OP_READ_FIXED,
IORING_OP_WRITE_FIXED,
IORING_OP_POLL_ADD,
IORING_OP_POLL_REMOVE,
IORING_OP_SYNC_FILE_RANGE,
IORING_OP_SENDMSG,
IORING_OP_RECVMSG,
IORING_OP_TIMEOUT,
IORING_OP_TIMEOUT_REMOVE,
IORING_OP_ACCEPT,
IORING_OP_ASYNC_CANCEL,
IORING_OP_LINK_TIMEOUT,
IORING_OP_CONNECT,
IORING_OP_FALLOCATE,
IORING_OP_OPENAT,
IORING_OP_CLOSE,
IORING_OP_FILES_UPDATE,
IORING_OP_STATX,
IORING_OP_READ,
IORING_OP_WRITE,
IORING_OP_FADVISE,
IORING_OP_MADVISE,
IORING_OP_SEND,
IORING_OP_RECV,
IORING_OP_OPENAT2,
IORING_OP_EPOLL_CTL,
/* this goes last, obviously */
IORING_OP_LAST,
};
/*
* sqe->fsync_flags
*/
#define IORING_FSYNC_DATASYNC (1U << 0)
/*
* sqe->timeout_flags
*/
#define IORING_TIMEOUT_ABS (1U << 0)
/*
* IO completion data structure (Completion Queue Entry)
*/
struct io_uring_cqe {
uint64_t user_data; /* sqe->data submission passed back */
int32_t res; /* result code for this event */
uint32_t flags;
};
/*
* Magic offsets for the application to mmap the data it needs
*/
#define IORING_OFF_SQ_RING 0ULL
#define IORING_OFF_CQ_RING 0x8000000ULL
#define IORING_OFF_SQES 0x10000000ULL
/*
* Filled with the offset for mmap(2)
*/
struct io_sqring_offsets {
uint32_t head;
uint32_t tail;
uint32_t ring_mask;
uint32_t ring_entries;
uint32_t flags;
uint32_t dropped;
uint32_t array;
uint32_t resv1;
uint64_t resv2;
};
/*
* sq_ring->flags
*/
#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */
struct io_cqring_offsets {
uint32_t head;
uint32_t tail;
uint32_t ring_mask;
uint32_t ring_entries;
uint32_t overflow;
uint32_t cqes;
uint64_t resv[2];
};
/*
* io_uring_enter(2) flags
*/
#define IORING_ENTER_GETEVENTS (1U << 0)
#define IORING_ENTER_SQ_WAKEUP (1U << 1)
/*
* Passed in for io_uring_setup(2). Copied back with updated info on success
*/
struct io_uring_params {
uint32_t sq_entries;
uint32_t cq_entries;
uint32_t flags;
uint32_t sq_thread_cpu;
uint32_t sq_thread_idle;
uint32_t features;
uint32_t wq_fd;
uint32_t resv[3];
struct io_sqring_offsets sq_off;
struct io_cqring_offsets cq_off;
};
/*
* io_uring_params->features flags
*/
#define IORING_FEAT_SINGLE_MMAP (1U << 0)
#define IORING_FEAT_NODROP (1U << 1)
#define IORING_FEAT_SUBMIT_STABLE (1U << 2)
#define IORING_FEAT_RW_CUR_POS (1U << 3)
#define IORING_FEAT_CUR_PERSONALITY (1U << 4)
/*
* io_uring_register(2) opcodes and arguments
*/
#define IORING_REGISTER_BUFFERS 0
#define IORING_UNREGISTER_BUFFERS 1
#define IORING_REGISTER_FILES 2
#define IORING_UNREGISTER_FILES 3
#define IORING_REGISTER_EVENTFD 4
#define IORING_UNREGISTER_EVENTFD 5
#define IORING_REGISTER_FILES_UPDATE 6
#define IORING_REGISTER_EVENTFD_ASYNC 7
#define IORING_REGISTER_PROBE 8
#define IORING_REGISTER_PERSONALITY 9
#define IORING_UNREGISTER_PERSONALITY 10
struct io_uring_files_update {
uint32_t offset;
uint32_t resv;
uint64_t __attribute__((aligned(8))) fds;
};
#define IO_URING_OP_SUPPORTED (1U << 0)
struct io_uring_probe_op {
uint8_t op;
uint8_t resv;
uint16_t flags; /* IO_URING_OP_* flags */
uint32_t resv2;
};
struct io_uring_probe {
uint8_t last_op; /* last opcode supported */
uint8_t ops_len; /* length of ops[] array below */
uint16_t resv;
uint32_t resv2[3];
struct io_uring_probe_op ops[0];
};
#endif /* IOSQE_FIXED_FILE */
#ifndef HAVE_IO_URING_REGISTER
int io_uring_register(int fd, unsigned int opcode, void *arg,
unsigned int nr_args)
{
return tst_syscall(__NR_io_uring_register, fd, opcode, arg, nr_args);
}
#endif /* HAVE_IO_URING_REGISTER */
#ifndef HAVE_IO_URING_SETUP
int io_uring_setup(unsigned int entries, struct io_uring_params *p)
{
return tst_syscall(__NR_io_uring_setup, entries, p);
}
#endif /* HAVE_IO_URING_SETUP */
#ifndef HAVE_IO_URING_ENTER
int io_uring_enter(int fd, unsigned int to_submit, unsigned int min_complete,
unsigned int flags, sigset_t *sig)
{
return tst_syscall(__NR_io_uring_enter, fd, to_submit, min_complete,
flags, sig, _NSIG / 8);
}
#endif /* HAVE_IO_URING_ENTER */
void io_uring_setup_supported_by_kernel(void)
{
if ((tst_kvercmp(5, 1, 0)) < 0) {
TEST(syscall(__NR_io_uring_setup, NULL, 0));
if (TST_RET != -1)
SAFE_CLOSE(TST_RET);
else if (TST_ERR == ENOSYS)
tst_brk(TCONF,
"Test not supported on kernel version < v5.1");
}
}
#endif /* IO_URING_H__ */
|
