summaryrefslogtreecommitdiffstats
path: root/util
diff options
context:
space:
mode:
Diffstat (limited to 'util')
-rw-r--r--util/aio-posix.c88
-rw-r--r--util/memfd.c1
-rw-r--r--util/qht.c138
-rw-r--r--util/qsp.c11
-rw-r--r--util/trace-events4
5 files changed, 163 insertions, 79 deletions
diff --git a/util/aio-posix.c b/util/aio-posix.c
index 131ba6b4a8..621b3025d8 100644
--- a/util/aio-posix.c
+++ b/util/aio-posix.c
@@ -211,6 +211,7 @@ void aio_set_fd_handler(AioContext *ctx,
AioHandler *node;
bool is_new = false;
bool deleted = false;
+ int poll_disable_change;
qemu_lockcnt_lock(&ctx->list_lock);
@@ -244,11 +245,9 @@ void aio_set_fd_handler(AioContext *ctx,
QLIST_REMOVE(node, node);
deleted = true;
}
-
- if (!node->io_poll) {
- ctx->poll_disable_cnt--;
- }
+ poll_disable_change = -!node->io_poll;
} else {
+ poll_disable_change = !io_poll - (node && !node->io_poll);
if (node == NULL) {
/* Alloc and insert if it's not already there */
node = g_new0(AioHandler, 1);
@@ -257,10 +256,6 @@ void aio_set_fd_handler(AioContext *ctx,
g_source_add_poll(&ctx->source, &node->pfd);
is_new = true;
-
- ctx->poll_disable_cnt += !io_poll;
- } else {
- ctx->poll_disable_cnt += !io_poll - !node->io_poll;
}
/* Update handler with latest information */
@@ -274,6 +269,15 @@ void aio_set_fd_handler(AioContext *ctx,
node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);
}
+ /* No need to order poll_disable_cnt writes against other updates;
+ * the counter is only used to avoid wasting time and latency on
+ * iterated polling when the system call will be ultimately necessary.
+ * Changing handlers is a rare event, and a little wasted polling until
+ * the aio_notify below is not an issue.
+ */
+ atomic_set(&ctx->poll_disable_cnt,
+ atomic_read(&ctx->poll_disable_cnt) + poll_disable_change);
+
aio_epoll_update(ctx, node, is_new);
qemu_lockcnt_unlock(&ctx->list_lock);
aio_notify(ctx);
@@ -486,7 +490,7 @@ static void add_pollfd(AioHandler *node)
npfd++;
}
-static bool run_poll_handlers_once(AioContext *ctx)
+static bool run_poll_handlers_once(AioContext *ctx, int64_t *timeout)
{
bool progress = false;
AioHandler *node;
@@ -494,9 +498,11 @@ static bool run_poll_handlers_once(AioContext *ctx)
QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
if (!node->deleted && node->io_poll &&
aio_node_check(ctx, node->is_external) &&
- node->io_poll(node->opaque) &&
- node->opaque != &ctx->notifier) {
- progress = true;
+ node->io_poll(node->opaque)) {
+ *timeout = 0;
+ if (node->opaque != &ctx->notifier) {
+ progress = true;
+ }
}
/* Caller handles freeing deleted nodes. Don't do it here. */
@@ -518,31 +524,38 @@ static bool run_poll_handlers_once(AioContext *ctx)
*
* Returns: true if progress was made, false otherwise
*/
-static bool run_poll_handlers(AioContext *ctx, int64_t max_ns)
+static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout)
{
bool progress;
- int64_t end_time;
+ int64_t start_time, elapsed_time;
assert(ctx->notify_me);
assert(qemu_lockcnt_count(&ctx->list_lock) > 0);
- assert(ctx->poll_disable_cnt == 0);
-
- trace_run_poll_handlers_begin(ctx, max_ns);
- end_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + max_ns;
+ trace_run_poll_handlers_begin(ctx, max_ns, *timeout);
+ start_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
do {
- progress = run_poll_handlers_once(ctx);
- } while (!progress && qemu_clock_get_ns(QEMU_CLOCK_REALTIME) < end_time);
+ progress = run_poll_handlers_once(ctx, timeout);
+ elapsed_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time;
+ } while (!progress && elapsed_time < max_ns
+ && !atomic_read(&ctx->poll_disable_cnt));
- trace_run_poll_handlers_end(ctx, progress);
+ /* If time has passed with no successful polling, adjust *timeout to
+ * keep the same ending time.
+ */
+ if (*timeout != -1) {
+ *timeout -= MIN(*timeout, elapsed_time);
+ }
+ trace_run_poll_handlers_end(ctx, progress, *timeout);
return progress;
}
/* try_poll_mode:
* @ctx: the AioContext
- * @blocking: busy polling is only attempted when blocking is true
+ * @timeout: timeout for blocking wait, computed by the caller and updated if
+ * polling succeeds.
*
* ctx->notify_me must be non-zero so this function can detect aio_notify().
*
@@ -550,19 +563,16 @@ static bool run_poll_handlers(AioContext *ctx, int64_t max_ns)
*
* Returns: true if progress was made, false otherwise
*/
-static bool try_poll_mode(AioContext *ctx, bool blocking)
+static bool try_poll_mode(AioContext *ctx, int64_t *timeout)
{
- if (blocking && ctx->poll_max_ns && ctx->poll_disable_cnt == 0) {
- /* See qemu_soonest_timeout() uint64_t hack */
- int64_t max_ns = MIN((uint64_t)aio_compute_timeout(ctx),
- (uint64_t)ctx->poll_ns);
+ /* See qemu_soonest_timeout() uint64_t hack */
+ int64_t max_ns = MIN((uint64_t)*timeout, (uint64_t)ctx->poll_ns);
- if (max_ns) {
- poll_set_started(ctx, true);
+ if (max_ns && !atomic_read(&ctx->poll_disable_cnt)) {
+ poll_set_started(ctx, true);
- if (run_poll_handlers(ctx, max_ns)) {
- return true;
- }
+ if (run_poll_handlers(ctx, max_ns, timeout)) {
+ return true;
}
}
@@ -571,7 +581,7 @@ static bool try_poll_mode(AioContext *ctx, bool blocking)
/* Even if we don't run busy polling, try polling once in case it can make
* progress and the caller will be able to avoid ppoll(2)/epoll_wait(2).
*/
- return run_poll_handlers_once(ctx);
+ return run_poll_handlers_once(ctx, timeout);
}
bool aio_poll(AioContext *ctx, bool blocking)
@@ -601,8 +611,14 @@ bool aio_poll(AioContext *ctx, bool blocking)
start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
}
- progress = try_poll_mode(ctx, blocking);
- if (!progress) {
+ timeout = blocking ? aio_compute_timeout(ctx) : 0;
+ progress = try_poll_mode(ctx, &timeout);
+ assert(!(timeout && progress));
+
+ /* If polling is allowed, non-blocking aio_poll does not need the
+ * system call---a single round of run_poll_handlers_once suffices.
+ */
+ if (timeout || atomic_read(&ctx->poll_disable_cnt)) {
assert(npfd == 0);
/* fill pollfds */
@@ -616,8 +632,6 @@ bool aio_poll(AioContext *ctx, bool blocking)
}
}
- timeout = blocking ? aio_compute_timeout(ctx) : 0;
-
/* wait until next event */
if (aio_epoll_check_poll(ctx, pollfds, npfd, timeout)) {
AioHandler epoll_handler;
diff --git a/util/memfd.c b/util/memfd.c
index d248a53c3c..6287946b61 100644
--- a/util/memfd.c
+++ b/util/memfd.c
@@ -187,6 +187,7 @@ bool qemu_memfd_alloc_check(void)
int fd;
void *ptr;
+ fd = -1;
ptr = qemu_memfd_alloc("test", 4096, 0, &fd, NULL);
memfd_check = ptr ? MEMFD_OK : MEMFD_KO;
qemu_memfd_free(ptr, 4096, fd);
diff --git a/util/qht.c b/util/qht.c
index 1e3a072e25..aa51be3c52 100644
--- a/util/qht.c
+++ b/util/qht.c
@@ -89,6 +89,19 @@
#define QHT_BUCKET_ENTRIES 4
#endif
+enum qht_iter_type {
+ QHT_ITER_VOID, /* do nothing; use retvoid */
+ QHT_ITER_RM, /* remove element if retbool returns true */
+};
+
+struct qht_iter {
+ union {
+ qht_iter_func_t retvoid;
+ qht_iter_bool_func_t retbool;
+ } f;
+ enum qht_iter_type type;
+};
+
/*
* Do _not_ use qemu_mutex_[try]lock directly! Use these macros, otherwise
* the profiler (QSP) will deadlock.
@@ -223,7 +236,7 @@ static inline void qht_head_init(struct qht_bucket *b)
}
static inline
-struct qht_bucket *qht_map_to_bucket(struct qht_map *map, uint32_t hash)
+struct qht_bucket *qht_map_to_bucket(const struct qht_map *map, uint32_t hash)
{
return &map->buckets[hash & (map->n_buckets - 1)];
}
@@ -255,7 +268,8 @@ static void qht_map_unlock_buckets(struct qht_map *map)
* Call with at least a bucket lock held.
* @map should be the value read before acquiring the lock (or locks).
*/
-static inline bool qht_map_is_stale__locked(struct qht *ht, struct qht_map *map)
+static inline bool qht_map_is_stale__locked(const struct qht *ht,
+ const struct qht_map *map)
{
return map != ht->map;
}
@@ -324,12 +338,12 @@ struct qht_bucket *qht_bucket_lock__no_stale(struct qht *ht, uint32_t hash,
return b;
}
-static inline bool qht_map_needs_resize(struct qht_map *map)
+static inline bool qht_map_needs_resize(const struct qht_map *map)
{
return atomic_read(&map->n_added_buckets) > map->n_added_buckets_threshold;
}
-static inline void qht_chain_destroy(struct qht_bucket *head)
+static inline void qht_chain_destroy(const struct qht_bucket *head)
{
struct qht_bucket *curr = head->next;
struct qht_bucket *prev;
@@ -469,10 +483,10 @@ bool qht_reset_size(struct qht *ht, size_t n_elems)
}
static inline
-void *qht_do_lookup(struct qht_bucket *head, qht_lookup_func_t func,
+void *qht_do_lookup(const struct qht_bucket *head, qht_lookup_func_t func,
const void *userp, uint32_t hash)
{
- struct qht_bucket *b = head;
+ const struct qht_bucket *b = head;
int i;
do {
@@ -496,7 +510,7 @@ void *qht_do_lookup(struct qht_bucket *head, qht_lookup_func_t func,
}
static __attribute__((noinline))
-void *qht_lookup__slowpath(struct qht_bucket *b, qht_lookup_func_t func,
+void *qht_lookup__slowpath(const struct qht_bucket *b, qht_lookup_func_t func,
const void *userp, uint32_t hash)
{
unsigned int version;
@@ -509,11 +523,11 @@ void *qht_lookup__slowpath(struct qht_bucket *b, qht_lookup_func_t func,
return ret;
}
-void *qht_lookup_custom(struct qht *ht, const void *userp, uint32_t hash,
+void *qht_lookup_custom(const struct qht *ht, const void *userp, uint32_t hash,
qht_lookup_func_t func)
{
- struct qht_bucket *b;
- struct qht_map *map;
+ const struct qht_bucket *b;
+ const struct qht_map *map;
unsigned int version;
void *ret;
@@ -532,13 +546,16 @@ void *qht_lookup_custom(struct qht *ht, const void *userp, uint32_t hash,
return qht_lookup__slowpath(b, func, userp, hash);
}
-void *qht_lookup(struct qht *ht, const void *userp, uint32_t hash)
+void *qht_lookup(const struct qht *ht, const void *userp, uint32_t hash)
{
return qht_lookup_custom(ht, userp, hash, ht->cmp);
}
-/* call with head->lock held */
-static void *qht_insert__locked(struct qht *ht, struct qht_map *map,
+/*
+ * call with head->lock held
+ * @ht is const since it is only used for ht->cmp()
+ */
+static void *qht_insert__locked(const struct qht *ht, struct qht_map *map,
struct qht_bucket *head, void *p, uint32_t hash,
bool *needs_resize)
{
@@ -632,7 +649,7 @@ bool qht_insert(struct qht *ht, void *p, uint32_t hash, void **existing)
return false;
}
-static inline bool qht_entry_is_last(struct qht_bucket *b, int pos)
+static inline bool qht_entry_is_last(const struct qht_bucket *b, int pos)
{
if (pos == QHT_BUCKET_ENTRIES - 1) {
if (b->next == NULL) {
@@ -658,7 +675,7 @@ qht_entry_move(struct qht_bucket *to, int i, struct qht_bucket *from, int j)
}
/*
- * Find the last valid entry in @head, and swap it with @orig[pos], which has
+ * Find the last valid entry in @orig, and swap it with @orig[pos], which has
* just been invalidated.
*/
static inline void qht_bucket_remove_entry(struct qht_bucket *orig, int pos)
@@ -692,8 +709,7 @@ static inline void qht_bucket_remove_entry(struct qht_bucket *orig, int pos)
/* call with b->lock held */
static inline
-bool qht_remove__locked(struct qht_map *map, struct qht_bucket *head,
- const void *p, uint32_t hash)
+bool qht_remove__locked(struct qht_bucket *head, const void *p, uint32_t hash)
{
struct qht_bucket *b = head;
int i;
@@ -728,15 +744,16 @@ bool qht_remove(struct qht *ht, const void *p, uint32_t hash)
qht_debug_assert(p);
b = qht_bucket_lock__no_stale(ht, hash, &map);
- ret = qht_remove__locked(map, b, p, hash);
+ ret = qht_remove__locked(b, p, hash);
qht_bucket_debug__locked(b);
qemu_spin_unlock(&b->lock);
return ret;
}
-static inline void qht_bucket_iter(struct qht *ht, struct qht_bucket *b,
- qht_iter_func_t func, void *userp)
+static inline void qht_bucket_iter(struct qht_bucket *head,
+ const struct qht_iter *iter, void *userp)
{
+ struct qht_bucket *b = head;
int i;
do {
@@ -744,37 +761,83 @@ static inline void qht_bucket_iter(struct qht *ht, struct qht_bucket *b,
if (b->pointers[i] == NULL) {
return;
}
- func(ht, b->pointers[i], b->hashes[i], userp);
+ switch (iter->type) {
+ case QHT_ITER_VOID:
+ iter->f.retvoid(b->pointers[i], b->hashes[i], userp);
+ break;
+ case QHT_ITER_RM:
+ if (iter->f.retbool(b->pointers[i], b->hashes[i], userp)) {
+ /* replace i with the last valid element in the bucket */
+ seqlock_write_begin(&head->sequence);
+ qht_bucket_remove_entry(b, i);
+ seqlock_write_end(&head->sequence);
+ qht_bucket_debug__locked(b);
+ /* reevaluate i, since it just got replaced */
+ i--;
+ continue;
+ }
+ break;
+ default:
+ g_assert_not_reached();
+ }
}
b = b->next;
} while (b);
}
/* call with all of the map's locks held */
-static inline void qht_map_iter__all_locked(struct qht *ht, struct qht_map *map,
- qht_iter_func_t func, void *userp)
+static inline void qht_map_iter__all_locked(struct qht_map *map,
+ const struct qht_iter *iter,
+ void *userp)
{
size_t i;
for (i = 0; i < map->n_buckets; i++) {
- qht_bucket_iter(ht, &map->buckets[i], func, userp);
+ qht_bucket_iter(&map->buckets[i], iter, userp);
}
}
-void qht_iter(struct qht *ht, qht_iter_func_t func, void *userp)
+static inline void
+do_qht_iter(struct qht *ht, const struct qht_iter *iter, void *userp)
{
struct qht_map *map;
map = atomic_rcu_read(&ht->map);
qht_map_lock_buckets(map);
- /* Note: ht here is merely for carrying ht->mode; ht->map won't be read */
- qht_map_iter__all_locked(ht, map, func, userp);
+ qht_map_iter__all_locked(map, iter, userp);
qht_map_unlock_buckets(map);
}
-static void qht_map_copy(struct qht *ht, void *p, uint32_t hash, void *userp)
+void qht_iter(struct qht *ht, qht_iter_func_t func, void *userp)
+{
+ const struct qht_iter iter = {
+ .f.retvoid = func,
+ .type = QHT_ITER_VOID,
+ };
+
+ do_qht_iter(ht, &iter, userp);
+}
+
+void qht_iter_remove(struct qht *ht, qht_iter_bool_func_t func, void *userp)
+{
+ const struct qht_iter iter = {
+ .f.retbool = func,
+ .type = QHT_ITER_RM,
+ };
+
+ do_qht_iter(ht, &iter, userp);
+}
+
+struct qht_map_copy_data {
+ struct qht *ht;
+ struct qht_map *new;
+};
+
+static void qht_map_copy(void *p, uint32_t hash, void *userp)
{
- struct qht_map *new = userp;
+ struct qht_map_copy_data *data = userp;
+ struct qht *ht = data->ht;
+ struct qht_map *new = data->new;
struct qht_bucket *b = qht_map_to_bucket(new, hash);
/* no need to acquire b->lock because no thread has seen this map yet */
@@ -788,6 +851,11 @@ static void qht_map_copy(struct qht *ht, void *p, uint32_t hash, void *userp)
static void qht_do_resize_reset(struct qht *ht, struct qht_map *new, bool reset)
{
struct qht_map *old;
+ const struct qht_iter iter = {
+ .f.retvoid = qht_map_copy,
+ .type = QHT_ITER_VOID,
+ };
+ struct qht_map_copy_data data;
old = ht->map;
qht_map_lock_buckets(old);
@@ -802,7 +870,9 @@ static void qht_do_resize_reset(struct qht *ht, struct qht_map *new, bool reset)
}
g_assert(new->n_buckets != old->n_buckets);
- qht_map_iter__all_locked(ht, old, qht_map_copy, new);
+ data.ht = ht;
+ data.new = new;
+ qht_map_iter__all_locked(old, &iter, &data);
qht_map_debug__all_locked(new);
atomic_rcu_set(&ht->map, new);
@@ -829,9 +899,9 @@ bool qht_resize(struct qht *ht, size_t n_elems)
}
/* pass @stats to qht_statistics_destroy() when done */
-void qht_statistics_init(struct qht *ht, struct qht_stats *stats)
+void qht_statistics_init(const struct qht *ht, struct qht_stats *stats)
{
- struct qht_map *map;
+ const struct qht_map *map;
int i;
map = atomic_rcu_read(&ht->map);
@@ -848,8 +918,8 @@ void qht_statistics_init(struct qht *ht, struct qht_stats *stats)
stats->head_buckets = map->n_buckets;
for (i = 0; i < map->n_buckets; i++) {
- struct qht_bucket *head = &map->buckets[i];
- struct qht_bucket *b;
+ const struct qht_bucket *head = &map->buckets[i];
+ const struct qht_bucket *b;
unsigned int version;
size_t buckets;
size_t entries;
diff --git a/util/qsp.c b/util/qsp.c
index b0c2575d10..2de3a97594 100644
--- a/util/qsp.c
+++ b/util/qsp.c
@@ -533,7 +533,7 @@ static gint qsp_tree_cmp(gconstpointer ap, gconstpointer bp, gpointer up)
}
}
-static void qsp_sort(struct qht *ht, void *p, uint32_t h, void *userp)
+static void qsp_sort(void *p, uint32_t h, void *userp)
{
QSPEntry *e = p;
GTree *tree = userp;
@@ -541,7 +541,7 @@ static void qsp_sort(struct qht *ht, void *p, uint32_t h, void *userp)
g_tree_insert(tree, e, NULL);
}
-static void qsp_aggregate(struct qht *global_ht, void *p, uint32_t h, void *up)
+static void qsp_aggregate(void *p, uint32_t h, void *up)
{
struct qht *ht = up;
const QSPEntry *e = p;
@@ -553,7 +553,7 @@ static void qsp_aggregate(struct qht *global_ht, void *p, uint32_t h, void *up)
qsp_entry_aggregate(agg, e);
}
-static void qsp_iter_diff(struct qht *orig, void *p, uint32_t hash, void *htp)
+static void qsp_iter_diff(void *p, uint32_t hash, void *htp)
{
struct qht *ht = htp;
QSPEntry *old = p;
@@ -583,8 +583,7 @@ static void qsp_diff(struct qht *orig, struct qht *new)
qht_iter(orig, qsp_iter_diff, new);
}
-static void
-qsp_iter_callsite_coalesce(struct qht *orig, void *p, uint32_t h, void *htp)
+static void qsp_iter_callsite_coalesce(void *p, uint32_t h, void *htp)
{
struct qht *ht = htp;
QSPEntry *old = p;
@@ -603,7 +602,7 @@ qsp_iter_callsite_coalesce(struct qht *orig, void *p, uint32_t h, void *htp)
e->n_acqs += old->n_acqs;
}
-static void qsp_ht_delete(struct qht *ht, void *p, uint32_t h, void *htp)
+static void qsp_ht_delete(void *p, uint32_t h, void *htp)
{
g_free(p);
}
diff --git a/util/trace-events b/util/trace-events
index 4822434c89..79569b7fdf 100644
--- a/util/trace-events
+++ b/util/trace-events
@@ -1,8 +1,8 @@
# See docs/devel/tracing.txt for syntax documentation.
# util/aio-posix.c
-run_poll_handlers_begin(void *ctx, int64_t max_ns) "ctx %p max_ns %"PRId64
-run_poll_handlers_end(void *ctx, bool progress) "ctx %p progress %d"
+run_poll_handlers_begin(void *ctx, int64_t max_ns, int64_t timeout) "ctx %p max_ns %"PRId64 " timeout %"PRId64
+run_poll_handlers_end(void *ctx, bool progress, int64_t timeout) "ctx %p progress %d new timeout %"PRId64
poll_shrink(void *ctx, int64_t old, int64_t new) "ctx %p old %"PRId64" new %"PRId64
poll_grow(void *ctx, int64_t old, int64_t new) "ctx %p old %"PRId64" new %"PRId64
generated by cgit v1.2.3-55-g7522 (git 2.39.0) at 2025-04-24 00:49:58 +0200