summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/rhashtable.h51
-rw-r--r--lib/rhashtable.c15
2 files changed, 43 insertions, 23 deletions
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h
index ccbbafdf5547..460c0eaf6b96 100644
--- a/include/linux/rhashtable.h
+++ b/include/linux/rhashtable.h
@@ -82,6 +82,8 @@ struct bucket_table {
struct bucket_table __rcu *future_tbl;
+ struct lockdep_map dep_map;
+
struct rhash_lock_head __rcu *buckets[] ____cacheline_aligned_in_smp;
};
@@ -102,23 +104,38 @@ struct bucket_table {
* this is safe.
*/
-static inline void rht_lock(struct rhash_lock_head **bkt)
+static inline void rht_lock(struct bucket_table *tbl,
+ struct rhash_lock_head **bkt)
{
local_bh_disable();
bit_spin_lock(1, (unsigned long *)bkt);
+ lock_map_acquire(&tbl->dep_map);
+}
+
+static inline void rht_lock_nested(struct bucket_table *tbl,
+ struct rhash_lock_head **bucket,
+ unsigned int subclass)
+{
+ local_bh_disable();
+ bit_spin_lock(1, (unsigned long *)bucket);
+ lock_acquire_exclusive(&tbl->dep_map, subclass, 0, NULL, _THIS_IP_);
}
-static inline void rht_unlock(struct rhash_lock_head **bkt)
+static inline void rht_unlock(struct bucket_table *tbl,
+ struct rhash_lock_head **bkt)
{
+ lock_map_release(&tbl->dep_map);
bit_spin_unlock(1, (unsigned long *)bkt);
local_bh_enable();
}
-static inline void rht_assign_unlock(struct rhash_lock_head **bkt,
+static inline void rht_assign_unlock(struct bucket_table *tbl,
+ struct rhash_lock_head **bkt,
struct rhash_head *obj)
{
struct rhash_head **p = (struct rhash_head **)bkt;
+ lock_map_release(&tbl->dep_map);
rcu_assign_pointer(*p, obj);
preempt_enable();
__release(bitlock);
@@ -672,11 +689,11 @@ static inline void *__rhashtable_insert_fast(
if (!bkt)
goto out;
pprev = NULL;
- rht_lock(bkt);
+ rht_lock(tbl, bkt);
if (unlikely(rcu_access_pointer(tbl->future_tbl))) {
slow_path:
- rht_unlock(bkt);
+ rht_unlock(tbl, bkt);
rcu_read_unlock();
return rhashtable_insert_slow(ht, key, obj);
}
@@ -708,9 +725,9 @@ slow_path:
RCU_INIT_POINTER(list->rhead.next, head);
if (pprev) {
rcu_assign_pointer(*pprev, obj);
- rht_unlock(bkt);
+ rht_unlock(tbl, bkt);
} else
- rht_assign_unlock(bkt, obj);
+ rht_assign_unlock(tbl, bkt, obj);
data = NULL;
goto out;
}
@@ -737,7 +754,7 @@ slow_path:
}
atomic_inc(&ht->nelems);
- rht_assign_unlock(bkt, obj);
+ rht_assign_unlock(tbl, bkt, obj);
if (rht_grow_above_75(ht, tbl))
schedule_work(&ht->run_work);
@@ -749,7 +766,7 @@ out:
return data;
out_unlock:
- rht_unlock(bkt);
+ rht_unlock(tbl, bkt);
goto out;
}
@@ -951,7 +968,7 @@ static inline int __rhashtable_remove_fast_one(
if (!bkt)
return -ENOENT;
pprev = NULL;
- rht_lock(bkt);
+ rht_lock(tbl, bkt);
rht_for_each_from(he, rht_ptr(*bkt), tbl, hash) {
struct rhlist_head *list;
@@ -995,14 +1012,14 @@ static inline int __rhashtable_remove_fast_one(
if (pprev) {
rcu_assign_pointer(*pprev, obj);
- rht_unlock(bkt);
+ rht_unlock(tbl, bkt);
} else {
- rht_assign_unlock(bkt, obj);
+ rht_assign_unlock(tbl, bkt, obj);
}
goto unlocked;
}
- rht_unlock(bkt);
+ rht_unlock(tbl, bkt);
unlocked:
if (err > 0) {
atomic_dec(&ht->nelems);
@@ -1110,7 +1127,7 @@ static inline int __rhashtable_replace_fast(
return -ENOENT;
pprev = NULL;
- rht_lock(bkt);
+ rht_lock(tbl, bkt);
rht_for_each_from(he, rht_ptr(*bkt), tbl, hash) {
if (he != obj_old) {
@@ -1121,15 +1138,15 @@ static inline int __rhashtable_replace_fast(
rcu_assign_pointer(obj_new->next, obj_old->next);
if (pprev) {
rcu_assign_pointer(*pprev, obj_new);
- rht_unlock(bkt);
+ rht_unlock(tbl, bkt);
} else {
- rht_assign_unlock(bkt, obj_new);
+ rht_assign_unlock(tbl, bkt, obj_new);
}
err = 0;
goto unlocked;
}
- rht_unlock(bkt);
+ rht_unlock(tbl, bkt);
unlocked:
return err;
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index c5d0974467ee..a8583af43b59 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -173,6 +173,7 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
struct bucket_table *tbl = NULL;
size_t size;
int i;
+ static struct lock_class_key __key;
size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]);
tbl = kvzalloc(size, gfp);
@@ -187,6 +188,8 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
if (tbl == NULL)
return NULL;
+ lockdep_init_map(&tbl->dep_map, "rhashtable_bucket", &__key, 0);
+
tbl->size = size;
rcu_head_init(&tbl->rcu);
@@ -244,14 +247,14 @@ static int rhashtable_rehash_one(struct rhashtable *ht,
new_hash = head_hashfn(ht, new_tbl, entry);
- rht_lock(&new_tbl->buckets[new_hash]);
+ rht_lock_nested(new_tbl, &new_tbl->buckets[new_hash], SINGLE_DEPTH_NESTING);
head = rht_ptr(rht_dereference_bucket(new_tbl->buckets[new_hash],
new_tbl, new_hash));
RCU_INIT_POINTER(entry->next, head);
- rht_assign_unlock(&new_tbl->buckets[new_hash], entry);
+ rht_assign_unlock(new_tbl, &new_tbl->buckets[new_hash], entry);
if (pprev)
rcu_assign_pointer(*pprev, next);
@@ -272,14 +275,14 @@ static int rhashtable_rehash_chain(struct rhashtable *ht,
if (!bkt)
return 0;
- rht_lock(bkt);
+ rht_lock(old_tbl, bkt);
while (!(err = rhashtable_rehash_one(ht, bkt, old_hash)))
;
if (err == -ENOENT)
err = 0;
- rht_unlock(bkt);
+ rht_unlock(old_tbl, bkt);
return err;
}
@@ -600,7 +603,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key,
new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);
data = ERR_PTR(-EAGAIN);
} else {
- rht_lock(bkt);
+ rht_lock(tbl, bkt);
data = rhashtable_lookup_one(ht, bkt, tbl,
hash, key, obj);
new_tbl = rhashtable_insert_one(ht, bkt, tbl,
@@ -608,7 +611,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key,
if (PTR_ERR(new_tbl) != -EEXIST)
data = ERR_CAST(new_tbl);
- rht_unlock(bkt);
+ rht_unlock(tbl, bkt);
}
} while (!IS_ERR_OR_NULL(new_tbl));
generated by cgit v1.2.3-55-g7522 (git 2.39.0) at 2024-06-11 05:51:11 +0200