From d7b627277b57370223d682cede979a279284b12a Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Mon, 13 Feb 2017 15:58:24 -0500 Subject: radix-tree: Fix __rcu annotations Many places were missing __rcu annotations. A few places needed a few lines of explanation about why it was safe to not use RCU accessors. Add a custom CFLAGS setting to the Makefile to ensure that new patches don't miss RCU annotations. Signed-off-by: Matthew Wilcox --- include/linux/radix-tree.h | 110 ++++++++++++++++++++------------------- lib/Makefile | 2 + lib/radix-tree.c | 125 ++++++++++++++++++++++++--------------------- 3 files changed, 122 insertions(+), 115 deletions(-) diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 0b502de7d23a..3e5735064b71 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -221,10 +221,8 @@ static inline unsigned int iter_shift(const struct radix_tree_iter *iter) */ /** - * radix_tree_deref_slot - dereference a slot - * @pslot: pointer to slot, returned by radix_tree_lookup_slot - * Returns: item that was stored in that slot with any direct pointer flag - * removed. + * radix_tree_deref_slot - dereference a slot + * @slot: slot pointer, returned by radix_tree_lookup_slot * * For use with radix_tree_lookup_slot(). Caller must hold tree at least read * locked across slot lookup and dereference. Not required if write lock is @@ -232,26 +230,27 @@ static inline unsigned int iter_shift(const struct radix_tree_iter *iter) * * radix_tree_deref_retry must be used to confirm validity of the pointer if * only the read lock is held. + * + * Return: entry stored in that slot. */ -static inline void *radix_tree_deref_slot(void **pslot) +static inline void *radix_tree_deref_slot(void __rcu **slot) { - return rcu_dereference(*pslot); + return rcu_dereference(*slot); } /** - * radix_tree_deref_slot_protected - dereference a slot without RCU lock but with tree lock held - * @pslot: pointer to slot, returned by radix_tree_lookup_slot - * Returns: item that was stored in that slot with any direct pointer flag - * removed. - * - * Similar to radix_tree_deref_slot but only used during migration when a pages - * mapping is being moved. The caller does not hold the RCU read lock but it - * must hold the tree lock to prevent parallel updates. + * radix_tree_deref_slot_protected - dereference a slot with tree lock held + * @slot: slot pointer, returned by radix_tree_lookup_slot + * + * Similar to radix_tree_deref_slot. The caller does not hold the RCU read + * lock but it must hold the tree lock to prevent parallel updates. + * + * Return: entry stored in that slot. */ -static inline void *radix_tree_deref_slot_protected(void **pslot, +static inline void *radix_tree_deref_slot_protected(void __rcu **slot, spinlock_t *treelock) { - return rcu_dereference_protected(*pslot, lockdep_is_held(treelock)); + return rcu_dereference_protected(*slot, lockdep_is_held(treelock)); } /** @@ -287,9 +286,9 @@ static inline int radix_tree_exception(void *arg) return unlikely((unsigned long)arg & RADIX_TREE_ENTRY_MASK); } -int __radix_tree_create(struct radix_tree_root *root, unsigned long index, +int __radix_tree_create(struct radix_tree_root *, unsigned long index, unsigned order, struct radix_tree_node **nodep, - void ***slotp); + void __rcu ***slotp); int __radix_tree_insert(struct radix_tree_root *, unsigned long index, unsigned order, void *); static inline int radix_tree_insert(struct radix_tree_root *root, @@ -298,42 +297,41 @@ static inline int radix_tree_insert(struct radix_tree_root *root, return __radix_tree_insert(root, index, 0, entry); } void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index, - struct radix_tree_node **nodep, void ***slotp); + struct radix_tree_node **nodep, void __rcu ***slotp); void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); -void **radix_tree_lookup_slot(const struct radix_tree_root *, unsigned long); +void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *, + unsigned long index); typedef void (*radix_tree_update_node_t)(struct radix_tree_node *, void *); -void __radix_tree_replace(struct radix_tree_root *root, - struct radix_tree_node *node, - void **slot, void *item, +void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *, + void __rcu **slot, void *entry, radix_tree_update_node_t update_node, void *private); void radix_tree_iter_replace(struct radix_tree_root *, - const struct radix_tree_iter *, void **slot, void *item); -void radix_tree_replace_slot(struct radix_tree_root *root, - void **slot, void *item); -void __radix_tree_delete_node(struct radix_tree_root *root, - struct radix_tree_node *node, + const struct radix_tree_iter *, void __rcu **slot, void *entry); +void radix_tree_replace_slot(struct radix_tree_root *, + void __rcu **slot, void *entry); +void __radix_tree_delete_node(struct radix_tree_root *, + struct radix_tree_node *, radix_tree_update_node_t update_node, void *private); void radix_tree_iter_delete(struct radix_tree_root *, - struct radix_tree_iter *iter, void **slot); + struct radix_tree_iter *iter, void __rcu **slot); void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *); void *radix_tree_delete(struct radix_tree_root *, unsigned long); -void radix_tree_clear_tags(struct radix_tree_root *root, - struct radix_tree_node *node, - void **slot); +void radix_tree_clear_tags(struct radix_tree_root *, struct radix_tree_node *, + void __rcu **slot); unsigned int radix_tree_gang_lookup(const struct radix_tree_root *, void **results, unsigned long first_index, unsigned int max_items); unsigned int radix_tree_gang_lookup_slot(const struct radix_tree_root *, - void ***results, unsigned long *indices, + void __rcu ***results, unsigned long *indices, unsigned long first_index, unsigned int max_items); int radix_tree_preload(gfp_t gfp_mask); int radix_tree_maybe_preload(gfp_t gfp_mask); int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order); void radix_tree_init(void); -void *radix_tree_tag_set(struct radix_tree_root *root, +void *radix_tree_tag_set(struct radix_tree_root *, unsigned long index, unsigned int tag); -void *radix_tree_tag_clear(struct radix_tree_root *root, +void *radix_tree_tag_clear(struct radix_tree_root *, unsigned long index, unsigned int tag); int radix_tree_tag_get(const struct radix_tree_root *, unsigned long index, unsigned int tag); @@ -341,15 +339,13 @@ void radix_tree_iter_tag_set(struct radix_tree_root *, const struct radix_tree_iter *iter, unsigned int tag); void radix_tree_iter_tag_clear(struct radix_tree_root *, const struct radix_tree_iter *iter, unsigned int tag); -unsigned int -radix_tree_gang_lookup_tag(const struct radix_tree_root *, void **results, - unsigned long first_index, unsigned int max_items, - unsigned int tag); -unsigned int -radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *, void ***results, - unsigned long first_index, unsigned int max_items, - unsigned int tag); -int radix_tree_tagged(const struct radix_tree_root *root, unsigned int tag); +unsigned int radix_tree_gang_lookup_tag(const struct radix_tree_root *, + void **results, unsigned long first_index, + unsigned int max_items, unsigned int tag); +unsigned int radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *, + void __rcu ***results, unsigned long first_index, + unsigned int max_items, unsigned int tag); +int radix_tree_tagged(const struct radix_tree_root *, unsigned int tag); static inline void radix_tree_preload_end(void) { @@ -361,7 +357,7 @@ int radix_tree_split(struct radix_tree_root *, unsigned long index, unsigned new_order); int radix_tree_join(struct radix_tree_root *, unsigned long index, unsigned new_order, void *); -void **idr_get_free(struct radix_tree_root *, struct radix_tree_iter *, +void __rcu **idr_get_free(struct radix_tree_root *, struct radix_tree_iter *, gfp_t, int end); enum { @@ -377,7 +373,7 @@ enum { * @start: iteration starting index * Returns: NULL */ -static __always_inline void ** +static __always_inline void __rcu ** radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start) { /* @@ -406,7 +402,7 @@ radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start) * Also it fills @iter with data about chunk: position in the tree (index), * its end (next_index), and constructs a bit mask for tagged iterating (tags). */ -void **radix_tree_next_chunk(const struct radix_tree_root *, +void __rcu **radix_tree_next_chunk(const struct radix_tree_root *, struct radix_tree_iter *iter, unsigned flags); /** @@ -419,7 +415,8 @@ void **radix_tree_next_chunk(const struct radix_tree_root *, * containing it and updates @iter to describe the entry. If @index is not * present, it returns NULL. */ -static inline void **radix_tree_iter_lookup(const struct radix_tree_root *root, +static inline void __rcu ** +radix_tree_iter_lookup(const struct radix_tree_root *root, struct radix_tree_iter *iter, unsigned long index) { radix_tree_iter_init(iter, index); @@ -436,7 +433,8 @@ static inline void **radix_tree_iter_lookup(const struct radix_tree_root *root, * which is at least @index. If @index is larger than any present entry, this * function returns NULL. The @iter is updated to describe the entry found. */ -static inline void **radix_tree_iter_find(const struct radix_tree_root *root, +static inline void __rcu ** +radix_tree_iter_find(const struct radix_tree_root *root, struct radix_tree_iter *iter, unsigned long index) { radix_tree_iter_init(iter, index); @@ -453,7 +451,7 @@ static inline void **radix_tree_iter_find(const struct radix_tree_root *root, * and continue the iteration. */ static inline __must_check -void **radix_tree_iter_retry(struct radix_tree_iter *iter) +void __rcu **radix_tree_iter_retry(struct radix_tree_iter *iter) { iter->next_index = iter->index; iter->tags = 0; @@ -476,7 +474,7 @@ __radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots) * have been invalidated by an insertion or deletion. Call this function * before releasing the lock to continue the iteration from the next index. */ -void **__must_check radix_tree_iter_resume(void **slot, +void __rcu **__must_check radix_tree_iter_resume(void __rcu **slot, struct radix_tree_iter *iter); /** @@ -492,11 +490,11 @@ radix_tree_chunk_size(struct radix_tree_iter *iter) } #ifdef CONFIG_RADIX_TREE_MULTIORDER -void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, - unsigned flags); +void __rcu **__radix_tree_next_slot(void __rcu **slot, + struct radix_tree_iter *iter, unsigned flags); #else /* Can't happen without sibling entries, but the compiler can't tell that */ -static inline void ** __radix_tree_next_slot(void **slot, +static inline void __rcu **__radix_tree_next_slot(void __rcu **slot, struct radix_tree_iter *iter, unsigned flags) { return slot; @@ -522,8 +520,8 @@ static inline void ** __radix_tree_next_slot(void **slot, * b) we are doing non-tagged iteration, and iter->index and iter->next_index * have been set up so that radix_tree_chunk_size() returns 1 or 0. */ -static __always_inline void ** -radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, unsigned flags) +static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot, + struct radix_tree_iter *iter, unsigned flags) { if (flags & RADIX_TREE_ITER_TAGGED) { iter->tags >>= 1; diff --git a/lib/Makefile b/lib/Makefile index bc4073a8cd08..2fc096985b21 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -24,6 +24,8 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o nmi_backtrace.o nodemask.o win_minmax.o +CFLAGS_radix-tree.o += -DCONFIG_SPARSE_RCU_POINTER + lib-$(CONFIG_MMU) += ioremap.o lib-$(CONFIG_SMP) += cpumask.o lib-$(CONFIG_HAS_DMA) += dma-noop.o diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 723bebe40eef..9c0fa4df736b 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -104,7 +104,7 @@ static inline void *node_to_entry(void *ptr) static inline bool is_sibling_entry(const struct radix_tree_node *parent, void *node) { - void **ptr = node; + void __rcu **ptr = node; return (parent->slots <= ptr) && (ptr < parent->slots + RADIX_TREE_MAP_SIZE); } @@ -116,8 +116,8 @@ bool is_sibling_entry(const struct radix_tree_node *parent, void *node) } #endif -static inline -unsigned long get_slot_offset(const struct radix_tree_node *parent, void **slot) +static inline unsigned long +get_slot_offset(const struct radix_tree_node *parent, void __rcu **slot) { return slot - parent->slots; } @@ -126,12 +126,13 @@ static unsigned int radix_tree_descend(const struct radix_tree_node *parent, struct radix_tree_node **nodep, unsigned long index) { unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK; - void **entry = rcu_dereference_raw(parent->slots[offset]); + void __rcu **entry = rcu_dereference_raw(parent->slots[offset]); #ifdef CONFIG_RADIX_TREE_MULTIORDER if (radix_tree_is_internal_node(entry)) { if (is_sibling_entry(parent, entry)) { - void **sibentry = (void **) entry_to_node(entry); + void __rcu **sibentry; + sibentry = (void __rcu **) entry_to_node(entry); offset = get_slot_offset(parent, sibentry); entry = rcu_dereference_raw(*sibentry); } @@ -618,7 +619,7 @@ static unsigned radix_tree_load_root(const struct radix_tree_root *root, static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, unsigned long index, unsigned int shift) { - struct radix_tree_node *slot; + void *entry; unsigned int maxshift; int tag; @@ -627,8 +628,8 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, while (index > shift_maxindex(maxshift)) maxshift += RADIX_TREE_MAP_SHIFT; - slot = rcu_dereference_raw(root->rnode); - if (!slot && (!is_idr(root) || root_tag_get(root, IDR_FREE))) + entry = rcu_dereference_raw(root->rnode); + if (!entry && (!is_idr(root) || root_tag_get(root, IDR_FREE))) goto out; do { @@ -652,15 +653,19 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, } BUG_ON(shift > BITS_PER_LONG); - if (radix_tree_is_internal_node(slot)) { - entry_to_node(slot)->parent = node; - } else if (radix_tree_exceptional_entry(slot)) { + if (radix_tree_is_internal_node(entry)) { + entry_to_node(entry)->parent = node; + } else if (radix_tree_exceptional_entry(entry)) { /* Moving an exceptional root->rnode to a node */ node->exceptional = 1; } - node->slots[0] = slot; - slot = node_to_entry(node); - rcu_assign_pointer(root->rnode, slot); + /* + * entry was already in the radix tree, so we do not need + * rcu_assign_pointer here + */ + node->slots[0] = (void __rcu *)entry; + entry = node_to_entry(node); + rcu_assign_pointer(root->rnode, entry); shift += RADIX_TREE_MAP_SHIFT; } while (shift <= maxshift); out: @@ -708,7 +713,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, * (node->slots[0]), it will be safe to dereference the new * one (root->rnode) as far as dependent read barriers go. */ - root->rnode = child; + root->rnode = (void __rcu *)child; if (is_idr(root) && !tag_get(node, IDR_FREE, 0)) root_tag_clear(root, IDR_FREE); @@ -732,7 +737,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, */ node->count = 0; if (!radix_tree_is_internal_node(child)) { - node->slots[0] = RADIX_TREE_RETRY; + node->slots[0] = (void __rcu *)RADIX_TREE_RETRY; if (update_node) update_node(node, private); } @@ -805,10 +810,10 @@ static bool delete_node(struct radix_tree_root *root, */ int __radix_tree_create(struct radix_tree_root *root, unsigned long index, unsigned order, struct radix_tree_node **nodep, - void ***slotp) + void __rcu ***slotp) { struct radix_tree_node *node = NULL, *child; - void **slot = (void **)&root->rnode; + void __rcu **slot = (void __rcu **)&root->rnode; unsigned long maxindex; unsigned int shift, offset = 0; unsigned long max = index | ((1UL << order) - 1); @@ -890,8 +895,8 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) } #ifdef CONFIG_RADIX_TREE_MULTIORDER -static inline int insert_entries(struct radix_tree_node *node, void **slot, - void *item, unsigned order, bool replace) +static inline int insert_entries(struct radix_tree_node *node, + void __rcu **slot, void *item, unsigned order, bool replace) { struct radix_tree_node *child; unsigned i, n, tag, offset, tags = 0; @@ -953,8 +958,8 @@ static inline int insert_entries(struct radix_tree_node *node, void **slot, return n; } #else -static inline int insert_entries(struct radix_tree_node *node, void **slot, - void *item, unsigned order, bool replace) +static inline int insert_entries(struct radix_tree_node *node, + void __rcu **slot, void *item, unsigned order, bool replace) { if (*slot) return -EEXIST; @@ -981,7 +986,7 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, unsigned order, void *item) { struct radix_tree_node *node; - void **slot; + void __rcu **slot; int error; BUG_ON(radix_tree_is_internal_node(item)); @@ -1023,15 +1028,15 @@ EXPORT_SYMBOL(__radix_tree_insert); */ void *__radix_tree_lookup(const struct radix_tree_root *root, unsigned long index, struct radix_tree_node **nodep, - void ***slotp) + void __rcu ***slotp) { struct radix_tree_node *node, *parent; unsigned long maxindex; - void **slot; + void __rcu **slot; restart: parent = NULL; - slot = (void **)&root->rnode; + slot = (void __rcu **)&root->rnode; radix_tree_load_root(root, &node, &maxindex); if (index > maxindex) return NULL; @@ -1066,10 +1071,10 @@ void *__radix_tree_lookup(const struct radix_tree_root *root, * exclusive from other writers. Any dereference of the slot must be done * using radix_tree_deref_slot. */ -void **radix_tree_lookup_slot(const struct radix_tree_root *root, +void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *root, unsigned long index) { - void **slot; + void __rcu **slot; if (!__radix_tree_lookup(root, index, NULL, &slot)) return NULL; @@ -1096,7 +1101,7 @@ void *radix_tree_lookup(const struct radix_tree_root *root, unsigned long index) EXPORT_SYMBOL(radix_tree_lookup); static inline void replace_sibling_entries(struct radix_tree_node *node, - void **slot, int count, int exceptional) + void __rcu **slot, int count, int exceptional) { #ifdef CONFIG_RADIX_TREE_MULTIORDER void *ptr = node_to_entry(slot); @@ -1115,8 +1120,8 @@ static inline void replace_sibling_entries(struct radix_tree_node *node, #endif } -static void replace_slot(void **slot, void *item, struct radix_tree_node *node, - int count, int exceptional) +static void replace_slot(void __rcu **slot, void *item, + struct radix_tree_node *node, int count, int exceptional) { if (WARN_ON_ONCE(radix_tree_is_internal_node(item))) return; @@ -1147,7 +1152,7 @@ static bool node_tag_get(const struct radix_tree_root *root, * deleted. */ static int calculate_count(struct radix_tree_root *root, - struct radix_tree_node *node, void **slot, + struct radix_tree_node *node, void __rcu **slot, void *item, void *old) { if (is_idr(root)) { @@ -1175,7 +1180,7 @@ static int calculate_count(struct radix_tree_root *root, */ void __radix_tree_replace(struct radix_tree_root *root, struct radix_tree_node *node, - void **slot, void *item, + void __rcu **slot, void *item, radix_tree_update_node_t update_node, void *private) { void *old = rcu_dereference_raw(*slot); @@ -1188,7 +1193,7 @@ void __radix_tree_replace(struct radix_tree_root *root, * deleting entries, but that needs accounting against the * node unless the slot is root->rnode. */ - WARN_ON_ONCE(!node && (slot != (void **)&root->rnode) && + WARN_ON_ONCE(!node && (slot != (void __rcu **)&root->rnode) && (count || exceptional)); replace_slot(slot, item, node, count, exceptional); @@ -1218,7 +1223,7 @@ void __radix_tree_replace(struct radix_tree_root *root, * radix_tree_iter_replace(). */ void radix_tree_replace_slot(struct radix_tree_root *root, - void **slot, void *item) + void __rcu **slot, void *item) { __radix_tree_replace(root, NULL, slot, item, NULL, NULL); } @@ -1233,7 +1238,8 @@ void radix_tree_replace_slot(struct radix_tree_root *root, * Caller must hold tree write locked across split and replacement. */ void radix_tree_iter_replace(struct radix_tree_root *root, - const struct radix_tree_iter *iter, void **slot, void *item) + const struct radix_tree_iter *iter, + void __rcu **slot, void *item) { __radix_tree_replace(root, iter->node, slot, item, NULL, NULL); } @@ -1257,7 +1263,7 @@ int radix_tree_join(struct radix_tree_root *root, unsigned long index, unsigned order, void *item) { struct radix_tree_node *node; - void **slot; + void __rcu **slot; int error; BUG_ON(radix_tree_is_internal_node(item)); @@ -1292,7 +1298,7 @@ int radix_tree_split(struct radix_tree_root *root, unsigned long index, unsigned order) { struct radix_tree_node *parent, *node, *child; - void **slot; + void __rcu **slot; unsigned int offset, end; unsigned n, tag, tags = 0; gfp_t gfp = root_gfp_mask(root); @@ -1603,8 +1609,8 @@ static void set_iter_tags(struct radix_tree_iter *iter, } #ifdef CONFIG_RADIX_TREE_MULTIORDER -static void **skip_siblings(struct radix_tree_node **nodep, - void **slot, struct radix_tree_iter *iter) +static void __rcu **skip_siblings(struct radix_tree_node **nodep, + void __rcu **slot, struct radix_tree_iter *iter) { void *sib = node_to_entry(slot - 1); @@ -1621,8 +1627,8 @@ static void **skip_siblings(struct radix_tree_node **nodep, return NULL; } -void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, - unsigned flags) +void __rcu **__radix_tree_next_slot(void __rcu **slot, + struct radix_tree_iter *iter, unsigned flags) { unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK; struct radix_tree_node *node = rcu_dereference_raw(*slot); @@ -1675,14 +1681,15 @@ void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, } EXPORT_SYMBOL(__radix_tree_next_slot); #else -static void **skip_siblings(struct radix_tree_node **nodep, - void **slot, struct radix_tree_iter *iter) +static void __rcu **skip_siblings(struct radix_tree_node **nodep, + void __rcu **slot, struct radix_tree_iter *iter) { return slot; } #endif -void **radix_tree_iter_resume(void **slot, struct radix_tree_iter *iter) +void __rcu **radix_tree_iter_resume(void __rcu **slot, + struct radix_tree_iter *iter) { struct radix_tree_node *node; @@ -1703,7 +1710,7 @@ EXPORT_SYMBOL(radix_tree_iter_resume); * @flags: RADIX_TREE_ITER_* flags and tag index * Returns: pointer to chunk first slot, or NULL if iteration is over */ -void **radix_tree_next_chunk(const struct radix_tree_root *root, +void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, struct radix_tree_iter *iter, unsigned flags) { unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK; @@ -1740,7 +1747,7 @@ void **radix_tree_next_chunk(const struct radix_tree_root *root, iter->tags = 1; iter->node = NULL; __set_iter_shift(iter, 0); - return (void **)&root->rnode; + return (void __rcu **)&root->rnode; } do { @@ -1819,7 +1826,7 @@ radix_tree_gang_lookup(const struct radix_tree_root *root, void **results, unsigned long first_index, unsigned int max_items) { struct radix_tree_iter iter; - void **slot; + void __rcu **slot; unsigned int ret = 0; if (unlikely(!max_items)) @@ -1861,11 +1868,11 @@ EXPORT_SYMBOL(radix_tree_gang_lookup); */ unsigned int radix_tree_gang_lookup_slot(const struct radix_tree_root *root, - void ***results, unsigned long *indices, + void __rcu ***results, unsigned long *indices, unsigned long first_index, unsigned int max_items) { struct radix_tree_iter iter; - void **slot; + void __rcu **slot; unsigned int ret = 0; if (unlikely(!max_items)) @@ -1902,7 +1909,7 @@ radix_tree_gang_lookup_tag(const struct radix_tree_root *root, void **results, unsigned int tag) { struct radix_tree_iter iter; - void **slot; + void __rcu **slot; unsigned int ret = 0; if (unlikely(!max_items)) @@ -1939,11 +1946,11 @@ EXPORT_SYMBOL(radix_tree_gang_lookup_tag); */ unsigned int radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *root, - void ***results, unsigned long first_index, + void __rcu ***results, unsigned long first_index, unsigned int max_items, unsigned int tag) { struct radix_tree_iter iter; - void **slot; + void __rcu **slot; unsigned int ret = 0; if (unlikely(!max_items)) @@ -1979,7 +1986,7 @@ void __radix_tree_delete_node(struct radix_tree_root *root, } static bool __radix_tree_delete(struct radix_tree_root *root, - struct radix_tree_node *node, void **slot) + struct radix_tree_node *node, void __rcu **slot) { void *old = rcu_dereference_raw(*slot); int exceptional = radix_tree_exceptional_entry(old) ? -1 : 0; @@ -2009,7 +2016,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root, * which can access this tree. */ void radix_tree_iter_delete(struct radix_tree_root *root, - struct radix_tree_iter *iter, void **slot) + struct radix_tree_iter *iter, void __rcu **slot) { if (__radix_tree_delete(root, iter->node, slot)) iter->index = iter->next_index; @@ -2030,7 +2037,7 @@ void *radix_tree_delete_item(struct radix_tree_root *root, unsigned long index, void *item) { struct radix_tree_node *node = NULL; - void **slot; + void __rcu **slot; void *entry; entry = __radix_tree_lookup(root, index, &node, &slot); @@ -2064,7 +2071,7 @@ EXPORT_SYMBOL(radix_tree_delete); void radix_tree_clear_tags(struct radix_tree_root *root, struct radix_tree_node *node, - void **slot) + void __rcu **slot) { if (node) { unsigned int tag, offset = get_slot_offset(node, slot); @@ -2129,11 +2136,11 @@ int ida_pre_get(struct ida *ida, gfp_t gfp) } EXPORT_SYMBOL(ida_pre_get); -void **idr_get_free(struct radix_tree_root *root, +void __rcu **idr_get_free(struct radix_tree_root *root, struct radix_tree_iter *iter, gfp_t gfp, int end) { struct radix_tree_node *node = NULL, *child; - void **slot = (void **)&root->rnode; + void __rcu **slot = (void __rcu **)&root->rnode; unsigned long maxindex, start = iter->next_index; unsigned long max = end > 0 ? end - 1 : INT_MAX; unsigned int shift, offset = 0; -- cgit v1.2.3-55-g7522