Add keyring patches to support krb5 (rhbz 1003043)

1 files changed, 6834 insertions, 0 deletions

diff --git a/keys-expand-keyring.patch b/keys-expand-keyring.patch
new file mode 100644
index 000000000..75618243b
--- /dev/null
+++ b/keys-expand-keyring.patch
@@ -0,0 +1,6834 @@
+From 96dcf8e91389e509021448ffd798cc68471fcf0f Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:50 +0100
+Subject: [PATCH 01/10] KEYS: Skip key state checks when checking for
+ possession
+
+Skip key state checks (invalidation, revocation and expiration) when checking
+for possession.  Without this, keys that have been marked invalid, revoked
+keys and expired keys are not given a possession attribute - which means the
+possessor is not granted any possession permits and cannot do anything with
+them unless they also have one a user, group or other permit.
+
+This causes failures in the keyutils test suite's revocation and expiration
+tests now that commit 96b5c8fea6c0861621051290d705ec2e971963f1 reduced the
+initial permissions granted to a key.
+
+The failures are due to accesses to revoked and expired keys being given
+EACCES instead of EKEYREVOKED or EKEYEXPIRED.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ security/keys/internal.h         | 1 +
+ security/keys/process_keys.c     | 8 +++++---
+ security/keys/request_key.c      | 6 ++++--
+ security/keys/request_key_auth.c | 2 +-
+ 4 files changed, 11 insertions(+), 6 deletions(-)
+
+diff --git a/security/keys/internal.h b/security/keys/internal.h
+index d4f1468..df971fe 100644
+--- a/security/keys/internal.h
++++ b/security/keys/internal.h
+@@ -124,6 +124,7 @@ extern key_ref_t search_my_process_keyrings(struct key_type *type,
+ extern key_ref_t search_process_keyrings(struct key_type *type,
+ 					 const void *description,
+ 					 key_match_func_t match,
++					 bool no_state_check,
+ 					 const struct cred *cred);
+ 
+ extern struct key *find_keyring_by_name(const char *name, bool skip_perm_check);
+diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c
+index 42defae..a3410d6 100644
+--- a/security/keys/process_keys.c
++++ b/security/keys/process_keys.c
+@@ -440,6 +440,7 @@ found:
+ key_ref_t search_process_keyrings(struct key_type *type,
+ 				  const void *description,
+ 				  key_match_func_t match,
++				  bool no_state_check,
+ 				  const struct cred *cred)
+ {
+ 	struct request_key_auth *rka;
+@@ -448,7 +449,7 @@ key_ref_t search_process_keyrings(struct key_type *type,
+ 	might_sleep();
+ 
+ 	key_ref = search_my_process_keyrings(type, description, match,
+-					     false, cred);
++					     no_state_check, cred);
+ 	if (!IS_ERR(key_ref))
+ 		goto found;
+ 	err = key_ref;
+@@ -468,7 +469,8 @@ key_ref_t search_process_keyrings(struct key_type *type,
+ 			rka = cred->request_key_auth->payload.data;
+ 
+ 			key_ref = search_process_keyrings(type, description,
+-							  match, rka->cred);
++							  match, no_state_check,
++							  rka->cred);
+ 
+ 			up_read(&cred->request_key_auth->sem);
+ 
+@@ -675,7 +677,7 @@ try_again:
+ 		/* check to see if we possess the key */
+ 		skey_ref = search_process_keyrings(key->type, key,
+ 						   lookup_user_key_possessed,
+-						   cred);
++						   true, cred);
+ 
+ 		if (!IS_ERR(skey_ref)) {
+ 			key_put(key);
+diff --git a/security/keys/request_key.c b/security/keys/request_key.c
+index c411f9b..172115b 100644
+--- a/security/keys/request_key.c
++++ b/security/keys/request_key.c
+@@ -390,7 +390,8 @@ static int construct_alloc_key(struct key_type *type,
+ 	 * waited for locks */
+ 	mutex_lock(&key_construction_mutex);
+ 
+-	key_ref = search_process_keyrings(type, description, type->match, cred);
++	key_ref = search_process_keyrings(type, description, type->match,
++					  false, cred);
+ 	if (!IS_ERR(key_ref))
+ 		goto key_already_present;
+ 
+@@ -539,7 +540,8 @@ struct key *request_key_and_link(struct key_type *type,
+ 	       dest_keyring, flags);
+ 
+ 	/* search all the process keyrings for a key */
+-	key_ref = search_process_keyrings(type, description, type->match, cred);
++	key_ref = search_process_keyrings(type, description, type->match,
++					  false, cred);
+ 
+ 	if (!IS_ERR(key_ref)) {
+ 		key = key_ref_to_ptr(key_ref);
+diff --git a/security/keys/request_key_auth.c b/security/keys/request_key_auth.c
+index 85730d5..92077de 100644
+--- a/security/keys/request_key_auth.c
++++ b/security/keys/request_key_auth.c
+@@ -247,7 +247,7 @@ struct key *key_get_instantiation_authkey(key_serial_t target_id)
+ 		&key_type_request_key_auth,
+ 		(void *) (unsigned long) target_id,
+ 		key_get_instantiation_authkey_match,
+-		cred);
++		false, cred);
+ 
+ 	if (IS_ERR(authkey_ref)) {
+ 		authkey = ERR_CAST(authkey_ref);
+-- 
+1.8.3.1
+
+
+From 9b1294158dd1fbca78541b5d55c057e46b1a9ca2 Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:51 +0100
+Subject: [PATCH 02/10] KEYS: Use bool in make_key_ref() and is_key_possessed()
+
+Make make_key_ref() take a bool possession parameter and make
+is_key_possessed() return a bool.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ Documentation/security/keys.txt | 7 +++----
+ include/linux/key.h             | 4 ++--
+ security/keys/keyring.c         | 5 +++--
+ 3 files changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/Documentation/security/keys.txt b/Documentation/security/keys.txt
+index 7b4145d..9ede670 100644
+--- a/Documentation/security/keys.txt
++++ b/Documentation/security/keys.txt
+@@ -865,15 +865,14 @@ encountered:
+      calling processes has a searchable link to the key from one of its
+      keyrings. There are three functions for dealing with these:
+ 
+-	key_ref_t make_key_ref(const struct key *key,
+-			       unsigned long possession);
++	key_ref_t make_key_ref(const struct key *key, bool possession);
+ 
+ 	struct key *key_ref_to_ptr(const key_ref_t key_ref);
+ 
+-	unsigned long is_key_possessed(const key_ref_t key_ref);
++	bool is_key_possessed(const key_ref_t key_ref);
+ 
+      The first function constructs a key reference from a key pointer and
+-     possession information (which must be 0 or 1 and not any other value).
++     possession information (which must be true or false).
+ 
+      The second function retrieves the key pointer from a reference and the
+      third retrieves the possession flag.
+diff --git a/include/linux/key.h b/include/linux/key.h
+index 4dfde11..51bce29 100644
+--- a/include/linux/key.h
++++ b/include/linux/key.h
+@@ -99,7 +99,7 @@ struct keyring_name;
+ typedef struct __key_reference_with_attributes *key_ref_t;
+ 
+ static inline key_ref_t make_key_ref(const struct key *key,
+-				     unsigned long possession)
++				     bool possession)
+ {
+ 	return (key_ref_t) ((unsigned long) key | possession);
+ }
+@@ -109,7 +109,7 @@ static inline struct key *key_ref_to_ptr(const key_ref_t key_ref)
+ 	return (struct key *) ((unsigned long) key_ref & ~1UL);
+ }
+ 
+-static inline unsigned long is_key_possessed(const key_ref_t key_ref)
++static inline bool is_key_possessed(const key_ref_t key_ref)
+ {
+ 	return (unsigned long) key_ref & 1UL;
+ }
+diff --git a/security/keys/keyring.c b/security/keys/keyring.c
+index 6ece7f2..f784063 100644
+--- a/security/keys/keyring.c
++++ b/security/keys/keyring.c
+@@ -329,9 +329,10 @@ key_ref_t keyring_search_aux(key_ref_t keyring_ref,
+ 
+ 	struct keyring_list *keylist;
+ 	struct timespec now;
+-	unsigned long possessed, kflags;
++	unsigned long kflags;
+ 	struct key *keyring, *key;
+ 	key_ref_t key_ref;
++	bool possessed;
+ 	long err;
+ 	int sp, nkeys, kix;
+ 
+@@ -542,8 +543,8 @@ key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+ 			       key_perm_t perm)
+ {
+ 	struct keyring_list *klist;
+-	unsigned long possessed;
+ 	struct key *keyring, *key;
++	bool possessed;
+ 	int nkeys, loop;
+ 
+ 	keyring = key_ref_to_ptr(keyring_ref);
+-- 
+1.8.3.1
+
+
+From 4a7e7536b9b728f1d912d0e4c047c885c95e13a1 Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:51 +0100
+Subject: [PATCH 03/10] KEYS: key_is_dead() should take a const key pointer
+ argument
+
+key_is_dead() should take a const key pointer argument as it doesn't modify
+what it points to.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ security/keys/internal.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/security/keys/internal.h b/security/keys/internal.h
+index df971fe..490aef5 100644
+--- a/security/keys/internal.h
++++ b/security/keys/internal.h
+@@ -203,7 +203,7 @@ extern struct key *key_get_instantiation_authkey(key_serial_t target_id);
+ /*
+  * Determine whether a key is dead.
+  */
+-static inline bool key_is_dead(struct key *key, time_t limit)
++static inline bool key_is_dead(const struct key *key, time_t limit)
+ {
+ 	return
+ 		key->flags & ((1 << KEY_FLAG_DEAD) |
+-- 
+1.8.3.1
+
+
+From 9007a0a7f8c135f0085e46db277de0cf7b944403 Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:52 +0100
+Subject: [PATCH 04/10] KEYS: Consolidate the concept of an 'index key' for key
+ access
+
+Consolidate the concept of an 'index key' for accessing keys.  The index key
+is the search term needed to find a key directly - basically the key type and
+the key description.  We can add to that the description length.
+
+This will be useful when turning a keyring into an associative array rather
+than just a pointer block.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ include/linux/key.h         | 21 +++++++++----
+ security/keys/internal.h    |  8 ++---
+ security/keys/key.c         | 72 +++++++++++++++++++++++----------------------
+ security/keys/keyring.c     | 37 +++++++++++------------
+ security/keys/request_key.c | 12 +++++---
+ 5 files changed, 83 insertions(+), 67 deletions(-)
+
+diff --git a/include/linux/key.h b/include/linux/key.h
+index 51bce29..d573e82 100644
+--- a/include/linux/key.h
++++ b/include/linux/key.h
+@@ -82,6 +82,12 @@ struct key_owner;
+ struct keyring_list;
+ struct keyring_name;
+ 
++struct keyring_index_key {
++	struct key_type		*type;
++	const char		*description;
++	size_t			desc_len;
++};
++
+ /*****************************************************************************/
+ /*
+  * key reference with possession attribute handling
+@@ -129,7 +135,6 @@ struct key {
+ 		struct list_head graveyard_link;
+ 		struct rb_node	serial_node;
+ 	};
+-	struct key_type		*type;		/* type of key */
+ 	struct rw_semaphore	sem;		/* change vs change sem */
+ 	struct key_user		*user;		/* owner of this key */
+ 	void			*security;	/* security data for this key */
+@@ -163,12 +168,18 @@ struct key {
+ #define KEY_FLAG_ROOT_CAN_CLEAR	6	/* set if key can be cleared by root without permission */
+ #define KEY_FLAG_INVALIDATED	7	/* set if key has been invalidated */
+ 
+-	/* the description string
+-	 * - this is used to match a key against search criteria
+-	 * - this should be a printable string
++	/* the key type and key description string
++	 * - the desc is used to match a key against search criteria
++	 * - it should be a printable string
+ 	 * - eg: for krb5 AFS, this might be "afs@REDHAT.COM"
+ 	 */
+-	char			*description;
++	union {
++		struct keyring_index_key index_key;
++		struct {
++			struct key_type	*type;		/* type of key */
++			char		*description;
++		};
++	};
+ 
+ 	/* type specific data
+ 	 * - this is used by the keyring type to index the name
+diff --git a/security/keys/internal.h b/security/keys/internal.h
+index 490aef5..77441dd 100644
+--- a/security/keys/internal.h
++++ b/security/keys/internal.h
+@@ -89,19 +89,17 @@ extern struct key_type *key_type_lookup(const char *type);
+ extern void key_type_put(struct key_type *ktype);
+ 
+ extern int __key_link_begin(struct key *keyring,
+-			    const struct key_type *type,
+-			    const char *description,
++			    const struct keyring_index_key *index_key,
+ 			    unsigned long *_prealloc);
+ extern int __key_link_check_live_key(struct key *keyring, struct key *key);
+ extern void __key_link(struct key *keyring, struct key *key,
+ 		       unsigned long *_prealloc);
+ extern void __key_link_end(struct key *keyring,
+-			   struct key_type *type,
++			   const struct keyring_index_key *index_key,
+ 			   unsigned long prealloc);
+ 
+ extern key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+-				      const struct key_type *type,
+-				      const char *description,
++				      const struct keyring_index_key *index_key,
+ 				      key_perm_t perm);
+ 
+ extern struct key *keyring_search_instkey(struct key *keyring,
+diff --git a/security/keys/key.c b/security/keys/key.c
+index 8fb7c7b..7e6bc39 100644
+--- a/security/keys/key.c
++++ b/security/keys/key.c
+@@ -242,8 +242,8 @@ struct key *key_alloc(struct key_type *type, const char *desc,
+ 		}
+ 	}
+ 
+-	desclen = strlen(desc) + 1;
+-	quotalen = desclen + type->def_datalen;
++	desclen = strlen(desc);
++	quotalen = desclen + 1 + type->def_datalen;
+ 
+ 	/* get hold of the key tracking for this user */
+ 	user = key_user_lookup(uid);
+@@ -277,7 +277,8 @@ struct key *key_alloc(struct key_type *type, const char *desc,
+ 		goto no_memory_2;
+ 
+ 	if (desc) {
+-		key->description = kmemdup(desc, desclen, GFP_KERNEL);
++		key->index_key.desc_len = desclen;
++		key->index_key.description = kmemdup(desc, desclen + 1, GFP_KERNEL);
+ 		if (!key->description)
+ 			goto no_memory_3;
+ 	}
+@@ -285,7 +286,7 @@ struct key *key_alloc(struct key_type *type, const char *desc,
+ 	atomic_set(&key->usage, 1);
+ 	init_rwsem(&key->sem);
+ 	lockdep_set_class(&key->sem, &type->lock_class);
+-	key->type = type;
++	key->index_key.type = type;
+ 	key->user = user;
+ 	key->quotalen = quotalen;
+ 	key->datalen = type->def_datalen;
+@@ -489,8 +490,7 @@ int key_instantiate_and_link(struct key *key,
+ 	}
+ 
+ 	if (keyring) {
+-		ret = __key_link_begin(keyring, key->type, key->description,
+-				       &prealloc);
++		ret = __key_link_begin(keyring, &key->index_key, &prealloc);
+ 		if (ret < 0)
+ 			goto error_free_preparse;
+ 	}
+@@ -499,7 +499,7 @@ int key_instantiate_and_link(struct key *key,
+ 					 &prealloc);
+ 
+ 	if (keyring)
+-		__key_link_end(keyring, key->type, prealloc);
++		__key_link_end(keyring, &key->index_key, prealloc);
+ 
+ error_free_preparse:
+ 	if (key->type->preparse)
+@@ -548,8 +548,7 @@ int key_reject_and_link(struct key *key,
+ 	ret = -EBUSY;
+ 
+ 	if (keyring)
+-		link_ret = __key_link_begin(keyring, key->type,
+-					    key->description, &prealloc);
++		link_ret = __key_link_begin(keyring, &key->index_key, &prealloc);
+ 
+ 	mutex_lock(&key_construction_mutex);
+ 
+@@ -581,7 +580,7 @@ int key_reject_and_link(struct key *key,
+ 	mutex_unlock(&key_construction_mutex);
+ 
+ 	if (keyring)
+-		__key_link_end(keyring, key->type, prealloc);
++		__key_link_end(keyring, &key->index_key, prealloc);
+ 
+ 	/* wake up anyone waiting for a key to be constructed */
+ 	if (awaken)
+@@ -780,25 +779,27 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 			       key_perm_t perm,
+ 			       unsigned long flags)
+ {
+-	unsigned long prealloc;
++	struct keyring_index_key index_key = {
++		.description	= description,
++	};
+ 	struct key_preparsed_payload prep;
+ 	const struct cred *cred = current_cred();
+-	struct key_type *ktype;
++	unsigned long prealloc;
+ 	struct key *keyring, *key = NULL;
+ 	key_ref_t key_ref;
+ 	int ret;
+ 
+ 	/* look up the key type to see if it's one of the registered kernel
+ 	 * types */
+-	ktype = key_type_lookup(type);
+-	if (IS_ERR(ktype)) {
++	index_key.type = key_type_lookup(type);
++	if (IS_ERR(index_key.type)) {
+ 		key_ref = ERR_PTR(-ENODEV);
+ 		goto error;
+ 	}
+ 
+ 	key_ref = ERR_PTR(-EINVAL);
+-	if (!ktype->match || !ktype->instantiate ||
+-	    (!description && !ktype->preparse))
++	if (!index_key.type->match || !index_key.type->instantiate ||
++	    (!index_key.description && !index_key.type->preparse))
+ 		goto error_put_type;
+ 
+ 	keyring = key_ref_to_ptr(keyring_ref);
+@@ -812,21 +813,22 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 	memset(&prep, 0, sizeof(prep));
+ 	prep.data = payload;
+ 	prep.datalen = plen;
+-	prep.quotalen = ktype->def_datalen;
+-	if (ktype->preparse) {
+-		ret = ktype->preparse(&prep);
++	prep.quotalen = index_key.type->def_datalen;
++	if (index_key.type->preparse) {
++		ret = index_key.type->preparse(&prep);
+ 		if (ret < 0) {
+ 			key_ref = ERR_PTR(ret);
+ 			goto error_put_type;
+ 		}
+-		if (!description)
+-			description = prep.description;
++		if (!index_key.description)
++			index_key.description = prep.description;
+ 		key_ref = ERR_PTR(-EINVAL);
+-		if (!description)
++		if (!index_key.description)
+ 			goto error_free_prep;
+ 	}
++	index_key.desc_len = strlen(index_key.description);
+ 
+-	ret = __key_link_begin(keyring, ktype, description, &prealloc);
++	ret = __key_link_begin(keyring, &index_key, &prealloc);
+ 	if (ret < 0) {
+ 		key_ref = ERR_PTR(ret);
+ 		goto error_free_prep;
+@@ -844,9 +846,8 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 	 * key of the same type and description in the destination keyring and
+ 	 * update that instead if possible
+ 	 */
+-	if (ktype->update) {
+-		key_ref = __keyring_search_one(keyring_ref, ktype, description,
+-					       0);
++	if (index_key.type->update) {
++		key_ref = __keyring_search_one(keyring_ref, &index_key, 0);
+ 		if (!IS_ERR(key_ref))
+ 			goto found_matching_key;
+ 	}
+@@ -856,16 +857,17 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 		perm = KEY_POS_VIEW | KEY_POS_SEARCH | KEY_POS_LINK | KEY_POS_SETATTR;
+ 		perm |= KEY_USR_VIEW;
+ 
+-		if (ktype->read)
++		if (index_key.type->read)
+ 			perm |= KEY_POS_READ;
+ 
+-		if (ktype == &key_type_keyring || ktype->update)
++		if (index_key.type == &key_type_keyring ||
++		    index_key.type->update)
+ 			perm |= KEY_POS_WRITE;
+ 	}
+ 
+ 	/* allocate a new key */
+-	key = key_alloc(ktype, description, cred->fsuid, cred->fsgid, cred,
+-			perm, flags);
++	key = key_alloc(index_key.type, index_key.description,
++			cred->fsuid, cred->fsgid, cred, perm, flags);
+ 	if (IS_ERR(key)) {
+ 		key_ref = ERR_CAST(key);
+ 		goto error_link_end;
+@@ -882,12 +884,12 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 	key_ref = make_key_ref(key, is_key_possessed(keyring_ref));
+ 
+ error_link_end:
+-	__key_link_end(keyring, ktype, prealloc);
++	__key_link_end(keyring, &index_key, prealloc);
+ error_free_prep:
+-	if (ktype->preparse)
+-		ktype->free_preparse(&prep);
++	if (index_key.type->preparse)
++		index_key.type->free_preparse(&prep);
+ error_put_type:
+-	key_type_put(ktype);
++	key_type_put(index_key.type);
+ error:
+ 	return key_ref;
+ 
+@@ -895,7 +897,7 @@ error:
+ 	/* we found a matching key, so we're going to try to update it
+ 	 * - we can drop the locks first as we have the key pinned
+ 	 */
+-	__key_link_end(keyring, ktype, prealloc);
++	__key_link_end(keyring, &index_key, prealloc);
+ 
+ 	key_ref = __key_update(key_ref, &prep);
+ 	goto error_free_prep;
+diff --git a/security/keys/keyring.c b/security/keys/keyring.c
+index f784063..c7f59f9 100644
+--- a/security/keys/keyring.c
++++ b/security/keys/keyring.c
+@@ -538,8 +538,7 @@ EXPORT_SYMBOL(keyring_search);
+  * to the returned key reference.
+  */
+ key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+-			       const struct key_type *ktype,
+-			       const char *description,
++			       const struct keyring_index_key *index_key,
+ 			       key_perm_t perm)
+ {
+ 	struct keyring_list *klist;
+@@ -558,9 +557,9 @@ key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+ 		smp_rmb();
+ 		for (loop = 0; loop < nkeys ; loop++) {
+ 			key = rcu_dereference(klist->keys[loop]);
+-			if (key->type == ktype &&
++			if (key->type == index_key->type &&
+ 			    (!key->type->match ||
+-			     key->type->match(key, description)) &&
++			     key->type->match(key, index_key->description)) &&
+ 			    key_permission(make_key_ref(key, possessed),
+ 					   perm) == 0 &&
+ 			    !(key->flags & ((1 << KEY_FLAG_INVALIDATED) |
+@@ -747,8 +746,8 @@ static void keyring_unlink_rcu_disposal(struct rcu_head *rcu)
+ /*
+  * Preallocate memory so that a key can be linked into to a keyring.
+  */
+-int __key_link_begin(struct key *keyring, const struct key_type *type,
+-		     const char *description, unsigned long *_prealloc)
++int __key_link_begin(struct key *keyring, const struct keyring_index_key *index_key,
++		     unsigned long *_prealloc)
+ 	__acquires(&keyring->sem)
+ 	__acquires(&keyring_serialise_link_sem)
+ {
+@@ -759,7 +758,8 @@ int __key_link_begin(struct key *keyring, const struct key_type *type,
+ 	size_t size;
+ 	int loop, lru, ret;
+ 
+-	kenter("%d,%s,%s,", key_serial(keyring), type->name, description);
++	kenter("%d,%s,%s,",
++	       key_serial(keyring), index_key->type->name, index_key->description);
+ 
+ 	if (keyring->type != &key_type_keyring)
+ 		return -ENOTDIR;
+@@ -772,7 +772,7 @@ int __key_link_begin(struct key *keyring, const struct key_type *type,
+ 
+ 	/* serialise link/link calls to prevent parallel calls causing a cycle
+ 	 * when linking two keyring in opposite orders */
+-	if (type == &key_type_keyring)
++	if (index_key->type == &key_type_keyring)
+ 		down_write(&keyring_serialise_link_sem);
+ 
+ 	klist = rcu_dereference_locked_keyring(keyring);
+@@ -784,8 +784,8 @@ int __key_link_begin(struct key *keyring, const struct key_type *type,
+ 		for (loop = klist->nkeys - 1; loop >= 0; loop--) {
+ 			struct key *key = rcu_deref_link_locked(klist, loop,
+ 								keyring);
+-			if (key->type == type &&
+-			    strcmp(key->description, description) == 0) {
++			if (key->type == index_key->type &&
++			    strcmp(key->description, index_key->description) == 0) {
+ 				/* Found a match - we'll replace the link with
+ 				 * one to the new key.  We record the slot
+ 				 * position.
+@@ -865,7 +865,7 @@ error_quota:
+ 	key_payload_reserve(keyring,
+ 			    keyring->datalen - KEYQUOTA_LINK_BYTES);
+ error_sem:
+-	if (type == &key_type_keyring)
++	if (index_key->type == &key_type_keyring)
+ 		up_write(&keyring_serialise_link_sem);
+ error_krsem:
+ 	up_write(&keyring->sem);
+@@ -957,16 +957,17 @@ void __key_link(struct key *keyring, struct key *key,
+  *
+  * Must be called with __key_link_begin() having being called.
+  */
+-void __key_link_end(struct key *keyring, struct key_type *type,
++void __key_link_end(struct key *keyring,
++		    const struct keyring_index_key *index_key,
+ 		    unsigned long prealloc)
+ 	__releases(&keyring->sem)
+ 	__releases(&keyring_serialise_link_sem)
+ {
+-	BUG_ON(type == NULL);
+-	BUG_ON(type->name == NULL);
+-	kenter("%d,%s,%lx", keyring->serial, type->name, prealloc);
++	BUG_ON(index_key->type == NULL);
++	BUG_ON(index_key->type->name == NULL);
++	kenter("%d,%s,%lx", keyring->serial, index_key->type->name, prealloc);
+ 
+-	if (type == &key_type_keyring)
++	if (index_key->type == &key_type_keyring)
+ 		up_write(&keyring_serialise_link_sem);
+ 
+ 	if (prealloc) {
+@@ -1007,12 +1008,12 @@ int key_link(struct key *keyring, struct key *key)
+ 	key_check(keyring);
+ 	key_check(key);
+ 
+-	ret = __key_link_begin(keyring, key->type, key->description, &prealloc);
++	ret = __key_link_begin(keyring, &key->index_key, &prealloc);
+ 	if (ret == 0) {
+ 		ret = __key_link_check_live_key(keyring, key);
+ 		if (ret == 0)
+ 			__key_link(keyring, key, &prealloc);
+-		__key_link_end(keyring, key->type, prealloc);
++		__key_link_end(keyring, &key->index_key, prealloc);
+ 	}
+ 
+ 	return ret;
+diff --git a/security/keys/request_key.c b/security/keys/request_key.c
+index 172115b..586cb79 100644
+--- a/security/keys/request_key.c
++++ b/security/keys/request_key.c
+@@ -352,6 +352,11 @@ static int construct_alloc_key(struct key_type *type,
+ 			       struct key_user *user,
+ 			       struct key **_key)
+ {
++	const struct keyring_index_key index_key = {
++		.type		= type,
++		.description	= description,
++		.desc_len	= strlen(description),
++	};
+ 	const struct cred *cred = current_cred();
+ 	unsigned long prealloc;
+ 	struct key *key;
+@@ -379,8 +384,7 @@ static int construct_alloc_key(struct key_type *type,
+ 	set_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags);
+ 
+ 	if (dest_keyring) {
+-		ret = __key_link_begin(dest_keyring, type, description,
+-				       &prealloc);
++		ret = __key_link_begin(dest_keyring, &index_key, &prealloc);
+ 		if (ret < 0)
+ 			goto link_prealloc_failed;
+ 	}
+@@ -400,7 +404,7 @@ static int construct_alloc_key(struct key_type *type,
+ 
+ 	mutex_unlock(&key_construction_mutex);
+ 	if (dest_keyring)
+-		__key_link_end(dest_keyring, type, prealloc);
++		__key_link_end(dest_keyring, &index_key, prealloc);
+ 	mutex_unlock(&user->cons_lock);
+ 	*_key = key;
+ 	kleave(" = 0 [%d]", key_serial(key));
+@@ -416,7 +420,7 @@ key_already_present:
+ 		ret = __key_link_check_live_key(dest_keyring, key);
+ 		if (ret == 0)
+ 			__key_link(dest_keyring, key, &prealloc);
+-		__key_link_end(dest_keyring, type, prealloc);
++		__key_link_end(dest_keyring, &index_key, prealloc);
+ 		if (ret < 0)
+ 			goto link_check_failed;
+ 	}
+-- 
+1.8.3.1
+
+
+From eca8dad5cd291d2baf2d20372fcb0af9e75e25ea Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:52 +0100
+Subject: [PATCH 05/10] KEYS: Introduce a search context structure
+
+Search functions pass around a bunch of arguments, each of which gets copied
+with each call.  Introduce a search context structure to hold these.
+
+Whilst we're at it, create a search flag that indicates whether the search
+should be directly to the description or whether it should iterate through all
+keys looking for a non-description match.
+
+This will be useful when keyrings use a generic data struct with generic
+routines to manage their content as the search terms can just be passed
+through to the iterator callback function.
+
+Also, for future use, the data to be supplied to the match function is
+separated from the description pointer in the search context.  This makes it
+clear which is being supplied.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ include/linux/key-type.h         |   5 ++
+ security/keys/internal.h         |  40 +++++++------
+ security/keys/keyring.c          |  70 +++++++++++------------
+ security/keys/proc.c             |  17 ++++--
+ security/keys/process_keys.c     | 117 +++++++++++++++++++--------------------
+ security/keys/request_key.c      |  56 +++++++++----------
+ security/keys/request_key_auth.c |  14 +++--
+ security/keys/user_defined.c     |  18 +++---
+ 8 files changed, 179 insertions(+), 158 deletions(-)
+
+diff --git a/include/linux/key-type.h b/include/linux/key-type.h
+index 518a53a..f58737b 100644
+--- a/include/linux/key-type.h
++++ b/include/linux/key-type.h
+@@ -63,6 +63,11 @@ struct key_type {
+ 	 */
+ 	size_t def_datalen;
+ 
++	/* Default key search algorithm. */
++	unsigned def_lookup_type;
++#define KEYRING_SEARCH_LOOKUP_DIRECT	0x0000	/* Direct lookup by description. */
++#define KEYRING_SEARCH_LOOKUP_ITERATE	0x0001	/* Iterative search. */
++
+ 	/* vet a description */
+ 	int (*vet_description)(const char *description);
+ 
+diff --git a/security/keys/internal.h b/security/keys/internal.h
+index 77441dd..f4bf938 100644
+--- a/security/keys/internal.h
++++ b/security/keys/internal.h
+@@ -107,23 +107,31 @@ extern struct key *keyring_search_instkey(struct key *keyring,
+ 
+ typedef int (*key_match_func_t)(const struct key *, const void *);
+ 
++struct keyring_search_context {
++	struct keyring_index_key index_key;
++	const struct cred	*cred;
++	key_match_func_t	match;
++	const void		*match_data;
++	unsigned		flags;
++#define KEYRING_SEARCH_LOOKUP_TYPE	0x0001	/* [as type->def_lookup_type] */
++#define KEYRING_SEARCH_NO_STATE_CHECK	0x0002	/* Skip state checks */
++#define KEYRING_SEARCH_DO_STATE_CHECK	0x0004	/* Override NO_STATE_CHECK */
++#define KEYRING_SEARCH_NO_UPDATE_TIME	0x0008	/* Don't update times */
++#define KEYRING_SEARCH_NO_CHECK_PERM	0x0010	/* Don't check permissions */
++#define KEYRING_SEARCH_DETECT_TOO_DEEP	0x0020	/* Give an error on excessive depth */
++
++	/* Internal stuff */
++	int			skipped_ret;
++	bool			possessed;
++	key_ref_t		result;
++	struct timespec		now;
++};
++
+ extern key_ref_t keyring_search_aux(key_ref_t keyring_ref,
+-				    const struct cred *cred,
+-				    struct key_type *type,
+-				    const void *description,
+-				    key_match_func_t match,
+-				    bool no_state_check);
+-
+-extern key_ref_t search_my_process_keyrings(struct key_type *type,
+-					    const void *description,
+-					    key_match_func_t match,
+-					    bool no_state_check,
+-					    const struct cred *cred);
+-extern key_ref_t search_process_keyrings(struct key_type *type,
+-					 const void *description,
+-					 key_match_func_t match,
+-					 bool no_state_check,
+-					 const struct cred *cred);
++				    struct keyring_search_context *ctx);
++
++extern key_ref_t search_my_process_keyrings(struct keyring_search_context *ctx);
++extern key_ref_t search_process_keyrings(struct keyring_search_context *ctx);
+ 
+ extern struct key *find_keyring_by_name(const char *name, bool skip_perm_check);
+ 
+diff --git a/security/keys/keyring.c b/security/keys/keyring.c
+index c7f59f9..b42f2d4 100644
+--- a/security/keys/keyring.c
++++ b/security/keys/keyring.c
+@@ -280,11 +280,7 @@ EXPORT_SYMBOL(keyring_alloc);
+ /**
+  * keyring_search_aux - Search a keyring tree for a key matching some criteria
+  * @keyring_ref: A pointer to the keyring with possession indicator.
+- * @cred: The credentials to use for permissions checks.
+- * @type: The type of key to search for.
+- * @description: Parameter for @match.
+- * @match: Function to rule on whether or not a key is the one required.
+- * @no_state_check: Don't check if a matching key is bad
++ * @ctx: The keyring search context.
+  *
+  * Search the supplied keyring tree for a key that matches the criteria given.
+  * The root keyring and any linked keyrings must grant Search permission to the
+@@ -314,11 +310,7 @@ EXPORT_SYMBOL(keyring_alloc);
+  * @keyring_ref is propagated to the returned key reference.
+  */
+ key_ref_t keyring_search_aux(key_ref_t keyring_ref,
+-			     const struct cred *cred,
+-			     struct key_type *type,
+-			     const void *description,
+-			     key_match_func_t match,
+-			     bool no_state_check)
++			     struct keyring_search_context *ctx)
+ {
+ 	struct {
+ 		/* Need a separate keylist pointer for RCU purposes */
+@@ -328,20 +320,18 @@ key_ref_t keyring_search_aux(key_ref_t keyring_ref,
+ 	} stack[KEYRING_SEARCH_MAX_DEPTH];
+ 
+ 	struct keyring_list *keylist;
+-	struct timespec now;
+ 	unsigned long kflags;
+ 	struct key *keyring, *key;
+ 	key_ref_t key_ref;
+-	bool possessed;
+ 	long err;
+ 	int sp, nkeys, kix;
+ 
+ 	keyring = key_ref_to_ptr(keyring_ref);
+-	possessed = is_key_possessed(keyring_ref);
++	ctx->possessed = is_key_possessed(keyring_ref);
+ 	key_check(keyring);
+ 
+ 	/* top keyring must have search permission to begin the search */
+-	err = key_task_permission(keyring_ref, cred, KEY_SEARCH);
++	err = key_task_permission(keyring_ref, ctx->cred, KEY_SEARCH);
+ 	if (err < 0) {
+ 		key_ref = ERR_PTR(err);
+ 		goto error;
+@@ -353,7 +343,7 @@ key_ref_t keyring_search_aux(key_ref_t keyring_ref,
+ 
+ 	rcu_read_lock();
+ 
+-	now = current_kernel_time();
++	ctx->now = current_kernel_time();
+ 	err = -EAGAIN;
+ 	sp = 0;
+ 
+@@ -361,16 +351,17 @@ key_ref_t keyring_search_aux(key_ref_t keyring_ref,
+ 	 * are looking for */
+ 	key_ref = ERR_PTR(-EAGAIN);
+ 	kflags = keyring->flags;
+-	if (keyring->type == type && match(keyring, description)) {
++	if (keyring->type == ctx->index_key.type &&
++	    ctx->match(keyring, ctx->match_data)) {
+ 		key = keyring;
+-		if (no_state_check)
++		if (ctx->flags & KEYRING_SEARCH_NO_STATE_CHECK)
+ 			goto found;
+ 
+ 		/* check it isn't negative and hasn't expired or been
+ 		 * revoked */
+ 		if (kflags & (1 << KEY_FLAG_REVOKED))
+ 			goto error_2;
+-		if (key->expiry && now.tv_sec >= key->expiry)
++		if (key->expiry && ctx->now.tv_sec >= key->expiry)
+ 			goto error_2;
+ 		key_ref = ERR_PTR(key->type_data.reject_error);
+ 		if (kflags & (1 << KEY_FLAG_NEGATIVE))
+@@ -384,7 +375,7 @@ key_ref_t keyring_search_aux(key_ref_t keyring_ref,
+ 	if (kflags & ((1 << KEY_FLAG_INVALIDATED) |
+ 		      (1 << KEY_FLAG_REVOKED) |
+ 		      (1 << KEY_FLAG_NEGATIVE)) ||
+-	    (keyring->expiry && now.tv_sec >= keyring->expiry))
++	    (keyring->expiry && ctx->now.tv_sec >= keyring->expiry))
+ 		goto error_2;
+ 
+ 	/* start processing a new keyring */
+@@ -406,29 +397,29 @@ descend:
+ 		kflags = key->flags;
+ 
+ 		/* ignore keys not of this type */
+-		if (key->type != type)
++		if (key->type != ctx->index_key.type)
+ 			continue;
+ 
+ 		/* skip invalidated, revoked and expired keys */
+-		if (!no_state_check) {
++		if (!(ctx->flags & KEYRING_SEARCH_NO_STATE_CHECK)) {
+ 			if (kflags & ((1 << KEY_FLAG_INVALIDATED) |
+ 				      (1 << KEY_FLAG_REVOKED)))
+ 				continue;
+ 
+-			if (key->expiry && now.tv_sec >= key->expiry)
++			if (key->expiry && ctx->now.tv_sec >= key->expiry)
+ 				continue;
+ 		}
+ 
+ 		/* keys that don't match */
+-		if (!match(key, description))
++		if (!ctx->match(key, ctx->match_data))
+ 			continue;
+ 
+ 		/* key must have search permissions */
+-		if (key_task_permission(make_key_ref(key, possessed),
+-					cred, KEY_SEARCH) < 0)
++		if (key_task_permission(make_key_ref(key, ctx->possessed),
++					ctx->cred, KEY_SEARCH) < 0)
+ 			continue;
+ 
+-		if (no_state_check)
++		if (ctx->flags & KEYRING_SEARCH_NO_STATE_CHECK)
+ 			goto found;
+ 
+ 		/* we set a different error code if we pass a negative key */
+@@ -456,8 +447,8 @@ ascend:
+ 		if (sp >= KEYRING_SEARCH_MAX_DEPTH)
+ 			continue;
+ 
+-		if (key_task_permission(make_key_ref(key, possessed),
+-					cred, KEY_SEARCH) < 0)
++		if (key_task_permission(make_key_ref(key, ctx->possessed),
++					ctx->cred, KEY_SEARCH) < 0)
+ 			continue;
+ 
+ 		/* stack the current position */
+@@ -489,12 +480,12 @@ not_this_keyring:
+ 	/* we found a viable match */
+ found:
+ 	atomic_inc(&key->usage);
+-	key->last_used_at = now.tv_sec;
+-	keyring->last_used_at = now.tv_sec;
++	key->last_used_at = ctx->now.tv_sec;
++	keyring->last_used_at = ctx->now.tv_sec;
+ 	while (sp > 0)
+-		stack[--sp].keyring->last_used_at = now.tv_sec;
++		stack[--sp].keyring->last_used_at = ctx->now.tv_sec;
+ 	key_check(key);
+-	key_ref = make_key_ref(key, possessed);
++	key_ref = make_key_ref(key, ctx->possessed);
+ error_2:
+ 	rcu_read_unlock();
+ error:
+@@ -514,11 +505,20 @@ key_ref_t keyring_search(key_ref_t keyring,
+ 			 struct key_type *type,
+ 			 const char *description)
+ {
+-	if (!type->match)
++	struct keyring_search_context ctx = {
++		.index_key.type		= type,
++		.index_key.description	= description,
++		.cred			= current_cred(),
++		.match			= type->match,
++		.match_data		= description,
++		.flags			= (type->def_lookup_type |
++					   KEYRING_SEARCH_DO_STATE_CHECK),
++	};
++
++	if (!ctx.match)
+ 		return ERR_PTR(-ENOKEY);
+ 
+-	return keyring_search_aux(keyring, current->cred,
+-				  type, description, type->match, false);
++	return keyring_search_aux(keyring, &ctx);
+ }
+ EXPORT_SYMBOL(keyring_search);
+ 
+diff --git a/security/keys/proc.c b/security/keys/proc.c
+index 217b685..88e9a46 100644
+--- a/security/keys/proc.c
++++ b/security/keys/proc.c
+@@ -182,7 +182,6 @@ static void proc_keys_stop(struct seq_file *p, void *v)
+ 
+ static int proc_keys_show(struct seq_file *m, void *v)
+ {
+-	const struct cred *cred = current_cred();
+ 	struct rb_node *_p = v;
+ 	struct key *key = rb_entry(_p, struct key, serial_node);
+ 	struct timespec now;
+@@ -191,15 +190,23 @@ static int proc_keys_show(struct seq_file *m, void *v)
+ 	char xbuf[12];
+ 	int rc;
+ 
++	struct keyring_search_context ctx = {
++		.index_key.type		= key->type,
++		.index_key.description	= key->description,
++		.cred			= current_cred(),
++		.match			= lookup_user_key_possessed,
++		.match_data		= key,
++		.flags			= (KEYRING_SEARCH_NO_STATE_CHECK |
++					   KEYRING_SEARCH_LOOKUP_DIRECT),
++	};
++
+ 	key_ref = make_key_ref(key, 0);
+ 
+ 	/* determine if the key is possessed by this process (a test we can
+ 	 * skip if the key does not indicate the possessor can view it
+ 	 */
+ 	if (key->perm & KEY_POS_VIEW) {
+-		skey_ref = search_my_process_keyrings(key->type, key,
+-						      lookup_user_key_possessed,
+-						      true, cred);
++		skey_ref = search_my_process_keyrings(&ctx);
+ 		if (!IS_ERR(skey_ref)) {
+ 			key_ref_put(skey_ref);
+ 			key_ref = make_key_ref(key, 1);
+@@ -211,7 +218,7 @@ static int proc_keys_show(struct seq_file *m, void *v)
+ 	 * - the caller holds a spinlock, and thus the RCU read lock, making our
+ 	 *   access to __current_cred() safe
+ 	 */
+-	rc = key_task_permission(key_ref, cred, KEY_VIEW);
++	rc = key_task_permission(key_ref, ctx.cred, KEY_VIEW);
+ 	if (rc < 0)
+ 		return 0;
+ 
+diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c
+index a3410d6..e68a3e0 100644
+--- a/security/keys/process_keys.c
++++ b/security/keys/process_keys.c
+@@ -319,11 +319,7 @@ void key_fsgid_changed(struct task_struct *tsk)
+  * In the case of a successful return, the possession attribute is set on the
+  * returned key reference.
+  */
+-key_ref_t search_my_process_keyrings(struct key_type *type,
+-				     const void *description,
+-				     key_match_func_t match,
+-				     bool no_state_check,
+-				     const struct cred *cred)
++key_ref_t search_my_process_keyrings(struct keyring_search_context *ctx)
+ {
+ 	key_ref_t key_ref, ret, err;
+ 
+@@ -339,10 +335,9 @@ key_ref_t search_my_process_keyrings(struct key_type *type,
+ 	err = ERR_PTR(-EAGAIN);
+ 
+ 	/* search the thread keyring first */
+-	if (cred->thread_keyring) {
++	if (ctx->cred->thread_keyring) {
+ 		key_ref = keyring_search_aux(
+-			make_key_ref(cred->thread_keyring, 1),
+-			cred, type, description, match, no_state_check);
++			make_key_ref(ctx->cred->thread_keyring, 1), ctx);
+ 		if (!IS_ERR(key_ref))
+ 			goto found;
+ 
+@@ -358,10 +353,9 @@ key_ref_t search_my_process_keyrings(struct key_type *type,
+ 	}
+ 
+ 	/* search the process keyring second */
+-	if (cred->process_keyring) {
++	if (ctx->cred->process_keyring) {
+ 		key_ref = keyring_search_aux(
+-			make_key_ref(cred->process_keyring, 1),
+-			cred, type, description, match, no_state_check);
++			make_key_ref(ctx->cred->process_keyring, 1), ctx);
+ 		if (!IS_ERR(key_ref))
+ 			goto found;
+ 
+@@ -379,11 +373,11 @@ key_ref_t search_my_process_keyrings(struct key_type *type,
+ 	}
+ 
+ 	/* search the session keyring */
+-	if (cred->session_keyring) {
++	if (ctx->cred->session_keyring) {
+ 		rcu_read_lock();
+ 		key_ref = keyring_search_aux(
+-			make_key_ref(rcu_dereference(cred->session_keyring), 1),
+-			cred, type, description, match, no_state_check);
++			make_key_ref(rcu_dereference(ctx->cred->session_keyring), 1),
++			ctx);
+ 		rcu_read_unlock();
+ 
+ 		if (!IS_ERR(key_ref))
+@@ -402,10 +396,10 @@ key_ref_t search_my_process_keyrings(struct key_type *type,
+ 		}
+ 	}
+ 	/* or search the user-session keyring */
+-	else if (cred->user->session_keyring) {
++	else if (ctx->cred->user->session_keyring) {
+ 		key_ref = keyring_search_aux(
+-			make_key_ref(cred->user->session_keyring, 1),
+-			cred, type, description, match, no_state_check);
++			make_key_ref(ctx->cred->user->session_keyring, 1),
++			ctx);
+ 		if (!IS_ERR(key_ref))
+ 			goto found;
+ 
+@@ -437,19 +431,14 @@ found:
+  *
+  * Return same as search_my_process_keyrings().
+  */
+-key_ref_t search_process_keyrings(struct key_type *type,
+-				  const void *description,
+-				  key_match_func_t match,
+-				  bool no_state_check,
+-				  const struct cred *cred)
++key_ref_t search_process_keyrings(struct keyring_search_context *ctx)
+ {
+ 	struct request_key_auth *rka;
+ 	key_ref_t key_ref, ret = ERR_PTR(-EACCES), err;
+ 
+ 	might_sleep();
+ 
+-	key_ref = search_my_process_keyrings(type, description, match,
+-					     no_state_check, cred);
++	key_ref = search_my_process_keyrings(ctx);
+ 	if (!IS_ERR(key_ref))
+ 		goto found;
+ 	err = key_ref;
+@@ -458,19 +447,21 @@ key_ref_t search_process_keyrings(struct key_type *type,
+ 	 * search the keyrings of the process mentioned there
+ 	 * - we don't permit access to request_key auth keys via this method
+ 	 */
+-	if (cred->request_key_auth &&
+-	    cred == current_cred() &&
+-	    type != &key_type_request_key_auth
++	if (ctx->cred->request_key_auth &&
++	    ctx->cred == current_cred() &&
++	    ctx->index_key.type != &key_type_request_key_auth
+ 	    ) {
++		const struct cred *cred = ctx->cred;
++
+ 		/* defend against the auth key being revoked */
+ 		down_read(&cred->request_key_auth->sem);
+ 
+-		if (key_validate(cred->request_key_auth) == 0) {
+-			rka = cred->request_key_auth->payload.data;
++		if (key_validate(ctx->cred->request_key_auth) == 0) {
++			rka = ctx->cred->request_key_auth->payload.data;
+ 
+-			key_ref = search_process_keyrings(type, description,
+-							  match, no_state_check,
+-							  rka->cred);
++			ctx->cred = rka->cred;
++			key_ref = search_process_keyrings(ctx);
++			ctx->cred = cred;
+ 
+ 			up_read(&cred->request_key_auth->sem);
+ 
+@@ -524,19 +515,23 @@ int lookup_user_key_possessed(const struct key *key, const void *target)
+ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags,
+ 			  key_perm_t perm)
+ {
++	struct keyring_search_context ctx = {
++		.match	= lookup_user_key_possessed,
++		.flags	= (KEYRING_SEARCH_NO_STATE_CHECK |
++			   KEYRING_SEARCH_LOOKUP_DIRECT),
++	};
+ 	struct request_key_auth *rka;
+-	const struct cred *cred;
+ 	struct key *key;
+ 	key_ref_t key_ref, skey_ref;
+ 	int ret;
+ 
+ try_again:
+-	cred = get_current_cred();
++	ctx.cred = get_current_cred();
+ 	key_ref = ERR_PTR(-ENOKEY);
+ 
+ 	switch (id) {
+ 	case KEY_SPEC_THREAD_KEYRING:
+-		if (!cred->thread_keyring) {
++		if (!ctx.cred->thread_keyring) {
+ 			if (!(lflags & KEY_LOOKUP_CREATE))
+ 				goto error;
+ 
+@@ -548,13 +543,13 @@ try_again:
+ 			goto reget_creds;
+ 		}
+ 
+-		key = cred->thread_keyring;
++		key = ctx.cred->thread_keyring;
+ 		atomic_inc(&key->usage);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+ 	case KEY_SPEC_PROCESS_KEYRING:
+-		if (!cred->process_keyring) {
++		if (!ctx.cred->process_keyring) {
+ 			if (!(lflags & KEY_LOOKUP_CREATE))
+ 				goto error;
+ 
+@@ -566,13 +561,13 @@ try_again:
+ 			goto reget_creds;
+ 		}
+ 
+-		key = cred->process_keyring;
++		key = ctx.cred->process_keyring;
+ 		atomic_inc(&key->usage);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+ 	case KEY_SPEC_SESSION_KEYRING:
+-		if (!cred->session_keyring) {
++		if (!ctx.cred->session_keyring) {
+ 			/* always install a session keyring upon access if one
+ 			 * doesn't exist yet */
+ 			ret = install_user_keyrings();
+@@ -582,13 +577,13 @@ try_again:
+ 				ret = join_session_keyring(NULL);
+ 			else
+ 				ret = install_session_keyring(
+-					cred->user->session_keyring);
++					ctx.cred->user->session_keyring);
+ 
+ 			if (ret < 0)
+ 				goto error;
+ 			goto reget_creds;
+-		} else if (cred->session_keyring ==
+-			   cred->user->session_keyring &&
++		} else if (ctx.cred->session_keyring ==
++			   ctx.cred->user->session_keyring &&
+ 			   lflags & KEY_LOOKUP_CREATE) {
+ 			ret = join_session_keyring(NULL);
+ 			if (ret < 0)
+@@ -597,32 +592,32 @@ try_again:
+ 		}
+ 
+ 		rcu_read_lock();
+-		key = rcu_dereference(cred->session_keyring);
++		key = rcu_dereference(ctx.cred->session_keyring);
+ 		atomic_inc(&key->usage);
+ 		rcu_read_unlock();
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+ 	case KEY_SPEC_USER_KEYRING:
+-		if (!cred->user->uid_keyring) {
++		if (!ctx.cred->user->uid_keyring) {
+ 			ret = install_user_keyrings();
+ 			if (ret < 0)
+ 				goto error;
+ 		}
+ 
+-		key = cred->user->uid_keyring;
++		key = ctx.cred->user->uid_keyring;
+ 		atomic_inc(&key->usage);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+ 	case KEY_SPEC_USER_SESSION_KEYRING:
+-		if (!cred->user->session_keyring) {
++		if (!ctx.cred->user->session_keyring) {
+ 			ret = install_user_keyrings();
+ 			if (ret < 0)
+ 				goto error;
+ 		}
+ 
+-		key = cred->user->session_keyring;
++		key = ctx.cred->user->session_keyring;
+ 		atomic_inc(&key->usage);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+@@ -633,7 +628,7 @@ try_again:
+ 		goto error;
+ 
+ 	case KEY_SPEC_REQKEY_AUTH_KEY:
+-		key = cred->request_key_auth;
++		key = ctx.cred->request_key_auth;
+ 		if (!key)
+ 			goto error;
+ 
+@@ -642,20 +637,20 @@ try_again:
+ 		break;
+ 
+ 	case KEY_SPEC_REQUESTOR_KEYRING:
+-		if (!cred->request_key_auth)
++		if (!ctx.cred->request_key_auth)
+ 			goto error;
+ 
+-		down_read(&cred->request_key_auth->sem);
++		down_read(&ctx.cred->request_key_auth->sem);
+ 		if (test_bit(KEY_FLAG_REVOKED,
+-			     &cred->request_key_auth->flags)) {
++			     &ctx.cred->request_key_auth->flags)) {
+ 			key_ref = ERR_PTR(-EKEYREVOKED);
+ 			key = NULL;
+ 		} else {
+-			rka = cred->request_key_auth->payload.data;
++			rka = ctx.cred->request_key_auth->payload.data;
+ 			key = rka->dest_keyring;
+ 			atomic_inc(&key->usage);
+ 		}
+-		up_read(&cred->request_key_auth->sem);
++		up_read(&ctx.cred->request_key_auth->sem);
+ 		if (!key)
+ 			goto error;
+ 		key_ref = make_key_ref(key, 1);
+@@ -675,9 +670,13 @@ try_again:
+ 		key_ref = make_key_ref(key, 0);
+ 
+ 		/* check to see if we possess the key */
+-		skey_ref = search_process_keyrings(key->type, key,
+-						   lookup_user_key_possessed,
+-						   true, cred);
++		ctx.index_key.type		= key->type;
++		ctx.index_key.description	= key->description;
++		ctx.index_key.desc_len		= strlen(key->description);
++		ctx.match_data			= key;
++		kdebug("check possessed");
++		skey_ref = search_process_keyrings(&ctx);
++		kdebug("possessed=%p", skey_ref);
+ 
+ 		if (!IS_ERR(skey_ref)) {
+ 			key_put(key);
+@@ -717,14 +716,14 @@ try_again:
+ 		goto invalid_key;
+ 
+ 	/* check the permissions */
+-	ret = key_task_permission(key_ref, cred, perm);
++	ret = key_task_permission(key_ref, ctx.cred, perm);
+ 	if (ret < 0)
+ 		goto invalid_key;
+ 
+ 	key->last_used_at = current_kernel_time().tv_sec;
+ 
+ error:
+-	put_cred(cred);
++	put_cred(ctx.cred);
+ 	return key_ref;
+ 
+ invalid_key:
+@@ -735,7 +734,7 @@ invalid_key:
+ 	/* if we attempted to install a keyring, then it may have caused new
+ 	 * creds to be installed */
+ reget_creds:
+-	put_cred(cred);
++	put_cred(ctx.cred);
+ 	goto try_again;
+ }
+ 
+diff --git a/security/keys/request_key.c b/security/keys/request_key.c
+index 586cb79..ab75df4 100644
+--- a/security/keys/request_key.c
++++ b/security/keys/request_key.c
+@@ -345,38 +345,34 @@ static void construct_get_dest_keyring(struct key **_dest_keyring)
+  * May return a key that's already under construction instead if there was a
+  * race between two thread calling request_key().
+  */
+-static int construct_alloc_key(struct key_type *type,
+-			       const char *description,
++static int construct_alloc_key(struct keyring_search_context *ctx,
+ 			       struct key *dest_keyring,
+ 			       unsigned long flags,
+ 			       struct key_user *user,
+ 			       struct key **_key)
+ {
+-	const struct keyring_index_key index_key = {
+-		.type		= type,
+-		.description	= description,
+-		.desc_len	= strlen(description),
+-	};
+-	const struct cred *cred = current_cred();
+ 	unsigned long prealloc;
+ 	struct key *key;
+ 	key_perm_t perm;
+ 	key_ref_t key_ref;
+ 	int ret;
+ 
+-	kenter("%s,%s,,,", type->name, description);
++	kenter("%s,%s,,,",
++	       ctx->index_key.type->name, ctx->index_key.description);
+ 
+ 	*_key = NULL;
+ 	mutex_lock(&user->cons_lock);
+ 
+ 	perm = KEY_POS_VIEW | KEY_POS_SEARCH | KEY_POS_LINK | KEY_POS_SETATTR;
+ 	perm |= KEY_USR_VIEW;
+-	if (type->read)
++	if (ctx->index_key.type->read)
+ 		perm |= KEY_POS_READ;
+-	if (type == &key_type_keyring || type->update)
++	if (ctx->index_key.type == &key_type_keyring ||
++	    ctx->index_key.type->update)
+ 		perm |= KEY_POS_WRITE;
+ 
+-	key = key_alloc(type, description, cred->fsuid, cred->fsgid, cred,
++	key = key_alloc(ctx->index_key.type, ctx->index_key.description,
++			ctx->cred->fsuid, ctx->cred->fsgid, ctx->cred,
+ 			perm, flags);
+ 	if (IS_ERR(key))
+ 		goto alloc_failed;
+@@ -384,7 +380,7 @@ static int construct_alloc_key(struct key_type *type,
+ 	set_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags);
+ 
+ 	if (dest_keyring) {
+-		ret = __key_link_begin(dest_keyring, &index_key, &prealloc);
++		ret = __key_link_begin(dest_keyring, &ctx->index_key, &prealloc);
+ 		if (ret < 0)
+ 			goto link_prealloc_failed;
+ 	}
+@@ -394,8 +390,7 @@ static int construct_alloc_key(struct key_type *type,
+ 	 * waited for locks */
+ 	mutex_lock(&key_construction_mutex);
+ 
+-	key_ref = search_process_keyrings(type, description, type->match,
+-					  false, cred);
++	key_ref = search_process_keyrings(ctx);
+ 	if (!IS_ERR(key_ref))
+ 		goto key_already_present;
+ 
+@@ -404,7 +399,7 @@ static int construct_alloc_key(struct key_type *type,
+ 
+ 	mutex_unlock(&key_construction_mutex);
+ 	if (dest_keyring)
+-		__key_link_end(dest_keyring, &index_key, prealloc);
++		__key_link_end(dest_keyring, &ctx->index_key, prealloc);
+ 	mutex_unlock(&user->cons_lock);
+ 	*_key = key;
+ 	kleave(" = 0 [%d]", key_serial(key));
+@@ -420,7 +415,7 @@ key_already_present:
+ 		ret = __key_link_check_live_key(dest_keyring, key);
+ 		if (ret == 0)
+ 			__key_link(dest_keyring, key, &prealloc);
+-		__key_link_end(dest_keyring, &index_key, prealloc);
++		__key_link_end(dest_keyring, &ctx->index_key, prealloc);
+ 		if (ret < 0)
+ 			goto link_check_failed;
+ 	}
+@@ -449,8 +444,7 @@ alloc_failed:
+ /*
+  * Commence key construction.
+  */
+-static struct key *construct_key_and_link(struct key_type *type,
+-					  const char *description,
++static struct key *construct_key_and_link(struct keyring_search_context *ctx,
+ 					  const char *callout_info,
+ 					  size_t callout_len,
+ 					  void *aux,
+@@ -469,8 +463,7 @@ static struct key *construct_key_and_link(struct key_type *type,
+ 
+ 	construct_get_dest_keyring(&dest_keyring);
+ 
+-	ret = construct_alloc_key(type, description, dest_keyring, flags, user,
+-				  &key);
++	ret = construct_alloc_key(ctx, dest_keyring, flags, user, &key);
+ 	key_user_put(user);
+ 
+ 	if (ret == 0) {
+@@ -534,18 +527,24 @@ struct key *request_key_and_link(struct key_type *type,
+ 				 struct key *dest_keyring,
+ 				 unsigned long flags)
+ {
+-	const struct cred *cred = current_cred();
++	struct keyring_search_context ctx = {
++		.index_key.type		= type,
++		.index_key.description	= description,
++		.cred			= current_cred(),
++		.match			= type->match,
++		.match_data		= description,
++		.flags			= KEYRING_SEARCH_LOOKUP_DIRECT,
++	};
+ 	struct key *key;
+ 	key_ref_t key_ref;
+ 	int ret;
+ 
+ 	kenter("%s,%s,%p,%zu,%p,%p,%lx",
+-	       type->name, description, callout_info, callout_len, aux,
+-	       dest_keyring, flags);
++	       ctx.index_key.type->name, ctx.index_key.description,
++	       callout_info, callout_len, aux, dest_keyring, flags);
+ 
+ 	/* search all the process keyrings for a key */
+-	key_ref = search_process_keyrings(type, description, type->match,
+-					  false, cred);
++	key_ref = search_process_keyrings(&ctx);
+ 
+ 	if (!IS_ERR(key_ref)) {
+ 		key = key_ref_to_ptr(key_ref);
+@@ -568,9 +567,8 @@ struct key *request_key_and_link(struct key_type *type,
+ 		if (!callout_info)
+ 			goto error;
+ 
+-		key = construct_key_and_link(type, description, callout_info,
+-					     callout_len, aux, dest_keyring,
+-					     flags);
++		key = construct_key_and_link(&ctx, callout_info, callout_len,
++					     aux, dest_keyring, flags);
+ 	}
+ 
+ error:
+diff --git a/security/keys/request_key_auth.c b/security/keys/request_key_auth.c
+index 92077de..8d09852 100644
+--- a/security/keys/request_key_auth.c
++++ b/security/keys/request_key_auth.c
+@@ -239,15 +239,17 @@ static int key_get_instantiation_authkey_match(const struct key *key,
+  */
+ struct key *key_get_instantiation_authkey(key_serial_t target_id)
+ {
+-	const struct cred *cred = current_cred();
++	struct keyring_search_context ctx = {
++		.index_key.type		= &key_type_request_key_auth,
++		.cred			= current_cred(),
++		.match			= key_get_instantiation_authkey_match,
++		.match_data		= (void *)(unsigned long)target_id,
++		.flags			= KEYRING_SEARCH_LOOKUP_DIRECT,
++	};
+ 	struct key *authkey;
+ 	key_ref_t authkey_ref;
+ 
+-	authkey_ref = search_process_keyrings(
+-		&key_type_request_key_auth,
+-		(void *) (unsigned long) target_id,
+-		key_get_instantiation_authkey_match,
+-		false, cred);
++	authkey_ref = search_process_keyrings(&ctx);
+ 
+ 	if (IS_ERR(authkey_ref)) {
+ 		authkey = ERR_CAST(authkey_ref);
+diff --git a/security/keys/user_defined.c b/security/keys/user_defined.c
+index 55dc889..faa2cae 100644
+--- a/security/keys/user_defined.c
++++ b/security/keys/user_defined.c
+@@ -25,14 +25,15 @@ static int logon_vet_description(const char *desc);
+  * arbitrary blob of data as the payload
+  */
+ struct key_type key_type_user = {
+-	.name		= "user",
+-	.instantiate	= user_instantiate,
+-	.update		= user_update,
+-	.match		= user_match,
+-	.revoke		= user_revoke,
+-	.destroy	= user_destroy,
+-	.describe	= user_describe,
+-	.read		= user_read,
++	.name			= "user",
++	.def_lookup_type	= KEYRING_SEARCH_LOOKUP_DIRECT,
++	.instantiate		= user_instantiate,
++	.update			= user_update,
++	.match			= user_match,
++	.revoke			= user_revoke,
++	.destroy		= user_destroy,
++	.describe		= user_describe,
++	.read			= user_read,
+ };
+ 
+ EXPORT_SYMBOL_GPL(key_type_user);
+@@ -45,6 +46,7 @@ EXPORT_SYMBOL_GPL(key_type_user);
+  */
+ struct key_type key_type_logon = {
+ 	.name			= "logon",
++	.def_lookup_type	= KEYRING_SEARCH_LOOKUP_DIRECT,
+ 	.instantiate		= user_instantiate,
+ 	.update			= user_update,
+ 	.match			= user_match,
+-- 
+1.8.3.1
+
+
+From 4dffed72b92a305bcdbb73b719570d8f4ec53f46 Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:52 +0100
+Subject: [PATCH 06/10] KEYS: Search for auth-key by name rather than target
+ key ID
+
+Search for auth-key by name rather than by target key ID as, in a future
+patch, we'll by searching directly by index key in preference to iteration
+over all keys.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ security/keys/request_key_auth.c | 21 +++++++--------------
+ 1 file changed, 7 insertions(+), 14 deletions(-)
+
+diff --git a/security/keys/request_key_auth.c b/security/keys/request_key_auth.c
+index 8d09852..7495a93 100644
+--- a/security/keys/request_key_auth.c
++++ b/security/keys/request_key_auth.c
+@@ -18,6 +18,7 @@
+ #include <linux/slab.h>
+ #include <asm/uaccess.h>
+ #include "internal.h"
++#include <keys/user-type.h>
+ 
+ static int request_key_auth_instantiate(struct key *,
+ 					struct key_preparsed_payload *);
+@@ -222,33 +223,25 @@ error_alloc:
+ }
+ 
+ /*
+- * See if an authorisation key is associated with a particular key.
+- */
+-static int key_get_instantiation_authkey_match(const struct key *key,
+-					       const void *_id)
+-{
+-	struct request_key_auth *rka = key->payload.data;
+-	key_serial_t id = (key_serial_t)(unsigned long) _id;
+-
+-	return rka->target_key->serial == id;
+-}
+-
+-/*
+  * Search the current process's keyrings for the authorisation key for
+  * instantiation of a key.
+  */
+ struct key *key_get_instantiation_authkey(key_serial_t target_id)
+ {
++	char description[16];
+ 	struct keyring_search_context ctx = {
+ 		.index_key.type		= &key_type_request_key_auth,
++		.index_key.description	= description,
+ 		.cred			= current_cred(),
+-		.match			= key_get_instantiation_authkey_match,
+-		.match_data		= (void *)(unsigned long)target_id,
++		.match			= user_match,
++		.match_data		= description,
+ 		.flags			= KEYRING_SEARCH_LOOKUP_DIRECT,
+ 	};
+ 	struct key *authkey;
+ 	key_ref_t authkey_ref;
+ 
++	sprintf(description, "%x", target_id);
++
+ 	authkey_ref = search_process_keyrings(&ctx);
+ 
+ 	if (IS_ERR(authkey_ref)) {
+-- 
+1.8.3.1
+
+
+From 5f3c76b0923620ddd5294270ac478819f06f21d1 Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:53 +0100
+Subject: [PATCH 07/10] KEYS: Define a __key_get() wrapper to use rather than
+ atomic_inc()
+
+Define a __key_get() wrapper to use rather than atomic_inc() on the key usage
+count as this makes it easier to hook in refcount error debugging.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ Documentation/security/keys.txt | 13 ++++++++-----
+ include/linux/key.h             | 10 +++++++---
+ security/keys/key.c             |  2 +-
+ security/keys/keyring.c         |  6 +++---
+ security/keys/process_keys.c    | 16 ++++++++--------
+ 5 files changed, 27 insertions(+), 20 deletions(-)
+
+diff --git a/Documentation/security/keys.txt b/Documentation/security/keys.txt
+index 9ede670..a4c33f1 100644
+--- a/Documentation/security/keys.txt
++++ b/Documentation/security/keys.txt
+@@ -960,14 +960,17 @@ payload contents" for more information.
+     the argument will not be parsed.
+ 
+ 
+-(*) Extra references can be made to a key by calling the following function:
++(*) Extra references can be made to a key by calling one of the following
++    functions:
+ 
++	struct key *__key_get(struct key *key);
+ 	struct key *key_get(struct key *key);
+ 
+-    These need to be disposed of by calling key_put() when they've been
+-    finished with. The key pointer passed in will be returned. If the pointer
+-    is NULL or CONFIG_KEYS is not set then the key will not be dereferenced and
+-    no increment will take place.
++    Keys so references will need to be disposed of by calling key_put() when
++    they've been finished with.  The key pointer passed in will be returned.
++
++    In the case of key_get(), if the pointer is NULL or CONFIG_KEYS is not set
++    then the key will not be dereferenced and no increment will take place.
+ 
+ 
+ (*) A key's serial number can be obtained by calling:
+diff --git a/include/linux/key.h b/include/linux/key.h
+index d573e82..ef596c7 100644
+--- a/include/linux/key.h
++++ b/include/linux/key.h
+@@ -219,13 +219,17 @@ extern void key_revoke(struct key *key);
+ extern void key_invalidate(struct key *key);
+ extern void key_put(struct key *key);
+ 
+-static inline struct key *key_get(struct key *key)
++static inline struct key *__key_get(struct key *key)
+ {
+-	if (key)
+-		atomic_inc(&key->usage);
++	atomic_inc(&key->usage);
+ 	return key;
+ }
+ 
++static inline struct key *key_get(struct key *key)
++{
++	return key ? __key_get(key) : key;
++}
++
+ static inline void key_ref_put(key_ref_t key_ref)
+ {
+ 	key_put(key_ref_to_ptr(key_ref));
+diff --git a/security/keys/key.c b/security/keys/key.c
+index 7e6bc39..1e23cc2 100644
+--- a/security/keys/key.c
++++ b/security/keys/key.c
+@@ -644,7 +644,7 @@ found:
+ 	/* this races with key_put(), but that doesn't matter since key_put()
+ 	 * doesn't actually change the key
+ 	 */
+-	atomic_inc(&key->usage);
++	__key_get(key);
+ 
+ error:
+ 	spin_unlock(&key_serial_lock);
+diff --git a/security/keys/keyring.c b/security/keys/keyring.c
+index b42f2d4..87eff32 100644
+--- a/security/keys/keyring.c
++++ b/security/keys/keyring.c
+@@ -479,7 +479,7 @@ not_this_keyring:
+ 
+ 	/* we found a viable match */
+ found:
+-	atomic_inc(&key->usage);
++	__key_get(key);
+ 	key->last_used_at = ctx->now.tv_sec;
+ 	keyring->last_used_at = ctx->now.tv_sec;
+ 	while (sp > 0)
+@@ -573,7 +573,7 @@ key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+ 	return ERR_PTR(-ENOKEY);
+ 
+ found:
+-	atomic_inc(&key->usage);
++	__key_get(key);
+ 	keyring->last_used_at = key->last_used_at =
+ 		current_kernel_time().tv_sec;
+ 	rcu_read_unlock();
+@@ -909,7 +909,7 @@ void __key_link(struct key *keyring, struct key *key,
+ 
+ 	klist = rcu_dereference_locked_keyring(keyring);
+ 
+-	atomic_inc(&key->usage);
++	__key_get(key);
+ 	keyring->last_used_at = key->last_used_at =
+ 		current_kernel_time().tv_sec;
+ 
+diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c
+index e68a3e0..68548ea 100644
+--- a/security/keys/process_keys.c
++++ b/security/keys/process_keys.c
+@@ -235,7 +235,7 @@ int install_session_keyring_to_cred(struct cred *cred, struct key *keyring)
+ 		if (IS_ERR(keyring))
+ 			return PTR_ERR(keyring);
+ 	} else {
+-		atomic_inc(&keyring->usage);
++		__key_get(keyring);
+ 	}
+ 
+ 	/* install the keyring */
+@@ -544,7 +544,7 @@ try_again:
+ 		}
+ 
+ 		key = ctx.cred->thread_keyring;
+-		atomic_inc(&key->usage);
++		__key_get(key);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+@@ -562,7 +562,7 @@ try_again:
+ 		}
+ 
+ 		key = ctx.cred->process_keyring;
+-		atomic_inc(&key->usage);
++		__key_get(key);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+@@ -593,7 +593,7 @@ try_again:
+ 
+ 		rcu_read_lock();
+ 		key = rcu_dereference(ctx.cred->session_keyring);
+-		atomic_inc(&key->usage);
++		__key_get(key);
+ 		rcu_read_unlock();
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+@@ -606,7 +606,7 @@ try_again:
+ 		}
+ 
+ 		key = ctx.cred->user->uid_keyring;
+-		atomic_inc(&key->usage);
++		__key_get(key);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+@@ -618,7 +618,7 @@ try_again:
+ 		}
+ 
+ 		key = ctx.cred->user->session_keyring;
+-		atomic_inc(&key->usage);
++		__key_get(key);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+@@ -632,7 +632,7 @@ try_again:
+ 		if (!key)
+ 			goto error;
+ 
+-		atomic_inc(&key->usage);
++		__key_get(key);
+ 		key_ref = make_key_ref(key, 1);
+ 		break;
+ 
+@@ -648,7 +648,7 @@ try_again:
+ 		} else {
+ 			rka = ctx.cred->request_key_auth->payload.data;
+ 			key = rka->dest_keyring;
+-			atomic_inc(&key->usage);
++			__key_get(key);
+ 		}
+ 		up_read(&ctx.cred->request_key_auth->sem);
+ 		if (!key)
+-- 
+1.8.3.1
+
+
+From 99b0f3185570bb92a61952673b9933d9c1999508 Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:53 +0100
+Subject: [PATCH 08/10] KEYS: Drop the permissions argument from
+ __keyring_search_one()
+
+Drop the permissions argument from __keyring_search_one() as the only caller
+passes 0 here - which causes all checks to be skipped.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ security/keys/internal.h | 3 +--
+ security/keys/key.c      | 2 +-
+ security/keys/keyring.c  | 9 +++------
+ 3 files changed, 5 insertions(+), 9 deletions(-)
+
+diff --git a/security/keys/internal.h b/security/keys/internal.h
+index f4bf938..73950bf 100644
+--- a/security/keys/internal.h
++++ b/security/keys/internal.h
+@@ -99,8 +99,7 @@ extern void __key_link_end(struct key *keyring,
+ 			   unsigned long prealloc);
+ 
+ extern key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+-				      const struct keyring_index_key *index_key,
+-				      key_perm_t perm);
++				      const struct keyring_index_key *index_key);
+ 
+ extern struct key *keyring_search_instkey(struct key *keyring,
+ 					  key_serial_t target_id);
+diff --git a/security/keys/key.c b/security/keys/key.c
+index 1e23cc2..7d716b8 100644
+--- a/security/keys/key.c
++++ b/security/keys/key.c
+@@ -847,7 +847,7 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 	 * update that instead if possible
+ 	 */
+ 	if (index_key.type->update) {
+-		key_ref = __keyring_search_one(keyring_ref, &index_key, 0);
++		key_ref = __keyring_search_one(keyring_ref, &index_key);
+ 		if (!IS_ERR(key_ref))
+ 			goto found_matching_key;
+ 	}
+diff --git a/security/keys/keyring.c b/security/keys/keyring.c
+index 87eff32..eeef1a0 100644
+--- a/security/keys/keyring.c
++++ b/security/keys/keyring.c
+@@ -531,15 +531,14 @@ EXPORT_SYMBOL(keyring_search);
+  * RCU is used to make it unnecessary to lock the keyring key list here.
+  *
+  * Returns a pointer to the found key with usage count incremented if
+- * successful and returns -ENOKEY if not found.  Revoked keys and keys not
+- * providing the requested permission are skipped over.
++ * successful and returns -ENOKEY if not found.  Revoked and invalidated keys
++ * are skipped over.
+  *
+  * If successful, the possession indicator is propagated from the keyring ref
+  * to the returned key reference.
+  */
+ key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+-			       const struct keyring_index_key *index_key,
+-			       key_perm_t perm)
++			       const struct keyring_index_key *index_key)
+ {
+ 	struct keyring_list *klist;
+ 	struct key *keyring, *key;
+@@ -560,8 +559,6 @@ key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+ 			if (key->type == index_key->type &&
+ 			    (!key->type->match ||
+ 			     key->type->match(key, index_key->description)) &&
+-			    key_permission(make_key_ref(key, possessed),
+-					   perm) == 0 &&
+ 			    !(key->flags & ((1 << KEY_FLAG_INVALIDATED) |
+ 					    (1 << KEY_FLAG_REVOKED)))
+ 			    )
+-- 
+1.8.3.1
+
+
+From cb720b39e41e62d55bf1e5f8243d78643d31154d Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:53 +0100
+Subject: [PATCH 09/10] Add a generic associative array implementation.
+
+Add a generic associative array implementation that can be used as the
+container for keyrings, thereby massively increasing the capacity available
+whilst also speeding up searching in keyrings that contain a lot of keys.
+
+This may also be useful in FS-Cache for tracking cookies.
+
+Documentation is added into Documentation/associative_array.txt
+
+Some of the properties of the implementation are:
+
+ (1) Objects are opaque pointers.  The implementation does not care where they
+     point (if anywhere) or what they point to (if anything).
+
+     [!] NOTE: Pointers to objects _must_ be zero in the two least significant
+     	       bits.
+
+ (2) Objects do not need to contain linkage blocks for use by the array.  This
+     permits an object to be located in multiple arrays simultaneously.
+     Rather, the array is made up of metadata blocks that point to objects.
+
+ (3) Objects are labelled as being one of two types (the type is a bool value).
+     This information is stored in the array, but has no consequence to the
+     array itself or its algorithms.
+
+ (4) Objects require index keys to locate them within the array.
+
+ (5) Index keys must be unique.  Inserting an object with the same key as one
+     already in the array will replace the old object.
+
+ (6) Index keys can be of any length and can be of different lengths.
+
+ (7) Index keys should encode the length early on, before any variation due to
+     length is seen.
+
+ (8) Index keys can include a hash to scatter objects throughout the array.
+
+ (9) The array can iterated over.  The objects will not necessarily come out in
+     key order.
+
+(10) The array can be iterated whilst it is being modified, provided the RCU
+     readlock is being held by the iterator.  Note, however, under these
+     circumstances, some objects may be seen more than once.  If this is a
+     problem, the iterator should lock against modification.  Objects will not
+     be missed, however, unless deleted.
+
+(11) Objects in the array can be looked up by means of their index key.
+
+(12) Objects can be looked up whilst the array is being modified, provided the
+     RCU readlock is being held by the thread doing the look up.
+
+The implementation uses a tree of 16-pointer nodes internally that are indexed
+on each level by nibbles from the index key.  To improve memory efficiency,
+shortcuts can be emplaced to skip over what would otherwise be a series of
+single-occupancy nodes.  Further, nodes pack leaf object pointers into spare
+space in the node rather than making an extra branch until as such time an
+object needs to be added to a full node.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ Documentation/assoc_array.txt    |  574 +++++++++++++
+ include/linux/assoc_array.h      |   92 ++
+ include/linux/assoc_array_priv.h |  182 ++++
+ lib/Kconfig                      |   14 +
+ lib/Makefile                     |    1 +
+ lib/assoc_array.c                | 1745 ++++++++++++++++++++++++++++++++++++++
+ 6 files changed, 2608 insertions(+)
+ create mode 100644 Documentation/assoc_array.txt
+ create mode 100644 include/linux/assoc_array.h
+ create mode 100644 include/linux/assoc_array_priv.h
+ create mode 100644 lib/assoc_array.c
+
+diff --git a/Documentation/assoc_array.txt b/Documentation/assoc_array.txt
+new file mode 100644
+index 0000000..f4faec0
+--- /dev/null
++++ b/Documentation/assoc_array.txt
+@@ -0,0 +1,574 @@
++		   ========================================
++		   GENERIC ASSOCIATIVE ARRAY IMPLEMENTATION
++		   ========================================
++
++Contents:
++
++ - Overview.
++
++ - The public API.
++   - Edit script.
++   - Operations table.
++   - Manipulation functions.
++   - Access functions.
++   - Index key form.
++
++ - Internal workings.
++   - Basic internal tree layout.
++   - Shortcuts.
++   - Splitting and collapsing nodes.
++   - Non-recursive iteration.
++   - Simultaneous alteration and iteration.
++
++
++========
++OVERVIEW
++========
++
++This associative array implementation is an object container with the following
++properties:
++
++ (1) Objects are opaque pointers.  The implementation does not care where they
++     point (if anywhere) or what they point to (if anything).
++
++     [!] NOTE: Pointers to objects _must_ be zero in the least significant bit.
++
++ (2) Objects do not need to contain linkage blocks for use by the array.  This
++     permits an object to be located in multiple arrays simultaneously.
++     Rather, the array is made up of metadata blocks that point to objects.
++
++ (3) Objects require index keys to locate them within the array.
++
++ (4) Index keys must be unique.  Inserting an object with the same key as one
++     already in the array will replace the old object.
++
++ (5) Index keys can be of any length and can be of different lengths.
++
++ (6) Index keys should encode the length early on, before any variation due to
++     length is seen.
++
++ (7) Index keys can include a hash to scatter objects throughout the array.
++
++ (8) The array can iterated over.  The objects will not necessarily come out in
++     key order.
++
++ (9) The array can be iterated over whilst it is being modified, provided the
++     RCU readlock is being held by the iterator.  Note, however, under these
++     circumstances, some objects may be seen more than once.  If this is a
++     problem, the iterator should lock against modification.  Objects will not
++     be missed, however, unless deleted.
++
++(10) Objects in the array can be looked up by means of their index key.
++
++(11) Objects can be looked up whilst the array is being modified, provided the
++     RCU readlock is being held by the thread doing the look up.
++
++The implementation uses a tree of 16-pointer nodes internally that are indexed
++on each level by nibbles from the index key in the same manner as in a radix
++tree.  To improve memory efficiency, shortcuts can be emplaced to skip over
++what would otherwise be a series of single-occupancy nodes.  Further, nodes
++pack leaf object pointers into spare space in the node rather than making an
++extra branch until as such time an object needs to be added to a full node.
++
++
++==============
++THE PUBLIC API
++==============
++
++The public API can be found in <linux/assoc_array.h>.  The associative array is
++rooted on the following structure:
++
++	struct assoc_array {
++		...
++	};
++
++The code is selected by enabling CONFIG_ASSOCIATIVE_ARRAY.
++
++
++EDIT SCRIPT
++-----------
++
++The insertion and deletion functions produce an 'edit script' that can later be
++applied to effect the changes without risking ENOMEM.  This retains the
++preallocated metadata blocks that will be installed in the internal tree and
++keeps track of the metadata blocks that will be removed from the tree when the
++script is applied.
++
++This is also used to keep track of dead blocks and dead objects after the
++script has been applied so that they can be freed later.  The freeing is done
++after an RCU grace period has passed - thus allowing access functions to
++proceed under the RCU read lock.
++
++The script appears as outside of the API as a pointer of the type:
++
++	struct assoc_array_edit;
++
++There are two functions for dealing with the script:
++
++ (1) Apply an edit script.
++
++	void assoc_array_apply_edit(struct assoc_array_edit *edit);
++
++     This will perform the edit functions, interpolating various write barriers
++     to permit accesses under the RCU read lock to continue.  The edit script
++     will then be passed to call_rcu() to free it and any dead stuff it points
++     to.
++
++ (2) Cancel an edit script.
++
++	void assoc_array_cancel_edit(struct assoc_array_edit *edit);
++
++     This frees the edit script and all preallocated memory immediately.  If
++     this was for insertion, the new object is _not_ released by this function,
++     but must rather be released by the caller.
++
++These functions are guaranteed not to fail.
++
++
++OPERATIONS TABLE
++----------------
++
++Various functions take a table of operations:
++
++	struct assoc_array_ops {
++		...
++	};
++
++This points to a number of methods, all of which need to be provided:
++
++ (1) Get a chunk of index key from caller data:
++
++	unsigned long (*get_key_chunk)(const void *index_key, int level);
++
++     This should return a chunk of caller-supplied index key starting at the
++     *bit* position given by the level argument.  The level argument will be a
++     multiple of ASSOC_ARRAY_KEY_CHUNK_SIZE and the function should return
++     ASSOC_ARRAY_KEY_CHUNK_SIZE bits.  No error is possible.
++
++
++ (2) Get a chunk of an object's index key.
++
++	unsigned long (*get_object_key_chunk)(const void *object, int level);
++
++     As the previous function, but gets its data from an object in the array
++     rather than from a caller-supplied index key.
++
++
++ (3) See if this is the object we're looking for.
++
++	bool (*compare_object)(const void *object, const void *index_key);
++
++     Compare the object against an index key and return true if it matches and
++     false if it doesn't.
++
++
++ (4) Diff the index keys of two objects.
++
++	int (*diff_objects)(const void *a, const void *b);
++
++     Return the bit position at which the index keys of two objects differ or
++     -1 if they are the same.
++
++
++ (5) Free an object.
++
++	void (*free_object)(void *object);
++
++     Free the specified object.  Note that this may be called an RCU grace
++     period after assoc_array_apply_edit() was called, so synchronize_rcu() may
++     be necessary on module unloading.
++
++
++MANIPULATION FUNCTIONS
++----------------------
++
++There are a number of functions for manipulating an associative array:
++
++ (1) Initialise an associative array.
++
++	void assoc_array_init(struct assoc_array *array);
++
++     This initialises the base structure for an associative array.  It can't
++     fail.
++
++
++ (2) Insert/replace an object in an associative array.
++
++	struct assoc_array_edit *
++	assoc_array_insert(struct assoc_array *array,
++			   const struct assoc_array_ops *ops,
++			   const void *index_key,
++			   void *object);
++
++     This inserts the given object into the array.  Note that the least
++     significant bit of the pointer must be zero as it's used to type-mark
++     pointers internally.
++
++     If an object already exists for that key then it will be replaced with the
++     new object and the old one will be freed automatically.
++
++     The index_key argument should hold index key information and is
++     passed to the methods in the ops table when they are called.
++
++     This function makes no alteration to the array itself, but rather returns
++     an edit script that must be applied.  -ENOMEM is returned in the case of
++     an out-of-memory error.
++
++     The caller should lock exclusively against other modifiers of the array.
++
++
++ (3) Delete an object from an associative array.
++
++	struct assoc_array_edit *
++	assoc_array_delete(struct assoc_array *array,
++			   const struct assoc_array_ops *ops,
++			   const void *index_key);
++
++     This deletes an object that matches the specified data from the array.
++
++     The index_key argument should hold index key information and is
++     passed to the methods in the ops table when they are called.
++
++     This function makes no alteration to the array itself, but rather returns
++     an edit script that must be applied.  -ENOMEM is returned in the case of
++     an out-of-memory error.  NULL will be returned if the specified object is
++     not found within the array.
++
++     The caller should lock exclusively against other modifiers of the array.
++
++
++ (4) Delete all objects from an associative array.
++
++	struct assoc_array_edit *
++	assoc_array_clear(struct assoc_array *array,
++			  const struct assoc_array_ops *ops);
++
++     This deletes all the objects from an associative array and leaves it
++     completely empty.
++
++     This function makes no alteration to the array itself, but rather returns
++     an edit script that must be applied.  -ENOMEM is returned in the case of
++     an out-of-memory error.
++
++     The caller should lock exclusively against other modifiers of the array.
++
++
++ (5) Destroy an associative array, deleting all objects.
++
++	void assoc_array_destroy(struct assoc_array *array,
++				 const struct assoc_array_ops *ops);
++
++     This destroys the contents of the associative array and leaves it
++     completely empty.  It is not permitted for another thread to be traversing
++     the array under the RCU read lock at the same time as this function is
++     destroying it as no RCU deferral is performed on memory release -
++     something that would require memory to be allocated.
++
++     The caller should lock exclusively against other modifiers and accessors
++     of the array.
++
++
++ (6) Garbage collect an associative array.
++
++	int assoc_array_gc(struct assoc_array *array,
++			   const struct assoc_array_ops *ops,
++			   bool (*iterator)(void *object, void *iterator_data),
++			   void *iterator_data);
++
++     This iterates over the objects in an associative array and passes each one
++     to iterator().  If iterator() returns true, the object is kept.  If it
++     returns false, the object will be freed.  If the iterator() function
++     returns true, it must perform any appropriate refcount incrementing on the
++     object before returning.
++
++     The internal tree will be packed down if possible as part of the iteration
++     to reduce the number of nodes in it.
++
++     The iterator_data is passed directly to iterator() and is otherwise
++     ignored by the function.
++
++     The function will return 0 if successful and -ENOMEM if there wasn't
++     enough memory.
++
++     It is possible for other threads to iterate over or search the array under
++     the RCU read lock whilst this function is in progress.  The caller should
++     lock exclusively against other modifiers of the array.
++
++
++ACCESS FUNCTIONS
++----------------
++
++There are two functions for accessing an associative array:
++
++ (1) Iterate over all the objects in an associative array.
++
++	int assoc_array_iterate(const struct assoc_array *array,
++				int (*iterator)(const void *object,
++						void *iterator_data),
++				void *iterator_data);
++
++     This passes each object in the array to the iterator callback function.
++     iterator_data is private data for that function.
++
++     This may be used on an array at the same time as the array is being
++     modified, provided the RCU read lock is held.  Under such circumstances,
++     it is possible for the iteration function to see some objects twice.  If
++     this is a problem, then modification should be locked against.  The
++     iteration algorithm should not, however, miss any objects.
++
++     The function will return 0 if no objects were in the array or else it will
++     return the result of the last iterator function called.  Iteration stops
++     immediately if any call to the iteration function results in a non-zero
++     return.
++
++
++ (2) Find an object in an associative array.
++
++	void *assoc_array_find(const struct assoc_array *array,
++			       const struct assoc_array_ops *ops,
++			       const void *index_key);
++
++     This walks through the array's internal tree directly to the object
++     specified by the index key..
++
++     This may be used on an array at the same time as the array is being
++     modified, provided the RCU read lock is held.
++
++     The function will return the object if found (and set *_type to the object
++     type) or will return NULL if the object was not found.
++
++
++INDEX KEY FORM
++--------------
++
++The index key can be of any form, but since the algorithms aren't told how long
++the key is, it is strongly recommended that the index key includes its length
++very early on before any variation due to the length would have an effect on
++comparisons.
++
++This will cause leaves with different length keys to scatter away from each
++other - and those with the same length keys to cluster together.
++
++It is also recommended that the index key begin with a hash of the rest of the
++key to maximise scattering throughout keyspace.
++
++The better the scattering, the wider and lower the internal tree will be.
++
++Poor scattering isn't too much of a problem as there are shortcuts and nodes
++can contain mixtures of leaves and metadata pointers.
++
++The index key is read in chunks of machine word.  Each chunk is subdivided into
++one nibble (4 bits) per level, so on a 32-bit CPU this is good for 8 levels and
++on a 64-bit CPU, 16 levels.  Unless the scattering is really poor, it is
++unlikely that more than one word of any particular index key will have to be
++used.
++
++
++=================
++INTERNAL WORKINGS
++=================
++
++The associative array data structure has an internal tree.  This tree is
++constructed of two types of metadata blocks: nodes and shortcuts.
++
++A node is an array of slots.  Each slot can contain one of four things:
++
++ (*) A NULL pointer, indicating that the slot is empty.
++
++ (*) A pointer to an object (a leaf).
++
++ (*) A pointer to a node at the next level.
++
++ (*) A pointer to a shortcut.
++
++
++BASIC INTERNAL TREE LAYOUT
++--------------------------
++
++Ignoring shortcuts for the moment, the nodes form a multilevel tree.  The index
++key space is strictly subdivided by the nodes in the tree and nodes occur on
++fixed levels.  For example:
++
++ Level:	0		1		2		3
++	===============	===============	===============	===============
++							NODE D
++			NODE B		NODE C	+------>+---+
++		+------>+---+	+------>+---+	|	| 0 |
++	NODE A	|	| 0 |	|	| 0 |	|	+---+
++	+---+	|	+---+	|	+---+	|	:   :
++	| 0 |	|	:   :	|	:   :	|	+---+
++	+---+	|	+---+	|	+---+	|	| f |
++	| 1 |---+	| 3 |---+	| 7 |---+	+---+
++	+---+		+---+		+---+
++	:   :		:   :		| 8 |---+
++	+---+		+---+		+---+	|	NODE E
++	| e |---+	| f |		:   :   +------>+---+
++	+---+	|	+---+		+---+		| 0 |
++	| f |	|			| f |		+---+
++	+---+	|			+---+		:   :
++		|	NODE F				+---+
++		+------>+---+				| f |
++			| 0 |		NODE G		+---+
++			+---+	+------>+---+
++			:   :	|	| 0 |
++			+---+	|	+---+
++			| 6 |---+	:   :
++			+---+		+---+
++			:   :		| f |
++			+---+		+---+
++			| f |
++			+---+
++
++In the above example, there are 7 nodes (A-G), each with 16 slots (0-f).
++Assuming no other meta data nodes in the tree, the key space is divided thusly:
++
++	KEY PREFIX	NODE
++	==========	====
++	137*		D
++	138*		E
++	13[0-69-f]*	C
++	1[0-24-f]*	B
++	e6*		G
++	e[0-57-f]*	F
++	[02-df]*	A
++
++So, for instance, keys with the following example index keys will be found in
++the appropriate nodes:
++
++	INDEX KEY	PREFIX	NODE
++	===============	=======	====
++	13694892892489	13	C
++	13795289025897	137	D
++	13889dde88793	138	E
++	138bbb89003093	138	E
++	1394879524789	12	C
++	1458952489	1	B
++	9431809de993ba	-	A
++	b4542910809cd	-	A
++	e5284310def98	e	F
++	e68428974237	e6	G
++	e7fffcbd443	e	F
++	f3842239082	-	A
++
++To save memory, if a node can hold all the leaves in its portion of keyspace,
++then the node will have all those leaves in it and will not have any metadata
++pointers - even if some of those leaves would like to be in the same slot.
++
++A node can contain a heterogeneous mix of leaves and metadata pointers.
++Metadata pointers must be in the slots that match their subdivisions of key
++space.  The leaves can be in any slot not occupied by a metadata pointer.  It
++is guaranteed that none of the leaves in a node will match a slot occupied by a
++metadata pointer.  If the metadata pointer is there, any leaf whose key matches
++the metadata key prefix must be in the subtree that the metadata pointer points
++to.
++
++In the above example list of index keys, node A will contain:
++
++	SLOT	CONTENT		INDEX KEY (PREFIX)
++	====	===============	==================
++	1	PTR TO NODE B	1*
++	any	LEAF		9431809de993ba
++	any	LEAF		b4542910809cd
++	e	PTR TO NODE F	e*
++	any	LEAF		f3842239082
++
++and node B:
++
++	3	PTR TO NODE C	13*
++	any	LEAF		1458952489
++
++
++SHORTCUTS
++---------
++
++Shortcuts are metadata records that jump over a piece of keyspace.  A shortcut
++is a replacement for a series of single-occupancy nodes ascending through the
++levels.  Shortcuts exist to save memory and to speed up traversal.
++
++It is possible for the root of the tree to be a shortcut - say, for example,
++the tree contains at least 17 nodes all with key prefix '1111'.  The insertion
++algorithm will insert a shortcut to skip over the '1111' keyspace in a single
++bound and get to the fourth level where these actually become different.
++
++
++SPLITTING AND COLLAPSING NODES
++------------------------------
++
++Each node has a maximum capacity of 16 leaves and metadata pointers.  If the
++insertion algorithm finds that it is trying to insert a 17th object into a
++node, that node will be split such that at least two leaves that have a common
++key segment at that level end up in a separate node rooted on that slot for
++that common key segment.
++
++If the leaves in a full node and the leaf that is being inserted are
++sufficiently similar, then a shortcut will be inserted into the tree.
++
++When the number of objects in the subtree rooted at a node falls to 16 or
++fewer, then the subtree will be collapsed down to a single node - and this will
++ripple towards the root if possible.
++
++
++NON-RECURSIVE ITERATION
++-----------------------
++
++Each node and shortcut contains a back pointer to its parent and the number of
++slot in that parent that points to it.  None-recursive iteration uses these to
++proceed rootwards through the tree, going to the parent node, slot N + 1 to
++make sure progress is made without the need for a stack.
++
++The backpointers, however, make simultaneous alteration and iteration tricky.
++
++
++SIMULTANEOUS ALTERATION AND ITERATION
++-------------------------------------
++
++There are a number of cases to consider:
++
++ (1) Simple insert/replace.  This involves simply replacing a NULL or old
++     matching leaf pointer with the pointer to the new leaf after a barrier.
++     The metadata blocks don't change otherwise.  An old leaf won't be freed
++     until after the RCU grace period.
++
++ (2) Simple delete.  This involves just clearing an old matching leaf.  The
++     metadata blocks don't change otherwise.  The old leaf won't be freed until
++     after the RCU grace period.
++
++ (3) Insertion replacing part of a subtree that we haven't yet entered.  This
++     may involve replacement of part of that subtree - but that won't affect
++     the iteration as we won't have reached the pointer to it yet and the
++     ancestry blocks are not replaced (the layout of those does not change).
++
++ (4) Insertion replacing nodes that we're actively processing.  This isn't a
++     problem as we've passed the anchoring pointer and won't switch onto the
++     new layout until we follow the back pointers - at which point we've
++     already examined the leaves in the replaced node (we iterate over all the
++     leaves in a node before following any of its metadata pointers).
++
++     We might, however, re-see some leaves that have been split out into a new
++     branch that's in a slot further along than we were at.
++
++ (5) Insertion replacing nodes that we're processing a dependent branch of.
++     This won't affect us until we follow the back pointers.  Similar to (4).
++
++ (6) Deletion collapsing a branch under us.  This doesn't affect us because the
++     back pointers will get us back to the parent of the new node before we
++     could see the new node.  The entire collapsed subtree is thrown away
++     unchanged - and will still be rooted on the same slot, so we shouldn't
++     process it a second time as we'll go back to slot + 1.
++
++Note:
++
++ (*) Under some circumstances, we need to simultaneously change the parent
++     pointer and the parent slot pointer on a node (say, for example, we
++     inserted another node before it and moved it up a level).  We cannot do
++     this without locking against a read - so we have to replace that node too.
++
++     However, when we're changing a shortcut into a node this isn't a problem
++     as shortcuts only have one slot and so the parent slot number isn't used
++     when traversing backwards over one.  This means that it's okay to change
++     the slot number first - provided suitable barriers are used to make sure
++     the parent slot number is read after the back pointer.
++
++Obsolete blocks and leaves are freed up after an RCU grace period has passed,
++so as long as anyone doing walking or iteration holds the RCU read lock, the
++old superstructure should not go away on them.
+diff --git a/include/linux/assoc_array.h b/include/linux/assoc_array.h
+new file mode 100644
+index 0000000..9a193b8
+--- /dev/null
++++ b/include/linux/assoc_array.h
+@@ -0,0 +1,92 @@
++/* Generic associative array implementation.
++ *
++ * See Documentation/assoc_array.txt for information.
++ *
++ * Copyright (C) 2013 Red Hat, Inc. All Rights Reserved.
++ * Written by David Howells (dhowells@redhat.com)
++ *
++ * This program is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU General Public Licence
++ * as published by the Free Software Foundation; either version
++ * 2 of the Licence, or (at your option) any later version.
++ */
++
++#ifndef _LINUX_ASSOC_ARRAY_H
++#define _LINUX_ASSOC_ARRAY_H
++
++#ifdef CONFIG_ASSOCIATIVE_ARRAY
++
++#include <linux/types.h>
++
++#define ASSOC_ARRAY_KEY_CHUNK_SIZE BITS_PER_LONG /* Key data retrieved in chunks of this size */
++
++/*
++ * Generic associative array.
++ */
++struct assoc_array {
++	struct assoc_array_ptr	*root;		/* The node at the root of the tree */
++	unsigned long		nr_leaves_on_tree;
++};
++
++/*
++ * Operations on objects and index keys for use by array manipulation routines.
++ */
++struct assoc_array_ops {
++	/* Method to get a chunk of an index key from caller-supplied data */
++	unsigned long (*get_key_chunk)(const void *index_key, int level);
++
++	/* Method to get a piece of an object's index key */
++	unsigned long (*get_object_key_chunk)(const void *object, int level);
++
++	/* Is this the object we're looking for? */
++	bool (*compare_object)(const void *object, const void *index_key);
++
++	/* How different are two objects, to a bit position in their keys? (or
++	 * -1 if they're the same)
++	 */
++	int (*diff_objects)(const void *a, const void *b);
++
++	/* Method to free an object. */
++	void (*free_object)(void *object);
++};
++
++/*
++ * Access and manipulation functions.
++ */
++struct assoc_array_edit;
++
++static inline void assoc_array_init(struct assoc_array *array)
++{
++	array->root = NULL;
++	array->nr_leaves_on_tree = 0;
++}
++
++extern int assoc_array_iterate(const struct assoc_array *array,
++			       int (*iterator)(const void *object,
++					       void *iterator_data),
++			       void *iterator_data);
++extern void *assoc_array_find(const struct assoc_array *array,
++			      const struct assoc_array_ops *ops,
++			      const void *index_key);
++extern void assoc_array_destroy(struct assoc_array *array,
++				const struct assoc_array_ops *ops);
++extern struct assoc_array_edit *assoc_array_insert(struct assoc_array *array,
++						   const struct assoc_array_ops *ops,
++						   const void *index_key,
++						   void *object);
++extern void assoc_array_insert_set_object(struct assoc_array_edit *edit,
++					  void *object);
++extern struct assoc_array_edit *assoc_array_delete(struct assoc_array *array,
++						   const struct assoc_array_ops *ops,
++						   const void *index_key);
++extern struct assoc_array_edit *assoc_array_clear(struct assoc_array *array,
++						  const struct assoc_array_ops *ops);
++extern void assoc_array_apply_edit(struct assoc_array_edit *edit);
++extern void assoc_array_cancel_edit(struct assoc_array_edit *edit);
++extern int assoc_array_gc(struct assoc_array *array,
++			  const struct assoc_array_ops *ops,
++			  bool (*iterator)(void *object, void *iterator_data),
++			  void *iterator_data);
++
++#endif /* CONFIG_ASSOCIATIVE_ARRAY */
++#endif /* _LINUX_ASSOC_ARRAY_H */
+diff --git a/include/linux/assoc_array_priv.h b/include/linux/assoc_array_priv.h
+new file mode 100644
+index 0000000..711275e
+--- /dev/null
++++ b/include/linux/assoc_array_priv.h
+@@ -0,0 +1,182 @@
++/* Private definitions for the generic associative array implementation.
++ *
++ * See Documentation/assoc_array.txt for information.
++ *
++ * Copyright (C) 2013 Red Hat, Inc. All Rights Reserved.
++ * Written by David Howells (dhowells@redhat.com)
++ *
++ * This program is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU General Public Licence
++ * as published by the Free Software Foundation; either version
++ * 2 of the Licence, or (at your option) any later version.
++ */
++
++#ifndef _LINUX_ASSOC_ARRAY_PRIV_H
++#define _LINUX_ASSOC_ARRAY_PRIV_H
++
++#ifdef CONFIG_ASSOCIATIVE_ARRAY
++
++#include <linux/assoc_array.h>
++
++#define ASSOC_ARRAY_FAN_OUT		16	/* Number of slots per node */
++#define ASSOC_ARRAY_FAN_MASK		(ASSOC_ARRAY_FAN_OUT - 1)
++#define ASSOC_ARRAY_LEVEL_STEP		(ilog2(ASSOC_ARRAY_FAN_OUT))
++#define ASSOC_ARRAY_LEVEL_STEP_MASK	(ASSOC_ARRAY_LEVEL_STEP - 1)
++#define ASSOC_ARRAY_KEY_CHUNK_MASK	(ASSOC_ARRAY_KEY_CHUNK_SIZE - 1)
++#define ASSOC_ARRAY_KEY_CHUNK_SHIFT	(ilog2(BITS_PER_LONG))
++
++/*
++ * Undefined type representing a pointer with type information in the bottom
++ * two bits.
++ */
++struct assoc_array_ptr;
++
++/*
++ * An N-way node in the tree.
++ *
++ * Each slot contains one of four things:
++ *
++ *	(1) Nothing (NULL).
++ *
++ *	(2) A leaf object (pointer types 0).
++ *
++ *	(3) A next-level node (pointer type 1, subtype 0).
++ *
++ *	(4) A shortcut (pointer type 1, subtype 1).
++ *
++ * The tree is optimised for search-by-ID, but permits reasonable iteration
++ * also.
++ *
++ * The tree is navigated by constructing an index key consisting of an array of
++ * segments, where each segment is ilog2(ASSOC_ARRAY_FAN_OUT) bits in size.
++ *
++ * The segments correspond to levels of the tree (the first segment is used at
++ * level 0, the second at level 1, etc.).
++ */
++struct assoc_array_node {
++	struct assoc_array_ptr	*back_pointer;
++	u8			parent_slot;
++	struct assoc_array_ptr	*slots[ASSOC_ARRAY_FAN_OUT];
++	unsigned long		nr_leaves_on_branch;
++};
++
++/*
++ * A shortcut through the index space out to where a collection of nodes/leaves
++ * with the same IDs live.
++ */
++struct assoc_array_shortcut {
++	struct assoc_array_ptr	*back_pointer;
++	int			parent_slot;
++	int			skip_to_level;
++	struct assoc_array_ptr	*next_node;
++	unsigned long		index_key[];
++};
++
++/*
++ * Preallocation cache.
++ */
++struct assoc_array_edit {
++	struct rcu_head			rcu;
++	struct assoc_array		*array;
++	const struct assoc_array_ops	*ops;
++	const struct assoc_array_ops	*ops_for_excised_subtree;
++	struct assoc_array_ptr		*leaf;
++	struct assoc_array_ptr		**leaf_p;
++	struct assoc_array_ptr		*dead_leaf;
++	struct assoc_array_ptr		*new_meta[3];
++	struct assoc_array_ptr		*excised_meta[1];
++	struct assoc_array_ptr		*excised_subtree;
++	struct assoc_array_ptr		**set_backpointers[ASSOC_ARRAY_FAN_OUT];
++	struct assoc_array_ptr		*set_backpointers_to;
++	struct assoc_array_node		*adjust_count_on;
++	long				adjust_count_by;
++	struct {
++		struct assoc_array_ptr	**ptr;
++		struct assoc_array_ptr	*to;
++	} set[2];
++	struct {
++		u8			*p;
++		u8			to;
++	} set_parent_slot[1];
++	u8				segment_cache[ASSOC_ARRAY_FAN_OUT + 1];
++};
++
++/*
++ * Internal tree member pointers are marked in the bottom one or two bits to
++ * indicate what type they are so that we don't have to look behind every
++ * pointer to see what it points to.
++ *
++ * We provide functions to test type annotations and to create and translate
++ * the annotated pointers.
++ */
++#define ASSOC_ARRAY_PTR_TYPE_MASK 0x1UL
++#define ASSOC_ARRAY_PTR_LEAF_TYPE 0x0UL	/* Points to leaf (or nowhere) */
++#define ASSOC_ARRAY_PTR_META_TYPE 0x1UL	/* Points to node or shortcut */
++#define ASSOC_ARRAY_PTR_SUBTYPE_MASK	0x2UL
++#define ASSOC_ARRAY_PTR_NODE_SUBTYPE	0x0UL
++#define ASSOC_ARRAY_PTR_SHORTCUT_SUBTYPE 0x2UL
++
++static inline bool assoc_array_ptr_is_meta(const struct assoc_array_ptr *x)
++{
++	return (unsigned long)x & ASSOC_ARRAY_PTR_TYPE_MASK;
++}
++static inline bool assoc_array_ptr_is_leaf(const struct assoc_array_ptr *x)
++{
++	return !assoc_array_ptr_is_meta(x);
++}
++static inline bool assoc_array_ptr_is_shortcut(const struct assoc_array_ptr *x)
++{
++	return (unsigned long)x & ASSOC_ARRAY_PTR_SUBTYPE_MASK;
++}
++static inline bool assoc_array_ptr_is_node(const struct assoc_array_ptr *x)
++{
++	return !assoc_array_ptr_is_shortcut(x);
++}
++
++static inline void *assoc_array_ptr_to_leaf(const struct assoc_array_ptr *x)
++{
++	return (void *)((unsigned long)x & ~ASSOC_ARRAY_PTR_TYPE_MASK);
++}
++
++static inline
++unsigned long __assoc_array_ptr_to_meta(const struct assoc_array_ptr *x)
++{
++	return (unsigned long)x &
++		~(ASSOC_ARRAY_PTR_SUBTYPE_MASK | ASSOC_ARRAY_PTR_TYPE_MASK);
++}
++static inline
++struct assoc_array_node *assoc_array_ptr_to_node(const struct assoc_array_ptr *x)
++{
++	return (struct assoc_array_node *)__assoc_array_ptr_to_meta(x);
++}
++static inline
++struct assoc_array_shortcut *assoc_array_ptr_to_shortcut(const struct assoc_array_ptr *x)
++{
++	return (struct assoc_array_shortcut *)__assoc_array_ptr_to_meta(x);
++}
++
++static inline
++struct assoc_array_ptr *__assoc_array_x_to_ptr(const void *p, unsigned long t)
++{
++	return (struct assoc_array_ptr *)((unsigned long)p | t);
++}
++static inline
++struct assoc_array_ptr *assoc_array_leaf_to_ptr(const void *p)
++{
++	return __assoc_array_x_to_ptr(p, ASSOC_ARRAY_PTR_LEAF_TYPE);
++}
++static inline
++struct assoc_array_ptr *assoc_array_node_to_ptr(const struct assoc_array_node *p)
++{
++	return __assoc_array_x_to_ptr(
++		p, ASSOC_ARRAY_PTR_META_TYPE | ASSOC_ARRAY_PTR_NODE_SUBTYPE);
++}
++static inline
++struct assoc_array_ptr *assoc_array_shortcut_to_ptr(const struct assoc_array_shortcut *p)
++{
++	return __assoc_array_x_to_ptr(
++		p, ASSOC_ARRAY_PTR_META_TYPE | ASSOC_ARRAY_PTR_SHORTCUT_SUBTYPE);
++}
++
++#endif /* CONFIG_ASSOCIATIVE_ARRAY */
++#endif /* _LINUX_ASSOC_ARRAY_PRIV_H */
+diff --git a/lib/Kconfig b/lib/Kconfig
+index 35da513..b7d3234 100644
+--- a/lib/Kconfig
++++ b/lib/Kconfig
+@@ -312,6 +312,20 @@ config TEXTSEARCH_FSM
+ config BTREE
+ 	boolean
+ 
++config ASSOCIATIVE_ARRAY
++	bool
++	help
++	  Generic associative array.  Can be searched and iterated over whilst
++	  it is being modified.  It is also reasonably quick to search and
++	  modify.  The algorithms are non-recursive, and the trees are highly
++	  capacious.
++
++	  See:
++
++		Documentation/assoc_array.txt
++
++	  for more information.
++
+ config HAS_IOMEM
+ 	boolean
+ 	depends on !NO_IOMEM
+diff --git a/lib/Makefile b/lib/Makefile
+index 7baccfd..2c59891 100644
+--- a/lib/Makefile
++++ b/lib/Makefile
+@@ -49,6 +49,7 @@ CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS))
+ obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o
+ 
+ obj-$(CONFIG_BTREE) += btree.o
++obj-$(CONFIG_ASSOCIATIVE_ARRAY) += assoc_array.o
+ obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o
+ obj-$(CONFIG_DEBUG_LIST) += list_debug.o
+ obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o
+diff --git a/lib/assoc_array.c b/lib/assoc_array.c
+new file mode 100644
+index 0000000..a095281
+--- /dev/null
++++ b/lib/assoc_array.c
+@@ -0,0 +1,1745 @@
++/* Generic associative array implementation.
++ *
++ * See Documentation/assoc_array.txt for information.
++ *
++ * Copyright (C) 2013 Red Hat, Inc. All Rights Reserved.
++ * Written by David Howells (dhowells@redhat.com)
++ *
++ * This program is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU General Public Licence
++ * as published by the Free Software Foundation; either version
++ * 2 of the Licence, or (at your option) any later version.
++ */
++//#define DEBUG
++#include <linux/slab.h>
++#include <linux/assoc_array_priv.h>
++
++/*
++ * Iterate over an associative array.  The caller must hold the RCU read lock
++ * or better.
++ */
++static int assoc_array_subtree_iterate(const struct assoc_array_ptr *root,
++				       const struct assoc_array_ptr *stop,
++				       int (*iterator)(const void *leaf,
++						       void *iterator_data),
++				       void *iterator_data)
++{
++	const struct assoc_array_shortcut *shortcut;
++	const struct assoc_array_node *node;
++	const struct assoc_array_ptr *cursor, *ptr, *parent;
++	unsigned long has_meta;
++	int slot, ret;
++
++	cursor = root;
++
++begin_node:
++	if (assoc_array_ptr_is_shortcut(cursor)) {
++		/* Descend through a shortcut */
++		shortcut = assoc_array_ptr_to_shortcut(cursor);
++		smp_read_barrier_depends();
++		cursor = ACCESS_ONCE(shortcut->next_node);
++	}
++
++	node = assoc_array_ptr_to_node(cursor);
++	smp_read_barrier_depends();
++	slot = 0;
++
++	/* We perform two passes of each node.
++	 *
++	 * The first pass does all the leaves in this node.  This means we
++	 * don't miss any leaves if the node is split up by insertion whilst
++	 * we're iterating over the branches rooted here (we may, however, see
++	 * some leaves twice).
++	 */
++	has_meta = 0;
++	for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++		ptr = ACCESS_ONCE(node->slots[slot]);
++		has_meta |= (unsigned long)ptr;
++		if (ptr && assoc_array_ptr_is_leaf(ptr)) {
++			/* We need a barrier between the read of the pointer
++			 * and dereferencing the pointer - but only if we are
++			 * actually going to dereference it.
++			 */
++			smp_read_barrier_depends();
++
++			/* Invoke the callback */
++			ret = iterator(assoc_array_ptr_to_leaf(ptr),
++				       iterator_data);
++			if (ret)
++				return ret;
++		}
++	}
++
++	/* The second pass attends to all the metadata pointers.  If we follow
++	 * one of these we may find that we don't come back here, but rather go
++	 * back to a replacement node with the leaves in a different layout.
++	 *
++	 * We are guaranteed to make progress, however, as the slot number for
++	 * a particular portion of the key space cannot change - and we
++	 * continue at the back pointer + 1.
++	 */
++	if (!(has_meta & ASSOC_ARRAY_PTR_META_TYPE))
++		goto finished_node;
++	slot = 0;
++
++continue_node:
++	node = assoc_array_ptr_to_node(cursor);
++	smp_read_barrier_depends();
++
++	for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++		ptr = ACCESS_ONCE(node->slots[slot]);
++		if (assoc_array_ptr_is_meta(ptr)) {
++			cursor = ptr;
++			goto begin_node;
++		}
++	}
++
++finished_node:
++	/* Move up to the parent (may need to skip back over a shortcut) */
++	parent = ACCESS_ONCE(node->back_pointer);
++	slot = node->parent_slot;
++	if (parent == stop)
++		return 0;
++
++	if (assoc_array_ptr_is_shortcut(parent)) {
++		shortcut = assoc_array_ptr_to_shortcut(parent);
++		smp_read_barrier_depends();
++		cursor = parent;
++		parent = ACCESS_ONCE(shortcut->back_pointer);
++		slot = shortcut->parent_slot;
++		if (parent == stop)
++			return 0;
++	}
++
++	/* Ascend to next slot in parent node */
++	cursor = parent;
++	slot++;
++	goto continue_node;
++}
++
++/**
++ * assoc_array_iterate - Pass all objects in the array to a callback
++ * @array: The array to iterate over.
++ * @iterator: The callback function.
++ * @iterator_data: Private data for the callback function.
++ *
++ * Iterate over all the objects in an associative array.  Each one will be
++ * presented to the iterator function.
++ *
++ * If the array is being modified concurrently with the iteration then it is
++ * possible that some objects in the array will be passed to the iterator
++ * callback more than once - though every object should be passed at least
++ * once.  If this is undesirable then the caller must lock against modification
++ * for the duration of this function.
++ *
++ * The function will return 0 if no objects were in the array or else it will
++ * return the result of the last iterator function called.  Iteration stops
++ * immediately if any call to the iteration function results in a non-zero
++ * return.
++ *
++ * The caller should hold the RCU read lock or better if concurrent
++ * modification is possible.
++ */
++int assoc_array_iterate(const struct assoc_array *array,
++			int (*iterator)(const void *object,
++					void *iterator_data),
++			void *iterator_data)
++{
++	struct assoc_array_ptr *root = ACCESS_ONCE(array->root);
++
++	if (!root)
++		return 0;
++	return assoc_array_subtree_iterate(root, NULL, iterator, iterator_data);
++}
++
++enum assoc_array_walk_status {
++	assoc_array_walk_tree_empty,
++	assoc_array_walk_found_terminal_node,
++	assoc_array_walk_found_wrong_shortcut,
++} status;
++
++struct assoc_array_walk_result {
++	struct {
++		struct assoc_array_node	*node;	/* Node in which leaf might be found */
++		int		level;
++		int		slot;
++	} terminal_node;
++	struct {
++		struct assoc_array_shortcut *shortcut;
++		int		level;
++		int		sc_level;
++		unsigned long	sc_segments;
++		unsigned long	dissimilarity;
++	} wrong_shortcut;
++};
++
++/*
++ * Navigate through the internal tree looking for the closest node to the key.
++ */
++static enum assoc_array_walk_status
++assoc_array_walk(const struct assoc_array *array,
++		 const struct assoc_array_ops *ops,
++		 const void *index_key,
++		 struct assoc_array_walk_result *result)
++{
++	struct assoc_array_shortcut *shortcut;
++	struct assoc_array_node *node;
++	struct assoc_array_ptr *cursor, *ptr;
++	unsigned long sc_segments, dissimilarity;
++	unsigned long segments;
++	int level, sc_level, next_sc_level;
++	int slot;
++
++	pr_devel("-->%s()\n", __func__);
++
++	cursor = ACCESS_ONCE(array->root);
++	if (!cursor)
++		return assoc_array_walk_tree_empty;
++
++	level = 0;
++
++	/* Use segments from the key for the new leaf to navigate through the
++	 * internal tree, skipping through nodes and shortcuts that are on
++	 * route to the destination.  Eventually we'll come to a slot that is
++	 * either empty or contains a leaf at which point we've found a node in
++	 * which the leaf we're looking for might be found or into which it
++	 * should be inserted.
++	 */
++jumped:
++	segments = ops->get_key_chunk(index_key, level);
++	pr_devel("segments[%d]: %lx\n", level, segments);
++
++	if (assoc_array_ptr_is_shortcut(cursor))
++		goto follow_shortcut;
++
++consider_node:
++	node = assoc_array_ptr_to_node(cursor);
++	smp_read_barrier_depends();
++
++	slot = segments >> (level & ASSOC_ARRAY_KEY_CHUNK_MASK);
++	slot &= ASSOC_ARRAY_FAN_MASK;
++	ptr = ACCESS_ONCE(node->slots[slot]);
++
++	pr_devel("consider slot %x [ix=%d type=%lu]\n",
++		 slot, level, (unsigned long)ptr & 3);
++
++	if (!assoc_array_ptr_is_meta(ptr)) {
++		/* The node doesn't have a node/shortcut pointer in the slot
++		 * corresponding to the index key that we have to follow.
++		 */
++		result->terminal_node.node = node;
++		result->terminal_node.level = level;
++		result->terminal_node.slot = slot;
++		pr_devel("<--%s() = terminal_node\n", __func__);
++		return assoc_array_walk_found_terminal_node;
++	}
++
++	if (assoc_array_ptr_is_node(ptr)) {
++		/* There is a pointer to a node in the slot corresponding to
++		 * this index key segment, so we need to follow it.
++		 */
++		cursor = ptr;
++		level += ASSOC_ARRAY_LEVEL_STEP;
++		if ((level & ASSOC_ARRAY_KEY_CHUNK_MASK) != 0)
++			goto consider_node;
++		goto jumped;
++	}
++
++	/* There is a shortcut in the slot corresponding to the index key
++	 * segment.  We follow the shortcut if its partial index key matches
++	 * this leaf's.  Otherwise we need to split the shortcut.
++	 */
++	cursor = ptr;
++follow_shortcut:
++	shortcut = assoc_array_ptr_to_shortcut(cursor);
++	smp_read_barrier_depends();
++	pr_devel("shortcut to %d\n", shortcut->skip_to_level);
++	sc_level = level + ASSOC_ARRAY_LEVEL_STEP;
++	BUG_ON(sc_level > shortcut->skip_to_level);
++
++	do {
++		/* Check the leaf against the shortcut's index key a word at a
++		 * time, trimming the final word (the shortcut stores the index
++		 * key completely from the root to the shortcut's target).
++		 */
++		if ((sc_level & ASSOC_ARRAY_KEY_CHUNK_MASK) == 0)
++			segments = ops->get_key_chunk(index_key, sc_level);
++
++		sc_segments = shortcut->index_key[sc_level >> ASSOC_ARRAY_KEY_CHUNK_SHIFT];
++		dissimilarity = segments ^ sc_segments;
++
++		if (round_up(sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE) > shortcut->skip_to_level) {
++			/* Trim segments that are beyond the shortcut */
++			int shift = shortcut->skip_to_level & ASSOC_ARRAY_KEY_CHUNK_MASK;
++			dissimilarity &= ~(ULONG_MAX << shift);
++			next_sc_level = shortcut->skip_to_level;
++		} else {
++			next_sc_level = sc_level + ASSOC_ARRAY_KEY_CHUNK_SIZE;
++			next_sc_level = round_down(next_sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
++		}
++
++		if (dissimilarity != 0) {
++			/* This shortcut points elsewhere */
++			result->wrong_shortcut.shortcut = shortcut;
++			result->wrong_shortcut.level = level;
++			result->wrong_shortcut.sc_level = sc_level;
++			result->wrong_shortcut.sc_segments = sc_segments;
++			result->wrong_shortcut.dissimilarity = dissimilarity;
++			return assoc_array_walk_found_wrong_shortcut;
++		}
++
++		sc_level = next_sc_level;
++	} while (sc_level < shortcut->skip_to_level);
++
++	/* The shortcut matches the leaf's index to this point. */
++	cursor = ACCESS_ONCE(shortcut->next_node);
++	if (((level ^ sc_level) & ~ASSOC_ARRAY_KEY_CHUNK_MASK) != 0) {
++		level = sc_level;
++		goto jumped;
++	} else {
++		level = sc_level;
++		goto consider_node;
++	}
++}
++
++/**
++ * assoc_array_find - Find an object by index key
++ * @array: The associative array to search.
++ * @ops: The operations to use.
++ * @index_key: The key to the object.
++ *
++ * Find an object in an associative array by walking through the internal tree
++ * to the node that should contain the object and then searching the leaves
++ * there.  NULL is returned if the requested object was not found in the array.
++ *
++ * The caller must hold the RCU read lock or better.
++ */
++void *assoc_array_find(const struct assoc_array *array,
++		       const struct assoc_array_ops *ops,
++		       const void *index_key)
++{
++	struct assoc_array_walk_result result;
++	const struct assoc_array_node *node;
++	const struct assoc_array_ptr *ptr;
++	const void *leaf;
++	int slot;
++
++	if (assoc_array_walk(array, ops, index_key, &result) !=
++	    assoc_array_walk_found_terminal_node)
++		return NULL;
++
++	node = result.terminal_node.node;
++	smp_read_barrier_depends();
++
++	/* If the target key is available to us, it's has to be pointed to by
++	 * the terminal node.
++	 */
++	for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++		ptr = ACCESS_ONCE(node->slots[slot]);
++		if (ptr && assoc_array_ptr_is_leaf(ptr)) {
++			/* We need a barrier between the read of the pointer
++			 * and dereferencing the pointer - but only if we are
++			 * actually going to dereference it.
++			 */
++			leaf = assoc_array_ptr_to_leaf(ptr);
++			smp_read_barrier_depends();
++			if (ops->compare_object(leaf, index_key))
++				return (void *)leaf;
++		}
++	}
++
++	return NULL;
++}
++
++/*
++ * Destructively iterate over an associative array.  The caller must prevent
++ * other simultaneous accesses.
++ */
++static void assoc_array_destroy_subtree(struct assoc_array_ptr *root,
++					const struct assoc_array_ops *ops)
++{
++	struct assoc_array_shortcut *shortcut;
++	struct assoc_array_node *node;
++	struct assoc_array_ptr *cursor, *parent = NULL;
++	int slot = -1;
++
++	pr_devel("-->%s()\n", __func__);
++
++	cursor = root;
++	if (!cursor) {
++		pr_devel("empty\n");
++		return;
++	}
++
++move_to_meta:
++	if (assoc_array_ptr_is_shortcut(cursor)) {
++		/* Descend through a shortcut */
++		pr_devel("[%d] shortcut\n", slot);
++		BUG_ON(!assoc_array_ptr_is_shortcut(cursor));
++		shortcut = assoc_array_ptr_to_shortcut(cursor);
++		BUG_ON(shortcut->back_pointer != parent);
++		BUG_ON(slot != -1 && shortcut->parent_slot != slot);
++		parent = cursor;
++		cursor = shortcut->next_node;
++		slot = -1;
++		BUG_ON(!assoc_array_ptr_is_node(cursor));
++	}
++
++	pr_devel("[%d] node\n", slot);
++	node = assoc_array_ptr_to_node(cursor);
++	BUG_ON(node->back_pointer != parent);
++	BUG_ON(slot != -1 && node->parent_slot != slot);
++	slot = 0;
++
++continue_node:
++	pr_devel("Node %p [back=%p]\n", node, node->back_pointer);
++	for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++		struct assoc_array_ptr *ptr = node->slots[slot];
++		if (!ptr)
++			continue;
++		if (assoc_array_ptr_is_meta(ptr)) {
++			parent = cursor;
++			cursor = ptr;
++			goto move_to_meta;
++		}
++
++		if (ops) {
++			pr_devel("[%d] free leaf\n", slot);
++			ops->free_object(assoc_array_ptr_to_leaf(ptr));
++		}
++	}
++
++	parent = node->back_pointer;
++	slot = node->parent_slot;
++	pr_devel("free node\n");
++	kfree(node);
++	if (!parent)
++		return; /* Done */
++
++	/* Move back up to the parent (may need to free a shortcut on
++	 * the way up) */
++	if (assoc_array_ptr_is_shortcut(parent)) {
++		shortcut = assoc_array_ptr_to_shortcut(parent);
++		BUG_ON(shortcut->next_node != cursor);
++		cursor = parent;
++		parent = shortcut->back_pointer;
++		slot = shortcut->parent_slot;
++		pr_devel("free shortcut\n");
++		kfree(shortcut);
++		if (!parent)
++			return;
++
++		BUG_ON(!assoc_array_ptr_is_node(parent));
++	}
++
++	/* Ascend to next slot in parent node */
++	pr_devel("ascend to %p[%d]\n", parent, slot);
++	cursor = parent;
++	node = assoc_array_ptr_to_node(cursor);
++	slot++;
++	goto continue_node;
++}
++
++/**
++ * assoc_array_destroy - Destroy an associative array
++ * @array: The array to destroy.
++ * @ops: The operations to use.
++ *
++ * Discard all metadata and free all objects in an associative array.  The
++ * array will be empty and ready to use again upon completion.  This function
++ * cannot fail.
++ *
++ * The caller must prevent all other accesses whilst this takes place as no
++ * attempt is made to adjust pointers gracefully to permit RCU readlock-holding
++ * accesses to continue.  On the other hand, no memory allocation is required.
++ */
++void assoc_array_destroy(struct assoc_array *array,
++			 const struct assoc_array_ops *ops)
++{
++	assoc_array_destroy_subtree(array->root, ops);
++	array->root = NULL;
++}
++
++/*
++ * Handle insertion into an empty tree.
++ */
++static bool assoc_array_insert_in_empty_tree(struct assoc_array_edit *edit)
++{
++	struct assoc_array_node *new_n0;
++
++	pr_devel("-->%s()\n", __func__);
++
++	new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
++	if (!new_n0)
++		return false;
++
++	edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
++	edit->leaf_p = &new_n0->slots[0];
++	edit->adjust_count_on = new_n0;
++	edit->set[0].ptr = &edit->array->root;
++	edit->set[0].to = assoc_array_node_to_ptr(new_n0);
++
++	pr_devel("<--%s() = ok [no root]\n", __func__);
++	return true;
++}
++
++/*
++ * Handle insertion into a terminal node.
++ */
++static bool assoc_array_insert_into_terminal_node(struct assoc_array_edit *edit,
++						  const struct assoc_array_ops *ops,
++						  const void *index_key,
++						  struct assoc_array_walk_result *result)
++{
++	struct assoc_array_shortcut *shortcut, *new_s0;
++	struct assoc_array_node *node, *new_n0, *new_n1, *side;
++	struct assoc_array_ptr *ptr;
++	unsigned long dissimilarity, base_seg, blank;
++	size_t keylen;
++	bool have_meta;
++	int level, diff;
++	int slot, next_slot, free_slot, i, j;
++
++	node	= result->terminal_node.node;
++	level	= result->terminal_node.level;
++	edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = result->terminal_node.slot;
++
++	pr_devel("-->%s()\n", __func__);
++
++	/* We arrived at a node which doesn't have an onward node or shortcut
++	 * pointer that we have to follow.  This means that (a) the leaf we
++	 * want must go here (either by insertion or replacement) or (b) we
++	 * need to split this node and insert in one of the fragments.
++	 */
++	free_slot = -1;
++
++	/* Firstly, we have to check the leaves in this node to see if there's
++	 * a matching one we should replace in place.
++	 */
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++		ptr = node->slots[i];
++		if (!ptr) {
++			free_slot = i;
++			continue;
++		}
++		if (ops->compare_object(assoc_array_ptr_to_leaf(ptr), index_key)) {
++			pr_devel("replace in slot %d\n", i);
++			edit->leaf_p = &node->slots[i];
++			edit->dead_leaf = node->slots[i];
++			pr_devel("<--%s() = ok [replace]\n", __func__);
++			return true;
++		}
++	}
++
++	/* If there is a free slot in this node then we can just insert the
++	 * leaf here.
++	 */
++	if (free_slot >= 0) {
++		pr_devel("insert in free slot %d\n", free_slot);
++		edit->leaf_p = &node->slots[free_slot];
++		edit->adjust_count_on = node;
++		pr_devel("<--%s() = ok [insert]\n", __func__);
++		return true;
++	}
++
++	/* The node has no spare slots - so we're either going to have to split
++	 * it or insert another node before it.
++	 *
++	 * Whatever, we're going to need at least two new nodes - so allocate
++	 * those now.  We may also need a new shortcut, but we deal with that
++	 * when we need it.
++	 */
++	new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
++	if (!new_n0)
++		return false;
++	edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
++	new_n1 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
++	if (!new_n1)
++		return false;
++	edit->new_meta[1] = assoc_array_node_to_ptr(new_n1);
++
++	/* We need to find out how similar the leaves are. */
++	pr_devel("no spare slots\n");
++	have_meta = false;
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++		ptr = node->slots[i];
++		if (assoc_array_ptr_is_meta(ptr)) {
++			edit->segment_cache[i] = 0xff;
++			have_meta = true;
++			continue;
++		}
++		base_seg = ops->get_object_key_chunk(
++			assoc_array_ptr_to_leaf(ptr), level);
++		base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
++		edit->segment_cache[i] = base_seg & ASSOC_ARRAY_FAN_MASK;
++	}
++
++	if (have_meta) {
++		pr_devel("have meta\n");
++		goto split_node;
++	}
++
++	/* The node contains only leaves */
++	dissimilarity = 0;
++	base_seg = edit->segment_cache[0];
++	for (i = 1; i < ASSOC_ARRAY_FAN_OUT; i++)
++		dissimilarity |= edit->segment_cache[i] ^ base_seg;
++
++	pr_devel("only leaves; dissimilarity=%lx\n", dissimilarity);
++
++	if ((dissimilarity & ASSOC_ARRAY_FAN_MASK) == 0) {
++		/* The old leaves all cluster in the same slot.  We will need
++		 * to insert a shortcut if the new node wants to cluster with them.
++		 */
++		if ((edit->segment_cache[ASSOC_ARRAY_FAN_OUT] ^ base_seg) == 0)
++			goto all_leaves_cluster_together;
++
++		/* Otherwise we can just insert a new node ahead of the old
++		 * one.
++		 */
++		goto present_leaves_cluster_but_not_new_leaf;
++	}
++
++split_node:
++	pr_devel("split node\n");
++
++	/* We need to split the current node; we know that the node doesn't
++	 * simply contain a full set of leaves that cluster together (it
++	 * contains meta pointers and/or non-clustering leaves).
++	 *
++	 * We need to expel at least two leaves out of a set consisting of the
++	 * leaves in the node and the new leaf.
++	 *
++	 * We need a new node (n0) to replace the current one and a new node to
++	 * take the expelled nodes (n1).
++	 */
++	edit->set[0].to = assoc_array_node_to_ptr(new_n0);
++	new_n0->back_pointer = node->back_pointer;
++	new_n0->parent_slot = node->parent_slot;
++	new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
++	new_n1->parent_slot = -1; /* Need to calculate this */
++
++do_split_node:
++	pr_devel("do_split_node\n");
++
++	new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
++	new_n1->nr_leaves_on_branch = 0;
++
++	/* Begin by finding two matching leaves.  There have to be at least two
++	 * that match - even if there are meta pointers - because any leaf that
++	 * would match a slot with a meta pointer in it must be somewhere
++	 * behind that meta pointer and cannot be here.  Further, given N
++	 * remaining leaf slots, we now have N+1 leaves to go in them.
++	 */
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++		slot = edit->segment_cache[i];
++		if (slot != 0xff)
++			for (j = i + 1; j < ASSOC_ARRAY_FAN_OUT + 1; j++)
++				if (edit->segment_cache[j] == slot)
++					goto found_slot_for_multiple_occupancy;
++	}
++found_slot_for_multiple_occupancy:
++	pr_devel("same slot: %x %x [%02x]\n", i, j, slot);
++	BUG_ON(i >= ASSOC_ARRAY_FAN_OUT);
++	BUG_ON(j >= ASSOC_ARRAY_FAN_OUT + 1);
++	BUG_ON(slot >= ASSOC_ARRAY_FAN_OUT);
++
++	new_n1->parent_slot = slot;
++
++	/* Metadata pointers cannot change slot */
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++)
++		if (assoc_array_ptr_is_meta(node->slots[i]))
++			new_n0->slots[i] = node->slots[i];
++		else
++			new_n0->slots[i] = NULL;
++	BUG_ON(new_n0->slots[slot] != NULL);
++	new_n0->slots[slot] = assoc_array_node_to_ptr(new_n1);
++
++	/* Filter the leaf pointers between the new nodes */
++	free_slot = -1;
++	next_slot = 0;
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++		if (assoc_array_ptr_is_meta(node->slots[i]))
++			continue;
++		if (edit->segment_cache[i] == slot) {
++			new_n1->slots[next_slot++] = node->slots[i];
++			new_n1->nr_leaves_on_branch++;
++		} else {
++			do {
++				free_slot++;
++			} while (new_n0->slots[free_slot] != NULL);
++			new_n0->slots[free_slot] = node->slots[i];
++		}
++	}
++
++	pr_devel("filtered: f=%x n=%x\n", free_slot, next_slot);
++
++	if (edit->segment_cache[ASSOC_ARRAY_FAN_OUT] != slot) {
++		do {
++			free_slot++;
++		} while (new_n0->slots[free_slot] != NULL);
++		edit->leaf_p = &new_n0->slots[free_slot];
++		edit->adjust_count_on = new_n0;
++	} else {
++		edit->leaf_p = &new_n1->slots[next_slot++];
++		edit->adjust_count_on = new_n1;
++	}
++
++	BUG_ON(next_slot <= 1);
++
++	edit->set_backpointers_to = assoc_array_node_to_ptr(new_n0);
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++		if (edit->segment_cache[i] == 0xff) {
++			ptr = node->slots[i];
++			BUG_ON(assoc_array_ptr_is_leaf(ptr));
++			if (assoc_array_ptr_is_node(ptr)) {
++				side = assoc_array_ptr_to_node(ptr);
++				edit->set_backpointers[i] = &side->back_pointer;
++			} else {
++				shortcut = assoc_array_ptr_to_shortcut(ptr);
++				edit->set_backpointers[i] = &shortcut->back_pointer;
++			}
++		}
++	}
++
++	ptr = node->back_pointer;
++	if (!ptr)
++		edit->set[0].ptr = &edit->array->root;
++	else if (assoc_array_ptr_is_node(ptr))
++		edit->set[0].ptr = &assoc_array_ptr_to_node(ptr)->slots[node->parent_slot];
++	else
++		edit->set[0].ptr = &assoc_array_ptr_to_shortcut(ptr)->next_node;
++	edit->excised_meta[0] = assoc_array_node_to_ptr(node);
++	pr_devel("<--%s() = ok [split node]\n", __func__);
++	return true;
++
++present_leaves_cluster_but_not_new_leaf:
++	/* All the old leaves cluster in the same slot, but the new leaf wants
++	 * to go into a different slot, so we create a new node to hold the new
++	 * leaf and a pointer to a new node holding all the old leaves.
++	 */
++	pr_devel("present leaves cluster but not new leaf\n");
++
++	new_n0->back_pointer = node->back_pointer;
++	new_n0->parent_slot = node->parent_slot;
++	new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
++	new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
++	new_n1->parent_slot = edit->segment_cache[0];
++	new_n1->nr_leaves_on_branch = node->nr_leaves_on_branch;
++	edit->adjust_count_on = new_n0;
++
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++)
++		new_n1->slots[i] = node->slots[i];
++
++	new_n0->slots[edit->segment_cache[0]] = assoc_array_node_to_ptr(new_n0);
++	edit->leaf_p = &new_n0->slots[edit->segment_cache[ASSOC_ARRAY_FAN_OUT]];
++
++	edit->set[0].ptr = &assoc_array_ptr_to_node(node->back_pointer)->slots[node->parent_slot];
++	edit->set[0].to = assoc_array_node_to_ptr(new_n0);
++	edit->excised_meta[0] = assoc_array_node_to_ptr(node);
++	pr_devel("<--%s() = ok [insert node before]\n", __func__);
++	return true;
++
++all_leaves_cluster_together:
++	/* All the leaves, new and old, want to cluster together in this node
++	 * in the same slot, so we have to replace this node with a shortcut to
++	 * skip over the identical parts of the key and then place a pair of
++	 * nodes, one inside the other, at the end of the shortcut and
++	 * distribute the keys between them.
++	 *
++	 * Firstly we need to work out where the leaves start diverging as a
++	 * bit position into their keys so that we know how big the shortcut
++	 * needs to be.
++	 *
++	 * We only need to make a single pass of N of the N+1 leaves because if
++	 * any keys differ between themselves at bit X then at least one of
++	 * them must also differ with the base key at bit X or before.
++	 */
++	pr_devel("all leaves cluster together\n");
++	diff = INT_MAX;
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++		int x = ops->diff_objects(assoc_array_ptr_to_leaf(edit->leaf),
++					  assoc_array_ptr_to_leaf(node->slots[i]));
++		if (x < diff) {
++			BUG_ON(x < 0);
++			diff = x;
++		}
++	}
++	BUG_ON(diff == INT_MAX);
++	BUG_ON(diff < level + ASSOC_ARRAY_LEVEL_STEP);
++
++	keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE);
++	keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
++
++	new_s0 = kzalloc(sizeof(struct assoc_array_shortcut) +
++			 keylen * sizeof(unsigned long), GFP_KERNEL);
++	if (!new_s0)
++		return false;
++	edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s0);
++
++	edit->set[0].to = assoc_array_shortcut_to_ptr(new_s0);
++	new_s0->back_pointer = node->back_pointer;
++	new_s0->parent_slot = node->parent_slot;
++	new_s0->next_node = assoc_array_node_to_ptr(new_n0);
++	new_n0->back_pointer = assoc_array_shortcut_to_ptr(new_s0);
++	new_n0->parent_slot = 0;
++	new_n1->back_pointer = assoc_array_node_to_ptr(new_n0);
++	new_n1->parent_slot = -1; /* Need to calculate this */
++
++	new_s0->skip_to_level = level = diff & ~ASSOC_ARRAY_LEVEL_STEP_MASK;
++	pr_devel("skip_to_level = %d [diff %d]\n", level, diff);
++	BUG_ON(level <= 0);
++
++	for (i = 0; i < keylen; i++)
++		new_s0->index_key[i] =
++			ops->get_key_chunk(index_key, i * ASSOC_ARRAY_KEY_CHUNK_SIZE);
++
++	blank = ULONG_MAX << (level & ASSOC_ARRAY_KEY_CHUNK_MASK);
++	pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, level, blank);
++	new_s0->index_key[keylen - 1] &= ~blank;
++
++	/* This now reduces to a node splitting exercise for which we'll need
++	 * to regenerate the disparity table.
++	 */
++	for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++		ptr = node->slots[i];
++		base_seg = ops->get_object_key_chunk(assoc_array_ptr_to_leaf(ptr),
++						     level);
++		base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
++		edit->segment_cache[i] = base_seg & ASSOC_ARRAY_FAN_MASK;
++	}
++
++	base_seg = ops->get_key_chunk(index_key, level);
++	base_seg >>= level & ASSOC_ARRAY_KEY_CHUNK_MASK;
++	edit->segment_cache[ASSOC_ARRAY_FAN_OUT] = base_seg & ASSOC_ARRAY_FAN_MASK;
++	goto do_split_node;
++}
++
++/*
++ * Handle insertion into the middle of a shortcut.
++ */
++static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit,
++					    const struct assoc_array_ops *ops,
++					    struct assoc_array_walk_result *result)
++{
++	struct assoc_array_shortcut *shortcut, *new_s0, *new_s1;
++	struct assoc_array_node *node, *new_n0, *side;
++	unsigned long sc_segments, dissimilarity, blank;
++	size_t keylen;
++	int level, sc_level, diff;
++	int sc_slot;
++
++	shortcut	= result->wrong_shortcut.shortcut;
++	level		= result->wrong_shortcut.level;
++	sc_level	= result->wrong_shortcut.sc_level;
++	sc_segments	= result->wrong_shortcut.sc_segments;
++	dissimilarity	= result->wrong_shortcut.dissimilarity;
++
++	pr_devel("-->%s(ix=%d dis=%lx scix=%d)\n",
++		 __func__, level, dissimilarity, sc_level);
++
++	/* We need to split a shortcut and insert a node between the two
++	 * pieces.  Zero-length pieces will be dispensed with entirely.
++	 *
++	 * First of all, we need to find out in which level the first
++	 * difference was.
++	 */
++	diff = __ffs(dissimilarity);
++	diff &= ~ASSOC_ARRAY_LEVEL_STEP_MASK;
++	diff += sc_level & ~ASSOC_ARRAY_KEY_CHUNK_MASK;
++	pr_devel("diff=%d\n", diff);
++
++	if (!shortcut->back_pointer) {
++		edit->set[0].ptr = &edit->array->root;
++	} else if (assoc_array_ptr_is_node(shortcut->back_pointer)) {
++		node = assoc_array_ptr_to_node(shortcut->back_pointer);
++		edit->set[0].ptr = &node->slots[shortcut->parent_slot];
++	} else {
++		BUG();
++	}
++
++	edit->excised_meta[0] = assoc_array_shortcut_to_ptr(shortcut);
++
++	/* Create a new node now since we're going to need it anyway */
++	new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
++	if (!new_n0)
++		return false;
++	edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
++	edit->adjust_count_on = new_n0;
++
++	/* Insert a new shortcut before the new node if this segment isn't of
++	 * zero length - otherwise we just connect the new node directly to the
++	 * parent.
++	 */
++	level += ASSOC_ARRAY_LEVEL_STEP;
++	if (diff > level) {
++		pr_devel("pre-shortcut %d...%d\n", level, diff);
++		keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE);
++		keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
++
++		new_s0 = kzalloc(sizeof(struct assoc_array_shortcut) +
++				 keylen * sizeof(unsigned long), GFP_KERNEL);
++		if (!new_s0)
++			return false;
++		edit->new_meta[1] = assoc_array_shortcut_to_ptr(new_s0);
++		edit->set[0].to = assoc_array_shortcut_to_ptr(new_s0);
++		new_s0->back_pointer = shortcut->back_pointer;
++		new_s0->parent_slot = shortcut->parent_slot;
++		new_s0->next_node = assoc_array_node_to_ptr(new_n0);
++		new_s0->skip_to_level = diff;
++
++		new_n0->back_pointer = assoc_array_shortcut_to_ptr(new_s0);
++		new_n0->parent_slot = 0;
++
++		memcpy(new_s0->index_key, shortcut->index_key,
++		       keylen * sizeof(unsigned long));
++
++		blank = ULONG_MAX << (diff & ASSOC_ARRAY_KEY_CHUNK_MASK);
++		pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, diff, blank);
++		new_s0->index_key[keylen - 1] &= ~blank;
++	} else {
++		pr_devel("no pre-shortcut\n");
++		edit->set[0].to = assoc_array_node_to_ptr(new_n0);
++		new_n0->back_pointer = shortcut->back_pointer;
++		new_n0->parent_slot = shortcut->parent_slot;
++	}
++
++	side = assoc_array_ptr_to_node(shortcut->next_node);
++	new_n0->nr_leaves_on_branch = side->nr_leaves_on_branch;
++
++	/* We need to know which slot in the new node is going to take a
++	 * metadata pointer.
++	 */
++	sc_slot = sc_segments >> (diff & ASSOC_ARRAY_KEY_CHUNK_MASK);
++	sc_slot &= ASSOC_ARRAY_FAN_MASK;
++
++	pr_devel("new slot %lx >> %d -> %d\n",
++		 sc_segments, diff & ASSOC_ARRAY_KEY_CHUNK_MASK, sc_slot);
++
++	/* Determine whether we need to follow the new node with a replacement
++	 * for the current shortcut.  We could in theory reuse the current
++	 * shortcut if its parent slot number doesn't change - but that's a
++	 * 1-in-16 chance so not worth expending the code upon.
++	 */
++	level = diff + ASSOC_ARRAY_LEVEL_STEP;
++	if (level < shortcut->skip_to_level) {
++		pr_devel("post-shortcut %d...%d\n", level, shortcut->skip_to_level);
++		keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
++		keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
++
++		new_s1 = kzalloc(sizeof(struct assoc_array_shortcut) +
++				 keylen * sizeof(unsigned long), GFP_KERNEL);
++		if (!new_s1)
++			return false;
++		edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s1);
++
++		new_s1->back_pointer = assoc_array_node_to_ptr(new_n0);
++		new_s1->parent_slot = sc_slot;
++		new_s1->next_node = shortcut->next_node;
++		new_s1->skip_to_level = shortcut->skip_to_level;
++
++		new_n0->slots[sc_slot] = assoc_array_shortcut_to_ptr(new_s1);
++
++		memcpy(new_s1->index_key, shortcut->index_key,
++		       keylen * sizeof(unsigned long));
++
++		edit->set[1].ptr = &side->back_pointer;
++		edit->set[1].to = assoc_array_shortcut_to_ptr(new_s1);
++	} else {
++		pr_devel("no post-shortcut\n");
++
++		/* We don't have to replace the pointed-to node as long as we
++		 * use memory barriers to make sure the parent slot number is
++		 * changed before the back pointer (the parent slot number is
++		 * irrelevant to the old parent shortcut).
++		 */
++		new_n0->slots[sc_slot] = shortcut->next_node;
++		edit->set_parent_slot[0].p = &side->parent_slot;
++		edit->set_parent_slot[0].to = sc_slot;
++		edit->set[1].ptr = &side->back_pointer;
++		edit->set[1].to = assoc_array_node_to_ptr(new_n0);
++	}
++
++	/* Install the new leaf in a spare slot in the new node. */
++	if (sc_slot == 0)
++		edit->leaf_p = &new_n0->slots[1];
++	else
++		edit->leaf_p = &new_n0->slots[0];
++
++	pr_devel("<--%s() = ok [split shortcut]\n", __func__);
++	return edit;
++}
++
++/**
++ * assoc_array_insert - Script insertion of an object into an associative array
++ * @array: The array to insert into.
++ * @ops: The operations to use.
++ * @index_key: The key to insert at.
++ * @object: The object to insert.
++ *
++ * Precalculate and preallocate a script for the insertion or replacement of an
++ * object in an associative array.  This results in an edit script that can
++ * either be applied or cancelled.
++ *
++ * The function returns a pointer to an edit script or -ENOMEM.
++ *
++ * The caller should lock against other modifications and must continue to hold
++ * the lock until assoc_array_apply_edit() has been called.
++ *
++ * Accesses to the tree may take place concurrently with this function,
++ * provided they hold the RCU read lock.
++ */
++struct assoc_array_edit *assoc_array_insert(struct assoc_array *array,
++					    const struct assoc_array_ops *ops,
++					    const void *index_key,
++					    void *object)
++{
++	struct assoc_array_walk_result result;
++	struct assoc_array_edit *edit;
++
++	pr_devel("-->%s()\n", __func__);
++
++	/* The leaf pointer we're given must not have the bottom bit set as we
++	 * use those for type-marking the pointer.  NULL pointers are also not
++	 * allowed as they indicate an empty slot but we have to allow them
++	 * here as they can be updated later.
++	 */
++	BUG_ON(assoc_array_ptr_is_meta(object));
++
++	edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
++	if (!edit)
++		return ERR_PTR(-ENOMEM);
++	edit->array = array;
++	edit->ops = ops;
++	edit->leaf = assoc_array_leaf_to_ptr(object);
++	edit->adjust_count_by = 1;
++
++	switch (assoc_array_walk(array, ops, index_key, &result)) {
++	case assoc_array_walk_tree_empty:
++		/* Allocate a root node if there isn't one yet */
++		if (!assoc_array_insert_in_empty_tree(edit))
++			goto enomem;
++		return edit;
++
++	case assoc_array_walk_found_terminal_node:
++		/* We found a node that doesn't have a node/shortcut pointer in
++		 * the slot corresponding to the index key that we have to
++		 * follow.
++		 */
++		if (!assoc_array_insert_into_terminal_node(edit, ops, index_key,
++							   &result))
++			goto enomem;
++		return edit;
++
++	case assoc_array_walk_found_wrong_shortcut:
++		/* We found a shortcut that didn't match our key in a slot we
++		 * needed to follow.
++		 */
++		if (!assoc_array_insert_mid_shortcut(edit, ops, &result))
++			goto enomem;
++		return edit;
++	}
++
++enomem:
++	/* Clean up after an out of memory error */
++	pr_devel("enomem\n");
++	assoc_array_cancel_edit(edit);
++	return ERR_PTR(-ENOMEM);
++}
++
++/**
++ * assoc_array_insert_set_object - Set the new object pointer in an edit script
++ * @edit: The edit script to modify.
++ * @object: The object pointer to set.
++ *
++ * Change the object to be inserted in an edit script.  The object pointed to
++ * by the old object is not freed.  This must be done prior to applying the
++ * script.
++ */
++void assoc_array_insert_set_object(struct assoc_array_edit *edit, void *object)
++{
++	BUG_ON(!object);
++	edit->leaf = assoc_array_leaf_to_ptr(object);
++}
++
++struct assoc_array_delete_collapse_context {
++	struct assoc_array_node	*node;
++	const void		*skip_leaf;
++	int			slot;
++};
++
++/*
++ * Subtree collapse to node iterator.
++ */
++static int assoc_array_delete_collapse_iterator(const void *leaf,
++						void *iterator_data)
++{
++	struct assoc_array_delete_collapse_context *collapse = iterator_data;
++
++	if (leaf == collapse->skip_leaf)
++		return 0;
++
++	BUG_ON(collapse->slot >= ASSOC_ARRAY_FAN_OUT);
++
++	collapse->node->slots[collapse->slot++] = assoc_array_leaf_to_ptr(leaf);
++	return 0;
++}
++
++/**
++ * assoc_array_delete - Script deletion of an object from an associative array
++ * @array: The array to search.
++ * @ops: The operations to use.
++ * @index_key: The key to the object.
++ *
++ * Precalculate and preallocate a script for the deletion of an object from an
++ * associative array.  This results in an edit script that can either be
++ * applied or cancelled.
++ *
++ * The function returns a pointer to an edit script if the object was found,
++ * NULL if the object was not found or -ENOMEM.
++ *
++ * The caller should lock against other modifications and must continue to hold
++ * the lock until assoc_array_apply_edit() has been called.
++ *
++ * Accesses to the tree may take place concurrently with this function,
++ * provided they hold the RCU read lock.
++ */
++struct assoc_array_edit *assoc_array_delete(struct assoc_array *array,
++					    const struct assoc_array_ops *ops,
++					    const void *index_key)
++{
++	struct assoc_array_delete_collapse_context collapse;
++	struct assoc_array_walk_result result;
++	struct assoc_array_node *node, *new_n0;
++	struct assoc_array_edit *edit;
++	struct assoc_array_ptr *ptr;
++	bool has_meta;
++	int slot, i;
++
++	pr_devel("-->%s()\n", __func__);
++
++	edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
++	if (!edit)
++		return ERR_PTR(-ENOMEM);
++	edit->array = array;
++	edit->ops = ops;
++	edit->adjust_count_by = -1;
++
++	switch (assoc_array_walk(array, ops, index_key, &result)) {
++	case assoc_array_walk_found_terminal_node:
++		/* We found a node that should contain the leaf we've been
++		 * asked to remove - *if* it's in the tree.
++		 */
++		pr_devel("terminal_node\n");
++		node = result.terminal_node.node;
++
++		for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++			ptr = node->slots[slot];
++			if (ptr &&
++			    assoc_array_ptr_is_leaf(ptr) &&
++			    ops->compare_object(assoc_array_ptr_to_leaf(ptr),
++						index_key))
++				goto found_leaf;
++		}
++	case assoc_array_walk_tree_empty:
++	case assoc_array_walk_found_wrong_shortcut:
++	default:
++		assoc_array_cancel_edit(edit);
++		pr_devel("not found\n");
++		return NULL;
++	}
++
++found_leaf:
++	BUG_ON(array->nr_leaves_on_tree <= 0);
++
++	/* In the simplest form of deletion we just clear the slot and release
++	 * the leaf after a suitable interval.
++	 */
++	edit->dead_leaf = node->slots[slot];
++	edit->set[0].ptr = &node->slots[slot];
++	edit->set[0].to = NULL;
++	edit->adjust_count_on = node;
++
++	/* If that concludes erasure of the last leaf, then delete the entire
++	 * internal array.
++	 */
++	if (array->nr_leaves_on_tree == 1) {
++		edit->set[1].ptr = &array->root;
++		edit->set[1].to = NULL;
++		edit->adjust_count_on = NULL;
++		edit->excised_subtree = array->root;
++		pr_devel("all gone\n");
++		return edit;
++	}
++
++	/* However, we'd also like to clear up some metadata blocks if we
++	 * possibly can.
++	 *
++	 * We go for a simple algorithm of: if this node has FAN_OUT or fewer
++	 * leaves in it, then attempt to collapse it - and attempt to
++	 * recursively collapse up the tree.
++	 *
++	 * We could also try and collapse in partially filled subtrees to take
++	 * up space in this node.
++	 */
++	if (node->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1) {
++		struct assoc_array_node *parent, *grandparent;
++		struct assoc_array_ptr *ptr;
++
++		/* First of all, we need to know if this node has metadata so
++		 * that we don't try collapsing if all the leaves are already
++		 * here.
++		 */
++		has_meta = false;
++		for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++			ptr = node->slots[i];
++			if (assoc_array_ptr_is_meta(ptr)) {
++				has_meta = true;
++				break;
++			}
++		}
++
++		pr_devel("leaves: %ld [m=%d]\n",
++			 node->nr_leaves_on_branch - 1, has_meta);
++
++		/* Look further up the tree to see if we can collapse this node
++		 * into a more proximal node too.
++		 */
++		parent = node;
++	collapse_up:
++		pr_devel("collapse subtree: %ld\n", parent->nr_leaves_on_branch);
++
++		ptr = parent->back_pointer;
++		if (!ptr)
++			goto do_collapse;
++		if (assoc_array_ptr_is_shortcut(ptr)) {
++			struct assoc_array_shortcut *s = assoc_array_ptr_to_shortcut(ptr);
++			ptr = s->back_pointer;
++			if (!ptr)
++				goto do_collapse;
++		}
++
++		grandparent = assoc_array_ptr_to_node(ptr);
++		if (grandparent->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT + 1) {
++			parent = grandparent;
++			goto collapse_up;
++		}
++
++	do_collapse:
++		/* There's no point collapsing if the original node has no meta
++		 * pointers to discard and if we didn't merge into one of that
++		 * node's ancestry.
++		 */
++		if (has_meta || parent != node) {
++			node = parent;
++
++			/* Create a new node to collapse into */
++			new_n0 = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
++			if (!new_n0)
++				goto enomem;
++			edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
++
++			new_n0->back_pointer = node->back_pointer;
++			new_n0->parent_slot = node->parent_slot;
++			new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
++			edit->adjust_count_on = new_n0;
++
++			collapse.node = new_n0;
++			collapse.skip_leaf = assoc_array_ptr_to_leaf(edit->dead_leaf);
++			collapse.slot = 0;
++			assoc_array_subtree_iterate(assoc_array_node_to_ptr(node),
++						    node->back_pointer,
++						    assoc_array_delete_collapse_iterator,
++						    &collapse);
++			pr_devel("collapsed %d,%lu\n", collapse.slot, new_n0->nr_leaves_on_branch);
++			BUG_ON(collapse.slot != new_n0->nr_leaves_on_branch - 1);
++
++			if (!node->back_pointer) {
++				edit->set[1].ptr = &array->root;
++			} else if (assoc_array_ptr_is_leaf(node->back_pointer)) {
++				BUG();
++			} else if (assoc_array_ptr_is_node(node->back_pointer)) {
++				struct assoc_array_node *p =
++					assoc_array_ptr_to_node(node->back_pointer);
++				edit->set[1].ptr = &p->slots[node->parent_slot];
++			} else if (assoc_array_ptr_is_shortcut(node->back_pointer)) {
++				struct assoc_array_shortcut *s =
++					assoc_array_ptr_to_shortcut(node->back_pointer);
++				edit->set[1].ptr = &s->next_node;
++			}
++			edit->set[1].to = assoc_array_node_to_ptr(new_n0);
++			edit->excised_subtree = assoc_array_node_to_ptr(node);
++		}
++	}
++
++	return edit;
++
++enomem:
++	/* Clean up after an out of memory error */
++	pr_devel("enomem\n");
++	assoc_array_cancel_edit(edit);
++	return ERR_PTR(-ENOMEM);
++}
++
++/**
++ * assoc_array_clear - Script deletion of all objects from an associative array
++ * @array: The array to clear.
++ * @ops: The operations to use.
++ *
++ * Precalculate and preallocate a script for the deletion of all the objects
++ * from an associative array.  This results in an edit script that can either
++ * be applied or cancelled.
++ *
++ * The function returns a pointer to an edit script if there are objects to be
++ * deleted, NULL if there are no objects in the array or -ENOMEM.
++ *
++ * The caller should lock against other modifications and must continue to hold
++ * the lock until assoc_array_apply_edit() has been called.
++ *
++ * Accesses to the tree may take place concurrently with this function,
++ * provided they hold the RCU read lock.
++ */
++struct assoc_array_edit *assoc_array_clear(struct assoc_array *array,
++					   const struct assoc_array_ops *ops)
++{
++	struct assoc_array_edit *edit;
++
++	pr_devel("-->%s()\n", __func__);
++
++	if (!array->root)
++		return NULL;
++
++	edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
++	if (!edit)
++		return ERR_PTR(-ENOMEM);
++	edit->array = array;
++	edit->ops = ops;
++	edit->set[1].ptr = &array->root;
++	edit->set[1].to = NULL;
++	edit->excised_subtree = array->root;
++	edit->ops_for_excised_subtree = ops;
++	pr_devel("all gone\n");
++	return edit;
++}
++
++/*
++ * Handle the deferred destruction after an applied edit.
++ */
++static void assoc_array_rcu_cleanup(struct rcu_head *head)
++{
++	struct assoc_array_edit *edit =
++		container_of(head, struct assoc_array_edit, rcu);
++	int i;
++
++	pr_devel("-->%s()\n", __func__);
++
++	if (edit->dead_leaf)
++		edit->ops->free_object(assoc_array_ptr_to_leaf(edit->dead_leaf));
++	for (i = 0; i < ARRAY_SIZE(edit->excised_meta); i++)
++		if (edit->excised_meta[i])
++			kfree(assoc_array_ptr_to_node(edit->excised_meta[i]));
++
++	if (edit->excised_subtree) {
++		BUG_ON(assoc_array_ptr_is_leaf(edit->excised_subtree));
++		if (assoc_array_ptr_is_node(edit->excised_subtree)) {
++			struct assoc_array_node *n =
++				assoc_array_ptr_to_node(edit->excised_subtree);
++			n->back_pointer = NULL;
++		} else {
++			struct assoc_array_shortcut *s =
++				assoc_array_ptr_to_shortcut(edit->excised_subtree);
++			s->back_pointer = NULL;
++		}
++		assoc_array_destroy_subtree(edit->excised_subtree,
++					    edit->ops_for_excised_subtree);
++	}
++
++	kfree(edit);
++}
++
++/**
++ * assoc_array_apply_edit - Apply an edit script to an associative array
++ * @edit: The script to apply.
++ *
++ * Apply an edit script to an associative array to effect an insertion,
++ * deletion or clearance.  As the edit script includes preallocated memory,
++ * this is guaranteed not to fail.
++ *
++ * The edit script, dead objects and dead metadata will be scheduled for
++ * destruction after an RCU grace period to permit those doing read-only
++ * accesses on the array to continue to do so under the RCU read lock whilst
++ * the edit is taking place.
++ */
++void assoc_array_apply_edit(struct assoc_array_edit *edit)
++{
++	struct assoc_array_shortcut *shortcut;
++	struct assoc_array_node *node;
++	struct assoc_array_ptr *ptr;
++	int i;
++
++	pr_devel("-->%s()\n", __func__);
++
++	smp_wmb();
++	if (edit->leaf_p)
++		*edit->leaf_p = edit->leaf;
++
++	smp_wmb();
++	for (i = 0; i < ARRAY_SIZE(edit->set_parent_slot); i++)
++		if (edit->set_parent_slot[i].p)
++			*edit->set_parent_slot[i].p = edit->set_parent_slot[i].to;
++
++	smp_wmb();
++	for (i = 0; i < ARRAY_SIZE(edit->set_backpointers); i++)
++		if (edit->set_backpointers[i])
++			*edit->set_backpointers[i] = edit->set_backpointers_to;
++
++	smp_wmb();
++	for (i = 0; i < ARRAY_SIZE(edit->set); i++)
++		if (edit->set[i].ptr)
++			*edit->set[i].ptr = edit->set[i].to;
++
++	if (edit->array->root == NULL) {
++		edit->array->nr_leaves_on_tree = 0;
++	} else if (edit->adjust_count_on) {
++		node = edit->adjust_count_on;
++		for (;;) {
++			node->nr_leaves_on_branch += edit->adjust_count_by;
++
++			ptr = node->back_pointer;
++			if (!ptr)
++				break;
++			if (assoc_array_ptr_is_shortcut(ptr)) {
++				shortcut = assoc_array_ptr_to_shortcut(ptr);
++				ptr = shortcut->back_pointer;
++				if (!ptr)
++					break;
++			}
++			BUG_ON(!assoc_array_ptr_is_node(ptr));
++			node = assoc_array_ptr_to_node(ptr);
++		}
++
++		edit->array->nr_leaves_on_tree += edit->adjust_count_by;
++	}
++
++	call_rcu(&edit->rcu, assoc_array_rcu_cleanup);
++}
++
++/**
++ * assoc_array_cancel_edit - Discard an edit script.
++ * @edit: The script to discard.
++ *
++ * Free an edit script and all the preallocated data it holds without making
++ * any changes to the associative array it was intended for.
++ *
++ * NOTE!  In the case of an insertion script, this does _not_ release the leaf
++ * that was to be inserted.  That is left to the caller.
++ */
++void assoc_array_cancel_edit(struct assoc_array_edit *edit)
++{
++	struct assoc_array_ptr *ptr;
++	int i;
++
++	pr_devel("-->%s()\n", __func__);
++
++	/* Clean up after an out of memory error */
++	for (i = 0; i < ARRAY_SIZE(edit->new_meta); i++) {
++		ptr = edit->new_meta[i];
++		if (ptr) {
++			if (assoc_array_ptr_is_node(ptr))
++				kfree(assoc_array_ptr_to_node(ptr));
++			else
++				kfree(assoc_array_ptr_to_shortcut(ptr));
++		}
++	}
++	kfree(edit);
++}
++
++/**
++ * assoc_array_gc - Garbage collect an associative array.
++ * @array: The array to clean.
++ * @ops: The operations to use.
++ * @iterator: A callback function to pass judgement on each object.
++ * @iterator_data: Private data for the callback function.
++ *
++ * Collect garbage from an associative array and pack down the internal tree to
++ * save memory.
++ *
++ * The iterator function is asked to pass judgement upon each object in the
++ * array.  If it returns false, the object is discard and if it returns true,
++ * the object is kept.  If it returns true, it must increment the object's
++ * usage count (or whatever it needs to do to retain it) before returning.
++ *
++ * This function returns 0 if successful or -ENOMEM if out of memory.  In the
++ * latter case, the array is not changed.
++ *
++ * The caller should lock against other modifications and must continue to hold
++ * the lock until assoc_array_apply_edit() has been called.
++ *
++ * Accesses to the tree may take place concurrently with this function,
++ * provided they hold the RCU read lock.
++ */
++int assoc_array_gc(struct assoc_array *array,
++		   const struct assoc_array_ops *ops,
++		   bool (*iterator)(void *object, void *iterator_data),
++		   void *iterator_data)
++{
++	struct assoc_array_shortcut *shortcut, *new_s;
++	struct assoc_array_node *node, *new_n;
++	struct assoc_array_edit *edit;
++	struct assoc_array_ptr *cursor, *ptr;
++	struct assoc_array_ptr *new_root, *new_parent, **new_ptr_pp;
++	unsigned long nr_leaves_on_tree;
++	int keylen, slot, nr_free, next_slot, i;
++
++	pr_devel("-->%s()\n", __func__);
++
++	if (!array->root)
++		return 0;
++
++	edit = kzalloc(sizeof(struct assoc_array_edit), GFP_KERNEL);
++	if (!edit)
++		return -ENOMEM;
++	edit->array = array;
++	edit->ops = ops;
++	edit->ops_for_excised_subtree = ops;
++	edit->set[0].ptr = &array->root;
++	edit->excised_subtree = array->root;
++
++	new_root = new_parent = NULL;
++	new_ptr_pp = &new_root;
++	cursor = array->root;
++
++descend:
++	/* If this point is a shortcut, then we need to duplicate it and
++	 * advance the target cursor.
++	 */
++	if (assoc_array_ptr_is_shortcut(cursor)) {
++		shortcut = assoc_array_ptr_to_shortcut(cursor);
++		keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE);
++		keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT;
++		new_s = kmalloc(sizeof(struct assoc_array_shortcut) +
++				keylen * sizeof(unsigned long), GFP_KERNEL);
++		if (!new_s)
++			goto enomem;
++		pr_devel("dup shortcut %p -> %p\n", shortcut, new_s);
++		memcpy(new_s, shortcut, (sizeof(struct assoc_array_shortcut) +
++					 keylen * sizeof(unsigned long)));
++		new_s->back_pointer = new_parent;
++		new_s->parent_slot = shortcut->parent_slot;
++		*new_ptr_pp = new_parent = assoc_array_shortcut_to_ptr(new_s);
++		new_ptr_pp = &new_s->next_node;
++		cursor = shortcut->next_node;
++	}
++
++	/* Duplicate the node at this position */
++	node = assoc_array_ptr_to_node(cursor);
++	new_n = kzalloc(sizeof(struct assoc_array_node), GFP_KERNEL);
++	if (!new_n)
++		goto enomem;
++	pr_devel("dup node %p -> %p\n", node, new_n);
++	new_n->back_pointer = new_parent;
++	new_n->parent_slot = node->parent_slot;
++	*new_ptr_pp = new_parent = assoc_array_node_to_ptr(new_n);
++	new_ptr_pp = NULL;
++	slot = 0;
++
++continue_node:
++	/* Filter across any leaves and gc any subtrees */
++	for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++		ptr = node->slots[slot];
++		if (!ptr)
++			continue;
++
++		if (assoc_array_ptr_is_leaf(ptr)) {
++			if (iterator(assoc_array_ptr_to_leaf(ptr),
++				     iterator_data))
++				/* The iterator will have done any reference
++				 * counting on the object for us.
++				 */
++				new_n->slots[slot] = ptr;
++			continue;
++		}
++
++		new_ptr_pp = &new_n->slots[slot];
++		cursor = ptr;
++		goto descend;
++	}
++
++	pr_devel("-- compress node %p --\n", new_n);
++
++	/* Count up the number of empty slots in this node and work out the
++	 * subtree leaf count.
++	 */
++	new_n->nr_leaves_on_branch = 0;
++	nr_free = 0;
++	for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++		ptr = new_n->slots[slot];
++		if (!ptr)
++			nr_free++;
++		else if (assoc_array_ptr_is_leaf(ptr))
++			new_n->nr_leaves_on_branch++;
++	}
++	pr_devel("free=%d, leaves=%lu\n", nr_free, new_n->nr_leaves_on_branch);
++
++	/* See what we can fold in */
++	next_slot = 0;
++	for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++		struct assoc_array_shortcut *s;
++		struct assoc_array_node *child;
++
++		ptr = new_n->slots[slot];
++		if (!ptr || assoc_array_ptr_is_leaf(ptr))
++			continue;
++
++		s = NULL;
++		if (assoc_array_ptr_is_shortcut(ptr)) {
++			s = assoc_array_ptr_to_shortcut(ptr);
++			ptr = s->next_node;
++		}
++
++		child = assoc_array_ptr_to_node(ptr);
++		new_n->nr_leaves_on_branch += child->nr_leaves_on_branch;
++
++		if (child->nr_leaves_on_branch <= nr_free + 1) {
++			/* Fold the child node into this one */
++			pr_devel("[%d] fold node %lu/%d [nx %d]\n",
++				 slot, child->nr_leaves_on_branch, nr_free + 1,
++				 next_slot);
++
++			/* We would already have reaped an intervening shortcut
++			 * on the way back up the tree.
++			 */
++			BUG_ON(s);
++
++			new_n->slots[slot] = NULL;
++			nr_free++;
++			if (slot < next_slot)
++				next_slot = slot;
++			for (i = 0; i < ASSOC_ARRAY_FAN_OUT; i++) {
++				struct assoc_array_ptr *p = child->slots[i];
++				if (!p)
++					continue;
++				BUG_ON(assoc_array_ptr_is_meta(p));
++				while (new_n->slots[next_slot])
++					next_slot++;
++				BUG_ON(next_slot >= ASSOC_ARRAY_FAN_OUT);
++				new_n->slots[next_slot++] = p;
++				nr_free--;
++			}
++			kfree(child);
++		} else {
++			pr_devel("[%d] retain node %lu/%d [nx %d]\n",
++				 slot, child->nr_leaves_on_branch, nr_free + 1,
++				 next_slot);
++		}
++	}
++
++	pr_devel("after: %lu\n", new_n->nr_leaves_on_branch);
++
++	nr_leaves_on_tree = new_n->nr_leaves_on_branch;
++
++	/* Excise this node if it is singly occupied by a shortcut */
++	if (nr_free == ASSOC_ARRAY_FAN_OUT - 1) {
++		for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++)
++			if ((ptr = new_n->slots[slot]))
++				break;
++
++		if (assoc_array_ptr_is_meta(ptr) &&
++		    assoc_array_ptr_is_shortcut(ptr)) {
++			pr_devel("excise node %p with 1 shortcut\n", new_n);
++			new_s = assoc_array_ptr_to_shortcut(ptr);
++			new_parent = new_n->back_pointer;
++			slot = new_n->parent_slot;
++			kfree(new_n);
++			if (!new_parent) {
++				new_s->back_pointer = NULL;
++				new_s->parent_slot = 0;
++				new_root = ptr;
++				goto gc_complete;
++			}
++
++			if (assoc_array_ptr_is_shortcut(new_parent)) {
++				/* We can discard any preceding shortcut also */
++				struct assoc_array_shortcut *s =
++					assoc_array_ptr_to_shortcut(new_parent);
++
++				pr_devel("excise preceding shortcut\n");
++
++				new_parent = new_s->back_pointer = s->back_pointer;
++				slot = new_s->parent_slot = s->parent_slot;
++				kfree(s);
++				if (!new_parent) {
++					new_s->back_pointer = NULL;
++					new_s->parent_slot = 0;
++					new_root = ptr;
++					goto gc_complete;
++				}
++			}
++
++			new_s->back_pointer = new_parent;
++			new_s->parent_slot = slot;
++			new_n = assoc_array_ptr_to_node(new_parent);
++			new_n->slots[slot] = ptr;
++			goto ascend_old_tree;
++		}
++	}
++
++	/* Excise any shortcuts we might encounter that point to nodes that
++	 * only contain leaves.
++	 */
++	ptr = new_n->back_pointer;
++	if (!ptr)
++		goto gc_complete;
++
++	if (assoc_array_ptr_is_shortcut(ptr)) {
++		new_s = assoc_array_ptr_to_shortcut(ptr);
++		new_parent = new_s->back_pointer;
++		slot = new_s->parent_slot;
++
++		if (new_n->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT) {
++			struct assoc_array_node *n;
++
++			pr_devel("excise shortcut\n");
++			new_n->back_pointer = new_parent;
++			new_n->parent_slot = slot;
++			kfree(new_s);
++			if (!new_parent) {
++				new_root = assoc_array_node_to_ptr(new_n);
++				goto gc_complete;
++			}
++
++			n = assoc_array_ptr_to_node(new_parent);
++			n->slots[slot] = assoc_array_node_to_ptr(new_n);
++		}
++	} else {
++		new_parent = ptr;
++	}
++	new_n = assoc_array_ptr_to_node(new_parent);
++
++ascend_old_tree:
++	ptr = node->back_pointer;
++	if (assoc_array_ptr_is_shortcut(ptr)) {
++		shortcut = assoc_array_ptr_to_shortcut(ptr);
++		slot = shortcut->parent_slot;
++		cursor = shortcut->back_pointer;
++	} else {
++		slot = node->parent_slot;
++		cursor = ptr;
++	}
++	BUG_ON(!ptr);
++	node = assoc_array_ptr_to_node(cursor);
++	slot++;
++	goto continue_node;
++
++gc_complete:
++	edit->set[0].to = new_root;
++	assoc_array_apply_edit(edit);
++	edit->array->nr_leaves_on_tree = nr_leaves_on_tree;
++	return 0;
++
++enomem:
++	pr_devel("enomem\n");
++	assoc_array_destroy_subtree(new_root, edit->ops);
++	kfree(edit);
++	return -ENOMEM;
++}
+-- 
+1.8.3.1
+
+
+From 03ac60b84587fa8e57e7ec5cd3d59b7fa8d97c79 Mon Sep 17 00:00:00 2001
+From: David Howells <dhowells@redhat.com>
+Date: Fri, 30 Aug 2013 15:37:54 +0100
+Subject: [PATCH 10/10] KEYS: Expand the capacity of a keyring
+
+Expand the capacity of a keyring to be able to hold a lot more keys by using
+the previously added associative array implementation.  Currently the maximum
+capacity is:
+
+	(PAGE_SIZE - sizeof(header)) / sizeof(struct key *)
+
+which, on a 64-bit system, is a little more 500.  However, since this is being
+used for the NFS uid mapper, we need more than that.  The new implementation
+gives us effectively unlimited capacity.
+
+With some alterations, the keyutils testsuite runs successfully to completion
+after this patch is applied.  The alterations are because (a) keyrings that
+are simply added to no longer appear ordered and (b) some of the errors have
+changed a bit.
+
+Signed-off-by: David Howells <dhowells@redhat.com>
+---
+ include/keys/keyring-type.h |   17 +-
+ include/linux/key.h         |   13 +-
+ lib/assoc_array.c           |    1 +
+ security/keys/Kconfig       |    1 +
+ security/keys/gc.c          |   33 +-
+ security/keys/internal.h    |   17 +-
+ security/keys/key.c         |   35 +-
+ security/keys/keyring.c     | 1436 ++++++++++++++++++++++---------------------
+ security/keys/request_key.c |   12 +-
+ 9 files changed, 803 insertions(+), 762 deletions(-)
+
+diff --git a/include/keys/keyring-type.h b/include/keys/keyring-type.h
+index cf49159..fca5c62 100644
+--- a/include/keys/keyring-type.h
++++ b/include/keys/keyring-type.h
+@@ -1,6 +1,6 @@
+ /* Keyring key type
+  *
+- * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
++ * Copyright (C) 2008, 2013 Red Hat, Inc. All Rights Reserved.
+  * Written by David Howells (dhowells@redhat.com)
+  *
+  * This program is free software; you can redistribute it and/or
+@@ -13,19 +13,6 @@
+ #define _KEYS_KEYRING_TYPE_H
+ 
+ #include <linux/key.h>
+-#include <linux/rcupdate.h>
+-
+-/*
+- * the keyring payload contains a list of the keys to which the keyring is
+- * subscribed
+- */
+-struct keyring_list {
+-	struct rcu_head	rcu;		/* RCU deletion hook */
+-	unsigned short	maxkeys;	/* max keys this list can hold */
+-	unsigned short	nkeys;		/* number of keys currently held */
+-	unsigned short	delkey;		/* key to be unlinked by RCU */
+-	struct key __rcu *keys[0];
+-};
+-
++#include <linux/assoc_array.h>
+ 
+ #endif /* _KEYS_KEYRING_TYPE_H */
+diff --git a/include/linux/key.h b/include/linux/key.h
+index ef596c7..2417f78 100644
+--- a/include/linux/key.h
++++ b/include/linux/key.h
+@@ -22,6 +22,7 @@
+ #include <linux/sysctl.h>
+ #include <linux/rwsem.h>
+ #include <linux/atomic.h>
++#include <linux/assoc_array.h>
+ 
+ #ifdef __KERNEL__
+ #include <linux/uidgid.h>
+@@ -196,11 +197,13 @@ struct key {
+ 	 *   whatever
+ 	 */
+ 	union {
+-		unsigned long		value;
+-		void __rcu		*rcudata;
+-		void			*data;
+-		struct keyring_list __rcu *subscriptions;
+-	} payload;
++		union {
++			unsigned long		value;
++			void __rcu		*rcudata;
++			void			*data;
++		} payload;
++		struct assoc_array keys;
++	};
+ };
+ 
+ extern struct key *key_alloc(struct key_type *type,
+diff --git a/lib/assoc_array.c b/lib/assoc_array.c
+index a095281..17edeaf 100644
+--- a/lib/assoc_array.c
++++ b/lib/assoc_array.c
+@@ -12,6 +12,7 @@
+  */
+ //#define DEBUG
+ #include <linux/slab.h>
++#include <linux/err.h>
+ #include <linux/assoc_array_priv.h>
+ 
+ /*
+diff --git a/security/keys/Kconfig b/security/keys/Kconfig
+index a90d6d3..15e0dfe 100644
+--- a/security/keys/Kconfig
++++ b/security/keys/Kconfig
+@@ -4,6 +4,7 @@
+ 
+ config KEYS
+ 	bool "Enable access key retention support"
++	select ASSOCIATIVE_ARRAY
+ 	help
+ 	  This option provides support for retaining authentication tokens and
+ 	  access keys in the kernel.
+diff --git a/security/keys/gc.c b/security/keys/gc.c
+index d67c97b..cce621c 100644
+--- a/security/keys/gc.c
++++ b/security/keys/gc.c
+@@ -130,6 +130,13 @@ void key_gc_keytype(struct key_type *ktype)
+ 	kleave("");
+ }
+ 
++static int key_gc_keyring_func(const void *object, void *iterator_data)
++{
++	const struct key *key = object;
++	time_t *limit = iterator_data;
++	return key_is_dead(key, *limit);
++}
++
+ /*
+  * Garbage collect pointers from a keyring.
+  *
+@@ -138,10 +145,9 @@ void key_gc_keytype(struct key_type *ktype)
+  */
+ static void key_gc_keyring(struct key *keyring, time_t limit)
+ {
+-	struct keyring_list *klist;
+-	int loop;
++	int result;
+ 
+-	kenter("%x", key_serial(keyring));
++	kenter("%x{%s}", keyring->serial, keyring->description ?: "");
+ 
+ 	if (keyring->flags & ((1 << KEY_FLAG_INVALIDATED) |
+ 			      (1 << KEY_FLAG_REVOKED)))
+@@ -149,27 +155,17 @@ static void key_gc_keyring(struct key *keyring, time_t limit)
+ 
+ 	/* scan the keyring looking for dead keys */
+ 	rcu_read_lock();
+-	klist = rcu_dereference(keyring->payload.subscriptions);
+-	if (!klist)
+-		goto unlock_dont_gc;
+-
+-	loop = klist->nkeys;
+-	smp_rmb();
+-	for (loop--; loop >= 0; loop--) {
+-		struct key *key = rcu_dereference(klist->keys[loop]);
+-		if (key_is_dead(key, limit))
+-			goto do_gc;
+-	}
+-
+-unlock_dont_gc:
++	result = assoc_array_iterate(&keyring->keys,
++				     key_gc_keyring_func, &limit);
+ 	rcu_read_unlock();
++	if (result == true)
++		goto do_gc;
++
+ dont_gc:
+ 	kleave(" [no gc]");
+ 	return;
+ 
+ do_gc:
+-	rcu_read_unlock();
+-
+ 	keyring_gc(keyring, limit);
+ 	kleave(" [gc]");
+ }
+@@ -392,7 +388,6 @@ found_unreferenced_key:
+ 	 */
+ found_keyring:
+ 	spin_unlock(&key_serial_lock);
+-	kdebug("scan keyring %d", key->serial);
+ 	key_gc_keyring(key, limit);
+ 	goto maybe_resched;
+ 
+diff --git a/security/keys/internal.h b/security/keys/internal.h
+index 73950bf..581c6f6 100644
+--- a/security/keys/internal.h
++++ b/security/keys/internal.h
+@@ -90,20 +90,23 @@ extern void key_type_put(struct key_type *ktype);
+ 
+ extern int __key_link_begin(struct key *keyring,
+ 			    const struct keyring_index_key *index_key,
+-			    unsigned long *_prealloc);
++			    struct assoc_array_edit **_edit);
+ extern int __key_link_check_live_key(struct key *keyring, struct key *key);
+-extern void __key_link(struct key *keyring, struct key *key,
+-		       unsigned long *_prealloc);
++extern void __key_link(struct key *key, struct assoc_array_edit **_edit);
+ extern void __key_link_end(struct key *keyring,
+ 			   const struct keyring_index_key *index_key,
+-			   unsigned long prealloc);
++			   struct assoc_array_edit *edit);
+ 
+-extern key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+-				      const struct keyring_index_key *index_key);
++extern key_ref_t find_key_to_update(key_ref_t keyring_ref,
++				    const struct keyring_index_key *index_key);
+ 
+ extern struct key *keyring_search_instkey(struct key *keyring,
+ 					  key_serial_t target_id);
+ 
++extern int iterate_over_keyring(const struct key *keyring,
++				int (*func)(const struct key *key, void *data),
++				void *data);
++
+ typedef int (*key_match_func_t)(const struct key *, const void *);
+ 
+ struct keyring_search_context {
+@@ -119,6 +122,8 @@ struct keyring_search_context {
+ #define KEYRING_SEARCH_NO_CHECK_PERM	0x0010	/* Don't check permissions */
+ #define KEYRING_SEARCH_DETECT_TOO_DEEP	0x0020	/* Give an error on excessive depth */
+ 
++	int (*iterator)(const void *object, void *iterator_data);
++
+ 	/* Internal stuff */
+ 	int			skipped_ret;
+ 	bool			possessed;
+diff --git a/security/keys/key.c b/security/keys/key.c
+index 7d716b8..a819b5c 100644
+--- a/security/keys/key.c
++++ b/security/keys/key.c
+@@ -409,7 +409,7 @@ static int __key_instantiate_and_link(struct key *key,
+ 				      struct key_preparsed_payload *prep,
+ 				      struct key *keyring,
+ 				      struct key *authkey,
+-				      unsigned long *_prealloc)
++				      struct assoc_array_edit **_edit)
+ {
+ 	int ret, awaken;
+ 
+@@ -436,7 +436,7 @@ static int __key_instantiate_and_link(struct key *key,
+ 
+ 			/* and link it into the destination keyring */
+ 			if (keyring)
+-				__key_link(keyring, key, _prealloc);
++				__key_link(key, _edit);
+ 
+ 			/* disable the authorisation key */
+ 			if (authkey)
+@@ -476,7 +476,7 @@ int key_instantiate_and_link(struct key *key,
+ 			     struct key *authkey)
+ {
+ 	struct key_preparsed_payload prep;
+-	unsigned long prealloc;
++	struct assoc_array_edit *edit;
+ 	int ret;
+ 
+ 	memset(&prep, 0, sizeof(prep));
+@@ -490,16 +490,15 @@ int key_instantiate_and_link(struct key *key,
+ 	}
+ 
+ 	if (keyring) {
+-		ret = __key_link_begin(keyring, &key->index_key, &prealloc);
++		ret = __key_link_begin(keyring, &key->index_key, &edit);
+ 		if (ret < 0)
+ 			goto error_free_preparse;
+ 	}
+ 
+-	ret = __key_instantiate_and_link(key, &prep, keyring, authkey,
+-					 &prealloc);
++	ret = __key_instantiate_and_link(key, &prep, keyring, authkey, &edit);
+ 
+ 	if (keyring)
+-		__key_link_end(keyring, &key->index_key, prealloc);
++		__key_link_end(keyring, &key->index_key, edit);
+ 
+ error_free_preparse:
+ 	if (key->type->preparse)
+@@ -537,7 +536,7 @@ int key_reject_and_link(struct key *key,
+ 			struct key *keyring,
+ 			struct key *authkey)
+ {
+-	unsigned long prealloc;
++	struct assoc_array_edit *edit;
+ 	struct timespec now;
+ 	int ret, awaken, link_ret = 0;
+ 
+@@ -548,7 +547,7 @@ int key_reject_and_link(struct key *key,
+ 	ret = -EBUSY;
+ 
+ 	if (keyring)
+-		link_ret = __key_link_begin(keyring, &key->index_key, &prealloc);
++		link_ret = __key_link_begin(keyring, &key->index_key, &edit);
+ 
+ 	mutex_lock(&key_construction_mutex);
+ 
+@@ -570,7 +569,7 @@ int key_reject_and_link(struct key *key,
+ 
+ 		/* and link it into the destination keyring */
+ 		if (keyring && link_ret == 0)
+-			__key_link(keyring, key, &prealloc);
++			__key_link(key, &edit);
+ 
+ 		/* disable the authorisation key */
+ 		if (authkey)
+@@ -580,7 +579,7 @@ int key_reject_and_link(struct key *key,
+ 	mutex_unlock(&key_construction_mutex);
+ 
+ 	if (keyring)
+-		__key_link_end(keyring, &key->index_key, prealloc);
++		__key_link_end(keyring, &key->index_key, edit);
+ 
+ 	/* wake up anyone waiting for a key to be constructed */
+ 	if (awaken)
+@@ -783,8 +782,8 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 		.description	= description,
+ 	};
+ 	struct key_preparsed_payload prep;
++	struct assoc_array_edit *edit;
+ 	const struct cred *cred = current_cred();
+-	unsigned long prealloc;
+ 	struct key *keyring, *key = NULL;
+ 	key_ref_t key_ref;
+ 	int ret;
+@@ -828,7 +827,7 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 	}
+ 	index_key.desc_len = strlen(index_key.description);
+ 
+-	ret = __key_link_begin(keyring, &index_key, &prealloc);
++	ret = __key_link_begin(keyring, &index_key, &edit);
+ 	if (ret < 0) {
+ 		key_ref = ERR_PTR(ret);
+ 		goto error_free_prep;
+@@ -847,8 +846,8 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 	 * update that instead if possible
+ 	 */
+ 	if (index_key.type->update) {
+-		key_ref = __keyring_search_one(keyring_ref, &index_key);
+-		if (!IS_ERR(key_ref))
++		key_ref = find_key_to_update(keyring_ref, &index_key);
++		if (key_ref)
+ 			goto found_matching_key;
+ 	}
+ 
+@@ -874,7 +873,7 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 	}
+ 
+ 	/* instantiate it and link it into the target keyring */
+-	ret = __key_instantiate_and_link(key, &prep, keyring, NULL, &prealloc);
++	ret = __key_instantiate_and_link(key, &prep, keyring, NULL, &edit);
+ 	if (ret < 0) {
+ 		key_put(key);
+ 		key_ref = ERR_PTR(ret);
+@@ -884,7 +883,7 @@ key_ref_t key_create_or_update(key_ref_t keyring_ref,
+ 	key_ref = make_key_ref(key, is_key_possessed(keyring_ref));
+ 
+ error_link_end:
+-	__key_link_end(keyring, &index_key, prealloc);
++	__key_link_end(keyring, &index_key, edit);
+ error_free_prep:
+ 	if (index_key.type->preparse)
+ 		index_key.type->free_preparse(&prep);
+@@ -897,7 +896,7 @@ error:
+ 	/* we found a matching key, so we're going to try to update it
+ 	 * - we can drop the locks first as we have the key pinned
+ 	 */
+-	__key_link_end(keyring, &index_key, prealloc);
++	__key_link_end(keyring, &index_key, edit);
+ 
+ 	key_ref = __key_update(key_ref, &prep);
+ 	goto error_free_prep;
+diff --git a/security/keys/keyring.c b/security/keys/keyring.c
+index eeef1a0..f7cdea2 100644
+--- a/security/keys/keyring.c
++++ b/security/keys/keyring.c
+@@ -1,6 +1,6 @@
+ /* Keyring handling
+  *
+- * Copyright (C) 2004-2005, 2008 Red Hat, Inc. All Rights Reserved.
++ * Copyright (C) 2004-2005, 2008, 2013 Red Hat, Inc. All Rights Reserved.
+  * Written by David Howells (dhowells@redhat.com)
+  *
+  * This program is free software; you can redistribute it and/or
+@@ -17,25 +17,11 @@
+ #include <linux/seq_file.h>
+ #include <linux/err.h>
+ #include <keys/keyring-type.h>
++#include <keys/user-type.h>
++#include <linux/assoc_array_priv.h>
+ #include <linux/uaccess.h>
+ #include "internal.h"
+ 
+-#define rcu_dereference_locked_keyring(keyring)				\
+-	(rcu_dereference_protected(					\
+-		(keyring)->payload.subscriptions,			\
+-		rwsem_is_locked((struct rw_semaphore *)&(keyring)->sem)))
+-
+-#define rcu_deref_link_locked(klist, index, keyring)			\
+-	(rcu_dereference_protected(					\
+-		(klist)->keys[index],					\
+-		rwsem_is_locked((struct rw_semaphore *)&(keyring)->sem)))
+-
+-#define MAX_KEYRING_LINKS						\
+-	min_t(size_t, USHRT_MAX - 1,					\
+-	      ((PAGE_SIZE - sizeof(struct keyring_list)) / sizeof(struct key *)))
+-
+-#define KEY_LINK_FIXQUOTA 1UL
+-
+ /*
+  * When plumbing the depths of the key tree, this sets a hard limit
+  * set on how deep we're willing to go.
+@@ -47,6 +33,28 @@
+  */
+ #define KEYRING_NAME_HASH_SIZE	(1 << 5)
+ 
++/*
++ * We mark pointers we pass to the associative array with bit 1 set if
++ * they're keyrings and clear otherwise.
++ */
++#define KEYRING_PTR_SUBTYPE	0x2UL
++
++static inline bool keyring_ptr_is_keyring(const struct assoc_array_ptr *x)
++{
++	return (unsigned long)x & KEYRING_PTR_SUBTYPE;
++}
++static inline struct key *keyring_ptr_to_key(const struct assoc_array_ptr *x)
++{
++	void *object = assoc_array_ptr_to_leaf(x);
++	return (struct key *)((unsigned long)object & ~KEYRING_PTR_SUBTYPE);
++}
++static inline void *keyring_key_to_ptr(struct key *key)
++{
++	if (key->type == &key_type_keyring)
++		return (void *)((unsigned long)key | KEYRING_PTR_SUBTYPE);
++	return key;
++}
++
+ static struct list_head	keyring_name_hash[KEYRING_NAME_HASH_SIZE];
+ static DEFINE_RWLOCK(keyring_name_lock);
+ 
+@@ -67,7 +75,6 @@ static inline unsigned keyring_hash(const char *desc)
+  */
+ static int keyring_instantiate(struct key *keyring,
+ 			       struct key_preparsed_payload *prep);
+-static int keyring_match(const struct key *keyring, const void *criterion);
+ static void keyring_revoke(struct key *keyring);
+ static void keyring_destroy(struct key *keyring);
+ static void keyring_describe(const struct key *keyring, struct seq_file *m);
+@@ -76,9 +83,9 @@ static long keyring_read(const struct key *keyring,
+ 
+ struct key_type key_type_keyring = {
+ 	.name		= "keyring",
+-	.def_datalen	= sizeof(struct keyring_list),
++	.def_datalen	= 0,
+ 	.instantiate	= keyring_instantiate,
+-	.match		= keyring_match,
++	.match		= user_match,
+ 	.revoke		= keyring_revoke,
+ 	.destroy	= keyring_destroy,
+ 	.describe	= keyring_describe,
+@@ -127,6 +134,7 @@ static int keyring_instantiate(struct key *keyring,
+ 
+ 	ret = -EINVAL;
+ 	if (prep->datalen == 0) {
++		assoc_array_init(&keyring->keys);
+ 		/* make the keyring available by name if it has one */
+ 		keyring_publish_name(keyring);
+ 		ret = 0;
+@@ -136,15 +144,226 @@ static int keyring_instantiate(struct key *keyring,
+ }
+ 
+ /*
+- * Match keyrings on their name
++ * Multiply 64-bits by 32-bits to 96-bits and fold back to 64-bit.  Ideally we'd
++ * fold the carry back too, but that requires inline asm.
++ */
++static u64 mult_64x32_and_fold(u64 x, u32 y)
++{
++	u64 hi = (u64)(u32)(x >> 32) * y;
++	u64 lo = (u64)(u32)(x) * y;
++	return lo + ((u64)(u32)hi << 32) + (u32)(hi >> 32);
++}
++
++/*
++ * Hash a key type and description.
++ */
++static unsigned long hash_key_type_and_desc(const struct keyring_index_key *index_key)
++{
++	const unsigned level_shift = ASSOC_ARRAY_LEVEL_STEP;
++	const unsigned long level_mask = ASSOC_ARRAY_LEVEL_STEP_MASK;
++	const char *description = index_key->description;
++	unsigned long hash, type;
++	u32 piece;
++	u64 acc;
++	int n, desc_len = index_key->desc_len;
++
++	type = (unsigned long)index_key->type;
++
++	acc = mult_64x32_and_fold(type, desc_len + 13);
++	acc = mult_64x32_and_fold(acc, 9207);
++	for (;;) {
++		n = desc_len;
++		if (n <= 0)
++			break;
++		if (n > 4)
++			n = 4;
++		piece = 0;
++		memcpy(&piece, description, n);
++		description += n;
++		desc_len -= n;
++		acc = mult_64x32_and_fold(acc, piece);
++		acc = mult_64x32_and_fold(acc, 9207);
++	}
++
++	/* Fold the hash down to 32 bits if need be. */
++	hash = acc;
++	if (ASSOC_ARRAY_KEY_CHUNK_SIZE == 32)
++		hash ^= acc >> 32;
++
++	/* Squidge all the keyrings into a separate part of the tree to
++	 * ordinary keys by making sure the lowest level segment in the hash is
++	 * zero for keyrings and non-zero otherwise.
++	 */
++	if (index_key->type != &key_type_keyring && (hash & level_mask) == 0)
++		return hash | (hash >> (ASSOC_ARRAY_KEY_CHUNK_SIZE - level_shift)) | 1;
++	if (index_key->type == &key_type_keyring && (hash & level_mask) != 0)
++		return (hash + (hash << level_shift)) & ~level_mask;
++	return hash;
++}
++
++/*
++ * Build the next index key chunk.
++ *
++ * On 32-bit systems the index key is laid out as:
++ *
++ *	0	4	5	9...
++ *	hash	desclen	typeptr	desc[]
++ *
++ * On 64-bit systems:
++ *
++ *	0	8	9	17...
++ *	hash	desclen	typeptr	desc[]
++ *
++ * We return it one word-sized chunk at a time.
+  */
+-static int keyring_match(const struct key *keyring, const void *description)
++static unsigned long keyring_get_key_chunk(const void *data, int level)
++{
++	const struct keyring_index_key *index_key = data;
++	unsigned long chunk = 0;
++	long offset = 0;
++	int desc_len = index_key->desc_len, n = sizeof(chunk);
++
++	level /= ASSOC_ARRAY_KEY_CHUNK_SIZE;
++	switch (level) {
++	case 0:
++		return hash_key_type_and_desc(index_key);
++	case 1:
++		return ((unsigned long)index_key->type << 8) | desc_len;
++	case 2:
++		if (desc_len == 0)
++			return (u8)((unsigned long)index_key->type >>
++				    (ASSOC_ARRAY_KEY_CHUNK_SIZE - 8));
++		n--;
++		offset = 1;
++	default:
++		offset += sizeof(chunk) - 1;
++		offset += (level - 3) * sizeof(chunk);
++		if (offset >= desc_len)
++			return 0;
++		desc_len -= offset;
++		if (desc_len > n)
++			desc_len = n;
++		offset += desc_len;
++		do {
++			chunk <<= 8;
++			chunk |= ((u8*)index_key->description)[--offset];
++		} while (--desc_len > 0);
++
++		if (level == 2) {
++			chunk <<= 8;
++			chunk |= (u8)((unsigned long)index_key->type >>
++				      (ASSOC_ARRAY_KEY_CHUNK_SIZE - 8));
++		}
++		return chunk;
++	}
++}
++
++static unsigned long keyring_get_object_key_chunk(const void *object, int level)
++{
++	const struct key *key = keyring_ptr_to_key(object);
++	return keyring_get_key_chunk(&key->index_key, level);
++}
++
++static bool keyring_compare_object(const void *object, const void *data)
+ {
+-	return keyring->description &&
+-		strcmp(keyring->description, description) == 0;
++	const struct keyring_index_key *index_key = data;
++	const struct key *key = keyring_ptr_to_key(object);
++
++	return key->index_key.type == index_key->type &&
++		key->index_key.desc_len == index_key->desc_len &&
++		memcmp(key->index_key.description, index_key->description,
++		       index_key->desc_len) == 0;
+ }
+ 
+ /*
++ * Compare the index keys of a pair of objects and determine the bit position
++ * at which they differ - if they differ.
++ */
++static int keyring_diff_objects(const void *_a, const void *_b)
++{
++	const struct key *key_a = keyring_ptr_to_key(_a);
++	const struct key *key_b = keyring_ptr_to_key(_b);
++	const struct keyring_index_key *a = &key_a->index_key;
++	const struct keyring_index_key *b = &key_b->index_key;
++	unsigned long seg_a, seg_b;
++	int level, i;
++
++	level = 0;
++	seg_a = hash_key_type_and_desc(a);
++	seg_b = hash_key_type_and_desc(b);
++	if ((seg_a ^ seg_b) != 0)
++		goto differ;
++
++	/* The number of bits contributed by the hash is controlled by a
++	 * constant in the assoc_array headers.  Everything else thereafter we
++	 * can deal with as being machine word-size dependent.
++	 */
++	level += ASSOC_ARRAY_KEY_CHUNK_SIZE / 8;
++	seg_a = a->desc_len;
++	seg_b = b->desc_len;
++	if ((seg_a ^ seg_b) != 0)
++		goto differ;
++
++	/* The next bit may not work on big endian */
++	level++;
++	seg_a = (unsigned long)a->type;
++	seg_b = (unsigned long)b->type;
++	if ((seg_a ^ seg_b) != 0)
++		goto differ;
++
++	level += sizeof(unsigned long);
++	if (a->desc_len == 0)
++		goto same;
++
++	i = 0;
++	if (((unsigned long)a->description | (unsigned long)b->description) &
++	    (sizeof(unsigned long) - 1)) {
++		do {
++			seg_a = *(unsigned long *)(a->description + i);
++			seg_b = *(unsigned long *)(b->description + i);
++			if ((seg_a ^ seg_b) != 0)
++				goto differ_plus_i;
++			i += sizeof(unsigned long);
++		} while (i < (a->desc_len & (sizeof(unsigned long) - 1)));
++	}
++
++	for (; i < a->desc_len; i++) {
++		seg_a = *(unsigned char *)(a->description + i);
++		seg_b = *(unsigned char *)(b->description + i);
++		if ((seg_a ^ seg_b) != 0)
++			goto differ_plus_i;
++	}
++
++same:
++	return -1;
++
++differ_plus_i:
++	level += i;
++differ:
++	i = level * 8 + __ffs(seg_a ^ seg_b);
++	return i;
++}
++
++/*
++ * Free an object after stripping the keyring flag off of the pointer.
++ */
++static void keyring_free_object(void *object)
++{
++	key_put(keyring_ptr_to_key(object));
++}
++
++/*
++ * Operations for keyring management by the index-tree routines.
++ */
++static const struct assoc_array_ops keyring_assoc_array_ops = {
++	.get_key_chunk		= keyring_get_key_chunk,
++	.get_object_key_chunk	= keyring_get_object_key_chunk,
++	.compare_object		= keyring_compare_object,
++	.diff_objects		= keyring_diff_objects,
++	.free_object		= keyring_free_object,
++};
++
++/*
+  * Clean up a keyring when it is destroyed.  Unpublish its name if it had one
+  * and dispose of its data.
+  *
+@@ -155,9 +374,6 @@ static int keyring_match(const struct key *keyring, const void *description)
+  */
+ static void keyring_destroy(struct key *keyring)
+ {
+-	struct keyring_list *klist;
+-	int loop;
+-
+ 	if (keyring->description) {
+ 		write_lock(&keyring_name_lock);
+ 
+@@ -168,12 +384,7 @@ static void keyring_destroy(struct key *keyring)
+ 		write_unlock(&keyring_name_lock);
+ 	}
+ 
+-	klist = rcu_access_pointer(keyring->payload.subscriptions);
+-	if (klist) {
+-		for (loop = klist->nkeys - 1; loop >= 0; loop--)
+-			key_put(rcu_access_pointer(klist->keys[loop]));
+-		kfree(klist);
+-	}
++	assoc_array_destroy(&keyring->keys, &keyring_assoc_array_ops);
+ }
+ 
+ /*
+@@ -181,76 +392,88 @@ static void keyring_destroy(struct key *keyring)
+  */
+ static void keyring_describe(const struct key *keyring, struct seq_file *m)
+ {
+-	struct keyring_list *klist;
+-
+ 	if (keyring->description)
+ 		seq_puts(m, keyring->description);
+ 	else
+ 		seq_puts(m, "[anon]");
+ 
+ 	if (key_is_instantiated(keyring)) {
+-		rcu_read_lock();
+-		klist = rcu_dereference(keyring->payload.subscriptions);
+-		if (klist)
+-			seq_printf(m, ": %u/%u", klist->nkeys, klist->maxkeys);
++		if (keyring->keys.nr_leaves_on_tree != 0)
++			seq_printf(m, ": %lu", keyring->keys.nr_leaves_on_tree);
+ 		else
+ 			seq_puts(m, ": empty");
+-		rcu_read_unlock();
+ 	}
+ }
+ 
++struct keyring_read_iterator_context {
++	size_t			qty;
++	size_t			count;
++	key_serial_t __user	*buffer;
++};
++
++static int keyring_read_iterator(const void *object, void *data)
++{
++	struct keyring_read_iterator_context *ctx = data;
++	const struct key *key = keyring_ptr_to_key(object);
++	int ret;
++
++	kenter("{%s,%d},,{%zu/%zu}",
++	       key->type->name, key->serial, ctx->count, ctx->qty);
++
++	if (ctx->count >= ctx->qty)
++		return 1;
++
++	ret = put_user(key->serial, ctx->buffer);
++	if (ret < 0)
++		return ret;
++	ctx->buffer++;
++	ctx->count += sizeof(key->serial);
++	return 0;
++}
++
+ /*
+  * Read a list of key IDs from the keyring's contents in binary form
+  *
+- * The keyring's semaphore is read-locked by the caller.
++ * The keyring's semaphore is read-locked by the caller.  This prevents someone
++ * from modifying it under us - which could cause us to read key IDs multiple
++ * times.
+  */
+ static long keyring_read(const struct key *keyring,
+ 			 char __user *buffer, size_t buflen)
+ {
+-	struct keyring_list *klist;
+-	struct key *key;
+-	size_t qty, tmp;
+-	int loop, ret;
++	struct keyring_read_iterator_context ctx;
++	unsigned long nr_keys;
++	int ret;
+ 
+-	ret = 0;
+-	klist = rcu_dereference_locked_keyring(keyring);
+-	if (klist) {
+-		/* calculate how much data we could return */
+-		qty = klist->nkeys * sizeof(key_serial_t);
+-
+-		if (buffer && buflen > 0) {
+-			if (buflen > qty)
+-				buflen = qty;
+-
+-			/* copy the IDs of the subscribed keys into the
+-			 * buffer */
+-			ret = -EFAULT;
+-
+-			for (loop = 0; loop < klist->nkeys; loop++) {
+-				key = rcu_deref_link_locked(klist, loop,
+-							    keyring);
+-
+-				tmp = sizeof(key_serial_t);
+-				if (tmp > buflen)
+-					tmp = buflen;
+-
+-				if (copy_to_user(buffer,
+-						 &key->serial,
+-						 tmp) != 0)
+-					goto error;
+-
+-				buflen -= tmp;
+-				if (buflen == 0)
+-					break;
+-				buffer += tmp;
+-			}
+-		}
++	kenter("{%d},,%zu", key_serial(keyring), buflen);
++
++	if (buflen & (sizeof(key_serial_t) - 1))
++		return -EINVAL;
++
++	nr_keys = keyring->keys.nr_leaves_on_tree;
++	if (nr_keys == 0)
++		return 0;
+ 
+-		ret = qty;
++	/* Calculate how much data we could return */
++	ctx.qty = nr_keys * sizeof(key_serial_t);
++
++	if (!buffer || !buflen)
++		return ctx.qty;
++
++	if (buflen > ctx.qty)
++		ctx.qty = buflen;
++
++	/* Copy the IDs of the subscribed keys into the buffer */
++	ctx.buffer = (key_serial_t __user *)buffer;
++	ctx.count = 0;
++	ret = assoc_array_iterate(&keyring->keys, keyring_read_iterator, &ctx);
++	if (ret < 0) {
++		kleave(" = %d [iterate]", ret);
++		return ret;
+ 	}
+ 
+-error:
+-	return ret;
++	kleave(" = %zu [ok]", ctx.count);
++	return ctx.count;
+ }
+ 
+ /*
+@@ -277,219 +500,360 @@ struct key *keyring_alloc(const char *description, kuid_t uid, kgid_t gid,
+ }
+ EXPORT_SYMBOL(keyring_alloc);
+ 
+-/**
+- * keyring_search_aux - Search a keyring tree for a key matching some criteria
+- * @keyring_ref: A pointer to the keyring with possession indicator.
+- * @ctx: The keyring search context.
+- *
+- * Search the supplied keyring tree for a key that matches the criteria given.
+- * The root keyring and any linked keyrings must grant Search permission to the
+- * caller to be searchable and keys can only be found if they too grant Search
+- * to the caller. The possession flag on the root keyring pointer controls use
+- * of the possessor bits in permissions checking of the entire tree.  In
+- * addition, the LSM gets to forbid keyring searches and key matches.
+- *
+- * The search is performed as a breadth-then-depth search up to the prescribed
+- * limit (KEYRING_SEARCH_MAX_DEPTH).
+- *
+- * Keys are matched to the type provided and are then filtered by the match
+- * function, which is given the description to use in any way it sees fit.  The
+- * match function may use any attributes of a key that it wishes to to
+- * determine the match.  Normally the match function from the key type would be
+- * used.
+- *
+- * RCU is used to prevent the keyring key lists from disappearing without the
+- * need to take lots of locks.
+- *
+- * Returns a pointer to the found key and increments the key usage count if
+- * successful; -EAGAIN if no matching keys were found, or if expired or revoked
+- * keys were found; -ENOKEY if only negative keys were found; -ENOTDIR if the
+- * specified keyring wasn't a keyring.
+- *
+- * In the case of a successful return, the possession attribute from
+- * @keyring_ref is propagated to the returned key reference.
++/*
++ * Iteration function to consider each key found.
+  */
+-key_ref_t keyring_search_aux(key_ref_t keyring_ref,
+-			     struct keyring_search_context *ctx)
++static int keyring_search_iterator(const void *object, void *iterator_data)
+ {
+-	struct {
+-		/* Need a separate keylist pointer for RCU purposes */
+-		struct key *keyring;
+-		struct keyring_list *keylist;
+-		int kix;
+-	} stack[KEYRING_SEARCH_MAX_DEPTH];
+-
+-	struct keyring_list *keylist;
+-	unsigned long kflags;
+-	struct key *keyring, *key;
+-	key_ref_t key_ref;
+-	long err;
+-	int sp, nkeys, kix;
++	struct keyring_search_context *ctx = iterator_data;
++	const struct key *key = keyring_ptr_to_key(object);
++	unsigned long kflags = key->flags;
+ 
+-	keyring = key_ref_to_ptr(keyring_ref);
+-	ctx->possessed = is_key_possessed(keyring_ref);
+-	key_check(keyring);
++	kenter("{%d}", key->serial);
+ 
+-	/* top keyring must have search permission to begin the search */
+-	err = key_task_permission(keyring_ref, ctx->cred, KEY_SEARCH);
+-	if (err < 0) {
+-		key_ref = ERR_PTR(err);
+-		goto error;
++	/* ignore keys not of this type */
++	if (key->type != ctx->index_key.type) {
++		kleave(" = 0 [!type]");
++		return 0;
+ 	}
+ 
+-	key_ref = ERR_PTR(-ENOTDIR);
+-	if (keyring->type != &key_type_keyring)
+-		goto error;
++	/* skip invalidated, revoked and expired keys */
++	if (ctx->flags & KEYRING_SEARCH_DO_STATE_CHECK) {
++		if (kflags & ((1 << KEY_FLAG_INVALIDATED) |
++			      (1 << KEY_FLAG_REVOKED))) {
++			ctx->result = ERR_PTR(-EKEYREVOKED);
++			kleave(" = %d [invrev]", ctx->skipped_ret);
++			goto skipped;
++		}
+ 
+-	rcu_read_lock();
++		if (key->expiry && ctx->now.tv_sec >= key->expiry) {
++			ctx->result = ERR_PTR(-EKEYEXPIRED);
++			kleave(" = %d [expire]", ctx->skipped_ret);
++			goto skipped;
++		}
++	}
+ 
+-	ctx->now = current_kernel_time();
+-	err = -EAGAIN;
+-	sp = 0;
+-
+-	/* firstly we should check to see if this top-level keyring is what we
+-	 * are looking for */
+-	key_ref = ERR_PTR(-EAGAIN);
+-	kflags = keyring->flags;
+-	if (keyring->type == ctx->index_key.type &&
+-	    ctx->match(keyring, ctx->match_data)) {
+-		key = keyring;
+-		if (ctx->flags & KEYRING_SEARCH_NO_STATE_CHECK)
+-			goto found;
++	/* keys that don't match */
++	if (!ctx->match(key, ctx->match_data)) {
++		kleave(" = 0 [!match]");
++		return 0;
++	}
+ 
+-		/* check it isn't negative and hasn't expired or been
+-		 * revoked */
+-		if (kflags & (1 << KEY_FLAG_REVOKED))
+-			goto error_2;
+-		if (key->expiry && ctx->now.tv_sec >= key->expiry)
+-			goto error_2;
+-		key_ref = ERR_PTR(key->type_data.reject_error);
+-		if (kflags & (1 << KEY_FLAG_NEGATIVE))
+-			goto error_2;
+-		goto found;
++	/* key must have search permissions */
++	if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM) &&
++	    key_task_permission(make_key_ref(key, ctx->possessed),
++				ctx->cred, KEY_SEARCH) < 0) {
++		ctx->result = ERR_PTR(-EACCES);
++		kleave(" = %d [!perm]", ctx->skipped_ret);
++		goto skipped;
+ 	}
+ 
+-	/* otherwise, the top keyring must not be revoked, expired, or
+-	 * negatively instantiated if we are to search it */
+-	key_ref = ERR_PTR(-EAGAIN);
+-	if (kflags & ((1 << KEY_FLAG_INVALIDATED) |
+-		      (1 << KEY_FLAG_REVOKED) |
+-		      (1 << KEY_FLAG_NEGATIVE)) ||
+-	    (keyring->expiry && ctx->now.tv_sec >= keyring->expiry))
+-		goto error_2;
+-
+-	/* start processing a new keyring */
+-descend:
+-	kflags = keyring->flags;
+-	if (kflags & ((1 << KEY_FLAG_INVALIDATED) |
+-		      (1 << KEY_FLAG_REVOKED)))
+-		goto not_this_keyring;
++	if (ctx->flags & KEYRING_SEARCH_DO_STATE_CHECK) {
++		/* we set a different error code if we pass a negative key */
++		if (kflags & (1 << KEY_FLAG_NEGATIVE)) {
++			ctx->result = ERR_PTR(key->type_data.reject_error);
++			kleave(" = %d [neg]", ctx->skipped_ret);
++			goto skipped;
++		}
++	}
+ 
+-	keylist = rcu_dereference(keyring->payload.subscriptions);
+-	if (!keylist)
+-		goto not_this_keyring;
++	/* Found */
++	ctx->result = make_key_ref(key, ctx->possessed);
++	kleave(" = 1 [found]");
++	return 1;
+ 
+-	/* iterate through the keys in this keyring first */
+-	nkeys = keylist->nkeys;
+-	smp_rmb();
+-	for (kix = 0; kix < nkeys; kix++) {
+-		key = rcu_dereference(keylist->keys[kix]);
+-		kflags = key->flags;
++skipped:
++	return ctx->skipped_ret;
++}
+ 
+-		/* ignore keys not of this type */
+-		if (key->type != ctx->index_key.type)
+-			continue;
++/*
++ * Search inside a keyring for a key.  We can search by walking to it
++ * directly based on its index-key or we can iterate over the entire
++ * tree looking for it, based on the match function.
++ */
++static int search_keyring(struct key *keyring, struct keyring_search_context *ctx)
++{
++	if ((ctx->flags & KEYRING_SEARCH_LOOKUP_TYPE) ==
++	    KEYRING_SEARCH_LOOKUP_DIRECT) {
++		const void *object;
++
++		object = assoc_array_find(&keyring->keys,
++					  &keyring_assoc_array_ops,
++					  &ctx->index_key);
++		return object ? ctx->iterator(object, ctx) : 0;
++	}
++	return assoc_array_iterate(&keyring->keys, ctx->iterator, ctx);
++}
+ 
+-		/* skip invalidated, revoked and expired keys */
+-		if (!(ctx->flags & KEYRING_SEARCH_NO_STATE_CHECK)) {
+-			if (kflags & ((1 << KEY_FLAG_INVALIDATED) |
+-				      (1 << KEY_FLAG_REVOKED)))
+-				continue;
++/*
++ * Search a tree of keyrings that point to other keyrings up to the maximum
++ * depth.
++ */
++static bool search_nested_keyrings(struct key *keyring,
++				   struct keyring_search_context *ctx)
++{
++	struct {
++		struct key *keyring;
++		struct assoc_array_node *node;
++		int slot;
++	} stack[KEYRING_SEARCH_MAX_DEPTH];
+ 
+-			if (key->expiry && ctx->now.tv_sec >= key->expiry)
+-				continue;
+-		}
++	struct assoc_array_shortcut *shortcut;
++	struct assoc_array_node *node;
++	struct assoc_array_ptr *ptr;
++	struct key *key;
++	int sp = 0, slot;
+ 
+-		/* keys that don't match */
+-		if (!ctx->match(key, ctx->match_data))
+-			continue;
++	kenter("{%d},{%s,%s}",
++	       keyring->serial,
++	       ctx->index_key.type->name,
++	       ctx->index_key.description);
+ 
+-		/* key must have search permissions */
+-		if (key_task_permission(make_key_ref(key, ctx->possessed),
+-					ctx->cred, KEY_SEARCH) < 0)
+-			continue;
++	if (ctx->index_key.description)
++		ctx->index_key.desc_len = strlen(ctx->index_key.description);
+ 
+-		if (ctx->flags & KEYRING_SEARCH_NO_STATE_CHECK)
++	/* Check to see if this top-level keyring is what we are looking for
++	 * and whether it is valid or not.
++	 */
++	if (ctx->flags & KEYRING_SEARCH_LOOKUP_ITERATE ||
++	    keyring_compare_object(keyring, &ctx->index_key)) {
++		ctx->skipped_ret = 2;
++		ctx->flags |= KEYRING_SEARCH_DO_STATE_CHECK;
++		switch (ctx->iterator(keyring_key_to_ptr(keyring), ctx)) {
++		case 1:
+ 			goto found;
+-
+-		/* we set a different error code if we pass a negative key */
+-		if (kflags & (1 << KEY_FLAG_NEGATIVE)) {
+-			err = key->type_data.reject_error;
+-			continue;
++		case 2:
++			return false;
++		default:
++			break;
+ 		}
++	}
+ 
++	ctx->skipped_ret = 0;
++	if (ctx->flags & KEYRING_SEARCH_NO_STATE_CHECK)
++		ctx->flags &= ~KEYRING_SEARCH_DO_STATE_CHECK;
++
++	/* Start processing a new keyring */
++descend_to_keyring:
++	kdebug("descend to %d", keyring->serial);
++	if (keyring->flags & ((1 << KEY_FLAG_INVALIDATED) |
++			      (1 << KEY_FLAG_REVOKED)))
++		goto not_this_keyring;
++
++	/* Search through the keys in this keyring before its searching its
++	 * subtrees.
++	 */
++	if (search_keyring(keyring, ctx))
+ 		goto found;
+-	}
+ 
+-	/* search through the keyrings nested in this one */
+-	kix = 0;
+-ascend:
+-	nkeys = keylist->nkeys;
+-	smp_rmb();
+-	for (; kix < nkeys; kix++) {
+-		key = rcu_dereference(keylist->keys[kix]);
+-		if (key->type != &key_type_keyring)
+-			continue;
++	/* Then manually iterate through the keyrings nested in this one.
++	 *
++	 * Start from the root node of the index tree.  Because of the way the
++	 * hash function has been set up, keyrings cluster on the leftmost
++	 * branch of the root node (root slot 0) or in the root node itself.
++	 * Non-keyrings avoid the leftmost branch of the root entirely (root
++	 * slots 1-15).
++	 */
++	ptr = ACCESS_ONCE(keyring->keys.root);
++	if (!ptr)
++		goto not_this_keyring;
+ 
+-		/* recursively search nested keyrings
+-		 * - only search keyrings for which we have search permission
++	if (assoc_array_ptr_is_shortcut(ptr)) {
++		/* If the root is a shortcut, either the keyring only contains
++		 * keyring pointers (everything clusters behind root slot 0) or
++		 * doesn't contain any keyring pointers.
+ 		 */
+-		if (sp >= KEYRING_SEARCH_MAX_DEPTH)
++		shortcut = assoc_array_ptr_to_shortcut(ptr);
++		smp_read_barrier_depends();
++		if ((shortcut->index_key[0] & ASSOC_ARRAY_FAN_MASK) != 0)
++			goto not_this_keyring;
++
++		ptr = ACCESS_ONCE(shortcut->next_node);
++		node = assoc_array_ptr_to_node(ptr);
++		goto begin_node;
++	}
++
++	node = assoc_array_ptr_to_node(ptr);
++	smp_read_barrier_depends();
++
++	ptr = node->slots[0];
++	if (!assoc_array_ptr_is_meta(ptr))
++		goto begin_node;
++
++descend_to_node:
++	/* Descend to a more distal node in this keyring's content tree and go
++	 * through that.
++	 */
++	kdebug("descend");
++	if (assoc_array_ptr_is_shortcut(ptr)) {
++		shortcut = assoc_array_ptr_to_shortcut(ptr);
++		smp_read_barrier_depends();
++		ptr = ACCESS_ONCE(shortcut->next_node);
++		BUG_ON(!assoc_array_ptr_is_node(ptr));
++		node = assoc_array_ptr_to_node(ptr);
++	}
++
++begin_node:
++	kdebug("begin_node");
++	smp_read_barrier_depends();
++	slot = 0;
++ascend_to_node:
++	/* Go through the slots in a node */
++	for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
++		ptr = ACCESS_ONCE(node->slots[slot]);
++
++		if (assoc_array_ptr_is_meta(ptr) && node->back_pointer)
++			goto descend_to_node;
++
++		if (!keyring_ptr_is_keyring(ptr))
+ 			continue;
+ 
+-		if (key_task_permission(make_key_ref(key, ctx->possessed),
++		key = keyring_ptr_to_key(ptr);
++
++		if (sp >= KEYRING_SEARCH_MAX_DEPTH) {
++			if (ctx->flags & KEYRING_SEARCH_DETECT_TOO_DEEP) {
++				ctx->result = ERR_PTR(-ELOOP);
++				return false;
++			}
++			goto not_this_keyring;
++		}
++
++		/* Search a nested keyring */
++		if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM) &&
++		    key_task_permission(make_key_ref(key, ctx->possessed),
+ 					ctx->cred, KEY_SEARCH) < 0)
+ 			continue;
+ 
+ 		/* stack the current position */
+ 		stack[sp].keyring = keyring;
+-		stack[sp].keylist = keylist;
+-		stack[sp].kix = kix;
++		stack[sp].node = node;
++		stack[sp].slot = slot;
+ 		sp++;
+ 
+ 		/* begin again with the new keyring */
+ 		keyring = key;
+-		goto descend;
++		goto descend_to_keyring;
++	}
++
++	/* We've dealt with all the slots in the current node, so now we need
++	 * to ascend to the parent and continue processing there.
++	 */
++	ptr = ACCESS_ONCE(node->back_pointer);
++	slot = node->parent_slot;
++
++	if (ptr && assoc_array_ptr_is_shortcut(ptr)) {
++		shortcut = assoc_array_ptr_to_shortcut(ptr);
++		smp_read_barrier_depends();
++		ptr = ACCESS_ONCE(shortcut->back_pointer);
++		slot = shortcut->parent_slot;
++	}
++	if (!ptr)
++		goto not_this_keyring;
++	node = assoc_array_ptr_to_node(ptr);
++	smp_read_barrier_depends();
++	slot++;
++
++	/* If we've ascended to the root (zero backpointer), we must have just
++	 * finished processing the leftmost branch rather than the root slots -
++	 * so there can't be any more keyrings for us to find.
++	 */
++	if (node->back_pointer) {
++		kdebug("ascend %d", slot);
++		goto ascend_to_node;
+ 	}
+ 
+-	/* the keyring we're looking at was disqualified or didn't contain a
+-	 * matching key */
++	/* The keyring we're looking at was disqualified or didn't contain a
++	 * matching key.
++	 */
+ not_this_keyring:
+-	if (sp > 0) {
+-		/* resume the processing of a keyring higher up in the tree */
+-		sp--;
+-		keyring = stack[sp].keyring;
+-		keylist = stack[sp].keylist;
+-		kix = stack[sp].kix + 1;
+-		goto ascend;
++	kdebug("not_this_keyring %d", sp);
++	if (sp <= 0) {
++		kleave(" = false");
++		return false;
+ 	}
+ 
+-	key_ref = ERR_PTR(err);
+-	goto error_2;
++	/* Resume the processing of a keyring higher up in the tree */
++	sp--;
++	keyring = stack[sp].keyring;
++	node = stack[sp].node;
++	slot = stack[sp].slot + 1;
++	kdebug("ascend to %d [%d]", keyring->serial, slot);
++	goto ascend_to_node;
+ 
+-	/* we found a viable match */
++	/* We found a viable match */
+ found:
+-	__key_get(key);
+-	key->last_used_at = ctx->now.tv_sec;
+-	keyring->last_used_at = ctx->now.tv_sec;
+-	while (sp > 0)
+-		stack[--sp].keyring->last_used_at = ctx->now.tv_sec;
++	key = key_ref_to_ptr(ctx->result);
+ 	key_check(key);
+-	key_ref = make_key_ref(key, ctx->possessed);
+-error_2:
++	if (!(ctx->flags & KEYRING_SEARCH_NO_UPDATE_TIME)) {
++		key->last_used_at = ctx->now.tv_sec;
++		keyring->last_used_at = ctx->now.tv_sec;
++		while (sp > 0)
++			stack[--sp].keyring->last_used_at = ctx->now.tv_sec;
++	}
++	kleave(" = true");
++	return true;
++}
++
++/**
++ * keyring_search_aux - Search a keyring tree for a key matching some criteria
++ * @keyring_ref: A pointer to the keyring with possession indicator.
++ * @ctx: The keyring search context.
++ *
++ * Search the supplied keyring tree for a key that matches the criteria given.
++ * The root keyring and any linked keyrings must grant Search permission to the
++ * caller to be searchable and keys can only be found if they too grant Search
++ * to the caller. The possession flag on the root keyring pointer controls use
++ * of the possessor bits in permissions checking of the entire tree.  In
++ * addition, the LSM gets to forbid keyring searches and key matches.
++ *
++ * The search is performed as a breadth-then-depth search up to the prescribed
++ * limit (KEYRING_SEARCH_MAX_DEPTH).
++ *
++ * Keys are matched to the type provided and are then filtered by the match
++ * function, which is given the description to use in any way it sees fit.  The
++ * match function may use any attributes of a key that it wishes to to
++ * determine the match.  Normally the match function from the key type would be
++ * used.
++ *
++ * RCU can be used to prevent the keyring key lists from disappearing without
++ * the need to take lots of locks.
++ *
++ * Returns a pointer to the found key and increments the key usage count if
++ * successful; -EAGAIN if no matching keys were found, or if expired or revoked
++ * keys were found; -ENOKEY if only negative keys were found; -ENOTDIR if the
++ * specified keyring wasn't a keyring.
++ *
++ * In the case of a successful return, the possession attribute from
++ * @keyring_ref is propagated to the returned key reference.
++ */
++key_ref_t keyring_search_aux(key_ref_t keyring_ref,
++			     struct keyring_search_context *ctx)
++{
++	struct key *keyring;
++	long err;
++
++	ctx->iterator = keyring_search_iterator;
++	ctx->possessed = is_key_possessed(keyring_ref);
++	ctx->result = ERR_PTR(-EAGAIN);
++
++	keyring = key_ref_to_ptr(keyring_ref);
++	key_check(keyring);
++
++	if (keyring->type != &key_type_keyring)
++		return ERR_PTR(-ENOTDIR);
++
++	if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM)) {
++		err = key_task_permission(keyring_ref, ctx->cred, KEY_SEARCH);
++		if (err < 0)
++			return ERR_PTR(err);
++	}
++
++	rcu_read_lock();
++	ctx->now = current_kernel_time();
++	if (search_nested_keyrings(keyring, ctx))
++		__key_get(key_ref_to_ptr(ctx->result));
+ 	rcu_read_unlock();
+-error:
+-	return key_ref;
++	return ctx->result;
+ }
+ 
+ /**
+@@ -499,7 +863,7 @@ error:
+  * @description: The name of the keyring we want to find.
+  *
+  * As keyring_search_aux() above, but using the current task's credentials and
+- * type's default matching function.
++ * type's default matching function and preferred search method.
+  */
+ key_ref_t keyring_search(key_ref_t keyring,
+ 			 struct key_type *type,
+@@ -523,58 +887,49 @@ key_ref_t keyring_search(key_ref_t keyring,
+ EXPORT_SYMBOL(keyring_search);
+ 
+ /*
+- * Search the given keyring only (no recursion).
++ * Search the given keyring for a key that might be updated.
+  *
+  * The caller must guarantee that the keyring is a keyring and that the
+- * permission is granted to search the keyring as no check is made here.
+- *
+- * RCU is used to make it unnecessary to lock the keyring key list here.
++ * permission is granted to modify the keyring as no check is made here.  The
++ * caller must also hold a lock on the keyring semaphore.
+  *
+  * Returns a pointer to the found key with usage count incremented if
+- * successful and returns -ENOKEY if not found.  Revoked and invalidated keys
+- * are skipped over.
++ * successful and returns NULL if not found.  Revoked and invalidated keys are
++ * skipped over.
+  *
+  * If successful, the possession indicator is propagated from the keyring ref
+  * to the returned key reference.
+  */
+-key_ref_t __keyring_search_one(key_ref_t keyring_ref,
+-			       const struct keyring_index_key *index_key)
++key_ref_t find_key_to_update(key_ref_t keyring_ref,
++			     const struct keyring_index_key *index_key)
+ {
+-	struct keyring_list *klist;
+ 	struct key *keyring, *key;
+-	bool possessed;
+-	int nkeys, loop;
++	const void *object;
+ 
+ 	keyring = key_ref_to_ptr(keyring_ref);
+-	possessed = is_key_possessed(keyring_ref);
+ 
+-	rcu_read_lock();
++	kenter("{%d},{%s,%s}",
++	       keyring->serial, index_key->type->name, index_key->description);
+ 
+-	klist = rcu_dereference(keyring->payload.subscriptions);
+-	if (klist) {
+-		nkeys = klist->nkeys;
+-		smp_rmb();
+-		for (loop = 0; loop < nkeys ; loop++) {
+-			key = rcu_dereference(klist->keys[loop]);
+-			if (key->type == index_key->type &&
+-			    (!key->type->match ||
+-			     key->type->match(key, index_key->description)) &&
+-			    !(key->flags & ((1 << KEY_FLAG_INVALIDATED) |
+-					    (1 << KEY_FLAG_REVOKED)))
+-			    )
+-				goto found;
+-		}
+-	}
++	object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops,
++				  index_key);
+ 
+-	rcu_read_unlock();
+-	return ERR_PTR(-ENOKEY);
++	if (object)
++		goto found;
++
++	kleave(" = NULL");
++	return NULL;
+ 
+ found:
++	key = keyring_ptr_to_key(object);
++	if (key->flags & ((1 << KEY_FLAG_INVALIDATED) |
++			  (1 << KEY_FLAG_REVOKED))) {
++		kleave(" = NULL [x]");
++		return NULL;
++	}
+ 	__key_get(key);
+-	keyring->last_used_at = key->last_used_at =
+-		current_kernel_time().tv_sec;
+-	rcu_read_unlock();
+-	return make_key_ref(key, possessed);
++	kleave(" = {%d}", key->serial);
++	return make_key_ref(key, is_key_possessed(keyring_ref));
+ }
+ 
+ /*
+@@ -637,6 +992,19 @@ out:
+ 	return keyring;
+ }
+ 
++static int keyring_detect_cycle_iterator(const void *object,
++					 void *iterator_data)
++{
++	struct keyring_search_context *ctx = iterator_data;
++	const struct key *key = keyring_ptr_to_key(object);
++
++	kenter("{%d}", key->serial);
++
++	BUG_ON(key != ctx->match_data);
++	ctx->result = ERR_PTR(-EDEADLK);
++	return 1;
++}
++
+ /*
+  * See if a cycle will will be created by inserting acyclic tree B in acyclic
+  * tree A at the topmost level (ie: as a direct child of A).
+@@ -646,117 +1014,39 @@ out:
+  */
+ static int keyring_detect_cycle(struct key *A, struct key *B)
+ {
+-	struct {
+-		struct keyring_list *keylist;
+-		int kix;
+-	} stack[KEYRING_SEARCH_MAX_DEPTH];
+-
+-	struct keyring_list *keylist;
+-	struct key *subtree, *key;
+-	int sp, nkeys, kix, ret;
++	struct keyring_search_context ctx = {
++		.index_key	= A->index_key,
++		.match_data	= A,
++		.iterator	= keyring_detect_cycle_iterator,
++		.flags		= (KEYRING_SEARCH_LOOKUP_DIRECT |
++				   KEYRING_SEARCH_NO_STATE_CHECK |
++				   KEYRING_SEARCH_NO_UPDATE_TIME |
++				   KEYRING_SEARCH_NO_CHECK_PERM |
++				   KEYRING_SEARCH_DETECT_TOO_DEEP),
++	};
+ 
+ 	rcu_read_lock();
+-
+-	ret = -EDEADLK;
+-	if (A == B)
+-		goto cycle_detected;
+-
+-	subtree = B;
+-	sp = 0;
+-
+-	/* start processing a new keyring */
+-descend:
+-	if (test_bit(KEY_FLAG_REVOKED, &subtree->flags))
+-		goto not_this_keyring;
+-
+-	keylist = rcu_dereference(subtree->payload.subscriptions);
+-	if (!keylist)
+-		goto not_this_keyring;
+-	kix = 0;
+-
+-ascend:
+-	/* iterate through the remaining keys in this keyring */
+-	nkeys = keylist->nkeys;
+-	smp_rmb();
+-	for (; kix < nkeys; kix++) {
+-		key = rcu_dereference(keylist->keys[kix]);
+-
+-		if (key == A)
+-			goto cycle_detected;
+-
+-		/* recursively check nested keyrings */
+-		if (key->type == &key_type_keyring) {
+-			if (sp >= KEYRING_SEARCH_MAX_DEPTH)
+-				goto too_deep;
+-
+-			/* stack the current position */
+-			stack[sp].keylist = keylist;
+-			stack[sp].kix = kix;
+-			sp++;
+-
+-			/* begin again with the new keyring */
+-			subtree = key;
+-			goto descend;
+-		}
+-	}
+-
+-	/* the keyring we're looking at was disqualified or didn't contain a
+-	 * matching key */
+-not_this_keyring:
+-	if (sp > 0) {
+-		/* resume the checking of a keyring higher up in the tree */
+-		sp--;
+-		keylist = stack[sp].keylist;
+-		kix = stack[sp].kix + 1;
+-		goto ascend;
+-	}
+-
+-	ret = 0; /* no cycles detected */
+-
+-error:
++	search_nested_keyrings(B, &ctx);
+ 	rcu_read_unlock();
+-	return ret;
+-
+-too_deep:
+-	ret = -ELOOP;
+-	goto error;
+-
+-cycle_detected:
+-	ret = -EDEADLK;
+-	goto error;
+-}
+-
+-/*
+- * Dispose of a keyring list after the RCU grace period, freeing the unlinked
+- * key
+- */
+-static void keyring_unlink_rcu_disposal(struct rcu_head *rcu)
+-{
+-	struct keyring_list *klist =
+-		container_of(rcu, struct keyring_list, rcu);
+-
+-	if (klist->delkey != USHRT_MAX)
+-		key_put(rcu_access_pointer(klist->keys[klist->delkey]));
+-	kfree(klist);
++	return PTR_ERR(ctx.result) == -EAGAIN ? 0 : PTR_ERR(ctx.result);
+ }
+ 
+ /*
+  * Preallocate memory so that a key can be linked into to a keyring.
+  */
+-int __key_link_begin(struct key *keyring, const struct keyring_index_key *index_key,
+-		     unsigned long *_prealloc)
++int __key_link_begin(struct key *keyring,
++		     const struct keyring_index_key *index_key,
++		     struct assoc_array_edit **_edit)
+ 	__acquires(&keyring->sem)
+ 	__acquires(&keyring_serialise_link_sem)
+ {
+-	struct keyring_list *klist, *nklist;
+-	unsigned long prealloc;
+-	unsigned max;
+-	time_t lowest_lru;
+-	size_t size;
+-	int loop, lru, ret;
++	struct assoc_array_edit *edit;
++	int ret;
+ 
+ 	kenter("%d,%s,%s,",
+-	       key_serial(keyring), index_key->type->name, index_key->description);
++	       keyring->serial, index_key->type->name, index_key->description);
++
++	BUG_ON(index_key->desc_len == 0);
+ 
+ 	if (keyring->type != &key_type_keyring)
+ 		return -ENOTDIR;
+@@ -772,88 +1062,25 @@ int __key_link_begin(struct key *keyring, const struct keyring_index_key *index_
+ 	if (index_key->type == &key_type_keyring)
+ 		down_write(&keyring_serialise_link_sem);
+ 
+-	klist = rcu_dereference_locked_keyring(keyring);
+-
+-	/* see if there's a matching key we can displace */
+-	lru = -1;
+-	if (klist && klist->nkeys > 0) {
+-		lowest_lru = TIME_T_MAX;
+-		for (loop = klist->nkeys - 1; loop >= 0; loop--) {
+-			struct key *key = rcu_deref_link_locked(klist, loop,
+-								keyring);
+-			if (key->type == index_key->type &&
+-			    strcmp(key->description, index_key->description) == 0) {
+-				/* Found a match - we'll replace the link with
+-				 * one to the new key.  We record the slot
+-				 * position.
+-				 */
+-				klist->delkey = loop;
+-				prealloc = 0;
+-				goto done;
+-			}
+-			if (key->last_used_at < lowest_lru) {
+-				lowest_lru = key->last_used_at;
+-				lru = loop;
+-			}
+-		}
+-	}
+-
+-	/* If the keyring is full then do an LRU discard */
+-	if (klist &&
+-	    klist->nkeys == klist->maxkeys &&
+-	    klist->maxkeys >= MAX_KEYRING_LINKS) {
+-		kdebug("LRU discard %d\n", lru);
+-		klist->delkey = lru;
+-		prealloc = 0;
+-		goto done;
+-	}
+-
+ 	/* check that we aren't going to overrun the user's quota */
+ 	ret = key_payload_reserve(keyring,
+ 				  keyring->datalen + KEYQUOTA_LINK_BYTES);
+ 	if (ret < 0)
+ 		goto error_sem;
+ 
+-	if (klist && klist->nkeys < klist->maxkeys) {
+-		/* there's sufficient slack space to append directly */
+-		klist->delkey = klist->nkeys;
+-		prealloc = KEY_LINK_FIXQUOTA;
+-	} else {
+-		/* grow the key list */
+-		max = 4;
+-		if (klist) {
+-			max += klist->maxkeys;
+-			if (max > MAX_KEYRING_LINKS)
+-				max = MAX_KEYRING_LINKS;
+-			BUG_ON(max <= klist->maxkeys);
+-		}
+-
+-		size = sizeof(*klist) + sizeof(struct key *) * max;
+-
+-		ret = -ENOMEM;
+-		nklist = kmalloc(size, GFP_KERNEL);
+-		if (!nklist)
+-			goto error_quota;
+-
+-		nklist->maxkeys = max;
+-		if (klist) {
+-			memcpy(nklist->keys, klist->keys,
+-			       sizeof(struct key *) * klist->nkeys);
+-			nklist->delkey = klist->nkeys;
+-			nklist->nkeys = klist->nkeys + 1;
+-			klist->delkey = USHRT_MAX;
+-		} else {
+-			nklist->nkeys = 1;
+-			nklist->delkey = 0;
+-		}
+-
+-		/* add the key into the new space */
+-		RCU_INIT_POINTER(nklist->keys[nklist->delkey], NULL);
+-		prealloc = (unsigned long)nklist | KEY_LINK_FIXQUOTA;
++	/* Create an edit script that will insert/replace the key in the
++	 * keyring tree.
++	 */
++	edit = assoc_array_insert(&keyring->keys,
++				  &keyring_assoc_array_ops,
++				  index_key,
++				  NULL);
++	if (IS_ERR(edit)) {
++		ret = PTR_ERR(edit);
++		goto error_quota;
+ 	}
+ 
+-done:
+-	*_prealloc = prealloc;
++	*_edit = edit;
+ 	kleave(" = 0");
+ 	return 0;
+ 
+@@ -893,60 +1120,12 @@ int __key_link_check_live_key(struct key *keyring, struct key *key)
+  * holds at most one link to any given key of a particular type+description
+  * combination.
+  */
+-void __key_link(struct key *keyring, struct key *key,
+-		unsigned long *_prealloc)
++void __key_link(struct key *key, struct assoc_array_edit **_edit)
+ {
+-	struct keyring_list *klist, *nklist;
+-	struct key *discard;
+-
+-	nklist = (struct keyring_list *)(*_prealloc & ~KEY_LINK_FIXQUOTA);
+-	*_prealloc = 0;
+-
+-	kenter("%d,%d,%p", keyring->serial, key->serial, nklist);
+-
+-	klist = rcu_dereference_locked_keyring(keyring);
+-
+ 	__key_get(key);
+-	keyring->last_used_at = key->last_used_at =
+-		current_kernel_time().tv_sec;
+-
+-	/* there's a matching key we can displace or an empty slot in a newly
+-	 * allocated list we can fill */
+-	if (nklist) {
+-		kdebug("reissue %hu/%hu/%hu",
+-		       nklist->delkey, nklist->nkeys, nklist->maxkeys);
+-
+-		RCU_INIT_POINTER(nklist->keys[nklist->delkey], key);
+-
+-		rcu_assign_pointer(keyring->payload.subscriptions, nklist);
+-
+-		/* dispose of the old keyring list and, if there was one, the
+-		 * displaced key */
+-		if (klist) {
+-			kdebug("dispose %hu/%hu/%hu",
+-			       klist->delkey, klist->nkeys, klist->maxkeys);
+-			call_rcu(&klist->rcu, keyring_unlink_rcu_disposal);
+-		}
+-	} else if (klist->delkey < klist->nkeys) {
+-		kdebug("replace %hu/%hu/%hu",
+-		       klist->delkey, klist->nkeys, klist->maxkeys);
+-
+-		discard = rcu_dereference_protected(
+-			klist->keys[klist->delkey],
+-			rwsem_is_locked(&keyring->sem));
+-		rcu_assign_pointer(klist->keys[klist->delkey], key);
+-		/* The garbage collector will take care of RCU
+-		 * synchronisation */
+-		key_put(discard);
+-	} else {
+-		/* there's sufficient slack space to append directly */
+-		kdebug("append %hu/%hu/%hu",
+-		       klist->delkey, klist->nkeys, klist->maxkeys);
+-
+-		RCU_INIT_POINTER(klist->keys[klist->delkey], key);
+-		smp_wmb();
+-		klist->nkeys++;
+-	}
++	assoc_array_insert_set_object(*_edit, keyring_key_to_ptr(key));
++	assoc_array_apply_edit(*_edit);
++	*_edit = NULL;
+ }
+ 
+ /*
+@@ -956,23 +1135,20 @@ void __key_link(struct key *keyring, struct key *key,
+  */
+ void __key_link_end(struct key *keyring,
+ 		    const struct keyring_index_key *index_key,
+-		    unsigned long prealloc)
++		    struct assoc_array_edit *edit)
+ 	__releases(&keyring->sem)
+ 	__releases(&keyring_serialise_link_sem)
+ {
+ 	BUG_ON(index_key->type == NULL);
+-	BUG_ON(index_key->type->name == NULL);
+-	kenter("%d,%s,%lx", keyring->serial, index_key->type->name, prealloc);
++	kenter("%d,%s,", keyring->serial, index_key->type->name);
+ 
+ 	if (index_key->type == &key_type_keyring)
+ 		up_write(&keyring_serialise_link_sem);
+ 
+-	if (prealloc) {
+-		if (prealloc & KEY_LINK_FIXQUOTA)
+-			key_payload_reserve(keyring,
+-					    keyring->datalen -
+-					    KEYQUOTA_LINK_BYTES);
+-		kfree((struct keyring_list *)(prealloc & ~KEY_LINK_FIXQUOTA));
++	if (edit) {
++		key_payload_reserve(keyring,
++				    keyring->datalen - KEYQUOTA_LINK_BYTES);
++		assoc_array_cancel_edit(edit);
+ 	}
+ 	up_write(&keyring->sem);
+ }
+@@ -999,20 +1175,24 @@ void __key_link_end(struct key *keyring,
+  */
+ int key_link(struct key *keyring, struct key *key)
+ {
+-	unsigned long prealloc;
++	struct assoc_array_edit *edit;
+ 	int ret;
+ 
++	kenter("{%d,%d}", keyring->serial, atomic_read(&keyring->usage));
++
+ 	key_check(keyring);
+ 	key_check(key);
+ 
+-	ret = __key_link_begin(keyring, &key->index_key, &prealloc);
++	ret = __key_link_begin(keyring, &key->index_key, &edit);
+ 	if (ret == 0) {
++		kdebug("begun {%d,%d}", keyring->serial, atomic_read(&keyring->usage));
+ 		ret = __key_link_check_live_key(keyring, key);
+ 		if (ret == 0)
+-			__key_link(keyring, key, &prealloc);
+-		__key_link_end(keyring, &key->index_key, prealloc);
++			__key_link(key, &edit);
++		__key_link_end(keyring, &key->index_key, edit);
+ 	}
+ 
++	kleave(" = %d {%d,%d}", ret, keyring->serial, atomic_read(&keyring->usage));
+ 	return ret;
+ }
+ EXPORT_SYMBOL(key_link);
+@@ -1036,90 +1216,36 @@ EXPORT_SYMBOL(key_link);
+  */
+ int key_unlink(struct key *keyring, struct key *key)
+ {
+-	struct keyring_list *klist, *nklist;
+-	int loop, ret;
++	struct assoc_array_edit *edit;
++	int ret;
+ 
+ 	key_check(keyring);
+ 	key_check(key);
+ 
+-	ret = -ENOTDIR;
+ 	if (keyring->type != &key_type_keyring)
+-		goto error;
++		return -ENOTDIR;
+ 
+ 	down_write(&keyring->sem);
+ 
+-	klist = rcu_dereference_locked_keyring(keyring);
+-	if (klist) {
+-		/* search the keyring for the key */
+-		for (loop = 0; loop < klist->nkeys; loop++)
+-			if (rcu_access_pointer(klist->keys[loop]) == key)
+-				goto key_is_present;
++	edit = assoc_array_delete(&keyring->keys, &keyring_assoc_array_ops,
++				  &key->index_key);
++	if (IS_ERR(edit)) {
++		ret = PTR_ERR(edit);
++		goto error;
+ 	}
+-
+-	up_write(&keyring->sem);
+ 	ret = -ENOENT;
+-	goto error;
+-
+-key_is_present:
+-	/* we need to copy the key list for RCU purposes */
+-	nklist = kmalloc(sizeof(*klist) +
+-			 sizeof(struct key *) * klist->maxkeys,
+-			 GFP_KERNEL);
+-	if (!nklist)
+-		goto nomem;
+-	nklist->maxkeys = klist->maxkeys;
+-	nklist->nkeys = klist->nkeys - 1;
+-
+-	if (loop > 0)
+-		memcpy(&nklist->keys[0],
+-		       &klist->keys[0],
+-		       loop * sizeof(struct key *));
+-
+-	if (loop < nklist->nkeys)
+-		memcpy(&nklist->keys[loop],
+-		       &klist->keys[loop + 1],
+-		       (nklist->nkeys - loop) * sizeof(struct key *));
+-
+-	/* adjust the user's quota */
+-	key_payload_reserve(keyring,
+-			    keyring->datalen - KEYQUOTA_LINK_BYTES);
+-
+-	rcu_assign_pointer(keyring->payload.subscriptions, nklist);
+-
+-	up_write(&keyring->sem);
+-
+-	/* schedule for later cleanup */
+-	klist->delkey = loop;
+-	call_rcu(&klist->rcu, keyring_unlink_rcu_disposal);
++	if (edit == NULL)
++		goto error;
+ 
++	assoc_array_apply_edit(edit);
+ 	ret = 0;
+ 
+ error:
+-	return ret;
+-nomem:
+-	ret = -ENOMEM;
+ 	up_write(&keyring->sem);
+-	goto error;
++	return ret;
+ }
+ EXPORT_SYMBOL(key_unlink);
+ 
+-/*
+- * Dispose of a keyring list after the RCU grace period, releasing the keys it
+- * links to.
+- */
+-static void keyring_clear_rcu_disposal(struct rcu_head *rcu)
+-{
+-	struct keyring_list *klist;
+-	int loop;
+-
+-	klist = container_of(rcu, struct keyring_list, rcu);
+-
+-	for (loop = klist->nkeys - 1; loop >= 0; loop--)
+-		key_put(rcu_access_pointer(klist->keys[loop]));
+-
+-	kfree(klist);
+-}
+-
+ /**
+  * keyring_clear - Clear a keyring
+  * @keyring: The keyring to clear.
+@@ -1130,33 +1256,25 @@ static void keyring_clear_rcu_disposal(struct rcu_head *rcu)
+  */
+ int keyring_clear(struct key *keyring)
+ {
+-	struct keyring_list *klist;
++	struct assoc_array_edit *edit;
+ 	int ret;
+ 
+-	ret = -ENOTDIR;
+-	if (keyring->type == &key_type_keyring) {
+-		/* detach the pointer block with the locks held */
+-		down_write(&keyring->sem);
+-
+-		klist = rcu_dereference_locked_keyring(keyring);
+-		if (klist) {
+-			/* adjust the quota */
+-			key_payload_reserve(keyring,
+-					    sizeof(struct keyring_list));
+-
+-			rcu_assign_pointer(keyring->payload.subscriptions,
+-					   NULL);
+-		}
+-
+-		up_write(&keyring->sem);
++	if (keyring->type != &key_type_keyring)
++		return -ENOTDIR;
+ 
+-		/* free the keys after the locks have been dropped */
+-		if (klist)
+-			call_rcu(&klist->rcu, keyring_clear_rcu_disposal);
++	down_write(&keyring->sem);
+ 
++	edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops);
++	if (IS_ERR(edit)) {
++		ret = PTR_ERR(edit);
++	} else {
++		if (edit)
++			assoc_array_apply_edit(edit);
++		key_payload_reserve(keyring, 0);
+ 		ret = 0;
+ 	}
+ 
++	up_write(&keyring->sem);
+ 	return ret;
+ }
+ EXPORT_SYMBOL(keyring_clear);
+@@ -1168,17 +1286,25 @@ EXPORT_SYMBOL(keyring_clear);
+  */
+ static void keyring_revoke(struct key *keyring)
+ {
+-	struct keyring_list *klist;
++	struct assoc_array_edit *edit;
+ 
+-	klist = rcu_dereference_locked_keyring(keyring);
++	edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops);
++	if (!IS_ERR(edit)) {
++		if (edit)
++			assoc_array_apply_edit(edit);
++		key_payload_reserve(keyring, 0);
++	}
++}
+ 
+-	/* adjust the quota */
+-	key_payload_reserve(keyring, 0);
++static bool gc_iterator(void *object, void *iterator_data)
++{
++	struct key *key = keyring_ptr_to_key(object);
++	time_t *limit = iterator_data;
+ 
+-	if (klist) {
+-		rcu_assign_pointer(keyring->payload.subscriptions, NULL);
+-		call_rcu(&klist->rcu, keyring_clear_rcu_disposal);
+-	}
++	if (key_is_dead(key, *limit))
++		return false;
++	key_get(key);
++	return true;
+ }
+ 
+ /*
+@@ -1191,88 +1317,12 @@ static void keyring_revoke(struct key *keyring)
+  */
+ void keyring_gc(struct key *keyring, time_t limit)
+ {
+-	struct keyring_list *klist, *new;
+-	struct key *key;
+-	int loop, keep, max;
+-
+ 	kenter("{%x,%s}", key_serial(keyring), keyring->description);
+ 
+ 	down_write(&keyring->sem);
+-
+-	klist = rcu_dereference_locked_keyring(keyring);
+-	if (!klist)
+-		goto no_klist;
+-
+-	/* work out how many subscriptions we're keeping */
+-	keep = 0;
+-	for (loop = klist->nkeys - 1; loop >= 0; loop--)
+-		if (!key_is_dead(rcu_deref_link_locked(klist, loop, keyring),
+-				 limit))
+-			keep++;
+-
+-	if (keep == klist->nkeys)
+-		goto just_return;
+-
+-	/* allocate a new keyring payload */
+-	max = roundup(keep, 4);
+-	new = kmalloc(sizeof(struct keyring_list) + max * sizeof(struct key *),
+-		      GFP_KERNEL);
+-	if (!new)
+-		goto nomem;
+-	new->maxkeys = max;
+-	new->nkeys = 0;
+-	new->delkey = 0;
+-
+-	/* install the live keys
+-	 * - must take care as expired keys may be updated back to life
+-	 */
+-	keep = 0;
+-	for (loop = klist->nkeys - 1; loop >= 0; loop--) {
+-		key = rcu_deref_link_locked(klist, loop, keyring);
+-		if (!key_is_dead(key, limit)) {
+-			if (keep >= max)
+-				goto discard_new;
+-			RCU_INIT_POINTER(new->keys[keep++], key_get(key));
+-		}
+-	}
+-	new->nkeys = keep;
+-
+-	/* adjust the quota */
+-	key_payload_reserve(keyring,
+-			    sizeof(struct keyring_list) +
+-			    KEYQUOTA_LINK_BYTES * keep);
+-
+-	if (keep == 0) {
+-		rcu_assign_pointer(keyring->payload.subscriptions, NULL);
+-		kfree(new);
+-	} else {
+-		rcu_assign_pointer(keyring->payload.subscriptions, new);
+-	}
+-
+-	up_write(&keyring->sem);
+-
+-	call_rcu(&klist->rcu, keyring_clear_rcu_disposal);
+-	kleave(" [yes]");
+-	return;
+-
+-discard_new:
+-	new->nkeys = keep;
+-	keyring_clear_rcu_disposal(&new->rcu);
++	assoc_array_gc(&keyring->keys, &keyring_assoc_array_ops,
++		       gc_iterator, &limit);
+ 	up_write(&keyring->sem);
+-	kleave(" [discard]");
+-	return;
+ 
+-just_return:
+-	up_write(&keyring->sem);
+-	kleave(" [no dead]");
+-	return;
+-
+-no_klist:
+-	up_write(&keyring->sem);
+-	kleave(" [no_klist]");
+-	return;
+-
+-nomem:
+-	up_write(&keyring->sem);
+-	kleave(" [oom]");
++	kleave("");
+ }
+diff --git a/security/keys/request_key.c b/security/keys/request_key.c
+index ab75df4..df94827 100644
+--- a/security/keys/request_key.c
++++ b/security/keys/request_key.c
+@@ -351,7 +351,7 @@ static int construct_alloc_key(struct keyring_search_context *ctx,
+ 			       struct key_user *user,
+ 			       struct key **_key)
+ {
+-	unsigned long prealloc;
++	struct assoc_array_edit *edit;
+ 	struct key *key;
+ 	key_perm_t perm;
+ 	key_ref_t key_ref;
+@@ -380,7 +380,7 @@ static int construct_alloc_key(struct keyring_search_context *ctx,
+ 	set_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags);
+ 
+ 	if (dest_keyring) {
+-		ret = __key_link_begin(dest_keyring, &ctx->index_key, &prealloc);
++		ret = __key_link_begin(dest_keyring, &ctx->index_key, &edit);
+ 		if (ret < 0)
+ 			goto link_prealloc_failed;
+ 	}
+@@ -395,11 +395,11 @@ static int construct_alloc_key(struct keyring_search_context *ctx,
+ 		goto key_already_present;
+ 
+ 	if (dest_keyring)
+-		__key_link(dest_keyring, key, &prealloc);
++		__key_link(key, &edit);
+ 
+ 	mutex_unlock(&key_construction_mutex);
+ 	if (dest_keyring)
+-		__key_link_end(dest_keyring, &ctx->index_key, prealloc);
++		__key_link_end(dest_keyring, &ctx->index_key, edit);
+ 	mutex_unlock(&user->cons_lock);
+ 	*_key = key;
+ 	kleave(" = 0 [%d]", key_serial(key));
+@@ -414,8 +414,8 @@ key_already_present:
+ 	if (dest_keyring) {
+ 		ret = __key_link_check_live_key(dest_keyring, key);
+ 		if (ret == 0)
+-			__key_link(dest_keyring, key, &prealloc);
+-		__key_link_end(dest_keyring, &ctx->index_key, prealloc);
++			__key_link(key, &edit);
++		__key_link_end(dest_keyring, &ctx->index_key, edit);
+ 		if (ret < 0)
+ 			goto link_check_failed;
+ 	}
+-- 
+1.8.3.1
+