wrap new style matching rule plugins for use in old style indexing code

Create wrappers for the new syntax plugin style matching rule code so that
we can use the old matching rule indexing functions.  Introduced a new type
of indexer for Slapi_Value values.  The old style used struct berval * values,
but the syntax plugins and a lot of newer code work with Slapi_Value* instead.

6 files changed, 268 insertions, 46 deletions

diff --git a/ldap/servers/slapd/back-ldbm/ldbm_attr.c b/ldap/servers/slapd/back-ldbm/ldbm_attr.c
index 3866e121..a73a5320 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_attr.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_attr.c
@@ -240,6 +240,25 @@ attr_index_config(
 			}
 
 			/* compute a->ai_index_rules: */
+			/* for index rules there are two uses:
+			 * 1) a simple way to define an ordered index to support <= and >= searches
+			 * for those attributes which do not have an ORDERING matching rule defined
+			 * for them in their schema definition.  The index generated is not a :RULE:
+			 * index, it is a normal = EQUALITY index, with the keys ordered using the
+			 * comparison function provided by the syntax plugin for the attribute.  For
+			 * example - the uidNumber attribute has INTEGER syntax, but the standard
+			 * definition of the attribute does not specify an ORDERING matching rule.
+			 * By default, this means that you cannot perform searches like
+			 * (uidNumber>=501) - but many users expect to be able to perform this type of
+			 * search.  By specifying that you want an ordered index, using an integer
+			 * matching rule, you can support indexed seaches of this type.
+			 * 2) a RULE index - the index key prefix is :NAMEOROID: - this is used
+			 * to support extensible match searches like (cn:fr-CA.3:=gilles), which would
+			 * find the index key :fr-CA.3:gilles in the cn index.
+			 * We check first to see if this is a simple ordered index - user specified an
+			 * ordering matching rule compatible with the attribute syntax, and there is
+			 * a compare function.  If not, we assume it is a RULE index definition.
+			 */
 			j = 0;
 			if (index_rules != NULL) for (; index_rules[j] != NULL; ++j);
 			if (j > 0) { /* there are some candidates */
@@ -250,47 +269,66 @@ attr_index_config(
 					/* Check that index_rules[j] is an official OID */
 					char* officialOID = NULL;
 					IFP mrINDEX = NULL;
-					Slapi_PBlock* pb = slapi_pblock_new();
+					Slapi_PBlock* pb = NULL;
+					int do_continue = 0; /* can we skip the RULE parsing stuff? */
+
+					if ((p = strstr(index_rules[j], INDEX_ATTR_SUBSTRBEGIN))) {
+						_set_attr_substrlen(INDEX_SUBSTRBEGIN, index_rules[j],
+											&substrlens);
+						do_continue = 1; /* done with j - next j */
+					} else if ((p = strstr(index_rules[j], INDEX_ATTR_SUBSTRMIDDLE))) {
+						_set_attr_substrlen(INDEX_SUBSTRMIDDLE, index_rules[j],
+											&substrlens);
+						do_continue = 1; /* done with j - next j */
+					} else if ((p = strstr(index_rules[j], INDEX_ATTR_SUBSTREND))) {
+						_set_attr_substrlen(INDEX_SUBSTREND, index_rules[j],
+											&substrlens);
+						do_continue = 1; /* done with j - next j */
+					/* check if this is a simple ordering specification
+					   for an attribute that has no ordering matching rule */
+					} else if (slapi_matchingrule_is_ordering(index_rules[j], attrsyntax_oid) &&
+							   !a->ai_sattr.a_mr_ord_plugin) { /* no ordering for this attribute */
+						need_compare_fn = 1; /* get compare func for this attr */
+						do_continue = 1; /* done with j - next j */
+					}
+
+					if (do_continue) {
+						continue; /* done with index_rules[j] */
+					}
+
+					/* must be a RULE specification */
+					pb = slapi_pblock_new();
+					/* next check if this is a RULE type index
+					   try to actually create an indexer and see if the indexer
+					   actually has a regular INDEX_FN or an INDEX_SV_FN */
 					if (!slapi_pblock_set (pb, SLAPI_PLUGIN_MR_OID, index_rules[j]) &&
 						!slapi_pblock_set (pb, SLAPI_PLUGIN_MR_TYPE, a->ai_type) &&
 						!slapi_mr_indexer_create (pb) &&
-						!slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX) &&
-						mrINDEX != NULL &&
+						((!slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX) &&
+						  mrINDEX != NULL) ||
+						 (!slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_SV_FN, &mrINDEX) &&
+						  mrINDEX != NULL)) &&
 						!slapi_pblock_get (pb, SLAPI_PLUGIN_MR_OID, &officialOID) &&
 						officialOID != NULL) {
 						if (!strcasecmp (index_rules[j], officialOID)) {
-						official_rules[k++] = slapi_ch_strdup (officialOID);
+							official_rules[k++] = slapi_ch_strdup (officialOID);
 						} else {
-						char* preamble = slapi_ch_smprintf("%s: line %d", fname, lineno);
-						LDAPDebug (LDAP_DEBUG_ANY, "%s: use \"%s\" instead of \"%s\" (ignored)\n",
-							   preamble, officialOID, index_rules[j] );
-						slapi_ch_free((void**)&preamble);
+							char* preamble = slapi_ch_smprintf("%s: line %d", fname, lineno);
+							LDAPDebug (LDAP_DEBUG_ANY, "%s: use \"%s\" instead of \"%s\" (ignored)\n",
+									   preamble, officialOID, index_rules[j] );
+							slapi_ch_free((void**)&preamble);
 						}
-					} else if ((p =
-                                strstr(index_rules[j], INDEX_ATTR_SUBSTRBEGIN))) {
-						_set_attr_substrlen(INDEX_SUBSTRBEGIN, index_rules[j],
-											&substrlens);
-					} else if ((p =
-                                strstr(index_rules[j], INDEX_ATTR_SUBSTRMIDDLE))) {
-						_set_attr_substrlen(INDEX_SUBSTRMIDDLE, index_rules[j],
-											&substrlens);
-					} else if ((p =
-                                strstr(index_rules[j], INDEX_ATTR_SUBSTREND))) {
-						_set_attr_substrlen(INDEX_SUBSTREND, index_rules[j],
-											&substrlens);
-					} else if (!slapi_matchingrule_is_ordering(index_rules[j], attrsyntax_oid)) {
+					} else { /* we don't know what this is */
 						LDAPDebug (LDAP_DEBUG_ANY, "%s: line %d: "
 							   "unknown or invalid matching rule \"%s\" in index configuration (ignored)\n",
 							   fname, lineno, index_rules[j] );
-					} else { /* assume builtin and use compare fn provided by syntax plugin */
-						need_compare_fn = 1;
 					}
 					{/* It would improve speed to save the indexer, for future use.
 						But, for simplicity, we destroy it now: */
 						IFP mrDESTROY = NULL;
 						if (!slapi_pblock_get (pb, SLAPI_PLUGIN_DESTROY_FN, &mrDESTROY) &&
-						mrDESTROY != NULL) {
-						mrDESTROY (pb);
+							mrDESTROY != NULL) {
+							mrDESTROY (pb);
 						}
 					}
 					slapi_pblock_destroy (pb);
@@ -317,12 +355,8 @@ attr_index_config(
 
 		/* if user didn't specify an ordering rule in the index config,
 		   see if the schema def for the attr defines one */
-		if (!need_compare_fn) {
-			asyntaxinfo *asi = attr_syntax_get_by_name( a->ai_type );
-			if (asi && asi->asi_mr_ordering) {
-			 	need_compare_fn = 1;
-			}
-			attr_syntax_return( asi );
+		if (!need_compare_fn && a->ai_sattr.a_mr_ord_plugin) {
+			need_compare_fn = 1;
 		}
 
 		if (need_compare_fn) {
diff --git a/ldap/servers/slapd/back-ldbm/matchrule.c b/ldap/servers/slapd/back-ldbm/matchrule.c
index 0734935f..043c9682 100644
--- a/ldap/servers/slapd/back-ldbm/matchrule.c
+++ b/ldap/servers/slapd/back-ldbm/matchrule.c
@@ -96,7 +96,17 @@ create_matchrule_indexer(Slapi_PBlock **pb,char* matchrule,char* type)
 
 	if ( (0 != return_value) || (mrINDEX == NULL) )
 	{
-		return LDAP_OPERATIONS_ERROR;
+		/* doesn't have an old MR_INDEX_FN - look for MR_INDEX_SV_FN */
+		return_value = slapi_pblock_get (*pb, SLAPI_PLUGIN_MR_INDEX_SV_FN, &mrINDEX);
+		
+		if ( (0 != return_value) || (mrINDEX == NULL) )
+		{
+			return LDAP_OPERATIONS_ERROR;
+		}
+		else
+		{
+			return LDAP_SUCCESS;
+		}
 	}
 	else
 	{
@@ -146,18 +156,10 @@ int
 matchrule_values_to_keys_sv(Slapi_PBlock *pb,Slapi_Value **input_values,Slapi_Value ***output_values)
 {
 	IFP mrINDEX = NULL;
-        struct berval **bvi, **bvo;
-
-        valuearray_get_bervalarray(input_values, &bvi);
 
-	slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX);
-	slapi_pblock_set (pb, SLAPI_PLUGIN_MR_VALUES, bvi);
+	slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_SV_FN, &mrINDEX);
+	slapi_pblock_set (pb, SLAPI_PLUGIN_MR_VALUES, input_values);
 	mrINDEX (pb);
-	slapi_pblock_get (pb, SLAPI_PLUGIN_MR_KEYS, &bvo);
-
-        slapi_pblock_set (pb, SLAPI_PLUGIN_MR_VALUES, NULL);
-        ber_bvecfree(bvi);
-
-        valuearray_init_bervalarray(bvo, output_values);
+	slapi_pblock_get (pb, SLAPI_PLUGIN_MR_KEYS, output_values);
 	return 0;
 }
diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c
index b5d994ad..7d885745 100644
--- a/ldap/servers/slapd/pblock.c
+++ b/ldap/servers/slapd/pblock.c
@@ -1349,6 +1349,9 @@ slapi_pblock_get( Slapi_PBlock *pblock, int arg, void *value )
 	case SLAPI_PLUGIN_MR_INDEX_FN:
 		(*(IFP *)value) = pblock->pb_mr_index_fn;
 		break;
+	case SLAPI_PLUGIN_MR_INDEX_SV_FN:
+		(*(IFP *)value) = pblock->pb_mr_index_sv_fn;
+		break;
 
 	/* matching rule plugin arguments */
 	case SLAPI_PLUGIN_MR_OID:
@@ -2722,6 +2725,9 @@ slapi_pblock_set( Slapi_PBlock *pblock, int arg, void *value )
 	case SLAPI_PLUGIN_MR_INDEX_FN:
 		pblock->pb_mr_index_fn = (IFP) value;
 		break;
+	case SLAPI_PLUGIN_MR_INDEX_SV_FN:
+		pblock->pb_mr_index_sv_fn = (IFP) value;
+		break;
 
 	/* matching rule plugin arguments */
 	case SLAPI_PLUGIN_MR_OID:
diff --git a/ldap/servers/slapd/plugin_mr.c b/ldap/servers/slapd/plugin_mr.c
index 8f62a7a0..014082b7 100644
--- a/ldap/servers/slapd/plugin_mr.c
+++ b/ldap/servers/slapd/plugin_mr.c
@@ -47,9 +47,16 @@
 
 #include "slap.h"
 
+struct mr_indexer_private {
+	Slapi_Value **sva; /* if using index_sv_fn */
+	struct berval **bva; /* if using index_fn */
+};
+
 static oid_item_t* global_mr_oids = NULL;
 static PRLock* global_mr_oids_lock = NULL;
 
+static int default_mr_indexer_create(Slapi_PBlock* pb);
+
 static void
 init_global_mr_lock()
 {
@@ -78,6 +85,27 @@ plugin_mr_find( const char *nameoroid )
 	return ( pi );
 }
 
+static int
+plugin_mr_get_type(struct slapdplugin *pi)
+{
+	int rc = LDAP_FILTER_EQUALITY;
+	if (pi) {
+		char **str = pi->plg_mr_names;
+		for (; str && *str; ++str) {
+			if (PL_strcasestr(*str, "substr")) {
+				rc = LDAP_FILTER_SUBSTRINGS;
+				break;
+			}
+			if (PL_strcasestr(*str, "approx")) {
+				rc = LDAP_FILTER_APPROX;
+				break;
+			}
+		}
+	}
+
+	return rc;
+}
+
 static struct slapdplugin*
 plugin_mr_find_registered (char* oid)
 {
@@ -144,15 +172,32 @@ slapi_mr_indexer_create (Slapi_PBlock* opb)
 				    !(rc = slapi_pblock_get (&pb, SLAPI_PLUGIN_MR_INDEXER_CREATE_FN, &createFn)) &&
 				    createFn != NULL &&
 				    !(rc = createFn (&pb)) &&
-				    !(rc = slapi_pblock_get (&pb, SLAPI_PLUGIN_MR_INDEX_FN, &indexFn)) &&
-				    indexFn != NULL)
+					((!(rc = slapi_pblock_get (&pb, SLAPI_PLUGIN_MR_INDEX_FN, &indexFn)) &&
+					 indexFn != NULL) ||
+					 (!(rc = slapi_pblock_get (&pb, SLAPI_PLUGIN_MR_INDEX_SV_FN, &indexFn)) &&
+					  indexFn != NULL)))
 				{
 				    /* Success: this plugin can handle it. */
 				    memcpy (opb, &pb, sizeof(Slapi_PBlock));
 				    plugin_mr_bind (oid, mrp); /* for future reference */
+					rc = 0; /* success */
 				    break;
 				}
 		    }
+			if (rc != 0) {
+				/* look for a new syntax-style mr plugin */
+				struct slapdplugin *pi = plugin_mr_find(oid);
+				if (pi) {
+					Slapi_PBlock pb;
+					memcpy (&pb, opb, sizeof(Slapi_PBlock));
+					slapi_pblock_set(&pb, SLAPI_PLUGIN, pi);
+					rc = default_mr_indexer_create(&pb);
+					if (!rc) {
+						memcpy (opb, &pb, sizeof(Slapi_PBlock));
+						plugin_mr_bind (oid, pi); /* for future reference */
+					}
+				}
+			}
 		}
     }
     return rc;
@@ -229,3 +274,136 @@ slapi_mr_filter_index (Slapi_Filter* f, Slapi_PBlock* pb)
     return rc;
 }
 
+static struct mr_indexer_private *
+mr_indexer_private_new()
+{
+	return (struct mr_indexer_private *)slapi_ch_calloc(1, sizeof(struct mr_indexer_private));
+}
+
+static void
+mr_indexer_private_done(struct mr_indexer_private *mrip)
+{
+	if (mrip && mrip->sva) {
+		valuearray_free(&mrip->sva);
+	} else if (mrip && mrip->bva) {
+		ber_bvecfree(mrip->bva);
+		mrip->bva = NULL;
+	}
+}
+
+static void
+mr_indexer_private_free(struct mr_indexer_private **mrip)
+{
+	if (mrip) {
+		mr_indexer_private_done(*mrip);
+		slapi_ch_free((void **)mrip);
+	}
+}
+
+/* this function takes SLAPI_PLUGIN_MR_VALUES as Slapi_Value ** and
+   returns SLAPI_PLUGIN_MR_KEYS as Slapi_Value **
+*/
+static int
+mr_wrap_mr_index_sv_fn(Slapi_PBlock* pb)
+{
+	int rc = -1;
+	Slapi_Value **in_vals = NULL;
+	Slapi_Value **out_vals = NULL;
+	struct slapdplugin *pi = NULL;
+	
+	slapi_pblock_set(pb, SLAPI_PLUGIN_MR_KEYS, out_vals); /* make sure output is cleared */
+	slapi_pblock_get(pb, SLAPI_PLUGIN, &pi);
+	if (!pi) {
+		LDAPDebug0Args(LDAP_DEBUG_ANY, "mr_wrap_mr_index_sv_fn: error - no plugin specified\n");
+	} else if (!pi->plg_mr_values2keys) {
+		LDAPDebug0Args(LDAP_DEBUG_ANY, "mr_wrap_mr_index_sv_fn: error - plugin has no plg_mr_values2keys function\n");
+	} else {
+		struct mr_indexer_private *mrip = NULL;
+		int ftype = plugin_mr_get_type(pi);
+		slapi_pblock_get(pb, SLAPI_PLUGIN_MR_VALUES, &in_vals);
+		(*pi->plg_mr_values2keys)(pb, in_vals, &out_vals, ftype);
+		slapi_pblock_set(pb, SLAPI_PLUGIN_MR_KEYS, out_vals);
+		/* we have to save out_vals to free next time or during destroy */
+		slapi_pblock_get(pb, SLAPI_PLUGIN_OBJECT, &mrip);
+		mr_indexer_private_done(mrip); /* free old vals, if any */
+		mrip->sva = out_vals; /* save pointer for later */
+		rc = 0;
+	}
+	return rc;
+}
+
+/* this function takes SLAPI_PLUGIN_MR_VALUES as struct berval ** and
+   returns SLAPI_PLUGIN_MR_KEYS as struct berval **
+*/
+static int
+mr_wrap_mr_index_fn(Slapi_PBlock* pb)
+{
+	int rc = -1;
+	struct berval **in_vals = NULL;
+	struct berval **out_vals = NULL;
+	struct mr_indexer_private *mrip = NULL;
+	Slapi_Value **in_vals_sv = NULL;
+	Slapi_Value **out_vals_sv = NULL;
+
+	slapi_pblock_get(pb, SLAPI_PLUGIN_MR_VALUES, &in_vals); /* get bervals */
+	/* convert bervals to sv ary */
+	valuearray_init_bervalarray(in_vals, &in_vals_sv);
+	slapi_pblock_set(pb, SLAPI_PLUGIN_MR_VALUES, in_vals_sv); /* use sv */
+	rc = mr_wrap_mr_index_sv_fn(pb);
+	/* get result sv keys */
+	slapi_pblock_get(pb, SLAPI_PLUGIN_MR_KEYS, &out_vals_sv);
+	/* convert to bvec */
+	valuearray_get_bervalarray(out_vals_sv, &out_vals);
+	valuearray_free(&out_vals_sv); /* don't need svals */
+	/* we have to save out_vals to free next time or during destroy */
+	slapi_pblock_get(pb, SLAPI_PLUGIN_OBJECT, &mrip);
+	mr_indexer_private_done(mrip); /* free old vals, if any */
+	mrip->bva = out_vals; /* save pointer for later */
+
+	return rc;
+}
+
+static int
+default_mr_indexer_destroy(Slapi_PBlock* pb)
+{
+	struct mr_indexer_private *mrip = NULL;
+
+	slapi_pblock_get(pb, SLAPI_PLUGIN_OBJECT, &mrip);
+	mr_indexer_private_free(&mrip);
+	mrip = NULL;
+	slapi_pblock_set(pb, SLAPI_PLUGIN_OBJECT, mrip);
+
+	return 0;
+}
+
+/* this is the default mr indexer create func
+   for new syntax-style mr plugins */
+static int
+default_mr_indexer_create(Slapi_PBlock* pb)
+{
+	int rc = -1;
+	struct slapdplugin *pi = NULL;
+	
+	slapi_pblock_get(pb, SLAPI_PLUGIN, &pi);
+	if (NULL == pi) {
+		LDAPDebug0Args(LDAP_DEBUG_ANY, "default_mr_indexer_create: error - no plugin specified\n");
+		goto done;
+	}
+
+	if (NULL == pi->plg_mr_values2keys) {
+		LDAPDebug1Arg(LDAP_DEBUG_ANY, "default_mr_indexer_create: error - plugin [%s] has no plg_mr_values2keys function\n",
+					  pi->plg_name);
+		goto done;
+	}
+
+	slapi_pblock_set(pb, SLAPI_PLUGIN_OBJECT, mr_indexer_private_new());
+	slapi_pblock_set(pb, SLAPI_PLUGIN_MR_INDEX_FN, mr_wrap_mr_index_fn);
+	slapi_pblock_set(pb, SLAPI_PLUGIN_MR_INDEX_SV_FN, mr_wrap_mr_index_sv_fn);
+	slapi_pblock_set(pb, SLAPI_PLUGIN_DESTROY_FN, default_mr_indexer_destroy);
+	slapi_pblock_set(pb, SLAPI_PLUGIN_MR_INDEXER_CREATE_FN, default_mr_indexer_create);
+	rc = 0;
+
+done:
+	return rc;
+}
+
diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h
index 9dea4525..adef7a8a 100644
--- a/ldap/servers/slapd/slap.h
+++ b/ldap/servers/slapd/slap.h
@@ -1440,7 +1440,7 @@ typedef struct slapi_pblock {
 	mrFilterMatchFn	pb_mr_filter_match_fn;
 	IFP		pb_mr_filter_index_fn;
 	IFP		pb_mr_filter_reset_fn;
-	IFP		pb_mr_index_fn;
+	IFP		pb_mr_index_fn; /* values and keys are struct berval ** */
 	char*		pb_mr_oid;
 	char*		pb_mr_type;
 	struct berval*	pb_mr_value;
@@ -1540,6 +1540,7 @@ typedef struct slapi_pblock {
 							   /* used in plugin init; pb_plugin is not ready, then */
 	LDAPControl	**pb_search_ctrls; /* for search operations, allows plugins to provide
 									  controls to pass for each entry or referral returned */
+	IFP		pb_mr_index_sv_fn; /* values and keys are Slapi_Value ** */
 } slapi_pblock;
 
 /* index if substrlens */
diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h
index 6ced381e..5f0fc47f 100644
--- a/ldap/servers/slapd/slapi-plugin.h
+++ b/ldap/servers/slapd/slapi-plugin.h
@@ -5456,6 +5456,7 @@ typedef struct slapi_plugindesc {
 #define SLAPI_PLUGIN_MR_FILTER_INDEX_FN		603
 #define SLAPI_PLUGIN_MR_FILTER_RESET_FN		604
 #define SLAPI_PLUGIN_MR_INDEX_FN		605
+#define SLAPI_PLUGIN_MR_INDEX_SV_FN		606
 
 /* matching rule plugin arguments */
 #define SLAPI_PLUGIN_MR_OID			610