From 63d68b122dc5efe6cd0e73873e26c38ba8a2d2f4 Mon Sep 17 00:00:00 2001
From: Noriko Hosoi <nhosoi@redhat.com>
Date: Fri, 13 Jul 2012 11:56:30 -0700
Subject: [PATCH] Trac Ticket #346 - Slow ldapmodify operation time for large 
                   quantities of multi-valued attribute
 values

https://fedorahosted.org/389/ticket/346

Fix Description:
Fix in "commit c0151f78509c35250095da6e2785842337963008" takes
advantage of the knowledge on normalized attribute DNs.  When
generating index keys from the attribute value, it has to be
case normalized, but the attribute DN is normalized w/o the
cases lowered.  This patch introduces attribute flags to
distinguish the 2 cases:
  SLAPI_ATTR_FLAG_NORMALIZED_CES - normalized but not case-
                                   normalized.
  SLAPI_ATTR_FLAG_NORMALIZED_CIS - case-normalized
And SLAPI_ATTR_FLAG_NORMALIZED is the combination of the 2
macros.
---
 ldap/servers/plugins/syntaxes/string.c  |   60 +++++++++++++++++++++++++------
 ldap/servers/slapd/attrlist.c           |    4 +-
 ldap/servers/slapd/back-ldbm/index.c    |    4 ++-
 ldap/servers/slapd/back-ldbm/ldbm_add.c |    2 +-
 ldap/servers/slapd/entry.c              |    2 +-
 ldap/servers/slapd/entrywsi.c           |    4 +-
 ldap/servers/slapd/slapi-plugin.h       |   10 +++++-
 ldap/servers/slapd/value.c              |    2 +-
 8 files changed, 68 insertions(+), 20 deletions(-)

diff --git a/ldap/servers/plugins/syntaxes/string.c b/ldap/servers/plugins/syntaxes/string.c
index 5c78d6a..8d04746 100644
--- a/ldap/servers/plugins/syntaxes/string.c
+++ b/ldap/servers/plugins/syntaxes/string.c
@@ -448,21 +448,31 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 
 		for ( bvlp = bvals, nbvlp = nbvals; bvlp && *bvlp; bvlp++, nbvlp++ )
 		{
+			unsigned long value_flags = slapi_value_get_flags(*bvlp);
 			c = slapi_ch_strdup(slapi_value_get_string(*bvlp));
 			/* if the NORMALIZED flag is set, skip normalizing */
-			if (!(slapi_value_get_flags(*bvlp) & SLAPI_ATTR_FLAG_NORMALIZED)) {
+			if (!(value_flags & SLAPI_ATTR_FLAG_NORMALIZED)) {
 				/* 3rd arg: 1 - trim leading blanks */
 				value_normalize_ext( c, syntax, 1, &alt );
+				value_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+			} else if ((syntax & SYNTAX_DN) &&
+			           (value_flags & SLAPI_ATTR_FLAG_NORMALIZED_CES)) {
+				/* This dn value is normalized, but not case-normalized. */
+				slapi_dn_ignore_case(c);
+				/* This dn value is case-normalized */
+				value_flags &= ~SLAPI_ATTR_FLAG_NORMALIZED_CES;
+				value_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CIS;
 			}
 			if (alt) {
 				slapi_ch_free_string(&c);
-		    	*nbvlp = slapi_value_new_string_passin(alt);
+				*nbvlp = slapi_value_new_string_passin(alt);
 				alt = NULL;
 			} else {
-		    	*nbvlp = slapi_value_new_string_passin(c);
+				*nbvlp = slapi_value_new_string_passin(c);
+				c = NULL;
 			}
 			/* new value is normalized */
-			slapi_value_set_flags(*nbvlp, slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
+			slapi_value_set_flags(*nbvlp, value_flags);
 		}
 		*ivals = nbvals;
 		break;
@@ -570,8 +580,9 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 
 		bvdup= slapi_value_new(); 
 		for ( bvlp = bvals; bvlp && *bvlp; bvlp++ ) {
+			unsigned long value_flags = slapi_value_get_flags(*bvlp);
 			/* 3rd arg: 1 - trim leading blanks */
-			if (!(slapi_value_get_flags(*bvlp) & SLAPI_ATTR_FLAG_NORMALIZED)) {
+			if (!(value_flags & SLAPI_ATTR_FLAG_NORMALIZED)) {
 				c = slapi_ch_strdup(slapi_value_get_string(*bvlp));
 				value_normalize_ext( c, syntax, 1, &alt );
 				if (alt) {
@@ -583,6 +594,17 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 					c = NULL;
 				}
 				bvp = slapi_value_get_berval(bvdup);
+				value_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+			} else if ((syntax & SYNTAX_DN) &&
+			           (value_flags & SLAPI_ATTR_FLAG_NORMALIZED_CES)) {
+				/* This dn value is normalized, but not case-normalized. */
+				c = slapi_ch_strdup(slapi_value_get_string(*bvlp));
+				slapi_dn_ignore_case(c);
+				slapi_value_set_string_passin(bvdup, c);
+				c = NULL;
+				/* This dn value is case-normalized */
+				value_flags &= ~SLAPI_ATTR_FLAG_NORMALIZED_CES;
+				value_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CIS;
 			} else {
 				bvp = slapi_value_get_berval(*bvlp);
 			}
@@ -595,7 +617,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				}
 				buf[substrlens[INDEX_SUBSTRBEGIN]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
-				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
+				slapi_value_set_flags((*ivals)[n], value_flags);
 				n++;
 			}
 
@@ -608,7 +630,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				}
 				buf[substrlens[INDEX_SUBSTRMIDDLE]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
-				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
+				slapi_value_set_flags((*ivals)[n], value_flags);
 				n++;
 			}
 
@@ -621,7 +643,7 @@ string_values2keys( Slapi_PBlock *pb, Slapi_Value **bvals,
 				buf[substrlens[INDEX_SUBSTREND] - 1] = '$';
 				buf[substrlens[INDEX_SUBSTREND]] = '\0';
 				(*ivals)[n] = slapi_value_new_string(buf);
-				slapi_value_set_flags((*ivals)[n], slapi_value_get_flags(*bvlp)|SLAPI_ATTR_FLAG_NORMALIZED);
+				slapi_value_set_flags((*ivals)[n], value_flags);
 				n++;
 			}
 		}
@@ -674,8 +696,16 @@ string_assertion2keys_ava(
                 alt = NULL;
             }
             tmpval->bv.bv_len=strlen(tmpval->bv.bv_val);
+            flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+        } else if ((syntax & SYNTAX_DN) &&
+                   (flags & SLAPI_ATTR_FLAG_NORMALIZED_CES)) {
+            /* This dn value is normalized, but not case-normalized. */
+            slapi_dn_ignore_case(tmpval->bv.bv_val);
+            /* This dn value is case-normalized */
+            flags &= ~SLAPI_ATTR_FLAG_NORMALIZED_CES;
+            flags |= SLAPI_ATTR_FLAG_NORMALIZED_CIS;
         }
-        slapi_value_set_flags(tmpval, flags|SLAPI_ATTR_FLAG_NORMALIZED);
+        slapi_value_set_flags(tmpval, flags);
         break;
 	case LDAP_FILTER_EQUALITY:
 		(*ivals) = (Slapi_Value **) slapi_ch_malloc( 2 * sizeof(Slapi_Value *) );
@@ -689,8 +719,16 @@ string_assertion2keys_ava(
 				(*ivals)[0]->bv.bv_len = strlen( (*ivals)[0]->bv.bv_val );
 				alt = NULL;
 			}
-			slapi_value_set_flags((*ivals)[0], flags|SLAPI_ATTR_FLAG_NORMALIZED);
-		}
+			flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+		} else if ((syntax & SYNTAX_DN) &&
+		           (flags & SLAPI_ATTR_FLAG_NORMALIZED_CES)) {
+            /* This dn value is normalized, but not case-normalized. */
+			slapi_dn_ignore_case((*ivals)[0]->bv.bv_val);
+			/* This dn value is case-normalized */
+			flags &= ~SLAPI_ATTR_FLAG_NORMALIZED_CES;
+			flags |= SLAPI_ATTR_FLAG_NORMALIZED_CIS;
+		}
+		slapi_value_set_flags((*ivals)[0], flags);
 		(*ivals)[1] = NULL;
 		break;
 
diff --git a/ldap/servers/slapd/attrlist.c b/ldap/servers/slapd/attrlist.c
index 4ae5fcf..e365f3d 100644
--- a/ldap/servers/slapd/attrlist.c
+++ b/ldap/servers/slapd/attrlist.c
@@ -292,7 +292,7 @@ int attrlist_replace(Slapi_Attr **alist, const char *type, struct berval **vals)
         valuearray_init_bervalarray(vals, &values);
         if (slapi_attr_is_dn_syntax_attr(*a)) {
             valuearray_dn_normalize_value(values);
-            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
         }
         rc = attr_replace(*a, values);
     }
@@ -319,7 +319,7 @@ int attrlist_replace_with_flags(Slapi_Attr **alist, const char *type, struct ber
         valuearray_init_bervalarray_with_flags(vals, &values, flags);
         if (slapi_attr_is_dn_syntax_attr(*a)) {
             valuearray_dn_normalize_value(values);
-            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+            (*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
         }
         rc = attr_replace(*a, values);
     }
diff --git a/ldap/servers/slapd/back-ldbm/index.c b/ldap/servers/slapd/back-ldbm/index.c
index 43b68f5..a5c6ac5 100644
--- a/ldap/servers/slapd/back-ldbm/index.c
+++ b/ldap/servers/slapd/back-ldbm/index.c
@@ -455,7 +455,9 @@ index_addordel_entry(
                     /* skip "entrydn" */
                     continue;
                 } else {
-                    slapi_values_set_flags(svals, SLAPI_ATTR_FLAG_NORMALIZED);
+                    /* entrydn is case-normalized */
+                    slapi_values_set_flags(svals,
+                                           SLAPI_ATTR_FLAG_NORMALIZED_CIS);
                 }
             }
             result = index_addordel_values_sv( be, type, svals, NULL,
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_add.c b/ldap/servers/slapd/back-ldbm/ldbm_add.c
index 18390c2..158dc8e 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_add.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_add.c
@@ -1189,7 +1189,7 @@ add_update_entrydn_operational_attributes(struct backentry *ep)
     bv.bv_val = (void*)backentry_get_ndn(ep);
     bv.bv_len = strlen( bv.bv_val );
     entry_replace_values_with_flags( ep->ep_entry, LDBM_ENTRYDN_STR, bvp,
-                                     SLAPI_ATTR_FLAG_NORMALIZED );
+                                     SLAPI_ATTR_FLAG_NORMALIZED_CIS );
 }
 
 /*
diff --git a/ldap/servers/slapd/entry.c b/ldap/servers/slapd/entry.c
index ddb62dc..2f69d31 100644
--- a/ldap/servers/slapd/entry.c
+++ b/ldap/servers/slapd/entry.c
@@ -3356,7 +3356,7 @@ slapi_entry_add_values_sv(Slapi_Entry *e,
 		attrlist_find_or_create(alist, type, &a);
 		if (slapi_attr_is_dn_syntax_attr(*a)) {
 			valuearray_dn_normalize_value(vals);
-			(*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+			(*a)->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
 		}
 		rc= attr_add_valuearray(*a,vals,slapi_entry_get_dn_const(e));
     }
diff --git a/ldap/servers/slapd/entrywsi.c b/ldap/servers/slapd/entrywsi.c
index 2f2cfd3..0419864 100644
--- a/ldap/servers/slapd/entrywsi.c
+++ b/ldap/servers/slapd/entrywsi.c
@@ -442,7 +442,7 @@ entry_add_present_values_wsi(Slapi_Entry *e, const char *type, struct berval **b
 		/* Check if the type of the to-be-added values has DN syntax or not. */
 		if (slapi_attr_is_dn_syntax_attr(a)) {
 			valuearray_dn_normalize_value(valuestoadd);
-			a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+			a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
 		}
 		if(urp)
 		{
@@ -564,7 +564,7 @@ entry_delete_present_values_wsi(Slapi_Entry *e, const char *type, struct berval
 			 * or not. */
 			if (slapi_attr_is_dn_syntax_attr(a)) {
 				valuearray_dn_normalize_value(valuestodelete);
-				a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED;
+				a->a_flags |= SLAPI_ATTR_FLAG_NORMALIZED_CES;
 			}
 			if(urp)
 			{
diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h
index 0aa764d..87f8302 100644
--- a/ldap/servers/slapd/slapi-plugin.h
+++ b/ldap/servers/slapd/slapi-plugin.h
@@ -177,7 +177,15 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...)
  * \see slapi_value_set_flags()
  * \see slapi_values_set_flags()
  */
-#define SLAPI_ATTR_FLAG_NORMALIZED	0x0200	/* the attr value is normalized */
+/* the attr value is normalized */
+#define SLAPI_ATTR_FLAG_NORMALIZED \
+        (SLAPI_ATTR_FLAG_NORMALIZED_CES|SLAPI_ATTR_FLAG_NORMALIZED_CIS)
+#define SLAPI_ATTR_FLAG_NORMALIZED_CES 0x0200 /* the attr value is normalized,
+                                                 but not case-normalized.
+                                                 Used for DN. */
+#define SLAPI_ATTR_FLAG_NORMALIZED_CIS 0x0400 /* the attr value is normalized 
+                                                 including case.
+                                                 Used for DN. */
 
 /* operation flags */
 #define SLAPI_OP_FLAG_INTERNAL		0x00020 /* An operation generated by the core server or a plugin. */
diff --git a/ldap/servers/slapd/value.c b/ldap/servers/slapd/value.c
index 6d30f63..73e5d20 100644
--- a/ldap/servers/slapd/value.c
+++ b/ldap/servers/slapd/value.c
@@ -595,7 +595,7 @@ value_dn_normalize_value(Slapi_Value *value)
 		value->bv.bv_val = slapi_ch_strdup(slapi_sdn_get_dn(sdn));
 		value->bv.bv_len = slapi_sdn_get_ndn_len(sdn);
 		slapi_sdn_free(&sdn);
-		slapi_value_set_flags(value, SLAPI_ATTR_FLAG_NORMALIZED);
+		slapi_value_set_flags(value, SLAPI_ATTR_FLAG_NORMALIZED_CES);
 	} else {
 		rc = 1;
 		slapi_ch_free((void **)&sdn); /* free just Slapi_DN */
-- 
1.7.6.5

