616618 - 389 v1.2.5 accepts 2 identical entries with different DN formats

https://bugzilla.redhat.com/show_bug.cgi?id=616618

Change Description:
. renamed 91upgradednformat.pl to 80upgradednformat.pl to run
  upgradednformat before 90subtreerename.pl.
. ported following patches applied to Directory_Server_8_2_Branch
  to master and 389-ds-base-1.2.6.
  612771 - RHDS 8.1/389 v1.2.5 accepts 2 identical entries with
           different DN formats
  * commit 50bb2ee6676e10cdda704982dfa831efa4011037
    Fix Description:
    This patch fixes the upgradednformat utility to eliminate the
    duplicated DNs found in the upgrading period.  Assuming DN:
    attr0=value0,attr1=value1,...,attrn=valuen exists in the DB
    and another identical DN is found, the second DN is renamed to
      nsuniqueid=<uuid>+attr0=value0,attr1=value1,...,attrn=valuen
    , where the <uuid> is the unique id assigned to the entry when
    the entry is added to the server.  The modification is logged
    in the error log.

    In addition, there was a bug to handle multi-valued RDNs in
    slapi_dn_normalize_ext.  In case multi-valued RDN appears with
    DN value (e.g., nsuniqueid=<uuid>+cn=uid\=<uid>\,o\=<org>,dc=<dc>),
    it was not normalized properly.  Introduced second rdn_av_stack
    (subinitial_rdn_av_stack) for the nested DN value.

    See also:
    http://directory.fedoraproject.org/wiki/Upgrade_to_New_DN_Format#Another_Upgrade_Scenario

  * commit ebaa18adaf201f75c709e078abd8e32605eddb81
    Fix Description:
    1) In the previous commit 50bb2ee6676e10cdda704982dfa831efa4011037,
    there were 3 places to miss replacing rdn_av_count with subrdn_av_count.
    2) A variable subtypestart is used to keep the head of nested
    multivalued RDNs.  For instance,
       seeAlso=cn\=<CN>\+l\=<L>\,o\=<O>,dc=examle,dc=com
               ^   or    ^   or  ^
    There was a bug to set the first position ("cn" in the above example).

    In addition, fixed a warning assigning "const char *" to "char *" in
    import_foreman (import-threads.c).

  * commit 472a2c0f2f21205390ba87c31c498b4419a932ba
    Fix Description:
    When newly creating an ancestorid index file, instead of
    opening the index file with DB_TRUNCATE option, delete
    the file then create it.

  616135 - Upgrade failure on Solaris :
           tar: db/NetscapeRoot/{DBVERSION,*.db4}:
           No such file or directory

  * commit e6723421e6dc69408edb2a396eb22da361cff206
    Fix Description:
    .../'{'DBVERSION,*.db4'}' is not a shell syntax. Process DBVERSION
    and *.db4 individually.

1 files changed, 75 insertions, 58 deletions

diff --git a/ldap/servers/slapd/dn.c b/ldap/servers/slapd/dn.c
index bd22f76e..e50ec768 100644
--- a/ldap/servers/slapd/dn.c
+++ b/ldap/servers/slapd/dn.c
@@ -493,13 +493,19 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
     char *d = NULL; /* work pointer for dest */
     char *ends = NULL;
     char *endd = NULL;
-    char *typestart = NULL;
-    char *subtypestart = NULL; /* used for the nested DN */
     char *lastesc = NULL;
+    /* rdn avs for the main DN */
+    char *typestart = NULL;
     int rdn_av_count = 0;
     struct berval *rdn_avs = NULL;
     struct berval initial_rdn_av_stack[ SLAPI_DNNORM_INITIAL_RDN_AVS ];
+    /* rdn avs for the nested DN */
+    char *subtypestart = NULL; /* used for nested rdn avs */
+    int subrdn_av_count = 0;
+    struct berval *subrdn_avs = NULL;
+    struct berval subinitial_rdn_av_stack[ SLAPI_DNNORM_INITIAL_RDN_AVS ];
     int chkblank = 0;
+    int avstat = 0;
 
     if (NULL == dest) {
         goto bail;
@@ -596,6 +602,12 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                 rc = -1;
                 goto bail;
             } /* otherwise, go through */
+            if (ISESCAPE(*s)) {
+                subtypestart = NULL; /* if escaped, can't be multivalued dn */
+            } else {
+                subtypestart = d; /* prepare for '+' in the nested DN, if any */
+            }
+            subrdn_av_count = 0;
         case INVALUE:    /* in value; cn=ABC */
                          /*               ^  */
             if (ISESCAPE(*s)) {
@@ -612,20 +624,6 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                         goto bail;
                     } else {
                         if (ISEQUAL(*(s+1))) {
-                            if (NULL == subtypestart) {
-                                /* e.g., cn=a\=b\,c\=d */
-                                /*            ^        */
-                                *d = '\0';
-                                subtypestart = strrchr(*dest, '=');
-                                if (subtypestart) {
-                                    /* pointing 'a' in the above example */
-                                    subtypestart++;
-                                }
-                                /* else 
-                                 *    No equal for the nested DN looking value.
-                                 *    Just through it.
-                                 */
-                            }
                             while (ISSPACE(*(d-1))) {
                                 /* remove trailing spaces */
                                 d--;
@@ -649,16 +647,18 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                              * multivalued RDNs.
                              */
                             if (subtypestart &&
-                                (ISPLUS(*(s+1)) || rdn_av_count > 0)) {
-                                add_rdn_av(subtypestart, d, &rdn_av_count,
-                                           &rdn_avs, initial_rdn_av_stack);
+                                (ISPLUS(*(s+1)) || subrdn_av_count > 0)) {
+                                add_rdn_av(subtypestart, d, &subrdn_av_count,
+                                          &subrdn_avs, subinitial_rdn_av_stack);
                             }
                             if (!ISPLUS(*(s+1))) {    /* at end of this RDN */
-                                if (rdn_av_count > 1) {
-                                    sort_rdn_avs( rdn_avs, rdn_av_count, 1 );
+                                if (subrdn_av_count > 1) {
+                                    sort_rdn_avs( subrdn_avs, 
+                                                  subrdn_av_count, 1 );
                                 }
-                                if (rdn_av_count > 0) {
-                                    reset_rdn_avs( &rdn_avs, &rdn_av_count );
+                                if (subrdn_av_count > 0) {
+                                    reset_rdn_avs( &subrdn_avs,
+                                                   &subrdn_av_count );
                                     subtypestart = NULL;
                                 }
                             }
@@ -667,9 +667,17 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                         *d++ = *s++;            /* '\\' */
                         PR_snprintf(d, 3, "%X", *s);    /* hexpair */
                         d += 2;
-                        if (ISPLUS(*s) && PL_strnstr(s, "\\=", ends - s)) {
+                        if (ISPLUS(*s)) {
                             /* next type start of multi values */
-                            subtypestart = d;
+                            /* should not be a escape char AND should be 
+                             * followed by \\= or \\3D */
+                            if (!ISESCAPE(*s) &&
+                                (PL_strnstr(s, "\\=", ends - s) ||
+                                 PL_strncaserstr(s, "\\3D", ends - s))) {
+                                subtypestart = d;
+                            } else {
+                                subtypestart = NULL;
+                            }
                         }
                         if (SEPARATOR(*s) || ISEQUAL(*s)) {
                             while (ISSPACE(*(s+1)))
@@ -681,26 +689,12 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                     }
                 } else if (((state == INVALUE1ST) &&
                             (s+2 < ends) && LEADNEEDSESCAPESTR(s+1)) ||
-				           ((state == INVALUE) && 
+                           ((state == INVALUE) && 
                             (((s+2 < ends) && NEEDSESCAPESTR(s+1)) ||
                              (ISEOV(s+3, ends) && ISBLANKSTR(s+1))))) {
                              /* e.g., cn=abc\20 ,... */
                              /*             ^        */
                     if (ISEQUALSTR(s+1)) {
-                        if (NULL == subtypestart) {
-                            /* e.g., cn=a\3Db\2Cc\3Dd */
-                            /*           ^            */
-                            *d = '\0';
-                            subtypestart = strrchr(*dest, '=');
-                            if (subtypestart) {
-                                /* pointing 'a' in the above example */
-                                subtypestart++;
-                            }
-                            /* else 
-                             *    No equal for the nested DN looking value.
-                             *    Just through it.
-                             */
-                        }
                         while (ISSPACE(*(d-1))) {
                             /* remove trailing spaces */
                             d--;
@@ -724,25 +718,33 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                          * multivalued RDNs.
                          */
                         if (subtypestart &&
-                            (ISPLUSSTR(s+1) || rdn_av_count > 0)) {
-                            add_rdn_av(subtypestart, d, &rdn_av_count,
-                                       &rdn_avs, initial_rdn_av_stack);
+                            (ISPLUSSTR(s+1) || subrdn_av_count > 0)) {
+                            add_rdn_av(subtypestart, d, &subrdn_av_count,
+                                       &subrdn_avs, subinitial_rdn_av_stack);
                         }
                         if (!ISPLUSSTR(s+1)) {    /* at end of this RDN */
-                            if (rdn_av_count > 1) {
-                                sort_rdn_avs( rdn_avs, rdn_av_count, 1 );
+                            if (subrdn_av_count > 1) {
+                                sort_rdn_avs( subrdn_avs, subrdn_av_count, 1 );
                             }
-                            if (rdn_av_count > 0) {
-                                reset_rdn_avs( &rdn_avs, &rdn_av_count );
+                            if (subrdn_av_count > 0) {
+                                reset_rdn_avs( &subrdn_avs, &subrdn_av_count );
+                                subtypestart = NULL;
                             }
                         }
                     }
                     *d++ = *s++;            /* '\\' */
                     *d++ = *s++;            /* HEX */
                     *d++ = *s++;            /* HEX */
-                    if (ISPLUSSTR(s-2) && PL_strnstr(s, "\\=", ends - s)) {
+                    if (ISPLUSSTR(s-2)) {
                         /* next type start of multi values */
-                        subtypestart = d;
+                        /* should not be a escape char AND should be followed
+                         * by \\= or \\3D */
+                        if (!ISESCAPE(*s) && (PL_strnstr(s, "\\=", ends - s) ||
+                            PL_strncaserstr(s, "\\3D", ends - s))) {
+                            subtypestart = d;
+                        } else {
+                            subtypestart = NULL;
+                        }
                     }
                     if (SEPARATORSTR(s-2) || ISEQUALSTR(s-2)) {
                         while (ISSPACE(*s)) /* remove leading spaces */
@@ -804,6 +806,7 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                 continue;
             }
             subtypestart = d; /* prepare for '+' in the quoted value, if any */
+            subrdn_av_count = 0;
         case INQUOTEDVALUE:
             if (ISQUOTE(*s)) {
                 if (ISESCAPE(*(d-1))) { /* the quote is escaped */
@@ -855,16 +858,17 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                          * Track and sort attribute values within
                          * multivalued RDNs.
                          */
-                        if (ISPLUS(*s) || rdn_av_count > 0) {
-                            add_rdn_av(subtypestart, d, &rdn_av_count,
-                                       &rdn_avs, initial_rdn_av_stack);
+                        if (subtypestart &&
+                            (ISPLUS(*s) || subrdn_av_count > 0)) {
+                            add_rdn_av(subtypestart, d, &subrdn_av_count,
+                                       &subrdn_avs, subinitial_rdn_av_stack);
                         }
                         if (!ISPLUS(*s)) {    /* at end of this RDN */
-                            if (rdn_av_count > 1) {
-                                sort_rdn_avs( rdn_avs, rdn_av_count, 1 );
+                            if (subrdn_av_count > 1) {
+                                sort_rdn_avs( subrdn_avs, subrdn_av_count, 1 );
                             }
-                            if (rdn_av_count > 0) {
-                                reset_rdn_avs( &rdn_avs, &rdn_av_count );
+                            if (subrdn_av_count > 0) {
+                                reset_rdn_avs( &subrdn_avs, &subrdn_av_count );
                                 subtypestart = NULL;
                             }
                         }
@@ -897,7 +901,8 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                  * Track and sort attribute values within
                  * multivalued RDNs.
                  */
-                if (ISPLUS(*s) || rdn_av_count > 0) {
+                if (typestart &&
+                    (ISPLUS(*s) || rdn_av_count > 0)) {
                     add_rdn_av(typestart, d, &rdn_av_count,
                                &rdn_avs, initial_rdn_av_stack);
                 }
@@ -907,6 +912,7 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
                     }
                     if (rdn_av_count > 0) {
                         reset_rdn_avs( &rdn_avs, &rdn_av_count );
+                        typestart = NULL;
                     }
                 }
 
@@ -931,7 +937,7 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len)
      * rdn to our list to sort.  We should only be in the INVALUE
      * or B4SEPARATOR state if we have a valid rdn component to 
      * be added. */
-    if ((rdn_av_count > 0) && ((state == INVALUE1ST) || 
+    if (typestart && (rdn_av_count > 0) && ((state == INVALUE1ST) || 
         (state == INVALUE) || (state == B4SEPARATOR))) {
         add_rdn_av(typestart, d, &rdn_av_count, &rdn_avs, initial_rdn_av_stack);
     }
@@ -1160,6 +1166,17 @@ rdn_av_swap( struct berval *av1, struct berval *av2, int escape )
 
 /* Introduced for the upgrade tool. DON'T USE THIS API! */
 char *
+slapi_dn_normalize_original( char *dn )
+{
+	/* LDAPDebug( LDAP_DEBUG_TRACE, "=> slapi_dn_normalize \"%s\"\n", dn, 0, 0 ); */
+	*(substr_dn_normalize_orig( dn, dn + strlen( dn ))) = '\0';
+	/* LDAPDebug( LDAP_DEBUG_TRACE, "<= slapi_dn_normalize \"%s\"\n", dn, 0, 0 ); */
+
+	return( dn );
+}
+
+/* Introduced for the upgrade tool. DON'T USE THIS API! */
+char *
 slapi_dn_normalize_case_original( char *dn )
 {
 	/* LDAPDebug( LDAP_DEBUG_TRACE, "=> slapi_dn_normalize \"%s\"\n", dn, 0, 0 ); */