616618 - 389 v1.2.5 accepts 2 identical entries with different DN formats

https://bugzilla.redhat.com/show_bug.cgi?id=616618

Change Description:
. renamed 91upgradednformat.pl to 80upgradednformat.pl to run
  upgradednformat before 90subtreerename.pl.
. ported following patches applied to Directory_Server_8_2_Branch
  to master and 389-ds-base-1.2.6.
  612771 - RHDS 8.1/389 v1.2.5 accepts 2 identical entries with
           different DN formats
  * commit 50bb2ee6676e10cdda704982dfa831efa4011037
    Fix Description:
    This patch fixes the upgradednformat utility to eliminate the
    duplicated DNs found in the upgrading period.  Assuming DN:
    attr0=value0,attr1=value1,...,attrn=valuen exists in the DB
    and another identical DN is found, the second DN is renamed to
      nsuniqueid=<uuid>+attr0=value0,attr1=value1,...,attrn=valuen
    , where the <uuid> is the unique id assigned to the entry when
    the entry is added to the server.  The modification is logged
    in the error log.

    In addition, there was a bug to handle multi-valued RDNs in
    slapi_dn_normalize_ext.  In case multi-valued RDN appears with
    DN value (e.g., nsuniqueid=<uuid>+cn=uid\=<uid>\,o\=<org>,dc=<dc>),
    it was not normalized properly.  Introduced second rdn_av_stack
    (subinitial_rdn_av_stack) for the nested DN value.

    See also:
    http://directory.fedoraproject.org/wiki/Upgrade_to_New_DN_Format#Another_Upgrade_Scenario

  * commit ebaa18adaf201f75c709e078abd8e32605eddb81
    Fix Description:
    1) In the previous commit 50bb2ee6676e10cdda704982dfa831efa4011037,
    there were 3 places to miss replacing rdn_av_count with subrdn_av_count.
    2) A variable subtypestart is used to keep the head of nested
    multivalued RDNs.  For instance,
       seeAlso=cn\=<CN>\+l\=<L>\,o\=<O>,dc=examle,dc=com
               ^   or    ^   or  ^
    There was a bug to set the first position ("cn" in the above example).

    In addition, fixed a warning assigning "const char *" to "char *" in
    import_foreman (import-threads.c).

  * commit 472a2c0f2f21205390ba87c31c498b4419a932ba
    Fix Description:
    When newly creating an ancestorid index file, instead of
    opening the index file with DB_TRUNCATE option, delete
    the file then create it.

  616135 - Upgrade failure on Solaris :
           tar: db/NetscapeRoot/{DBVERSION,*.db4}:
           No such file or directory

  * commit e6723421e6dc69408edb2a396eb22da361cff206
    Fix Description:
    .../'{'DBVERSION,*.db4'}' is not a shell syntax. Process DBVERSION
    and *.db4 individually.

Cherry-picked 700d7c1a199db4a690740636b8bd3e612304ce1e and
              0b6ddc9b06845bdf385e749f2c5a99497df20d98 from
389-ds-base-1.2.6 branch.

1 files changed, 171 insertions, 0 deletions

diff --git a/ldap/admin/src/scripts/80upgradednformat.pl b/ldap/admin/src/scripts/80upgradednformat.pl
new file mode 100644
index 00000000..cb44b852
--- /dev/null
+++ b/ldap/admin/src/scripts/80upgradednformat.pl
@@ -0,0 +1,171 @@
+use Mozilla::LDAP::Conn;
+use Mozilla::LDAP::Utils qw(normalizeDN);
+use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn);
+use File::Basename;
+use File::Copy;
+
+# Upgrade DN format if needed.
+# For each backend instance, 
+#     run upgradednformat with -N (dryrun mode),
+#     if it returns 0 (Upgrade candidates are found), 
+#     recursively copy the instance dir to the work dir (dnupgrade)
+#     run upgradednformat w/o -N against the DB in the work dir
+#     if it went ok, replace the original instance dir with the work dir.
+# Note: This script does nothing if the server is up.
+sub runinst {
+    my ($inf, $inst, $dseldif, $conn) = @_;
+
+    my @errs;
+
+    my $config = "cn=config";
+    my $config_entry = $conn->search($config, "base", "(cn=*)");
+    if (!$config_entry) {
+        return ("error_no_configuration_entry", $!);
+    }
+    # First, check if the server is up or down.
+    my $rundir = $config_entry->getValues('nsslapd-rundir');
+
+    # Check if the server is up or not
+    my $pidfile = $rundir . "/" . $inst . ".pid";
+    if (-e $pidfile) {
+        return (); # server is running; do nothing.
+    }
+    my $mappingtree = "cn=mapping tree,cn=config";
+    my $ldbmbase = "cn=ldbm database,cn=plugins,cn=config";
+
+    my $backend_entry;
+    my $mtentry = $conn->search($mappingtree, "onelevel", "(cn=*)", 0, @attr);
+    if (!$mtentry) {
+        return ("error_no_mapping_tree_entries", $!);
+    }
+
+    my $db_config_entry = 
+            $conn->search("cn=config,cn=ldbm database,cn=plugins,cn=config",
+            "base", "(objectclass=*)");
+    if (!$db_config_entry) {
+        return ('error_finding_config_entry',
+                'cn=config,cn=ldbm database,cn=plugins,cn=config',
+                $conn->getErrorString());
+    }
+    # If subtree rename swich is not found in the config file,
+    # set off to the switch and upgrade dn format using entrydn.
+    my $switch = $db_config_entry->getValues('nsslapd-subtree-rename-switch');
+    if ("" eq $switch) { 
+        $db_config_entry->addValue('nsslapd-subtree-rename-switch', "off");
+        $conn->update($db_config_entry);
+    }
+
+    # If a suffix in the mapping tree is doube-quoted and 
+    # the cn value has only the double-quoted value, e.g.
+    #   dn: cn="dc=example,dc=com",cn=mapping tree,cn=config
+    #   cn: "dc=example,dc=com"
+    # the following code adds non-quoted value:
+    #   cn: dc=example,dc=com
+    while ($mtentry) {
+        my $numvals = $mtentry->size("cn");
+        my $i;
+        my $withquotes = -1;
+        my $noquotes = -1;
+        for ($i = 0; $i < $numvals; $i++) {
+            if ($mtentry->{"cn"}[$i] =~ /^".*"$/) {
+                $withquotes = $i;
+            } else {
+                $noquotes = $i;
+            }
+        }
+        if ($withquotes >= 0 && $noquotes == -1) {
+            # Has only cn: "<suffix>"
+            # Adding cn: <suffix>
+            my $stripped = $mtentry->{"cn"}[$withquotes];
+            $stripped =~ s/^"(.*)"$/$1/;
+            $mtentry->addValue("cn", $stripped);
+            $conn->update($mtentry);
+        }
+        $mtentry = $conn->nextEntry();
+    }
+
+    my $instancedir = $config_entry->{"nsslapd-instancedir"}[0];
+    my $upgradednformat = $instancedir . "/upgradednformat";
+
+    # Scan through all of the backends to see if any of them
+    # contain escape characters in the DNs.  If we find any
+    # escapes, we need to run the conversion tool on that
+    # backend.
+    $backend_entry = $conn->search($ldbmbase, "onelevel", "(objectClass=nsBackendInstance)", 0, @attr);
+    if (!$backend_entry) {
+        return ("error_no_backend_entries", $!);
+    }
+
+    while ($backend_entry) {
+        my $backend = $backend_entry->{"cn"}[0];
+        my $dbinstdir = $backend_entry->{"nsslapd-directory"}[0];
+        my $workdir = $dbinstdir . "/dnupgrade";
+        my $dbdir = dirname($dbinstdir);
+        my $pdbdir = dirname($dbdir);
+        my $instname = basename($dbinstdir);
+
+        if ("$dbdir" eq "" || "$instname" eq "") {
+            push @errs, ["error_invalid_dbinst_dir", $dbinstdir];
+            return @errs;
+        }
+
+        # clean up db region files, which might contain the old pages
+        if ( -d $dbdir  && -f $dbdir."/__db.001") {
+            unlink <$dbdir/__db.*>;
+        }
+
+        if (-e "$dbinstdir/id2entry.db4") {
+            # Check if any DNs contain escape characters with dbscan.
+            # dryrun mode
+            # return values:  0 -- need to upgrade dn format
+            #                 1 -- no need to upgrade dn format
+            #                -1 -- error
+            my $escapes = system("$upgradednformat -n $backend -a $dbinstdir -N");
+            if (0 == $escapes) {
+                my $rc = 0;
+
+                if (system("cd $pdbdir; tar cf - db/DBVERSION | (cd $dbinstdir; tar xf -)") ||
+                    system("cd $pdbdir; tar cf - db/$instname/DBVERSION | (cd $dbinstdir; tar xf -)") ||
+                    system("cd $pdbdir; tar cf - db/$instname/*.db4 | (cd $dbinstdir; tar xf -)")) {
+                    push @errs, ["error_cant_backup_db", $backend, $!];
+                    return @errs;
+                }
+                my @stat = stat("$dbdir");
+                my $mode = $stat[2];
+                my $uid = $stat[4];
+                my $gid = $stat[5];
+
+                move("$dbinstdir/db", "$workdir");
+                chmod($mode, $workdir);
+                chown($uid, $gid, $workdir);
+
+                @stat = stat("$dbinstdir");
+                $mode = $stat[2];
+                $uid = $stat[4];
+                $gid = $stat[5];
+
+                chmod($mode, "$workdir/$instname");
+                chown($uid, $gid, "$workdir/$instname");
+
+                # call conversion tool here and get return status.
+                $rc = system("$upgradednformat -n $backend -a $workdir/$instname");
+                if ($rc == 0) { # success
+                    move("$dbinstdir", "$dbinstdir.orig");
+                    move("$dbinstdir.orig/dnupgrade/$instname", "$dbinstdir");
+                    copy("$dbinstdir.orig/dnupgrade/DBVERSION", "$dbdir");
+                } else {
+                    # Conversion failed. Cleanup and bail.
+                    unlink <$dbinstdir/dnupgrade/$backend/*>;
+                    rmdir("$dbinstdir/dnupgrade/$backend");
+                    unlink <$dbinstdir/dnupgrade/*>;
+                    rmdir("$dbinstdir/dnupgrade");
+                    return ("error_cant_convert_db", $backend, $rc);
+                }
+            }
+        }
+
+        $backend_entry = $conn->nextEntry();
+    }
+
+    return ();
+}