591336 - Implementing upgrade DN format tool

Change description:
. adding upgradednformat utility to each server instance.
. adding 91upgradednformat.pl for in-place-upgrade.
. implementing ldbm_back_upgradednformat sharing the import/
  reincexing codes.
. adding a new DBVERSION ID "dn-4514" for the upgraded db.
. fixing access logs (delete.c and modify.c)
. fixing compiler warnings.
. fixing memory leaks.
. fixing a bug in syntax plugin to free strings.
. adding templates for plugin id, version, vendor, and description,
  which are needed for the online upgrade.
. dbversion_write takes an additional bit flags, which indicates
  which extra DBVERSION strings are written to the DBVERSION file.
  It was introduced for the upgrade tools not to intervene each
  other's tasks (e.g., dn2rdn for converting entrydn to entryrdn
  and upgradednformat for upgrading the DN format).
. fixing a bug in entryrdn index code which was missing to normalize
  RDN.

See also:
https://bugzilla.redhat.com/show_bug.cgi?id=591336
http://directory.fedoraproject.org/wiki/Upgrade_to_New_DN_Format#Migration.2FUpgrade

4 files changed, 213 insertions, 0 deletions

diff --git a/ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif b/ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif
index 5375c9f1..6ef0fae2 100644
--- a/ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif
+++ b/ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif
@@ -6,3 +6,8 @@ nsslapd-pluginpath: libpwdstorage-plugin
 nsslapd-plugininitfunc: smd5_pwd_storage_scheme_init
 nsslapd-plugintype: pwdstoragescheme
 nsslapd-pluginenabled: on
+# these will be replaced when the server loads the plugin
+nsslapd-pluginId: ID
+nsslapd-pluginVersion: PACKAGE_VERSION
+nsslapd-pluginVendor: VENDOR
+nsslapd-pluginDescription: DESC
diff --git a/ldap/admin/src/scripts/91upgradednformat.pl b/ldap/admin/src/scripts/91upgradednformat.pl
new file mode 100644
index 00000000..1bed3cfd
--- /dev/null
+++ b/ldap/admin/src/scripts/91upgradednformat.pl
@@ -0,0 +1,145 @@
+use Mozilla::LDAP::Conn;
+use Mozilla::LDAP::Utils qw(normalizeDN);
+use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn);
+use File::Basename;
+use File::Copy;
+
+# Upgrade DN format if needed.
+# For each backend instance, 
+#     run upgradednformat with -N (dryrun mode),
+#     if it returns 0 (Upgrade candidates are found), 
+#     recursively copy the instance dir to the work dir (dnupgrade)
+#     run upgradednformat w/o -N against the DB in the work dir
+#     if it went ok, replace the original instance dir with the work dir.
+sub runinst {
+    my ($inf, $inst, $dseldif, $conn) = @_;
+
+    my @errs;
+
+    my $config = "cn=config";
+    my $mappingtree = "cn=mapping tree,cn=config";
+    my $ldbmbase = "cn=ldbm database,cn=plugins,cn=config";
+
+    my $backend_entry;
+    my $mtentry = $conn->search($mappingtree, "onelevel", "(cn=*)", 0, @attr);
+    if (!$mtentry) {
+        return ("error_no_mapping_tree_entries", $!);
+    }
+
+    # If a suffix in the mapping tree is doube-quoted and 
+    # the cn value has only the double-quoted value, e.g.
+    #   dn: cn="dc=example,dc=com",cn=mapping tree,cn=config
+    #   cn: "dc=example,dc=com"
+    # the following code adds non-quoted value:
+    #   cn: dc=example,dc=com
+    while ($mtentry) {
+        my $numvals = $mtentry->size("cn");
+        my $i;
+        my $withquotes = -1;
+        my $noquotes = -1;
+        for ($i = 0; $i < $numvals; $i++) {
+            if ($mtentry->{"cn"}[$i] =~ /^".*"$/) {
+                $withquotes = $i;
+            } else {
+                $noquotes = $i;
+            }
+        }
+        if ($withquotes >= 0 && $noquotes == -1) {
+            # Has only cn: "<suffix>"
+            # Adding cn: <suffix>
+            my $stripped = $mtentry->{"cn"}[$withquotes];
+            $stripped =~ s/^"(.*)"$/$1/;
+            $mtentry->addValue("cn", $stripped);
+            $conn->update($mtentry);
+        }
+        $mtentry = $conn->nextEntry();
+    }
+
+    my $config_entry = $conn->search($config, "base", "(cn=*)", 0, ("nsslapd-instancedir"));
+    if (!$config_entry) {
+        return ("error_no_configuration_entry", $!);
+    }
+    my $instancedir = $config_entry->{"nsslapd-instancedir"}[0];
+    my $upgradednformat = $instancedir . "/upgradednformat";
+
+    # Scan through all of the backends to see if any of them
+    # contain escape characters in the DNs.  If we find any
+    # escapes, we need to run the conversion tool on that
+    # backend.
+    $backend_entry = $conn->search($ldbmbase, "onelevel", "(objectClass=nsBackendInstance)", 0, @attr);
+    if (!$backend_entry) {
+        return ("error_no_backend_entries", $!);
+    }
+
+    while ($backend_entry) {
+        my $backend = $backend_entry->{"cn"}[0];
+        my $dbinstdir = $backend_entry->{"nsslapd-directory"}[0];
+        my $workdir = $dbinstdir . "/dnupgrade";
+        my $dbdir = dirname($dbinstdir);
+        my $pdbdir = dirname($dbdir);
+        my $instname = basename($dbinstdir);
+
+        if ("$dbdir" eq "" || "$instname" eq "") {
+            push @errs, ["error_invalid_dbinst_dir", $dbinstdir];
+            return @errs;
+        }
+
+        # clean up db region files, which might contain the old pages
+        if ( -d $dbdir  && -f $dbdir."/__db.001") {
+            unlink <$dbdir/__db.*>;
+        }
+
+        if (-e "$dbinstdir/id2entry.db4") {
+            # Check if any DNs contain escape characters with dbscan.
+            # dryrun mode
+            # return values:  0 -- need to upgrade dn format
+            #                 1 -- no need to upgrade dn format
+            #                -1 -- error
+            my $escapes = system("$upgradednformat -n $backend -a $dbinstdir -N");
+            if (0 == $escapes) {
+                my $rc = 0;
+
+                if (system("cd $pdbdir; tar cf - db/DBVERSION | (cd $dbinstdir; tar xf -)") ||
+                    system("cd $pdbdir; tar cf - db/$instname/{DBVERSION,*.db4} | (cd $dbinstdir; tar xf -)")) {
+                    push @errs, ["error_cant_backup_db", $backend, $!];
+                    return @errs;
+                }
+                my @stat = stat("$dbdir");
+                my $mode = $stat[2];
+                my $uid = $stat[4];
+                my $gid = $stat[5];
+
+                move("$dbinstdir/db", "$workdir");
+                chmod($mode, $workdir);
+                chown($uid, $gid, $workdir);
+
+                @stat = stat("$dbinstdir");
+                $mode = $stat[2];
+                $uid = $stat[4];
+                $gid = $stat[5];
+
+                chmod($mode, "$workdir/$instname");
+                chown($uid, $gid, "$workdir/$instname");
+
+                # call conversion tool here and get return status.
+                $rc = system("$upgradednformat -n $backend -a $workdir/$instname");
+                if ($rc == 0) { # success
+                    move("$dbinstdir", "$dbinstdir.orig");
+                    move("$dbinstdir.orig/dnupgrade/$instname", "$dbinstdir");
+                    copy("$dbinstdir.orig/dnupgrade/DBVERSION", "$dbdir");
+                } else {
+                    # Conversion failed. Cleanup and bail.
+                    unlink <$dbinstdir/dnupgrade/$backend/*>;
+                    rmdir("$dbinstdir/dnupgrade/$backend");
+                    unlink <$dbinstdir/dnupgrade/*>;
+                    rmdir("$dbinstdir/dnupgrade");
+                    return ("error_cant_convert_db", $backend, $rc);
+                }
+            }
+        }
+
+        $backend_entry = $conn->nextEntry();
+    }
+
+    return ();
+}
diff --git a/ldap/admin/src/scripts/setup-ds.res.in b/ldap/admin/src/scripts/setup-ds.res.in
index ca7b6e7e..25f84935 100644
--- a/ldap/admin/src/scripts/setup-ds.res.in
+++ b/ldap/admin/src/scripts/setup-ds.res.in
@@ -185,3 +185,10 @@ error_online_update = Could not open a connection to the server at %s port %s as
 Please make sure the server is up and running before using online mode,\
 or use offline mode.\n\n
 error_offline_update = Could not read the server config file '%s'. Error: %s\n\n
+error_no_mapping_tree_entries = Could not find a mapping tree entry.  Error: %s\n
+error_no_configuration_entry = Could not find a configuration entry.  Error: %s\n
+error_no_configuration_entry = Could not find a backend entry.  Error: %s\n
+error_invalid_dbinst_dir = Invalid database instance dir '%s'.\n
+error_cant_backup_db = Failed to back up backend instance '%s'.  Error: %s\n
+error_cant_convert_db = Failed to convert backend instance '%s'.  Error: %s\n
+error_missing_entrydn = Backend instance '%s' does not have database files to upgrade.\n
diff --git a/ldap/admin/src/scripts/template-upgradednformat.in b/ldap/admin/src/scripts/template-upgradednformat.in
new file mode 100755
index 00000000..ea4f18a4
--- /dev/null
+++ b/ldap/admin/src/scripts/template-upgradednformat.in
@@ -0,0 +1,56 @@
+#!/bin/sh
+
+# upgradednformat -- upgrade DN format to the new style (RFC 4514)
+# Usgae: upgradednformat [-N] -n backend_instance -a db_instance_directory
+#        -N: dryrun
+#            exit code: 0 -- needs upgrade; 1 -- no need to upgrade; -1 -- error
+#        -n backend_instance -- instance name to be examined or upgraded
+#        -a db_instance_directory -- full path to the db instance dir
+#                                    e.g., /var/lib/dirsrv/slapd-ID/db/userRoot
+prefix="{{DS-ROOT}}"
+if [ "$prefix" = "/" ] ; then
+    prefix=""
+fi
+LD_LIBRARY_PATH=$prefix/{{SERVER-DIR}}:$prefix@nss_libdir@:$prefix@libdir@:$prefix@pcre_libdir@
+if [ -n "$prefix" ] ; then
+    LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:@nss_libdir@"
+fi
+export LD_LIBRARY_PATH
+SHLIB_PATH=$LD_LIBRARY_PATH
+export SHLIB_PATH
+
+cd {{SERVERBIN-DIR}}
+
+dir=""
+be=""
+dryrun=0
+while [ "$1" != "" ]
+do
+    if [ "$1" = "-a" ]; then
+        shift
+        dir="$1"
+    elif [ "$1" = "-n" ]; then
+        shift
+        be="$1"
+    elif [ "$1" = "-N" ]; then
+        dryrun=1
+    fi
+    if [ "$1" != "" ]; then
+        shift
+    fi
+done
+
+if [ "$be" = "" ] || [ "$dir" = "" ]; then
+        echo "be: $be"
+        echo "dir: $dir"
+    echo "Usage: $0 [-N] -n backend_instance -a db_instance_directory"
+    exit 1
+fi
+
+if [ $dryrun -eq 0 ]; then
+    ./ns-slapd upgradednformat -D {{CONFIG-DIR}} -a $dir -n $be
+else
+    ./ns-slapd upgradednformat -D {{CONFIG-DIR}} -a $dir -n $be -N
+fi
+rc=$?
+exit $rc