diff options
| author | Noriko Hosoi <nhosoi@redhat.com> | 2010-05-12 11:14:16 -0700 |
|---|---|---|
| committer | Noriko Hosoi <nhosoi@redhat.com> | 2010-05-12 11:14:16 -0700 |
| commit | 142d7ada626173de4937330be6776fabbebe9f60 (patch) | |
| tree | c0931005e4f59fd0b91f7a5f48b900d232eca315 | |
| parent | e7a81ddaa86b284d8eb29906e63e7841958f8ef1 (diff) | |
| download | ds-142d7ada626173de4937330be6776fabbebe9f60.tar.gz ds-142d7ada626173de4937330be6776fabbebe9f60.tar.xz ds-142d7ada626173de4937330be6776fabbebe9f60.zip | |
591336 - Implementing upgrade DN format tool
Change description:
. adding upgradednformat utility to each server instance.
. adding 70upgradednformat.pl for in-place-upgrade.
. implementing ldbm_back_upgradednformat sharing the import/
reincexing codes.
. adding a new DBVERSION ID "dn-4514" for the upgraded db.
. fixing access logs (delete.c and modify.c)
. fixing compiler warnings.
. fixing a bug in syntax plugin to free strings.
. adding templates for plugin id, version, vendor, and description,
which are needed for the online upgrade.
See also:
https://bugzilla.redhat.com/show_bug.cgi?id=591336
http://directory.fedoraproject.org/wiki/Upgrade_to_New_DN_Format#Migration.2FUpgrade
29 files changed, 1720 insertions, 238 deletions
diff --git a/Makefile.am b/Makefile.am index 98c65449..1550e04e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -88,7 +88,10 @@ CLEANFILES = dberrstrs.h ns-slapd.properties \ ldap/admin/src/scripts/template-restart-slapd ldap/admin/src/scripts/template-restoreconfig \ ldap/admin/src/scripts/template-saveconfig ldap/admin/src/scripts/template-start-slapd \ ldap/admin/src/scripts/template-stop-slapd ldap/admin/src/scripts/template-suffix2instance \ - ldap/admin/src/scripts/template-upgradedb ldap/admin/src/scripts/template-verify-db.pl \ + ldap/admin/src/scripts/template-upgradedb \ + ldap/admin/src/scripts/template-upgradednformat \ + ldap/admin/src/scripts/template-usn-tombstone-cleanup.pl \ + ldap/admin/src/scripts/template-verify-db.pl \ ldap/admin/src/scripts/template-vlvindex ldap/admin/src/scripts/DSUtil.pm \ ldap/ldif/template-baseacis.ldif ldap/ldif/template-bitwise.ldif ldap/ldif/template-country.ldif \ ldap/ldif/template-dnaplugin.ldif ldap/ldif/template-domain.ldif ldap/ldif/template-dse.ldif \ @@ -312,12 +315,13 @@ task_SCRIPTS = ldap/admin/src/scripts/template-bak2db \ ldap/admin/src/scripts/template-stop-slapd \ ldap/admin/src/scripts/template-restart-slapd \ ldap/admin/src/scripts/template-suffix2instance \ + ldap/admin/src/scripts/template-upgradednformat \ ldap/admin/src/scripts/template-vlvindex \ ldap/admin/src/scripts/template-bak2db.pl \ ldap/admin/src/scripts/template-db2bak.pl \ ldap/admin/src/scripts/template-db2index.pl \ ldap/admin/src/scripts/template-db2ldif.pl \ - ldap/admin/src/scripts/template-fixup-linkedattrs.pl \ + ldap/admin/src/scripts/template-fixup-linkedattrs.pl \ ldap/admin/src/scripts/template-fixup-memberof.pl \ ldap/admin/src/scripts/template-ldif2db.pl \ ldap/admin/src/scripts/template-ns-accountstatus.pl \ @@ -409,6 +413,7 @@ update_DATA = ldap/admin/src/scripts/exampleupdate.pl \ ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif \ ldap/admin/src/scripts/50retroclprecedence.ldif \ ldap/admin/src/scripts/60upgradeschemafiles.pl \ + ldap/admin/src/scripts/70upgradednformat.pl \ ldap/admin/src/scripts/dnaplugindepends.ldif update_SCRIPTS = ldap/admin/src/scripts/exampleupdate.sh diff --git a/Makefile.in b/Makefile.in index 4a10ac04..86d1e1ac 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1190,7 +1190,10 @@ CLEANFILES = dberrstrs.h ns-slapd.properties \ ldap/admin/src/scripts/template-restart-slapd ldap/admin/src/scripts/template-restoreconfig \ ldap/admin/src/scripts/template-saveconfig ldap/admin/src/scripts/template-start-slapd \ ldap/admin/src/scripts/template-stop-slapd ldap/admin/src/scripts/template-suffix2instance \ - ldap/admin/src/scripts/template-upgradedb ldap/admin/src/scripts/template-verify-db.pl \ + ldap/admin/src/scripts/template-upgradedb \ + ldap/admin/src/scripts/template-upgradednformat \ + ldap/admin/src/scripts/template-usn-tombstone-cleanup.pl \ + ldap/admin/src/scripts/template-verify-db.pl \ ldap/admin/src/scripts/template-vlvindex ldap/admin/src/scripts/DSUtil.pm \ ldap/ldif/template-baseacis.ldif ldap/ldif/template-bitwise.ldif ldap/ldif/template-country.ldif \ ldap/ldif/template-dnaplugin.ldif ldap/ldif/template-domain.ldif ldap/ldif/template-dse.ldif \ @@ -1370,12 +1373,13 @@ task_SCRIPTS = ldap/admin/src/scripts/template-bak2db \ ldap/admin/src/scripts/template-stop-slapd \ ldap/admin/src/scripts/template-restart-slapd \ ldap/admin/src/scripts/template-suffix2instance \ + ldap/admin/src/scripts/template-upgradednformat \ ldap/admin/src/scripts/template-vlvindex \ ldap/admin/src/scripts/template-bak2db.pl \ ldap/admin/src/scripts/template-db2bak.pl \ ldap/admin/src/scripts/template-db2index.pl \ ldap/admin/src/scripts/template-db2ldif.pl \ - ldap/admin/src/scripts/template-fixup-linkedattrs.pl \ + ldap/admin/src/scripts/template-fixup-linkedattrs.pl \ ldap/admin/src/scripts/template-fixup-memberof.pl \ ldap/admin/src/scripts/template-ldif2db.pl \ ldap/admin/src/scripts/template-ns-accountstatus.pl \ @@ -1468,6 +1472,7 @@ update_DATA = ldap/admin/src/scripts/exampleupdate.pl \ ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif \ ldap/admin/src/scripts/50retroclprecedence.ldif \ ldap/admin/src/scripts/60upgradeschemafiles.pl \ + ldap/admin/src/scripts/70upgradednformat.pl \ ldap/admin/src/scripts/dnaplugindepends.ldif update_SCRIPTS = ldap/admin/src/scripts/exampleupdate.sh diff --git a/ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif b/ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif index 5375c9f1..6ef0fae2 100644 --- a/ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif +++ b/ldap/admin/src/scripts/50smd5pwdstorageplugin.ldif @@ -6,3 +6,8 @@ nsslapd-pluginpath: libpwdstorage-plugin nsslapd-plugininitfunc: smd5_pwd_storage_scheme_init nsslapd-plugintype: pwdstoragescheme nsslapd-pluginenabled: on +# these will be replaced when the server loads the plugin +nsslapd-pluginId: ID +nsslapd-pluginVersion: PACKAGE_VERSION +nsslapd-pluginVendor: VENDOR +nsslapd-pluginDescription: DESC diff --git a/ldap/admin/src/scripts/60upgradeschemafiles.pl b/ldap/admin/src/scripts/60upgradeschemafiles.pl index 984973f5..fb8a5c9b 100644 --- a/ldap/admin/src/scripts/60upgradeschemafiles.pl +++ b/ldap/admin/src/scripts/60upgradeschemafiles.pl @@ -11,7 +11,7 @@ sub runinst { # these schema files are obsolete, or we want to replace # them with newer versions - my @toremove = qw(00core.ldif 01common.ldif 05rfc2247.ldif 10presence.ldif 28pilot.ldif 50ns-directory.ldif 60mozilla.ldif); + my @toremove = qw(00core.ldif 01common.ldif 05rfc2247.ldif 10presence.ldif 28pilot.ldif 30ns-common.ldif 50ns-directory.ldif 60mozilla.ldif); # these hashes will be used to check for obsolete schema # in 99user.ldif diff --git a/ldap/admin/src/scripts/70upgradednformat.pl b/ldap/admin/src/scripts/70upgradednformat.pl new file mode 100644 index 00000000..b223e125 --- /dev/null +++ b/ldap/admin/src/scripts/70upgradednformat.pl @@ -0,0 +1,147 @@ +use Mozilla::LDAP::Conn; +use Mozilla::LDAP::Utils qw(normalizeDN); +use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn); +use File::Basename; +use File::Copy; + +# Upgrade DN format if needed. +# For each backend instance, +# run upgradednformat with -N (dryrun mode), +# if it returns 0 (Upgrade candidates are found), +# recursively copy the instance dir to the work dir (dnupgrade) +# run upgradednformat w/o -N against the DB in the work dir +# if it went ok, replace the original instance dir with the work dir. +sub runinst { + my ($inf, $inst, $dseldif, $conn) = @_; + + my @errs; + + my $config = "cn=config"; + my $mappingtree = "cn=mapping tree,cn=config"; + my $ldbmbase = "cn=ldbm database,cn=plugins,cn=config"; + + my $backend_entry; + my $mtentry = $conn->search($mappingtree, "onelevel", "(cn=*)", 0, @attr); + if (!$mtentry) { + return ("error_no_mapping_tree_entries", $!); + } + + # If a suffix in the mapping tree is doube-quoted and + # the cn value has only the double-quoted value, e.g. + # dn: cn="dc=example,dc=com",cn=mapping tree,cn=config + # cn: "dc=example,dc=com" + # the following code adds non-quoted value: + # cn: dc=example,dc=com + while ($mtentry) { + my $numvals = $mtentry->size("cn"); + my $i; + my $withquotes = -1; + my $noquotes = -1; + for ($i = 0; $i < $numvals; $i++) { + if ($mtentry->{"cn"}[$i] =~ /^".*"$/) { + $withquotes = $i; + } else { + $noquotes = $i; + } + } + if ($withquotes >= 0 && $noquotes == -1) { + # Has only cn: "<suffix>" + # Adding cn: <suffix> + my $stripped = $mtentry->{"cn"}[$withquotes]; + $stripped =~ s/^"(.*)"$/$1/; + $mtentry->addValue("cn", $stripped); + $conn->update($mtentry); + } + $mtentry = $conn->nextEntry(); + } + + my $config_entry = $conn->search($config, "base", "(cn=*)", 0, ("nsslapd-instancedir")); + if (!$config_entry) { + return ("error_no_configuration_entry", $!); + } + my $instancedir = $config_entry->{"nsslapd-instancedir"}[0]; + my $upgradednformat = $instancedir . "/upgradednformat"; + + # Scan through all of the backends to see if any of them + # contain escape characters in the DNs. If we find any + # escapes, we need to run the conversion tool on that + # backend. + $backend_entry = $conn->search($ldbmbase, "onelevel", "(objectClass=nsBackendInstance)", 0, @attr); + if (!$backend_entry) { + return ("error_no_backend_entries", $!); + } + + while ($backend_entry) { + my $backend = $backend_entry->{"cn"}[0]; + my $dbinstdir = $backend_entry->{"nsslapd-directory"}[0]; + my $workdir = $dbinstdir . "/dnupgrade"; + my $dbdir = dirname($dbinstdir); + my $pdbdir = dirname($dbdir); + my $instname = basename($dbinstdir); + + if ("$dbdir" eq "" || "$instname" eq "") { + push @errs, ["error_invalid_dbinst_dir", $dbinstdir]; + return @errs; + } + + # clean up db region files, which might contain the old pages + if ( -d $dbdir && -f $dbdir."/__db.001") { + unlink <$dbdir/__db.*>; + } + + if (-e "$dbinstdir/entrydn.db4") { + # Check if any DNs contain escape characters with dbscan. + # dryrun mode + # return values: 0 -- need to upgrade dn format + # 1 -- no need to upgrade dn format + # -1 -- error + my $escapes = system("$upgradednformat -n $backend -a $dbinstdir -N"); + if (0 == $escapes) { + my $rc = 0; + + if (system("cd $pdbdir; tar cf - db/DBVERSION | (cd $dbinstdir; tar xf -)") || + system("cd $pdbdir; tar cf - db/$instname/{DBVERSION,*.db4} | (cd $dbinstdir; tar xf -)")) { + push @errs, ["error_cant_backup_db", $backend, $!]; + return @errs; + } + my @stat = stat("$dbdir"); + my $mode = $stat[2]; + my $uid = $stat[4]; + my $gid = $stat[5]; + + move("$dbinstdir/db", "$workdir"); + chmod($mode, $workdir); + chown($uid, $gid, $workdir); + + @stat = stat("$dbinstdir"); + $mode = $stat[2]; + $uid = $stat[4]; + $gid = $stat[5]; + + chmod($mode, "$workdir/$instname"); + chown($uid, $gid, "$workdir/$instname"); + + # call conversion tool here and get return status. + $rc = system("$upgradednformat -n $backend -a $workdir/$instname"); + if ($rc == 0) { # success + move("$dbinstdir", "$dbinstdir.orig"); + move("$dbinstdir.orig/dnupgrade/$instname", "$dbinstdir"); + copy("$dbinstdir.orig/dnupgrade/DBVERSION", "$dbdir"); + } else { + # Conversion failed. Cleanup and bail. + unlink <$dbinstdir/dnupgrade/$backend/*>; + rmdir("$dbinstdir/dnupgrade/$backend"); + unlink <$dbinstdir/dnupgrade/*>; + rmdir("$dbinstdir/dnupgrade"); + return ("error_cant_convert_db", $backend, $rc); + } + } + } else { + return ("error_missing_entrydn", $backend); + } + + $backend_entry = $conn->nextEntry(); + } + + return (); +} diff --git a/ldap/admin/src/scripts/setup-ds.res.in b/ldap/admin/src/scripts/setup-ds.res.in index b0d88c21..79803a6a 100644 --- a/ldap/admin/src/scripts/setup-ds.res.in +++ b/ldap/admin/src/scripts/setup-ds.res.in @@ -184,3 +184,10 @@ error_online_update = Could not open a connection to the server at %s port %s as Please make sure the server is up and running before using online mode,\ or use offline mode.\n\n error_offline_update = Could not read the server config file '%s'. Error: %s\n\n +error_no_mapping_tree_entries = Could not find a mapping tree entry. Error: %s\n +error_no_configuration_entry = Could not find a configuration entry. Error: %s\n +error_no_configuration_entry = Could not find a backend entry. Error: %s\n +error_invalid_dbinst_dir = Invalid database instance dir '%s'.\n +error_cant_backup_db = Failed to back up backend instance '%s'. Error: %s\n +error_cant_convert_db = Failed to convert backend instance '%s'. Error: %s\n +error_missing_entrydn = Backend instance '%s' does not have database files to upgrade.\n diff --git a/ldap/admin/src/scripts/template-upgradednformat.in b/ldap/admin/src/scripts/template-upgradednformat.in new file mode 100755 index 00000000..ea4f18a4 --- /dev/null +++ b/ldap/admin/src/scripts/template-upgradednformat.in @@ -0,0 +1,56 @@ +#!/bin/sh + +# upgradednformat -- upgrade DN format to the new style (RFC 4514) +# Usgae: upgradednformat [-N] -n backend_instance -a db_instance_directory +# -N: dryrun +# exit code: 0 -- needs upgrade; 1 -- no need to upgrade; -1 -- error +# -n backend_instance -- instance name to be examined or upgraded +# -a db_instance_directory -- full path to the db instance dir +# e.g., /var/lib/dirsrv/slapd-ID/db/userRoot +prefix="{{DS-ROOT}}" +if [ "$prefix" = "/" ] ; then + prefix="" +fi +LD_LIBRARY_PATH=$prefix/{{SERVER-DIR}}:$prefix@nss_libdir@:$prefix@libdir@:$prefix@pcre_libdir@ +if [ -n "$prefix" ] ; then + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:@nss_libdir@" +fi +export LD_LIBRARY_PATH +SHLIB_PATH=$LD_LIBRARY_PATH +export SHLIB_PATH + +cd {{SERVERBIN-DIR}} + +dir="" +be="" +dryrun=0 +while [ "$1" != "" ] +do + if [ "$1" = "-a" ]; then + shift + dir="$1" + elif [ "$1" = "-n" ]; then + shift + be="$1" + elif [ "$1" = "-N" ]; then + dryrun=1 + fi + if [ "$1" != "" ]; then + shift + fi +done + +if [ "$be" = "" ] || [ "$dir" = "" ]; then + echo "be: $be" + echo "dir: $dir" + echo "Usage: $0 [-N] -n backend_instance -a db_instance_directory" + exit 1 +fi + +if [ $dryrun -eq 0 ]; then + ./ns-slapd upgradednformat -D {{CONFIG-DIR}} -a $dir -n $be +else + ./ns-slapd upgradednformat -D {{CONFIG-DIR}} -a $dir -n $be -N +fi +rc=$? +exit $rc diff --git a/ldap/ldif/template-bitwise.ldif.in b/ldap/ldif/template-bitwise.ldif.in index 6087fc2e..896d386d 100644 --- a/ldap/ldif/template-bitwise.ldif.in +++ b/ldap/ldif/template-bitwise.ldif.in @@ -7,4 +7,9 @@ nsslapd-pluginPath: libbitwise-plugin nsslapd-pluginInitfunc: bitwise_init nsslapd-pluginType: matchingRule nsslapd-pluginEnabled: on +# these will be replaced when the server loads the plugin +nsslapd-pluginId: ID +nsslapd-pluginVersion: PACKAGE_VERSION +nsslapd-pluginVendor: VENDOR +nsslapd-pluginDescription: DESC diff --git a/ldap/servers/plugins/syntaxes/string.c b/ldap/servers/plugins/syntaxes/string.c index fc4ca202..9131f9f3 100644 --- a/ldap/servers/plugins/syntaxes/string.c +++ b/ldap/servers/plugins/syntaxes/string.c @@ -616,7 +616,9 @@ string_assertion2keys_ava( /* 3rd arg: 1 - trim leading blanks */ value_normalize_ext(tmpval->bv.bv_val, syntax, 1, &alt ); if (alt) { - slapi_ch_free_string(&tmpval->bv.bv_val); + if (len >= tmpval->bv.bv_len) { + slapi_ch_free_string(&tmpval->bv.bv_val); + } tmpval->bv.bv_val = alt; } tmpval->bv.bv_len=strlen(tmpval->bv.bv_val); @@ -687,8 +689,11 @@ string_assertion2keys_sub( int maxsublen; char *comp_buf = NULL; char *altinit = NULL; + char *oaltinit = NULL; char **altany = NULL; + char **oaltany = NULL; char *altfinal = NULL; + char *oaltfinal = NULL; int anysize = 0; slapi_pblock_get(pb, SLAPI_SYNTAX_SUBSTRLENS, &substrlens); @@ -719,6 +724,7 @@ string_assertion2keys_sub( if ( initial != NULL ) { /* 3rd arg: 0 - DO NOT trim leading blanks */ value_normalize_ext( initial, syntax, 0, &altinit ); + oaltinit = altinit; if (NULL == altinit) { altinit = initial; } @@ -737,11 +743,14 @@ string_assertion2keys_sub( anysize++; } altany = (char **)slapi_ch_calloc(anysize + 1, sizeof(char *)); + oaltany = (char **)slapi_ch_calloc(anysize + 1, sizeof(char *)); for ( i = 0; any != NULL && any[i] != NULL; i++ ) { /* 3rd arg: 0 - DO NOT trim leading blanks */ value_normalize_ext( any[i], syntax, 0, &altany[i] ); if (NULL == altany[i]) { altany[i] = any[i]; + } else { + oaltany[i] = altany[i]; } len = strlen( altany[i] ); if ( len >= substrlens[INDEX_SUBSTRMIDDLE] ) { @@ -751,6 +760,7 @@ string_assertion2keys_sub( if ( final != NULL ) { /* 3rd arg: 0 - DO NOT trim leading blanks */ value_normalize_ext( final, syntax, 0, &altfinal ); + oaltfinal = altfinal; if (NULL == altfinal) { altfinal = final; } @@ -784,10 +794,8 @@ string_assertion2keys_sub( if ( altinit != NULL ) { substring_comp_keys( ivals, &nsubs, altinit, initiallen, '^', syntax, comp_buf, substrlens ); - if (altinit != initial) { - slapi_ch_free_string(&altinit); - } } + slapi_ch_free_string(&oaltinit); for ( i = 0; altany != NULL && altany[i] != NULL; i++ ) { len = strlen( altany[i] ); if ( len < substrlens[INDEX_SUBSTRMIDDLE] ) { @@ -795,18 +803,15 @@ string_assertion2keys_sub( } substring_comp_keys( ivals, &nsubs, altany[i], len, 0, syntax, comp_buf, substrlens ); - if (altany[i] != any[i]) { - slapi_ch_free_string(&altany[i]); - } + slapi_ch_free_string(&oaltany[i]); } + slapi_ch_free((void **)&oaltany); slapi_ch_free((void **)&altany); if ( altfinal != NULL ) { substring_comp_keys( ivals, &nsubs, altfinal, finallen, '$', syntax, comp_buf, substrlens ); - if (altfinal != final) { - slapi_ch_free_string(&final); - } } + slapi_ch_free_string(&oaltfinal); (*ivals)[nsubs] = NULL; slapi_ch_free_string(&comp_buf); diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h index ac886a39..1bf9a035 100644 --- a/ldap/servers/slapd/back-ldbm/back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h @@ -167,6 +167,7 @@ typedef unsigned short u_int16_t; */ #define BDB_IMPL "bdb" #define BDB_BACKEND "libback-ldbm" /* This backend plugin */ +#define BDB_DNFORMAT "dn-4514" /* DN format RFC 4514 compliant */ /* * While we support both new and old idl index, @@ -734,7 +735,7 @@ typedef struct _back_search_result_set #define BE_INDEX_TOMBSTONE 8 /* Index entry as a tombstone */ #define BE_INDEX_DONT_ENCRYPT 16 /* Disable any encryption if this flag is set */ #define BE_INDEX_EQUALITY 32 /* (w/DEL) remove the equality index */ -#define BE_INDEX_NORMALIZED SLAPI_ATTR_FLAG_NORMALIZED /* value already normalized */ +#define BE_INDEX_NORMALIZED SLAPI_ATTR_FLAG_NORMALIZED /* value already normalized (0x200) */ /* Name of attribute type used for binder-based look through limit */ #define LDBM_LOOKTHROUGHLIMIT_AT "nsLookThroughLimit" diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c index 3b778a8f..c60451fe 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.c +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -1826,7 +1826,7 @@ int dblayer_instance_start(backend *be, int mode) struct dblayer_private_env *pEnv; char inst_dir[MAXPATHLEN]; char *inst_dirp = NULL; - int return_value; + int return_value = -1; priv = (dblayer_private*)li->li_dblayer_private; pEnv = priv->dblayer_env; @@ -2216,7 +2216,7 @@ int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv) size_t cachesize; PRFileInfo prfinfo; PRStatus prst; - char *id2entry_file; + char *id2entry_file = NULL; char inst_dir[MAXPATHLEN]; char *inst_dirp = NULL; char *data_directories[2] = {0, 0}; @@ -2583,35 +2583,6 @@ int dblayer_post_close(struct ldbminfo *li, int dbmode) return_value = pEnv->dblayer_DB_ENV->close(pEnv->dblayer_DB_ENV, 0); dblayer_free_env(&priv->dblayer_env); /* pEnv is now garbage */ -#if 0 /* DBDB do NOT remove the environment: bad, bad idea */ - if (return_value == 0) { - DB_ENV *env = 0; - return_value = db_env_create(&env, 0); - /* don't be tempted to use the - previously nulled out env handle - as Sleepycat 3.x is unhappy if - the env handle handed to remove - was used elsewhere. rwagner */ - if (return_value == 0) { - char *home_dir = dblayer_get_home_dir(li, NULL); - if (home_dir) - return_value = env->remove(env, home_dir, 0); - if (0 == return_value - && !((DBLAYER_ARCHIVE_MODE|DBLAYER_EXPORT_MODE) & dbmode) - && !priv->dblayer_bad_stuff_happened) { - /* - * If we got here, we have a good consistent database, - * so we write the guard file - */ - commit_good_database(priv); - } else if (return_value == EBUSY) { - /* something else is using the env so ignore */ - /* but let's not make a guardian file */ - return_value = 0; - } - } - } -#endif if (0 == return_value && !((DBLAYER_ARCHIVE_MODE|DBLAYER_EXPORT_MODE) & dbmode) && !priv->dblayer_bad_stuff_happened) { @@ -2676,11 +2647,44 @@ int dblayer_close(struct ldbminfo *li, int dbmode) * for the transacted database, we interpret this as an instruction * to write a checkpoint. */ -int dblayer_flush(struct ldbminfo *li) +int +dblayer_flush(struct ldbminfo *li) { return 0; } +/* API to remove the environment */ +int +dblayer_remove_env(struct ldbminfo *li) +{ + DB_ENV *env = NULL; + dblayer_private *priv = NULL; + char *home_dir = NULL; + int rc = db_env_create(&env, 0); + if (rc) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "ERROR -- Failed to create DB_ENV (returned: %d)\n", rc); + return rc; + } + if (NULL == li) { + LDAPDebug0Args(LDAP_DEBUG_ANY, "ERROR -- No ldbm info is given\n"); + return -1; + } + priv = (dblayer_private *)li->li_dblayer_private; + + home_dir = dblayer_get_home_dir(li, NULL); + if (home_dir) { + rc = env->remove(env, home_dir, 0); + if (rc) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "ERROR -- Failed to remove DB environment files. " + "Please remove %s/__db.00# (# is 1 through 6)\n", + home_dir); + } + } + return rc; +} + #if !defined(DB_DUPSORT) #define DB_DUPSORT 0 #endif diff --git a/ldap/servers/slapd/back-ldbm/dbversion.c b/ldap/servers/slapd/back-ldbm/dbversion.c index 78780df8..ab2bd3ec 100644 --- a/ldap/servers/slapd/back-ldbm/dbversion.c +++ b/ldap/servers/slapd/back-ldbm/dbversion.c @@ -105,13 +105,15 @@ dbversion_write(struct ldbminfo *li, const char *directory, * (406922) */ if (idl_get_idl_new()) { - sprintf(buf, "%s/%d.%d/%s\n", - BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, BDB_BACKEND); + sprintf(buf, "%s/%d.%d/%s/%s\n", + BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, + BDB_BACKEND, BDB_DNFORMAT); } else { - sprintf(buf, "%s/%d.%d/%s\n", - BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, BDB_BACKEND); + sprintf(buf, "%s/%d.%d/%s/%s\n", + BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, + BDB_BACKEND, BDB_DNFORMAT); } len = strlen( buf ); if ( slapi_write_buffer( prfd, buf, len ) != len ) diff --git a/ldap/servers/slapd/back-ldbm/import-threads.c b/ldap/servers/slapd/back-ldbm/import-threads.c index d4b7af85..09a156f8 100644 --- a/ldap/servers/slapd/back-ldbm/import-threads.c +++ b/ldap/servers/slapd/back-ldbm/import-threads.c @@ -408,7 +408,6 @@ void import_producer(void *param) * as we read it. */ while (! finished) { - Slapi_Attr *attr = NULL; int flags = 0; int prev_lineno = 0; int lines_in_entry = 0; @@ -610,8 +609,6 @@ void import_producer(void *param) continue; } - /* generate uniqueid if necessary */ - import_generate_uniqueid(job, e); if (g_get_global_lastmod()) { import_add_created_attrs(e); } @@ -628,20 +625,11 @@ void import_producer(void *param) continue; } - /* not sure what this does, but it looked like it could be - * simplified. if it's broken, it's my fault. -robey - */ - if (slapi_entry_attr_find(ep->ep_entry, "userpassword", &attr) == 0) { - Slapi_Value **va = attr_get_present_values(attr); - - pw_encodevals( (Slapi_Value **)va ); /* jcm - cast away const */ + if (job->flags & FLAG_ABORT) { + backentry_free(&ep); + goto error; } - if (job->flags & FLAG_ABORT) { - backentry_free(&ep); - goto error; - } - /* Now we have this new entry, all decoded * Next thing we need to do is: @@ -972,6 +960,491 @@ error: info->state = ABORTED; } +struct upgradedn_attr { + char *ud_type; + char *ud_value; + struct upgradedn_attr *ud_next; + int ud_flags; +#define OLD_DN_NORMALIZE 0x1 +}; + +static void +upgradedn_free_list(struct upgradedn_attr **ud_list) +{ + struct upgradedn_attr *ptr = *ud_list; + + while (ptr) { + struct upgradedn_attr *next = ptr->ud_next; + slapi_ch_free_string(&ptr->ud_type); + slapi_ch_free_string(&ptr->ud_value); + slapi_ch_free((void **)&ptr); + ptr = next; + } + *ud_list = NULL; + return; +} + +static void +upgradedn_add_to_list(struct upgradedn_attr **ud_list, + char *type, char *value, int flag) +{ + struct upgradedn_attr *elem = + (struct upgradedn_attr *) slapi_ch_malloc(sizeof(struct upgradedn_attr)); + elem->ud_type = type; + elem->ud_value = value; + elem->ud_flags = flag; + elem->ud_next = *ud_list; + *ud_list = elem; + return; +} + +/* + * Producer thread for upgrading dn format + * FLAG_UPGRADEDNFORMAT | FLAG_DRYRUN -- check the necessity of dn upgrade + * FLAG_UPGRADEDNFORMAT -- execute dn upgrade + * + * Read id2entry, + * Check the DN syntax attributes if it contains '\' or not AND + * Check the RDNs of the attributes if the value is surrounded by + * double-quotes or not. + * If both are false, skip the entry and go to next + * If either is true, create an entry which contains a correctly normalized + * DN attribute values in e_attr list and the original entrydn in the + * deleted attribute list e_deleted_attrs. + * + * If FLAG_UPGRADEDNFORMAT is set, worker_threads for indexing DN syntax + * attributes are brought up. Foreman thread updates entrydn index + * as well as the entry itself in the id2entry.db#. + * + * Note: QUIT state for info->state is introduced for DRYRUN mode to + * distinguish the intentional QUIT (found the dn upgrade candidate) + * from ABORTED (aborted or error) and FINISHED (scan all the entries + * and found no candidate to upgrade) + */ +void +upgradedn_producer(void *param) +{ + ImportWorkerInfo *info = (ImportWorkerInfo *)param; + ImportJob *job = info->job; + ID id = job->first_ID; + Slapi_Entry *e = NULL; + struct backentry *ep = NULL, *old_ep = NULL; + ldbm_instance *inst = job->inst; + PRIntervalTime sleeptime; + int finished = 0; + int idx; + int rc = 0; + Slapi_Attr *a = NULL; + Slapi_DN *sdn = NULL; + char *workdn = NULL; + int doit = 0; + int skipit = 0; + int isentrydn = 0; + Slapi_Value *value = NULL; + struct upgradedn_attr *ud_list = NULL; + char **ud_vals = NULL; + char **ud_valp = NULL; + struct upgradedn_attr *ud_ptr = NULL; + Slapi_Attr *ud_attr = NULL; + char *ecopy = NULL; + + /* vars for Berkeley DB */ + DB_ENV *env = NULL; + DB *db = NULL; + DBC *dbc = NULL; + DBT key = {0}; + DBT data = {0}; + int db_rval = -1; + backend *be = inst->inst_be; + int isfirst = 1; + int curr_entry = 0; + size_t newesize = 0; + + PR_ASSERT(info != NULL); + PR_ASSERT(inst != NULL); + PR_ASSERT(be != NULL); + + if ( job->flags & FLAG_ABORT ) + goto error; + + sleeptime = PR_MillisecondsToInterval(import_sleep_time); + + /* pause until we're told to run */ + while ((info->command == PAUSE) && !(job->flags & FLAG_ABORT)) { + info->state = WAITING; + DS_Sleep(sleeptime); + } + info->state = RUNNING; + + /* open id2entry with dedicated db env and db handler */ + if ( dblayer_get_aux_id2entry( be, &db, &env ) != 0 || db == NULL || + env == NULL) { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open id2entry\n", 0, 0, 0 ); + goto error; + } + + /* get a cursor to we can walk over the table */ + db_rval = db->cursor(db, NULL, &dbc, 0); + if ( 0 != db_rval ) { + LDAPDebug( LDAP_DEBUG_ANY, + "Failed to get cursor for reindexing\n", 0, 0, 0 ); + dblayer_release_id2entry(be, db); + goto error; + } + + /* we loop around reading the input files and processing each entry + * as we read it. + */ + finished = 0; + while (!finished) { + ID temp_id; + + if (job->flags & FLAG_ABORT) { + goto error; + } + while ((info->command == PAUSE) && !(job->flags & FLAG_ABORT)){ + info->state = WAITING; + DS_Sleep(sleeptime); + } + info->state = RUNNING; + + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + if (isfirst) + { + db_rval = dbc->c_get(dbc, &key, &data, DB_FIRST); + isfirst = 0; + } + else + { + db_rval = dbc->c_get(dbc, &key, &data, DB_NEXT); + } + + if (0 != db_rval) { + if (DB_NOTFOUND != db_rval) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed to read database, " + "errno=%d (%s)\n", inst->inst_name, db_rval, + dblayer_strerror(db_rval)); + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Failed to read database, err %d (%s)", + inst->inst_name, db_rval, + dblayer_strerror(db_rval)); + } + } + finished = 1; + break; /* error or done */ + } + curr_entry++; + temp_id = id_stored_to_internal((char *)key.data); + slapi_ch_free(&(key.data)); + + /* call post-entry plugin */ + plugin_call_entryfetch_plugins((char **)&data.dptr, &data.dsize); + ecopy = (char *)slapi_ch_malloc(data.dsize + 1); + memcpy(ecopy, data.dptr, data.dsize); + *(ecopy + data.dsize) = '\0'; + e = slapi_str2entry(data.data, 0); + if ( NULL == e ) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: WARNING: skipping badly formatted entry (id %lu)", + inst->inst_name, (u_long)temp_id); + } + LDAPDebug(LDAP_DEBUG_ANY, + "%s: WARNING: skipping badly formatted entry (id %lu)\n", + inst->inst_name, (u_long)temp_id, 0); + continue; + } + slapi_ch_free(&(data.data)); + + /* Check DN syntax attr values if it contains '\\' or not */ + for (a = e->e_attrs; a; a = a->a_next) { + if (slapi_attr_is_dn_syntax_attr(a)) { /* is dn syntax attr? */ + rc = get_values_from_string((const char *)ecopy, + a->a_type, &ud_vals); + if (rc || (NULL == ud_vals)) { + continue; /* empty; ignore it */ + } + + for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) { + char **rdns = NULL; + char **rdnsp = NULL; + char *valueptr = NULL; + int valuelen; + + /* ud_valp contains '\\'. We have to update the value */ + if (PL_strchr(*ud_valp, '\\')) { + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(a->a_type), + slapi_ch_strdup(*ud_valp), + 0); + LDAPDebug(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: %s (id %lu)\n", + inst->inst_name, *ud_valp, (u_long)temp_id); + doit = 1; + continue; + } + /* Also check RDN contains double quoted values */ + if (strcasecmp(a->a_type, "entrydn")) { + /* except entrydn */ + workdn = slapi_ch_strdup(*ud_valp); + isentrydn = 0; + } else { + /* entrydn: Get Slapi DN */ + sdn = slapi_entry_get_sdn(e); + workdn = slapi_ch_strdup(slapi_sdn_get_dn(sdn)); + isentrydn = 1; + } + rdns = ldap_explode_dn(workdn, 0); + skipit = 0; + for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) { + valueptr = PL_strchr(*rdnsp, '='); + if (NULL == valueptr) { + skipit = 1; + break; + } + valueptr++; + while ((' ' == *valueptr) || ('\t' == *valueptr)) { + valueptr++; + } + valuelen = strlen(valueptr); + if (0 == valuelen) { + skipit = 1; + break; + } + /* DN contains an RDN <type>="<value>" ? */ + if (('"' == *valueptr) && + ('"' == *(valueptr + valuelen - 1))) { + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(a->a_type), + slapi_ch_strdup(*ud_valp), + isentrydn?0:OLD_DN_NORMALIZE); + LDAPDebug(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: %s (id %lu)\n", + inst->inst_name, valueptr, (u_long)temp_id); + doit = 1; + break; + } + } + if (rdns) { + slapi_ldap_value_free(rdns); + } else { + skipit = 1; + } + if (skipit) { + break; + } + slapi_ch_free_string(&workdn); + } /* for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) */ + charray_free(ud_vals); + ud_vals = NULL; + if (skipit) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: WARNING: skipping an entry " + "with a corrupted dn (syntax value): %s " + "(id %lu)\n", + inst->inst_name, + workdn?workdn:"unknown", (u_long)temp_id); + slapi_ch_free_string(&workdn); + upgradedn_free_list(&ud_list); + break; + } + } /* if (slapi_attr_is_dn_syntax_attr(a)) */ + } /* for (a = e->e_attrs; a; a = a->a_next) */ + slapi_ch_free_string(&ecopy); + if (skipit) { + upgradedn_free_list(&ud_list); + slapi_entry_free(e); e = NULL; + continue; + } + + if (!doit) { + /* We don't have to update dn syntax values. */ + upgradedn_free_list(&ud_list); + slapi_entry_free(e); e = NULL; + continue; + } + + /* doit */ + if (job->flags & FLAG_DRYRUN) { + /* We can return SUCCESS (== found upgrade dn candidates) */ + finished = 0; /* make it sure ... */ + upgradedn_free_list(&ud_list); + slapi_entry_free(e); e = NULL; + goto bail; + } + + skipit = 0; + for (ud_ptr = ud_list; ud_ptr; ud_ptr = ud_ptr->ud_next) { + /* Move the current value to e_deleted_attrs. */ + ud_attr = attrlist_find(e->e_attrs, ud_ptr->ud_type); + if (ud_attr) { + if (0 == strcmp(ud_ptr->ud_type, "entrydn")) { + /* entrydn contains half normalized value in id2entry, + thus we have to replace it in id2entry. + The other DN syntax attribute values store + the originals. They are taken care by the normalizer. + */ + attrlist_remove(&e->e_attrs, ud_ptr->ud_type); + rc = slapi_attr_first_value(ud_attr, &value); + if (rc < 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: WARNING: skipping an entry with no entrydn: " + "%s (id %lu)\n", + inst->inst_name, ud_ptr->ud_value, (u_long)temp_id); + skipit = 1; + break; + } else { + rc = slapi_value_set_string(value, ud_ptr->ud_value); + if (rc) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: WARNING: skipping an entry; " + "failed to replace entrydn: %s " + "(id %lu)\n", inst->inst_name, + ud_ptr->ud_value, (u_long)temp_id); + skipit = 1; + break; + } + } + attrlist_add(&e->e_deleted_attrs, ud_attr); + } else { + /* We have to normalize the orignal string to generate + the key in the index. + */ + a = attrlist_find(e->e_deleted_attrs, ud_ptr->ud_type); + if (!a) { + a = slapi_attr_new(); + slapi_attr_init(a, ud_ptr->ud_type); + } else { + a = attrlist_remove(&e->e_deleted_attrs, + ud_ptr->ud_type); + } + slapi_dn_normalize_case_original(ud_ptr->ud_value); + value = slapi_value_new_string(ud_ptr->ud_value); + slapi_attr_add_value(a, value); + slapi_value_free(&value); + attrlist_add(&e->e_deleted_attrs, a); + } + } + } + upgradedn_free_list(&ud_list); + if (skipit) { + slapi_entry_free(e); e = NULL; + continue; + } + + ep = import_make_backentry(e, temp_id); + if (!ep) { + slapi_entry_free(e); e = NULL; + goto error; + } + + /* Add the newly case-normalized dn to entrydn in the e_attrs list. */ + add_update_entrydn_operational_attributes(ep); + + if (job->flags & FLAG_ABORT) + goto error; + + /* Now we have this new entry, all decoded + * Next thing we need to do is: + * (1) see if the appropriate fifo location contains an + * entry which had been processed by the indexers. + * If so, proceed. + * If not, spin waiting for it to become free. + * (2) free the old entry and store the new one there. + * (3) Update the job progress indicators so the indexers + * can use the new entry. + */ + idx = id % job->fifo.size; + old_ep = job->fifo.item[idx].entry; + if (old_ep) { + /* for the slot to be recycled, it needs to be already absorbed + * by the foreman (id >= ready_EID), and all the workers need to + * be finished with it (refcount = 0). + */ + while (((old_ep->ep_refcnt > 0) || + (old_ep->ep_id >= job->ready_EID)) + && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + info->state = WAITING; + DS_Sleep(sleeptime); + } + if (job->flags & FLAG_ABORT) + goto error; + + info->state = RUNNING; + PR_ASSERT(old_ep == job->fifo.item[idx].entry); + job->fifo.item[idx].entry = NULL; + if (job->fifo.c_bsize > job->fifo.item[idx].esize) + job->fifo.c_bsize -= job->fifo.item[idx].esize; + else + job->fifo.c_bsize = 0; + backentry_free(&old_ep); + } + + newesize = (slapi_entry_size(ep->ep_entry) + sizeof(struct backentry)); + if (newesize > job->fifo.bsize) { /* entry too big */ + char ebuf[BUFSIZ]; + import_log_notice(job, "WARNING: skipping entry \"%s\"", + escape_string(slapi_entry_get_dn(e), ebuf)); + import_log_notice(job, "REASON: entry too large (%lu bytes) for " + "the buffer size (%lu bytes)", newesize, job->fifo.bsize); + backentry_free(&ep); + job->skipped++; + continue; + } + /* Now check if fifo has enough space for the new entry */ + if ((job->fifo.c_bsize + newesize) > job->fifo.bsize) { + import_wait_for_space_in_fifo( job, newesize ); + } + + /* We have enough space */ + job->fifo.item[idx].filename = ID2ENTRY LDBM_FILENAME_SUFFIX; + job->fifo.item[idx].line = curr_entry; + job->fifo.item[idx].entry = ep; + job->fifo.item[idx].bad = 0; + job->fifo.item[idx].esize = newesize; + + /* Add the entry size to total fifo size */ + job->fifo.c_bsize += ep->ep_entry? job->fifo.item[idx].esize : 0; + + /* Update the job to show our progress */ + job->lead_ID = id; + if ((id - info->first_ID) <= job->fifo.size) { + job->trailing_ID = info->first_ID; + } else { + job->trailing_ID = id - job->fifo.size; + } + + /* Update our progress meter too */ + info->last_ID_processed = id; + id++; + if (job->flags & FLAG_ABORT) + goto error; + if (info->command == STOP) + { + finished = 1; + } + } +bail: + dbc->c_close(dbc); + dblayer_release_aux_id2entry( be, db, env ); + if (job->flags & FLAG_DRYRUN) { + if (finished) { /* Set if dn upgrade candidates are not found */ + info->state = FINISHED; + } else { /* At least one dn upgrade candidate is found */ + info->state = QUIT; + } + } else { + info->state = FINISHED; + } + return; + +error: + dbc->c_close(dbc); + dblayer_release_aux_id2entry( be, db, env ); + info->state = ABORTED; +} + static void import_wait_for_space_in_fifo(ImportJob *job, size_t new_esize) { @@ -1046,6 +1519,39 @@ static int foreman_do_entrydn(ImportJob *job, FifoItem *fi) int err = 0, ret = 0; IDList *IDL; + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* Get the entrydn attribute value from deleted attr list */ + Slapi_Value *value = NULL; + Slapi_Attr *entrydn_to_del = + attrlist_remove(&fi->entry->ep_entry->e_deleted_attrs, "entrydn"); + + if (entrydn_to_del) { + /* Delete it. */ + ret = slapi_attr_first_value(entrydn_to_del, &value); + if (ret < 0) { + import_log_notice(job, + "Error: retrieving entrydn value (error %d)", + ret); + } else { + const struct berval *bval = + slapi_value_get_berval((const Slapi_Value *)value); + ret = index_addordel_string(be, "entrydn", + bval->bv_val, + fi->entry->ep_id, + BE_INDEX_DEL|BE_INDEX_EQUALITY|BE_INDEX_NORMALIZED, + NULL); + if (ret) { + import_log_notice(job, + "Error: deleting %s from entrydn index " + "(error %d: %s)", + bval->bv_val, ret, dblayer_strerror(ret)); + return ret; + } + } + slapi_attr_free(&entrydn_to_del); + } + } + /* insert into the entrydn index */ bv.bv_val = (void*)backentry_get_ndn(fi->entry); /* jcm - Had to cast away const */ bv.bv_len = strlen(bv.bv_val); @@ -1058,29 +1564,49 @@ static int foreman_do_entrydn(ImportJob *job, FifoItem *fi) /* So, we do an index read first */ err = 0; IDL = index_read(be, "entrydn", indextype_EQUALITY, &bv, NULL, &err); - - /* Did this work ? */ - if (NULL != IDL) { - /* IMPOSTER ! Get thee hence... */ - import_log_notice(job, "WARNING: Skipping duplicate entry " - "\"%s\" found at line %d of file \"%s\"", - slapi_entry_get_dn(fi->entry->ep_entry), - fi->line, fi->filename); - idl_free(IDL); - /* skip this one */ - fi->bad = 1; - job->skipped++; - return -1; /* skip to next entry */ - } - if ((ret = index_addordel_string(be, "entrydn", - bv.bv_val, - fi->entry->ep_id, - BE_INDEX_ADD|BE_INDEX_NORMALIZED, NULL)) != 0) { - import_log_notice(job, "Error writing entrydn index " - "(error %d: %s)", - ret, dblayer_strerror(ret)); - return ret; + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* + * In the UPGRADEDNFORMAT case, if entrydn value exists, + * that means entrydn is not upgraded. And it is normal. + * We could add entrydn only when the value is not found in the db. + */ + if (IDL) { + idl_free(IDL); + } else { + ret = index_addordel_string(be, "entrydn", + bv.bv_val, fi->entry->ep_id, + BE_INDEX_ADD|BE_INDEX_NORMALIZED, NULL); + if (ret) { + import_log_notice(job, "Error writing entrydn index " + "(error %d: %s)", + ret, dblayer_strerror(ret)); + return ret; + } + } + } else { + /* Did this work ? */ + if (IDL) { + /* IMPOSTER ! Get thee hence... */ + import_log_notice(job, "WARNING: Skipping duplicate entry " + "\"%s\" found at line %d of file \"%s\"", + slapi_entry_get_dn(fi->entry->ep_entry), + fi->line, fi->filename); + idl_free(IDL); + /* skip this one */ + fi->bad = 1; + job->skipped++; + return -1; /* skip to next entry */ + } + ret = index_addordel_string(be, "entrydn", bv.bv_val, fi->entry->ep_id, + BE_INDEX_ADD|BE_INDEX_NORMALIZED, NULL); + if (ret) { + import_log_notice(job, "Error writing entrydn index " + "(error %d: %s)", + ret, dblayer_strerror(ret)); + return ret; + } } + return 0; } @@ -1120,13 +1646,13 @@ void import_foreman(void *param) FifoItem *fi = NULL; int parent_status = 0; - if (job->flags & FLAG_ABORT) { goto error; } while ( ((info->command == PAUSE) || (id > job->lead_ID)) && - (info->command != STOP) && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + (info->command != STOP) && (info->command != ABORT) && + !(job->flags & FLAG_ABORT) ) { /* Check to see if we've been told to stop */ info->state = WAITING; DS_Sleep(sleeptime); @@ -1167,9 +1693,9 @@ void import_foreman(void *param) * Only check for a parent and add to the entry2dn index if * the entry is not a tombstone. */ - if (job->flags & FLAG_ABORT) { - goto error; - } + if (job->flags & FLAG_ABORT) { + goto error; + } if (parent_status == IMPORT_ADD_OP_ATTRS_NO_PARENT) { /* If this entry is a suffix entry, this is not a problem */ @@ -1190,10 +1716,9 @@ void import_foreman(void *param) goto cont; /* below */ } } - if (job->flags & FLAG_ABORT) { - goto error; - } - + if (job->flags & FLAG_ABORT) { + goto error; + } /* insert into the entrydn index */ ret = foreman_do_entrydn(job, fi); @@ -1213,6 +1738,9 @@ void import_foreman(void *param) * (that isn't really an index -- it's the storehouse of the entries * themselves.) */ + /* id2entry_add_ext replaces an entry if it already exists. + * therefore, the Entry ID stays the same. + */ if ((ret = id2entry_add_ext(be, fi->entry, NULL, job->encrypt)) != 0) { /* DB_RUNRECOVERY usually occurs if disk fills */ if (LDBM_OS_ERR_IS_DISKFULL(ret)) { @@ -1238,7 +1766,10 @@ void import_foreman(void *param) goto error; } - if (! slapi_entry_flag_is_set(fi->entry->ep_entry, + if (!(job->flags & FLAG_UPGRADEDNFORMAT) && /* Upgrade dn format mode + does not need to update + parentid index */ + !slapi_entry_flag_is_set(fi->entry->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE)) { /* parentid index * (we have to do this here, because the parentID is dependent on @@ -1253,9 +1784,9 @@ void import_foreman(void *param) vlv code to see whether it's within the scope a VLV index. */ vlv_grok_new_import_entry(fi->entry, be); } - if (job->flags & FLAG_ABORT) { - goto error; - } + if (job->flags & FLAG_ABORT) { + goto error; + } /* Remove the entry from the cache (Put in the cache in id2entry_add) */ @@ -1372,7 +1903,8 @@ void import_worker(void *param) * thread, and the state is neither STOP nor ABORT */ while (((info->command == PAUSE) || (id > job->ready_ID)) && - (info->command != STOP) && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + (info->command != STOP) && (info->command != ABORT) && + !(job->flags & FLAG_ABORT)) { /* Check to see if we've been told to stop */ info->state = WAITING; DS_Sleep(sleeptime); @@ -1419,6 +1951,43 @@ void import_worker(void *param) slapi_pblock_destroy(pb); } else { /* No, process regular index */ + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* Get the attribute value from deleted attr list */ + Slapi_Value *value = NULL; + const struct berval *bval = NULL; + Slapi_Attr *key_to_del = + attrlist_remove(&fi->entry->ep_entry->e_deleted_attrs, + info->index_info->name); + + if (key_to_del) { + int idx = 0; + /* Delete it. */ + for (idx = slapi_attr_first_value(key_to_del, &value); + idx >= 0; + idx = slapi_attr_next_value(key_to_del, idx, + &value)) { + bval = + slapi_value_get_berval((const Slapi_Value *)value); + ret = index_addordel_string(be, + info->index_info->name, + bval->bv_val, + fi->entry->ep_id, + BE_INDEX_DEL|BE_INDEX_EQUALITY| + BE_INDEX_NORMALIZED, + NULL); + if (ret) { + import_log_notice(job, + "Error deleting %s from %s index " + "(error %d: %s)", + bval->bv_val, info->index_info->name, + ret, dblayer_strerror(ret)); + goto error; + } + } + slapi_attr_free(&key_to_del); + } + } + /* Look for the attribute we're indexing and its subtypes */ /* For each attr write to the index */ attrlist_cursor = NULL; diff --git a/ldap/servers/slapd/back-ldbm/import.c b/ldap/servers/slapd/back-ldbm/import.c index df0fa5d5..8b66705f 100644 --- a/ldap/servers/slapd/back-ldbm/import.c +++ b/ldap/servers/slapd/back-ldbm/import.c @@ -51,6 +51,7 @@ #include "import.h" #define ERR_IMPORT_ABORTED -23 +#define DRYRUN_QUIT -24 /********** routines to manipulate the entry fifo **********/ @@ -193,8 +194,16 @@ void import_log_notice(ImportJob *job, char *format, ...) slapi_task_log_notice(job->task, "%s", buffer); } /* also save it in the logs for posterity */ - LDAPDebug(LDAP_DEBUG_ANY, "import %s: %s\n", job->inst->inst_name, + if (job->flags & FLAG_UPGRADEDNFORMAT) { + LDAPDebug(LDAP_DEBUG_ANY, "upgradedn %s: %s\n", job->inst->inst_name, buffer, 0); + } else if (job->flags & FLAG_REINDEXING) { + LDAPDebug(LDAP_DEBUG_ANY, "reindex %s: %s\n", job->inst->inst_name, + buffer, 0); + } else { + LDAPDebug(LDAP_DEBUG_ANY, "import %s: %s\n", job->inst->inst_name, + buffer, 0); + } } static void import_task_destroy(Slapi_Task *task) @@ -245,6 +254,22 @@ static int import_attr_callback(void *node, void *param) ImportJob *job = (ImportJob *)param; struct attrinfo *a = (struct attrinfo *)node; + if (job->flags & FLAG_DRYRUN) { /* dryrun; we don't need the workers */ + return 0; + } + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* Bring up import workers just for indexes having DN syntax + * attribute type. (except entrydn -- taken care below) */ + int rc = 0; + Slapi_Attr attr = {0}; + slapi_attr_init(&attr, a->ai_type); + rc = slapi_attr_is_dn_syntax_attr(&attr); + attr_done(&attr); + if (0 == rc) { + return 0; + } + } + /* OK, so we now have hold of the attribute structure and the job info, * let's see what we have. Remember that although this function is called * many times, all these calls are in the context of a single thread, so we @@ -255,28 +280,28 @@ static int import_attr_callback(void *node, void *param) * we build those in the foreman thread. */ if (IS_INDEXED(a->ai_indexmask) && - (strcasecmp(a->ai_type, "entrydn") != 0) && - (strcasecmp(a->ai_type, "parentid") != 0) && - (strcasecmp(a->ai_type, "ancestorid") != 0) && - (strcasecmp(a->ai_type, numsubordinates) != 0)) { - /* Make an import_index_info structure, fill it in and insert into the - * job's list */ - IndexInfo *info = CALLOC(IndexInfo); + (strcasecmp(a->ai_type, "entrydn") != 0) && + (strcasecmp(a->ai_type, "parentid") != 0) && + (strcasecmp(a->ai_type, "ancestorid") != 0) && + (strcasecmp(a->ai_type, numsubordinates) != 0)) { + /* Make an import_index_info structure, fill it in and insert into the + * job's list */ + IndexInfo *info = CALLOC(IndexInfo); - if (NULL == info) { - /* Memory allocation error */ - return -1; - } - info->name = slapi_ch_strdup(a->ai_type); - info->ai = a; - if (NULL == info->name) { - /* Memory allocation error */ - FREE(info); - return -1; - } - info->next = job->index_list; - job->index_list = info; - job->number_indexers++; + if (NULL == info) { + /* Memory allocation error */ + return -1; + } + info->name = slapi_ch_strdup(a->ai_type); + info->ai = a; + if (NULL == info->name) { + /* Memory allocation error */ + FREE(info); + return -1; + } + info->next = job->index_list; + job->index_list = info; + job->number_indexers++; } return 0; } @@ -401,12 +426,12 @@ static int import_start_threads(ImportJob *job) import_init_worker_info(foreman, job); foreman->work_type = FOREMAN; if (! CREATE_THREAD(PR_USER_THREAD, (VFP)import_foreman, foreman, - PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, - PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE)) { + PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, + PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE)) { PRErrorCode prerr = PR_GetError(); LDAPDebug(LDAP_DEBUG_ANY, "unable to spawn import foreman thread, " - SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", - prerr, slapd_pr_strerror(prerr), 0); + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); FREE(foreman); goto error; } @@ -658,7 +683,7 @@ static int import_monitor_threads(ImportJob *job, int *status) time_t time_now = 0; time_t last_time = 0; time_t time_interval = 0; - + int rc = 0; for (current_worker = job->worker_list; current_worker != NULL; current_worker = current_worker->next) { @@ -721,7 +746,10 @@ static int import_monitor_threads(ImportJob *job, int *status) import_calc_rate(current_worker, time_interval); import_print_worker_status(current_worker); } - if (current_worker->state != FINISHED) { + if (current_worker->state == QUIT) { + rc = DRYRUN_QUIT; /* Set the RC; Don't abort now; + We have to stop other threads */ + } else if (current_worker->state != FINISHED) { finished = 0; } if (current_worker->state == ABORTED) { @@ -764,8 +792,10 @@ static int import_monitor_threads(ImportJob *job, int *status) /* if the producer is finished, and the foreman has caught up... */ if (producer) { - producer_done = (producer->state == FINISHED); + producer_done = (producer->state == FINISHED) || + (producer->state == QUIT); } else { + /* set in ldbm_back_wire_import */ producer_done = (job->flags & FLAG_PRODUCER_DONE); } if (producer_done && (job->lead_ID == job->ready_ID)) { @@ -801,6 +831,7 @@ static int import_monitor_threads(ImportJob *job, int *status) for (current_worker = job->worker_list; current_worker != NULL; ) { if ((current_worker->state != FINISHED) && (current_worker->state != ABORTED) && + (current_worker->state != QUIT) && (current_worker->work_type != PRODUCER)) { DS_Sleep(tenthsecond); /* Only sleep if we hit a thread that is still not done */ continue; @@ -813,7 +844,7 @@ static int import_monitor_threads(ImportJob *job, int *status) /* If we're here and giveup is true, and the primary hadn't finished * processing the input files, we need to return IMPORT_INCOMPLETE_PASS */ if (giveup && (job->input_filenames || (job->flags & FLAG_ONLINE) || - (job->flags & FLAG_REINDEXING /* support multi-pass */))) { + (job->flags & FLAG_REINDEXING /* support multi-paths */))) { if (producer_done && (job->ready_ID == job->lead_ID)) { /* foreman caught up with the producer, and the producer is * done. @@ -825,7 +856,7 @@ static int import_monitor_threads(ImportJob *job, int *status) } else { *status = IMPORT_COMPLETE_PASS; } - return 0; + return rc; error_abort: return ERR_IMPORT_ABORTED; @@ -842,16 +873,16 @@ static int import_run_pass(ImportJob *job, int *status) ret = import_start_threads(job); if (ret != 0) { import_log_notice(job, "Starting threads failed: %d\n", ret); - goto error; + goto error; } /* Monitor the threads until we're done or fail */ ret = import_monitor_threads(job, status); - if (ret == ERR_IMPORT_ABORTED) { + if ((ret == ERR_IMPORT_ABORTED) || (ret == DRYRUN_QUIT)) { goto error; } else if (ret != 0) { import_log_notice(job, "Thread monitoring aborted: %d\n", ret); - goto error; + goto error; } error: @@ -875,20 +906,18 @@ static void import_set_abort_flag_all(ImportJob *job, int wait_for_them) /* allow all the aborts to be processed */ DS_Sleep(PR_MillisecondsToInterval(3000)); - if (wait_for_them) { + if (wait_for_them) { /* Having done that, wait for them to say that they've stopped */ for (worker = job->worker_list; worker != NULL; ) { DS_Sleep(PR_MillisecondsToInterval(100)); - if ((worker->state != FINISHED) && - (worker->state != ABORTED)){ + if ((worker->state != FINISHED) && (worker->state != ABORTED) && + (worker->state != QUIT)) { continue; - } - else{ + } else { worker = worker->next; - } + } } } - } @@ -907,11 +936,12 @@ void import_abort_all(ImportJob *job, int wait_for_them) /* Having done that, wait for them to say that they've stopped */ for (worker = job->worker_list; worker != NULL; ) { DS_Sleep(PR_MillisecondsToInterval(100)); - if ((worker->state != FINISHED) && - (worker->state != ABORTED)) + if ((worker->state != FINISHED) && (worker->state != ABORTED) && + (worker->state != QUIT)) { continue; - else + } else { worker = worker->next; + } } } } @@ -927,11 +957,10 @@ int import_make_merge_filenames(char *directory, char *indexname, int pass, *oldname = slapi_ch_smprintf("%s/%s%s", directory, indexname, LDBM_FILENAME_SUFFIX); *newname = slapi_ch_smprintf("%s/%s.%d%s", directory, indexname, pass, LDBM_FILENAME_SUFFIX); - if (!*oldname || !*newname) { - slapi_ch_free_string(oldname); - slapi_ch_free_string(newname); - return -1; - } + if (!*oldname || !*newname) { slapi_ch_free_string(oldname); + slapi_ch_free_string(newname); + return -1; + } return 0; } @@ -1018,8 +1047,8 @@ static int import_all_done(ImportJob *job, int ret) ldbm_instance *inst = job->inst; /* Writing this file indicates to future server startups that - * the db is OK */ - if (ret == 0) { + * the db is OK unless it's in the dry run mode. */ + if ((ret == 0) && !(job->flags & FLAG_DRYRUN)) { char inst_dir[MAXPATHLEN*2]; char *inst_dirp = NULL; inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, @@ -1077,10 +1106,20 @@ int import_main_offline(void *arg) int verbose = 1; int aborted = 0; ImportWorkerInfo *producer = NULL; + char *opstr = "Import"; if (job->task) slapi_task_inc_refcount(job->task); + if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & FLAG_DRYRUN) { + opstr = "Upgrade Dn Dryrun"; + } else { + opstr = "Upgrade Dn"; + } + } else if (job->flags & FLAG_REINDEXING) { + opstr = "Reindexing"; + } PR_ASSERT(inst != NULL); time(&beginning); @@ -1119,7 +1158,20 @@ int import_main_offline(void *arg) /* start the producer */ import_init_worker_info(producer, job); producer->work_type = PRODUCER; - if (job->flags & FLAG_REINDEXING) + if (job->flags & FLAG_UPGRADEDNFORMAT) + { + if (! CREATE_THREAD(PR_USER_THREAD, (VFP)upgradedn_producer, + producer, PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, + PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE)) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, + "unable to spawn upgrade dn producer thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + goto error; + } + } + else if (job->flags & FLAG_REINDEXING) { if (! CREATE_THREAD(PR_USER_THREAD, (VFP)index_producer, producer, PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, @@ -1188,15 +1240,18 @@ int import_main_offline(void *arg) if (ret == ERR_IMPORT_ABORTED) { /* at least one of the threads has aborted -- shut down ALL * of the threads */ - import_log_notice(job, "Aborting all import threads..."); + import_log_notice(job, "Aborting all %s threads...", opstr); /* this abort sets the abort flag on the threads and will block for * the exit of all threads */ import_set_abort_flag_all(job, 1); - import_log_notice(job, "Import threads aborted."); + import_log_notice(job, "%s threads aborted.", opstr); aborted = 1; goto error; } + if (ret == DRYRUN_QUIT) { + goto error; /* Found the candidate; close the db files and quit */ + } if (0 != ret) { /* Some horrible fate has befallen the import */ @@ -1293,7 +1348,8 @@ int import_main_offline(void *arg) * Database. */ error: - /* If we fail, the database is now in a mess, so we delete it */ + /* If we fail, the database is now in a mess, so we delete it + except dry run mode */ import_log_notice(job, "Closing files..."); cache_clear(&job->inst->inst_cache); if (aborted) { @@ -1307,7 +1363,10 @@ error: } } if (0 != ret) { - dblayer_delete_instance_dir(be); + if (!(job->flags & FLAG_DRYRUN)) { /* If not dryrun */ + /* if running in the dry run mode, don't touch the db */ + dblayer_delete_instance_dir(be); + } dblayer_instance_close(job->inst->inst_be); } else { if (0 != (ret = dblayer_instance_close(job->inst->inst_be)) ) { @@ -1321,52 +1380,82 @@ error: if (verbose && (0 == ret)) { int seconds_to_import = end - beginning; size_t entries_processed = job->lead_ID - (job->starting_ID - 1); - double entries_per_second = (double) entries_processed / - (double) seconds_to_import; - - if (job->not_here_skipped) - { - if (job->skipped) - import_log_notice(job, "Import complete. Processed %lu entries " - "(%d bad entries were skipped, " - "%d entries were skipped because they don't " - "belong to this database) in %d seconds. " - "(%.2f entries/sec)", entries_processed, - job->skipped, job->not_here_skipped, - seconds_to_import, entries_per_second); - else - import_log_notice(job, "Import complete. Processed %lu entries " - "(%d entries were skipped because they don't " - "belong to this database) " - "in %d seconds. (%.2f entries/sec)", - entries_processed, job->not_here_skipped, - seconds_to_import, entries_per_second); - } - else - { - if (job->skipped) - import_log_notice(job, "Import complete. Processed %lu entries " - "(%d were skipped) in %d seconds. " - "(%.2f entries/sec)", entries_processed, - job->skipped, seconds_to_import, - entries_per_second); - else - import_log_notice(job, "Import complete. Processed %lu entries " - "in %d seconds. (%.2f entries/sec)", - entries_processed, seconds_to_import, - entries_per_second); + double entries_per_second = + seconds_to_import ? + (double)entries_processed / (double)seconds_to_import : 0; + + if (job->not_here_skipped) { + if (job->skipped) { + import_log_notice(job, + "%s complete. Processed %lu entries " + "(%d bad entries were skipped, " + "%d entries were skipped because they don't " + "belong to this database) in %d seconds. " + "(%.2f entries/sec)", + opstr, entries_processed, + job->skipped, job->not_here_skipped, + seconds_to_import, entries_per_second); + } else { + import_log_notice(job, + "%s complete. Processed %lu entries " + "(%d entries were skipped because they don't " + "belong to this database) " + "in %d seconds. (%.2f entries/sec)", + opstr, entries_processed, + job->not_here_skipped, seconds_to_import, + entries_per_second); + } + } else { + if (job->skipped) { + import_log_notice(job, + "%s complete. Processed %lu entries " + "(%d were skipped) in %d seconds. " + "(%.2f entries/sec)", + opstr, entries_processed, + job->skipped, seconds_to_import, + entries_per_second); + } else { + import_log_notice(job, + "%s complete. Processed %lu entries " + "in %d seconds. (%.2f entries/sec)", + opstr, entries_processed, + seconds_to_import, entries_per_second); + } } } - if (0 != ret) { - import_log_notice(job, "Import failed."); + if (job->flags & FLAG_DRYRUN) { + if (0 == ret) { + import_log_notice(job, "%s complete. %s is up-to-date.", + opstr, job->inst->inst_name); + ret = 1; + if (job->task) { + slapi_task_dec_refcount(job->task); + } + import_all_done(job, ret); + } else if (DRYRUN_QUIT == ret) { + import_log_notice(job, "%s complete. %s needs upgradednformat.", + opstr, job->inst->inst_name); + if (job->task) { + slapi_task_dec_refcount(job->task); + } + import_all_done(job, ret); + ret = 0; + } else { + ret = -1; + if (job->task != NULL) { + slapi_task_finish(job->task, ret); + } + } + } else if (0 != ret) { + import_log_notice(job, "%s failed.", opstr); if (job->task != NULL) { slapi_task_finish(job->task, ret); } } else { - if (job->task) + if (job->task) { slapi_task_dec_refcount(job->task); - + } import_all_done(job, ret); } @@ -1376,7 +1465,6 @@ error: import_free_job(job); if (producer) FREE(producer); - return(ret); } @@ -1399,6 +1487,7 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) char **name_array = NULL; int total_files, i; PRThread *thread = NULL; + int ud_flags = 0; job = CALLOC(ImportJob); if (job == NULL) { @@ -1412,6 +1501,7 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) job->inst = (ldbm_instance *)be->be_instance_info; slapi_pblock_get( pb, SLAPI_LDIF2DB_NOATTRINDEXES, &noattrindexes ); slapi_pblock_get( pb, SLAPI_LDIF2DB_FILE, &name_array ); + slapi_pblock_get( pb, SLAPI_SEQ_TYPE, &ud_flags ); /* For upgrade dn */ /* the removedupvals field is blatantly overloaded here to mean * the chunk size too. (chunk size = number of entries that should @@ -1437,12 +1527,22 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) } job->flags = FLAG_USE_FILES; - if (NULL == name_array) /* no ldif file is given -> reindexing */ - job->flags |= FLAG_REINDEXING; - if (!noattrindexes) - job->flags |= FLAG_INDEX_ATTRS; - for (i = 0; name_array && name_array[i] != NULL; i++) + if (NULL == name_array) { /* no ldif file is given -> reindexing */ + if (ud_flags & SLAPI_UPGRADEDNFORMAT) { + job->flags |= FLAG_UPGRADEDNFORMAT; + if (ud_flags & SLAPI_DRYRUN) { + job->flags |= FLAG_DRYRUN; + } + } else { + job->flags |= FLAG_REINDEXING; + } + } + if (!noattrindexes) { + job->flags |= FLAG_INDEX_ATTRS; + } + for (i = 0; name_array && name_array[i] != NULL; i++) { charray_add(&job->input_filenames, slapi_ch_strdup(name_array[i])); + } job->starting_ID = 1; job->first_ID = 1; job->mothers = CALLOC(import_subcount_stuff); @@ -1450,10 +1550,10 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) /* how much space should we allocate to index buffering? */ job->job_index_buffer_size = import_get_index_buffer_size(); if (job->job_index_buffer_size == 0) { - /* 10% of the allocated cache size + one meg */ + /* 10% of the allocated cache size + one meg */ PR_Lock(job->inst->inst_li->li_config_mutex); - job->job_index_buffer_size = (job->inst->inst_li->li_import_cachesize/10) + - (1024*1024); + job->job_index_buffer_size = + (job->inst->inst_li->li_import_cachesize/10) + (1024*1024); PR_Unlock(job->inst->inst_li->li_config_mutex); } import_subcount_stuff_init(job->mothers); @@ -1466,18 +1566,19 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) /* add 1 to account for post-import cleanup (which can take a * significant amount of time) */ - /* NGK - This should eventually be cleaned up to use the public - * task API. */ - if (0 == total_files) /* reindexing */ - job->task->task_work = 2; - else - job->task->task_work = total_files + 1; - job->task->task_progress = 0; - job->task->task_state = SLAPI_TASK_RUNNING; - slapi_task_set_data(job->task, job); - slapi_task_set_destructor_fn(job->task, import_task_destroy); - slapi_task_set_cancel_fn(job->task, import_task_abort); - job->flags |= FLAG_ONLINE; + /* NGK - This should eventually be cleaned up to use the public + * task API. */ + if (0 == total_files) { /* reindexing */ + job->task->task_work = 2; + } else { + job->task->task_work = total_files + 1; + } + job->task->task_progress = 0; + job->task->task_state = SLAPI_TASK_RUNNING; + slapi_task_set_data(job->task, job); + slapi_task_set_destructor_fn(job->task, import_task_destroy); + slapi_task_set_cancel_fn(job->task, import_task_abort); + job->flags |= FLAG_ONLINE; /* create thread for import_main, so we can return */ thread = PR_CreateThread(PR_USER_THREAD, import_main, (void *)job, diff --git a/ldap/servers/slapd/back-ldbm/import.h b/ldap/servers/slapd/back-ldbm/import.h index aa3201ce..e5569d34 100644 --- a/ldap/servers/slapd/back-ldbm/import.h +++ b/ldap/servers/slapd/back-ldbm/import.h @@ -159,11 +159,13 @@ typedef struct { #define FLAG_INDEX_ATTRS 0x01 /* should we index the attributes? */ #define FLAG_USE_FILES 0x02 /* import from files */ -#define FLAG_PRODUCER_DONE 0x04 /* frontend is done sending entries - * for replica initialization */ -#define FLAG_ABORT 0x08 /* import has been aborted */ -#define FLAG_ONLINE 0x10 /* bring backend online when done */ -#define FLAG_REINDEXING 0x20 /* read from id2entry and do indexing */ +#define FLAG_PRODUCER_DONE 0x04 /* frontend is done sending entries + * for replica initialization */ +#define FLAG_ABORT 0x08 /* import has been aborted */ +#define FLAG_ONLINE 0x10 /* bring backend online when done */ +#define FLAG_REINDEXING 0x20 /* read from id2entry and do indexing */ +#define FLAG_UPGRADEDNFORMAT 0x40 /* read from id2entry and do upgrade dn */ +#define FLAG_DRYRUN 0x80 /* dryrun for upgrade dn */ /* Structure holding stuff about a worker thread and what it's up to */ @@ -197,7 +199,8 @@ struct _import_worker_info { #define WAITING 1 #define RUNNING 2 #define FINISHED 3 -#define ABORTED 4 +#define ABORTED 4 +#define QUIT 5 /* quit intentionally. to distinguish from ABORTED & FINISHED */ /* this is just a convenience, because the slapi_ch_* calls are annoying */ #define CALLOC(name) (name *)slapi_ch_calloc(1, sizeof(name)) @@ -234,5 +237,6 @@ int add_op_attrs(Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *ep, /* import-threads.c */ void import_producer(void *param); void index_producer(void *param); +void upgradedn_producer(void *param); void import_foreman(void *param); void import_worker(void *param); diff --git a/ldap/servers/slapd/back-ldbm/init.c b/ldap/servers/slapd/back-ldbm/init.c index e1c57ba6..d25efc99 100644 --- a/ldap/servers/slapd/back-ldbm/init.c +++ b/ldap/servers/slapd/back-ldbm/init.c @@ -226,6 +226,8 @@ ldbm_back_init( Slapi_PBlock *pb ) (void *) ldbm_back_ldbm2archive ); rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_UPGRADEDB_FN, (void *) ldbm_back_upgradedb ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_UPGRADEDNFORMAT_FN, + (void *) ldbm_back_upgradednformat ); rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_DBVERIFY_FN, (void *) ldbm_back_dbverify ); rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_BEGIN_FN, diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c index 4540c149..fc07443e 100644 --- a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c +++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c @@ -2016,7 +2016,7 @@ ldbm_exclude_attr_from_export( struct ldbminfo *li , const char *attr, void upgradedb_core(Slapi_PBlock *pb, ldbm_instance *inst); int upgradedb_copy_logfiles(struct ldbminfo *li, char *destination_dir, int restore, int *cnt); int upgradedb_delete_indices_4cmd(ldbm_instance *inst); -void normalize_dir(char *dir); +static void normalize_dir(char *dir); /* * ldbm_back_upgradedb - @@ -2347,13 +2347,23 @@ fail0: return rval; } -void normalize_dir(char *dir) +static void +normalize_dir(char *dir) { - int l = strlen(dir); - if ('/' == dir[l-1] || '\\' == dir[l-1]) - { - dir[l-1] = '\0'; + char *p = NULL; + int l = 0; + + if (NULL == dir) { + return; + } + l = strlen(dir); + + for (p = dir + l - 1; p && *p; p--) { + if (' ' != *p && '\t' != *p && '/' != *p && '\\' != *p) { + break; + } } + *(p+1) = '\0'; } #define LOG "log." @@ -2546,3 +2556,205 @@ void upgradedb_core(Slapi_PBlock *pb, ldbm_instance *inst) ldbm_back_ldif2ldbm_deluxe(pb); } +/* + * ldbm_back_upgradednformat + * + * Update old DN format in entrydn and the leaf attr value to the new one + * + * The implementation would be similar to the upgradedb for new idl. + * Scan each entry, checking the entrydn value with the result of + * slapi_dn_normalize_ext_case(dn). + * If they don't match, + * replace the old entrydn value with the new one in the entry + * in id2entry.db4. + * also get the leaf RDN attribute value, unescape it, and check + * if it is in the entry. If not, add it. + * Then, update the key in the entrydn index and the leaf RDN attribute + * (if need it). + * + * Return value: 0: success (the backend instance includes update + * candidates for DRYRUN mode) + * 1: the backend instance is up-to-date (DRYRUN mode only) + * -1: error + * + * standalone only -- not allowed to run while DS is up. + */ +int ldbm_back_upgradednformat(Slapi_PBlock *pb) +{ + int rc = -1; + struct ldbminfo *li = NULL; + int run_from_cmdline = 0; + int task_flags = 0; + int server_running = 0; + Slapi_Task *task; + ldbm_instance *inst = NULL; + char *instance_name = NULL; + backend *be = NULL; + PRStatus prst = 0; + PRFileInfo prfinfo = {0}; + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + size_t id2entrylen = 0; + int found = 0; + char *rawworkdbdir = NULL; + char *workdbdir = NULL; + char *origdbdir = NULL; + char *origlogdir = NULL; + char *originstparentdir = NULL; + char *sep = NULL; + char *ldbmversion = NULL; + char *dataversion = NULL; + int ud_flags = 0; + + slapi_pblock_get(pb, SLAPI_TASK_FLAGS, &task_flags); + slapi_pblock_get(pb, SLAPI_BACKEND_TASK, &task); + slapi_pblock_get(pb, SLAPI_DB2LDIF_SERVER_RUNNING, &server_running); + slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name); + slapi_pblock_get( pb, SLAPI_SEQ_TYPE, &ud_flags ); + + run_from_cmdline = (task_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE); + slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &li); + if (run_from_cmdline) { + ldbm_config_load_dse_info(li); + autosize_import_cache(li); + } else { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + " Online mode is not supported. " + "Shutdown the server and run the tool\n"); + goto bail; + } + + /* Find the instance that the ldif2db will be done on. */ + inst = ldbm_instance_find_by_name(li, instance_name); + if (NULL == inst) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Unknown ldbm instance %s\n", instance_name); + goto bail; + } + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "%s: Start upgrade dn format.\n", inst->inst_name); + + slapi_pblock_set(pb, SLAPI_BACKEND, inst->inst_be); + slapi_pblock_get(pb, SLAPI_SEQ_VAL, &rawworkdbdir); + normalize_dir(rawworkdbdir); /* remove trailing spaces and slashes */ + + prst = PR_GetFileInfo(rawworkdbdir, &prfinfo); + if (PR_FAILURE == prst || PR_FILE_DIRECTORY != prfinfo.type) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Working DB instance dir %s is not a directory\n", + rawworkdbdir); + goto bail; + } + dirhandle = PR_OpenDir(rawworkdbdir); + if (!dirhandle) + { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Failed to open working DB instance dir %s\n", + rawworkdbdir); + goto bail; + } + id2entrylen = strlen(ID2ENTRY); + while ((direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { + if (!direntry->name) + break; + if (0 == strncasecmp(ID2ENTRY, direntry->name, id2entrylen)) { + found = 1; + break; + } + } + PR_CloseDir(dirhandle); + + if (!found) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Working DB instance dir %s does not include %s file\n", + rawworkdbdir, ID2ENTRY); + goto bail; + } + + if (run_from_cmdline) { + ldbm_config_internal_set(li, CONFIG_DB_TRANSACTION_LOGGING, "off"); + } + + /* We have to work on the copied db. So, the path should be set here. */ + origdbdir = li->li_directory; + origlogdir = li->li_dblayer_private->dblayer_log_directory; + originstparentdir = inst->inst_parent_dir_name; + + workdbdir = rel2abspath(rawworkdbdir); + + dbversion_read(li, workdbdir, &ldbmversion, &dataversion); + if (ldbmversion && PL_strstr(ldbmversion, BDB_DNFORMAT)) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Instance %s in %s is up-to-date\n", + instance_name, workdbdir); + rc = 1; /* 1: up-to-date; 0: need upgrade; otherwise: error */ + goto bail; + } + + sep = PL_strrchr(workdbdir, '/'); + if (!sep) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Working DB instance dir %s does not include %s file\n", + workdbdir, ID2ENTRY); + goto bail; + } + *sep = '\0'; + li->li_directory = workdbdir; + li->li_dblayer_private->dblayer_log_directory = workdbdir; + inst->inst_parent_dir_name = workdbdir; + + if (run_from_cmdline) { + if (0 != dblayer_start(li, DBLAYER_IMPORT_MODE)) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Failed to init database\n"); + goto bail; + } + } + + /* dblayer_instance_start will init the id2entry index. */ + be = inst->inst_be; + if (0 != dblayer_instance_start(be, DBLAYER_IMPORT_MODE)) + { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DB Format", + "Failed to init instance %s\n", inst->inst_name); + goto bail; + } + + if (run_from_cmdline) { + vlv_init(inst); /* Initialise the Virtual List View code */ + } + + rc = ldbm_back_ldif2ldbm_deluxe(pb); + + /* close the database */ + if (run_from_cmdline) { + if (0 != dblayer_flush(li)) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Failed to flush database\n"); + } + if (0 != dblayer_close(li,DBLAYER_IMPORT_MODE)) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Failed to close database\n"); + goto bail; + } + } + *sep = '/'; + if (((0 == rc) && !(ud_flags & SLAPI_DRYRUN)) || + ((rc > 0) && (ud_flags & SLAPI_DRYRUN))) { + /* modify the DBVERSION files if the DN upgrade was successful OR + * if DRYRUN, the backend instance is up-to-date. */ + dbversion_write(li, workdbdir, NULL); /* inst db dir */ + } + /* Remove the DB env files */ + dblayer_remove_env(li); + + li->li_directory = origdbdir; + li->li_dblayer_private->dblayer_log_directory = origlogdir; + inst->inst_parent_dir_name = originstparentdir; + +bail: + slapi_ch_free_string(&workdbdir); + slapi_ch_free_string(&ldbmversion); + slapi_ch_free_string(&dataversion); + return rc; +} diff --git a/ldap/servers/slapd/back-ldbm/misc.c b/ldap/servers/slapd/back-ldbm/misc.c index 37622157..dc1d2b01 100644 --- a/ldap/servers/slapd/back-ldbm/misc.c +++ b/ldap/servers/slapd/back-ldbm/misc.c @@ -388,3 +388,185 @@ is_fullpath(char *path) } return 0; } + +/* + * Get value of type from string. + * Note: this function does not support multi values. + * This could be used to retrieve a single value as a string from raw data + * read from db. + */ +/* caller is responsible to release "value" */ +int +get_value_from_string(const char *string, char *type, char **value) +{ + int rc = -1; + size_t typelen = 0; + char *ptr = NULL; + char *copy = NULL; + char *tmpptr = NULL; + char *tmptype = NULL; + char *valueptr = NULL; +#if defined (USE_OPENLDAP) + ber_len_t valuelen; +#else + int valuelen; +#endif + + if (NULL == string || NULL == type || NULL == value) { + return rc; + } + *value = NULL; + tmpptr = (char *)string; + ptr = PL_strcasestr(tmpptr, type); + if (NULL == ptr) { + return rc; + } + + typelen = strlen(type); + while (NULL != (ptr = ldif_getline(&tmpptr))) { + if ((0 != PL_strncasecmp(ptr, type, typelen)) || + (*(ptr + typelen) != ';' && *(ptr + typelen) != ':')) { + /* did not match */ + /* ldif_getline replaces '\n' and '\r' with '\0' */ + if ('\0' == *(tmpptr - 1)) { + *(tmpptr - 1) = '\n'; + } + if ('\0' == *(tmpptr - 2)) { + *(tmpptr - 2) = '\r'; + } + continue; + } + /* matched */ + copy = slapi_ch_strdup(ptr); + /* ldif_getline replaces '\n' and '\r' with '\0' */ + if ('\0' == *(tmpptr - 1)) { + *(tmpptr - 1) = '\n'; + } + if ('\0' == *(tmpptr - 2)) { + *(tmpptr - 2) = '\r'; + } + rc = ldif_parse_line(copy, &tmptype, &valueptr, &valuelen); + if (0 > rc || NULL == valueptr || 0 >= valuelen) { + continue; + } + if (0 != strcasecmp(type, tmptype)) { + char *p = PL_strchr(tmptype, ';'); /* subtype ? */ + if (p) { + if (0 != strncasecmp(type, tmptype, p - tmptype)) { + slapi_log_error(SLAPI_LOG_FATAL, "get_value_from_string", + "type does not match: %s != %s\n", + type, tmptype); + goto bail; + } + } else { + slapi_log_error(SLAPI_LOG_FATAL, "get_value_from_string", + "type does not match: %s != %s\n", + type, tmptype); + goto bail; + } + } + *value = (char *)slapi_ch_malloc(valuelen + 1); + memcpy(*value, valueptr, valuelen); + *(*value + valuelen) = '\0'; + break; + } +bail: + slapi_ch_free_string(©); + return rc; +} + +/* + * Get value array of type from string. + * multi-value support for get_value_from_string + */ +/* caller is responsible to release "valuearray" */ +int +get_values_from_string(const char *string, char *type, char ***valuearray) +{ + int rc = -1; + size_t typelen = 0; + char *ptr = NULL; + char *copy = NULL; + char *tmpptr = NULL; + char *tmptype = NULL; + char *valueptr = NULL; +#if defined (USE_OPENLDAP) + ber_len_t valuelen; +#else + int valuelen; +#endif + char *value = NULL; + int idx = 0; +#define get_values_INITIALMAXCNT 1 + int maxcnt = get_values_INITIALMAXCNT; + + if (NULL == string || NULL == type || NULL == valuearray) { + return rc; + } + *valuearray = NULL; + tmpptr = (char *)string; + ptr = PL_strcasestr(tmpptr, type); + if (NULL == ptr) { + return rc; + } + + typelen = strlen(type); + while (NULL != (ptr = ldif_getline(&tmpptr))) { + if ((0 != PL_strncasecmp(ptr, type, typelen)) || + (*(ptr + typelen) != ';' && *(ptr + typelen) != ':')) { + /* did not match */ + /* ldif_getline replaces '\n' and '\r' with '\0' */ + if ('\0' == *(tmpptr - 1)) { + *(tmpptr - 1) = '\n'; + } + if ('\0' == *(tmpptr - 2)) { + *(tmpptr - 2) = '\r'; + } + continue; + } + /* matched */ + copy = slapi_ch_strdup(ptr); + /* ldif_getline replaces '\n' and '\r' with '\0' */ + if ('\0' == *(tmpptr - 1)) { + *(tmpptr - 1) = '\n'; + } + if ('\0' == *(tmpptr - 2)) { + *(tmpptr - 2) = '\r'; + } + rc = ldif_parse_line(copy, &tmptype, &valueptr, &valuelen); + if (0 > rc || NULL == valueptr || 0 >= valuelen) { + continue; + } + if (0 != strcasecmp(type, tmptype)) { + char *p = PL_strchr(tmptype, ';'); /* subtype ? */ + if (p) { + if (0 != strncasecmp(type, tmptype, p - tmptype)) { + slapi_log_error(SLAPI_LOG_FATAL, "get_values_from_string", + "type does not match: %s != %s\n", + type, tmptype); + goto bail; + } + } else { + slapi_log_error(SLAPI_LOG_FATAL, "get_values_from_string", + "type does not match: %s != %s\n", + type, tmptype); + goto bail; + } + } + value = (char *)slapi_ch_malloc(valuelen + 1); + memcpy(value, valueptr, valuelen); + *(value + valuelen) = '\0'; + if ((get_values_INITIALMAXCNT == maxcnt) || !valuearray || + (idx + 1 >= maxcnt)) { + maxcnt *= 2; + *valuearray = (char **)slapi_ch_realloc((char *)*valuearray, + sizeof(char *) * maxcnt); + } + (*valuearray)[idx++] = value; + (*valuearray)[idx] = NULL; + slapi_ch_free_string(©); + } +bail: + slapi_ch_free_string(©); + return rc; +} diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h index 6f7d9994..c8bf3e2c 100644 --- a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h @@ -171,7 +171,7 @@ int dblayer_db_uses_transactions(DB_ENV *db_env); int dblayer_db_uses_mpool(DB_ENV *db_env); int dblayer_db_uses_logging(DB_ENV *db_env); int dblayer_bt_compare(DB *db, const DBT *dbt1, const DBT *dbt2); - +int dblayer_remove_env(struct ldbminfo *li); /* * dn2entry.c @@ -352,6 +352,9 @@ int ldbm_delete_dirs(char *path); int mkdir_p(char *dir, unsigned int mode); int is_fullpath(char *path); char get_sep(char *path); +int get_value_from_string(const char *string, char *type, char **value); +int get_values_from_string(const char *string, char *type, char ***valuearray); + /* * nextid.c @@ -451,6 +454,7 @@ int ldbm_back_ldif2ldbm( Slapi_PBlock *pb ); int ldbm_back_ldbm2ldif( Slapi_PBlock *pb ); int ldbm_back_ldbm2ldifalt( Slapi_PBlock *pb ); int ldbm_back_ldbm2index( Slapi_PBlock *pb ); +int ldbm_back_upgradednformat( Slapi_PBlock *pb ); int ldbm_back_archive2ldbm( Slapi_PBlock *pb ); int ldbm_back_ldbm2archive( Slapi_PBlock *pb ); int ldbm_back_upgradedb( Slapi_PBlock *pb ); @@ -602,6 +606,7 @@ int ldbm_instance_attrcrypt_config_modify_callback(Slapi_PBlock *pb, Slapi_Entry void replace_ldbm_config_value(char *conftype, char *val, struct ldbminfo *li); + /* * ancestorid.c */ diff --git a/ldap/servers/slapd/delete.c b/ldap/servers/slapd/delete.c index 3a80eeba..acb2e440 100644 --- a/ldap/servers/slapd/delete.c +++ b/ldap/servers/slapd/delete.c @@ -113,7 +113,7 @@ do_delete( Slapi_PBlock *pb ) } err = slapi_dn_normalize_ext(rawdn, 0, &dn, &dnlen); if (err < 0) { - op_shared_log_error_access(pb, "DEL", "???", "invalid dn"); + op_shared_log_error_access(pb, "DEL", rawdn?rawdn:"", "invalid dn"); send_ldap_result(pb, LDAP_INVALID_DN_SYNTAX, NULL, "invalid dn", 0, NULL); slapi_ch_free_string(&rawdn); diff --git a/ldap/servers/slapd/dn.c b/ldap/servers/slapd/dn.c index c3aaf4cc..fc3695c9 100644 --- a/ldap/servers/slapd/dn.c +++ b/ldap/servers/slapd/dn.c @@ -449,7 +449,6 @@ static int ISEOV(char *s, char *ends) { char *p; - int rc = 1; for (p = s; p && *p && p < ends; p++) { if (SEPARATOR(*p)) { return 1; @@ -1174,6 +1173,18 @@ slapi_dn_normalize( char *dn ) return dn; } +/* Introduced for the upgrade tool. DON'T USE THIS API! */ +char * +slapi_dn_normalize_case_original( char *dn ) +{ + /* LDAPDebug( LDAP_DEBUG_TRACE, "=> slapi_dn_normalize \"%s\"\n", dn, 0, 0 ); */ + *(substr_dn_normalize_orig( dn, dn + strlen( dn ))) = '\0'; + /* LDAPDebug( LDAP_DEBUG_TRACE, "<= slapi_dn_normalize \"%s\"\n", dn, 0, 0 ); */ + + /* normalize case */ + return( slapi_dn_ignore_case( dn )); +} + /* * DEPRECATED: this function does nothing. * Note that this routine normalizes to the end and doesn't null terminate diff --git a/ldap/servers/slapd/main.c b/ldap/servers/slapd/main.c index 67be7902..82f0089d 100644 --- a/ldap/servers/slapd/main.c +++ b/ldap/servers/slapd/main.c @@ -113,6 +113,7 @@ static int slapd_exemode_db2index(); static int slapd_exemode_archive2db(); static int slapd_exemode_db2archive(); static int slapd_exemode_upgradedb(); +static int slapd_exemode_upgradednformat(); static int slapd_exemode_dbverify(); static int slapd_exemode_dbtest(); static int slapd_exemode_suffix2instance(); @@ -374,6 +375,8 @@ name2exemode( char *progname, char *s, int exit_if_unknown ) exemode = SLAPD_EXEMODE_SUFFIX2INSTANCE; } else if ( strcmp( s, "upgradedb" ) == 0 ) { exemode = SLAPD_EXEMODE_UPGRADEDB; + } else if ( strcmp( s, "upgradednformat" ) == 0 ) { + exemode = SLAPD_EXEMODE_UPGRADEDNFORMAT; } else if ( strcmp( s, "dbverify" ) == 0 ) { exemode = SLAPD_EXEMODE_DBVERIFY; } @@ -381,7 +384,7 @@ name2exemode( char *progname, char *s, int exit_if_unknown ) fprintf( stderr, "usage: %s -D configdir " "[ldif2db | db2ldif | archive2db " "| db2archive | db2index | refer | suffix2instance" - " | upgradedb | dbverify] " + " | upgradedb | upgradednformat | dbverify] " "[options]\n", progname ); exit( 1 ); } else { @@ -441,6 +444,9 @@ usage( char *name, char *extraname ) case SLAPD_EXEMODE_UPGRADEDB: usagestr = "usage: %s %s%s-D configdir [-d debuglevel] [-f] -a archivedir\n"; break; + case SLAPD_EXEMODE_UPGRADEDNFORMAT: + usagestr = "usage: %s %s%s-D configdir [-d debuglevel] [-N] -n backend-instance-name -a fullpath-backend-instance-dir-full\n"; + break; case SLAPD_EXEMODE_DBVERIFY: usagestr = "usage: %s %s%s-D configdir [-d debuglevel] [-n backend-instance-name]\n"; break; @@ -484,6 +490,7 @@ static int dbverify_verbose = 0; static char *ldif2db_namespaceid = NULL; int importexport_encrypt = 0; static int upgradedb_force = 0; +static int upgradednformat_dryrun = 0; /* taken from idsktune */ #if defined(__sun) @@ -990,6 +997,11 @@ main( int argc, char **argv) goto cleanup; break; + case SLAPD_EXEMODE_UPGRADEDNFORMAT: + return_value = slapd_exemode_upgradednformat(); + goto cleanup; + break; + case SLAPD_EXEMODE_DBVERIFY: return_value = slapd_exemode_dbverify(); goto cleanup; @@ -1415,6 +1427,16 @@ process_command_line(int argc, char **argv, char *myname, {"configDir",ArgRequired,'D'}, {0,0,0}}; + char *opts_upgradednformat = "vd:a:n:D:N"; + struct opt_ext long_options_upgradednformat[] = { + {"version",ArgNone,'v'}, + {"debug",ArgRequired,'d'}, + {"backend",ArgRequired,'n'}, + {"archive",ArgRequired,'a'}, /* Path to the work db instance dir */ + {"configDir",ArgRequired,'D'}, + {"dryrun",ArgNone,'N'}, + {0,0,0}}; + char *opts_dbverify = "vVfd:n:D:"; struct opt_ext long_options_dbverify[] = { {"version",ArgNone,'v'}, @@ -1520,6 +1542,10 @@ process_command_line(int argc, char **argv, char *myname, opts = opts_upgradedb; long_opts = long_options_upgradedb; break; + case SLAPD_EXEMODE_UPGRADEDNFORMAT: + opts = opts_upgradednformat; + long_opts = long_options_upgradednformat; + break; case SLAPD_EXEMODE_DBVERIFY: opts = opts_dbverify; long_opts = long_options_dbverify; @@ -1622,6 +1648,7 @@ process_command_line(int argc, char **argv, char *myname, break; case 'n': /* which backend to do ldif2db/bak2db for */ if (slapd_exemode == SLAPD_EXEMODE_LDIF2DB || + slapd_exemode == SLAPD_EXEMODE_UPGRADEDNFORMAT || slapd_exemode == SLAPD_EXEMODE_DBTEST || slapd_exemode == SLAPD_EXEMODE_DB2INDEX || slapd_exemode == SLAPD_EXEMODE_ARCHIVE2DB) { @@ -1663,8 +1690,10 @@ process_command_line(int argc, char **argv, char *myname, db2ldif_dump_replica = 1; break; case 'N': /* do not do ldif2db duplicate value check */ + /* Or dryrun mode for upgradednformat */ if ( slapd_exemode != SLAPD_EXEMODE_LDIF2DB && - slapd_exemode != SLAPD_EXEMODE_DB2LDIF) { + slapd_exemode != SLAPD_EXEMODE_DB2LDIF && + slapd_exemode != SLAPD_EXEMODE_UPGRADEDNFORMAT) { usage( myname, *extraname ); exit( 1 ); } @@ -1679,6 +1708,9 @@ process_command_line(int argc, char **argv, char *myname, if ( slapd_exemode == SLAPD_EXEMODE_DB2LDIF ) { ldif_printkey &= ~EXPORT_PRINTKEY; } + if ( slapd_exemode == SLAPD_EXEMODE_UPGRADEDNFORMAT ) { + upgradednformat_dryrun = 1; + } break; @@ -2653,7 +2685,7 @@ slapd_exemode_upgradedb() } /* check for slapi v2 support */ if (! SLAPI_PLUGIN_IS_V2(backend_plugin)) { - LDAPDebug(LDAP_DEBUG_ANY, "ERROR: %s is too old to do convert idl.\n", + LDAPDebug(LDAP_DEBUG_ANY, "ERROR: %s is too old to reindex all.\n", backend_plugin->plg_name, 0, 0); return 1; } @@ -2684,6 +2716,83 @@ slapd_exemode_upgradedb() return( return_value ); } +/* Command to upgrade the old dn format to the new style */ +static int +slapd_exemode_upgradednformat() +{ + int rc = -1; /* error, by default */ + Slapi_PBlock pb; + struct slapdplugin *backend_plugin; + slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig(); + + if ( archive_name == NULL ) { + LDAPDebug0Args(LDAP_DEBUG_ANY, "ERROR: Required argument " + "\"-a <path to work db instance dir>\" is missing\n"); + usage( myname, extraname ); + goto bail; + } + + /* this should be the first time to be called! if the init order + * is ever changed, these lines should be changed (or erased)! + */ + mapping_tree_init(); + + if ((backend_plugin = plugin_get_by_name("ldbm database")) == NULL) { + LDAPDebug0Args(LDAP_DEBUG_ANY, + "ERROR: Could not find the ldbm backend plugin.\n"); + goto bail; + } + + /* Make sure we aren't going to run slapd in + * a mode that is going to conflict with other + * slapd processes that are currently running + * Pretending to execute import. + */ + if (add_new_slapd_process(slapd_exemode, 0, skip_db_protect_check) + == -1) { + LDAPDebug0Args(LDAP_DEBUG_ANY, "Shutting down due to possible " + "conflicts with other slapd processes\n"); + goto bail; + } + /* check for slapi v2 support */ + if (! SLAPI_PLUGIN_IS_V2(backend_plugin)) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "ERROR: %s is too old to upgrade dn format.\n", + backend_plugin->plg_name); + goto bail; + } + + memset( &pb, '\0', sizeof(pb) ); + pb.pb_backend = NULL; + pb.pb_plugin = backend_plugin; + pb.pb_instance_name = cmd_line_instance_name; + if (upgradednformat_dryrun) { + pb.pb_seq_type = SLAPI_UPGRADEDNFORMAT|SLAPI_DRYRUN; + } else { + pb.pb_seq_type = SLAPI_UPGRADEDNFORMAT; + } + pb.pb_seq_val = archive_name; /* Path to the work db instance dir */ + pb.pb_task_flags = SLAPI_TASK_RUNNING_FROM_COMMANDLINE; + /* borrowing import code, so need to set up the import variables */ + pb.pb_ldif_generate_uniqueid = ldif2db_generate_uniqueid; + pb.pb_ldif_namespaceid = ldif2db_namespaceid; + pb.pb_ldif2db_noattrindexes = 0; + pb.pb_removedupvals = 0; +#ifndef _WIN32 + main_setuid(slapdFrontendConfig->localuser); +#endif + if ( backend_plugin->plg_upgradednformat != NULL ) { + rc = (*backend_plugin->plg_upgradednformat)( &pb ); + } else { + LDAPDebug( LDAP_DEBUG_ANY, + "ERROR: no upgradednformat function defined for " + "%s\n", backend_plugin->plg_name, 0, 0 ); + } +bail: + slapi_ch_free((void**)&myname ); + return( rc ); +} + /* * function to perform DB verify */ diff --git a/ldap/servers/slapd/mapping_tree.c b/ldap/servers/slapd/mapping_tree.c index 42ce6105..29e3639b 100644 --- a/ldap/servers/slapd/mapping_tree.c +++ b/ldap/servers/slapd/mapping_tree.c @@ -1080,6 +1080,15 @@ int mapping_tree_entry_modify_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefor { parent_node = mapping_tree_root; } + else if ((strcasecmp(mods[i]->mod_type, "cn") == 0) && + SLAPI_IS_MOD_ADD(mods[i]->mod_op)) + { + /* Allow to add an additional cn. + * e.g., cn: "<suffix>" for the backward compatibility. + * No need to update the mapping tree node itself. + */ + continue; + } else { /* we have to find the new parent node */ @@ -2641,6 +2650,11 @@ mtn_get_mapping_tree_node_by_entry(mapping_tree_node* node, const Slapi_DN *dn) return NULL; } + if(NULL == dn){ + /* bad mapping tree entry operation */ + return NULL; + } + if (slapi_sdn_compare(node->mtn_subtree, dn) == 0) { return node; diff --git a/ldap/servers/slapd/modify.c b/ldap/servers/slapd/modify.c index 027bbefb..955ef820 100644 --- a/ldap/servers/slapd/modify.c +++ b/ldap/servers/slapd/modify.c @@ -193,7 +193,7 @@ do_modify( Slapi_PBlock *pb ) } rc = slapi_dn_normalize_ext(rawdn, 0, &dn, &dnlen); if (rc < 0) { - op_shared_log_error_access(pb, "MOD", "???", "invalid dn"); + op_shared_log_error_access(pb, "MOD", rawdn?rawdn:"", "invalid dn"); send_ldap_result(pb, LDAP_INVALID_DN_SYNTAX, NULL, "invalid dn", 0, NULL); slapi_ch_free((void **) &rawdn); diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c index 20ead29a..7a9167f1 100644 --- a/ldap/servers/slapd/pblock.c +++ b/ldap/servers/slapd/pblock.c @@ -666,6 +666,12 @@ slapi_pblock_get( Slapi_PBlock *pblock, int arg, void *value ) } (*(IFP *)value) = pblock->pb_plugin->plg_upgradedb; break; + case SLAPI_PLUGIN_DB_UPGRADEDNFORMAT_FN: + if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_DATABASE ) { + return( -1 ); + } + (*(IFP *)value) = pblock->pb_plugin->plg_upgradednformat; + break; case SLAPI_PLUGIN_DB_DBVERIFY_FN: if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_DATABASE ) { return( -1 ); @@ -1988,6 +1994,12 @@ slapi_pblock_set( Slapi_PBlock *pblock, int arg, void *value ) } pblock->pb_plugin->plg_upgradedb = (IFP) value; break; + case SLAPI_PLUGIN_DB_UPGRADEDNFORMAT_FN: + if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_DATABASE ) { + return( -1 ); + } + pblock->pb_plugin->plg_upgradednformat = (IFP) value; + break; case SLAPI_PLUGIN_DB_DBVERIFY_FN: if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_DATABASE ) { return( -1 ); diff --git a/ldap/servers/slapd/protect_db.c b/ldap/servers/slapd/protect_db.c index a6ff87a8..c90c8001 100644 --- a/ldap/servers/slapd/protect_db.c +++ b/ldap/servers/slapd/protect_db.c @@ -490,6 +490,15 @@ add_new_slapd_process(int exec_mode, int r_flag, int skip_flag) result = 0; } break; + case SLAPD_EXEMODE_UPGRADEDNFORMAT: + if (running || importing || exporting) { + LDAPDebug(LDAP_DEBUG_ANY, NO_UPGRADEDNFORMAT_DUE_TO_USE, 0, 0, 0); + result = -1; + } else { + add_this_process_to(import_dir); + result = 0; + } + break; case SLAPD_EXEMODE_DBTEST: if (running || importing || exporting) { LDAPDebug(LDAP_DEBUG_ANY, NO_DBTEST_DUE_TO_USE, 0, 0, 0); diff --git a/ldap/servers/slapd/protect_db.h b/ldap/servers/slapd/protect_db.h index 47b84c69..1def34c9 100644 --- a/ldap/servers/slapd/protect_db.h +++ b/ldap/servers/slapd/protect_db.h @@ -102,6 +102,8 @@ void remove_slapd_process(); #define NO_UPGRADEDB_DUE_TO_USE "Unable to recreate index files because the database is being used by another slapd process.\n" +#define NO_UPGRADEDNFORMAT_DUE_TO_USE "Unable to upgrade dn format because the database is being used by another slapd process.\n" + #define CREATE_MUTEX_ERROR "Error - CreateMutex failed: %s\n" /* reason for failure */ diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h index e590421a..98aee57e 100644 --- a/ldap/servers/slapd/slap.h +++ b/ldap/servers/slapd/slap.h @@ -74,6 +74,7 @@ static char ptokDes[34] = "Internal (Software) Token "; #define SLAPD_EXEMODE_PRINTVERSION 10 #define SLAPD_EXEMODE_UPGRADEDB 11 #define SLAPD_EXEMODE_DBVERIFY 12 +#define SLAPD_EXEMODE_UPGRADEDNFORMAT 13 #ifdef _WIN32 #ifndef DONT_DECLARE_SLAPD_LDAP_DEBUG @@ -806,6 +807,7 @@ struct slapdplugin { IFP plg_un_db_archive2db; /* ldif 2 database */ IFP plg_un_db_db2archive; /* database 2 ldif */ IFP plg_un_db_upgradedb; /* convert old idl to new */ + IFP plg_un_db_upgradednformat; /* convert old dn format to new */ IFP plg_un_db_begin; /* dbase txn begin */ IFP plg_un_db_commit; /* dbase txn commit */ IFP plg_un_db_abort; /* dbase txn abort */ @@ -845,6 +847,7 @@ struct slapdplugin { #define plg_archive2db plg_un.plg_un_db.plg_un_db_archive2db #define plg_db2archive plg_un.plg_un_db.plg_un_db_db2archive #define plg_upgradedb plg_un.plg_un_db.plg_un_db_upgradedb +#define plg_upgradednformat plg_un.plg_un_db.plg_un_db_upgradednformat #define plg_dbverify plg_un.plg_un_db.plg_un_db_verify #define plg_dbsize plg_un.plg_un_db.plg_un_db_dbsize #define plg_dbtest plg_un.plg_un_db.plg_un_db_dbtest @@ -1095,6 +1098,7 @@ typedef struct backend { #define be_seq be_database->plg_seq #define be_ldif2db be_database->plg_ldif2db #define be_upgradedb be_database->plg_upgradedb +#define be_upgradednformat be_database->plg_upgradednformat #define be_db2ldif be_database->plg_db2ldif #define be_db2index be_database->plg_db2index #define be_archive2db be_database->plg_archive2db diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h index bbe3b5e4..7c8d9647 100644 --- a/ldap/servers/slapd/slapi-private.h +++ b/ldap/servers/slapd/slapi-private.h @@ -375,6 +375,7 @@ Slapi_DN *slapi_sdn_init_dn_passin(Slapi_DN *sdn,const char *dn); Slapi_DN *slapi_sdn_init_ndn_byref(Slapi_DN *sdn,const char *dn); Slapi_DN *slapi_sdn_init_ndn_byval(Slapi_DN *sdn,const char *dn); Slapi_DN *slapi_sdn_init_dn_ndn_byref(Slapi_DN *sdn,const char *dn); +char * slapi_dn_normalize_case_original( char *dn ); /* filter.c */ int filter_flag_is_set(const Slapi_Filter *f,unsigned char flag); @@ -872,6 +873,7 @@ int valuearray_normalize_value(Slapi_Value **vals); #define SLAPI_PLUGIN_DB_ADD_SCHEMA_FN 237 #define SLAPI_PLUGIN_DB_SEARCH_RESULTS_RELEASE_FN 238 #define SLAPI_PLUGIN_DB_PREV_SEARCH_RESULTS_FN 239 +#define SLAPI_PLUGIN_DB_UPGRADEDNFORMAT_FN 240 /* database plugin-specific parameters */ #define SLAPI_PLUGIN_DB_NO_ACL 250 #define SLAPI_PLUGIN_DB_RMDB_FN 280 @@ -1185,9 +1187,11 @@ void bervalarray_add_berval_fast(struct berval ***vals, const struct berval *add void DS_Sleep(PRIntervalTime ticks); -/* macro to specify the behavior of upgradedb */ +/* macro to specify the behavior of upgradedb & upgradednformat */ #define SLAPI_UPGRADEDB_FORCE 0x1 /* reindex all (no check w/ idl switch) */ #define SLAPI_UPGRADEDB_SKIPINIT 0x2 /* call upgradedb as part of other op */ +#define SLAPI_UPGRADEDNFORMAT 0x4 /* specify this op is upgradednformat */ +#define SLAPI_DRYRUN 0x8 /* dryrun mode for upgradednformat */ /* * Macro to set port to the 'port' field of a NSPR PRNetAddr union. |