diff options
Diffstat (limited to 'ldap/servers/slapd')
22 files changed, 1582 insertions, 246 deletions
diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h index b9545b21..9efcd160 100644 --- a/ldap/servers/slapd/back-ldbm/back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h @@ -167,7 +167,14 @@ typedef unsigned short u_int16_t; */ #define BDB_IMPL "bdb" #define BDB_BACKEND "libback-ldbm" /* This backend plugin */ +#define BDB_NEWIDL "newidl" /* new idl format */ #define BDB_RDNFORMAT "rdn-format" /* Subtree rename enabled */ +#define BDB_DNFORMAT "dn-4514" /* DN format RFC 4514 compliant */ + +#define DBVERSION_NEWIDL 0x1 +#define DBVERSION_RDNFORMAT 0x2 +#define DBVERSION_DNFORMAT 0x4 +#define DBVERSION_ALL 0xffffffff /* * While we support both new and old idl index, @@ -789,7 +796,7 @@ typedef struct _back_search_result_set #define BE_INDEX_TOMBSTONE 8 /* Index entry as a tombstone */ #define BE_INDEX_DONT_ENCRYPT 16 /* Disable any encryption if this flag is set */ #define BE_INDEX_EQUALITY 32 /* (w/DEL) remove the equality index */ -#define BE_INDEX_NORMALIZED SLAPI_ATTR_FLAG_NORMALIZED /* value already normalized */ +#define BE_INDEX_NORMALIZED SLAPI_ATTR_FLAG_NORMALIZED /* value already normalized (0x200) */ /* Name of attribute type used for binder-based look through limit */ #define LDBM_LOOKTHROUGHLIMIT_AT "nsLookThroughLimit" diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c index e744321e..35c392db 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.c +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -1706,7 +1706,7 @@ int dblayer_start(struct ldbminfo *li, int dbmode) if ( (DBLAYER_NORMAL_MODE == dbmode ) && (0 == return_value)) { /* update the dbversion file */ - dbversion_write(li, region_dir, NULL); + dbversion_write(li, region_dir, NULL, DBVERSION_ALL); /* if dblayer_close then dblayer_start is called, this flag is set */ @@ -2013,7 +2013,7 @@ int dblayer_instance_start(backend *be, int mode) } } else { /* The dbversion file didn't exist, so we'll create one. */ - dbversion_write(li, inst_dirp, NULL); + dbversion_write(li, inst_dirp, NULL, DBVERSION_ALL); } } /* on import we don't mess with the dbversion file except to write it * when done with the import. */ @@ -2210,7 +2210,7 @@ out: } if (mode & DBLAYER_NORMAL_MODE) { - dbversion_write(li, inst_dirp, NULL); + dbversion_write(li, inst_dirp, NULL, DBVERSION_ALL); /* richm - not sure if need to acquire the be lock first? */ /* need to set state back to started - set to stopped in dblayer_instance_close */ @@ -2284,7 +2284,7 @@ int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv) size_t cachesize; PRFileInfo prfinfo; PRStatus prst; - char *id2entry_file; + char *id2entry_file = NULL; char inst_dir[MAXPATHLEN]; char *inst_dirp = NULL; char *data_directories[2] = {0, 0}; @@ -2651,35 +2651,6 @@ int dblayer_post_close(struct ldbminfo *li, int dbmode) return_value = pEnv->dblayer_DB_ENV->close(pEnv->dblayer_DB_ENV, 0); dblayer_free_env(&priv->dblayer_env); /* pEnv is now garbage */ -#if 0 /* DBDB do NOT remove the environment: bad, bad idea */ - if (return_value == 0) { - DB_ENV *env = 0; - return_value = db_env_create(&env, 0); - /* don't be tempted to use the - previously nulled out env handle - as Sleepycat 3.x is unhappy if - the env handle handed to remove - was used elsewhere. rwagner */ - if (return_value == 0) { - char *home_dir = dblayer_get_home_dir(li, NULL); - if (home_dir) - return_value = env->remove(env, home_dir, 0); - if (0 == return_value - && !((DBLAYER_ARCHIVE_MODE|DBLAYER_EXPORT_MODE) & dbmode) - && !priv->dblayer_bad_stuff_happened) { - /* - * If we got here, we have a good consistent database, - * so we write the guard file - */ - commit_good_database(priv); - } else if (return_value == EBUSY) { - /* something else is using the env so ignore */ - /* but let's not make a guardian file */ - return_value = 0; - } - } - } -#endif if (0 == return_value && !((DBLAYER_ARCHIVE_MODE|DBLAYER_EXPORT_MODE) & dbmode) && !priv->dblayer_bad_stuff_happened) { @@ -2744,11 +2715,44 @@ int dblayer_close(struct ldbminfo *li, int dbmode) * for the transacted database, we interpret this as an instruction * to write a checkpoint. */ -int dblayer_flush(struct ldbminfo *li) +int +dblayer_flush(struct ldbminfo *li) { return 0; } +/* API to remove the environment */ +int +dblayer_remove_env(struct ldbminfo *li) +{ + DB_ENV *env = NULL; + dblayer_private *priv = NULL; + char *home_dir = NULL; + int rc = db_env_create(&env, 0); + if (rc) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "ERROR -- Failed to create DB_ENV (returned: %d)\n", rc); + return rc; + } + if (NULL == li) { + LDAPDebug0Args(LDAP_DEBUG_ANY, "ERROR -- No ldbm info is given\n"); + return -1; + } + priv = (dblayer_private *)li->li_dblayer_private; + + home_dir = dblayer_get_home_dir(li, NULL); + if (home_dir) { + rc = env->remove(env, home_dir, 0); + if (rc) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "ERROR -- Failed to remove DB environment files. " + "Please remove %s/__db.00# (# is 1 through 6)\n", + home_dir); + } + } + return rc; +} + #if !defined(DB_DUPSORT) #define DB_DUPSORT 0 #endif diff --git a/ldap/servers/slapd/back-ldbm/dbversion.c b/ldap/servers/slapd/back-ldbm/dbversion.c index 689282ae..5c5cdeb4 100644 --- a/ldap/servers/slapd/back-ldbm/dbversion.c +++ b/ldap/servers/slapd/back-ldbm/dbversion.c @@ -74,7 +74,7 @@ mk_dbversion_fullpath(struct ldbminfo *li, const char *directory, char *filename */ int dbversion_write(struct ldbminfo *li, const char *directory, - const char *dataversion) + const char *dataversion, PRUint32 flags) { char filename[ MAXPATHLEN*2 ]; PRFileDesc *prfd; @@ -99,33 +99,28 @@ dbversion_write(struct ldbminfo *li, const char *directory, else { /* Write the file */ - PRInt32 len; char buf[ LDBM_VERSION_MAXBUF ]; - /* recognize the difference between an old/new database regarding idl - * (406922) */ - if (idl_get_idl_new()) - { - if (entryrdn_get_switch()) { - sprintf(buf, "%s/%d.%d/%s/%s\n", - BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, - BDB_BACKEND, BDB_RDNFORMAT); - } else { - sprintf(buf, "%s/%d.%d/%s\n", - BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, - BDB_BACKEND); - } + char *ptr = NULL; + size_t len = 0; + /* Base DB Version */ + PR_snprintf(buf, sizeof(buf), "%s/%d.%d/%s", + BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, BDB_BACKEND); + len = strlen(buf); + ptr = buf + len; + if (idl_get_idl_new() && (flags & DBVERSION_NEWIDL)) { + PR_snprintf(ptr, sizeof(buf) - len, "/%s", BDB_NEWIDL); + len = strlen(buf); + ptr = buf + len; } - else - { - if (entryrdn_get_switch()) { - sprintf(buf, "%s/%d.%d/%s/%s\n", - BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, - BDB_BACKEND, BDB_RDNFORMAT); - } else { - sprintf(buf, "%s/%d.%d/%s\n", - BDB_IMPL, DB_VERSION_MAJOR, DB_VERSION_MINOR, - BDB_BACKEND); - } + if (entryrdn_get_switch() && (flags & DBVERSION_RDNFORMAT)) { + PR_snprintf(ptr, sizeof(buf) - len, "/%s", BDB_RDNFORMAT); + len = strlen(buf); + ptr = buf + len; + } + if (flags & DBVERSION_DNFORMAT) { + PR_snprintf(ptr, sizeof(buf) - len, "/%s", BDB_DNFORMAT); + len = strlen(buf); + ptr = buf + len; } len = strlen( buf ); if ( slapi_write_buffer( prfd, buf, len ) != len ) diff --git a/ldap/servers/slapd/back-ldbm/import-threads.c b/ldap/servers/slapd/back-ldbm/import-threads.c index 264eb90b..dcc44517 100644 --- a/ldap/servers/slapd/back-ldbm/import-threads.c +++ b/ldap/servers/slapd/back-ldbm/import-threads.c @@ -378,6 +378,7 @@ import_producer(void *param) ldif_context c; int my_version = 0; size_t newesize = 0; + Slapi_Attr *attr = NULL; PR_ASSERT(info != NULL); PR_ASSERT(inst != NULL); @@ -405,7 +406,6 @@ import_producer(void *param) * as we read it. */ while (! finished) { - Slapi_Attr *attr = NULL; int flags = 0; int prev_lineno = 0; int lines_in_entry = 0; @@ -586,7 +586,6 @@ import_producer(void *param) /* If we are importing pre-encrypted attributes, we need * to skip syntax checks for the encrypted values. */ if (!(job->encrypt) && inst->attrcrypt_configured) { - Slapi_Attr *attr = NULL; Slapi_Entry *e_copy = NULL; /* Scan through the entry to see if any present @@ -1042,6 +1041,7 @@ index_producer(void *param) rc = 0; /* assume this is a suffix */ } else { ID pid = (ID)strtol(pid_str, (char **)NULL, 10); + slapi_ch_free_string(&pid_str); /* if pid is larger than the current pid temp_id, * the parent entry hasn't */ rc = import_get_and_add_parent_rdns(info, inst, db, @@ -1123,6 +1123,593 @@ error: info->state = ABORTED; } +struct upgradedn_attr { + char *ud_type; + char *ud_value; + struct upgradedn_attr *ud_next; + int ud_flags; +#define OLD_DN_NORMALIZE 0x1 +}; + +static void +upgradedn_free_list(struct upgradedn_attr **ud_list) +{ + struct upgradedn_attr *ptr = *ud_list; + + while (ptr) { + struct upgradedn_attr *next = ptr->ud_next; + slapi_ch_free_string(&ptr->ud_type); + slapi_ch_free_string(&ptr->ud_value); + slapi_ch_free((void **)&ptr); + ptr = next; + } + *ud_list = NULL; + return; +} + +static void +upgradedn_add_to_list(struct upgradedn_attr **ud_list, + char *type, char *value, int flag) +{ + struct upgradedn_attr *elem = + (struct upgradedn_attr *) slapi_ch_malloc(sizeof(struct upgradedn_attr)); + elem->ud_type = type; + elem->ud_value = value; + elem->ud_flags = flag; + elem->ud_next = *ud_list; + *ud_list = elem; + return; +} + +/* + * Producer thread for upgrading dn format + * FLAG_UPGRADEDNFORMAT | FLAG_DRYRUN -- check the necessity of dn upgrade + * FLAG_UPGRADEDNFORMAT -- execute dn upgrade + * + * Read id2entry, + * Check the DN syntax attributes if it contains '\' or not AND + * Check the RDNs of the attributes if the value is surrounded by + * double-quotes or not. + * If both are false, skip the entry and go to next + * If either is true, create an entry which contains a correctly normalized + * DN attribute values in e_attr list and the original entrydn in the + * deleted attribute list e_deleted_attrs. + * + * If FLAG_UPGRADEDNFORMAT is set, worker_threads for indexing DN syntax + * attributes are brought up. Foreman thread updates entrydn index + * as well as the entry itself in the id2entry.db#. + * + * Note: QUIT state for info->state is introduced for DRYRUN mode to + * distinguish the intentional QUIT (found the dn upgrade candidate) + * from ABORTED (aborted or error) and FINISHED (scan all the entries + * and found no candidate to upgrade) + */ +void +upgradedn_producer(void *param) +{ + ImportWorkerInfo *info = (ImportWorkerInfo *)param; + ImportJob *job = info->job; + ID id = job->first_ID; + Slapi_Entry *e = NULL; + struct backentry *ep = NULL, *old_ep = NULL; + ldbm_instance *inst = job->inst; + PRIntervalTime sleeptime; + int finished = 0; + int idx; + int rc = 0; + Slapi_Attr *a = NULL; + Slapi_DN *sdn = NULL; + char *workdn = NULL; + int doit = 0; + int skipit = 0; + int isentrydn = 0; + Slapi_Value *value = NULL; + struct upgradedn_attr *ud_list = NULL; + char **ud_vals = NULL; + char **ud_valp = NULL; + struct upgradedn_attr *ud_ptr = NULL; + Slapi_Attr *ud_attr = NULL; + char *ecopy = NULL; + + /* vars for Berkeley DB */ + DB_ENV *env = NULL; + DB *db = NULL; + DBC *dbc = NULL; + DBT key = {0}; + DBT data = {0}; + int db_rval = -1; + backend *be = inst->inst_be; + int isfirst = 1; + int curr_entry = 0; + size_t newesize = 0; + + PR_ASSERT(info != NULL); + PR_ASSERT(inst != NULL); + PR_ASSERT(be != NULL); + + if ( job->flags & FLAG_ABORT ) + goto error; + + sleeptime = PR_MillisecondsToInterval(import_sleep_time); + + /* pause until we're told to run */ + while ((info->command == PAUSE) && !(job->flags & FLAG_ABORT)) { + info->state = WAITING; + DS_Sleep(sleeptime); + } + info->state = RUNNING; + + /* open id2entry with dedicated db env and db handler */ + if ( dblayer_get_aux_id2entry( be, &db, &env ) != 0 || db == NULL || + env == NULL) { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open id2entry\n", 0, 0, 0 ); + goto error; + } + + /* get a cursor to we can walk over the table */ + db_rval = db->cursor(db, NULL, &dbc, 0); + if ( 0 != db_rval ) { + LDAPDebug( LDAP_DEBUG_ANY, + "Failed to get cursor for reindexing\n", 0, 0, 0 ); + dblayer_release_id2entry(be, db); + goto error; + } + + /* we loop around reading the input files and processing each entry + * as we read it. + */ + finished = 0; + while (!finished) { + ID temp_id; + + if (job->flags & FLAG_ABORT) { + goto error; + } + while ((info->command == PAUSE) && !(job->flags & FLAG_ABORT)){ + info->state = WAITING; + DS_Sleep(sleeptime); + } + info->state = RUNNING; + + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + if (isfirst) + { + db_rval = dbc->c_get(dbc, &key, &data, DB_FIRST); + isfirst = 0; + } + else + { + db_rval = dbc->c_get(dbc, &key, &data, DB_NEXT); + } + + if (0 != db_rval) { + if (DB_NOTFOUND != db_rval) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed to read database, " + "errno=%d (%s)\n", inst->inst_name, db_rval, + dblayer_strerror(db_rval)); + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Failed to read database, err %d (%s)", + inst->inst_name, db_rval, + dblayer_strerror(db_rval)); + } + } + finished = 1; + break; /* error or done */ + } + curr_entry++; + temp_id = id_stored_to_internal((char *)key.data); + slapi_ch_free(&(key.data)); + + /* call post-entry plugin */ + plugin_call_entryfetch_plugins((char **)&data.dptr, &data.dsize); + ecopy = (char *)slapi_ch_malloc(data.dsize + 1); + memcpy(ecopy, data.dptr, data.dsize); + *(ecopy + data.dsize) = '\0'; + if (entryrdn_get_switch()) { + char *rdn = NULL; + + /* rdn is allocated in get_value_from_string */ + rc = get_value_from_string((const char *)data.dptr, "rdn", &rdn); + if (rc) { + /* data.dptr may not include rdn: ..., try "dn: ..." */ + e = slapi_str2entry( data.dptr, 0 ); + } else { + char *dn = NULL; + struct backdn *bdn = + dncache_find_id(&inst->inst_dncache, temp_id); + if (bdn) { + /* don't free dn */ + dn = (char *)slapi_sdn_get_dn(bdn->dn_sdn); + CACHE_RETURN(&inst->inst_dncache, &bdn); + } else { + Slapi_DN *sdn = NULL; + rc = entryrdn_lookup_dn(be, rdn, temp_id, &dn, NULL); + if (rc) { + /* We cannot use the entryrdn index; + * Compose dn from the entries in id2entry */ + Slapi_RDN psrdn = {0}; + char *pid_str = NULL; + char *pdn = NULL; + + LDAPDebug2Args( LDAP_DEBUG_TRACE, + "index_producer: entryrdn is not available; " + "composing dn (rdn: %s, ID: %d)\n", + rdn, temp_id); + rc = get_value_from_string((const char *)data.dptr, + LDBM_PARENTID_STR, &pid_str); + if (rc) { + rc = 0; /* assume this is a suffix */ + } else { + ID pid = (ID)strtol(pid_str, (char **)NULL, 10); + slapi_ch_free_string(&pid_str); + /* if pid is larger than the current pid temp_id, + * the parent entry hasn't */ + rc = import_get_and_add_parent_rdns(info, inst, db, + pid, &id, &psrdn, &curr_entry); + if (rc) { + LDAPDebug2Args( LDAP_DEBUG_ANY, + "ldbm2index: Failed to compose dn for " + "(rdn: %s, ID: %d)\n", rdn, temp_id); + slapi_ch_free_string(&rdn); + slapi_rdn_done(&psrdn); + continue; + } + /* Generate DN string from Slapi_RDN */ + rc = slapi_rdn_get_dn(&psrdn, &pdn); + slapi_rdn_done(&psrdn); + if (rc) { + LDAPDebug2Args( LDAP_DEBUG_ANY, + "ldbm2index: Failed to compose dn for " + "(rdn: %s, ID: %d) from Slapi_RDN\n", + rdn, temp_id); + slapi_ch_free_string(&rdn); + continue; + } + } + dn = slapi_ch_smprintf("%s%s%s", + rdn, pdn?",":"", pdn?pdn:""); + slapi_ch_free_string(&pdn); + } + /* dn is not dup'ed in slapi_sdn_new_dn_byref. + * It's set to bdn and put in the dn cache. */ + sdn = slapi_sdn_new_dn_byref(dn); + bdn = backdn_init(sdn, temp_id, 0); + CACHE_ADD( &inst->inst_dncache, bdn, NULL ); + CACHE_RETURN(&inst->inst_dncache, &bdn); + slapi_log_error(SLAPI_LOG_CACHE, "ldbm2index", + "entryrdn_lookup_dn returned: %s, " + "and set to dn cache\n", dn); + } + e = slapi_str2entry_ext( dn, data.dptr, 0 ); + slapi_ch_free_string(&rdn); + } + } else { + e = slapi_str2entry(data.data, 0); + } + slapi_ch_free(&(data.data)); + if ( NULL == e ) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: WARNING: skipping badly formatted entry (id %lu)", + inst->inst_name, (u_long)temp_id); + } + LDAPDebug(LDAP_DEBUG_ANY, + "%s: WARNING: skipping badly formatted entry (id %lu)\n", + inst->inst_name, (u_long)temp_id, 0); + continue; + } + + /* Check DN syntax attr values if it contains '\\' or not */ + /* Start from the rdn */ + if (entryrdn_get_switch()) { /* subtree-rename: on */ + char *rdn = NULL; + size_t rdnlen = 0; + rc = get_value_from_string((const char *)ecopy, "rdn", &rdn); + if (rc || (NULL == rdn)) { + LDAPDebug2Args(LDAP_DEBUG_ANY, + "%s: WARNING: skipping an entry with no RDN (id %lu)\n", + inst->inst_name, (u_long)temp_id); + continue; + } + + /* rdn contains '\\'. We have to update the value */ + if (PL_strchr(rdn, '\\')) { + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(LDBM_ENTRYRDN_STR), + slapi_ch_strdup(rdn), 0); + LDAPDebug(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: %s (id %lu)\n", + inst->inst_name, *ud_valp, (u_long)temp_id); + doit = 1; + } else { + rdnlen = strlen(rdn); + /* DN contains an RDN <type>="<value>" ? */ + if (('"' == *rdn) && + ('"' == *(rdn + rdnlen - 1))) { + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(LDBM_ENTRYRDN_STR), + slapi_ch_strdup(rdn), 0); + LDAPDebug(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: %s (id %lu)\n", + inst->inst_name, rdn, (u_long)temp_id); + doit = 1; + } + } + slapi_ch_free_string(&rdn); + } + for (a = e->e_attrs; a; a = a->a_next) { + if (slapi_attr_is_dn_syntax_attr(a)) { /* is dn syntax attr? */ + rc = get_values_from_string((const char *)ecopy, + a->a_type, &ud_vals); + if (rc || (NULL == ud_vals)) { + continue; /* empty; ignore it */ + } + + for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) { + char **rdns = NULL; + char **rdnsp = NULL; + char *valueptr = NULL; + int valuelen; + + /* ud_valp contains '\\'. We have to update the value */ + if (PL_strchr(*ud_valp, '\\')) { + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(a->a_type), + slapi_ch_strdup(*ud_valp), + 0); + LDAPDebug(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: %s (id %lu)\n", + inst->inst_name, *ud_valp, (u_long)temp_id); + doit = 1; + continue; + } + /* Also check RDN contains double quoted values */ + if (strcasecmp(a->a_type, "entrydn")) { + /* except entrydn */ + workdn = slapi_ch_strdup(*ud_valp); + isentrydn = 0; + } else { + /* entrydn: Get Slapi DN */ + sdn = slapi_entry_get_sdn(e); + workdn = slapi_ch_strdup(slapi_sdn_get_dn(sdn)); + isentrydn = 1; + } + rdns = ldap_explode_dn(workdn, 0); + skipit = 0; + for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) { + valueptr = PL_strchr(*rdnsp, '='); + if (NULL == valueptr) { + skipit = 1; + break; + } + valueptr++; + while ((' ' == *valueptr) || ('\t' == *valueptr)) { + valueptr++; + } + valuelen = strlen(valueptr); + if (0 == valuelen) { + skipit = 1; + break; + } + /* DN contains an RDN <type>="<value>" ? */ + if (('"' == *valueptr) && + ('"' == *(valueptr + valuelen - 1))) { + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(a->a_type), + slapi_ch_strdup(*ud_valp), + isentrydn?0:OLD_DN_NORMALIZE); + LDAPDebug(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: %s (id %lu)\n", + inst->inst_name, valueptr, (u_long)temp_id); + doit = 1; + break; + } + } + if (rdns) { + slapi_ldap_value_free(rdns); + } else { + skipit = 1; + } + if (skipit) { + break; + } + slapi_ch_free_string(&workdn); + } /* for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) */ + charray_free(ud_vals); + ud_vals = NULL; + if (skipit) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: WARNING: skipping an entry " + "with a corrupted dn (syntax value): %s " + "(id %lu)\n", + inst->inst_name, + workdn?workdn:"unknown", (u_long)temp_id); + slapi_ch_free_string(&workdn); + upgradedn_free_list(&ud_list); + break; + } + } /* if (slapi_attr_is_dn_syntax_attr(a)) */ + } /* for (a = e->e_attrs; a; a = a->a_next) */ + slapi_ch_free_string(&ecopy); + if (skipit) { + upgradedn_free_list(&ud_list); + slapi_entry_free(e); e = NULL; + continue; + } + + if (!doit) { + /* We don't have to update dn syntax values. */ + upgradedn_free_list(&ud_list); + slapi_entry_free(e); e = NULL; + continue; + } + + /* doit */ + if (job->flags & FLAG_DRYRUN) { + /* We can return SUCCESS (== found upgrade dn candidates) */ + finished = 0; /* make it sure ... */ + upgradedn_free_list(&ud_list); + slapi_entry_free(e); e = NULL; + goto bail; + } + + skipit = 0; + for (ud_ptr = ud_list; ud_ptr; ud_ptr = ud_ptr->ud_next) { + /* Move the current value to e_deleted_attrs. */ + /* entryrdn is special since it does not have an attribute in db */ + if (0 == strcmp(ud_ptr->ud_type, LDBM_ENTRYRDN_STR)) { + /* entrydn contains half normalized value in id2entry, + thus we have to replace it in id2entry. + The other DN syntax attribute values store + the originals. They are taken care by the normalizer. + */ + a = slapi_attr_new(); + slapi_attr_init(a, ud_ptr->ud_type); + value = slapi_value_new_string(ud_ptr->ud_value); + slapi_attr_add_value(a, value); + slapi_value_free(&value); + attrlist_add(&e->e_deleted_attrs, a); + } else { /* except "entryrdn" */ + ud_attr = attrlist_find(e->e_attrs, ud_ptr->ud_type); + if (ud_attr) { + /* We have to normalize the orignal string to generate + the key in the index. + */ + a = attrlist_find(e->e_deleted_attrs, ud_ptr->ud_type); + if (!a) { + a = slapi_attr_new(); + slapi_attr_init(a, ud_ptr->ud_type); + } else { + a = attrlist_remove(&e->e_deleted_attrs, + ud_ptr->ud_type); + } + slapi_dn_normalize_case_original(ud_ptr->ud_value); + value = slapi_value_new_string(ud_ptr->ud_value); + slapi_attr_add_value(a, value); + slapi_value_free(&value); + attrlist_add(&e->e_deleted_attrs, a); + } + } + } + upgradedn_free_list(&ud_list); + if (skipit) { + slapi_entry_free(e); e = NULL; + continue; + } + + ep = import_make_backentry(e, temp_id); + if (!ep) { + slapi_entry_free(e); e = NULL; + goto error; + } + + /* Add the newly case-normalized dn to entrydn in the e_attrs list. */ + add_update_entrydn_operational_attributes(ep); + + if (job->flags & FLAG_ABORT) + goto error; + + /* Now we have this new entry, all decoded + * Next thing we need to do is: + * (1) see if the appropriate fifo location contains an + * entry which had been processed by the indexers. + * If so, proceed. + * If not, spin waiting for it to become free. + * (2) free the old entry and store the new one there. + * (3) Update the job progress indicators so the indexers + * can use the new entry. + */ + idx = id % job->fifo.size; + old_ep = job->fifo.item[idx].entry; + if (old_ep) { + /* for the slot to be recycled, it needs to be already absorbed + * by the foreman (id >= ready_EID), and all the workers need to + * be finished with it (refcount = 0). + */ + while (((old_ep->ep_refcnt > 0) || + (old_ep->ep_id >= job->ready_EID)) + && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + info->state = WAITING; + DS_Sleep(sleeptime); + } + if (job->flags & FLAG_ABORT) + goto error; + + info->state = RUNNING; + PR_ASSERT(old_ep == job->fifo.item[idx].entry); + job->fifo.item[idx].entry = NULL; + if (job->fifo.c_bsize > job->fifo.item[idx].esize) + job->fifo.c_bsize -= job->fifo.item[idx].esize; + else + job->fifo.c_bsize = 0; + backentry_free(&old_ep); + } + + newesize = (slapi_entry_size(ep->ep_entry) + sizeof(struct backentry)); + if (newesize > job->fifo.bsize) { /* entry too big */ + char ebuf[BUFSIZ]; + import_log_notice(job, "WARNING: skipping entry \"%s\"", + escape_string(slapi_entry_get_dn(e), ebuf)); + import_log_notice(job, "REASON: entry too large (%lu bytes) for " + "the buffer size (%lu bytes)", newesize, job->fifo.bsize); + backentry_free(&ep); + job->skipped++; + continue; + } + /* Now check if fifo has enough space for the new entry */ + if ((job->fifo.c_bsize + newesize) > job->fifo.bsize) { + import_wait_for_space_in_fifo( job, newesize ); + } + + /* We have enough space */ + job->fifo.item[idx].filename = ID2ENTRY LDBM_FILENAME_SUFFIX; + job->fifo.item[idx].line = curr_entry; + job->fifo.item[idx].entry = ep; + job->fifo.item[idx].bad = 0; + job->fifo.item[idx].esize = newesize; + + /* Add the entry size to total fifo size */ + job->fifo.c_bsize += ep->ep_entry? job->fifo.item[idx].esize : 0; + + /* Update the job to show our progress */ + job->lead_ID = id; + if ((id - info->first_ID) <= job->fifo.size) { + job->trailing_ID = info->first_ID; + } else { + job->trailing_ID = id - job->fifo.size; + } + + /* Update our progress meter too */ + info->last_ID_processed = id; + id++; + if (job->flags & FLAG_ABORT) + goto error; + if (info->command == STOP) + { + finished = 1; + } + } +bail: + dbc->c_close(dbc); + dblayer_release_aux_id2entry( be, db, env ); + if (job->flags & FLAG_DRYRUN) { + if (finished) { /* Set if dn upgrade candidates are not found */ + info->state = FINISHED; + } else { /* At least one dn upgrade candidate is found */ + info->state = QUIT; + } + } else { + info->state = FINISHED; + } + return; + +error: + dbc->c_close(dbc); + dblayer_release_aux_id2entry( be, db, env ); + info->state = ABORTED; +} + static void import_wait_for_space_in_fifo(ImportJob *job, size_t new_esize) { @@ -1198,6 +1785,39 @@ foreman_do_entrydn(ImportJob *job, FifoItem *fi) int err = 0, ret = 0; IDList *IDL; + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* Get the entrydn attribute value from deleted attr list */ + Slapi_Value *value = NULL; + Slapi_Attr *entrydn_to_del = + attrlist_remove(&fi->entry->ep_entry->e_deleted_attrs, "entrydn"); + + if (entrydn_to_del) { + /* Delete it. */ + ret = slapi_attr_first_value(entrydn_to_del, &value); + if (ret < 0) { + import_log_notice(job, + "Error: retrieving entrydn value (error %d)", + ret); + } else { + const struct berval *bval = + slapi_value_get_berval((const Slapi_Value *)value); + ret = index_addordel_string(be, "entrydn", + bval->bv_val, + fi->entry->ep_id, + BE_INDEX_DEL|BE_INDEX_EQUALITY|BE_INDEX_NORMALIZED, + NULL); + if (ret) { + import_log_notice(job, + "Error: deleting %s from entrydn index " + "(error %d: %s)", + bval->bv_val, ret, dblayer_strerror(ret)); + return ret; + } + } + slapi_attr_free(&entrydn_to_del); + } + } + /* insert into the entrydn index */ bv.bv_val = (void*)backentry_get_ndn(fi->entry); /* jcm - Had to cast away const */ bv.bv_len = strlen(bv.bv_val); @@ -1210,29 +1830,49 @@ foreman_do_entrydn(ImportJob *job, FifoItem *fi) /* So, we do an index read first */ err = 0; IDL = index_read(be, LDBM_ENTRYDN_STR, indextype_EQUALITY, &bv, NULL, &err); - - /* Did this work ? */ - if (NULL != IDL) { - /* IMPOSTER ! Get thee hence... */ - import_log_notice(job, "WARNING: Skipping duplicate entry " - "\"%s\" found at line %d of file \"%s\"", - slapi_entry_get_dn(fi->entry->ep_entry), - fi->line, fi->filename); - idl_free(IDL); - /* skip this one */ - fi->bad = 1; - job->skipped++; - return -1; /* skip to next entry */ - } - if ((ret = index_addordel_string(be, LDBM_ENTRYDN_STR, - bv.bv_val, - fi->entry->ep_id, - BE_INDEX_ADD|BE_INDEX_NORMALIZED, NULL)) != 0) { - import_log_notice(job, "Error writing entrydn index " - "(error %d: %s)", - ret, dblayer_strerror(ret)); - return ret; + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* + * In the UPGRADEDNFORMAT case, if entrydn value exists, + * that means entrydn is not upgraded. And it is normal. + * We could add entrydn only when the value is not found in the db. + */ + if (IDL) { + idl_free(IDL); + } else { + ret = index_addordel_string(be, "entrydn", + bv.bv_val, fi->entry->ep_id, + BE_INDEX_ADD|BE_INDEX_NORMALIZED, NULL); + if (ret) { + import_log_notice(job, "Error writing entrydn index " + "(error %d: %s)", + ret, dblayer_strerror(ret)); + return ret; + } + } + } else { + /* Did this work ? */ + if (IDL) { + /* IMPOSTER ! Get thee hence... */ + import_log_notice(job, "WARNING: Skipping duplicate entry " + "\"%s\" found at line %d of file \"%s\"", + slapi_entry_get_dn(fi->entry->ep_entry), + fi->line, fi->filename); + idl_free(IDL); + /* skip this one */ + fi->bad = 1; + job->skipped++; + return -1; /* skip to next entry */ + } + ret = index_addordel_string(be, "entrydn", bv.bv_val, fi->entry->ep_id, + BE_INDEX_ADD|BE_INDEX_NORMALIZED, NULL); + if (ret) { + import_log_notice(job, "Error writing entrydn index " + "(error %d: %s)", + ret, dblayer_strerror(ret)); + return ret; + } } + return 0; } @@ -1243,6 +1883,34 @@ foreman_do_entryrdn(ImportJob *job, FifoItem *fi) backend *be = job->inst->inst_be; int ret = 0; + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* Get the entrydn attribute value from deleted attr list */ + Slapi_Value *value = NULL; + Slapi_Attr *entryrdn_to_del = NULL; + entryrdn_to_del = attrlist_remove(&fi->entry->ep_entry->e_deleted_attrs, + LDBM_ENTRYRDN_STR); + if (entryrdn_to_del) { + /* Delete it. */ + ret = slapi_attr_first_value(entryrdn_to_del, &value); + if (ret < 0) { + import_log_notice(job, + "Error: retrieving entryrdn value (error %d)", + ret); + } else { + const struct berval *bval = + slapi_value_get_berval((const Slapi_Value *)value); + ret = entryrdn_index_entry(be, fi->entry, BE_INDEX_DEL, NULL); + if (ret) { + import_log_notice(job, + "Error: deleting %s from entrydn index " + "(error %d: %s)", + bval->bv_val, ret, dblayer_strerror(ret)); + return ret; + } + } + slapi_attr_free(&entryrdn_to_del); + } + } if ((ret = entryrdn_index_entry(be, fi->entry, BE_INDEX_ADD, NULL)) != 0) { import_log_notice(job, "Error writing entryrdn index " "(error %d: %s)", @@ -1294,7 +1962,8 @@ import_foreman(void *param) } while ( ((info->command == PAUSE) || (id > job->lead_ID)) && - (info->command != STOP) && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + (info->command != STOP) && (info->command != ABORT) && + !(job->flags & FLAG_ABORT) ) { /* Check to see if we've been told to stop */ info->state = WAITING; DS_Sleep(sleeptime); @@ -1335,9 +2004,9 @@ import_foreman(void *param) * Only check for a parent and add to the entry2dn index if * the entry is not a tombstone. */ - if (job->flags & FLAG_ABORT) { - goto error; - } + if (job->flags & FLAG_ABORT) { + goto error; + } if (parent_status == IMPORT_ADD_OP_ATTRS_NO_PARENT) { /* If this entry is a suffix entry, this is not a problem */ @@ -1386,6 +2055,9 @@ import_foreman(void *param) * (that isn't really an index -- it's the storehouse of the entries * themselves.) */ + /* id2entry_add_ext replaces an entry if it already exists. + * therefore, the Entry ID stays the same. + */ ret = id2entry_add_ext(be, fi->entry, NULL, job->encrypt); if (ret) { /* DB_RUNRECOVERY usually occurs if disk fills */ @@ -1412,7 +2084,10 @@ import_foreman(void *param) goto error; } - if (! slapi_entry_flag_is_set(fi->entry->ep_entry, + if (!(job->flags & FLAG_UPGRADEDNFORMAT) && /* Upgrade dn format mode + does not need to update + parentid index */ + !slapi_entry_flag_is_set(fi->entry->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE)) { /* parentid index * (we have to do this here, because the parentID is dependent on @@ -1427,9 +2102,9 @@ import_foreman(void *param) vlv code to see whether it's within the scope a VLV index. */ vlv_grok_new_import_entry(fi->entry, be); } - if (job->flags & FLAG_ABORT) { - goto error; - } + if (job->flags & FLAG_ABORT) { + goto error; + } /* Remove the entry from the cache (Put in the cache in id2entry_add) */ @@ -1547,7 +2222,8 @@ import_worker(void *param) * thread, and the state is neither STOP nor ABORT */ while (((info->command == PAUSE) || (id > job->ready_ID)) && - (info->command != STOP) && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + (info->command != STOP) && (info->command != ABORT) && + !(job->flags & FLAG_ABORT)) { /* Check to see if we've been told to stop */ info->state = WAITING; DS_Sleep(sleeptime); @@ -1594,6 +2270,43 @@ import_worker(void *param) slapi_pblock_destroy(pb); } else { /* No, process regular index */ + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* Get the attribute value from deleted attr list */ + Slapi_Value *value = NULL; + const struct berval *bval = NULL; + Slapi_Attr *key_to_del = + attrlist_remove(&fi->entry->ep_entry->e_deleted_attrs, + info->index_info->name); + + if (key_to_del) { + int idx = 0; + /* Delete it. */ + for (idx = slapi_attr_first_value(key_to_del, &value); + idx >= 0; + idx = slapi_attr_next_value(key_to_del, idx, + &value)) { + bval = + slapi_value_get_berval((const Slapi_Value *)value); + ret = index_addordel_string(be, + info->index_info->name, + bval->bv_val, + fi->entry->ep_id, + BE_INDEX_DEL|BE_INDEX_EQUALITY| + BE_INDEX_NORMALIZED, + NULL); + if (ret) { + import_log_notice(job, + "Error deleting %s from %s index " + "(error %d: %s)", + bval->bv_val, info->index_info->name, + ret, dblayer_strerror(ret)); + goto error; + } + } + slapi_attr_free(&key_to_del); + } + } + /* Look for the attribute we're indexing and its subtypes */ /* For each attr write to the index */ attrlist_cursor = NULL; @@ -2488,6 +3201,7 @@ import_get_and_add_parent_rdns(ImportWorkerInfo *info, rc = 0; /* assume this is a suffix */ } else { ID pid = (ID)strtol(pid_str, (char **)NULL, 10); + slapi_ch_free_string(&pid_str); rc = import_get_and_add_parent_rdns(info, inst, db, pid, total_id, &mysrdn, curr_entry); if (rc) { diff --git a/ldap/servers/slapd/back-ldbm/import.c b/ldap/servers/slapd/back-ldbm/import.c index 9939b654..0c643721 100644 --- a/ldap/servers/slapd/back-ldbm/import.c +++ b/ldap/servers/slapd/back-ldbm/import.c @@ -51,6 +51,7 @@ #include "import.h" #define ERR_IMPORT_ABORTED -23 +#define DRYRUN_QUIT -24 /********** routines to manipulate the entry fifo **********/ @@ -193,8 +194,16 @@ void import_log_notice(ImportJob *job, char *format, ...) slapi_task_log_notice(job->task, "%s", buffer); } /* also save it in the logs for posterity */ - LDAPDebug(LDAP_DEBUG_ANY, "import %s: %s\n", job->inst->inst_name, - buffer, 0); + if (job->flags & FLAG_UPGRADEDNFORMAT) { + LDAPDebug(LDAP_DEBUG_ANY, "upgradedn %s: %s\n", job->inst->inst_name, + buffer, 0); + } else if (job->flags & FLAG_REINDEXING) { + LDAPDebug(LDAP_DEBUG_ANY, "reindex %s: %s\n", job->inst->inst_name, + buffer, 0); + } else { + LDAPDebug(LDAP_DEBUG_ANY, "import %s: %s\n", job->inst->inst_name, + buffer, 0); + } } static void import_task_destroy(Slapi_Task *task) @@ -245,6 +254,22 @@ static int import_attr_callback(void *node, void *param) ImportJob *job = (ImportJob *)param; struct attrinfo *a = (struct attrinfo *)node; + if (job->flags & FLAG_DRYRUN) { /* dryrun; we don't need the workers */ + return 0; + } + if (job->flags & FLAG_UPGRADEDNFORMAT) { + /* Bring up import workers just for indexes having DN syntax + * attribute type. (except entrydn -- taken care below) */ + int rc = 0; + Slapi_Attr attr = {0}; + slapi_attr_init(&attr, a->ai_type); + rc = slapi_attr_is_dn_syntax_attr(&attr); + attr_done(&attr); + if (0 == rc) { + return 0; + } + } + /* OK, so we now have hold of the attribute structure and the job info, * let's see what we have. Remember that although this function is called * many times, all these calls are in the context of a single thread, so we @@ -255,29 +280,29 @@ static int import_attr_callback(void *node, void *param) * ancestorid indexes because we build those in the foreman thread. */ if (IS_INDEXED(a->ai_indexmask) && - (strcasecmp(a->ai_type, LDBM_ENTRYDN_STR) != 0) && - (strcasecmp(a->ai_type, LDBM_ENTRYRDN_STR) != 0) && - (strcasecmp(a->ai_type, LDBM_PARENTID_STR) != 0) && - (strcasecmp(a->ai_type, LDBM_ANCESTORID_STR) != 0) && - (strcasecmp(a->ai_type, numsubordinates) != 0)) { - /* Make an import_index_info structure, fill it in and insert into the - * job's list */ - IndexInfo *info = CALLOC(IndexInfo); + (strcasecmp(a->ai_type, LDBM_ENTRYDN_STR) != 0) && + (strcasecmp(a->ai_type, LDBM_ENTRYRDN_STR) != 0) && + (strcasecmp(a->ai_type, LDBM_PARENTID_STR) != 0) && + (strcasecmp(a->ai_type, LDBM_ANCESTORID_STR) != 0) && + (strcasecmp(a->ai_type, numsubordinates) != 0)) { + /* Make an import_index_info structure, fill it in and insert into the + * job's list */ + IndexInfo *info = CALLOC(IndexInfo); - if (NULL == info) { - /* Memory allocation error */ - return -1; - } - info->name = slapi_ch_strdup(a->ai_type); - info->ai = a; - if (NULL == info->name) { - /* Memory allocation error */ - FREE(info); - return -1; - } - info->next = job->index_list; - job->index_list = info; - job->number_indexers++; + if (NULL == info) { + /* Memory allocation error */ + return -1; + } + info->name = slapi_ch_strdup(a->ai_type); + info->ai = a; + if (NULL == info->name) { + /* Memory allocation error */ + FREE(info); + return -1; + } + info->next = job->index_list; + job->index_list = info; + job->number_indexers++; } return 0; } @@ -402,12 +427,12 @@ static int import_start_threads(ImportJob *job) import_init_worker_info(foreman, job); foreman->work_type = FOREMAN; if (! CREATE_THREAD(PR_USER_THREAD, (VFP)import_foreman, foreman, - PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, - PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE)) { + PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, + PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE)) { PRErrorCode prerr = PR_GetError(); LDAPDebug(LDAP_DEBUG_ANY, "unable to spawn import foreman thread, " - SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", - prerr, slapd_pr_strerror(prerr), 0); + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); FREE(foreman); goto error; } @@ -659,7 +684,7 @@ static int import_monitor_threads(ImportJob *job, int *status) time_t time_now = 0; time_t last_time = 0; time_t time_interval = 0; - + int rc = 0; for (current_worker = job->worker_list; current_worker != NULL; current_worker = current_worker->next) { @@ -722,7 +747,10 @@ static int import_monitor_threads(ImportJob *job, int *status) import_calc_rate(current_worker, time_interval); import_print_worker_status(current_worker); } - if (current_worker->state != FINISHED) { + if (current_worker->state == QUIT) { + rc = DRYRUN_QUIT; /* Set the RC; Don't abort now; + We have to stop other threads */ + } else if (current_worker->state != FINISHED) { finished = 0; } if (current_worker->state == ABORTED) { @@ -765,8 +793,10 @@ static int import_monitor_threads(ImportJob *job, int *status) /* if the producer is finished, and the foreman has caught up... */ if (producer) { - producer_done = (producer->state == FINISHED); + producer_done = (producer->state == FINISHED) || + (producer->state == QUIT); } else { + /* set in ldbm_back_wire_import */ producer_done = (job->flags & FLAG_PRODUCER_DONE); } if (producer_done && (job->lead_ID == job->ready_ID)) { @@ -802,6 +832,7 @@ static int import_monitor_threads(ImportJob *job, int *status) for (current_worker = job->worker_list; current_worker != NULL; ) { if ((current_worker->state != FINISHED) && (current_worker->state != ABORTED) && + (current_worker->state != QUIT) && (current_worker->work_type != PRODUCER)) { DS_Sleep(tenthsecond); /* Only sleep if we hit a thread that is still not done */ continue; @@ -826,7 +857,7 @@ static int import_monitor_threads(ImportJob *job, int *status) } else { *status = IMPORT_COMPLETE_PASS; } - return 0; + return rc; error_abort: return ERR_IMPORT_ABORTED; @@ -843,16 +874,16 @@ static int import_run_pass(ImportJob *job, int *status) ret = import_start_threads(job); if (ret != 0) { import_log_notice(job, "Starting threads failed: %d\n", ret); - goto error; + goto error; } /* Monitor the threads until we're done or fail */ ret = import_monitor_threads(job, status); - if (ret == ERR_IMPORT_ABORTED) { + if ((ret == ERR_IMPORT_ABORTED) || (ret == DRYRUN_QUIT)) { goto error; } else if (ret != 0) { import_log_notice(job, "Thread monitoring aborted: %d\n", ret); - goto error; + goto error; } error: @@ -876,20 +907,18 @@ static void import_set_abort_flag_all(ImportJob *job, int wait_for_them) /* allow all the aborts to be processed */ DS_Sleep(PR_MillisecondsToInterval(3000)); - if (wait_for_them) { + if (wait_for_them) { /* Having done that, wait for them to say that they've stopped */ for (worker = job->worker_list; worker != NULL; ) { DS_Sleep(PR_MillisecondsToInterval(100)); - if ((worker->state != FINISHED) && - (worker->state != ABORTED)){ + if ((worker->state != FINISHED) && (worker->state != ABORTED) && + (worker->state != QUIT)){ continue; - } - else{ + } else { worker = worker->next; - } + } } } - } @@ -908,11 +937,12 @@ void import_abort_all(ImportJob *job, int wait_for_them) /* Having done that, wait for them to say that they've stopped */ for (worker = job->worker_list; worker != NULL; ) { DS_Sleep(PR_MillisecondsToInterval(100)); - if ((worker->state != FINISHED) && - (worker->state != ABORTED)) + if ((worker->state != FINISHED) && (worker->state != ABORTED) && + (worker->state != QUIT)) { continue; - else + } else { worker = worker->next; + } } } } @@ -1019,13 +1049,13 @@ static int import_all_done(ImportJob *job, int ret) ldbm_instance *inst = job->inst; /* Writing this file indicates to future server startups that - * the db is OK */ - if (ret == 0) { + * the db is OK unless it's in the dry run mode. */ + if ((ret == 0) && !(job->flags & FLAG_DRYRUN)) { char inst_dir[MAXPATHLEN*2]; char *inst_dirp = NULL; inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, inst_dir, MAXPATHLEN*2); - ret = dbversion_write(inst->inst_li, inst_dirp, NULL); + ret = dbversion_write(inst->inst_li, inst_dirp, NULL, DBVERSION_ALL); if (inst_dirp != inst_dir) slapi_ch_free_string(&inst_dirp); } @@ -1078,10 +1108,20 @@ int import_main_offline(void *arg) int verbose = 1; int aborted = 0; ImportWorkerInfo *producer = NULL; + char *opstr = "Import"; if (job->task) slapi_task_inc_refcount(job->task); + if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & FLAG_DRYRUN) { + opstr = "Upgrade Dn Dryrun"; + } else { + opstr = "Upgrade Dn"; + } + } else if (job->flags & FLAG_REINDEXING) { + opstr = "Reindexing"; + } PR_ASSERT(inst != NULL); time(&beginning); @@ -1120,7 +1160,20 @@ int import_main_offline(void *arg) /* start the producer */ import_init_worker_info(producer, job); producer->work_type = PRODUCER; - if (job->flags & FLAG_REINDEXING) + if (job->flags & FLAG_UPGRADEDNFORMAT) + { + if (! CREATE_THREAD(PR_USER_THREAD, (VFP)upgradedn_producer, + producer, PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, + PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE)) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, + "unable to spawn upgrade dn producer thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + goto error; + } + } + else if (job->flags & FLAG_REINDEXING) { if (! CREATE_THREAD(PR_USER_THREAD, (VFP)index_producer, producer, PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, @@ -1128,9 +1181,9 @@ int import_main_offline(void *arg) SLAPD_DEFAULT_THREAD_STACKSIZE)) { PRErrorCode prerr = PR_GetError(); LDAPDebug(LDAP_DEBUG_ANY, - "unable to spawn index producer thread, " - SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", - prerr, slapd_pr_strerror(prerr), 0); + "unable to spawn index producer thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); goto error; } } @@ -1189,15 +1242,18 @@ int import_main_offline(void *arg) if (ret == ERR_IMPORT_ABORTED) { /* at least one of the threads has aborted -- shut down ALL * of the threads */ - import_log_notice(job, "Aborting all import threads..."); + import_log_notice(job, "Aborting all %s threads...", opstr); /* this abort sets the abort flag on the threads and will block for * the exit of all threads */ import_set_abort_flag_all(job, 1); - import_log_notice(job, "Import threads aborted."); + import_log_notice(job, "%s threads aborted.", opstr); aborted = 1; goto error; } + if (ret == DRYRUN_QUIT) { + goto error; /* Found the candidate; close the db files and quit */ + } if (0 != ret) { /* Some horrible fate has befallen the import */ @@ -1297,7 +1353,8 @@ int import_main_offline(void *arg) * Database. */ error: - /* If we fail, the database is now in a mess, so we delete it */ + /* If we fail, the database is now in a mess, so we delete it + except dry run mode */ import_log_notice(job, "Closing files..."); cache_clear(&job->inst->inst_cache, CACHE_TYPE_ENTRY); if (entryrdn_get_switch()) { @@ -1324,7 +1381,10 @@ error: } } if (0 != ret) { - dblayer_delete_instance_dir(be); + if (!(job->flags & FLAG_DRYRUN)) { /* If not dryrun */ + /* if running in the dry run mode, don't touch the db */ + dblayer_delete_instance_dir(be); + } dblayer_instance_close(job->inst->inst_be); } else { if (0 != (ret = dblayer_instance_close(job->inst->inst_be)) ) { @@ -1338,52 +1398,82 @@ error: if (verbose && (0 == ret)) { int seconds_to_import = end - beginning; size_t entries_processed = job->lead_ID - (job->starting_ID - 1); - double entries_per_second = (double) entries_processed / - (double) seconds_to_import; - - if (job->not_here_skipped) - { - if (job->skipped) - import_log_notice(job, "Import complete. Processed %lu entries " - "(%d bad entries were skipped, " - "%d entries were skipped because they don't " - "belong to this database) in %d seconds. " - "(%.2f entries/sec)", entries_processed, - job->skipped, job->not_here_skipped, - seconds_to_import, entries_per_second); - else - import_log_notice(job, "Import complete. Processed %lu entries " - "(%d entries were skipped because they don't " - "belong to this database) " - "in %d seconds. (%.2f entries/sec)", - entries_processed, job->not_here_skipped, - seconds_to_import, entries_per_second); - } - else - { - if (job->skipped) - import_log_notice(job, "Import complete. Processed %lu entries " - "(%d were skipped) in %d seconds. " - "(%.2f entries/sec)", entries_processed, - job->skipped, seconds_to_import, - entries_per_second); - else - import_log_notice(job, "Import complete. Processed %lu entries " - "in %d seconds. (%.2f entries/sec)", - entries_processed, seconds_to_import, - entries_per_second); + double entries_per_second = + seconds_to_import ? + (double)entries_processed / (double)seconds_to_import : 0; + + if (job->not_here_skipped) { + if (job->skipped) { + import_log_notice(job, + "%s complete. Processed %lu entries " + "(%d bad entries were skipped, " + "%d entries were skipped because they don't " + "belong to this database) in %d seconds. " + "(%.2f entries/sec)", + opstr, entries_processed, + job->skipped, job->not_here_skipped, + seconds_to_import, entries_per_second); + } else { + import_log_notice(job, + "%s complete. Processed %lu entries " + "(%d entries were skipped because they don't " + "belong to this database) " + "in %d seconds. (%.2f entries/sec)", + opstr, entries_processed, + job->not_here_skipped, seconds_to_import, + entries_per_second); + } + } else { + if (job->skipped) { + import_log_notice(job, + "%s complete. Processed %lu entries " + "(%d were skipped) in %d seconds. " + "(%.2f entries/sec)", + opstr, entries_processed, + job->skipped, seconds_to_import, + entries_per_second); + } else { + import_log_notice(job, + "%s complete. Processed %lu entries " + "in %d seconds. (%.2f entries/sec)", + opstr, entries_processed, + seconds_to_import, entries_per_second); + } } } - if (0 != ret) { - import_log_notice(job, "Import failed."); + if (job->flags & FLAG_DRYRUN) { + if (0 == ret) { + import_log_notice(job, "%s complete. %s is up-to-date.", + opstr, job->inst->inst_name); + ret = 1; + if (job->task) { + slapi_task_dec_refcount(job->task); + } + import_all_done(job, ret); + } else if (DRYRUN_QUIT == ret) { + import_log_notice(job, "%s complete. %s needs upgradednformat.", + opstr, job->inst->inst_name); + if (job->task) { + slapi_task_dec_refcount(job->task); + } + import_all_done(job, ret); + ret = 0; + } else { + ret = -1; + if (job->task != NULL) { + slapi_task_finish(job->task, ret); + } + } + } else if (0 != ret) { + import_log_notice(job, "%s failed.", opstr); if (job->task != NULL) { slapi_task_finish(job->task, ret); } } else { - if (job->task) + if (job->task) { slapi_task_dec_refcount(job->task); - + } import_all_done(job, ret); } @@ -1393,7 +1483,6 @@ error: import_free_job(job); if (producer) FREE(producer); - return(ret); } @@ -1430,6 +1519,8 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) job->inst = (ldbm_instance *)be->be_instance_info; slapi_pblock_get( pb, SLAPI_LDIF2DB_NOATTRINDEXES, &noattrindexes ); slapi_pblock_get( pb, SLAPI_LDIF2DB_FILE, &name_array ); + slapi_pblock_get(pb, SLAPI_SEQ_TYPE, &up_flags); /* For upgrade dn and + dn2rdn */ /* the removedupvals field is blatantly overloaded here to mean * the chunk size too. (chunk size = number of entries that should @@ -1454,27 +1545,37 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) job->uuid_namespace = slapi_ch_strdup(namespaceid); } - slapi_pblock_get(pb, SLAPI_SEQ_TYPE, &up_flags); job->flags = FLAG_USE_FILES; - if (NULL == name_array) { /* no ldif file is given */ - job->flags |= FLAG_REINDEXING; /* call index_producer */ - if (up_flags & SLAPI_UPGRADEDB_DN2RDN) { - if (entryrdn_get_switch()) { - job->flags |= FLAG_DN2RDN; /* migrate to the rdn format */ - } else { - LDAPDebug1Arg(LDAP_DEBUG_ANY, "DN to RDN option is specified, " - "but %s is not enabled\n", - CONFIG_ENTRYRDN_SWITCH); - import_free_job(job); - FREE(job); - return -1; + if (NULL == name_array) { /* no ldif file is given -> reindexing or + upgradedn */ + if (up_flags & SLAPI_UPGRADEDNFORMAT) { + job->flags |= FLAG_UPGRADEDNFORMAT; + if (up_flags & SLAPI_DRYRUN) { + job->flags |= FLAG_DRYRUN; + } + } else { + job->flags |= FLAG_REINDEXING; /* call index_producer */ + if (up_flags & SLAPI_UPGRADEDB_DN2RDN) { + if (entryrdn_get_switch()) { + job->flags |= FLAG_DN2RDN; /* migrate to the rdn format */ + } else { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "DN to RDN option is specified, " + "but %s is not enabled\n", + CONFIG_ENTRYRDN_SWITCH); + import_free_job(job); + FREE(job); + return -1; + } } } } - if (!noattrindexes) - job->flags |= FLAG_INDEX_ATTRS; - for (i = 0; name_array && name_array[i] != NULL; i++) + if (!noattrindexes) { + job->flags |= FLAG_INDEX_ATTRS; + } + for (i = 0; name_array && name_array[i] != NULL; i++) { charray_add(&job->input_filenames, slapi_ch_strdup(name_array[i])); + } job->starting_ID = 1; job->first_ID = 1; job->mothers = CALLOC(import_subcount_stuff); @@ -1482,10 +1583,10 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) /* how much space should we allocate to index buffering? */ job->job_index_buffer_size = import_get_index_buffer_size(); if (job->job_index_buffer_size == 0) { - /* 10% of the allocated cache size + one meg */ + /* 10% of the allocated cache size + one meg */ PR_Lock(job->inst->inst_li->li_config_mutex); - job->job_index_buffer_size = (job->inst->inst_li->li_import_cachesize/10) + - (1024*1024); + job->job_index_buffer_size = + (job->inst->inst_li->li_import_cachesize/10) + (1024*1024); PR_Unlock(job->inst->inst_li->li_config_mutex); } import_subcount_stuff_init(job->mothers); @@ -1498,12 +1599,13 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) /* add 1 to account for post-import cleanup (which can take a * significant amount of time) */ - /* NGK - This should eventually be cleaned up to use the public - * task API. */ - if (0 == total_files) /* reindexing */ + /* NGK - This should eventually be cleaned up to use the public + * task API. */ + if (0 == total_files) { /* reindexing */ job->task->task_work = 2; - else + } else { job->task->task_work = total_files + 1; + } job->task->task_progress = 0; job->task->task_state = SLAPI_TASK_RUNNING; slapi_task_set_data(job->task, job); diff --git a/ldap/servers/slapd/back-ldbm/import.h b/ldap/servers/slapd/back-ldbm/import.h index e11e1bc5..472f5551 100644 --- a/ldap/servers/slapd/back-ldbm/import.h +++ b/ldap/servers/slapd/back-ldbm/import.h @@ -159,12 +159,14 @@ typedef struct { #define FLAG_INDEX_ATTRS 0x01 /* should we index the attributes? */ #define FLAG_USE_FILES 0x02 /* import from files */ -#define FLAG_PRODUCER_DONE 0x04 /* frontend is done sending entries - * for replica initialization */ -#define FLAG_ABORT 0x08 /* import has been aborted */ -#define FLAG_ONLINE 0x10 /* bring backend online when done */ -#define FLAG_REINDEXING 0x20 /* read from id2entry and do indexing */ -#define FLAG_DN2RDN 0x40 /* modify backend to the rdn format */ +#define FLAG_PRODUCER_DONE 0x04 /* frontend is done sending entries + * for replica initialization */ +#define FLAG_ABORT 0x08 /* import has been aborted */ +#define FLAG_ONLINE 0x10 /* bring backend online when done */ +#define FLAG_REINDEXING 0x20 /* read from id2entry and do indexing */ +#define FLAG_DN2RDN 0x40 /* modify backend to the rdn format */ +#define FLAG_UPGRADEDNFORMAT 0x80 /* read from id2entry and do upgrade dn */ +#define FLAG_DRYRUN 0x100 /* dryrun for upgrade dn */ /* Structure holding stuff about a worker thread and what it's up to */ @@ -198,7 +200,8 @@ struct _import_worker_info { #define WAITING 1 #define RUNNING 2 #define FINISHED 3 -#define ABORTED 4 +#define ABORTED 4 +#define QUIT 5 /* quit intentionally. to distinguish from ABORTED & FINISHED */ /* this is just a convenience, because the slapi_ch_* calls are annoying */ #define CALLOC(name) (name *)slapi_ch_calloc(1, sizeof(name)) @@ -235,5 +238,6 @@ int add_op_attrs(Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *ep, /* import-threads.c */ void import_producer(void *param); void index_producer(void *param); +void upgradedn_producer(void *param); void import_foreman(void *param); void import_worker(void *param); diff --git a/ldap/servers/slapd/back-ldbm/init.c b/ldap/servers/slapd/back-ldbm/init.c index 3bac1090..f1f71840 100644 --- a/ldap/servers/slapd/back-ldbm/init.c +++ b/ldap/servers/slapd/back-ldbm/init.c @@ -227,6 +227,8 @@ ldbm_back_init( Slapi_PBlock *pb ) (void *) ldbm_back_ldbm2archive ); rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_UPGRADEDB_FN, (void *) ldbm_back_upgradedb ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_UPGRADEDNFORMAT_FN, + (void *) ldbm_back_upgradednformat ); rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_DBVERIFY_FN, (void *) ldbm_back_dbverify ); rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_BEGIN_FN, diff --git a/ldap/servers/slapd/back-ldbm/ldbm_entryrdn.c b/ldap/servers/slapd/back-ldbm/ldbm_entryrdn.c index bcf0293d..4e8fa7fe 100644 --- a/ldap/servers/slapd/back-ldbm/ldbm_entryrdn.c +++ b/ldap/servers/slapd/back-ldbm/ldbm_entryrdn.c @@ -999,7 +999,9 @@ entryrdn_lookup_dn(backend *be, static char buffer[RDN_BULK_FETCH_BUFFER_SIZE]; char *keybuf = NULL; Slapi_RDN *srdn = NULL; + char *orignrdn = NULL; char *nrdn = NULL; + size_t nrdn_len = 0; ID workid = id; /* starting from the given id */ ID previd = id; rdn_elem *elem = NULL; @@ -1037,8 +1039,18 @@ entryrdn_lookup_dn(backend *be, goto bail; } srdn = slapi_rdn_new_all_dn(rdn); - nrdn = slapi_ch_strdup(rdn); - slapi_dn_normalize_case(nrdn); /* normalize in place */ + orignrdn = slapi_ch_strdup(rdn); + rc = slapi_dn_normalize_case_ext(orignrdn, 0, &nrdn, &nrdn_len); + if (rc < 0) { + slapi_log_error(SLAPI_LOG_FATAL, ENTRYRDN_TAG, + "entryrdn_get_parent: Failed to normalize %s\n", rdn); + goto bail; + } + if (rc == 0) { /* orignrdn is passed in */ + *(nrdn + nrdn_len) = '\0'; + } else { + slapi_ch_free_string(&orignrdn); + } /* Setting the bulk fetch buffer */ memset(&data, 0, sizeof(data)); @@ -1159,7 +1171,9 @@ entryrdn_get_parent(backend *be, DB_TXN *db_txn = (txn != NULL) ? txn->back_txn_txn : NULL; DBT key, data; char *keybuf = NULL; + char *orignrdn = NULL; char *nrdn = NULL; + size_t nrdn_len = 0; rdn_elem *elem = NULL; slapi_log_error(SLAPI_LOG_TRACE, ENTRYRDN_TAG, @@ -1195,8 +1209,18 @@ entryrdn_get_parent(backend *be, cursor = NULL; goto bail; } - nrdn = slapi_ch_strdup(rdn); - slapi_dn_normalize_case(nrdn); /* normalize in place */ + orignrdn = slapi_ch_strdup(rdn); + rc = slapi_dn_normalize_case_ext(orignrdn, 0, &nrdn, &nrdn_len); + if (rc < 0) { + slapi_log_error(SLAPI_LOG_FATAL, ENTRYRDN_TAG, + "entryrdn_get_parent: Failed to normalize %s\n", rdn); + goto bail; + } + if (rc == 0) { /* orignrdn is passed in */ + *(nrdn + nrdn_len) = '\0'; + } else { + slapi_ch_free_string(&orignrdn); + } /* Setting the bulk fetch buffer */ memset(&data, 0, sizeof(data)); diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c index 76404d5a..2858f5b1 100644 --- a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c +++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c @@ -2438,7 +2438,7 @@ ldbm_exclude_attr_from_export( struct ldbminfo *li , const char *attr, void upgradedb_core(Slapi_PBlock *pb, ldbm_instance *inst); int upgradedb_copy_logfiles(struct ldbminfo *li, char *destination_dir, int restore, int *cnt); int upgradedb_delete_indices_4cmd(ldbm_instance *inst, int flags); -void normalize_dir(char *dir); +static void normalize_dir(char *dir); /* * ldbm_back_upgradedb - @@ -2467,6 +2467,7 @@ int ldbm_back_upgradedb(Slapi_PBlock *pb) char *inst_dirp = NULL; int cnt = 0; PRFileInfo info = {0}; + PRUint32 dbversion_flags = DBVERSION_ALL; slapi_pblock_get(pb, SLAPI_SEQ_TYPE, &up_flags); slapi_log_error(SLAPI_LOG_TRACE, "upgrade DB", "Reindexing all...\n"); @@ -2685,15 +2686,19 @@ int ldbm_back_upgradedb(Slapi_PBlock *pb) home_dir = dblayer_get_home_dir(li, NULL); /* write db version files */ - dbversion_write(li, home_dir, NULL); + dbversion_write(li, home_dir, NULL, DBVERSION_ALL); + if ((up_flags & SLAPI_UPGRADEDB_DN2RDN) && entryrdn_get_switch()) { + /* exclude dnformat to allow upgradednformat later */ + dbversion_flags = DBVERSION_ALL ^ DBVERSION_DNFORMAT;; + } inst_obj = objset_first_obj(li->li_instance_set); while (NULL != inst_obj) { char *inst_dirp = NULL; inst_dirp = dblayer_get_full_inst_dir(li, inst, inst_dir, MAXPATHLEN); inst = (ldbm_instance *)object_get_data(inst_obj); - dbversion_write(li, inst_dirp, NULL); + dbversion_write(li, inst_dirp, NULL, dbversion_flags); inst_obj = objset_next_obj(li->li_instance_set, inst_obj); if (inst_dirp != inst_dir) slapi_ch_free_string(&inst_dirp); @@ -2781,15 +2786,26 @@ fail0: return rval; } -void normalize_dir(char *dir) +static void +normalize_dir(char *dir) { - int l = strlen(dir); - if ('/' == dir[l-1] || '\\' == dir[l-1]) - { - dir[l-1] = '\0'; + char *p = NULL; + int l = 0; + + if (NULL == dir) { + return; } + l = strlen(dir); + + for (p = dir + l - 1; p && *p; p--) { + if (' ' != *p && '\t' != *p && '/' != *p && '\\' != *p) { + break; + } + } + *(p+1) = '\0'; } + #define LOG "log." #define LOGLEN 4 int upgradedb_copy_logfiles(struct ldbminfo *li, char *destination_dir, @@ -3330,3 +3346,208 @@ bail: slapi_ch_free_string(&prdn); return rc; } + +/* + * ldbm_back_upgradednformat + * + * Update old DN format in entrydn and the leaf attr value to the new one + * + * The implementation would be similar to the upgradedb for new idl. + * Scan each entry, checking the entrydn value with the result of + * slapi_dn_normalize_ext_case(dn). + * If they don't match, + * replace the old entrydn value with the new one in the entry + * in id2entry.db4. + * also get the leaf RDN attribute value, unescape it, and check + * if it is in the entry. If not, add it. + * Then, update the key in the entrydn index and the leaf RDN attribute + * (if need it). + * + * Return value: 0: success (the backend instance includes update + * candidates for DRYRUN mode) + * 1: the backend instance is up-to-date (DRYRUN mode only) + * -1: error + * + * standalone only -- not allowed to run while DS is up. + */ +int ldbm_back_upgradednformat(Slapi_PBlock *pb) +{ + int rc = -1; + struct ldbminfo *li = NULL; + int run_from_cmdline = 0; + int task_flags = 0; + int server_running = 0; + Slapi_Task *task; + ldbm_instance *inst = NULL; + char *instance_name = NULL; + backend *be = NULL; + PRStatus prst = 0; + PRFileInfo prfinfo = {0}; + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + size_t id2entrylen = 0; + int found = 0; + char *rawworkdbdir = NULL; + char *workdbdir = NULL; + char *origdbdir = NULL; + char *origlogdir = NULL; + char *originstparentdir = NULL; + char *sep = NULL; + char *ldbmversion = NULL; + char *dataversion = NULL; + int ud_flags = 0; + + slapi_pblock_get(pb, SLAPI_TASK_FLAGS, &task_flags); + slapi_pblock_get(pb, SLAPI_BACKEND_TASK, &task); + slapi_pblock_get(pb, SLAPI_DB2LDIF_SERVER_RUNNING, &server_running); + slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name); + slapi_pblock_get( pb, SLAPI_SEQ_TYPE, &ud_flags ); + + run_from_cmdline = (task_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE); + slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &li); + if (run_from_cmdline) { + ldbm_config_load_dse_info(li); + if (check_and_set_import_cache(li) < 0) { + return -1; + } + } else { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + " Online mode is not supported. " + "Shutdown the server and run the tool\n"); + goto bail; + } + + /* Find the instance that the ldif2db will be done on. */ + inst = ldbm_instance_find_by_name(li, instance_name); + if (NULL == inst) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Unknown ldbm instance %s\n", instance_name); + goto bail; + } + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "%s: Start upgrade dn format.\n", inst->inst_name); + + slapi_pblock_set(pb, SLAPI_BACKEND, inst->inst_be); + slapi_pblock_get(pb, SLAPI_SEQ_VAL, &rawworkdbdir); + normalize_dir(rawworkdbdir); /* remove trailing spaces and slashes */ + + prst = PR_GetFileInfo(rawworkdbdir, &prfinfo); + if (PR_FAILURE == prst || PR_FILE_DIRECTORY != prfinfo.type) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Working DB instance dir %s is not a directory\n", + rawworkdbdir); + goto bail; + } + dirhandle = PR_OpenDir(rawworkdbdir); + if (!dirhandle) + { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Failed to open working DB instance dir %s\n", + rawworkdbdir); + goto bail; + } + id2entrylen = strlen(ID2ENTRY); + while ((direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { + if (!direntry->name) + break; + if (0 == strncasecmp(ID2ENTRY, direntry->name, id2entrylen)) { + found = 1; + break; + } + } + PR_CloseDir(dirhandle); + + if (!found) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Working DB instance dir %s does not include %s file\n", + rawworkdbdir, ID2ENTRY); + goto bail; + } + + if (run_from_cmdline) { + ldbm_config_internal_set(li, CONFIG_DB_TRANSACTION_LOGGING, "off"); + } + + /* We have to work on the copied db. So, the path should be set here. */ + origdbdir = li->li_directory; + origlogdir = li->li_dblayer_private->dblayer_log_directory; + originstparentdir = inst->inst_parent_dir_name; + + workdbdir = rel2abspath(rawworkdbdir); + + dbversion_read(li, workdbdir, &ldbmversion, &dataversion); + if (ldbmversion && PL_strstr(ldbmversion, BDB_DNFORMAT)) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Instance %s in %s is up-to-date\n", + instance_name, workdbdir); + rc = 1; /* 1: up-to-date; 0: need upgrade; otherwise: error */ + goto bail; + } + + sep = PL_strrchr(workdbdir, '/'); + if (!sep) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Working DB instance dir %s does not include %s file\n", + workdbdir, ID2ENTRY); + goto bail; + } + *sep = '\0'; + li->li_directory = workdbdir; + li->li_dblayer_private->dblayer_log_directory = workdbdir; + inst->inst_parent_dir_name = workdbdir; + + if (run_from_cmdline) { + if (0 != dblayer_start(li, DBLAYER_IMPORT_MODE)) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Failed to init database\n"); + goto bail; + } + } + + /* dblayer_instance_start will init the id2entry index. */ + be = inst->inst_be; + if (0 != dblayer_instance_start(be, DBLAYER_IMPORT_MODE)) + { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DB Format", + "Failed to init instance %s\n", inst->inst_name); + goto bail; + } + + if (run_from_cmdline) { + vlv_init(inst); /* Initialise the Virtual List View code */ + } + + rc = ldbm_back_ldif2ldbm_deluxe(pb); + + /* close the database */ + if (run_from_cmdline) { + if (0 != dblayer_flush(li)) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Failed to flush database\n"); + } + if (0 != dblayer_close(li,DBLAYER_IMPORT_MODE)) { + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Failed to close database\n"); + goto bail; + } + } + *sep = '/'; + if (((0 == rc) && !(ud_flags & SLAPI_DRYRUN)) || + ((rc > 0) && (ud_flags & SLAPI_DRYRUN))) { + /* modify the DBVERSION files if the DN upgrade was successful OR + * if DRYRUN, the backend instance is up-to-date. */ + dbversion_write(li, workdbdir, NULL, DBVERSION_ALL); /* inst db dir */ + } + /* Remove the DB env files */ + dblayer_remove_env(li); + + li->li_directory = origdbdir; + li->li_dblayer_private->dblayer_log_directory = origlogdir; + inst->inst_parent_dir_name = originstparentdir; + +bail: + slapi_ch_free_string(&workdbdir); + slapi_ch_free_string(&ldbmversion); + slapi_ch_free_string(&dataversion); + return rc; +} diff --git a/ldap/servers/slapd/back-ldbm/misc.c b/ldap/servers/slapd/back-ldbm/misc.c index 759e37f3..ad7128de 100644 --- a/ldap/servers/slapd/back-ldbm/misc.c +++ b/ldap/servers/slapd/back-ldbm/misc.c @@ -482,3 +482,99 @@ bail: slapi_ch_free_string(©); return rc; } + +/* + * Get value array of type from string. + * multi-value support for get_value_from_string + */ +/* caller is responsible to release "valuearray" */ +int +get_values_from_string(const char *string, char *type, char ***valuearray) +{ + int rc = -1; + size_t typelen = 0; + char *ptr = NULL; + char *copy = NULL; + char *tmpptr = NULL; + char *tmptype = NULL; + char *valueptr = NULL; +#if defined (USE_OPENLDAP) + ber_len_t valuelen; +#else + int valuelen; +#endif + char *value = NULL; + int idx = 0; +#define get_values_INITIALMAXCNT 1 + int maxcnt = get_values_INITIALMAXCNT; + + if (NULL == string || NULL == type || NULL == valuearray) { + return rc; + } + *valuearray = NULL; + tmpptr = (char *)string; + ptr = PL_strcasestr(tmpptr, type); + if (NULL == ptr) { + return rc; + } + + typelen = strlen(type); + while (NULL != (ptr = ldif_getline(&tmpptr))) { + if ((0 != PL_strncasecmp(ptr, type, typelen)) || + (*(ptr + typelen) != ';' && *(ptr + typelen) != ':')) { + /* did not match */ + /* ldif_getline replaces '\n' and '\r' with '\0' */ + if ('\0' == *(tmpptr - 1)) { + *(tmpptr - 1) = '\n'; + } + if ('\0' == *(tmpptr - 2)) { + *(tmpptr - 2) = '\r'; + } + continue; + } + /* matched */ + copy = slapi_ch_strdup(ptr); + /* ldif_getline replaces '\n' and '\r' with '\0' */ + if ('\0' == *(tmpptr - 1)) { + *(tmpptr - 1) = '\n'; + } + if ('\0' == *(tmpptr - 2)) { + *(tmpptr - 2) = '\r'; + } + rc = ldif_parse_line(copy, &tmptype, &valueptr, &valuelen); + if (0 > rc || NULL == valueptr || 0 >= valuelen) { + continue; + } + if (0 != strcasecmp(type, tmptype)) { + char *p = PL_strchr(tmptype, ';'); /* subtype ? */ + if (p) { + if (0 != strncasecmp(type, tmptype, p - tmptype)) { + slapi_log_error(SLAPI_LOG_FATAL, "get_values_from_string", + "type does not match: %s != %s\n", + type, tmptype); + goto bail; + } + } else { + slapi_log_error(SLAPI_LOG_FATAL, "get_values_from_string", + "type does not match: %s != %s\n", + type, tmptype); + goto bail; + } + } + value = (char *)slapi_ch_malloc(valuelen + 1); + memcpy(value, valueptr, valuelen); + *(value + valuelen) = '\0'; + if ((get_values_INITIALMAXCNT == maxcnt) || !valuearray || + (idx + 1 >= maxcnt)) { + maxcnt *= 2; + *valuearray = (char **)slapi_ch_realloc((char *)*valuearray, + sizeof(char *) * maxcnt); + } + (*valuearray)[idx++] = value; + (*valuearray)[idx] = NULL; + slapi_ch_free_string(©); + } +bail: + slapi_ch_free_string(©); + return rc; +} diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h index 41fd54cf..12361f42 100644 --- a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h @@ -171,7 +171,7 @@ int dblayer_db_uses_transactions(DB_ENV *db_env); int dblayer_db_uses_mpool(DB_ENV *db_env); int dblayer_db_uses_logging(DB_ENV *db_env); int dblayer_bt_compare(DB *db, const DBT *dbt1, const DBT *dbt2); - +int dblayer_remove_env(struct ldbminfo *li); /* * dn2entry.c @@ -361,6 +361,7 @@ int mkdir_p(char *dir, unsigned int mode); int is_fullpath(char *path); char get_sep(char *path); int get_value_from_string(const char *string, char *type, char **value); +int get_values_from_string(const char *string, char *type, char ***valuearray); /* @@ -465,6 +466,7 @@ int ldbm_back_ldif2ldbm( Slapi_PBlock *pb ); int ldbm_back_ldbm2ldif( Slapi_PBlock *pb ); int ldbm_back_ldbm2ldifalt( Slapi_PBlock *pb ); int ldbm_back_ldbm2index( Slapi_PBlock *pb ); +int ldbm_back_upgradednformat( Slapi_PBlock *pb ); int ldbm_back_archive2ldbm( Slapi_PBlock *pb ); int ldbm_back_ldbm2archive( Slapi_PBlock *pb ); int ldbm_back_upgradedb( Slapi_PBlock *pb ); @@ -586,7 +588,7 @@ int ldbm_attribute_always_indexed(const char *attrtype); /* * dbversion.c */ -int dbversion_write(struct ldbminfo *li, const char *dir, const char *dversion); +int dbversion_write(struct ldbminfo *li, const char *directory, const char *dataversion, PRUint32 flags); int dbversion_read(struct ldbminfo *li, const char *directory, char **ldbmversion, char **dataversion); int dbversion_exists(struct ldbminfo *li, const char *directory); diff --git a/ldap/servers/slapd/back-ldbm/start.c b/ldap/servers/slapd/back-ldbm/start.c index 49109c3b..06bf7deb 100644 --- a/ldap/servers/slapd/back-ldbm/start.c +++ b/ldap/servers/slapd/back-ldbm/start.c @@ -205,7 +205,7 @@ ldbm_back_start( Slapi_PBlock *pb ) home_dir = dblayer_get_home_dir(li, NULL); if (!dbversion_exists(li, home_dir)) { - dbversion_write (li, home_dir, NULL); + dbversion_write (li, home_dir, NULL, DBVERSION_ALL); } diff --git a/ldap/servers/slapd/delete.c b/ldap/servers/slapd/delete.c index 3a80eeba..acb2e440 100644 --- a/ldap/servers/slapd/delete.c +++ b/ldap/servers/slapd/delete.c @@ -113,7 +113,7 @@ do_delete( Slapi_PBlock *pb ) } err = slapi_dn_normalize_ext(rawdn, 0, &dn, &dnlen); if (err < 0) { - op_shared_log_error_access(pb, "DEL", "???", "invalid dn"); + op_shared_log_error_access(pb, "DEL", rawdn?rawdn:"", "invalid dn"); send_ldap_result(pb, LDAP_INVALID_DN_SYNTAX, NULL, "invalid dn", 0, NULL); slapi_ch_free_string(&rawdn); diff --git a/ldap/servers/slapd/dn.c b/ldap/servers/slapd/dn.c index f30578f6..bd229d72 100644 --- a/ldap/servers/slapd/dn.c +++ b/ldap/servers/slapd/dn.c @@ -449,7 +449,6 @@ static int ISEOV(char *s, char *ends) { char *p; - int rc = 1; for (p = s; p && *p && p < ends; p++) { if (SEPARATOR(*p)) { return 1; @@ -1159,6 +1158,17 @@ rdn_av_swap( struct berval *av1, struct berval *av2, int escape ) } } +/* Introduced for the upgrade tool. DON'T USE THIS API! */ +char * +slapi_dn_normalize_case_original( char *dn ) +{ + /* LDAPDebug( LDAP_DEBUG_TRACE, "=> slapi_dn_normalize \"%s\"\n", dn, 0, 0 ); */ + *(substr_dn_normalize_orig( dn, dn + strlen( dn ))) = '\0'; + /* LDAPDebug( LDAP_DEBUG_TRACE, "<= slapi_dn_normalize \"%s\"\n", dn, 0, 0 ); */ + + /* normalize case */ + return( slapi_dn_ignore_case( dn )); +} /* * DEPRECATED: this function does nothing. diff --git a/ldap/servers/slapd/main.c b/ldap/servers/slapd/main.c index 3772a757..322f1b8f 100644 --- a/ldap/servers/slapd/main.c +++ b/ldap/servers/slapd/main.c @@ -113,6 +113,7 @@ static int slapd_exemode_db2index(); static int slapd_exemode_archive2db(); static int slapd_exemode_db2archive(); static int slapd_exemode_upgradedb(); +static int slapd_exemode_upgradednformat(); static int slapd_exemode_dbverify(); static int slapd_exemode_dbtest(); static int slapd_exemode_suffix2instance(); @@ -374,14 +375,16 @@ name2exemode( char *progname, char *s, int exit_if_unknown ) exemode = SLAPD_EXEMODE_SUFFIX2INSTANCE; } else if ( strcmp( s, "upgradedb" ) == 0 ) { exemode = SLAPD_EXEMODE_UPGRADEDB; + } else if ( strcmp( s, "upgradednformat" ) == 0 ) { + exemode = SLAPD_EXEMODE_UPGRADEDNFORMAT; } else if ( strcmp( s, "dbverify" ) == 0 ) { exemode = SLAPD_EXEMODE_DBVERIFY; } else if ( exit_if_unknown ) { fprintf( stderr, "usage: %s -D configdir " "[ldif2db | db2ldif | archive2db " - "| db2archive | db2index | refer | suffix2instance" - " | upgradedb | dbverify] " + "| db2archive | db2index | refer | suffix2instance " + "| upgradedb | upgradednformat | dbverify] " "[options]\n", progname ); exit( 1 ); } else { @@ -441,6 +444,9 @@ usage( char *name, char *extraname ) case SLAPD_EXEMODE_UPGRADEDB: usagestr = "usage: %s %s%s-D configdir [-d debuglevel] [-f] [-r] -a archivedir\n"; break; + case SLAPD_EXEMODE_UPGRADEDNFORMAT: + usagestr = "usage: %s %s%s-D configdir [-d debuglevel] [-N] -n backend-instance-name -a fullpath-backend-instance-dir-full\n"; + break; case SLAPD_EXEMODE_DBVERIFY: usagestr = "usage: %s %s%s-D configdir [-d debuglevel] [-n backend-instance-name]\n"; break; @@ -484,6 +490,7 @@ static int dbverify_verbose = 0; static char *ldif2db_namespaceid = NULL; int importexport_encrypt = 0; static int upgradedb_flags = 0; +static int upgradednformat_dryrun = 0; /* taken from idsktune */ #if defined(__sun) @@ -990,6 +997,11 @@ main( int argc, char **argv) goto cleanup; break; + case SLAPD_EXEMODE_UPGRADEDNFORMAT: + return_value = slapd_exemode_upgradednformat(); + goto cleanup; + break; + case SLAPD_EXEMODE_DBVERIFY: return_value = slapd_exemode_dbverify(); goto cleanup; @@ -1416,6 +1428,16 @@ process_command_line(int argc, char **argv, char *myname, {"configDir",ArgRequired,'D'}, {0,0,0}}; + char *opts_upgradednformat = "vd:a:n:D:N"; + struct opt_ext long_options_upgradednformat[] = { + {"version",ArgNone,'v'}, + {"debug",ArgRequired,'d'}, + {"backend",ArgRequired,'n'}, + {"archive",ArgRequired,'a'}, /* Path to the work db instance dir */ + {"configDir",ArgRequired,'D'}, + {"dryrun",ArgNone,'N'}, + {0,0,0}}; + char *opts_dbverify = "vVfd:n:D:"; struct opt_ext long_options_dbverify[] = { {"version",ArgNone,'v'}, @@ -1521,6 +1543,10 @@ process_command_line(int argc, char **argv, char *myname, opts = opts_upgradedb; long_opts = long_options_upgradedb; break; + case SLAPD_EXEMODE_UPGRADEDNFORMAT: + opts = opts_upgradednformat; + long_opts = long_options_upgradednformat; + break; case SLAPD_EXEMODE_DBVERIFY: opts = opts_dbverify; long_opts = long_options_dbverify; @@ -1623,6 +1649,7 @@ process_command_line(int argc, char **argv, char *myname, break; case 'n': /* which backend to do ldif2db/bak2db for */ if (slapd_exemode == SLAPD_EXEMODE_LDIF2DB || + slapd_exemode == SLAPD_EXEMODE_UPGRADEDNFORMAT || slapd_exemode == SLAPD_EXEMODE_DBTEST || slapd_exemode == SLAPD_EXEMODE_DB2INDEX || slapd_exemode == SLAPD_EXEMODE_ARCHIVE2DB) { @@ -1665,8 +1692,10 @@ process_command_line(int argc, char **argv, char *myname, db2ldif_dump_replica = 1; break; case 'N': /* do not do ldif2db duplicate value check */ + /* Or dryrun mode for upgradednformat */ if ( slapd_exemode != SLAPD_EXEMODE_LDIF2DB && - slapd_exemode != SLAPD_EXEMODE_DB2LDIF) { + slapd_exemode != SLAPD_EXEMODE_DB2LDIF && + slapd_exemode != SLAPD_EXEMODE_UPGRADEDNFORMAT) { usage( myname, *extraname ); exit( 1 ); } @@ -1681,6 +1710,9 @@ process_command_line(int argc, char **argv, char *myname, if ( slapd_exemode == SLAPD_EXEMODE_DB2LDIF ) { ldif_printkey &= ~EXPORT_PRINTKEY; } + if ( slapd_exemode == SLAPD_EXEMODE_UPGRADEDNFORMAT ) { + upgradednformat_dryrun = 1; + } break; @@ -2171,7 +2203,7 @@ slapd_exemode_ldif2db() } /* check for slapi v2 support */ if (! SLAPI_PLUGIN_IS_V2(plugin)) { - LDAPDebug(LDAP_DEBUG_ANY, "ERROR: %s is too old to do imports.\n", + LDAPDebug(LDAP_DEBUG_ANY, "ERROR: %s is too old to reindex all.\n", plugin->plg_name, 0, 0); return 1; } @@ -2686,6 +2718,83 @@ slapd_exemode_upgradedb() return( return_value ); } +/* Command to upgrade the old dn format to the new style */ +static int +slapd_exemode_upgradednformat() +{ + int rc = -1; /* error, by default */ + Slapi_PBlock pb; + struct slapdplugin *backend_plugin; + slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig(); + + if ( archive_name == NULL ) { + LDAPDebug0Args(LDAP_DEBUG_ANY, "ERROR: Required argument " + "\"-a <path to work db instance dir>\" is missing\n"); + usage( myname, extraname ); + goto bail; + } + + /* this should be the first time to be called! if the init order + * is ever changed, these lines should be changed (or erased)! + */ + mapping_tree_init(); + + if ((backend_plugin = plugin_get_by_name("ldbm database")) == NULL) { + LDAPDebug0Args(LDAP_DEBUG_ANY, + "ERROR: Could not find the ldbm backend plugin.\n"); + goto bail; + } + + /* Make sure we aren't going to run slapd in + * a mode that is going to conflict with other + * slapd processes that are currently running + * Pretending to execute import. + */ + if (add_new_slapd_process(slapd_exemode, 0, skip_db_protect_check) + == -1) { + LDAPDebug0Args(LDAP_DEBUG_ANY, "Shutting down due to possible " + "conflicts with other slapd processes\n"); + goto bail; + } + /* check for slapi v2 support */ + if (! SLAPI_PLUGIN_IS_V2(backend_plugin)) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "ERROR: %s is too old to upgrade dn format.\n", + backend_plugin->plg_name); + goto bail; + } + + memset( &pb, '\0', sizeof(pb) ); + pb.pb_backend = NULL; + pb.pb_plugin = backend_plugin; + pb.pb_instance_name = cmd_line_instance_name; + if (upgradednformat_dryrun) { + pb.pb_seq_type = SLAPI_UPGRADEDNFORMAT|SLAPI_DRYRUN; + } else { + pb.pb_seq_type = SLAPI_UPGRADEDNFORMAT; + } + pb.pb_seq_val = archive_name; /* Path to the work db instance dir */ + pb.pb_task_flags = SLAPI_TASK_RUNNING_FROM_COMMANDLINE; + /* borrowing import code, so need to set up the import variables */ + pb.pb_ldif_generate_uniqueid = ldif2db_generate_uniqueid; + pb.pb_ldif_namespaceid = ldif2db_namespaceid; + pb.pb_ldif2db_noattrindexes = 0; + pb.pb_removedupvals = 0; +#ifndef _WIN32 + main_setuid(slapdFrontendConfig->localuser); +#endif + if ( backend_plugin->plg_upgradednformat != NULL ) { + rc = (*backend_plugin->plg_upgradednformat)( &pb ); + } else { + LDAPDebug( LDAP_DEBUG_ANY, + "ERROR: no upgradednformat function defined for " + "%s\n", backend_plugin->plg_name, 0, 0 ); + } +bail: + slapi_ch_free((void**)&myname ); + return( rc ); +} + /* * function to perform DB verify */ diff --git a/ldap/servers/slapd/mapping_tree.c b/ldap/servers/slapd/mapping_tree.c index 7503929d..a67ec2ac 100644 --- a/ldap/servers/slapd/mapping_tree.c +++ b/ldap/servers/slapd/mapping_tree.c @@ -1080,6 +1080,15 @@ int mapping_tree_entry_modify_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefor { parent_node = mapping_tree_root; } + else if ((strcasecmp(mods[i]->mod_type, "cn") == 0) && + SLAPI_IS_MOD_ADD(mods[i]->mod_op)) + { + /* Allow to add an additional cn. + * e.g., cn: "<suffix>" for the backward compatibility. + * No need to update the mapping tree node itself. + */ + continue; + } else { /* we have to find the new parent node */ @@ -2684,6 +2693,11 @@ mtn_get_mapping_tree_node_by_entry(mapping_tree_node* node, const Slapi_DN *dn) return NULL; } + if(NULL == dn){ + /* bad mapping tree entry operation */ + return NULL; + } + if (slapi_sdn_compare(node->mtn_subtree, dn) == 0) { return node; diff --git a/ldap/servers/slapd/modify.c b/ldap/servers/slapd/modify.c index 027bbefb..955ef820 100644 --- a/ldap/servers/slapd/modify.c +++ b/ldap/servers/slapd/modify.c @@ -193,7 +193,7 @@ do_modify( Slapi_PBlock *pb ) } rc = slapi_dn_normalize_ext(rawdn, 0, &dn, &dnlen); if (rc < 0) { - op_shared_log_error_access(pb, "MOD", "???", "invalid dn"); + op_shared_log_error_access(pb, "MOD", rawdn?rawdn:"", "invalid dn"); send_ldap_result(pb, LDAP_INVALID_DN_SYNTAX, NULL, "invalid dn", 0, NULL); slapi_ch_free((void **) &rawdn); diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c index 7d885745..dd126976 100644 --- a/ldap/servers/slapd/pblock.c +++ b/ldap/servers/slapd/pblock.c @@ -666,6 +666,12 @@ slapi_pblock_get( Slapi_PBlock *pblock, int arg, void *value ) } (*(IFP *)value) = pblock->pb_plugin->plg_upgradedb; break; + case SLAPI_PLUGIN_DB_UPGRADEDNFORMAT_FN: + if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_DATABASE ) { + return( -1 ); + } + (*(IFP *)value) = pblock->pb_plugin->plg_upgradednformat; + break; case SLAPI_PLUGIN_DB_DBVERIFY_FN: if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_DATABASE ) { return( -1 ); @@ -2041,6 +2047,12 @@ slapi_pblock_set( Slapi_PBlock *pblock, int arg, void *value ) } pblock->pb_plugin->plg_upgradedb = (IFP) value; break; + case SLAPI_PLUGIN_DB_UPGRADEDNFORMAT_FN: + if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_DATABASE ) { + return( -1 ); + } + pblock->pb_plugin->plg_upgradednformat = (IFP) value; + break; case SLAPI_PLUGIN_DB_DBVERIFY_FN: if ( pblock->pb_plugin->plg_type != SLAPI_PLUGIN_DATABASE ) { return( -1 ); diff --git a/ldap/servers/slapd/protect_db.c b/ldap/servers/slapd/protect_db.c index a6ff87a8..c90c8001 100644 --- a/ldap/servers/slapd/protect_db.c +++ b/ldap/servers/slapd/protect_db.c @@ -490,6 +490,15 @@ add_new_slapd_process(int exec_mode, int r_flag, int skip_flag) result = 0; } break; + case SLAPD_EXEMODE_UPGRADEDNFORMAT: + if (running || importing || exporting) { + LDAPDebug(LDAP_DEBUG_ANY, NO_UPGRADEDNFORMAT_DUE_TO_USE, 0, 0, 0); + result = -1; + } else { + add_this_process_to(import_dir); + result = 0; + } + break; case SLAPD_EXEMODE_DBTEST: if (running || importing || exporting) { LDAPDebug(LDAP_DEBUG_ANY, NO_DBTEST_DUE_TO_USE, 0, 0, 0); diff --git a/ldap/servers/slapd/protect_db.h b/ldap/servers/slapd/protect_db.h index 47b84c69..1def34c9 100644 --- a/ldap/servers/slapd/protect_db.h +++ b/ldap/servers/slapd/protect_db.h @@ -102,6 +102,8 @@ void remove_slapd_process(); #define NO_UPGRADEDB_DUE_TO_USE "Unable to recreate index files because the database is being used by another slapd process.\n" +#define NO_UPGRADEDNFORMAT_DUE_TO_USE "Unable to upgrade dn format because the database is being used by another slapd process.\n" + #define CREATE_MUTEX_ERROR "Error - CreateMutex failed: %s\n" /* reason for failure */ diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h index ba550c72..7e60ca55 100644 --- a/ldap/servers/slapd/slap.h +++ b/ldap/servers/slapd/slap.h @@ -74,6 +74,7 @@ static char ptokDes[34] = "Internal (Software) Token "; #define SLAPD_EXEMODE_PRINTVERSION 10 #define SLAPD_EXEMODE_UPGRADEDB 11 #define SLAPD_EXEMODE_DBVERIFY 12 +#define SLAPD_EXEMODE_UPGRADEDNFORMAT 13 #ifdef _WIN32 #ifndef DONT_DECLARE_SLAPD_LDAP_DEBUG @@ -824,6 +825,7 @@ struct slapdplugin { IFP plg_un_db_archive2db; /* ldif 2 database */ IFP plg_un_db_db2archive; /* database 2 ldif */ IFP plg_un_db_upgradedb; /* convert old idl to new */ + IFP plg_un_db_upgradednformat; /* convert old dn format to new */ IFP plg_un_db_begin; /* dbase txn begin */ IFP plg_un_db_commit; /* dbase txn commit */ IFP plg_un_db_abort; /* dbase txn abort */ @@ -863,6 +865,7 @@ struct slapdplugin { #define plg_archive2db plg_un.plg_un_db.plg_un_db_archive2db #define plg_db2archive plg_un.plg_un_db.plg_un_db_db2archive #define plg_upgradedb plg_un.plg_un_db.plg_un_db_upgradedb +#define plg_upgradednformat plg_un.plg_un_db.plg_un_db_upgradednformat #define plg_dbverify plg_un.plg_un_db.plg_un_db_verify #define plg_dbsize plg_un.plg_un_db.plg_un_db_dbsize #define plg_dbtest plg_un.plg_un_db.plg_un_db_dbtest @@ -1132,6 +1135,7 @@ typedef struct backend { #define be_seq be_database->plg_seq #define be_ldif2db be_database->plg_ldif2db #define be_upgradedb be_database->plg_upgradedb +#define be_upgradednformat be_database->plg_upgradednformat #define be_db2ldif be_database->plg_db2ldif #define be_db2index be_database->plg_db2index #define be_archive2db be_database->plg_archive2db diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h index 382cded0..343b8219 100644 --- a/ldap/servers/slapd/slapi-private.h +++ b/ldap/servers/slapd/slapi-private.h @@ -375,6 +375,7 @@ Slapi_DN *slapi_sdn_init_dn_passin(Slapi_DN *sdn,const char *dn); Slapi_DN *slapi_sdn_init_ndn_byref(Slapi_DN *sdn,const char *dn); Slapi_DN *slapi_sdn_init_ndn_byval(Slapi_DN *sdn,const char *dn); Slapi_DN *slapi_sdn_init_dn_ndn_byref(Slapi_DN *sdn,const char *dn); +char *slapi_dn_normalize_case_original( char *dn ); /* filter.c */ int filter_flag_is_set(const Slapi_Filter *f,unsigned char flag); @@ -872,6 +873,7 @@ int valuearray_normalize_value(Slapi_Value **vals); #define SLAPI_PLUGIN_DB_ADD_SCHEMA_FN 237 #define SLAPI_PLUGIN_DB_SEARCH_RESULTS_RELEASE_FN 238 #define SLAPI_PLUGIN_DB_PREV_SEARCH_RESULTS_FN 239 +#define SLAPI_PLUGIN_DB_UPGRADEDNFORMAT_FN 240 /* database plugin-specific parameters */ #define SLAPI_PLUGIN_DB_NO_ACL 250 #define SLAPI_PLUGIN_DB_RMDB_FN 280 @@ -1185,11 +1187,14 @@ void bervalarray_add_berval_fast(struct berval ***vals, const struct berval *add void DS_Sleep(PRIntervalTime ticks); -/* macro to specify the behavior of upgradedb */ -#define SLAPI_UPGRADEDB_FORCE 0x1 /* reindex all (no check w/ idl switch) */ -#define SLAPI_UPGRADEDB_SKIPINIT 0x2 /* call upgradedb as part of other op */ -#define SLAPI_UPGRADEDB_DN2RDN 0x4 /* modify id2entry from dn format to rdn; - generate entryrdn index */ +/* macro to specify the behavior of upgradedb & upgradednformat */ +#define SLAPI_UPGRADEDB_FORCE 0x1 /* reindex all (no check w/ idl switch) */ +#define SLAPI_UPGRADEDB_SKIPINIT 0x2 /* call upgradedb as part of other op */ +#define SLAPI_UPGRADEDB_DN2RDN 0x4 /* modify id2entry from dn format to rdn; + generate entryrdn index */ +#define SLAPI_UPGRADEDNFORMAT 0x8 /* specify this op is upgradednformat */ +#define SLAPI_DRYRUN 0x10 /* dryrun mode for upgradednformat */ + /* * Macro to set port to the 'port' field of a NSPR PRNetAddr union. |