diff options
Diffstat (limited to 'ldap/servers/slapd/back-ldbm')
72 files changed, 41138 insertions, 0 deletions
diff --git a/ldap/servers/slapd/back-ldbm/Makefile b/ldap/servers/slapd/back-ldbm/Makefile new file mode 100644 index 00000000..c86ec0b3 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/Makefile @@ -0,0 +1,190 @@ +# +# BEGIN COPYRIGHT BLOCK +# Copyright 2001 Sun Microsystems, Inc. +# Portions copyright 1999, 2001-2003 Netscape Communications Corporation. +# All rights reserved. +# END COPYRIGHT BLOCK +# +# +# GNU Makefile for Directory Server libback-ldbm +# +LDAP_SRC = ../../.. +MCOM_ROOT = ../../../../.. + +NOSTDCLEAN=true # don't let nsconfig.mk define target clean +NOSTDSTRIP=true # don't let nsconfig.mk define target strip +NSPR20=true # probably should be defined somewhere else (not sure where) + +OBJDEST = $(OBJDIR)/lib/libback-ldbm + +ifndef INSTDIR +INSTDIR = /netscape/server4/ +endif + +include $(MCOM_ROOT)/ldapserver/nsconfig.mk +include $(LDAP_SRC)/nsldap.mk +ifndef LDAP_USE_OLD_DB +include $(MCOM_ROOT)/ldapserver/ns_usedb.mk +INCLUDES+=-I$(DB_INCLUDE) +else +CFLAGS+=-DLDAP_USE_DB185 +endif +include $(LDAP_SRC)/nsdeps.mk + +CFLAGS+=$(SLCFLAGS) + +INCLUDES += -I$(LDAP_SRC)/servers/slapd + +LIBBACK_LDBM_OBJS= idl.o idl_shim.o idl_new.o idl_common.o cache.o dn2entry.o \ + id2entry.o index.o haschildren.o nextid.o init.o \ + filterindex.o close.o backentry.o monitor.o seq.o start.o \ + rmdb.o ldif2ldbm.o dblayer.o findentry.o archive.o \ + sort.o dbsize.o dbtest.o vlv.o vlv_key.o \ + vlv_srch.o matchrule.o entrystore.o parents.o misc.o \ + upgrade.o dbversion.o cleanup.o uniqueid2entry.o \ + perfctrs.o instance.o import-threads.o import.o import-merge.o \ + ldbm_config.o ldbm_instance_config.o ldbm_index_config.o ldbm_attrcrypt_config.o \ + ldbm_attr.o \ + ldbm_abandon.o \ + ldbm_compare.o \ + ldbm_add.o \ + ldbm_search.o \ + ldbm_modify.o \ + ldbm_modrdn.o \ + ldbm_delete.o \ + ldbm_bind.o \ + ldbm_unbind.o \ + ancestorid.o \ + ldbm_attrcrypt.o + +OBJS = $(addprefix $(OBJDEST)/, $(LIBBACK_LDBM_OBJS)) + +ifeq ($(ARCH), WINNT) +LIBBACK_LDBM_DLL_OBJ = $(addprefix $(OBJDEST)/, dllmain.o) +endif + +LIBBACK_LDBM= $(addprefix $(LDAP_LIBBACK_LDBM_DLLDIR)/, $(LIBBACK_LDBM_DLL).$(DLL_SUFFIX)) + +ifeq ($(ARCH), WINNT) +EXTRA_LIBS_DEP += \ + $(LDAPSDK_DEP) \ + $(LDAP_LIBLDIF_DEP) \ + $(LDAP_LIBAVL_DEP) +EXTRA_LIBS += \ + $(NSPRLINK) \ + $(LDAP_SDK_LIBLDAP_DLL) \ + $(LDAP_LIBLDIF) \ + $(LDAP_LIBAVL) +CFLAGS+= /WX +endif + +ifeq ($(ARCH), AIX) +EXTRA_LIBS_DEP += \ + $(LDAP_SDK_LIBLDAP_DLL_DEP) \ + $(LDAP_LIBLDIF_DEP) \ + $(LDAP_LIBAVL_DEP) +EXTRA_LIBS += \ + $(NSPRLINK) \ + $(LDAP_SDK_LIBLDAP_DLL) \ + $(LDAP_LIBLDIF) \ + $(LDAP_LIBAVL) \ + $(DLL_EXTRA_LIBS) +LD=ld +endif + +ifeq ($(ARCH), SOLARIS) +EXTRA_LIBS_DEP += \ + $(LDAPSDK_DEP) \ + $(LDAP_LIBLDIF_DEP) \ + $(LDAP_LIBAVL_DEP) +EXTRA_LIBS += \ + $(LDAPLINK) $(SECURITYLINK) $(NSPRLINK) \ + $(LDAP_LIBLDIF) \ + $(LDAP_LIBAVL) \ + $(DLL_EXTRA_LIBS) -lc +LINK_DLL += -z defs +endif + +ifeq ($(ARCH), HPUX) +EXTRA_LIBS_DEP += \ + $(LDAPSDK_DEP) \ + $(LDAP_LIBLDIF_DEP) \ + $(LDAP_LIBAVL_DEP) +EXTRA_LIBS += \ + $(LDAPLINK) $(SECURITYLINK) $(NSPRLINK) \ + $(LDAP_LIBLDIF) \ + $(LDAP_LIBAVL) \ + $(DLL_EXTRA_LIBS) -lc +endif + +ifeq ($(ARCH), WINNT) +DLL_LDFLAGS += -def:"./libback-ldbm.def" +IMPLIB= /IMPLIB:$(LDAP_LIBBACK_LDBM_LIBDIR)/$(LIBBACK_LDBM_DLL).lib +MAPFILE= /MAP:$(LDAP_LIBBACK_LDBM_LIBDIR)/$(LIBBACK_LDBM_DLL).map +endif # WINNT + +ifeq ($(ARCH), UnixWare) +EXTRA_LIBS_DEP += $(LDAP_LIBAVL_DEP) +EXTRA_LIBS += $(LDAP_LIBAVL) +endif # UnixWare + +ifeq ($(ARCH), Linux) +EXTRA_LIBS_DEP += $(LDAP_LIBLDBM_DEP) $(LDAP_LIBAVL_DEP) $(LDAP_LIBLDIF_DEP) +EXTRA_LIBS += $(LDAP_LIBLDBM) $(LDAP_LIBAVL) $(LDAP_LIBLDIF) +EXTRA_LIBS += $(DLL_EXTRA_LIBS) +endif # Linux + +EXTRA_LIBS_DEP += $(DB_LIB_DEP) + +clientSDK: + +all: $(OBJDEST) $(LDAP_LIBBACK_LDBM_LIBDIR) $(LDAP_LIBBACK_LDBM_DLLDIR) \ + $(BUILD_DEP) $(LIBBACK_LDBM) +ifeq ($(ARCH), WINNT) + cd ntdbperfdll; $(MAKE) $(MFLAGS) all +endif + +dummy: + -@echo LDAP_LIBDIR = $(LDAP_LIBDIR) + -@echo LDAP_LIBBACK_LDBM_LIBDIR = $(LDAP_LIBBACK_LDBM_LIBDIR) + -@echo LIB_RELDIR = $(LIB_RELDIR) + -@echo LDAP_LIBBACK_LDBM_DLLDIR = $(LDAP_LIBBACK_LDBM_DLLDIR) + -@echo LDAP_LIBBACK_LDBM = $(LDAP_LIBBACK_LDBM) + -@echo LIBBACK_LDBM = $(LIBBACK_LDBM) + abort + +$(LIBBACK_LDBM): $(OBJS) $(LIBBACK_LDBM_DLL_OBJ) $(EXTRA_LIBS_DEP) $(LIBSLAPD_DEP) + $(LINK_DLL) $(IMPLIB) $(MAPFILE) $(LIBBACK_LDBM_DLL_OBJ) $(EXTRA_LIBS) $(DB_LIB) $(LIBSLAPD) + +veryclean: clean + +clean: + $(RM) $(OBJS) +ifeq ($(ARCH), WINNT) + $(RM) $(LIBBACK_LDBM_DLL_OBJ) +endif + $(RM) $(LIBBACK_LDBM) + +$(OBJDEST) $(LIBBACK_LDBM_LIBDIR): + $(MKDIR) $@ + +ifeq ($(ARCH), AIX) +ifeq ($(DEBUG), optimize) + +# For some reason compiling ldif2ldbm.c with the -O flag on AIX causes +# the new import code to hang. For now we will avoid the -O flag. + +TEMP_CFLAGS = $(subst -O,,$(CFLAGS)) + +$(OBJDEST)/ldif2ldbm.o: ldif2ldbm.c + $(CC) -o $(OBJDEST)/ldif2ldbm.o -c $(TEMP_CFLAGS) $(MCC_INCLUDE) ldif2ldbm.c +endif +endif + +# Target to push the built binary to an installed server +LDBM_PUSH = $(addprefix $(INSTDIR)/, lib/libback-ldbm.dll) +push: $(LDBM_PUSH) + +$(LDBM_PUSH): $(LIBBACK_LDBM) + cp $(LIBBACK_LDBM) $(LDBM_PUSH) + diff --git a/ldap/servers/slapd/back-ldbm/ancestorid.c b/ldap/servers/slapd/back-ldbm/ancestorid.c new file mode 100644 index 00000000..0ec76a17 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ancestorid.c @@ -0,0 +1,747 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +#include "back-ldbm.h" + +static char *sourcefile = "ancestorid"; + +/* Start of definitions for a simple cache using a hash table */ + +typedef struct id2idl { + ID keyid; + IDList *idl; + struct id2idl *next; +} id2idl; + +static void id2idl_free(id2idl **ididl); +static int id2idl_same_key(const void *ididl, const void *k); + +typedef Hashtable id2idl_hash; + +#define id2idl_new_hash(size) new_hash(size,HASHLOC(id2idl,next),NULL,id2idl_same_key) +#define id2idl_hash_lookup(ht,key,he) find_hash(ht,key,sizeof(ID),(void**)(he)) +#define id2idl_hash_add(ht,key,he,alt) add_hash(ht,key,sizeof(ID),he,(void**)(alt)) +#define id2idl_hash_remove(ht,key) remove_hash(ht,key,sizeof(ID)) + +static void id2idl_hash_destroy(id2idl_hash *ht); + +/* End of definitions for a simple cache using a hash table */ + +static int ldbm_parentid(backend *be, DB_TXN *txn, ID id, ID *ppid); +static int check_cache(id2idl_hash *ht); +static IDList *idl_union_allids(backend *be, struct attrinfo *ai, IDList *a, IDList *b); + +static int ldbm_get_nonleaf_ids(backend *be, DB_TXN *txn, IDList **idl) +{ + int ret = 0; + DB *db = NULL; + DBC *dbc = NULL; + DBT key = {0}; + DBT data = {0}; + struct attrinfo *ai = NULL; + IDList *nodes = NULL; + ID id; + + /* Open the parentid index */ + ainfo_get( be, "parentid", &ai ); + + /* Open the parentid index file */ + ret = dblayer_get_index_file(be, ai, &db, DBOPEN_CREATE); + if (ret != 0) { + ldbm_nasty(sourcefile,13010,ret); + goto out; + } + + /* Get a cursor so we can walk through the parentid */ + ret = db->cursor(db,txn,&dbc,0); + if (ret != 0 ) { + ldbm_nasty(sourcefile,13020,ret); + goto out; + } + + /* For each key which is an equality key */ + do { + ret = dbc->c_get(dbc,&key,&data,DB_NEXT_NODUP); + if ((ret == 0) && (*(char*)key.data == EQ_PREFIX)) { + id = (ID) strtoul((char*)key.data+1, NULL, 10); + idl_insert(&nodes, id); + } + } while (ret == 0); + + /* Check for success */ + if (ret == DB_NOTFOUND) ret = 0; + if (ret != 0) ldbm_nasty(sourcefile,13030,ret); + + out: + /* Close the cursor */ + if (dbc != NULL) { + if (ret == 0) { + ret = dbc->c_close(dbc); + if (ret != 0) ldbm_nasty(sourcefile,13040,ret); + } else { + (void)dbc->c_close(dbc); + } + } + + /* Release the parentid file */ + if (db != NULL) { + dblayer_release_index_file( be, ai, db ); + } + + /* Return the idlist */ + if (ret == 0) { + *idl = nodes; + LDAPDebug(LDAP_DEBUG_TRACE, "found %lu nodes for ancestorid\n", + (u_long)IDL_NIDS(nodes), 0, 0); + } else { + idl_free(nodes); + *idl = NULL; + } + + return ret; +} + +/* + * XXX: This function creates ancestorid index, which is a sort of hack. + * This function handles idl directly, + * which should have been implemented in the idl file(s). + * When the idl code would be updated in the future, + * this function may also get affected. + * (see also bug#: 605535) + * + * Construct the ancestorid index. Requirements: + * - The backend is read only. + * - The parentid index is accurate. + * - Non-leaf entries have IDs less than their descendants + * (guaranteed after a database import but not after a subtree move) + * + */ +int ldbm_ancestorid_create_index(backend *be) +{ + int ret = 0; + DB *db_pid = NULL; + DB *db_aid = NULL; + DBT key = {0}; + DB_TXN *txn = NULL; + struct attrinfo *ai_pid = NULL; + struct attrinfo *ai_aid = NULL; + char keybuf[24]; + IDList *nodes = NULL; + IDList *children = NULL, *descendants = NULL; + NIDS nids; + ID id, parentid; + id2idl_hash *ht = NULL; + id2idl *ididl; + + /* + * We need to iterate depth-first through the non-leaf nodes + * in the tree amassing an idlist of descendant ids for each node. + * We would prefer to go through the parentid keys just once from + * highest id to lowest id but the btree ordering is by string + * rather than number. So we go through the parentid keys in btree + * order first of all to create an idlist of all the non-leaf nodes. + * Then we can use the idlist to iterate through parentid in the + * correct order. + */ + + LDAPDebug(LDAP_DEBUG_TRACE, "Creating ancestorid index\n", 0,0,0); + + /* Get the non-leaf node IDs */ + ret = ldbm_get_nonleaf_ids(be, txn, &nodes); + if (ret != 0) return ret; + + /* Get the ancestorid index */ + ainfo_get(be, "ancestorid", &ai_aid); + + /* Prevent any other use of the index */ + ai_aid->ai_indexmask |= INDEX_OFFLINE; + + /* Open the ancestorid index file */ + ret = dblayer_get_index_file(be, ai_aid, &db_aid, DBOPEN_CREATE); + if (ret != 0) { + ldbm_nasty(sourcefile,13050,ret); + goto out; + } + + /* Maybe nothing to do */ + if (nodes == NULL || nodes->b_nids == 0) { + LDAPDebug(LDAP_DEBUG_ANY, "Nothing to do to build ancestorid index\n", + 0, 0, 0); + goto out; + } + + /* Create an ancestorid cache */ + ht = id2idl_new_hash(nodes->b_nids); + + /* Get the parentid index */ + ainfo_get( be, "parentid", &ai_pid ); + + /* Open the parentid index file */ + ret = dblayer_get_index_file(be, ai_pid, &db_pid, DBOPEN_CREATE); + if (ret != 0) { + ldbm_nasty(sourcefile,13060,ret); + goto out; + } + + /* Initialize key DBT */ + key.data = keybuf; + key.ulen = sizeof(keybuf); + key.flags = DB_DBT_USERMEM; + + /* Iterate from highest to lowest ID */ + nids = nodes->b_nids; + do { + + nids--; + id = nodes->b_ids[nids]; + + /* Get immediate children from parentid index */ + key.size = PR_snprintf(key.data, key.ulen, "%c%lu", + EQ_PREFIX, (u_long)id); + key.size++; /* include the null terminator */ + ret = NEW_IDL_NO_ALLID; + children = idl_fetch(be, db_pid, &key, txn, ai_pid, &ret); + if (ret != 0) { + ldbm_nasty(sourcefile,13070,ret); + break; + } + + /* Insert into ancestorid for this node */ + if (id2idl_hash_lookup(ht, &id, &ididl)) { + descendants = idl_union_allids(be, ai_aid, ididl->idl, children); + idl_free(children); + if (id2idl_hash_remove(ht, &id) == 0) { + LDAPDebug(LDAP_DEBUG_ANY, "ancestorid hash_remove failed\n", 0,0,0); + } else { + id2idl_free(&ididl); + } + } else { + descendants = children; + } + ret = idl_store_block(be, db_aid, &key, descendants, txn, ai_aid); + if (ret != 0) break; + + /* Get parentid for this entry */ + ret = ldbm_parentid(be, txn, id, &parentid); + if (ret != 0) { + idl_free(descendants); + break; + } + + /* A suffix entry does not have a parent */ + if (parentid == NOID) { + idl_free(descendants); + continue; + } + + /* Insert into ancestorid for this node's parent */ + if (id2idl_hash_lookup(ht, &parentid, &ididl)) { + IDList *idl = idl_union_allids(be, ai_aid, ididl->idl, descendants); + idl_free(descendants); + idl_free(ididl->idl); + ididl->idl = idl; + } else { + ididl = (id2idl*)slapi_ch_calloc(1,sizeof(id2idl)); + ididl->keyid = parentid; + ididl->idl = descendants; + if (id2idl_hash_add(ht, &parentid, ididl, NULL) == 0) { + LDAPDebug(LDAP_DEBUG_ANY, "ancestorid hash_add failed\n", 0,0,0); + } + } + + } while (nids > 0); + + if (ret != 0) { + goto out; + } + + /* We're expecting the cache to be empty */ + ret = check_cache(ht); + + out: + if (ret == 0) { + LDAPDebug(LDAP_DEBUG_TRACE, "Created ancestorid index\n", 0,0,0); + } else { + LDAPDebug(LDAP_DEBUG_ANY, "Failed to create ancestorid index\n", 0,0,0); + } + + /* Destroy the cache */ + id2idl_hash_destroy(ht); + + /* Free any leftover idlists */ + idl_free(nodes); + + /* Release the parentid file */ + if (db_pid != NULL) { + dblayer_release_index_file( be, ai_pid, db_pid ); + } + + /* Release the ancestorid file */ + if (db_aid != NULL) { + dblayer_release_index_file( be, ai_aid, db_aid ); + } + + /* Enable the index */ + if (ret == 0) { + ai_aid->ai_indexmask &= ~INDEX_OFFLINE; + } + + return ret; +} + +/* + * Get parentid of an id by reading the operational attr from id2entry. + */ +static int ldbm_parentid(backend *be, DB_TXN *txn, ID id, ID *ppid) +{ + int ret = 0; + DB *db = NULL; + DBT key = {0}; + DBT data = {0}; + ID stored_id; + char *p; + + /* Open the id2entry file */ + ret = dblayer_get_id2entry(be, &db); + if (ret != 0) { + ldbm_nasty(sourcefile,13100,ret); + goto out; + } + + /* Initialize key and data DBTs */ + id_internal_to_stored(id, (char *)&stored_id); + key.data = (char *)&stored_id; + key.size = sizeof(stored_id); + key.flags = DB_DBT_USERMEM; + data.flags = DB_DBT_MALLOC; + + /* Read id2entry */ + ret = db->get(db, txn, &key, &data, 0); + if (ret != 0) { + ldbm_nasty(sourcefile,13110,ret); + goto out; + } + + /* Extract the parentid value */ +#define PARENTID_STR "\nparentid:" + p = strstr(data.data, PARENTID_STR); + if (p == NULL) { + *ppid = NOID; + goto out; + } + *ppid = strtoul(p + strlen(PARENTID_STR), NULL, 10); + + out: + /* Free the entry value */ + if (data.data != NULL) free(data.data); + + /* Release the id2entry file */ + if (db != NULL) { + dblayer_release_id2entry(be, db); + } + return ret; +} + +static void id2idl_free(id2idl **ididl) +{ + idl_free((*ididl)->idl); + slapi_ch_free((void**)ididl); +} + +static int id2idl_same_key(const void *ididl, const void *k) +{ + return (((id2idl *)ididl)->keyid == *(ID *)k); +} + +static int check_cache(id2idl_hash *ht) +{ + id2idl *e; + u_long i, found = 0; + int ret = 0; + + if (ht == NULL) return 0; + + for (i = 0; i < ht->size; i++) { + e = (id2idl *)ht->slot[i]; + while (e) { + found++; + e = e->next; + } + } + + if (found > 0) { + LDAPDebug(LDAP_DEBUG_ANY, "ERROR: parentid index is not complete (%lu extra keys in ancestorid cache)\n", found,0,0); + ret = -1; + } + + return ret; +} + +static void id2idl_hash_destroy(id2idl_hash *ht) +{ + u_long i; + id2idl *e, *next; + + if (ht == NULL) return; + + for (i = 0; i < ht->size; i++) { + e = (id2idl *)ht->slot[i]; + while (e) { + next = e->next; + id2idl_free(&e); + e = next; + } + } + slapi_ch_free((void **)&ht); +} + +/* + * idl_union_allids - return a union b + * takes attr index allids setting into account + */ +static IDList *idl_union_allids(backend *be, struct attrinfo *ai, IDList *a, IDList *b) +{ + if (!idl_get_idl_new()) { + if (a != NULL && b != NULL) { + if (ALLIDS( a ) || ALLIDS( b ) || + (IDL_NIDS(a) + IDL_NIDS(b) > idl_get_allidslimit(ai))) { + return( idl_allids( be ) ); + } + } + } + return idl_union(be, a, b); +} + +static int ancestorid_addordel( + backend *be, + DB* db, + ID node_id, + ID id, + DB_TXN *txn, + struct attrinfo *ai, + int flags, + int *allids +) +{ + DBT key = {0}; + char keybuf[24]; + int ret = 0; + + /* Initialize key DBT */ + key.data = keybuf; + key.ulen = sizeof(keybuf); + key.flags = DB_DBT_USERMEM; + key.size = PR_snprintf(key.data, key.ulen, "%c%lu", + EQ_PREFIX, (u_long)node_id); + key.size++; /* include the null terminator */ + + if (flags & BE_INDEX_ADD) { +#if 1 + LDAPDebug(LDAP_DEBUG_TRACE, "insert ancestorid %lu:%lu\n", + (u_long)node_id, (u_long)id, 0); +#endif + ret = idl_insert_key(be, db, &key, id, txn, ai, allids); + } else { +#if 1 + LDAPDebug(LDAP_DEBUG_TRACE, "delete ancestorid %lu:%lu\n", + (u_long)node_id, (u_long)id, 0); +#endif + ret = idl_delete_key(be, db, &key, id, txn, ai); + } + + if (ret != 0) { + ldbm_nasty(sourcefile,13120,ret); + } + + return ret; +} + +/* + * Update ancestorid index inserting or deleting depending on flags. + * The entry ids to be indexed are given by id (a base object) + * and optionally subtree_idl (descendants of the base object). + * The ancestorid keys to be updated are derived from nodes + * in the tree from low up to high. Whether the low and high nodes + * themselves are updated is given by include_low and include_high. + */ +static int ldbm_ancestorid_index_update( + backend *be, + const Slapi_DN *low, + const Slapi_DN *high, + int include_low, + int include_high, + ID id, + IDList *subtree_idl, + int flags, /* BE_INDEX_ADD, BE_INDEX_DEL */ + back_txn *txn +) +{ + DB *db = NULL; + int allids = IDL_INSERT_NORMAL; + Slapi_DN dn = {0}; + Slapi_DN nextdn = {0}; + struct attrinfo *ai = NULL; + struct berval ndnv; + ID node_id, sub_id; + IDList *idl; + idl_iterator iter; + int err = 0, ret = 0; + DB_TXN *db_txn = txn != NULL ? txn->back_txn_txn : NULL; + + /* Open the ancestorid index */ + ainfo_get(be, "ancestorid", &ai); + ret = dblayer_get_index_file(be, ai, &db, DBOPEN_CREATE); + if (ret != 0) { + ldbm_nasty(sourcefile,13130,ret); + goto out; + } + + slapi_sdn_copy(low, &dn); + + if (include_low == 0) { + if (slapi_sdn_compare(&dn, high) == 0) { + goto out; + } + /* Get the next highest DN */ + slapi_sdn_get_parent(&dn, &nextdn); + slapi_sdn_copy(&nextdn, &dn); + } + + /* Iterate up through the tree */ + do { + if (slapi_sdn_isempty(&dn)) { + break; + } + + /* Have we reached the high node? */ + if (include_high == 0 && slapi_sdn_compare(&dn, high) == 0) { + break; + } + + /* Get the id for that DN */ + ndnv.bv_val = (void*)slapi_sdn_get_ndn(&dn); + ndnv.bv_len = slapi_sdn_get_ndn_len(&dn); + err = 0; + idl = index_read(be, "entrydn", indextype_EQUALITY, &ndnv, txn, &err); + if (idl == NULL) { + if (err != 0 && err != DB_NOTFOUND) { + ldbm_nasty(sourcefile,13140,ret); + ret = err; + } + break; + } + node_id = idl_firstid(idl); + idl_free(idl); + + /* Update ancestorid for the base entry */ + ret = ancestorid_addordel(be, db, node_id, id, db_txn, ai, flags, &allids); + if (ret != 0) break; + + /* + * If this node was already allids then all higher nodes must already + * be at allids since the higher nodes must have a greater number + * of descendants. Therefore no point continuing. + */ + if (allids == IDL_INSERT_ALLIDS) break; + + /* Update ancestorid for any subtree entries */ + if (subtree_idl != NULL && ((flags & BE_INDEX_ADD) || (!ALLIDS(subtree_idl)))) { + iter = idl_iterator_init(subtree_idl); + while ((sub_id = idl_iterator_dereference_increment(&iter, subtree_idl)) != NOID) { + ret = ancestorid_addordel(be, db, node_id, sub_id, db_txn, ai, flags, &allids); + if (ret != 0) break; + } + if (ret != 0) break; + } + + /* Have we reached the high node? */ + if (slapi_sdn_compare(&dn, high) == 0) { + break; + } + + /* Get the next highest DN */ + slapi_sdn_get_parent(&dn, &nextdn); + slapi_sdn_copy(&nextdn, &dn); + + } while (ret == 0); + + out: + slapi_sdn_done(&dn); + slapi_sdn_done(&nextdn); + + /* Release the ancestorid file */ + if (db != NULL) { + dblayer_release_index_file(be, ai, db); + } + + return ret; +} + +/* + * Update the ancestorid index for a single entry. + * This function depends on the integrity of the entrydn index. + */ +int ldbm_ancestorid_index_entry( + backend *be, + struct backentry *e, + int flags, /* BE_INDEX_ADD, BE_INDEX_DEL */ + back_txn *txn +) +{ + int ret = 0; + + ret = ldbm_ancestorid_index_update(be, + slapi_entry_get_sdn_const(e->ep_entry), + slapi_be_getsuffix(be, 0), + 0, 1, e->ep_id, NULL, flags, txn); + + return ret; +} + +/* + * Returns <0, 0, >0 according to whether right is a suffix of left, + * neither is a suffix of the other, or left is a suffix of right. + * If common is non-null then the common suffix of left and right + * is returned in *common. + */ +int slapi_sdn_suffix_cmp( + const Slapi_DN *left, + const Slapi_DN *right, + Slapi_DN *common +) +{ + char **rdns1, **rdns2; + int count1, count2, i, ret = 0; + size_t len = 0; + char *p, *ndnstr; + + rdns1 = ldap_explode_dn(slapi_sdn_get_ndn(left), 0); + rdns2 = ldap_explode_dn(slapi_sdn_get_ndn(right), 0); + + for(count1 = 0; rdns1[count1]!=NULL; count1++){ + } + count1--; + + for(count2 = 0; rdns2[count2]!=NULL; count2++){ + } + count2--; + + while (count1 >= 0 && count2 >= 0) { + if (strcmp(rdns1[count1], rdns2[count2]) != 0) break; + count1--; + count2--; + } + + count1++; + count2++; + + if (count1 == 0 && count2 == 0) { + /* equal */ + ret = 0; + } else if (count1 == 0) { + /* left is suffix of right */ + ret = 1; + } else if (count2 == 0) { + /* right is suffix of left */ + ret = -1; + } else { + /* common prefix (possibly root), not left nor right */ + ret = 0; + } + + /* if caller does not want the common prefix then we're done */ + if (common == NULL) goto out; + + /* figure out how much space we need */ + for (i = count1; rdns1[i] != NULL; i++) { + len += strlen(rdns1[i]) + 1; + } + + /* write the string */ + p = ndnstr = slapi_ch_calloc(len+1,sizeof(char)); + for (i = count1; rdns1[i] != NULL; i++) { + sprintf(p, "%s%s", (p != ndnstr) ? "," : "", rdns1[i]); + p += strlen(p); + } + + /* return the DN */ + slapi_sdn_set_dn_passin(common, ndnstr); + + LDAPDebug(LDAP_DEBUG_TRACE, "common suffix <%s>\n", + slapi_sdn_get_dn(common), 0, 0); + + out: + ldap_value_free(rdns1); + ldap_value_free(rdns2); + + LDAPDebug(LDAP_DEBUG_TRACE, "slapi_sdn_suffix_cmp(<%s>, <%s>) => %d\n", + slapi_sdn_get_dn(left), slapi_sdn_get_dn(right), ret); + + return ret; +} + +int ldbm_ancestorid_move_subtree( + backend *be, + const Slapi_DN *olddn, + const Slapi_DN *newdn, + ID id, + IDList *subtree_idl, + back_txn *txn +) +{ + int ret = 0; + Slapi_DN commondn = {0}; + + /* Determine the common ancestor */ + (void)slapi_sdn_suffix_cmp(olddn, newdn, &commondn); + + /* Delete from old ancestors */ + ret = ldbm_ancestorid_index_update(be, + olddn, + &commondn, + 0, + 0, + id, + subtree_idl, + BE_INDEX_DEL, + txn); + if (ret != 0) goto out; + + /* Add to new ancestors */ + ret = ldbm_ancestorid_index_update(be, + newdn, + &commondn, + 0, + 0, + id, + subtree_idl, + BE_INDEX_ADD, + txn); + + out: + slapi_sdn_done(&commondn); + return ret; +} + +int ldbm_ancestorid_read( + backend *be, + back_txn *txn, + ID id, + IDList **idl +) +{ + int ret = 0; + struct berval bv; + char keybuf[24]; + + bv.bv_val = keybuf; + bv.bv_len = PR_snprintf(keybuf, sizeof(keybuf), "%lu", (u_long)id); + + *idl = index_read(be, "ancestorid", indextype_EQUALITY, &bv, txn, &ret); + + return ret; +} + diff --git a/ldap/servers/slapd/back-ldbm/archive.c b/ldap/servers/slapd/back-ldbm/archive.c new file mode 100644 index 00000000..2daee2c5 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/archive.c @@ -0,0 +1,332 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* archive.c - ldap ldbm back-end archive and restore entry points */ + +#include "back-ldbm.h" + +int ldbm_back_archive2ldbm( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + char *directory = NULL; + int return_value = -1; + int task_flags = 0; + int run_from_cmdline = 0; + Slapi_Task *task; + int is_old_to_new = 0; + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_SEQ_VAL, &directory ); + slapi_pblock_get( pb, SLAPI_BACKEND_TASK, &task ); + slapi_pblock_get( pb, SLAPI_TASK_FLAGS, &task_flags ); + li->li_flags = run_from_cmdline = (task_flags & TASK_RUNNING_FROM_COMMANDLINE); + + /* check the current idl format vs backup DB version */ + if (idl_get_idl_new()) + { + char dbversion[LDBM_VERSION_MAXBUF]; + char dataversion[LDBM_VERSION_MAXBUF]; + int value = 0; + + if (dbversion_read(li, directory, dbversion, dataversion) != 0) + { + LDAPDebug(LDAP_DEBUG_ANY, "Warning: Unable to read dbversion " + "file in %s\n", directory, 0, 0); + } + value = lookup_dbversion(dbversion, DBVERSION_TYPE); + if (value & DBVERSION_OLD_IDL) + { + is_old_to_new = 1; + } + } + + /* No ldbm be's exist until we process the config information. */ + if (run_from_cmdline) { + mapping_tree_init(); + ldbm_config_load_dse_info(li); + } else { + ldbm_instance *inst; + Object *inst_obj, *inst_obj2; + + /* task does not support restore old idl onto new idl server */ + if (is_old_to_new) + { + LDAPDebug(LDAP_DEBUG_ANY, + "backup has old idl format; " + "to restore old formated backup onto the new server, " + "please use command line utility \"bak2db\" .\n", + 0, 0, 0); + if (task) + { + slapi_task_log_notice(task, + "backup has old idl format; " + "to restore old formated backup onto the new server, " + "please use command line utility \"bak2db\" .\n"); + } + return -1; + } + /* server is up -- mark all backends busy */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + + /* check if an import/restore is already ongoing... */ + if (instance_set_busy(inst) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm: '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + inst->inst_name, 0, 0); + if (task) { + slapi_task_log_notice(task, + "Backend '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + inst->inst_name); + } + + /* painfully, we have to clear the BUSY flags on the + * backends we'd already marked... + */ + for (inst_obj2 = objset_first_obj(li->li_instance_set); + inst_obj2 && (inst_obj2 != inst_obj); + inst_obj2 = objset_next_obj(li->li_instance_set, + inst_obj2)) { + inst = (ldbm_instance *)object_get_data(inst_obj2); + instance_set_not_busy(inst); + } + object_release(inst_obj2); + object_release(inst_obj); + return -1; + } + } + + /* now take down ALL BACKENDS */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + LDAPDebug(LDAP_DEBUG_ANY, "Bringing %s offline...\n", + inst->inst_name, 0, 0); + if (task) { + slapi_task_log_notice(task, "Bringing %s offline...", + inst->inst_name); + } + slapi_mtn_be_disable(inst->inst_be); + cache_clear(&inst->inst_cache); + } + /* now we know nobody's using any of the backend instances, so we + * can shutdown the dblayer -- this closes all instances too. + * Use DBLAYER_RESTORE_MODE to prevent loss of perfctr memory. + */ + dblayer_close(li, DBLAYER_RESTORE_MODE); + } + + /* tell the database to restore */ + return_value = dblayer_restore(li, directory, task); + if (0 != return_value) { + LDAPDebug( LDAP_DEBUG_ANY, + "archive2db: Failed to read backup file set. " + "Either the directory specified doesn't exist, " + "or it exists but doesn't contain a valid backup set, " + "or file permissions prevent the server reading " + "the backup set. error=%d (%s)\n", + return_value, dblayer_strerror(return_value), 0 ); + if (task) { + slapi_task_log_notice(task, "Failed to read the backup file set " + "from %s", directory); + } + } + + if (run_from_cmdline) + { + if (is_old_to_new) + { + /* does not exist */ + char *p; + char c; + char *bakup_dir = NULL; + int skipinit = SLAPI_UPGRADEDB_SKIPINIT; + + p = strrchr(directory, '/'); + if (NULL == p) + { + p = strrchr(directory, '\\'); + } + + if (NULL == p) /* never happen, I guess */ + { + directory = "."; + c = '/'; + } + else + { + c = *p; + *p = '\0'; + } + bakup_dir = (char *)slapi_ch_malloc(strlen(directory) + + sizeof("tmp") + 13); + sprintf(bakup_dir, "%s%ctmp_%010d", directory, c, time(0)); + LDAPDebug( LDAP_DEBUG_ANY, + "archive2db: backup dir: %s\n", bakup_dir, 0, 0); + *p = c; + + slapi_pblock_set( pb, SLAPI_SEQ_VAL, bakup_dir ); + slapi_pblock_set( pb, SLAPI_SEQ_TYPE, &skipinit ); + return_value = ldbm_back_upgradedb( pb ); + } + } + else + { + ldbm_instance *inst; + Object *inst_obj; + int ret; + + if (0 != return_value) { + /* error case (607331) + * just to go back to the previous state if possible */ + dblayer_start(li, DBLAYER_NORMAL_MODE); + } + /* bring all backends back online */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + ret = dblayer_instance_start(inst->inst_be, DBLAYER_NORMAL_MODE); + if (ret != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "archive2db: Unable to restart '%s'\n", + inst->inst_name, 0, 0); + if (task) { + slapi_task_log_notice(task, "Unable to restart '%s'", + inst->inst_name); + } + } else { + slapi_mtn_be_enable(inst->inst_be); + instance_set_not_busy(inst); + } + } + } + + return return_value; +} + +int ldbm_back_ldbm2archive( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + char *directory = NULL; + int return_value = -1; + int task_flags = 0; + int run_from_cmdline = 0; + Slapi_Task *task; + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_SEQ_VAL, &directory ); + slapi_pblock_get( pb, SLAPI_TASK_FLAGS, &task_flags ); + li->li_flags = run_from_cmdline = (task_flags & TASK_RUNNING_FROM_COMMANDLINE); + + slapi_pblock_get( pb, SLAPI_BACKEND_TASK, &task ); + + /* No ldbm be's exist until we process the config information. */ + if (run_from_cmdline) { + mapping_tree_init(); + ldbm_config_load_dse_info(li); + } + /* to avoid conflict w/ import, do this check for commandline, as well */ + { + Object *inst_obj, *inst_obj2; + ldbm_instance *inst = NULL; + + /* server is up -- mark all backends busy */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + + /* check if an import/restore is already ongoing... */ + if (instance_set_busy(inst) != 0 || dblayer_in_import(inst) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm: '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + inst->inst_name, 0, 0); + if (task) { + slapi_task_log_notice(task, + "Backend '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + inst->inst_name); + } + + /* painfully, we have to clear the BUSY flags on the + * backends we'd already marked... + */ + for (inst_obj2 = objset_first_obj(li->li_instance_set); + inst_obj2 && (inst_obj2 != inst_obj); + inst_obj2 = objset_next_obj(li->li_instance_set, + inst_obj2)) { + inst = (ldbm_instance *)object_get_data(inst_obj2); + instance_set_not_busy(inst); + } + object_release(inst_obj2); + object_release(inst_obj); + return -1; + } + } + } + + if ( !directory || !*directory ) { + LDAPDebug( LDAP_DEBUG_ANY, "db2archive: no archive name\n", + 0, 0, 0 ); + return( -1 ); + } + if (0 != MKDIR(directory,SLAPD_DEFAULT_DIR_MODE) && EEXIST != errno) { + char *msg = dblayer_strerror(errno); + + LDAPDebug(LDAP_DEBUG_ANY, + "db2archive: mkdir(%s) failed; errno %i (%s)\n", + directory, errno, msg ? msg : "unknown"); + if (task) { + slapi_task_log_notice(task, + "mkdir(%s) failed; errno %i (%s)", + directory, errno, msg ? msg : "unknown"); + } + } + + /* start the database code up, do not attempt to perform recovery */ + if (run_from_cmdline && + 0 != dblayer_start(li,DBLAYER_ARCHIVE_MODE|DBLAYER_CMDLINE_MODE)) { + LDAPDebug(LDAP_DEBUG_ANY, "db2archive: Failed to init database\n", + 0, 0, 0); + if (task) { + slapi_task_log_notice(task, "Failed to init database"); + } + return( -1 ); + } + + /* tell it to archive */ + return_value = dblayer_backup(li, directory, task); + + /* close the database down again */ + if (run_from_cmdline && + 0 != dblayer_close(li,DBLAYER_ARCHIVE_MODE|DBLAYER_CMDLINE_MODE)) { + LDAPDebug(LDAP_DEBUG_ANY, "db2archive: Failed to close database\n", + 0, 0, 0); + if (task) { + slapi_task_log_notice(task, "Failed to close database"); + } + + /* The backup succeeded, so a failed close is not really a + total error... */ + /*return( -1 );*/ + } + + if (! run_from_cmdline) { + ldbm_instance *inst; + Object *inst_obj; + + /* none of these backends are busy anymore */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + instance_set_not_busy(inst); + } + } + + return return_value; +} diff --git a/ldap/servers/slapd/back-ldbm/attrcrypt.h b/ldap/servers/slapd/back-ldbm/attrcrypt.h new file mode 100644 index 00000000..b6ba50fb --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/attrcrypt.h @@ -0,0 +1,33 @@ +/** BEGIN COPYRIGHT BLOCK + * Portions copyright 2004 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* Private tructures and #defines used in the attribute encryption code. */ + +#ifndef _ATTRCRYPT_H_ +#define _ATTRCRYPT_H_ + +/* structure which holds our stuff in the attrinfo objects */ +struct attrcrypt_private +{ + int attrcrypt_cipher; +}; + +typedef struct _attrcrypt_cipher_entry +{ + int cipher_number; + char *cipher_display_name; + CK_MECHANISM_TYPE cipher_mechanism; + CK_MECHANISM_TYPE wrap_mechanism; + CK_MECHANISM_TYPE key_gen_mechanism; + int key_size; + int iv_length; +} attrcrypt_cipher_entry; + +extern attrcrypt_cipher_entry attrcrypt_cipher_list[]; + +/* The ciphers we support (used in attrcrypt_cipher above) */ +#define ATTRCRYPT_CIPHER_AES 1 +#define ATTRCRYPT_CIPHER_DES3 2 + +#endif /* _ATTRCRYPT_H_ */ diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h new file mode 100644 index 00000000..efe24ed7 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h @@ -0,0 +1,629 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* back-ldbm.h - ldap ldbm back-end header file */ + +#ifndef _BACK_LDBM_H_ +#define _BACK_LDBM_H_ + +#define SLAPD_LOGGING 1 + +#if defined(irix) || defined(AIX) || defined(HPUX11) || defined(OS_solaris) || defined(linux) +/* built-in 64-bit file I/O support */ +#define DB_USE_64LFS +#endif + +/* needed by at least HPUX and Solaris, to define off64_t */ +#ifdef DB_USE_64LFS +#define _LARGEFILE64_SOURCE +#endif + +/* A bunch of random system headers taken from all the source files, no source file should #include + any system headers now */ +#include <stdio.h> +#include <string.h> +#include <sys/types.h> +#include <errno.h> +#include <sys/stat.h> +#include <stdlib.h> +#include "prio.h" /* for PR_OpenDir etc */ +#include "prlog.h" /* for PR_ASSERT */ +/* The following cruft is for ldif2db only */ +#ifndef XP_WIN32 +#include <unistd.h> /* write/close (ldbm2ldif_write) */ +#else +#include <io.h> /* write/close (ldbm2ldif_write) */ +#endif +#include <fcntl.h> +#include <time.h> +/* And this cruft is from nextid.c */ +#ifndef _WIN32 +#include <sys/param.h> +#endif /* ! _WIN32 */ +#include <limits.h> /* Used in search.c (why?) */ + + + +#ifndef _WIN32 +/* for MAXPATHLEN */ +#include <sys/param.h> +#define MKDIR(path,mode) mkdir((path),(mode)) +#else +/* for mkdir */ +#include <direct.h> +#define MKDIR(path,mode) mkdir(path) +#endif + +#ifdef HPUX11 +#define __BIT_TYPES_DEFINED__ +typedef unsigned char u_int8_t; +typedef unsigned int u_int32_t; +typedef unsigned short u_int16_t; +#endif +#include "db.h" + +#define dptr data +#define dsize size + +#define ID2ENTRY "id2entry" /* main db file name: ID2ENTRY+LDBM_SUFFIX */ + +#define LDBM_SUFFIX_OLD ".db3" +#define LDBM_SUFFIX ".db4" + +#define MEGABYTE (1024 * 1024) +#define GIGABYTE (1024 * MEGABYTE) + + +/* include NSPR header files */ +#include "nspr.h" +#include "plhash.h" + +#include "slap.h" +#include "slapi-plugin.h" +#include "slapi-private.h" +#include "avl.h" +#include "ldaplog.h" +#include "portable.h" +#include "proto-slap.h" + +/* We should only change the LDBM_VERSION when the format of the db files + * is changing in some (possibly incompatible) way -- so we can detect and + * treat older ldbm versions. Thus, f.e., DS4.1 will still use the same + * LDBM_VERSION as 4.0 and so on... + * Don't make the length of LDBM_VERSION longer than LDBM_VERSION_MAXBUF - 1 + */ +#define LDBM_VERSION_MAXBUF 64 +#define LDBM_DATABASE_TYPE_NAME "ldbm database" +/* + * While we support both new and old idl index, + * we distinguish them by the following 2 macros. + * When we drop the old idl code, we eliminate LDBM_VERSION_OLD. + * bug #604922 + */ +/* To set new idl default, uncomment it. */ +#define USE_NEW_IDL 1 + +#define LDBM_VERSION_BASE "Netscape-ldbm/" +#define LDBM_VERSION "Netscape-ldbm/7.0" /* db42: new idl -> old */ +#define LDBM_VERSION_NEW "Netscape-ldbm/7.0_NEW" /* db42: new idl */ + /* used only when + * USE_NEW_IDL is + * NOT defined + */ +#define LDBM_VERSION_OLD "Netscape-ldbm/7.0_CLASSIC" /* db42: old idl */ + /* used only when + * USE_NEW_IDL is + * defined + */ +#define LDBM_VERSION_62 "Netscape-ldbm/6.2" /* db33: new idl */ +#define LDBM_VERSION_61 "Netscape-ldbm/6.1" /* db33: new idl */ +#define LDBM_VERSION_60 "Netscape-ldbm/6.0" /* db33: old idl */ + +#define LDBM_VERSION_50 "Netscape-ldbm/5.0" +#define LDBM_VERSION_40 "Netscape-ldbm/4.0" +#define LDBM_VERSION_30 "Netscape-ldbm/3.0" +#define LDBM_VERSION_31 "Netscape-ldbm/3.1" +#define LDBM_FILENAME_SUFFIX ".db4" +#define DBVERSION_FILENAME "DBVERSION" +#define DEFAULT_CACHE_SIZE (size_t)10485760 +#define DEFAULT_CACHE_ENTRIES -1 /* no limit */ +#define DEFAULT_DBCACHE_SIZE 1000000 +#define DEFAULT_MODE 0600 +#define DEFAULT_ALLIDSTHRESHOLD 4000 +#define DEFAULT_LOOKTHROUGHLIMIT 5000 +#define DEFAULT_IDL_TUNE 1 +#define DEFAULT_SEARCH_TUNE 0 +#define DEFAULT_IMPORT_INDEX_BUFFER_SIZE 0 +#define SUBLEN 3 +#define LDBM_CACHE_RETRY_COUNT 1000 /* Number of times we re-try a cache operation */ +#define IDL_FETCH_RETRY_COUNT 5 /* Number of times we re-try idl_fetch if it returns deadlock */ +#define IMPORT_SUBCOUNT_HASHTABLE_SIZE 500 /* Number of buckets in hash used to accumulate subcount for broody parents */ + +/* minimum max ids that a single index entry can map to in ldbm */ +#define SLAPD_LDBM_MIN_MAXIDS 4000 + +/* clear the following flag to suppress "database files do not exist" warning */ +extern int ldbm_warn_if_no_db; + +/* + * there is a single index for each attribute. these prefixes insure + * that there is no collision among keys. + */ +#define EQ_PREFIX '=' /* prefix for equality keys */ +#define APPROX_PREFIX '~' /* prefix for approx keys */ +#define SUB_PREFIX '*' /* prefix for substring keys */ +#define CONT_PREFIX '\\' /* prefix for continuation keys */ +#define RULE_PREFIX ':' /* prefix for matchingRule keys */ +#define PRES_PREFIX '+' + +/* Values for "disposition" value in idl_insert_key() */ +#define IDL_INSERT_NORMAL 1 +#define IDL_INSERT_ALLIDS 2 +#define IDL_INSERT_NOW_ALLIDS 3 + +#define DEFAULT_BLOCKSIZE 8192 + +/* + * The candidate list size at which it is cheaper to apply the filter test + * to the whole list than to continue ANDing in IDLs. + */ +#define FILTER_TEST_THRESHOLD (NIDS)10 + +/* flags to indicate what kind of startup the dblayer should do */ +#define DBLAYER_IMPORT_MODE 0x1 +#define DBLAYER_NORMAL_MODE 0x2 +#define DBLAYER_EXPORT_MODE 0x4 +#define DBLAYER_ARCHIVE_MODE 0x8 +#define DBLAYER_RESTORE_MODE 0x10 +#define DBLAYER_RESTORE_NO_RECOVERY_MODE 0x20 +#define DBLAYER_TEST_MODE 0x40 +#define DBLAYER_INDEX_MODE 0x80 +#define DBLAYER_CLEAN_RECOVER_MODE 0x100 + +#define DBLAYER_CMDLINE_MODE 0x1000 + +#define DBLAYER_RESTORE_MASK (DBLAYER_RESTORE_MODE|DBLAYER_RESTORE_NO_RECOVERY_MODE) + + +/* + * the id used in the indexes to refer to an entry + */ +typedef u_int32_t ID; +#define MAXID ((ID)-3) +#define NOID ((ID)-2) +#define ALLID ((ID)-1) + +/* + * effective only on idl_new_fetch + */ +#define NEW_IDL_NOOP 1 /* no need to fetch on new idl */ +#define NEW_IDL_NO_ALLID 2 /* force to return full idl (no allids) */ +#define NEW_IDL_DEFAULT 0 + +/* + * if the id of any backend instance is above the threshold, then warning + * message will be logged about the need of rebuilding the database in question + */ +#define ID_WARNING_THRESHOLD (MAXID * 0.9) + +/* + * Use this to count and index into an array of ID. + */ +typedef u_int32_t NIDS; + +/* + * This structure represents an id block on disk and an id list + * in core. + * + * The fields have the following meanings: + * + * b_nmax maximum number of ids in this block. if this is == ALLIDSBLOCK, + * then this block represents all ids. + * b_nids current number of ids in use in this block. if this + * is == INDBLOCK, then this block is an indirect block + * containing a list of other blocks containing actual ids. + * the list is terminated by an id of NOID. + * b_ids a list of the actual ids themselves + */ +typedef struct block { + NIDS b_nmax; /* max number of ids in this list */ +#define ALLIDSBLOCK 0 /* == 0 => this is an allid block */ + NIDS b_nids; /* current number of ids used */ +#define INDBLOCK 0 /* == 0 => this is an indirect blk */ + ID b_ids[1]; /* the ids - actually bigger */ +} Block, IDList; + +#define ALLIDS( idl ) ((idl)->b_nmax == ALLIDSBLOCK) +#define INDIRECT_BLOCK( idl ) ((idl)->b_nids == INDBLOCK) +#define IDL_NIDS(idl) (idl ? (idl)->b_nids : (NIDS)0) + +typedef size_t idl_iterator; + +/* small hashtable implementation used in the entry cache -- the table + * stores little identical structs, and relies on using a (void *) inside + * the struct to store linkage information. + */ +typedef int (*HashTestFn)(const void *, const void *); +typedef unsigned long (*HashFn)(const void *, size_t); +typedef struct { + u_long offset; /* offset of linkage info in user struct */ + u_long size; /* members in array below */ + HashFn hashfn; /* compute a hash value on a key */ + HashTestFn testfn; /* function to test if two entries are equal */ + void * slot[1]; /* actually much bigger */ +} Hashtable; + +/* use this macro to find the offset of the linkage info into your structure + * (required for hashtable to work correctly) + * HASHLOC(struct mything, linkptr) + */ +#define HASHLOC(mem, node) (u_long)&(((mem *)0L)->node) + +struct backentry { + Slapi_Entry *ep_entry; /* real entry */ + Slapi_Entry *ep_vlventry; + ID ep_id; /* entry id */ + char ep_state; /* state in the cache */ +#define ENTRY_STATE_DELETED 0x1 /* entry is marked as deleted */ +#define ENTRY_STATE_CREATING 0x2 /* entry is being created; don't touch it */ +#define ENTRY_STATE_NOTINCACHE 0x4 /* cache_add failed; not in the cache */ + int ep_refcnt; /* entry reference cnt */ + void * ep_dn_link; /* linkage for the 3 hash */ + void * ep_id_link; /* tables used for */ + void * ep_uuid_link; /* looking up entries */ + struct backentry *ep_lrunext; /* for the cache */ + struct backentry *ep_lruprev; /* for the cache */ + PRLock *ep_mutexp; /* protection for mods */ + size_t size; /* for cache tracking */ +}; + +/* for the in-core cache of entries */ +struct cache { + size_t c_maxsize; /* max size in bytes */ + size_t c_cursize; /* size in bytes */ + long c_maxentries; /* max entries allowed (-1: no limit) */ + long c_curentries; /* current # entries in cache */ + Hashtable *c_dntable; + Hashtable *c_idtable; +#ifdef UUIDCACHE_ON + Hashtable *c_uuidtable; +#endif + u_long c_hits; /* for analysis of hits/misses */ + u_long c_tries; + struct backentry *c_lruhead; /* add entries here */ + struct backentry *c_lrutail; /* remove entries here */ + PRLock *c_mutex; /* lock for cache operations */ + PRLock *c_emutexalloc_mutex; +}; + +/* various modules keep private data inside the attrinfo structure */ +typedef struct dblayer_private dblayer_private; +typedef struct dblayer_private_env dblayer_private_env; +typedef struct idl_private idl_private; +typedef struct attrcrypt_private attrcrypt_private; + + +/* for the cache of attribute information (which are indexed, etc.) */ +struct attrinfo { + char *ai_type; /* type name (cn, sn, ...) */ + int ai_indexmask; /* how the attr is indexed */ +#define INDEX_PRESENCE 0x01 +#define INDEX_EQUALITY 0x02 +#define INDEX_APPROX 0x04 +#define INDEX_SUB 0x08 +#define INDEX_UNKNOWN 0x10 +#define INDEX_FROMINIT 0x20 +#define INDEX_RULES 0x40 +#define INDEX_VLV 0x80 +#define INDEX_ANY (INDEX_PRESENCE | INDEX_EQUALITY | INDEX_APPROX | INDEX_SUB | INDEX_RULES | INDEX_VLV) + +#define INDEX_OFFLINE 0x1000 /* index is being generated, or + * has been created but not indexed + * yet. */ + +#define IS_INDEXED( a ) ( a & INDEX_ANY ) + void *ai_plugin; + char **ai_index_rules; /* matching rule OIDs */ + void *ai_dblayer; /* private data used by the dblayer code */ + PRInt32 ai_dblayer_count; /* used by the dblayer code */ + idl_private *ai_idl; /* private data used by the IDL code (eg locking the IDLs) */ + attrcrypt_private *ai_attrcrypt; /* private data used by the attribute encryption code (eg is it enabled or not) */ +}; + +#define MAXDBCACHE 20 + +struct id_array { + int ida_next_index; /*The next index that is free*/ + int ida_size; /*The size of this puppy*/ + ID *ida_ids; /*The array of ids*/ + +}; +typedef struct id_array Id_Array; + +struct _db_upgrade_info { + char* old_version_string; + int type; + int action; +}; +typedef struct _db_upgrade_info db_upgrade_info; +/* Values for dbversion_stuff->type */ +#define DBVERSION_COMPATIBLE 0x10 +#define DBVERSION_UPGRADABLE 0x20 +#define DBVERSION_SOL 0x40 +#define DBVERSION_OLD_IDL 0x1 +#define DBVERSION_NEW_IDL 0x2 + +/* Values for dbversion_stuff->action + return value */ +#define DBVERSION_NO_UPGRADE 0x0 +#define DBVERSION_NEED_IDL_OLD2NEW 0x100 +#define DBVERSION_NEED_IDL_NEW2OLD 0x200 +#define DBVERSION_UPGRADE_3_4 0x400 +#define DBVERSION_NOT_SUPPORTED 0x800 + +#define DBVERSION_TYPE 0x1 +#define DBVERSION_ACTION 0x2 + +struct ldbminfo { + int li_mode; + int li_lookthroughlimit; + int li_allidsthreshold; + char *li_directory; + int li_reslimit_lookthrough_handle; + size_t li_dbcachesize; + int li_dbncache; + int li_import_cache_autosize; /* % of free memory to use + * for the import caches + * (-1=default, 80% on cmd import) + * (0 = off) -- overrides + * import cache size settings */ + int li_cache_autosize; /* % of free memory to use + * for the combined caches + * (0 = off) -- overrides + * other cache size settings */ + int li_cache_autosize_split; /* % of li_cache_autosize to + * use for the libdb cache. + * the rest is split up among + * the instance entry caches */ + unsigned long li_cache_autosize_ec; /* new instances created while + * the server is up, should + * use this as the entry cache + * size (0 = autosize off) */ + size_t li_import_cachesize; /* size of the mpool for + * imports */ + PRLock *li_dbcache_mutex; + PRCondVar *li_dbcache_cv; + int li_shutdown; /* flag to tell any BE threads + * to end */ + PRLock *li_shutdown_mutex; /* protect shutdown flag */ + dblayer_private *li_dblayer_private; /* session ptr for databases */ + int li_noparentcheck; /* check if parent exists on + * add */ + + /* the next 2 fields are for the params that don't get changed until + * the server is restarted (used by the admin console) + */ + char *li_new_directory; + size_t li_new_dbcachesize; + + int li_new_dbncache; + + db_upgrade_info *upgrade_info; + int li_filter_bypass; /* bypass filter testing, + * when possible */ + int li_filter_bypass_check; /* check that filter bypass + * is doing the right thing */ + int li_use_vlv; /* use vlv indexes to short- + * circuit matches when + * possible */ + void *li_identity; /* The ldbm plugin needs to keep + * track of its identity so it can + * perform internal ops. Its + * identity is given to it when + * its init function is called. */ + + Objset *li_instance_set; /* A set containing the ldbm + * instances. */ + + PRLock *li_config_mutex; + + /* There are times when we need a pointer to the ldbm database + * plugin, so we will store a pointer to it here. Examples of + * when we need it are when we create a new instance and when + * we need the name of the plugin to do internal ops. */ + struct slapdplugin *li_plugin; + + /* factory extension markers for the Connection struct -- bulk import + * uses this to store state info on a Connection. + */ + int li_bulk_import_object; + int li_bulk_import_handle; + /* maximum number of pass before merging the files during an import */ + int li_maxpassbeforemerge; + + /* charray of attributes to exclude from LDIF export */ + char **li_attrs_to_exclude_from_export; + + int li_flags; + int li_fat_lock; /* 608146 -- make this configurable, first */ + int li_legacy_errcode; /* 615428 -- in case legacy err code is expected */ +}; + +/* li_flags could store these bits defined in ../slap.h + * task flag (pb_task_flags) * + * #define TASK_RUNNING_AS_TASK 0x0 + * #define TASK_RUNNING_FROM_COMMANDLINE 0x1 + */ +/* allow conf w/o CONFIG_FLAG_ALLOW_RUNNING_CHANGE to be updated */ +#define LI_FORCE_MOD_CONFIG 0x10 + +/* Structure used to hold stuff for the lifetime of an LDAP transaction */ +/* If we do clever stuff like LDAP transactions, we'll need a stack of TXN ID's */ +typedef struct back_txn back_txn; +struct back_txn { + DB_TXN *back_txn_txn; /* Transaction ID for the database */ +}; +typedef void * back_txnid; + +#define RETRY_TIMES 50 + +/* Structure used to communicate information about subordinatecount on import/upgrade */ +struct _import_subcount_stuff { + PLHashTable *hashtable; +}; +typedef struct _import_subcount_stuff import_subcount_stuff; + +/* Handy structures for modify operations */ + +struct _modify_context { + int new_entry_in_cache; + struct backentry *old_entry; + struct backentry *new_entry; + Slapi_Mods *smods; +}; +typedef struct _modify_context modify_context; + +#define INSTANCE_DB_SUFFIX "-db" +#define INSTANCE_CHANGELOG_SUFFIX "-changelog" + + +/* This structure was moved here from dblayer.c because the ldbm_instance + * structure uses the dblayer_handle structure. */ +struct tag_dblayer_handle; typedef struct tag_dblayer_handle dblayer_handle; +struct tag_dblayer_handle +{ + DB* dblayer_dbp; + PRLock *dblayer_lock; /* used when anyone wants exclusive access to a file */ + dblayer_handle *dblayer_handle_next; + void **dblayer_handle_ai_backpointer; /* Voodo magic pointer to the place where we store a + pointer to this handle in the attrinfo structure */ +}; + +/* This structure was moved here from perfctrs.c so the ldbm_instance structure + * could use it. */ +struct _perfctrs_private { +#if defined(_WIN32) + /* Handle to the shared memory object */ + HANDLE hMemory; + /* Handle to the update event */ + HANDLE hEvent; +#else + /* Nothing yet */ +#endif + /* Pointer to the shared memory */ + void *memory; +}; +typedef struct _perfctrs_private perfctrs_private; + +typedef struct _attrcrypt_state_private attrcrypt_state_private; + +/* flags for ldbm_instance */ +/* please lock inst_config_mutex before changing inst_flags */ +#define INST_FLAG_BUSY 0x0001 /* instance is doing an import or + * restore. */ +#define INST_FLAG_READONLY 0x0002 /* instance is truly readonly */ + +/* Structure used to hold instance specific information. */ +typedef struct ldbm_instance { + char *inst_name; /* Name given for this instance. */ + backend *inst_be; /* pointer back to the backend */ + struct ldbminfo *inst_li; /* pointer back to global info */ + int inst_flags; /* see above */ + + PRLock *inst_config_mutex; + + PRInt32 *inst_ref_count; /* Keeps track of how many operations + * are currently using this instance */ + + char *inst_dir_name; /* The name of the directory in the db + * directory that holds the index files + * for this instance. Relative to the + * parent of the instance name dir */ + char *inst_parent_dir_name; /* Absolute parent dir for this inst */ + + PRLock *inst_db_mutex; /* Used to synchronize modify operations + * on this instance. */ + + dblayer_handle *inst_handle_head; /* These are used to maintain a list */ + dblayer_handle *inst_handle_tail; /* of open db handles for this instance */ + PRLock *inst_handle_list_mutex; + + DB *inst_id2entry; /* id2entry for this instance. */ + + perfctrs_private inst_perf_private; /* Private data for the performace + * counters specific to this instance */ + attrcrypt_state_private *inst_attrcrypt_state_private; + int attrcrypt_configured; /* Are any attributes configured for encryption ? */ + + Avlnode *inst_attrs; /* Keeps track of what's indexed for + * this instance. */ + + struct cache inst_cache; /* The entry cache for this instance. */ + + PRLock *inst_nextid_mutex; + ID inst_nextid; + + PRCondVar *inst_indexer_cv; /* indexer thread cond var */ + PRThread *inst_indexer_tid; /* for the indexer thread */ + + long inst_cache_hits; /* used during imports to figure out when + * a pass should end. */ + long inst_cache_misses; + + char *inst_dataversion; /* The user data version tag. Used by + * replication. */ + dblayer_private_env *import_env; /* use a different DB_ENV for imports */ + int require_index; /* set to 1 to require an index be used + * in search */ +} ldbm_instance; + +/* + * This structure is passed through the PBlock from ldbm_back_search to + * ldbm_back_next_search_entry. It contains the candidate result set + * determined by ldbm_back_search, to be served up by ldbm_back_next_search_entry. + */ +typedef struct _back_search_result_set +{ + IDList* sr_candidates; /* the search results */ + idl_iterator sr_current; /* the current position in the search results */ + struct backentry* sr_entry; /* the last entry returned */ + int sr_lookthroughcount; /* how many have we examined? */ + int sr_lookthroughlimit; /* how many can we examine? */ + int sr_virtuallistview; /* is this a VLV Search */ + Slapi_Entry* sr_vlventry; /* a special VLV Entry for when the ACL check fails */ + int sr_flags; /* Magic flags, defined below */ +} back_search_result_set; +#define SR_FLAG_CAN_SKIP_FILTER_TEST 1 /* If set in sr_flags, means that we can safely skip the filter test */ + +#include "proto-back-ldbm.h" +#include "ldbm_config.h" + +/* flags used when adding/removing index items */ +#define BE_INDEX_ADD 1 +#define BE_INDEX_DEL 2 +#define BE_INDEX_PRESENCE 4 /* (w/DEL) remove the presence index */ +#define BE_INDEX_TOMBSTONE 8 /* Index entry as a tombstone */ +#define BE_INDEX_DONT_ENCRYPT 16 /* Disable any encryption if this flag is set */ + +/* Name of attribute type used for binder-based look through limit */ +#define LDBM_LOOKTHROUGHLIMIT_AT "nsLookThroughLimit" + +/* OIDs for attribute types used internally */ +#define LDBM_ENTRYDN_OID "2.16.840.1.113730.3.1.602" +#define LDBM_DNCOMP_OID "2.16.840.1.113730.3.1.603" +#define LDBM_PARENTID_OID "2.16.840.1.113730.3.1.604" +#define LDBM_ENTRYID_OID "2.16.840.1.113730.3.1.605" + +/* Name of psuedo attribute used to track default indexes */ +#define LDBM_PSEUDO_ATTR_DEFAULT ".default" + +/* for checking disk full errors. */ +#define LDBM_OS_ERR_IS_DISKFULL( err ) ((err)==ENOSPC || (err)==EFBIG) + +/* flag: open_flag for dblayer_get_index_file -> dblayer_open_file */ +#define DBOPEN_CREATE 0x1 /* oprinary mode: create a db file if needed */ + +/* whether we call fat lock or not [608146] */ +#define SERIALLOCK(li) (li->li_fat_lock) +#endif /* _back_ldbm_h_ */ diff --git a/ldap/servers/slapd/back-ldbm/backentry.c b/ldap/servers/slapd/back-ldbm/backentry.c new file mode 100644 index 00000000..f38953d8 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/backentry.c @@ -0,0 +1,89 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* backentry.c - wrapper routines to deal with entries */ + +#include "back-ldbm.h" + +void +backentry_free( struct backentry **bep ) +{ + struct backentry *ep; + if ( NULL == bep || NULL == *bep ) { + return; + } + ep = *bep; + if ( ep->ep_entry != NULL ) { + slapi_entry_free( ep->ep_entry ); + } + if ( ep->ep_mutexp != NULL ) { + PR_DestroyLock( ep->ep_mutexp ); + } + slapi_ch_free( (void**)&ep ); + *bep = NULL; +} + +struct backentry * +backentry_alloc() +{ + struct backentry *ec; + ec = (struct backentry *) slapi_ch_calloc( 1, sizeof(struct backentry) ) ; + ec->ep_state = ENTRY_STATE_NOTINCACHE; +#ifdef LDAP_CACHE_DEBUG + ec->debug_sig = 0x45454545; +#endif + return ec; +} + +void backentry_clear_entry( struct backentry *ep ) +{ + if (ep) + { + ep->ep_entry = NULL; + } +} + +struct backentry * +backentry_init( Slapi_Entry *e ) +{ + struct backentry *ep; + + ep = (struct backentry *) slapi_ch_calloc( 1, sizeof(struct backentry) ); + ep->ep_entry= e; + ep->ep_state = ENTRY_STATE_NOTINCACHE; +#ifdef LDAP_CACHE_DEBUG + ep->debug_sig = 0x23232323; +#endif + + return( ep ); +} + +struct backentry * +backentry_dup( struct backentry *e ) +{ + struct backentry *ec; + + ec = (struct backentry *) slapi_ch_calloc( 1, sizeof(struct backentry) ); + ec->ep_id = e->ep_id; + ec->ep_entry = slapi_entry_dup( e->ep_entry ); + ec->ep_state = ENTRY_STATE_NOTINCACHE; +#ifdef LDAP_CACHE_DEBUG + ec->debug_sig = 0x12121212; +#endif + + return( ec ); +} + +char * +backentry_get_ndn(const struct backentry *e) +{ + return (char *)slapi_sdn_get_ndn(slapi_entry_get_sdn_const(e->ep_entry)); +} + +const Slapi_DN * +backentry_get_sdn(const struct backentry *e) +{ + return slapi_entry_get_sdn_const(e->ep_entry); +} diff --git a/ldap/servers/slapd/back-ldbm/cache.c b/ldap/servers/slapd/back-ldbm/cache.c new file mode 100644 index 00000000..e55bddb2 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/cache.c @@ -0,0 +1,1195 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* cache.c - routines to maintain an in-core cache of entries */ + +#include "back-ldbm.h" + +#ifdef DEBUG +#define LDAP_CACHE_DEBUG +/* #define LDAP_CACHE_DEBUG_LRU */ /* causes slowdown */ +#endif + +/* cache can't get any smaller than this (in bytes) */ +#define MINCACHESIZE (size_t)200000 + +/* don't let hash be smaller than this # of slots */ +#define MINHASHSIZE 1024 + +/* + * the cache has three entry points (ways to find things): + * + * by entry e.g., if you already have an entry from the cache + * and want to delete it. (really by entry ptr) + * by dn e.g., when looking for the base object of a search + * by id e.g., for search candidates + * by uniqueid + * + * these correspond to three different avl trees that are maintained. + * those avl trees are being destroyed as we speak. + */ + +#ifdef LDAP_CACHE_DEBUG +#define ASSERT(_x) do { \ + if (!(_x)) { \ + LDAPDebug(LDAP_DEBUG_ANY, "BAD CACHE ASSERTION at %s/%d: %s\n", \ + __FILE__, __LINE__, #_x); \ + *(char *)0L = 23; \ + } \ +} while (0) +#define LOG(_a, _x1, _x2, _x3) LDAPDebug(LDAP_DEBUG_CACHE, _a, _x1, _x2, _x3) +#else +#define ASSERT(_x) ; +#define LOG(_a, _x1, _x2, _x3) ; +#endif + + +/***** tiny hashtable implementation *****/ + +#define HASH_VALUE(_key, _keylen) \ + ((ht->hashfn == NULL) ? (*(unsigned int *)(_key)) : \ + ((*ht->hashfn)(_key, _keylen))) +#define HASH_NEXT(ht, entry) (*(void **)((char *)(entry) + (ht)->offset)) + +static int entry_same_id(const void *e, const void *k) +{ + return (((struct backentry *)e)->ep_id == *(ID *)k); +} + +static unsigned long dn_hash(const void *key, size_t keylen) +{ + unsigned char *x = (unsigned char *)key; + ssize_t i; + unsigned long val = 0; + + for (i = keylen-1; i >= 0; i--) + val += ((val << 5) + (*x++)) & 0xffffffff; + return val; +} + +#ifdef UUIDCACHE_ON +static unsigned long uuid_hash(const void *key, size_t keylen) +{ + unsigned char *x = (unsigned char *)key; + size_t i; + unsigned long val = 0; + + for (i = 0; i < keylen; i++, x++) { + char c = (*x <= '9' ? (*x - '0') : (*x - 'A' + 10)); + val = ((val << 4) ^ (val >> 28) ^ c) & 0xffffffff; + } + return val; +} + +static int entry_same_uuid(const void *e, const void *k) +{ + struct backentry *be = (struct backentry *)e; + const char *uuid = slapi_entry_get_uniqueid(be->ep_entry); + + return (strcmp(uuid, (char *)k) == 0); +} +#endif + +static int entry_same_dn(const void *e, const void *k) +{ + struct backentry *be = (struct backentry *)e; + const char *ndn = slapi_sdn_get_ndn(backentry_get_sdn(be)); + + return (strcmp(ndn, (char *)k) == 0); +} + +Hashtable *new_hash(u_long size, u_long offset, HashFn hfn, + HashTestFn tfn) +{ + static u_long prime[] = { 3, 5, 7, 11, 13, 17, 19 }; + Hashtable *ht; + int ok = 0, i; + + if (size < MINHASHSIZE) + size = MINHASHSIZE; + /* move up to nearest relative prime (it's a statistical thing) */ + size |= 1; + do { + ok = 1; + for (i = 0; i < (sizeof(prime) / sizeof(prime[0])); i++) + if (!(size % prime[i])) + ok = 0; + if (!ok) + size += 2; + } while (!ok); + + ht = (Hashtable*)slapi_ch_calloc(1, sizeof(Hashtable) + size*sizeof(void *)); + if (!ht) + return NULL; + ht->size = size; + ht->offset = offset; + ht->hashfn = hfn; + ht->testfn = tfn; + /* calloc zeroes out the slots automagically */ + return ht; +} + +/* adds an entry to the hash -- returns 1 on success, 0 if the key was + * already there (filled into 'alt' if 'alt' is not NULL) + */ +int add_hash(Hashtable *ht, void *key, size_t keylen, void *entry, + void **alt) +{ + u_long val, slot; + void *e; + + val = HASH_VALUE(key, keylen); + slot = (val % ht->size); + /* first, check if this key is already in the table */ + e = ht->slot[slot]; + while (e) { + if ((*ht->testfn)(e, key)) { + /* ack! already in! */ + if (alt) + *alt = e; + return 0; + } + e = HASH_NEXT(ht, e); + } + /* ok, it's not already there, so add it */ + HASH_NEXT(ht, entry) = ht->slot[slot]; + ht->slot[slot] = entry; + return 1; +} + +/* returns 1 if the item was found, and puts a ptr to it in 'entry' */ +int find_hash(Hashtable *ht, const void *key, size_t keylen, void **entry) +{ + u_long val, slot; + void *e; + + val = HASH_VALUE(key, keylen); + slot = (val % ht->size); + e = ht->slot[slot]; + while (e) { + if ((*ht->testfn)(e, key)) { + *entry = e; + return 1; + } + e = HASH_NEXT(ht, e); + } + /* no go */ + *entry = NULL; + return 0; +} + +/* returns 1 if the item was found and removed */ +int remove_hash(Hashtable *ht, const void *key, size_t keylen) +{ + u_long val, slot; + void *e, *laste = NULL; + + val = HASH_VALUE(key, keylen); + slot = (val % ht->size); + e = ht->slot[slot]; + while (e) { + if ((*ht->testfn)(e, key)) { + /* remove this one */ + if (laste) + HASH_NEXT(ht, laste) = HASH_NEXT(ht, e); + else + ht->slot[slot] = HASH_NEXT(ht, e); + HASH_NEXT(ht, e) = NULL; + return 1; + } + laste = e; + e = HASH_NEXT(ht, e); + } + /* nope */ + return 0; +} + +/* hashtable distribution stats -- + * slots: # of slots in the hashtable + * total_entries: # of entries in the hashtable + * max_entries_per_slot: highest number of chained entries in a single slot + * slot_stats: if X is the number of entries in a given slot, then + * slot_stats[X] will hold the number of slots that held X entries + */ +static void hash_stats(Hashtable *ht, u_long *slots, int *total_entries, + int *max_entries_per_slot, int **slot_stats) +{ +#define MAX_SLOT_STATS 50 + u_long i; + int x; + void *e; + + *slot_stats = (int *)slapi_ch_malloc(MAX_SLOT_STATS * sizeof(int)); + for (i = 0; i < MAX_SLOT_STATS; i++) + (*slot_stats)[i] = 0; + + *slots = ht->size; + *max_entries_per_slot = 0; + *total_entries = 0; + for (i = 0; i < ht->size; i++) { + e = ht->slot[i]; + x = 0; + while (e) { + x++; + (*total_entries)++; + e = HASH_NEXT(ht, e); + } + if (x < MAX_SLOT_STATS) + (*slot_stats)[x]++; + if (x > *max_entries_per_slot) + *max_entries_per_slot = x; + } +} + + +/***** add/remove entries to/from the LRU list *****/ + +#ifdef LDAP_CACHE_DEBUG_LRU +/* for debugging -- painstakingly verify the lru list is ok -- if 'in' is + * true, then entry 'e' should be in the list right now; otherwise, it + * should NOT be in the list. + */ +static void lru_verify(struct cache *cache, struct backentry *e, int in) +{ + int is_in = 0; + int count = 0; + struct backentry *ep; + + ep = cache->c_lruhead; + while (ep) { + count++; + if (ep == e) { + is_in = 1; + } + if (ep->ep_lruprev) { + ASSERT(ep->ep_lruprev->ep_lrunext == ep); + } else { + ASSERT(ep == cache->c_lruhead); + } + if (ep->ep_lrunext) { + ASSERT(ep->ep_lrunext->ep_lruprev == ep); + } else { + ASSERT(ep == cache->c_lrutail); + } + + ep = ep->ep_lrunext; + } + ASSERT(is_in == in); +} +#endif + +/* assume lock is held */ +static void lru_detach(struct cache *cache, struct backentry *e) +{ +#ifdef LDAP_CACHE_DEBUG_LRU + lru_verify(cache, e, 1); +#endif + if (e->ep_lruprev) + { + e->ep_lruprev->ep_lrunext = NULL; + cache->c_lrutail = e->ep_lruprev; + } + else + { + cache->c_lruhead = NULL; + cache->c_lrutail = NULL; + } +#ifdef LDAP_CACHE_DEBUG_LRU + lru_verify(cache, e, 0); +#endif +} + +/* assume lock is held */ +static void lru_delete(struct cache *cache, struct backentry *e) +{ +#ifdef LDAP_CACHE_DEBUG_LRU + lru_verify(cache, e, 1); +#endif + if (e->ep_lruprev) + e->ep_lruprev->ep_lrunext = e->ep_lrunext; + else + cache->c_lruhead = e->ep_lrunext; + if (e->ep_lrunext) + e->ep_lrunext->ep_lruprev = e->ep_lruprev; + else + cache->c_lrutail = e->ep_lruprev; +#ifdef LDAP_CACHE_DEBUG_LRU + e->ep_lrunext = e->ep_lruprev = NULL; + lru_verify(cache, e, 0); +#endif +} + +/* assume lock is held */ +static void lru_add(struct cache *cache, struct backentry *e) +{ +#ifdef LDAP_CACHE_DEBUG_LRU + lru_verify(cache, e, 0); +#endif + e->ep_lruprev = NULL; + e->ep_lrunext = cache->c_lruhead; + cache->c_lruhead = e; + if (e->ep_lrunext) + e->ep_lrunext->ep_lruprev = e; + if (! cache->c_lrutail) + cache->c_lrutail = e; +#ifdef LDAP_CACHE_DEBUG_LRU + lru_verify(cache, e, 1); +#endif +} + + +/***** cache overhead *****/ + +static int cache_remove_int(struct cache *cache, struct backentry *e); + +static void cache_make_hashes(struct cache *cache) +{ + u_long hashsize = (cache->c_maxentries > 0) ? cache->c_maxentries : + (cache->c_maxsize/512); + + cache->c_dntable = new_hash(hashsize, + HASHLOC(struct backentry, ep_dn_link), + dn_hash, entry_same_dn); + cache->c_idtable = new_hash(hashsize, + HASHLOC(struct backentry, ep_id_link), + NULL, entry_same_id); +#ifdef UUIDCACHE_ON + cache->c_uuidtable = new_hash(hashsize, + HASHLOC(struct backentry, ep_uuid_link), + uuid_hash, entry_same_uuid); +#endif +} + +/* initialize the cache */ +int cache_init(struct cache *cache, size_t maxsize, long maxentries) +{ + LDAPDebug(LDAP_DEBUG_TRACE, "=> cache_init\n", 0, 0, 0); + cache->c_maxsize = maxsize; + cache->c_maxentries = maxentries; + cache->c_cursize = cache->c_curentries = 0; + cache->c_hits = cache->c_tries = 0; + cache->c_lruhead = cache->c_lrutail = NULL; + cache_make_hashes(cache); + + if (((cache->c_mutex = PR_NewLock()) == NULL) || + ((cache->c_emutexalloc_mutex = PR_NewLock()) == NULL)) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: cache_init: PR_NewLock failed\n", + 0, 0, 0); + return 0; + } + LDAPDebug(LDAP_DEBUG_TRACE, "<= cache_init\n", 0, 0, 0); + return 1; +} + +#define CACHE_FULL(cache) \ + (((cache)->c_cursize > (cache)->c_maxsize) || \ + (((cache)->c_maxentries > 0) && \ + ((cache)->c_curentries > cache->c_maxentries))) + + +/* clear out the cache to make room for new entries + * you must be holding cache->c_mutex !! + * return a pointer on the list of entries that get kicked out + * of the cache. + * These entries should be freed outside of the cache->c_mutex + */ +static struct backentry * cache_flush(struct cache *cache) +{ + struct backentry *e = NULL; + + LOG("=> cache_flush\n", 0, 0, 0); + + /* all entries on the LRU list are guaranteed to have a refcnt = 0 + * (iow, nobody's using them), so just delete from the tail down + * until the cache is a managable size again. + * (cache->c_mutex is locked when we enter this) + */ + while ((cache->c_lrutail != NULL) && CACHE_FULL(cache)) { + if (e == NULL) + { + e = cache->c_lrutail; + } + else + { + e = e->ep_lruprev; + } + ASSERT(e->ep_refcnt == 0); + e->ep_refcnt++; + if (cache_remove_int(cache, e) < 0) { + LDAPDebug(LDAP_DEBUG_ANY, "cache flush: unable to delete entry\n", + 0, 0, 0); + break; + } + if(e == cache->c_lruhead) { + break; + } + } + if (e) + lru_detach(cache, e); + LOG("<= cache_flush (down to %lu entries, %lu bytes)\n", cache->c_curentries, + cache->c_cursize, 0); + return e; +} + +/* remove everything from the cache */ +static void cache_clear_int(struct cache *cache) +{ + struct backentry *eflush = NULL; + struct backentry *eflushtemp = NULL; + size_t size = cache->c_maxsize; + + cache->c_maxsize = 0; + eflush = cache_flush(cache); + while (eflush) + { + eflushtemp = eflush->ep_lrunext; + backentry_free(&eflush); + eflush = eflushtemp; + } + cache->c_maxsize = size; + if (cache->c_curentries > 0) { + LDAPDebug(LDAP_DEBUG_ANY, "somehow, there are still %ld entries " + "in the entry cache. :/\n", cache->c_curentries, 0, 0); + } +} + +void cache_clear(struct cache *cache) +{ + PR_Lock(cache->c_mutex); + cache_clear_int(cache); + PR_Unlock(cache->c_mutex); +} + +static void erase_cache(struct cache *cache) +{ + cache_clear_int(cache); + slapi_ch_free((void **)&cache->c_dntable); + slapi_ch_free((void **)&cache->c_idtable); +#ifdef UUIDCACHE_ON + slapi_ch_free((void **)&cache->c_uuidtable); +#endif +} + +/* to be used on shutdown or when destroying a backend instance */ +void cache_destroy_please(struct cache *cache) +{ + erase_cache(cache); + PR_DestroyLock(cache->c_mutex); + PR_DestroyLock(cache->c_emutexalloc_mutex); +} + +void cache_set_max_size(struct cache *cache, size_t bytes) +{ + struct backentry *eflush = NULL; + struct backentry *eflushtemp = NULL; + + if (bytes < MINCACHESIZE) { + bytes = MINCACHESIZE; + LDAPDebug(LDAP_DEBUG_ANY, + "WARNING -- Minimum cache size is %lu -- rounding up\n", + MINCACHESIZE, 0, 0); + } + PR_Lock(cache->c_mutex); + cache->c_maxsize = bytes; + LOG("entry cache size set to %lu\n", bytes, 0, 0); + /* check for full cache, and clear out if necessary */ + if (CACHE_FULL(cache)) + eflush = cache_flush(cache); + while (eflush) + { + eflushtemp = eflush->ep_lrunext; + backentry_free(&eflush); + eflush = eflushtemp; + } + if (cache->c_curentries < 50) { + /* there's hardly anything left in the cache -- clear it out and + * resize the hashtables for efficiency. + */ + erase_cache(cache); + cache_make_hashes(cache); + } + PR_Unlock(cache->c_mutex); + if (! dblayer_is_cachesize_sane(&bytes)) { + LDAPDebug(LDAP_DEBUG_ANY, + "WARNING -- Possible CONFIGURATION ERROR -- cachesize " + "(%lu) may be configured to use more than the available " + "physical memory.\n", bytes, 0, 0); + } +} + +void cache_set_max_entries(struct cache *cache, long entries) +{ + struct backentry *eflush = NULL; + struct backentry *eflushtemp = NULL; + + /* this is a dumb remnant of pre-5.0 servers, where the cache size + * was given in # entries instead of memory footprint. hopefully, + * we can eventually drop this. + */ + PR_Lock(cache->c_mutex); + cache->c_maxentries = entries; + if (entries >= 0) { + LOG("entry cache entry-limit set to %lu\n", entries, 0, 0); + } else { + LOG("entry cache entry-limit turned off\n", 0, 0, 0); + } + + /* check for full cache, and clear out if necessary */ + if (CACHE_FULL(cache)) + eflush = cache_flush(cache); + PR_Unlock(cache->c_mutex); + while (eflush) + { + eflushtemp = eflush->ep_lrunext; + backentry_free(&eflush); + eflush = eflushtemp; + } +} + +size_t cache_get_max_size(struct cache *cache) +{ + size_t n; + + PR_Lock(cache->c_mutex); + n = cache->c_maxsize; + PR_Unlock(cache->c_mutex); + return n; +} + +long cache_get_max_entries(struct cache *cache) +{ + long n; + + PR_Lock(cache->c_mutex); + n = cache->c_maxentries; + PR_Unlock(cache->c_mutex); + return n; +} + +/* determine the general size of a cache entry */ +static size_t cache_entry_size(struct backentry *e) +{ + size_t size = 0; + + if (e->ep_entry) + size += slapi_entry_size(e->ep_entry); + if (e->ep_vlventry) + size += slapi_entry_size(e->ep_vlventry); + /* cannot size ep_mutexp (PRLock) */ + size += sizeof(struct backentry); + return size; +} + +/* the monitor code wants to be able to safely fetch the cache stats -- + * if it ever wants to pull out more info, we might want to change all + * these u_long *'s to a struct + */ +void cache_get_stats(struct cache *cache, u_long *hits, u_long *tries, + long *nentries, long *maxentries, + size_t *size, size_t *maxsize) +{ + PR_Lock(cache->c_mutex); + if (hits) *hits = cache->c_hits; + if (tries) *tries = cache->c_tries; + if (nentries) *nentries = cache->c_curentries; + if (maxentries) *maxentries = cache->c_maxentries; + if (size) *size = cache->c_cursize; + if (maxsize) *maxsize = cache->c_maxsize; + PR_Unlock(cache->c_mutex); +} + +void cache_debug_hash(struct cache *cache, char **out) +{ + u_long slots; + int total_entries, max_entries_per_slot, *slot_stats; + int i, j; + Hashtable *ht; + char *name; + + PR_Lock(cache->c_mutex); + *out = (char *)slapi_ch_malloc(1024); + **out = 0; + + for (i = 0; i < 3; i++) { + if (i > 0) + sprintf(*out + strlen(*out), "; "); + switch(i) { + case 0: + ht = cache->c_dntable; + name = "dn"; + break; + case 1: + ht = cache->c_idtable; + name = "id"; + break; +#ifdef UUIDCACHE_ON + case 2: + default: + ht = cache->c_uuidtable; + name = "uuid"; + break; +#endif + } + hash_stats(ht, &slots, &total_entries, &max_entries_per_slot, + &slot_stats); + sprintf(*out + strlen(*out), "%s hash: %lu slots, %d entries (%d max " + "entries per slot) -- ", name, slots, total_entries, + max_entries_per_slot); + for (j = 0; j <= max_entries_per_slot; j++) + sprintf(*out + strlen(*out), "%d[%d] ", j, slot_stats[j]); + slapi_ch_free((void **)&slot_stats); + } + PR_Unlock(cache->c_mutex); +} + + +/***** general-purpose cache stuff *****/ + +/* remove an entry from the cache */ +/* you must be holding c_mutex !! */ +static int cache_remove_int(struct cache *cache, struct backentry *e) +{ + int ret = 1; /* assume not in cache */ + const char *ndn; +#ifdef UUIDCACHE_ON + const char *uuid; +#endif + + LOG("=> cache_remove (%s)\n", backentry_get_ndn(e), 0, 0); + if (e->ep_state & ENTRY_STATE_NOTINCACHE) + { + return ret; + } + + /* remove from all hashtables -- this function may be called from places + * where the entry isn't in all the tables yet, so we don't care if any + * of these return errors. + */ + ndn = slapi_sdn_get_ndn(backentry_get_sdn(e)); + if (remove_hash(cache->c_dntable, (void *)ndn, strlen(ndn))) + { + ret = 0; + } + else + { + LOG("remove %s from dn hash failed\n", ndn, 0, 0); + } + if (remove_hash(cache->c_idtable, &(e->ep_id), sizeof(ID))) + { + ret = 0; + } + else + { + LOG("remove %d from id hash failed\n", e->ep_id, 0, 0); + } +#ifdef UUIDCACHE_ON + uuid = slapi_entry_get_uniqueid(e->ep_entry); + if (remove_hash(cache->c_uuidtable, (void *)uuid, strlen(uuid))) + { + ret = 0; + } + else + { + LOG("remove %d from uuid hash failed\n", uuid, 0, 0); + } +#endif + if (ret == 0) { + /* won't be on the LRU list since it has a refcount on it */ + /* adjust cache size */ + cache->c_cursize -= e->size; + cache->c_curentries--; + LOG("<= cache_remove (size %lu): cache now %lu entries, %lu bytes\n", + e->size, cache->c_curentries, cache->c_cursize); + } + + /* mark for deletion (will be erased when refcount drops to zero) */ + e->ep_state |= ENTRY_STATE_DELETED; + LOG("<= cache_remove: %d\n", ret, 0, 0); + return ret; +} + +/* remove an entry from the cache. + * you must have a refcount on e (iow, fetched via cache_find_*). the + * entry is removed from the cache, but NOT freed! you are responsible + * for freeing the entry yourself when done with it, preferrably via + * cache_return (called AFTER cache_remove). some code still does this + * via backentry_free, which is okay, as long as you know you're the only + * thread holding a reference to the deleted entry. + * returns: 0 on success + * 1 if the entry wasn't in the cache at all (not even partially) + */ +int cache_remove(struct cache *cache, struct backentry *e) +{ + int ret; + + PR_Lock(cache->c_mutex); + ASSERT(e->ep_refcnt > 0); + ret = cache_remove_int(cache, e); + PR_Unlock(cache->c_mutex); + return ret; +} + +/* replace an entry in the cache. + * returns: 0 on success + * 1 if the entry wasn't in the cache + */ +int cache_replace(struct cache *cache, struct backentry *olde, + struct backentry *newe) +{ + int found; + const char *oldndn; + const char *newndn; +#ifdef UUIDCACHE_ON + const char *olduuid; + const char *newuuid; +#endif + + LOG("=> cache_replace (%s) -> (%s)\n", backentry_get_ndn(olde), + backentry_get_ndn(newe), 0); + + /* remove from all hashtables -- this function may be called from places + * where the entry isn't in all the tables yet, so we don't care if any + * of these return errors. + */ + oldndn = slapi_sdn_get_ndn(backentry_get_sdn(olde)); +#ifdef UUIDCACHE_ON + olduuid = slapi_entry_get_uniqueid(olde->ep_entry); + newuuid = slapi_entry_get_uniqueid(newe->ep_entry); +#endif + newndn = slapi_sdn_get_ndn(backentry_get_sdn(newe)); + PR_Lock(cache->c_mutex); + + /* + * First, remove the old entry from all the hashtables. + * If the old entry is in cache but not in at least one of the + * cache tables, operation error + */ + if ( (olde->ep_state & ENTRY_STATE_NOTINCACHE) == 0 ) { + + found = remove_hash(cache->c_dntable, (void *)oldndn, strlen(oldndn)); + found &= remove_hash(cache->c_idtable, &(olde->ep_id), sizeof(ID)); +#ifdef UUIDCACHE_ON + found &= remove_hash(cache->c_uuidtable, (void *)olduuid, strlen(olduuid)); +#endif + if (!found) { + LOG("cache replace: cache index tables out of sync\n", 0, 0, 0); + PR_Unlock(cache->c_mutex); + return 1; + } + } + if (! entry_same_dn(newe, (void *)oldndn) && + (newe->ep_state & ENTRY_STATE_NOTINCACHE) == 0) { + /* if we're doing a modrdn, the new entry can be in the dn table + * already, so we need to remove that too. + */ + if (remove_hash(cache->c_dntable, (void *)newndn, strlen(newndn))) + { + cache->c_cursize -= newe->size; + cache->c_curentries--; + LOG("cache replace remove entry size %lu\n", newe->size, 0, 0); + } + } + + /* now, add the new entry to the hashtables */ + /* (probably don't need such extensive error handling, once this has been + * tested enough that we believe it works.) + */ + if (!add_hash(cache->c_dntable, (void *)newndn, strlen(newndn), newe, NULL)) { + LOG("cache replace: can't add dn\n", 0, 0, 0); + PR_Unlock(cache->c_mutex); + return 1; + } + if (!add_hash(cache->c_idtable, &(newe->ep_id), sizeof(ID), newe, NULL)) { + LOG("cache replace: can't add id\n", 0, 0, 0); + remove_hash(cache->c_dntable, (void *)newndn, strlen(newndn)); + PR_Unlock(cache->c_mutex); + return 1; + } +#ifdef UUIDCACHE_ON + if (newuuid && !add_hash(cache->c_uuidtable, (void *)newuuid, strlen(newuuid), + newe, NULL)) { + LOG("cache replace: can't add uuid\n", 0, 0, 0); + remove_hash(cache->c_dntable, (void *)newndn, strlen(newndn)); + remove_hash(cache->c_idtable, &(newe->ep_id), sizeof(ID)); + PR_Unlock(cache->c_mutex); + return 1; + } +#endif + /* adjust cache meta info */ + newe->ep_refcnt = 1; + newe->size = cache_entry_size(newe); + cache->c_cursize += (newe->size - olde->size); + olde->ep_state = ENTRY_STATE_DELETED; + newe->ep_state = 0; + PR_Unlock(cache->c_mutex); + LOG("<= cache_replace OK, cache size now %lu cache count now %ld\n", + cache->c_cursize, cache->c_curentries, 0); + return 0; +} + +/* call this when you're done with an entry that was fetched via one of + * the cache_find_* calls. + */ +void cache_return(struct cache *cache, struct backentry **bep) +{ + struct backentry *eflush = NULL; + struct backentry *eflushtemp = NULL; + struct backentry *e; + if (NULL == bep || NULL == *bep) + { + LOG("=> cache_return (null entry)\n", 0, 0, 0); + return; + } + e = *bep; + LOG("=> cache_return (%s) entry count: %d, entry in cache:%ld\n", backentry_get_ndn(e), e->ep_refcnt, cache->c_curentries); + + PR_Lock(cache->c_mutex); + if (e->ep_state & ENTRY_STATE_NOTINCACHE) + { + backentry_free(bep); + } + else + { + ASSERT(e->ep_refcnt > 0); + if (! --e->ep_refcnt) { + if (e->ep_state & ENTRY_STATE_DELETED) { + backentry_free(bep); + } else { + lru_add(cache, e); + /* the cache might be overfull... */ + if (CACHE_FULL(cache)) + eflush = cache_flush(cache); + } + } + } + PR_Unlock(cache->c_mutex); + while (eflush) + { + eflushtemp = eflush->ep_lrunext; + backentry_free(&eflush); + eflush = eflushtemp; + } +} + + +/* lookup entry by DN (assume cache lock is held) */ +struct backentry *cache_find_dn(struct cache *cache, const char *dn, unsigned long ndnlen) +{ + struct backentry *e; + + LOG("=> cache_find_dn (%s)\n", dn, 0, 0); + + /*entry normalized by caller (dn2entry.c) */ + PR_Lock(cache->c_mutex); + if (find_hash(cache->c_dntable, (void *)dn, ndnlen, (void **)&e)) { + /* need to check entry state */ + if (e->ep_state != 0) { + /* entry is deleted or not fully created yet */ + PR_Unlock(cache->c_mutex); + LOG("<= cache_find_dn (NOT FOUND)\n", 0, 0, 0); + return NULL; + } + if (e->ep_refcnt == 0) + lru_delete(cache, e); + e->ep_refcnt++; + cache->c_hits++; + } + cache->c_tries++; + PR_Unlock(cache->c_mutex); + + LOG("<= cache_find_dn (%sFOUND)\n", e ? "" : "NOT ", 0, 0); + return e; +} + + +/* lookup an entry in the cache by its id# (you must return it later) */ +struct backentry *cache_find_id(struct cache *cache, ID id) +{ + struct backentry *e; + + LOG("=> cache_find_id (%lu)\n", (u_long)id, 0, 0); + + PR_Lock(cache->c_mutex); + if (find_hash(cache->c_idtable, &id, sizeof(ID), (void **)&e)) { + /* need to check entry state */ + if (e->ep_state != 0) { + /* entry is deleted or not fully created yet */ + PR_Unlock(cache->c_mutex); + LOG("<= cache_find_id (NOT FOUND)\n", 0, 0, 0); + return NULL; + } + if (e->ep_refcnt == 0) + lru_delete(cache, e); + e->ep_refcnt++; + cache->c_hits++; + } + cache->c_tries++; + PR_Unlock(cache->c_mutex); + + LOG("<= cache_find_id (%sFOUND)\n", e ? "" : "NOT ", 0, 0); + return e; +} + +#ifdef UUIDCACHE_ON +/* lookup an entry in the cache by it's uuid (you must return it later) */ +struct backentry *cache_find_uuid(struct cache *cache, const char *uuid) +{ + struct backentry *e; + + LOG("=> cache_find_uuid (%s)\n", uuid, 0, 0); + + PR_Lock(cache->c_mutex); + if (find_hash(cache->c_uuidtable, uuid, strlen(uuid), (void **)&e)) { + /* need to check entry state */ + if (e->ep_state != 0) { + /* entry is deleted or not fully created yet */ + PR_Unlock(cache->c_mutex); + LOG("<= cache_find_uuid (NOT FOUND)\n", 0, 0, 0); + return NULL; + } + if (e->ep_refcnt == 0) + lru_delete(cache, e); + e->ep_refcnt++; + cache->c_hits++; + } + cache->c_tries++; + PR_Unlock(cache->c_mutex); + + LOG("<= cache_find_uuid (%sFOUND)\n", e ? "" : "NOT ", 0, 0); + return e; +} +#endif + +/* add an entry to the cache */ +static int cache_add_int(struct cache *cache, struct backentry *e, int state, + struct backentry **alt) +{ + struct backentry *eflush = NULL; + struct backentry *eflushtemp = NULL; + const char *ndn = slapi_sdn_get_ndn(backentry_get_sdn(e)); +#ifdef UUIDCACHE_ON + const char *uuid = slapi_entry_get_uniqueid(e->ep_entry); +#endif + struct backentry *my_alt; + int already_in = 0; + + LOG("=> cache_add_int( \"%s\", %ld )\n", backentry_get_ndn(e), + e->ep_id, 0); + + PR_Lock(cache->c_mutex); + if (! add_hash(cache->c_dntable, (void *)ndn, strlen(ndn), e, + (void **)&my_alt)) { + LOG("entry \"%s\" already in dn cache\n", backentry_get_ndn(e), 0, 0); + /* add_hash filled in 'my_alt' if necessary */ + if (my_alt == e) + { + if ((e->ep_state & ENTRY_STATE_CREATING) && (state == 0)) + { + /* attempting to "add" an entry that's already in the cache, + * and the old entry was a placeholder and the new one isn't? + * sounds like a confirmation of a previous add! + */ + LOG("confirming a previous add\n", 0, 0, 0); + already_in = 1; + } + else + { + /* the entry already in the cache and either one of these: + * 1) ep_state: CREATING && state: CREATING + * ==> keep protecting the entry; increase the refcnt + * 2) ep_state: 0 && state: CREATING + * ==> change the state to CREATING (protect it); + * increase the refcnt + * 3) ep_state: 0 && state: 0 + * ==> increase the refcnt + */ + if (e->ep_refcnt == 0) + lru_delete(cache, e); + e->ep_refcnt++; + e->ep_state = state; /* might be CREATING */ + /* returning 1 (entry already existed), but don't set to alt + * to prevent that the caller accidentally thinks the existing + * entry is not the same one the caller has and releases it. + */ + PR_Unlock(cache->c_mutex); + return 1; + } + } + else + { + if (my_alt->ep_state & ENTRY_STATE_CREATING) + { + LOG("the entry is reserved\n", 0, 0, 0); + e->ep_state |= ENTRY_STATE_NOTINCACHE; + PR_Unlock(cache->c_mutex); + return -1; + } + else if (state != 0) + { + LOG("the entry already exists. cannot reserve it.\n", 0, 0, 0); + e->ep_state |= ENTRY_STATE_NOTINCACHE; + PR_Unlock(cache->c_mutex); + return -1; + } + else + { + if (alt) { + *alt = my_alt; + if ((*alt)->ep_refcnt == 0) + lru_delete(cache, *alt); + (*alt)->ep_refcnt++; + } + PR_Unlock(cache->c_mutex); + return 1; + } + } + } + + /* creating an entry with ENTRY_STATE_CREATING just creates a stub + * which is only stored in the dn table (basically, reserving the dn) -- + * doing an add later with state==0 will "confirm" the add + */ + if (state == 0) { + /* neither of these should fail, or something is very wrong. */ + if (! add_hash(cache->c_idtable, &(e->ep_id), sizeof(ID), e, NULL)) { + LOG("entry %s already in id cache!\n", backentry_get_ndn(e), 0, 0); + if (already_in) { + /* there's a bug in the implementatin of 'modify' and 'modrdn' + * that i'm working around here. basically they do a + * tentative add of the new (modified) entry, which places + * the new entry in the cache, indexed only by dn. + * + * later they call id2entry_add() on the new entry, which + * "adds" the new entry to the cache. unfortunately, that + * add will fail, since the old entry is still in the cache, + * and both the old and new entries have the same ID and UUID. + * + * i catch that here, and just return 0 for success, without + * messing with either entry. a later cache_replace() will + * remove the old entry and add the new one, and all will be + * fine (i think). + */ + LOG("<= cache_add_int (ignoring)\n", 0, 0, 0); + PR_Unlock(cache->c_mutex); + return 0; + } + remove_hash(cache->c_dntable, (void *)ndn, strlen(ndn)); + e->ep_state |= ENTRY_STATE_NOTINCACHE; + PR_Unlock(cache->c_mutex); + return -1; + } +#ifdef UUIDCACHE_ON + if (uuid) { + /* (only insert entries with a uuid) */ + if (! add_hash(cache->c_uuidtable, (void *)uuid, strlen(uuid), e, + NULL)) { + LOG("entry %s already in uuid cache!\n", backentry_get_ndn(e), + 0, 0); + remove_hash(cache->c_dntable, (void *)ndn, strlen(ndn)); + remove_hash(cache->c_idtable, &(e->ep_id), sizeof(ID)); + e->ep_state |= ENTRY_STATE_NOTINCACHE; + PR_Unlock(cache->c_mutex); + return -1; + } + } +#endif + } + + e->ep_state = state; + + if (! already_in) { + e->ep_refcnt = 1; + e->size = cache_entry_size(e); + + cache->c_cursize += e->size; + cache->c_curentries++; + /* don't add to lru since refcnt = 1 */ + LOG("added entry of size %lu -> total now %lu out of max %lu\n", + e->size, cache->c_cursize, cache->c_maxsize); + if (cache->c_maxentries >= 0) { + LOG(" total entries %ld out of %ld\n", + cache->c_curentries, cache->c_maxentries, 0); + } + /* check for full cache, and clear out if necessary */ + if (CACHE_FULL(cache)) + eflush = cache_flush(cache); + } + PR_Unlock(cache->c_mutex); + + while (eflush) + { + eflushtemp = eflush->ep_lrunext; + backentry_free(&eflush); + eflush = eflushtemp; + } + LOG("<= cache_add_int OK\n", 0, 0, 0); + return 0; +} + +/* create an entry in the cache, and increase its refcount (you must + * return it when you're done). + * returns: 0 entry has been created & locked + * 1 entry already existed + * -1 something bad happened + * + * if 'alt' is not NULL, and the entry is found to already exist in the + * cache, a refcounted pointer to that entry will be placed in 'alt'. + * (this means code which suffered from race conditions between multiple + * entry modifiers can now work.) + */ +int cache_add(struct cache *cache, struct backentry *e, + struct backentry **alt) +{ + return cache_add_int(cache, e, 0, alt); +} + +/* same as above, but add it tentatively: nobody else can use this entry + * from the cache until you later call cache_add. + */ +int cache_add_tentative(struct cache *cache, struct backentry *e, + struct backentry **alt) +{ + return cache_add_int(cache, e, ENTRY_STATE_CREATING, alt); +} + +/* locks an entry so that it can be modified (you should have gotten the + * entry via cache_find_*). + * returns 0 on success, 1 if the entry is scheduled for deletion. + */ +int cache_lock_entry(struct cache *cache, struct backentry *e) +{ + LOG("=> cache_lock_entry (%s)\n", backentry_get_ndn(e), 0, 0); + + if (! e->ep_mutexp) { + /* make sure only one thread does this */ + PR_Lock(cache->c_emutexalloc_mutex); + if (! e->ep_mutexp) + e->ep_mutexp = PR_NewLock(); + PR_Unlock(cache->c_emutexalloc_mutex); + } + + /* wait on entry lock (done w/o holding the cache lock) */ + PR_Lock(e->ep_mutexp); + + /* make sure entry hasn't been deleted now */ + PR_Lock(cache->c_mutex); + if (e->ep_state & (ENTRY_STATE_DELETED|ENTRY_STATE_NOTINCACHE)) { + PR_Unlock(cache->c_mutex); + PR_Unlock(e->ep_mutexp); + LOG("<= cache_lock_entry (DELETED)\n", 0, 0, 0); + return 1; + } + PR_Unlock(cache->c_mutex); + + LOG("<= cache_lock_entry (FOUND)\n", 0, 0, 0); + return 0; +} + +/* the opposite of above */ +void cache_unlock_entry(struct cache *cache, struct backentry *e) +{ + LOG("=> cache_unlock_entry\n", 0, 0, 0); + PR_Unlock(e->ep_mutexp); +} diff --git a/ldap/servers/slapd/back-ldbm/cleanup.c b/ldap/servers/slapd/back-ldbm/cleanup.c new file mode 100644 index 00000000..ef539134 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/cleanup.c @@ -0,0 +1,51 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* cleanup.c - cleans up ldbm backend */ + +#include "back-ldbm.h" + +int ldbm_back_cleanup( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + Slapi_Backend *be; + + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm backend cleaning up\n", 0, 0, 0 ); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_BACKEND, &be ); + + if (be->be_state != BE_STATE_STOPPED && + be->be_state != BE_STATE_DELETED) + { + LDAPDebug( LDAP_DEBUG_TRACE, + "ldbm_back_cleanup: warning - backend is in a wrong state - %d\n", + be->be_state, 0, 0 ); + return 0; + } + + PR_Lock (be->be_state_lock); + + if (be->be_state != BE_STATE_STOPPED && + be->be_state != BE_STATE_DELETED) + { + LDAPDebug( LDAP_DEBUG_TRACE, + "ldbm_back_cleanup: warning - backend is in a wrong state - %d\n", + be->be_state, 0, 0 ); + PR_Unlock (be->be_state_lock); + return 0; + } + + dblayer_terminate( li ); + +/* JCM I tried adding this to tidy up memory on shutdown. */ +/* JCM But, the result was very messy. */ +/* JCM objset_delete(&li->li_instance_set); */ + + be->be_state = BE_STATE_CLEANED; + + PR_Unlock (be->be_state_lock); + + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/close.c b/ldap/servers/slapd/back-ldbm/close.c new file mode 100644 index 00000000..ce2be3b9 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/close.c @@ -0,0 +1,52 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* close.c - close ldbm backend */ + +#include "back-ldbm.h" + +int ldbm_back_close( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm backend syncing\n", 0, 0, 0 ); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + + /* Kill off any sleeping threads by setting this flag */ + PR_Lock(li->li_shutdown_mutex); + li->li_shutdown = 1; + PR_Unlock(li->li_shutdown_mutex); + + dblayer_flush( li ); /* just be doubly sure! */ + + /* close down all the ldbm instances */ + dblayer_close( li, DBLAYER_NORMAL_MODE ); + + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm backend done syncing\n", 0, 0, 0 ); + return 0; +} + +int ldbm_back_flush( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm backend flushing\n", 0, 0, 0 ); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + dblayer_flush( li ); + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm backend done flushing\n", 0, 0, 0 ); + return 0; +} + +void ldbm_back_instance_set_destructor(void **arg) +{ + /* + Objset *instance_set = (Objset *) *arg; + */ + + /* This function is called when the instance set is destroyed. + * I can't really think of anything we should do here, but that + * may change in the future. */ + LDAPDebug(LDAP_DEBUG_ANY, "Set of instances destroyed\n", 0, 0, 0); +} diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c new file mode 100644 index 00000000..c852b475 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -0,0 +1,5395 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2004 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +/* + Abstraction layer which sits between db2.0 and + higher layers in the directory server---typically + the back-end. + This module's purposes are 1) to hide messy stuff which + db2.0 needs, and with which we don't want to pollute the back-end + code. 2) Provide some degree of portability to other databases + if that becomes a requirement. Note that it is NOT POSSIBLE + to revert to db1.85 because the backend is now using features + from db2.0 which db1.85 does not have. + Also provides an emulation of the ldbm_ functions, for anyone + who is still calling those. The use of these functions is + deprecated. Only for backwards-compatibility. + Blame: dboreham +*/ + +/* Return code conventions: + Unless otherwise advertised, all the functions in this module + return an int which is zero if the operation was successful + and non-zero if it wasn't. If the return'ed value was > 0, + it can be interpreted as a system errno value. If it was < 0, + its meaning is defined in dblayer.h +*/ + +/* + Some information about how this stuff is to be used: + + Call dblayer_init() near the beginning of the application's life. + This allocates some resources and allows the config line processing + stuff to work. + Call dblayer_start() when you're sure all config stuff has been seen. + This needs to be called before you can do anything else. + Call dblayer_close() when you're finished using the db and want to exit. + This closes and flushes all files opened by your application since calling + dblayer_start. If you do NOT call dblayer_close(), we assume that the + application crashed, and initiate recover next time you call dblayer_start(). + Call dblayer_terminate() after close. This releases resources. + + DB* handles are retrieved from dblayer via these functions: + + dblayer_get_id2entry() + dblayer_get_index_file() + + the caller must honour the protocol that these handles are released back + to dblayer when you're done using them, use thse functions to do this: + + dblayer_release_id2entry() + dblayer_release_index_file() + + +*/ + +#include "back-ldbm.h" +#include "dblayer.h" +#include <prrwlock.h> + +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 4100 +#define DB_OPEN(oflags, db, txnid, file, database, type, flags, mode, rval) \ +{ \ + if (((oflags) & DB_INIT_TXN) && ((oflags) & DB_INIT_LOG)) \ + { \ + (rval) = (db)->open((db), (txnid), (file), (database), (type), (flags)|DB_AUTO_COMMIT, (mode)); \ + } \ + else \ + { \ + (rval) = (db)->open((db), (txnid), (file), (database), (type), (flags), (mode)); \ + } \ +} +/* 608145: db4.1 and newer does not require exclusive lock for checkpointing + * and transactions */ +#define DB_CHECKPOINT_LOCK(use_lock, lock) ; +#define DB_CHECKPOINT_UNLOCK(use_lock, lock) ; +#else /* older then db 41 */ +#define DB_OPEN(oflags, db, txnid, file, database, type, flags, mode, rval) \ + (rval) = (db)->open((db), (file), (database), (type), (flags), (mode)) +#define DB_CHECKPOINT_LOCK(use_lock, lock) if(use_lock) PR_RWLock_Wlock(lock); +#define DB_CHECKPOINT_UNLOCK(use_lock, lock) if(use_lock) PR_RWLock_Unlock(lock); +#endif + +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 4000 +#define DB_ENV_SET_REGION_INIT(env) (env)->set_flags((env), DB_REGION_INIT, 1) +#define DB_ENV_SET_TAS_SPINS(env, tas_spins) \ + (env)->set_tas_spins((env), (tas_spins)) +#define TXN_BEGIN(env, parent_txn, tid, flags) \ + (env)->txn_begin((env), (parent_txn), (tid), (flags)) +#define TXN_COMMIT(txn, flags) (txn)->commit((txn), (flags)) +#define TXN_ABORT(txn) (txn)->abort(txn) +#define TXN_CHECKPOINT(env, kbyte, min, flags) \ + (env)->txn_checkpoint((env), (kbyte), (min), (flags)) +#define MEMP_STAT(env, gsp, fsp, flags, malloc) \ + (env)->memp_stat((env), (gsp), (fsp), (flags)) +#define MEMP_TRICKLE(env, pct, nwrotep) \ + (env)->memp_trickle((env), (pct), (nwrotep)) +#define LOG_ARCHIVE(env, listp, flags, malloc) \ + (env)->log_archive((env), (listp), (flags)) +#define LOG_FLUSH(env, lsn) (env)->log_flush((env), (lsn)) +#define LOCK_DETECT(env, flags, atype, aborted) \ + (env)->lock_detect((env), (flags), (atype), (aborted)) + +#else /* older than db 4.0 */ +#define DB_ENV_SET_REGION_INIT(env) db_env_set_region_init(1) +#define DB_ENV_SET_TAS_SPINS(env, tas_spins) \ + db_env_set_tas_spins((tas_spins)) +#define TXN_BEGIN(env, parent_txn, tid, flags) \ + txn_begin((env), (parent_txn), (tid), (flags)) +#define TXN_COMMIT(txn, flags) txn_commit((txn), (flags)) +#define TXN_ABORT(txn) txn_abort((txn)) +#define TXN_CHECKPOINT(env, kbyte, min, flags) \ + txn_checkpoint((env), (kbyte), (min), (flags)) +#define MEMP_TRICKLE(env, pct, nwrotep) memp_trickle((env), (pct), (nwrotep)) +#define LOG_FLUSH(env, lsn) log_flush((env), (lsn)) +#define LOCK_DETECT(env, flags, atype, aborted) \ + lock_detect((env), (flags), (atype), (aborted)) + +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 3300 +#define MEMP_STAT(env, gsp, fsp, flags, malloc) memp_stat((env), (gsp), (fsp)) +#define LOG_ARCHIVE(env, listp, flags, malloc) \ + log_archive((env), (listp), (flags)) + +#else /* older than db 3.3 */ +#define MEMP_STAT(env, gsp, fsp, flags, malloc) \ + memp_stat((env), (gsp), (fsp), (malloc)) +#define LOG_ARCHIVE(env, listp, flags, malloc) \ + log_archive((env), (listp), (flags), (malloc)) +#endif +#endif + +static int perf_threadmain(void *param); +static int checkpoint_threadmain(void *param); +static int trickle_threadmain(void *param); +static int deadlock_threadmain(void *param); +static int commit_good_database(dblayer_private *priv); +static int read_metadata(struct ldbminfo *li); +static int count_dbfiles_in_dir(char *directory, int *count, int recurse); +static int dblayer_override_libdb_functions(DB_ENV *pEnv, dblayer_private *priv); +static int dblayer_force_checkpoint(struct ldbminfo *li); +static int log_flush_threadmain(void *param); +static int dblayer_delete_transaction_logs(const char * log_dir); + +static int dblayer_start_log_flush_thread(dblayer_private *priv); +static int dblayer_start_deadlock_thread(struct ldbminfo *li); +static int dblayer_start_checkpoint_thread(struct ldbminfo *li); +static int dblayer_start_trickle_thread(struct ldbminfo *li); +static int dblayer_start_perf_thread(struct ldbminfo *li); +static int trans_batch_count=1; +static int trans_batch_limit=0; +static PRBool log_flush_thread=PR_FALSE; +static int dblayer_db_remove_ex(dblayer_private_env *env, char const path[], char const dbName[], PRBool use_lock); +static char* last_four_chars(const char* s); + +#define MEGABYTE (1024 * 1024) +#define GIGABYTE (1024 * MEGABYTE) + +/* this flag use if user remotely turned batching off */ + +#define FLUSH_REMOTEOFF -1 +/* routine that allows batch value to be changed remotely: + + 1. value = 0 turns batching off + 2. value = 1 makes behavior be like 5.0 but leaves batching on + 3. value > 1 changes batch value + + 2 and 3 assume that nsslapd-db-transaction-batch-val is greater 0 at startup +*/ + +int +dblayer_set_batch_transactions(void *arg, void *value, char *errorbuf, int phase, int apply) { + int val = (int) value; + int retval = LDAP_SUCCESS; + + if (apply) { + if(phase == CONFIG_PHASE_STARTUP) { + trans_batch_limit=val; + } else if(trans_batch_limit != FLUSH_REMOTEOFF ) { + if((val == 0) && (log_flush_thread)) { + log_flush_thread=PR_FALSE; + trans_batch_limit = FLUSH_REMOTEOFF; + } else if(val > 0) { + trans_batch_limit=val; + } + } + } + return retval; +} + +void * +dblayer_get_batch_transactions(void *arg) { + return (void *)trans_batch_limit; +} + + +/* + Threading: dblayer isolates upper layers from threading considerations + Everything in dblayer is free-threaded. That is, you can have multiple + threads performing operations on a database and not worry about things. + Obviously, if you do something stupid, like move a cursor forward in + one thread, and backwards in another at the same time, you get what you + deserve. However, such a calling pattern will not crash your application ! +*/ + +static int +dblayer_txn_checkpoint(struct ldbminfo *li, struct dblayer_private_env *env, + PRBool use_lock, PRBool busy_skip) +{ + int ret = 0; + if (busy_skip && is_anyinstance_busy(li)) + { + return ret; + } + DB_CHECKPOINT_LOCK(use_lock, env->dblayer_env_lock); + ret = TXN_CHECKPOINT(env->dblayer_DB_ENV, 0, 0, DB_FORCE); + DB_CHECKPOINT_UNLOCK(use_lock, env->dblayer_env_lock); + return ret; +} + +static int _dblayer_check_version(dblayer_private *priv) +{ + int major, minor = 0; + char *string = 0; + int ret = 0; + + string = db_version(&major,&minor,NULL); + if (major < DB_VERSION_MAJOR) + { + ret = -1; + } + else + { + ret = 0; + } + /* DB3X: always POST 24 :) */ + priv->dblayer_lib_version = DBLAYER_LIB_VERSION_POST_24; + LDAPDebug(LDAP_DEBUG_TRACE,"version check: %s (%d.%d)\n", string, major, minor); + return ret; +} + + +/* + * return nsslapd-db-home-directory (dblayer_dbhome_directory), if exists. + * Otherwise, return nsslapd-directory (dblayer_home_directory). + * + * if dblayer_dbhome_directory exists, set 1 to dbhome. + */ +char * +dblayer_get_home_dir(struct ldbminfo *li, int *dbhome) +{ + dblayer_private *priv = (dblayer_private*)li->li_dblayer_private; + char *home_dir = priv->dblayer_home_directory; + if (dbhome) + *dbhome = 0; + + if (priv->dblayer_dbhome_directory && *(priv->dblayer_dbhome_directory)) + { + if (dbhome) + *dbhome = 1; + home_dir = priv->dblayer_dbhome_directory; + } + if (NULL == home_dir) + { + LDAPDebug(LDAP_DEBUG_ANY,"Db home directory is not set. " + "Possibly %s (optinally %s) is missing in the config file.\n", + CONFIG_DIRECTORY, CONFIG_DB_HOME_DIRECTORY, 0); + } + return home_dir; +} + +/* Helper function which deletes the persistent state of the database library + * IMHO this should be in inside libdb, but keith won't have it. + * Stop press---libdb now does delete these files on recovery, so we don't call this any more. + */ +static void dblayer_reset_env(struct ldbminfo *li) +{ + /* Remove the memory regions */ + dblayer_private *priv = (dblayer_private*)li->li_dblayer_private; + DB_ENV *pEnv = priv->dblayer_env->dblayer_DB_ENV; + char *home_dir = dblayer_get_home_dir(li, NULL); + if (home_dir) + pEnv->remove(pEnv, home_dir, DB_FORCE); +} + +/* Callback function for libdb to spit error info into our log */ +static void dblayer_log_print(const char* prefix, char *buffer) +{ + /* We ignore the prefix since we know who we are anyway */ + LDAPDebug(LDAP_DEBUG_ANY,"libdb: %s\n", buffer, 0, 0); +} + +void dblayer_remember_disk_filled(struct ldbminfo *li) +{ + dblayer_private *priv = NULL; + + PR_ASSERT(NULL != li); + priv = li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + priv->dblayer_bad_stuff_happened = 1; + +} + +/* Function which calls libdb to override some system calls which + * the library makes. We call this before calling any other function + * in libdb. + * Several OS use this, either partially or completely. + * This will eventually change---we will simply pass to libdb + * the addresses of a bunch of NSPR functions, and everything + * will magically work on all platforms (Ha!) + */ + +#ifdef DB_USE_64LFS +/* What is going on here ? + * Well, some platforms now support an extended API for dealing with + * files larger than 2G. (This apparently comes from the LFS -- "Large + * File Summit"... Summit, indeed.) Anyway, we try to detect at runtime + * whether this machine has the extended API, and use it if it's present. + * + */ + + +/* helper function for open64 */ +static int dblayer_open_large(const char *path, int oflag, mode_t mode) +{ + int err; + + err = open64(path, oflag, mode); + /* weird but necessary: */ + if (err >= 0) errno = 0; + return err; +} + +/* this is REALLY dumb. but nspr 19980529(x) doesn't support 64-bit files + * because of some weirdness we're doing at initialization (?), so we need + * to export some function that can open huge files, so that exporting + * can work right. when we fix the nspr problem (or get a more recent + * version of nspr that might magically work?), this should be blown away. + * (call mode_t an int because NT can't handle that in prototypes.) + * -robey, 28oct98 + */ +int dblayer_open_huge_file(const char *path, int oflag, int mode) +{ + return dblayer_open_large(path, oflag, (mode_t)mode); +} + + +/* Helper function for large seeks, db2.4 */ +static int dblayer_seek24_large(int fd, size_t pgsize, db_pgno_t pageno, + u_long relative, int isrewind, int whence) +{ + off64_t offset = 0, ret; + + offset = (off64_t)pgsize * pageno + relative; + if (isrewind) offset = -offset; + ret = lseek64(fd, offset, whence); + + return (ret < 0) ? errno : 0; +} + +/* helper function for large fstat -- this depends on 'struct stat64' having + * the following members: + * off64_t st_size; + * long st_blksize; + */ +static int dblayer_ioinfo_large(const char *path, int fd, u_int32_t *mbytesp, + u_int32_t *bytesp, u_int32_t *iosizep) +{ + struct stat64 sb; + + if (fstat64(fd, &sb) < 0) + return (errno); + + /* Return the size of the file. */ + if (mbytesp) + *mbytesp = (u_int32_t) (sb.st_size / (off64_t) MEGABYTE); + if (bytesp) + *bytesp = (u_int32_t) (sb.st_size % (off64_t) MEGABYTE); + + if (iosizep) + *iosizep = (u_int32_t)(sb.st_blksize); + return 0; +} +/* Helper function to tell if a file exists */ +/* On Solaris, if you use stat() on a file >4Gbytes, it fails with EOVERFLOW, + causing us to think that the file does not exist when it in fact does */ +static int dblayer_exists_large(char *path, int *isdirp) +{ + struct stat64 sb; + + if (stat64(path, &sb) != 0) + return (errno); + + if (isdirp != NULL) + *isdirp = S_ISDIR(sb.st_mode); + + return (0); +} + +#else /* DB_USE_64LFS */ + +int dblayer_open_huge_file(const char *path, int oflag, int mode) +{ + return open(path, oflag, mode); +} + +#endif /* DB_USE_64LFS */ + + +static int dblayer_override_libdb_functions(DB_ENV *pEnv, dblayer_private *priv) +{ +#ifdef DB_USE_64LFS + int major = 0; + int minor = 0; + + /* Find out whether we are talking to a 2.3 or 2.4+ libdb */ + db_version(&major, &minor, NULL); + +#ifndef irix + /* irix doesn't have open64() */ + db_env_set_func_open((int (*)(const char *, int, ...))dblayer_open_large); +#endif /* !irix */ + db_env_set_func_ioinfo(dblayer_ioinfo_large); + db_env_set_func_exists((int (*)())dblayer_exists_large); + db_env_set_func_seek((int (*)())dblayer_seek24_large); + + LDAPDebug(LDAP_DEBUG_TRACE, "Enabled 64-bit files\n", 0, 0, 0); +#endif /* DB_USE_64LFS */ + return 0; +} + + + +/* This function is called in the initialization code, before the + * config file is read in, so we can't do much here + */ +int dblayer_init(struct ldbminfo *li) +{ + /* Allocate memory we need, create mutexes etc. */ + dblayer_private *priv = NULL; + int ret = 0; + + PR_ASSERT(NULL != li); + if (NULL != li->li_dblayer_private) + { + return -1; + } + + priv = (dblayer_private*) slapi_ch_calloc(1,sizeof(dblayer_private)); + if (NULL == priv) + { + /* Memory allocation failed */ + return -1; + } + li->li_dblayer_private = priv; + + /* For now, we call this to get debug printout */ + _dblayer_check_version(priv); + + /* moved db_env_create to dblayer_start */ + return ret; +} + +int dblayer_terminate(struct ldbminfo *li) +{ + /* We assume that dblayer_close has been called already */ + dblayer_private *priv = (dblayer_private*)li->li_dblayer_private; + Object *inst_obj; + ldbm_instance *inst; + int rval = 0; + + if (NULL == priv) /* already terminated. nothing to do */ + return rval; + + /* clean up mutexes */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + if (NULL != inst->inst_db_mutex) { + PR_DestroyLock(inst->inst_db_mutex); + } + if (NULL != inst->inst_handle_list_mutex) { + PR_DestroyLock(inst->inst_handle_list_mutex); + } + } + + slapi_ch_free_string(&priv->dblayer_log_directory); + /* no need to release dblayer_home_directory, + * which is one of dblayer_data_directories */ + charray_free(priv->dblayer_data_directories); + slapi_ch_free((void**)&priv); + li->li_dblayer_private = NULL; + + return 0; +} + +static void dblayer_select_ncache(size_t cachesize, int *ncachep) +{ + /* First thing, if the user asked to use a particular ncache, + * we let them, and don't override it here. + */ + if (*ncachep) { + return; + } + /* If the user asked for a cache that's larger than 4G, + * we _must_ select an ncache >0 , such that each + * chunk is <4G. This is because DB won't accept a + * larger chunk. + */ +#if defined(__LP64__) || defined (_LP64) + if ( (sizeof(cachesize) > 4) && (cachesize > (4L * GIGABYTE))) { + *ncachep = (cachesize / (4L * GIGABYTE)) + 1; + LDAPDebug(LDAP_DEBUG_ANY,"Setting ncache to: %d to keep each chunk below 4Gbytes\n", + *ncachep, 0, 0); + } +#endif + /* On Windows, we know that it's hard to allocate more than some + * maximum chunk. In that case + * we set ncache to a sensible value. + */ +#if defined(_WIN32) + { + size_t max_windows_chunk = (300 * MEGABYTE); /* This number was determined empirically on Win2k */ + if (cachesize > max_windows_chunk) { + *ncachep = (cachesize / max_windows_chunk) + 1; + LDAPDebug(LDAP_DEBUG_ANY,"Setting ncache to: %d for Windows memory address space fragmentation\n", + *ncachep, 0, 0); + } + } +#endif +} + +/* This function is no longer called : + It attempts to find the maximum allocatable chunk size + by performing the actual allocations. However as a result + it allocates pretty much _all_ the available memory, + causing the actual cache allocation to fail later. + If there turns out to be a good safe way to determine + the maximum chunk size, then we should use that instead. + For now we just guess in dblayer_pick_ncache(). + */ +static void dblayer_get_ncache(size_t cachesize, int *ncachep) +{ + int myncache; + int mymaxncache; + int found = 0; + char **head; + + if (*ncachep <= 0) /* negative ncache is not allowed */ + myncache = 1; + else + myncache = *ncachep; + + mymaxncache = myncache + 20; /* should be reasonable */ + + head = (char **)slapi_ch_malloc(mymaxncache * sizeof(char *)); + do { + int i; + int end; + size_t sz; + size_t firstsz; + size_t rest; + + rest = cachesize % myncache; + sz = cachesize / myncache; + firstsz = sz + rest; + end = myncache; + for (i = 0; i < myncache; i++) { + if (i == 0) + head[i] = (char *)malloc(firstsz); + else + head[i] = (char *)malloc(sz); + if (NULL == head[i]) { + end = i; + myncache++; + goto cleanup; + } + } + found = 1; +cleanup: + for (i = 0; i < end; i++) { + slapi_ch_free((void **)&head[i]); + } + if (myncache == mymaxncache) { + LDAPDebug(LDAP_DEBUG_ANY, + "WARNING: dbcachesize %lu too big\n", cachesize, 0, 0); + myncache = 0; + found = -1; + } + } while (0 == found); + *ncachep = myncache; + slapi_ch_free((void **)&head); + return; +} + +static void dblayer_init_dbenv(DB_ENV *pEnv, dblayer_private *priv) +{ + size_t mysize; + int myncache = 1; + + mysize = priv->dblayer_cachesize; + myncache = priv->dblayer_ncache; + dblayer_select_ncache(mysize, &myncache); + priv->dblayer_ncache = myncache; + + pEnv->set_errpfx(pEnv, "ns-slapd"); + pEnv->set_lg_max(pEnv, priv->dblayer_logfile_size); + pEnv->set_cachesize(pEnv, mysize / GIGABYTE, mysize % GIGABYTE, myncache); + pEnv->set_lk_max_locks(pEnv, priv->dblayer_lock_config); + pEnv->set_lk_max_objects(pEnv, priv->dblayer_lock_config); + pEnv->set_lk_max_lockers(pEnv, priv->dblayer_lock_config); + if (priv->dblayer_verbose) { + pEnv->set_verbose(pEnv, DB_VERB_CHKPOINT, 1); /* 1 means on */ + pEnv->set_verbose(pEnv, DB_VERB_DEADLOCK, 1); /* 1 means on */ + pEnv->set_verbose(pEnv, DB_VERB_RECOVERY, 1); /* 1 means on */ + pEnv->set_verbose(pEnv, DB_VERB_WAITSFOR, 1); /* 1 means on */ + } + if (priv->dblayer_debug) { + pEnv->set_errcall(pEnv, dblayer_log_print); + } + + /* shm_key required for named_regions (DB_SYSTEM_MEM) */ + pEnv->set_shm_key(pEnv, priv->dblayer_shm_key); + + /* increase max number of active transactions */ + pEnv->set_tx_max(pEnv, priv->dblayer_tx_max); + +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 3300 + pEnv->set_alloc(pEnv, malloc, realloc, free); + + /* + * The log region is used to store filenames and so needs to be + * increased in size from the default for a large number of files. + */ + pEnv->set_lg_regionmax(pEnv, 1 * 1048576); /* 1 MB */ +#endif +} + +/* returns system pagesize (in bytes) and the number of pages of physical + * RAM this machine has. + * as a bonus, if 'procpages' is non-NULL, it will be filled in with the + * approximate number of pages this process is using! + * on platforms that we haven't figured out how to do this yet, both fields + * are filled with zero and you're on your own. + * + * platforms supported so far: + * Solaris, Linux, Windows + */ +#ifdef OS_solaris +#include <sys/procfs.h> +#include <sys/resource.h> +#endif +#ifdef LINUX +#include <linux/kernel.h> +#include <linux/sys.h> +#include <sys/sysinfo.h> /* undocumented (?) */ +#include <sys/resource.h> +#endif +#if defined ( hpux ) +#include <sys/pstat.h> +#include <sys/resource.h> +#endif + +#if !defined(_WIN32) +static size_t dblayer_getvirtualmemsize() +{ + struct rlimit rl; + + /* the maximum size of a process's total available memory, in bytes */ + getrlimit(RLIMIT_AS, &rl); + return rl.rlim_cur; +} +#endif + +/* pages = number of pages of physical ram on the machine (corrected for 32-bit build on 64-bit machine). + * procpages = pages currently used by this process (or working set size, sometimes) + * availpages = some notion of the number of pages 'free'. Typically this number is not useful. + */ +void dblayer_sys_pages(size_t *pagesize, size_t *pages, size_t *procpages, size_t *availpages) +{ + *pagesize = *pages = *availpages = 0; + if (procpages) + *procpages = 0; + +#ifdef _WIN32 + { + SYSTEM_INFO si; + MEMORYSTATUS ms; + + GetSystemInfo(&si); + ms.dwLength = sizeof(ms); + GlobalMemoryStatus(&ms); + *pagesize = si.dwPageSize; + *pages = ms.dwTotalPhys / si.dwPageSize; + *availpages = ms.dwAvailVirtual / *pagesize; + if (procpages) { + DWORD minwss = 0, maxwss = 0; + + GetProcessWorkingSetSize(GetCurrentProcess(), &minwss, &maxwss); + *procpages = (int)(maxwss / si.dwPageSize); + } + } +#endif + +#ifdef OS_solaris + *pagesize = (int)sysconf(_SC_PAGESIZE); + *pages = (int)sysconf(_SC_PHYS_PAGES); + *availpages = dblayer_getvirtualmemsize() / *pagesize; + /* solaris has THE most annoying way to get this info */ + if (procpages) { + struct prpsinfo psi; + char fn[40]; + int fd; + + sprintf(fn, "/proc/%d", getpid()); + fd = open(fn, O_RDONLY); + if (fd >= 0) { + memset(&psi, 0, sizeof(psi)); + if (ioctl(fd, PIOCPSINFO, (void *)&psi) == 0) + *procpages = psi.pr_size; + close(fd); + } + } +#endif + +#ifdef LINUX + { + struct sysinfo si; + size_t pages_per_mem_unit = 0; + size_t mem_units_per_page = 0; /* We don't know if these units are really pages */ + + sysinfo(&si); + *pagesize = getpagesize(); + if (si.mem_unit > *pagesize) { + pages_per_mem_unit = si.mem_unit / *pagesize; + *pages = si.totalram * pages_per_mem_unit; + } else { + mem_units_per_page = *pagesize / si.mem_unit; + *pages = si.totalram / mem_units_per_page; + } + *availpages = dblayer_getvirtualmemsize() / *pagesize; + /* okay i take that back, linux's method is more retarded here. + * hopefully linux doesn't have the FILE* problem that solaris does + * (where you can't use FILE if you have more than 256 fd's open) + */ + if (procpages) { + FILE *f; + char fn[40], s[80]; + + sprintf(fn, "/proc/%d/status", getpid()); + f = fopen(fn, "r"); + while (! feof(f)) { + fgets(s, 79, f); + if (feof(f)) + break; + if (strncmp(s, "VmSize:", 7) == 0) { + sscanf(s+7, "%d", procpages); + break; + } + } + fclose(f); + /* procpages is now in 1k chunks, not pages... */ + *procpages /= (*pagesize / 1024); + } + } +#endif + +#if defined ( hpux ) + { + struct pst_static pst; + struct pst_dynamic pst_dyn; + int rval = pstat_getstatic(&pst, sizeof(pst), (size_t)1, 0); + if (rval < 0) /* pstat_getstatic failed */ + return; + *pagesize = pst.page_size; + *pages = pst.physical_memory; + *availpages = dblayer_getvirtualmemsize() / *pagesize; + rval = pstat_getdynamic(&pst_dyn, sizeof(pst_dyn), (size_t)1, 0); + if (rval < 0) /* pstat_getdynamic failed */ + return; + if (procpages) + { +#define BURST (size_t)32 /* get BURST proc info at one time... */ + struct pst_status psts[BURST]; + int i, count; + int idx = 0; /* index within the context */ + int mypid = getpid(); + + *procpages = 0; + /* loop until count == 0, will occur all have been returned */ + while ((count = pstat_getproc(psts, sizeof(psts[0]), BURST, idx)) > 0) { + /* got count (max of BURST) this time. process them */ + for (i = 0; i < count; i++) { + if (psts[i].pst_pid == mypid) + { + *procpages = (size_t)(psts[i].pst_dsize + psts[i].pst_tsize + psts[i].pst_ssize); + break; + } + } + if (i < count) + break; + + /* + * now go back and do it again, using the next index after + * the current 'burst' + */ + idx = psts[count-1].pst_idx + 1; + } + } + } +#endif + /* If this is a 32-bit build, it might be running on a 64-bit machine, + * in which case, if the box has tons of ram, we can end up telling + * the auto cache code to use more memory than the process can address. + * so we cap the number returned here. + */ +#if defined(__LP64__) || defined (_LP64) +#else + { + size_t one_gig_pages = GIGABYTE / *pagesize; + if (*pages > (2 * one_gig_pages) ) { + LDAPDebug(LDAP_DEBUG_TRACE,"More than 2Gbytes physical memory detected. Since this is a 32-bit process, truncating memory size used for auto cache calculations to 2Gbytes\n", + 0, 0, 0); + *pages = (2 * one_gig_pages); + } + } +#endif +} + + +int dblayer_is_cachesize_sane(size_t *cachesize) +{ + size_t pages = 0, pagesize = 0, procpages = 0, availpages = 0; + int issane = 1; + + dblayer_sys_pages(&pagesize, &pages, &procpages, &availpages); + if (!pagesize || !pages) + return 1; /* do nothing when we can't get the avail mem */ + /* If the requested cache size is larger than the remaining pysical memory + * after the current working set size for this process has been subtracted, + * then we say that's insane and try to correct. + */ + issane = (int)(*cachesize / pagesize) <= (pages - procpages); + if (!issane) { + *cachesize = (size_t)((pages - procpages) * pagesize); + } + /* We now compensate for DB's own compensation for metadata size + * They increase the actual cache size by 25%, but only for sizes + * less than 500Meg. + */ + if (*cachesize < 500*MEGABYTE) { + *cachesize = (size_t)((double)*cachesize * (double)0.8); + } + + return issane; +} + + +static void dblayer_dump_config_tracing(dblayer_private *priv) +{ + if (priv->dblayer_home_directory) { + LDAPDebug(LDAP_DEBUG_TRACE,"home_directory=%s\n",priv->dblayer_home_directory,0,0); + } + if (priv->dblayer_log_directory) { + LDAPDebug(LDAP_DEBUG_TRACE,"log_directory=%s\n",priv->dblayer_log_directory,0,0); + } + if (priv->dblayer_dbhome_directory) { + LDAPDebug(LDAP_DEBUG_TRACE,"dbhome_directory=%s\n",priv->dblayer_dbhome_directory,0,0); + } + LDAPDebug(LDAP_DEBUG_TRACE,"trickle_percentage=%d\n",priv->dblayer_trickle_percentage,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"page_size=%lu\n",priv->dblayer_page_size,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"index_page_size=%lu\n",priv->dblayer_index_page_size,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"cachesize=%lu\n",priv->dblayer_cachesize,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"previous_cachesize=%lu\n",priv->dblayer_previous_cachesize,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"ncache=%d\n",priv->dblayer_ncache,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"previous_ncache=%d\n",priv->dblayer_previous_ncache,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"recovery_required=%d\n",priv->dblayer_recovery_required,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"durable_transactions=%d\n",priv->dblayer_durable_transactions,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"checkpoint_interval=%d\n",priv->dblayer_checkpoint_interval,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"transaction_batch_val=%d\n",trans_batch_limit,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"circular_logging=%d\n",priv->dblayer_circular_logging,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"idl_divisor=%d\n",priv->dblayer_idl_divisor,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"logfile_size=%lu\n",priv->dblayer_logfile_size,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"logbuf_size=%lu\n",priv->dblayer_logbuf_size,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"file_mode=%d\n",priv->dblayer_file_mode,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"cache_config=%d\n",priv->dblayer_cache_config,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"lib_version=%d\n",priv->dblayer_lib_version,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"spin_count=%d\n",priv->dblayer_spin_count,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"named_regions=%d\n",priv->dblayer_named_regions,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"private mem=%d\n",priv->dblayer_private_mem,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"private import mem=%d\n",priv->dblayer_private_import_mem,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"shm_key=%ld\n",priv->dblayer_shm_key,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"lockdown=%d\n",priv->dblayer_lockdown,0,0); + LDAPDebug(LDAP_DEBUG_TRACE,"tx_max=%d\n",priv->dblayer_tx_max,0,0); +} + +/* Check a given filesystem directory for access we need */ +#define DBLAYER_DIRECTORY_READ_ACCESS 1 +#define DBLAYER_DIRECTORY_WRITE_ACCESS 2 +#define DBLAYER_DIRECTORY_READWRITE_ACCESS 3 +static int dblayer_grok_directory(char *directory, int flags) +{ + /* First try to open the directory using NSPR */ + /* If that fails, we can tell whether it's because it cannot be created or + * we don't have any permission to access it */ + /* If that works, proceed to try to access files in the directory */ + char filename[MAXPATHLEN]; + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + PRFileInfo info; + + dirhandle = PR_OpenDir(directory); + if (NULL == dirhandle) + { + /* it does not exist or wrong file is there */ + /* try delete and mkdir */ + PR_Delete(directory); + return mkdir_p(directory, 0700); + } + + while (NULL != + (direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { + if (NULL == direntry->name) + { + break; + } + sprintf(filename,"%s/%s",directory,direntry->name); + + /* Right now this is set up to only look at files here. + * With multiple instances of the backend the are now other directories + * in the db home directory. This function wasn't ment to deal with + * other directories, so we skip them. */ + if (PR_GetFileInfo(filename, &info) == PR_SUCCESS && + info.type == PR_FILE_DIRECTORY) { + /* go into it (instance dir) */ + int retval = dblayer_grok_directory(filename, flags); + PR_CloseDir(dirhandle); + return retval; + } + + /* If we are here, it means that the directory exists, that we can read + * from it, and that there is at least one file there */ + /* We will try to open that file now if we were asked for read access */ + if (flags) { + PRFileDesc *prfd; + PRIntn open_flags = 0; + char *access_string = NULL; + + if (DBLAYER_DIRECTORY_READ_ACCESS & flags) { + open_flags = PR_RDONLY; + } + if (DBLAYER_DIRECTORY_WRITE_ACCESS & flags) { + open_flags = PR_RDWR; + } + /* Let's hope that on Solaris we get to open large files OK */ + prfd = PR_Open(filename,open_flags,0); + if (NULL == prfd) { + if (DBLAYER_DIRECTORY_READ_ACCESS == flags) { + access_string = "read"; + } else { + if (DBLAYER_DIRECTORY_READ_ACCESS & flags) { + access_string = "write"; + } else { + access_string = "****"; + } + } + /* If we're here, it means that we did not have the requested + * permission on this file */ + LDAPDebug(LDAP_DEBUG_ANY, + "WARNING---no %s permission to file %s\n", + access_string,filename,0); + } else { + PR_Close(prfd); /* okay */ + } + } + } + PR_CloseDir(dirhandle); + return 0; +} + +static void +dblayer_set_data_dir(dblayer_private *priv, struct dblayer_private_env *pEnv, + char **data_directories) +{ + char **dirp; + + if (!(pEnv->dblayer_priv_flags & DBLAYER_PRIV_SET_DATA_DIR)) + { + for (dirp = data_directories; dirp && *dirp; dirp++) + { + pEnv->dblayer_DB_ENV->set_data_dir(pEnv->dblayer_DB_ENV, *dirp); + } + pEnv->dblayer_priv_flags |= DBLAYER_PRIV_SET_DATA_DIR; + } +} + +static int +dblayer_inst_exists(ldbm_instance *inst, char *dbname) +{ + PRStatus prst; + char id2entry_file[MAXPATHLEN]; + char *parent_dir = inst->inst_parent_dir_name; + char sep = get_sep(parent_dir); + char *dbnamep; + if (dbname) + dbnamep = dbname; + else + dbnamep = ID2ENTRY LDBM_FILENAME_SUFFIX; + sprintf(id2entry_file, "%s%c%s%c%s", parent_dir, sep, inst->inst_dir_name, + sep, dbnamep); + prst = PR_Access(id2entry_file, PR_ACCESS_EXISTS); + if (PR_SUCCESS == prst) + return 1; + return 0; +} + +/* + * create a new DB_ENV and fill it with the goodies from dblayer_private + */ +#define INIT_MAX_DIRS 32 +static int +dblayer_make_env(struct dblayer_private_env **env, struct ldbminfo *li) +{ + dblayer_private *priv = (dblayer_private*)li->li_dblayer_private; + struct dblayer_private_env *pEnv; + char *home_dir = NULL; + int ret; + int data_dirs = INIT_MAX_DIRS; + Object *inst_obj; + ldbm_instance *inst = NULL; + + pEnv = + (struct dblayer_private_env *) PR_Calloc(1, sizeof(dblayer_private_env)); + + if ((ret = db_env_create(&pEnv->dblayer_DB_ENV, 0)) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR -- Failed to create DB_ENV (returned: %d).\n", + ret, 0, 0); + } + + DB_ENV_SET_REGION_INIT(pEnv->dblayer_DB_ENV); + + /* Here we overide various system functions called by libdb */ + ret = dblayer_override_libdb_functions(pEnv->dblayer_DB_ENV, priv); + if (ret != 0) + return ret; + + if (priv->dblayer_spin_count != 0) { + DB_ENV_SET_TAS_SPINS(pEnv->dblayer_DB_ENV, priv->dblayer_spin_count); + } + + dblayer_dump_config_tracing(priv); + + /* set data dir to avoid having absolute paths in the transaction log */ + priv->dblayer_data_directories = NULL; + for (inst_obj = objset_first_obj(li->li_instance_set); + inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) + { + inst = (ldbm_instance *)object_get_data(inst_obj); + if (inst->inst_parent_dir_name) + { + if (!charray_utf8_inlist(priv->dblayer_data_directories, + inst->inst_parent_dir_name)) + { + charray_add(&(priv->dblayer_data_directories), + inst->inst_parent_dir_name); + } + } + } + home_dir = dblayer_get_home_dir(li, NULL); + /* user specified db home */ + if (!charray_utf8_inlist(priv->dblayer_data_directories, home_dir)) + { + charray_add(&(priv->dblayer_data_directories), home_dir); + } + + /* user specified log dir */ + if (priv->dblayer_log_directory && *(priv->dblayer_log_directory)) { + pEnv->dblayer_DB_ENV->set_lg_dir(pEnv->dblayer_DB_ENV, + priv->dblayer_log_directory); + } + + /* set up cache sizes */ + dblayer_init_dbenv(pEnv->dblayer_DB_ENV, priv); + + pEnv->dblayer_env_lock = PR_NewRWLock(PR_RWLOCK_RANK_NONE, "checkpointer"); + + if (pEnv->dblayer_env_lock) { + *env = pEnv; + } else { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR -- Failed to create RWLock (returned: %d).\n", + ret, 0, 0); + } + + return ret; +} + +/* generate an absolute path if the given instance dir is not. */ +char * +dblayer_get_full_inst_dir(struct ldbminfo *li, ldbm_instance *inst, + char *buf, int buflen) +{ + char *parent_dir; + int mylen; + + if (!inst) + return NULL; + + if (inst->inst_parent_dir_name) + { + parent_dir = inst->inst_parent_dir_name; + if (inst->inst_parent_dir_name) + { + mylen = strlen(parent_dir) + strlen(inst->inst_dir_name) + 2; + } + else + { + mylen = strlen(parent_dir) + 1; + } + } + else + { + parent_dir = dblayer_get_home_dir(li, NULL); + mylen = strlen(parent_dir); + inst->inst_parent_dir_name = slapi_ch_strdup(parent_dir); + } + + + if (inst->inst_dir_name) + { + mylen += strlen(inst->inst_dir_name) + 2; + if (!buf || mylen > buflen) + buf = slapi_ch_malloc(mylen); + sprintf(buf, "%s%c%s", + parent_dir, get_sep(parent_dir), inst->inst_dir_name); + } + else if (inst->inst_name) + { + inst->inst_dir_name = slapi_ch_strdup(inst->inst_name); + mylen += strlen(inst->inst_dir_name) + 2; + if (!buf || mylen > buflen) + buf = slapi_ch_malloc(mylen); + sprintf(buf, "%s%c%s", + parent_dir, get_sep(parent_dir), inst->inst_dir_name); + } + else + { + mylen += 1; + if (!buf || mylen > buflen) + buf = slapi_ch_malloc(mylen); + sprintf(buf, "%s", parent_dir); + } + return buf; +} + +#if defined( OS_solaris ) +#include <sys/types.h> +#include <sys/statvfs.h> +#endif + +#if defined( hpux ) +#undef f_type +#include <sys/types.h> +#include <sys/statvfs.h> +#define f_type f_un.f_un_type +#endif + +#if defined( linux ) +#undef f_type +#include <sys/vfs.h> +#define f_type f_un.f_un_type +#endif + +static int +no_diskspace(struct ldbminfo *li) +{ + int rval = 0; +#if defined( OS_solaris ) || defined( hpux ) + struct statvfs fsbuf; + if (statvfs(li->li_directory, &fsbuf) < 0) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Cannot get file system info; file system corrupted?\n", 0, 0, 0); + rval = 1; + } + else + { + double fsiz = ((double)fsbuf.f_bavail) * fsbuf.f_frsize; + double expected_siz = ((double)li->li_dbcachesize) * 1.5; /* dbcache + + region files */ + if (fsiz < expected_siz) + { + LDAPDebug(LDAP_DEBUG_ANY, + "No enough space left on device (%lu bytes); " + "at least %lu bytes space is needed for db region files\n", + fsiz, expected_siz, 0); + rval = 1; + } + } +#endif +#if defined( linux ) + struct statfs fsbuf; + if (statfs(li->li_directory, &fsbuf) < 0) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Cannot get file system info; file system corrupted?\n", 0, 0, 0); + rval = 1; + } + else + { + double fsiz = ((double)fsbuf.f_bavail) * fsbuf.f_bsize; + double expected_siz = ((double)li->li_dbcachesize) * 1.5; /* dbcache + + region files */ + if (fsiz < expected_siz) + { + LDAPDebug(LDAP_DEBUG_ANY, + "No enough space left on device (%lu bytes); " + "at least %lu bytes space is needed for db region files\n", + fsiz, expected_siz, 0); + rval = 1; + } + } +#endif + return rval; +} + +/* + * This function is called after all the config options have been read in, + * so we can do real initialization work here. + */ +#define DBCONFLEN 3 +#define CATASTROPHIC (struct dblayer_private_env *)-1 + +int dblayer_start(struct ldbminfo *li, int dbmode) +{ + /* + * So, here we open our DB_ENV session. We store it away for future use. + * We also check to see if we exited cleanly last time. If we didn't, + * we try to recover. If recovery fails, we're hosed. + * We also create the thread which handles checkpointing and logfile + * truncation here. + */ + int return_value = -1; + dblayer_private *priv = NULL; + struct dblayer_private_env *pEnv = NULL; + char *region_dir = NULL; /* directory to place region files */ + char *log_dir = NULL; /* directory to place txn log files */ + int open_flags = 0; + + PR_ASSERT(NULL != li); + + priv = (dblayer_private*)li->li_dblayer_private; + + if (NULL == priv) { + /* you didn't call init successfully */ + return -1; + } + + if (NULL != priv->dblayer_env) { + if (CATASTROPHIC == priv->dblayer_env) { + LDAPDebug(LDAP_DEBUG_ANY, + "Error: DB previously failed to start.\n", 0, 0, 0); + return -1; + } else { + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: DB already started.\n", 0, 0, 0); + return 0; + } + } + + /* DBDB we should pick these up in our config routine, and do away with + * the li_ one */ + PR_Lock(li->li_config_mutex); + priv->dblayer_home_directory = li->li_directory; /* nsslapd-directory */ + priv->dblayer_cachesize = li->li_dbcachesize; + priv->dblayer_file_mode = li->li_mode; + priv->dblayer_ncache = li->li_dbncache; + PR_Unlock(li->li_config_mutex); + + /* use nsslapd-db-home-directory (dblayer_dbhome_directory), if set */ + /* Otherwise, nsslapd-directory (dblayer_home_directory). */ + region_dir = dblayer_get_home_dir(li, NULL); + if (!region_dir || !(*region_dir)) { + return -1; + } + + /* Check here that the database directory both exists, and that we have + * the appropriate access to it */ + return_value = dblayer_grok_directory(region_dir, + DBLAYER_DIRECTORY_READWRITE_ACCESS); + if (0 != return_value) { + LDAPDebug(LDAP_DEBUG_ANY,"Can't start because the database " + "directory \"%s\" either doesn't exist, or is not " + "accessible\n", region_dir, 0, 0); + return return_value; + } + + log_dir = priv->dblayer_log_directory; /* nsslapd-db-logdirectory */ + if (log_dir && *log_dir) { + return_value = dblayer_grok_directory(log_dir, + DBLAYER_DIRECTORY_READWRITE_ACCESS); + if (0 != return_value) { + LDAPDebug(LDAP_DEBUG_ANY,"Can't start because the log " + "directory \"%s\" either doesn't exist, or is not " + "accessible\n", log_dir, 0, 0); + return return_value; + } + } + + /* Sanity check on cache size on platforms which allow us to figure out + * the available phys mem */ + if (!dblayer_is_cachesize_sane(&(priv->dblayer_cachesize))) { + /* Oops---looks like the admin misconfigured, let's warn them */ + LDAPDebug(LDAP_DEBUG_ANY,"WARNING---Likely CONFIGURATION ERROR---" + "dbcachesize is configured to use more than the available " + "physical memory, decreased to the largest available size (%lu bytes).\n", + priv->dblayer_cachesize, 0, 0); + li->li_dbcachesize = priv->dblayer_cachesize; + } + + /* fill in DB_ENV stuff from the common configuration */ + return_value = dblayer_make_env(&pEnv, li); + if (return_value != 0) + return return_value; + + if ((DBLAYER_NORMAL_MODE|DBLAYER_CLEAN_RECOVER_MODE) & dbmode) + { + /* Now, we read our metadata */ + return_value = read_metadata(li); + if (0 != return_value) { + /* The error message was output by read_metadata() */ + return -1; + } + } + + priv->dblayer_env = pEnv; + + open_flags = DB_CREATE | DB_INIT_MPOOL | DB_THREAD; + + if (priv->dblayer_enable_transactions) { + open_flags |= (DB_INIT_TXN | DB_INIT_LOG | DB_INIT_LOCK); + if (priv->dblayer_recovery_required) { + open_flags |= DB_RECOVER; + if (DBLAYER_RESTORE_MODE & dbmode) { + LDAPDebug(LDAP_DEBUG_ANY, "Recovering database after restore " + "from archive.\n", 0, 0, 0); + } else if (DBLAYER_CLEAN_RECOVER_MODE & dbmode) { + LDAPDebug(LDAP_DEBUG_ANY, "Clean up db environment and start " + "from archive.\n", 0, 0, 0); + } else { + LDAPDebug(LDAP_DEBUG_ANY, "Detected Disorderly Shutdown last " + "time Directory Server was running, recovering " + "database.\n", 0, 0, 0); + } + } + switch (dbmode&DBLAYER_RESTORE_MASK) { + case DBLAYER_RESTORE_MODE: + open_flags |= DB_RECOVER_FATAL; + open_flags &= ~DB_RECOVER; /* shouldn't set both */ + if (!(dbmode & DBLAYER_CMDLINE_MODE)) + dbmode = DBLAYER_NORMAL_MODE; /* to restart helper threads */ + break; + case DBLAYER_RESTORE_NO_RECOVERY_MODE: + open_flags &= ~(DB_RECOVER | DB_RECOVER_FATAL); + if (!(dbmode & DBLAYER_CMDLINE_MODE)) + dbmode = DBLAYER_NORMAL_MODE; /* to restart helper threads */ + } + } + + if (priv->dblayer_private_mem) { + LDAPDebug(LDAP_DEBUG_ANY, + "WARNING: Server is running with nsslapd-db-private-mem on; " + "No other process is allowed to access the database\n", + 0, 0, 0); + open_flags |= DB_PRIVATE; + } + + if (priv->dblayer_named_regions) { + open_flags |= DB_SYSTEM_MEM; + } + + if (priv->dblayer_lockdown) { + open_flags |= DB_LOCKDOWN; + } + + + /* Is the cache being re-sized ? (If we're just doing an archive or export, + * we don't care if the cache is being re-sized) */ + if ( (priv->dblayer_previous_cachesize || priv->dblayer_previous_ncache) && + ((priv->dblayer_cachesize != priv->dblayer_previous_cachesize) || + (priv->dblayer_ncache != priv->dblayer_previous_ncache)) && + !(dbmode & (DBLAYER_ARCHIVE_MODE|DBLAYER_EXPORT_MODE)) ) { + LDAPDebug(LDAP_DEBUG_ANY, + "I'm resizing my cache now...cache was %lu and is now %lu\n", + priv->dblayer_previous_cachesize, priv->dblayer_cachesize, 0); + dblayer_reset_env(li); + /* + * Once pEnv->remove (via dblayer_reset_env) has been called, + * the DB_ENV (pEnv) needs to be created again. + */ + if ((return_value = dblayer_make_env(&pEnv, li)) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR -- Failed to create DBENV (returned: %d).\n", + return_value, 0, 0); + } + priv->dblayer_env = pEnv; + } + + /* transactions enabled and logbuf size greater than sleepycat's default */ + if(priv->dblayer_enable_transactions && (priv->dblayer_logbuf_size > 0)) { + if(priv->dblayer_logbuf_size >= 32768) { + pEnv->dblayer_DB_ENV->set_lg_bsize(pEnv->dblayer_DB_ENV,priv->dblayer_logbuf_size); + } else { + LDAPDebug(LDAP_DEBUG_ANY, "using default value for log bufsize because configured value (%lu) is too small.\n", + priv->dblayer_logbuf_size, 0, 0); + } + } + + /* check if there's enough disk space to start */ + if (no_diskspace(li)) + { + return ENOSPC; + } + + dblayer_set_data_dir(priv, pEnv, priv->dblayer_data_directories); + /* If we're doing recovery, we MUST open the env single-threaded ! */ + if ( (open_flags & DB_RECOVER) || (open_flags & DB_RECOVER_FATAL) ) { + /* Recover, then close, then open again */ + int recover_flags = open_flags & ~DB_THREAD; + + if (DBLAYER_CLEAN_RECOVER_MODE & dbmode) /* upgrade case */ + { + DB_ENV *thisenv = pEnv->dblayer_DB_ENV; + return_value = thisenv->remove(thisenv, region_dir, DB_FORCE); + if (0 != return_value) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dblayer_start: failed to remove old db env " + "in %s: %s\n", region_dir, + dblayer_strerror(return_value), 0); + return return_value; + } + dbmode = DBLAYER_NORMAL_MODE; + + if ((return_value = dblayer_make_env(&pEnv, li)) != 0) + { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR -- Failed to create DBENV (returned: %d).\n", + return_value, 0, 0); + return return_value; + } + } + + return_value = pEnv->dblayer_DB_ENV->open( + pEnv->dblayer_DB_ENV, + region_dir, + recover_flags, + priv->dblayer_file_mode + ); + if (0 != return_value) { + if (return_value == ENOMEM) { + /* + * https://blackflag.mcom.com/show_bug.cgi?id=557319 + * Crash ns-slapd while running scalab01 after restart slapd + */ + LDAPDebug(LDAP_DEBUG_ANY, + "mmap in opening database environment (recovery mode) " + "failed trying to allocate %lu bytes. (OS err %d - %s)\n", + li->li_dbcachesize, return_value, dblayer_strerror(return_value)); + priv->dblayer_env = CATASTROPHIC; + } else { + LDAPDebug(LDAP_DEBUG_ANY, "Database Recovery Process FAILED. " + "The database is not recoverable. err=%d: %s\n", + return_value, dblayer_strerror(return_value), 0); + LDAPDebug(LDAP_DEBUG_ANY, + "Please make sure there is enough disk space for " + "dbcache (%lu bytes) and db region files\n", + li->li_dbcachesize, 0, 0); + } + return return_value; + } else { + open_flags &= ~(DB_RECOVER | DB_RECOVER_FATAL); + pEnv->dblayer_DB_ENV->close(pEnv->dblayer_DB_ENV, 0); + if ((return_value = dblayer_make_env(&pEnv, li)) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR -- Failed to create DBENV (returned: %d).\n", + return_value, 0, 0); + return return_value; + } + priv->dblayer_env = pEnv; + dblayer_set_data_dir(priv, pEnv, priv->dblayer_data_directories); + } + } + + if ((!priv->dblayer_durable_transactions) || + ((priv->dblayer_enable_transactions) && (trans_batch_limit > 0))){ +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 3200 +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 4100 /* db4.1 and newer */ + pEnv->dblayer_DB_ENV->set_flags(pEnv->dblayer_DB_ENV, DB_TXN_WRITE_NOSYNC, 1); +#else /* db3.3 */ + pEnv->dblayer_DB_ENV->set_flags(pEnv->dblayer_DB_ENV, DB_TXN_NOSYNC, 1); +#endif +#else /* older */ + open_flags |= DB_TXN_NOSYNC; +#endif + } + if (!((DBLAYER_IMPORT_MODE|DBLAYER_INDEX_MODE) & dbmode)) + { + pEnv->dblayer_openflags = open_flags; + return_value = pEnv->dblayer_DB_ENV->open( + pEnv->dblayer_DB_ENV, + region_dir, + open_flags, + priv->dblayer_file_mode + ); + + + /* Now attempt to start up the checkpoint and deadlock threads */ + if ( (DBLAYER_NORMAL_MODE & dbmode ) && (0 == return_value)) { + /* update the dbversion file */ + dbversion_write(li, region_dir, NULL); + + /* if dblayer_close then dblayer_start is called, + this flag is set */ + priv->dblayer_stop_threads = 0; + if (0 != (return_value = dblayer_start_deadlock_thread(li))) { + return return_value; + } + + if (0 != (return_value = dblayer_start_checkpoint_thread(li))) { + return return_value; + } + + if (0 != (return_value = dblayer_start_log_flush_thread(priv))) { + return return_value; + } + + if (0 != (return_value = dblayer_start_trickle_thread(li))) { + return return_value; + } + + if (0 != (return_value = dblayer_start_perf_thread(li))) { + return return_value; + } + + /* Now open the performance counters stuff */ + perfctrs_init(li,&(priv->perf_private)); + } + if (return_value != 0) { + if (return_value == ENOMEM) { + /* + * https://blackflag.mcom.com/show_bug.cgi?id=557319 + * Crash ns-slapd while running scalab01 after restart slapd + */ + LDAPDebug(LDAP_DEBUG_ANY, + "mmap in opening database environment " + "failed trying to allocate %d bytes. (OS err %lu - %s)\n", + li->li_dbcachesize, return_value, dblayer_strerror(return_value)); + priv->dblayer_env = CATASTROPHIC; + } else { + LDAPDebug(LDAP_DEBUG_ANY, + "Opening database environment (%s) failed. err=%d: %s\n", + region_dir, return_value, dblayer_strerror(return_value)); + } + } + return return_value; + } + return 0; +} + +void +autosize_import_cache(struct ldbminfo *li) +{ + /* + * default behavior for ldif2db import cache, + * nsslapd-import-cache-autosize==-1, + * autosize 50% mem to import cache + */ + if (li->li_import_cache_autosize == -1) { + li->li_import_cache_autosize = 50; + } + + /* sanity check */ + if (li->li_import_cache_autosize > 100) { + LDAPDebug(LDAP_DEBUG_ANY, + "cache autosizing: bad setting, " + "import cache autosizing value should not be larger than 100(%).\n" + "set: 100(%).\n", NULL, NULL, NULL); + li->li_import_cache_autosize = 100; + } + + /* autosizing importCache */ + if (li->li_import_cache_autosize > 0) { + size_t pagesize, pages, procpages, availpages; + + dblayer_sys_pages(&pagesize, &pages, &procpages, &availpages); + LDAPDebug(LDAP_DEBUG_ANY, "dblayer_instance_start: " + "pagesize: %d, pages: %d, procpages: %d\n", + pagesize, pages, procpages); + if (pagesize) { + char s[32]; /* big enough to hold %ld */ + int import_pages; + int pages_limit = (200 * 1024) / (pagesize/1024); + import_pages = (li->li_import_cache_autosize * pages) / 125; + /* We don't want to go wild with memory when auto-sizing, cap the + * cache size at 200 Megs to try to avoid situations where we + * attempt to allocate more memory than there is free page pool for, or + * where there's some system limit on the size of process memory + */ + if (import_pages > pages_limit) { + import_pages = pages_limit; + } + LDAPDebug(LDAP_DEBUG_ANY, "cache autosizing: import cache: %dk \n", + import_pages*(pagesize/1024), NULL, NULL); + LDAPDebug(LDAP_DEBUG_ANY, + "li_import_cache_autosize: %d, import_pages: %d, pagesize: %d\n", + li->li_import_cache_autosize, import_pages, + pagesize); + + sprintf(s, "%lu", (unsigned long)(import_pages * pagesize)); + ldbm_config_internal_set(li, CONFIG_IMPORT_CACHESIZE, s); + } + } +} + +/* mode is one of + * DBLAYER_NORMAL_MODE, + * DBLAYER_INDEX_MODE, + * DBLAYER_IMPORT_MODE, + * DBLAYER_EXPORT_MODE + */ +int dblayer_instance_start(backend *be, int mode) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + dblayer_private *priv; + struct dblayer_private_env *pEnv; + char inst_dir[MAXPATHLEN]; + char *inst_dirp = NULL; + int return_value; + + priv = (dblayer_private*)li->li_dblayer_private; + pEnv = priv->dblayer_env; + if (CATASTROPHIC == pEnv || NULL == pEnv) { + LDAPDebug(LDAP_DEBUG_ANY, + "instance %s: dbenv is not available (0x%x).\n", + inst?inst->inst_name:"unknown", pEnv, 0); + return -1; + } + + if (NULL != inst->inst_id2entry) { + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: DB instance \"%s\" already started.\n", + inst->inst_name, 0, 0); + return 0; + } + + attrcrypt_init(inst); + + /* Get the name of the directory that holds index files + * for this instance. */ + if (dblayer_get_instance_data_dir(be) != 0) { + /* Problem getting the name of the directory that holds the + * index files for this instance. */ + return -1; + } + + inst_dirp = dblayer_get_full_inst_dir(li, inst, inst_dir, MAXPATHLEN); + return_value = dblayer_grok_directory(inst_dirp, + DBLAYER_DIRECTORY_READWRITE_ACCESS); + if (0 != return_value) { + LDAPDebug(LDAP_DEBUG_ANY,"Can't start because the database instance " + "directory \"%s\" either doesn't exist, " + "or the db files are not accessible\n", + inst_dirp, 0, 0); + goto errout; + } + + if (mode & DBLAYER_NORMAL_MODE) { + /* In normal mode (not db2ldif, ldif2db, etc.) we need to deal with + * the dbversion file here. */ + + /* Read the dbversion file if there is one, and create it + * if it doesn't exist. */ + if (dbversion_exists(li, inst_dirp)) { + char ldbmversion[LDBM_VERSION_MAXBUF]; + char dataversion[LDBM_VERSION_MAXBUF]; + + if (dbversion_read(li, inst_dirp, ldbmversion, dataversion) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "Warning: Unable to read dbversion " + "file in %s\n", inst->inst_dir_name, 0, 0); + } else { + int rval = 0; +#if defined(UPGRADEDB) + /* check the DBVERSION and reset idl-switch if needed (DS6.2) */ + /* from the next major rel, we won't do this and just upgrade */ + if (!(li->li_flags & LI_FORCE_MOD_CONFIG)) + adjust_idl_switch(ldbmversion, li); +#endif + + /* check to make sure these instance was made with the correct + * version. */ + rval = check_db_inst_version(inst); + if (rval & DBVERSION_NOT_SUPPORTED) + { + LDAPDebug(LDAP_DEBUG_ANY, "Instance %s does not have the " + "expected version\n", inst->inst_name, 0, 0); + PR_ASSERT(0); + return_value = -1; + goto errout; + } + if (rval & DBVERSION_NEED_IDL_OLD2NEW) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Instance %s: idl-switch is new while db idl format is " + "old, modify nsslapd-idl-switch in dse.ldif to old\n", + inst->inst_name, 0, 0); + } + if (rval & DBVERSION_NEED_IDL_NEW2OLD) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Instance %s: idl-switch is old while db idl format is " + "new, modify nsslapd-idl-switch in dse.ldif to new\n", + inst->inst_name, 0, 0); + } + + /* record the dataversion */ + if (dataversion[0] != '\0') { + inst->inst_dataversion = slapi_ch_strdup(dataversion); + } + + rval = ldbm_upgrade(inst, rval); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, "Upgrading instance %s failed\n", + inst->inst_name, 0, 0); + PR_ASSERT(0); + return_value = -1; + goto errout; + } + } + } else { + /* The dbversion file didn't exist, so we'll create one. */ + dbversion_write(li, inst_dirp, NULL); + } + } /* on import we don't mess with the dbversion file except to write it + * when done with the import. */ + + /* Now attempt to open id2entry */ + { + char *id2entry_file; + int open_flags = 0; + DB *dbp; + char *subname; + struct dblayer_private_env *mypEnv; + + id2entry_file = slapi_ch_malloc(strlen(inst->inst_dir_name) + + strlen(ID2ENTRY LDBM_FILENAME_SUFFIX) + 2); + sprintf(id2entry_file, "%s/%s", inst->inst_dir_name, + ID2ENTRY LDBM_FILENAME_SUFFIX); + + open_flags = DB_CREATE | DB_THREAD; + + /* The subname argument allows applications to have + * subdatabases, i.e., multiple databases inside of a single + * physical file. This is useful when the logical databases + * are both numerous and reasonably small, in order to + * avoid creating a large number of underlying files. + */ + subname = NULL; + mypEnv = NULL; + if (mode & (DBLAYER_IMPORT_MODE|DBLAYER_INDEX_MODE)) { + size_t cachesize; + char *data_directories[2] = {0, 0}; + /* [605974] delete DB_PRIVATE: + * to make import visible to the other process */ + int oflags = DB_CREATE | DB_INIT_MPOOL | DB_THREAD; + /* + * but nsslapd-db-private-import-mem should work with import, + * as well */ + if (priv->dblayer_private_import_mem) { + LDAPDebug(LDAP_DEBUG_ANY, + "WARNING: Import is running with " + "nsslapd-db-private-import-mem on; " + "No other process is allowed to access the database\n", + 0, 0, 0); + oflags |= DB_PRIVATE; + } + PR_Lock(li->li_config_mutex); + if ((li->li_flags & TASK_RUNNING_FROM_COMMANDLINE) && + (li->li_import_cache_autosize)) /* Autosizing importCache + * Need to re-eval every time + * to guarantee the memory is + * really available + * (just for command line I/F) + */ + { + autosize_import_cache(li); + } + cachesize = li->li_import_cachesize; + PR_Unlock(li->li_config_mutex); + + if (cachesize < 1048576) { + /* make it at least 1M */ + cachesize = 1048576; + } + priv->dblayer_cachesize = cachesize; + /* We always auto-calculate ncache for the import region */ + priv->dblayer_ncache = 0; + + /* use our own env */ + return_value = dblayer_make_env(&mypEnv, li); + if (return_value != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "Unable to create new DB_ENV for import/export! %d\n", + return_value, 0, 0); + goto out; + } + /* do not assume import cache size is under 1G */ + mypEnv->dblayer_DB_ENV->set_cachesize(mypEnv->dblayer_DB_ENV, + cachesize/GIGABYTE, + cachesize%GIGABYTE, + priv->dblayer_ncache); + /* probably want to change this -- but for now, create the + * mpool files in the instance directory. + */ + mypEnv->dblayer_openflags = oflags; + data_directories[0] = inst->inst_parent_dir_name; + dblayer_set_data_dir(priv, mypEnv, data_directories); + return_value = mypEnv->dblayer_DB_ENV->open(mypEnv->dblayer_DB_ENV, + inst_dirp, + oflags, + priv->dblayer_file_mode); + if (return_value != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "Unable to open new DB_ENV for import/export! %d\n", + return_value, 0, 0); + goto out; + } + inst->import_env = mypEnv; + } else { + mypEnv = pEnv; + } + + inst->inst_id2entry = NULL; + return_value = db_create(&inst->inst_id2entry, mypEnv->dblayer_DB_ENV, 0); + if (0 != return_value) { + LDAPDebug(LDAP_DEBUG_ANY, + "Unable to create id2entry db file! %d\n", + return_value, 0, 0); + goto out; + } + dbp = inst->inst_id2entry; + + return_value = dbp->set_pagesize(dbp, + (priv->dblayer_page_size == 0) ? DBLAYER_PAGESIZE : + priv->dblayer_page_size); + if (0 != return_value) { + LDAPDebug(LDAP_DEBUG_ANY, + "dbp->set_pagesize(%lu or %lu) failed %d\n", + priv->dblayer_page_size, DBLAYER_PAGESIZE, + return_value); + goto out; + } + +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR < 3300 + return_value = dbp->set_malloc(dbp, malloc); + if (0 != return_value) { + LDAPDebug(LDAP_DEBUG_ANY, + "dbp->set_malloc failed %d\n", + return_value, 0, 0); + + goto out; + } +#endif + if ((charray_get_index(priv->dblayer_data_directories, + inst->inst_parent_dir_name) != 0) && + !dblayer_inst_exists(inst, NULL)) + { + char *abs_id2entry_file = NULL; + int abs_len; + /* create a file with abs path, then try again */ + + abs_len = strlen(inst_dirp) + + strlen(ID2ENTRY LDBM_FILENAME_SUFFIX) + 2; + abs_id2entry_file = (char *)slapi_ch_malloc(abs_len); + sprintf(abs_id2entry_file, "%s%c%s", inst_dirp, + get_sep(inst_dirp), ID2ENTRY LDBM_FILENAME_SUFFIX); + DB_OPEN(mypEnv->dblayer_openflags, + dbp, NULL/* txnid */, abs_id2entry_file, subname, DB_BTREE, + open_flags, priv->dblayer_file_mode, return_value); + dbp->close(dbp, 0); + return_value = db_create(&inst->inst_id2entry, + mypEnv->dblayer_DB_ENV, 0); + if (0 != return_value) + goto out; + dbp = inst->inst_id2entry; + slapi_ch_free_string(&abs_id2entry_file); + } + DB_OPEN(mypEnv->dblayer_openflags, + dbp, NULL/* txnid */, id2entry_file, subname, DB_BTREE, + open_flags, priv->dblayer_file_mode, return_value); + if (0 != return_value) { + LDAPDebug(LDAP_DEBUG_ANY, + "dbp->open(\"%s\") failed: %s (%d)\n", + id2entry_file, dblayer_strerror(return_value), + return_value); + /* if it's a newly created backend instance, + * need to check the inst_parent_dir already exists and + * set as a data dir */ + if (strstr(dblayer_strerror(return_value), + "No such file or directory")) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Instance %s is not registered as a db data directory. " + "Please restart the server to create it.\n", + inst?inst->inst_name:"unknown", pEnv, 0); + } + else if (strstr(dblayer_strerror(return_value), + "Permission denied")) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Instance directory %s may not be writable\n", + inst_dirp, 0, 0); + } + + goto out; + } +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR == 4100 + /* lower the buffer cache priority to avoid sleep in memp_alloc */ + /* W/ DB_PRIORITY_LOW, the db buffer page priority is calculated as: + * priority = lru_count + pages / (-1) + * (by default, priority = lru_count) + * When upgraded to db4.2, this setting may not needed, hopefully. + * ask sleepycat [#8301]; blackflag #619964 + */ + dbp->set_cache_priority(dbp, DB_PRIORITY_LOW); +#endif +out: + slapi_ch_free((void**)&id2entry_file); + } + + if (0 == return_value) { + /* get nextid from disk now */ + get_ids_from_disk(be); + } + + if (mode & DBLAYER_NORMAL_MODE) { + dbversion_write(li, inst_dirp, NULL); + } + + /* + * check if nextid is valid: it only matters if the database is either + * being imported or is in normal mode + */ + if (inst->inst_nextid > MAXID && !(mode & DBLAYER_EXPORT_MODE)) { + LDAPDebug(LDAP_DEBUG_ANY, "dblayer_instance_start fail: backend '%s' " + "has no IDs left. DATABASE MUST BE REBUILT.\n", + be->be_name, 0, 0); + return 1; + } + + if (return_value != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "dblayer_instance_start fail: %s (%d)\n", + dblayer_strerror(return_value), return_value, 0); + } +errout: + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return return_value; +} + + +/* This returns a DB* for the primary index. + * If the database library is non-reentrant, we lock it. + * the caller MUST call to unlock the db library once they're + * finished with the handle. Luckily, the back-end already has + * these semantics for the older dbcache stuff. + */ +/* Things have changed since the above comment was + * written. The database library is reentrant. */ +int dblayer_get_id2entry(backend *be, DB **ppDB) +{ + ldbm_instance *inst; + + PR_ASSERT(NULL != be); + + inst = (ldbm_instance *) be->be_instance_info; + + *ppDB = inst->inst_id2entry; + return 0; +} + +int dblayer_release_id2entry(backend *be, DB *pDB) +{ + return 0; +} + +#ifdef UPGRADEDB +/* + * dblayer_get_aux_id2entry: + * - create a dedicated db env and db handler for id2entry. + * - introduced for upgradedb not to share the env and db handler with + * other index files to support multiple passes and merge. + */ +int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv) +{ + ldbm_instance *inst; + struct dblayer_private_env *mypEnv = NULL; + DB *dbp = NULL; + int rval = 1; + struct ldbminfo *li; + dblayer_private *opriv; + dblayer_private *priv; + char *subname = NULL; + int envflags; + size_t cachesize; + PRFileInfo prfinfo; + PRStatus prst; + char *id2entry_file; + char inst_dir[MAXPATHLEN]; + char *inst_dirp = NULL; + char *data_directories[2] = {0, 0}; + + PR_ASSERT(NULL != be); + + *ppDB = NULL; + *ppEnv = NULL; + inst = (ldbm_instance *) be->be_instance_info; + if (NULL == inst) + { + LDAPDebug(LDAP_DEBUG_ANY, + "No instance/env: persistent id2entry is not available\n", 0, 0, 0); + goto done; + } + + li = inst->inst_li; + if (NULL == li) + { + LDAPDebug(LDAP_DEBUG_ANY, + "No ldbm info: persistent id2entry is not available\n", 0, 0, 0); + goto done; + } + + opriv = li->li_dblayer_private; + if (NULL == opriv) + { + LDAPDebug(LDAP_DEBUG_ANY, + "No dblayer info: persistent id2entry is not available\n", 0, 0, 0); + goto done; + } + priv = (dblayer_private *)slapi_ch_malloc(sizeof(dblayer_private)); + memcpy(priv, opriv, sizeof(dblayer_private)); + priv->dblayer_spin_count = 0; + + inst_dirp = dblayer_get_full_inst_dir(li, inst, inst_dir, MAXPATHLEN); + priv->dblayer_home_directory = + slapi_ch_malloc(strlen(inst_dirp) + strlen("dbenv") + 2); + sprintf(priv->dblayer_home_directory, "%s/dbenv", inst_dirp); + priv->dblayer_log_directory = slapi_ch_strdup(priv->dblayer_home_directory); + + prst = PR_GetFileInfo(inst_dirp, &prfinfo); + if (PR_FAILURE == prst || PR_FILE_DIRECTORY != prfinfo.type) + { + LDAPDebug(LDAP_DEBUG_ANY, + "No inst dir: persistent id2entry is not available\n", 0, 0, 0); + goto done; + } + + prst = PR_GetFileInfo(priv->dblayer_home_directory, &prfinfo); + if (PR_SUCCESS == prst) + { + ldbm_delete_dirs(priv->dblayer_home_directory); + } + rval = mkdir_p(priv->dblayer_home_directory, 0700); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, + "can't create env dir: persistent id2entry is not available\n", 0, 0, 0); + goto done; + } + + /* use our own env */ + rval = dblayer_make_env(&mypEnv, li); + if (rval != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "Unable to create new DB_ENV for import/export! %d\n", rval, 0, 0); + goto err; + } + + envflags = DB_CREATE | DB_INIT_MPOOL | DB_PRIVATE; + cachesize = 1048576; /* 1M */ + + mypEnv->dblayer_DB_ENV->set_cachesize(mypEnv->dblayer_DB_ENV, + 0, cachesize, priv->dblayer_ncache); + + /* probably want to change this -- but for now, create the + * mpool files in the instance directory. + */ + mypEnv->dblayer_openflags = envflags; + data_directories[0] = inst->inst_parent_dir_name; + dblayer_set_data_dir(priv, mypEnv, data_directories); + rval = mypEnv->dblayer_DB_ENV->open(mypEnv->dblayer_DB_ENV, + priv->dblayer_home_directory, envflags, priv->dblayer_file_mode); + if (rval != 0) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Unable to open new DB_ENV for upgradedb/reindex %d\n", rval, 0, 0); + goto err; + } + *ppEnv = mypEnv->dblayer_DB_ENV; + + rval = db_create(&dbp, mypEnv->dblayer_DB_ENV, 0); + if (0 != rval) { + LDAPDebug(LDAP_DEBUG_ANY, + "Unable to create id2entry db handler! %d\n", rval, 0, 0); + goto err; + } + + rval = dbp->set_pagesize(dbp, (priv->dblayer_page_size == 0) ? + DBLAYER_PAGESIZE : priv->dblayer_page_size); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dbp->set_pagesize(%lu or %lu) failed %d\n", + priv->dblayer_page_size, DBLAYER_PAGESIZE, rval); + goto err; + } + + id2entry_file = slapi_ch_malloc(strlen(inst->inst_dir_name) + + strlen(ID2ENTRY LDBM_FILENAME_SUFFIX) + 2); + sprintf(id2entry_file, "%s/%s", + inst->inst_dir_name, ID2ENTRY LDBM_FILENAME_SUFFIX); + + PR_ASSERT(dblayer_inst_exists(inst, NULL)); + DB_OPEN(mypEnv->dblayer_openflags, + dbp, NULL/* txnid */, id2entry_file, subname, DB_BTREE, + 0, priv->dblayer_file_mode, rval); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dbp->open(\"%s\") failed: %s (%d)\n", + id2entry_file, dblayer_strerror(rval), rval); + if (strstr(dblayer_strerror(rval), "Permission denied")) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Instance directory %s may not be writable\n", inst_dirp, 0, 0); + } + goto err; + } + *ppDB = dbp; + goto done; +err: + if (*ppEnv) + (*ppEnv)->close(*ppEnv, 0); + if (priv->dblayer_home_directory) + ldbm_delete_dirs(priv->dblayer_home_directory); +done: + slapi_ch_free_string(&id2entry_file); + slapi_ch_free_string(&priv->dblayer_home_directory); + slapi_ch_free_string(&priv->dblayer_log_directory); + /* Don't free priv->dblayer_data_directories since priv doesn't own the memory */ + slapi_ch_free((void **)&priv); + slapi_ch_free((void **)&mypEnv); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return rval; +} + +int dblayer_release_aux_id2entry(backend *be, DB *pDB, DB_ENV *pEnv) +{ + ldbm_instance *inst; + char *envdir = NULL; + char inst_dir[MAXPATHLEN]; + char *inst_dirp = NULL; + + inst = (ldbm_instance *) be->be_instance_info; + if (NULL == inst) + { + LDAPDebug(LDAP_DEBUG_ANY, + "No instance/env: persistent id2entry is not available\n", 0, 0, 0); + goto done; + } + + inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN); + envdir = slapi_ch_malloc(strlen(inst_dirp) + strlen("dbenv") + 2); + sprintf(envdir, "%s/dbenv", inst_dirp); + +done: + if (pDB) + pDB->close(pDB, 0); + if (pEnv) + pEnv->close(pEnv, 0); + if (envdir) + ldbm_delete_dirs(envdir); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return 0; +} +#endif + +int dblayer_close_indexes(backend *be) +{ + ldbm_instance *inst; + DB *pDB = NULL; + dblayer_handle *handle = NULL; + dblayer_handle *next = NULL; + int return_value = 0; + + PR_ASSERT(NULL != be); + inst = (ldbm_instance *) be->be_instance_info; + PR_ASSERT(NULL != inst); + + for (handle = inst->inst_handle_head; handle != NULL; handle = next) { + /* Close it, and remove from the list */ + pDB = handle->dblayer_dbp; + return_value |= pDB->close(pDB,0); + next = handle->dblayer_handle_next; + *(handle->dblayer_handle_ai_backpointer) = NULL; + slapi_ch_free((void**)&handle); + } + + /* reset the list to make sure we don't use it again */ + inst->inst_handle_tail = NULL; + inst->inst_handle_head = NULL; + + return return_value; +} + +int dblayer_instance_close(backend *be) +{ + DB *pDB = NULL; + int return_value = 0; + DB_ENV * env = 0; + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; + + if (NULL == inst) + return -1; + + return_value = dblayer_close_indexes(be); + + /* Now close id2entry if it's open */ + pDB = inst->inst_id2entry; + if (NULL != pDB) { + return_value |= pDB->close(pDB,0); + } + inst->inst_id2entry = NULL; + + if (inst->import_env) { + /* ignore the value of env, close, because at this point, work is done with import env + by calling env.close, env and all the associated db handles will be closed, ignore, + if sleepycat complains, that db handles are open at env close time */ + return_value |= inst->import_env->dblayer_DB_ENV->close(inst->import_env->dblayer_DB_ENV, 0); + return_value = db_env_create(&env, 0); + if (return_value == 0) { + char inst_dir[MAXPATHLEN]; + char *inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN); + return_value = env->remove(env, inst_dirp, 0); + if (return_value == EBUSY) { + return_value = 0; /* something else is using the env so ignore */ + } + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + } + PR_DestroyRWLock(inst->import_env->dblayer_env_lock); + PR_Free((void *)inst->import_env); + inst->import_env = NULL; + } else { + be->be_state = BE_STATE_STOPPED; + } + + return return_value; +} + +void dblayer_pre_close(struct ldbminfo *li) +{ + dblayer_private *priv = 0; + + PR_ASSERT(NULL != li); + priv = (dblayer_private*)li->li_dblayer_private; + + if (priv->dblayer_stop_threads) /* already stopped. do nothing... */ + return; + + /* Now stop the make sure they've stopped, + * the various threads we have running */ + priv->dblayer_stop_threads = 1; + if (priv->dblayer_thread_count > 0) { + int sleep_counter = 0; + /* Print handy-dandy log message */ + LDAPDebug(LDAP_DEBUG_ANY,"Waiting for %d database threads to stop\n", + priv->dblayer_thread_count, 0,0); + while(priv->dblayer_thread_count > 0) { + /* We have no alternative here but to wait for them to finish, + * since if any thread activity, + * such as a checkpoint, wasn't finished when we shut down, + * the database would need recovery on startup */ + PRIntervalTime interval; + + sleep_counter++; + if (0 == sleep_counter % 10) { + if (priv->dblayer_thread_count > 1) { + LDAPDebug(LDAP_DEBUG_ANY, + "Still waiting on %d database threads...\n", + priv->dblayer_thread_count,0,0); + } else { + LDAPDebug(LDAP_DEBUG_ANY, + "Still waiting one database thread...\n",0,0,0); + } + } + + interval = PR_MillisecondsToInterval(DBLAYER_SLEEP_INTERVAL); + DS_Sleep(interval); + if (sleep_counter >= 100) { + LDAPDebug(LDAP_DEBUG_ANY,"Timeout; leave %d thread(s)...\n", + priv->dblayer_thread_count,0,0); + /* + * The guardian file should not + * get created under the timeout condition. + */ + priv->dblayer_bad_stuff_happened = 1; + goto timeout_escape; + } + } + LDAPDebug(LDAP_DEBUG_ANY,"All database threads now stopped\n",0,0,0); + } +timeout_escape: + return; +} + +int dblayer_post_close(struct ldbminfo *li, int dbmode) +{ + dblayer_private *priv = 0; + int return_value = 0; + dblayer_private_env *pEnv; + + PR_ASSERT(NULL != li); + priv = (dblayer_private*)li->li_dblayer_private; + + /* We close all the files we ever opened, and call pEnv->close. */ + if (NULL == priv->dblayer_env) /* db env is already closed. do nothing. */ + return return_value; + /* Shutdown the performance counter stuff */ + if (DBLAYER_NORMAL_MODE & dbmode) { + if (priv->perf_private) { + perfctrs_terminate(&priv->perf_private); + } + } + + /* Now release the db environment */ + pEnv = priv->dblayer_env; + return_value = pEnv->dblayer_DB_ENV->close(pEnv->dblayer_DB_ENV, 0); + PR_DestroyRWLock(priv->dblayer_env->dblayer_env_lock); + PR_Free((void *) priv->dblayer_env); + + priv->dblayer_env = NULL; /* pEnv is now garbage */ + + if (return_value == 0) { + DB_ENV *env = 0; + return_value = db_env_create(&env, 0); + /* don't be tempted to use the + previously nulled out env handle + as Sleepycat 3.x is unhappy if + the env handle handed to remove + was used elsewhere. rwagner */ + if (return_value == 0) { + char *home_dir = dblayer_get_home_dir(li, NULL); + if (home_dir) + /* DBDB do NOT remove the environment: bad, bad idea + * return_value = env->remove(env, home_dir, 0); + */ + if (0 == return_value + && !((DBLAYER_ARCHIVE_MODE|DBLAYER_EXPORT_MODE) & dbmode) + && !priv->dblayer_bad_stuff_happened) { + /* + * If we got here, we have a good consistent database, + * so we write the guard file + */ + commit_good_database(priv); + } else if (return_value == EBUSY) { + /* something else is using the env so ignore */ + /* but let's not make a guardian file */ + return_value = 0; + } + } + } + + return return_value; +} + +/* + * This function is called when the server is shutting down. + * This is not safe to call while other threads are calling into the open + * databases !!! So: DON'T ! + */ +int dblayer_close(struct ldbminfo *li, int dbmode) +{ + backend *be = NULL; + ldbm_instance *inst; + Object *inst_obj; + int return_value = 0; + + dblayer_pre_close(li); + + /* + * dblayer_close_indexes and pDB->close used to be located above loop: + * while(priv->dblayer_thread_count > 0) in pre_close. + * This order fixes a bug: shutdown under the stress makes txn_checkpoint + * (checkpoint_thread) fail b/c the mpool might have been already closed. + */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + be = inst->inst_be; + if (NULL != be->be_instance_info) { + return_value |= dblayer_instance_close(be); + } + } + + if (return_value != 0) { + /* force recovery next startup if any close failed */ + dblayer_private *priv; + PR_ASSERT(NULL != li); + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + priv->dblayer_bad_stuff_happened = 1; + } + + return_value |= dblayer_post_close(li, dbmode); + + return return_value; +} + +/* + * Called to tell us to flush any data not on disk to the disk + * for the transacted database, we interpret this as an instruction + * to write a checkpoint. + */ +int dblayer_flush(struct ldbminfo *li) +{ + return 0; +} + +#if !defined(DB_DUPSORT) +#define DB_DUPSORT 0 +#endif + +/* Routines for opening and closing random files in the DB_ENV. + Used by ldif2db merging code currently. + + Return value: + Success: 0 + Failure: -1 + */ +int dblayer_open_file(backend *be, char* indexname, int open_flag, int index_flags, DB **ppDB) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int open_flags = 0; + char *file_name = NULL; + char *rel_path = NULL; + dblayer_private_env *pENV = 0; + dblayer_private *priv = NULL; + int len; + int return_value = 0; + DB *dbp = NULL; + char *subname = NULL; + + PR_ASSERT(NULL != li); + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + if (NULL == inst->inst_dir_name) + { + if (dblayer_get_instance_data_dir(be) != 0) + return -1; + } + + if (NULL != inst->inst_parent_dir_name) + { + if (!charray_utf8_inlist(priv->dblayer_data_directories, + inst->inst_parent_dir_name) && + !is_fullpath(inst->inst_dir_name)) + + { + LDAPDebug(LDAP_DEBUG_ANY, + "The instance path %s is not registered for db_data_dir, " + "although %s is a relative path.\n", + inst->inst_parent_dir_name, inst->inst_dir_name, 0); + return -1; + } + } + len = strlen(indexname) + strlen(LDBM_FILENAME_SUFFIX) + 1; + file_name = slapi_ch_malloc(len); + len += strlen(inst->inst_dir_name) + 1; + rel_path = slapi_ch_malloc(len); + + pENV = priv->dblayer_env; + if (inst->import_env) + pENV = inst->import_env; + + PR_ASSERT(NULL != pENV); + sprintf(file_name, "%s%s", indexname, LDBM_FILENAME_SUFFIX); + sprintf(rel_path, "%s/%s", inst->inst_dir_name, file_name); + + open_flags = DB_THREAD; + if (open_flag & DBOPEN_CREATE) + open_flags |= DB_CREATE; + + if (!ppDB) + goto out; + return_value = db_create(ppDB, pENV->dblayer_DB_ENV, 0); + if (0 != return_value) + goto out; + + dbp = *ppDB; +/* With the new idl design, the large 8Kbyte pages we use are not + optimal. The page pool churns very quickly as we add new IDs under a + sustained add load. Smaller pages stop this happening so much and + consequently make us spend less time flushing dirty pages on checkpoints. + But 8K is still a good page size for id2entry. So we now allow different + page sizes for the primary and secondary indices. + Filed as bug: 604654 + */ + if (idl_get_idl_new()) { + return_value = dbp->set_pagesize( + dbp, + (priv->dblayer_index_page_size == 0) ? + DBLAYER_INDEX_PAGESIZE : priv->dblayer_index_page_size + ); + } else { + return_value = dbp->set_pagesize( + dbp, + (priv->dblayer_page_size == 0) ? + DBLAYER_PAGESIZE : priv->dblayer_page_size + ); + } + if (0 != return_value) + goto out; + +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR < 3300 + return_value = dbp->set_malloc(dbp, malloc); + if (0 != return_value) { + goto out; + } +#endif + + if (idl_get_idl_new() && !(index_flags & INDEX_VLV)) { + return_value = dbp->set_flags(dbp, DB_DUP | DB_DUPSORT); + if (0 != return_value) + goto out; + + return_value = dbp->set_dup_compare( + dbp, + idl_new_compare_dups + ); + if (0 != return_value) + goto out; + } + + if (index_flags & INDEX_VLV) { + /* + * Need index with record numbers for + * Virtual List View index + */ + return_value = dbp->set_flags(dbp, DB_RECNUM); + if (0 != return_value) + goto out; + } + + /* The subname argument allows applications to have + * subdatabases, i.e., multiple databases inside of a single + * physical file. This is useful when the logical databases + * are both numerous and reasonably small, in order to + * avoid creating a large number of underlying files. + */ + if ((charray_get_index(priv->dblayer_data_directories, + inst->inst_parent_dir_name) != 0) && + !dblayer_inst_exists(inst, file_name)) + { + char inst_dir[MAXPATHLEN]; + char *inst_dirp = NULL; + char *abs_file_name = NULL; + int abs_len; + /* create a file with abs path, then try again */ + + inst_dirp = dblayer_get_full_inst_dir(li, inst, inst_dir, MAXPATHLEN); + abs_len = strlen(inst_dirp) + strlen(file_name) + 2; + abs_file_name = (char *)slapi_ch_malloc(abs_len); + sprintf(abs_file_name, "%s%c%s", + inst_dirp, get_sep(inst_dirp), file_name); + DB_OPEN(pENV->dblayer_openflags, + dbp, NULL/* txnid */, abs_file_name, subname, DB_BTREE, + open_flags, priv->dblayer_file_mode, return_value); + dbp->close(dbp, 0); + return_value = db_create(ppDB, pENV->dblayer_DB_ENV, 0); + if (0 != return_value) + goto out; + dbp = *ppDB; + + slapi_ch_free_string(&abs_file_name); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + } + DB_OPEN(pENV->dblayer_openflags, + dbp, NULL, /* txnid */ rel_path, subname, DB_BTREE, + open_flags, priv->dblayer_file_mode, return_value); +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR == 4100 + /* lower the buffer cache priority to avoid sleep in memp_alloc */ + /* W/ DB_PRIORITY_LOW, the db buffer page priority is calculated as: + * priority = lru_count + pages / (-1) + * (by default, priority = lru_count) + * When upgraded to db4.2, this setting may not needed, hopefully. + * ask sleepycat [#8301]; blackflag #619964 + */ + dbp->set_cache_priority(dbp, DB_PRIORITY_LOW); +#endif +out: + slapi_ch_free((void**)&file_name); + slapi_ch_free((void**)&rel_path); + /* close the database handle to avoid handle leak */ + if (dbp && (return_value != 0)) { + dblayer_close_file(dbp); + } + return return_value; +} + +int dblayer_close_file(DB *db) +{ + return db->close(db,0); +} + +/* + * OK, this is a tricky one. We store open DB* handles within an AVL + * structure used in other parts of the back-end. This is nasty, because + * that code has no idea we're doing this, and we don't have much control + * over what it does. But, the reason is that we want to get fast lookup + * of the index file pertaining to each particular attribute. Putting the + * DB* handles in the attribute info structures is an easy way to achieve this + * because we already lookup this structure as part of an index lookup. + */ + +/* + * This function takes an attrinfo structure and returns a valid + * DB* handle for the index file corresponding to this attribute. + */ + +/* + * If the db library is non-reentrant, we lock the mutex here, + * see comments above for id2entry for the details on this. + */ + +/* + * The create flag determines if the index file should be created if it + * does not already exist. + */ + +int dblayer_get_index_file(backend *be, struct attrinfo *a, DB** ppDB, int open_flags) +{ + /* + * We either already have a DB* handle in the attrinfo structure. + * in which case we simply return it to the caller, OR: + * we need to make one. We do this as follows: + * 1a) acquire the mutex that protects the handle list. + * 1b) check that the DB* is still null. + * 2) get the filename, and call libdb to open it + * 3) if successful, store the result in the attrinfo stucture + * 4) store the DB* in our own list so we can close it later. + * 5) release the mutex. + */ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int return_value = -1; + DB *pDB = NULL; + char *attribute_name = a->ai_type; + + *ppDB = NULL; + + /* it's like a semaphore -- when count > 0, any file handle that's in + * the attrinfo will remain valid from here on. + */ + PR_AtomicIncrement(&a->ai_dblayer_count); + + if (NULL != a->ai_dblayer) { + /* This means that the pointer is valid, so we should return it. */ + *ppDB = ((dblayer_handle*)(a->ai_dblayer))->dblayer_dbp; + return 0; + } + + /* attrinfo handle is NULL, at least for now -- grab the mutex and try + * again. + */ + PR_Lock(inst->inst_handle_list_mutex); + if (NULL != a->ai_dblayer) { + /* another thread set the handle while we were waiting on the lock */ + *ppDB = ((dblayer_handle*)(a->ai_dblayer))->dblayer_dbp; + PR_Unlock(inst->inst_handle_list_mutex); + return 0; + } + + /* attrinfo handle is still blank, and we have the mutex: open the + * index file and stuff it in the attrinfo. + */ + return_value = dblayer_open_file(be, attribute_name, open_flags, + a->ai_indexmask, &pDB); + if (0 == return_value) { + /* Opened it OK */ + dblayer_handle *handle = (dblayer_handle *) + slapi_ch_calloc(1, sizeof(dblayer_handle)); + + if (NULL == handle) { + /* Memory allocation failed */ + return_value = -1; + } else { + dblayer_handle *prev_handle = inst->inst_handle_tail; + + PR_ASSERT(NULL != pDB); + /* Store the returned DB* in our own private list of + * open files */ + if (NULL == prev_handle) { + /* List was empty */ + inst->inst_handle_tail = handle; + inst->inst_handle_head = handle; + } else { + /* Chain the handle onto the last structure in the + * list */ + inst->inst_handle_tail = handle; + prev_handle->dblayer_handle_next = handle; + } + /* Stash a pointer to our wrapper structure in the + * attrinfo structure */ + handle->dblayer_dbp = pDB; + /* And, most importantly, return something to the caller!*/ + *ppDB = pDB; + /* and save the hande in the attrinfo structure for + * next time */ + a->ai_dblayer = handle; + /* don't need to update count -- we incr'd it already */ + handle->dblayer_handle_ai_backpointer = &(a->ai_dblayer); + } + } else { + /* Did not open it OK ! */ + /* Do nothing, because return value and fact that we didn't + * store a DB* in the attrinfo is enough + */ + } + PR_Unlock(inst->inst_handle_list_mutex); + + if (return_value != 0) { + /* some sort of error -- we didn't open a handle at all. + * decrement the refcount back to where it was. + */ + PR_AtomicDecrement(&a->ai_dblayer_count); + } + + return return_value; +} + +/* + * Unlock the db lib mutex here if we need to. + */ +int dblayer_release_index_file(backend *be,struct attrinfo *a, DB* pDB) +{ + PR_AtomicDecrement(&a->ai_dblayer_count); + return 0; +} + +/* + dblayer_db_remove assumptions: + + No environment has the given database open. + +*/ + +static int +dblayer_db_remove_ex(dblayer_private_env *env, char const path[], char const dbName[], PRBool use_lock) { + DB_ENV * db_env = 0; + int rc; + DB *db; + + if (env) { + if(use_lock) PR_RWLock_Wlock(env->dblayer_env_lock); /* We will be causing logging activity */ + db_env = env->dblayer_DB_ENV; + } + + rc = db_create(&db, db_env, 0); /* must use new handle to database */ + if (0 != rc) { + LDAPDebug(LDAP_DEBUG_ANY, "db_remove: Failed to create db (%d) %s\n", + rc, dblayer_strerror(rc), 0); + goto done; + } + rc = db->remove(db, path, dbName, 0); /* kiss the db goodbye! */ + +done: + if (env) { + if(use_lock) PR_RWLock_Unlock(env->dblayer_env_lock); + } + + return rc; +} + + +int +dblayer_db_remove(dblayer_private_env * env, char const path[], char const dbName[]) { + return(dblayer_db_remove_ex(env,path,dbName,PR_TRUE)); +} + +#define DBLAYER_CACHE_DELAY PR_MillisecondsToInterval(250) +int dblayer_erase_index_file_ex(backend *be, struct attrinfo *a, + PRBool use_lock, int no_force_checkpoint) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + dblayer_private *priv = (dblayer_private*) li->li_dblayer_private; + struct dblayer_private_env *pEnv = priv->dblayer_env; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + dblayer_handle *handle; + char dbName[MAXPATHLEN]; + char *dbNamep; + char *p; + int dbbasenamelen, dbnamelen; + int rc = 0; + DB *db = 0; + + if (NULL == pEnv) /* index file does not exist */ + return rc; + + /* Added for bug 600401. Somehow the checkpoint thread deadlocked on + index file with this function, index file couldn't be removed on win2k. + Force a checkpoint here to break deadlock. + */ + if (0 == no_force_checkpoint) { + dblayer_force_checkpoint(li); + } + + if (dblayer_get_index_file(be, a, &db, DBOPEN_CREATE) == 0) { + /* first, remove the file handle for this index, if we have it open */ + PR_Lock(inst->inst_handle_list_mutex); + if (a->ai_dblayer) { + /* there is a handle */ + handle = (dblayer_handle *)a->ai_dblayer; + + /* when we successfully called dblayer_get_index_file we bumped up + the reference count of how many threads are using the index. So we + must manually back off the count by one here.... rwagner */ + + dblayer_release_index_file(be, a, db); + + while (a->ai_dblayer_count > 0) { + /* someone is using this index file */ + /* ASSUMPTION: you have already set the INDEX_OFFLINE flag, because + * you intend to mess with this index. therefore no new requests + * for this indexfile should happen, so the dblayer_count should + * NEVER increase. + */ + PR_ASSERT(a->ai_indexmask & INDEX_OFFLINE); + PR_Unlock(inst->inst_handle_list_mutex); + DS_Sleep(DBLAYER_CACHE_DELAY); + PR_Lock(inst->inst_handle_list_mutex); + } + dblayer_close_file(handle->dblayer_dbp); + + /* remove handle from handle-list */ + if (inst->inst_handle_head == handle) { + inst->inst_handle_head = handle->dblayer_handle_next; + if (inst->inst_handle_tail == handle) { + inst->inst_handle_tail = NULL; + } + } else { + dblayer_handle *hp; + + for (hp = inst->inst_handle_head; hp; hp = hp->dblayer_handle_next) { + if (hp->dblayer_handle_next == handle) { + hp->dblayer_handle_next = handle->dblayer_handle_next; + if (inst->inst_handle_tail == handle) { + inst->inst_handle_tail = hp; + } + break; + } + } + } + dbNamep = dblayer_get_full_inst_dir(li, inst, dbName, MAXPATHLEN); + dbbasenamelen = strlen(dbNamep); + dbnamelen = dbbasenamelen + strlen(a->ai_type) + 6; + if (dbnamelen > MAXPATHLEN) + { + dbNamep = (char *)slapi_ch_realloc(dbNamep, dbnamelen); + } + p = dbNamep + dbbasenamelen; + sprintf(p, "%c%s%s", get_sep(dbNamep), a->ai_type, LDBM_FILENAME_SUFFIX); + rc = dblayer_db_remove_ex(pEnv, dbNamep, 0, use_lock); + a->ai_dblayer = NULL; + slapi_ch_free((void **)&handle); + if (dbNamep != dbName) + slapi_ch_free_string(&dbNamep); + } else { + /* no handle to close */ + } + PR_Unlock(inst->inst_handle_list_mutex); + + } + + return rc; +} + +int dblayer_erase_index_file_nolock(backend *be, struct attrinfo *a, int no_force_chkpt) { + return dblayer_erase_index_file_ex(be,a,PR_FALSE,no_force_chkpt); +} + +int dblayer_erase_index_file(backend *be, struct attrinfo *a, int no_force_chkpt) { + return dblayer_erase_index_file_ex(be,a,PR_TRUE,no_force_chkpt); +} + + +/* + * Transaction stuff. The idea is that the caller doesn't need to + * know the transaction mechanism underneath (because the caller is + * typically a few calls up the stack from any DB stuff). + * Sadly, in slapd there was no handy structure associated with + * an LDAP operation, and passed around evberywhere, so we had + * to invent the back_txn structure. + * The lower levels of the back-end look into this structure, and + * take out the DB_TXN they need. + */ +int dblayer_txn_init(struct ldbminfo *li, back_txn *txn) +{ + PR_ASSERT(NULL != txn); + + txn->back_txn_txn = NULL; + return 0; +} + + +int dblayer_txn_begin_ext(struct ldbminfo *li, back_txnid parent_txn, back_txn *txn, PRBool use_lock) +{ + int return_value = -1; + dblayer_private *priv = NULL; + PR_ASSERT(NULL != txn); + PR_ASSERT(NULL != li); + /* + * When server is shutting down, some components need to + * flush some data (e.g. replication to write ruv). + * So don't check shutdown signal unless we can't write. + */ + if ( g_get_shutdown() == SLAPI_SHUTDOWN_DISKFULL ) { + return return_value; + } + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + if (priv->dblayer_enable_transactions) + { + dblayer_private_env *pEnv = priv->dblayer_env; + if(use_lock) PR_RWLock_Rlock(pEnv->dblayer_env_lock); + return_value = TXN_BEGIN(pEnv->dblayer_DB_ENV, + (DB_TXN*)parent_txn, + &txn->back_txn_txn, + 0); + if (0 != return_value) + { + if(use_lock) PR_RWLock_Unlock(priv->dblayer_env->dblayer_env_lock); + txn->back_txn_txn = NULL; + } + } else + { + return_value = 0; + } + if (0 != return_value) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Serious Error---Failed in dblayer_txn_begin, err=%d (%s)\n", + return_value, dblayer_strerror(return_value), 0); + } + return return_value; +} + +int dblayer_read_txn_begin(struct ldbminfo *li, back_txnid parent_txn, back_txn *txn) { + return (dblayer_txn_begin_ext(li,parent_txn,txn,PR_FALSE)); +} + +int dblayer_txn_begin(struct ldbminfo *li, back_txnid parent_txn, back_txn *txn) { + return (dblayer_txn_begin_ext(li,parent_txn,txn,PR_TRUE)); +} + + +int dblayer_txn_commit_ext(struct ldbminfo *li, back_txn *txn, PRBool use_lock) +{ + int return_value = -1; + dblayer_private *priv = NULL; + DB_TXN *db_txn; + + PR_ASSERT(NULL != txn); + PR_ASSERT(NULL != li); + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + db_txn = txn->back_txn_txn; + if (NULL != db_txn && + 1 != priv->dblayer_stop_threads && + priv->dblayer_env && + priv->dblayer_enable_transactions) + { + return_value = TXN_COMMIT(db_txn, 0); + if ((priv->dblayer_durable_transactions) && use_lock ) { + if(trans_batch_limit > 0) { + if(trans_batch_count % trans_batch_limit) { + trans_batch_count++; + } else { + LOG_FLUSH(priv->dblayer_env->dblayer_DB_ENV,0); + trans_batch_count=1; + } + } else if(trans_batch_limit == FLUSH_REMOTEOFF) { /* user remotely turned batching off */ + LOG_FLUSH(priv->dblayer_env->dblayer_DB_ENV,0); + } + } + if(use_lock) PR_RWLock_Unlock(priv->dblayer_env->dblayer_env_lock); + } else + { + return_value = 0; + } + + if (0 != return_value) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Serious Error---Failed in dblayer_txn_commit, err=%d (%s)\n", + return_value, dblayer_strerror(return_value), 0); + if (LDBM_OS_ERR_IS_DISKFULL(return_value)) { + operation_out_of_disk_space(); + } + } + return return_value; +} + +int dblayer_read_txn_commit(struct ldbminfo *li, back_txn *txn) { + return(dblayer_txn_commit_ext(li,txn,PR_FALSE)); +} + +int dblayer_txn_commit(struct ldbminfo *li, back_txn *txn) { + return(dblayer_txn_commit_ext(li,txn,PR_TRUE)); +} + +int dblayer_txn_abort_ext(struct ldbminfo *li, back_txn *txn, PRBool use_lock) +{ + int return_value = -1; + dblayer_private *priv = NULL; + DB_TXN *db_txn; + + PR_ASSERT(NULL != txn); + PR_ASSERT(NULL != li); + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + db_txn = txn->back_txn_txn; + if (NULL != db_txn && + priv->dblayer_env && + priv->dblayer_enable_transactions) + { + return_value = TXN_ABORT(db_txn); + if(use_lock) PR_RWLock_Unlock(priv->dblayer_env->dblayer_env_lock); + } else + { + return_value = 0; + } + + if (0 != return_value) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Serious Error---Failed in dblayer_txn_abort, err=%d (%s)\n", + return_value, dblayer_strerror(return_value), 0); + if (LDBM_OS_ERR_IS_DISKFULL(return_value)) { + operation_out_of_disk_space(); + } + } + return return_value; +} + +int dblayer_read_txn_abort(struct ldbminfo *li, back_txn *txn){ + return(dblayer_txn_abort_ext(li,txn,PR_FALSE)); +} + +int dblayer_txn_abort(struct ldbminfo *li, back_txn *txn){ + return(dblayer_txn_abort_ext(li,txn,PR_TRUE)); +} + + +size_t dblayer_get_optimal_block_size(struct ldbminfo *li) +{ + dblayer_private *priv = NULL; + size_t page_size = 0; + + PR_ASSERT(NULL != li); + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + page_size = (priv->dblayer_page_size == 0) ? DBLAYER_PAGESIZE : priv->dblayer_page_size; + if (priv->dblayer_idl_divisor == 0) + { + return page_size - DB_EXTN_PAGE_HEADER_SIZE; + } else + { + return page_size / priv->dblayer_idl_divisor; + } +} + + +/* + * The dblock serializes writes to the database, + * which reduces deadlocking in the db code, + * which means that we run faster. + */ +void dblayer_lock_backend(backend *be) +{ + ldbm_instance *inst; + + PR_ASSERT(NULL != be); + inst = (ldbm_instance *) be->be_instance_info; + PR_ASSERT(NULL != inst); + + if (NULL != inst->inst_db_mutex) { + PR_Lock(inst->inst_db_mutex); + } +} + +void dblayer_unlock_backend(backend *be) +{ + ldbm_instance *inst; + + PR_ASSERT(NULL != be); + inst = (ldbm_instance *) be->be_instance_info; + PR_ASSERT(NULL != inst); + + if (NULL != inst->inst_db_mutex) { + PR_Unlock(inst->inst_db_mutex); + } +} + + +/* code which implements checkpointing and log file truncation */ + +/* + * create a thread for perf_threadmain + */ +static int +dblayer_start_perf_thread(struct ldbminfo *li) +{ + int return_value = 0; + if (NULL == PR_CreateThread (PR_USER_THREAD, + (VFP) (void *) perf_threadmain, li, + PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE) ) + { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, "failed to create database perf thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + return_value = -1; + } + return return_value; +} + +/* Performance thread */ +static int perf_threadmain(void *param) +{ + dblayer_private *priv = NULL; + struct ldbminfo *li = NULL; + + PR_ASSERT(NULL != param); + li = (struct ldbminfo*)param; + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + PR_AtomicIncrement(&(priv->dblayer_thread_count)); + while (!priv->dblayer_stop_threads) { + /* sleep for a while, updating perf counters if we need to */ + perfctrs_wait(1000,priv->perf_private,priv->dblayer_env->dblayer_DB_ENV); + } + PR_AtomicDecrement(&(priv->dblayer_thread_count)); + return 0; +} + +/* + * create a thread for deadlock_threadmain + */ +static int +dblayer_start_deadlock_thread(struct ldbminfo *li) +{ + int return_value = 0; + if (NULL == PR_CreateThread (PR_USER_THREAD, + (VFP) (void *) deadlock_threadmain, li, + PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE) ) + { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, "failed to create database deadlock thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + return_value = -1; + } + return return_value; +} + +/* checkpoint thread main function */ + +static int deadlock_threadmain(void *param) +{ + int rval = -1; + dblayer_private *priv = NULL; + struct ldbminfo *li = NULL; + PRIntervalTime interval; /*NSPR timeout stuffy*/ + + PR_ASSERT(NULL != param); + li = (struct ldbminfo*)param; + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + interval = PR_MillisecondsToInterval(100); + PR_AtomicIncrement(&(priv->dblayer_thread_count)); + while (!priv->dblayer_stop_threads) + { + if (priv->dblayer_enable_transactions) + { + if (NULL != priv->dblayer_env->dblayer_DB_ENV->lk_handle) { + int aborted; + if ((rval = LOCK_DETECT(priv->dblayer_env->dblayer_DB_ENV, + 0, + DB_LOCK_YOUNGEST, + &aborted)) + != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "Serious Error---Failed in deadlock detect (aborted at 0x%x), err=%d (%s)\n", + aborted, rval, dblayer_strerror(rval)); + } + } + } + DS_Sleep(interval); + } + PR_AtomicDecrement(&(priv->dblayer_thread_count)); + return 0; +} + +#define checkpoint_debug_message(debug, fmt, a1, a2, a3) \ + if (debug) { LDAPDebug(LDAP_DEBUG_ANY,fmt,a1,a2,a3); } + +/* this thread tries to do two things: + 1. catch a group of transactions that are pending allowing a worker thread + to work + 2. flush any left over transactions ( a single transaction for example) +*/ + +static int +dblayer_start_log_flush_thread(dblayer_private *priv) +{ + int return_value = 0; + + if ((priv->dblayer_durable_transactions) && + (priv->dblayer_enable_transactions) && (trans_batch_limit > 0)) { + log_flush_thread=PR_TRUE; + if (NULL == PR_CreateThread (PR_USER_THREAD, + (VFP) (void *) log_flush_threadmain, priv, + PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE) ) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, + "failed to create database log flush thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + return_value = -1; + } + } + return return_value; +} + +/* this thread tries to do two things: + 1. catch a group of transactions that are pending allowing a worker thread + to work + 2. flush any left over transactions ( a single transaction for example) +*/ + +static int log_flush_threadmain(void *param) +{ + dblayer_private *priv = NULL; + PRIntervalTime interval; + + + PR_ASSERT(NULL != param); + priv = (dblayer_private *) param; + PR_ASSERT(NULL != priv); + interval = PR_MillisecondsToInterval(300); + PR_AtomicIncrement(&(priv->dblayer_thread_count)); + while ((!priv->dblayer_stop_threads) && (log_flush_thread)) + { + if (priv->dblayer_enable_transactions) + { + DB_CHECKPOINT_LOCK(1, priv->dblayer_env->dblayer_env_lock); + if(trans_batch_limit > 0) { + if(trans_batch_count > 1) { + LOG_FLUSH(priv->dblayer_env->dblayer_DB_ENV,0); + trans_batch_count=1; + } + } + DB_CHECKPOINT_UNLOCK(1, priv->dblayer_env->dblayer_env_lock); + } + DS_Sleep(interval); + } + PR_AtomicDecrement(&(priv->dblayer_thread_count)); + return 0; +} + +/* + * create a thread for checkpoint_threadmain + */ +static int +dblayer_start_checkpoint_thread(struct ldbminfo *li) +{ + int return_value = 0; + if (NULL == PR_CreateThread (PR_USER_THREAD, + (VFP) (void *) checkpoint_threadmain, li, + PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE) ) + { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, + "failed to create database checkpoint thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + return_value = -1; + } + return return_value; +} + +static int checkpoint_threadmain(void *param) +{ + time_t time_of_last_checkpoint_completion = 0; /* seconds since epoch */ + PRIntervalTime interval; /*NSPR timeout stuffy*/ + int rval = -1; + dblayer_private *priv = NULL; + struct ldbminfo *li = NULL; + int debug_checkpointing = 0; + int checkpoint_interval; + char *home_dir = NULL; + + PR_ASSERT(NULL != param); + li = (struct ldbminfo*)param; + + home_dir = dblayer_get_home_dir(li, NULL); + if (NULL == home_dir) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Checkpoint thread failed due to missing db home directory info\n", + 0, 0, 0); + return rval; + } + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + /* work around a problem with newly created environments */ + dblayer_force_checkpoint(li); + + interval = PR_MillisecondsToInterval(DBLAYER_SLEEP_INTERVAL); + debug_checkpointing = priv->db_debug_checkpointing; + PR_AtomicIncrement(&(priv->dblayer_thread_count)); + /* assumes dblayer_force_checkpoint worked */ + time_of_last_checkpoint_completion = current_time(); + while (!priv->dblayer_stop_threads) + { + /* sleep for a while */ + /* why aren't we sleeping exactly the right amount of time ? */ + /* answer---because the interval might be changed after the server + * starts up */ + DS_Sleep(interval); + + if (0 == priv->dblayer_enable_transactions) + continue; + + PR_Lock(li->li_config_mutex); + checkpoint_interval = priv->dblayer_checkpoint_interval; + PR_Unlock(li->li_config_mutex); + + /* Check to see if the checkpoint interval has elapsed */ + if (current_time() - time_of_last_checkpoint_completion < + checkpoint_interval) + continue; + + if (NULL == priv->dblayer_env->dblayer_DB_ENV->tx_handle) + continue; + + /* now checkpoint */ + checkpoint_debug_message(debug_checkpointing, + "Starting checkpoint\n", 0, 0, 0); + rval = dblayer_txn_checkpoint(li, priv->dblayer_env, PR_TRUE, PR_TRUE); +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR < 4100 + if (DB_INCOMPLETE == rval) + { + checkpoint_debug_message(debug_checkpointing, + "Retrying checkpoint\n", 0, 0, 0); + } else +#endif + { + checkpoint_debug_message(debug_checkpointing, + "Checkpoint Done\n", 0, 0, 0); + if (rval != 0) { + /* bad error */ + LDAPDebug(LDAP_DEBUG_ANY, + "Serious Error---Failed to checkpoint database, " + "err=%d (%s)\n", rval, dblayer_strerror(rval), 0); + if (LDBM_OS_ERR_IS_DISKFULL(rval)) { + operation_out_of_disk_space(); + goto diskfull_return; + } + } else { + time_of_last_checkpoint_completion = current_time(); + } + } + + checkpoint_debug_message(debug_checkpointing, + "Starting checkpoint\n", 0, 0, 0); + rval = dblayer_txn_checkpoint(li, priv->dblayer_env, PR_TRUE, PR_TRUE); +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR < 4100 + if (DB_INCOMPLETE == rval) + { + checkpoint_debug_message(debug_checkpointing, + "Retrying checkpoint\n", 0, 0, 0); + } else +#endif + { + checkpoint_debug_message(debug_checkpointing, + "Checkpoint Done\n", 0, 0, 0); + if (rval != 0) { + /* bad error */ + LDAPDebug(LDAP_DEBUG_ANY, + "Serious Error---Failed to checkpoint database, " + "err=%d (%s)\n", rval, dblayer_strerror(rval), 0); + if (LDBM_OS_ERR_IS_DISKFULL(rval)) { + operation_out_of_disk_space(); + goto diskfull_return; + } + } else { + time_of_last_checkpoint_completion = current_time(); + } + } + { + char **list = NULL; + char **listp = NULL; + int return_value = -1; + char filename[MAXPATHLEN]; + char *prefix = NULL; + struct dblayer_private_env *penv = priv->dblayer_env; + if ((NULL != priv->dblayer_log_directory) && + (0 != strlen(priv->dblayer_log_directory))) + { + prefix = priv->dblayer_log_directory; + } + else + { + prefix = home_dir; + } + /* find out which log files don't contain active txns */ + DB_CHECKPOINT_LOCK(PR_TRUE, penv->dblayer_env_lock); + return_value = LOG_ARCHIVE(penv->dblayer_DB_ENV, &list, + 0, malloc); + DB_CHECKPOINT_UNLOCK(PR_TRUE, penv->dblayer_env_lock); + checkpoint_debug_message(debug_checkpointing, + "Got list of logfiles not needed %d %p\n", + return_value,list, 0); + if (0 == return_value && NULL != list) + { + /* zap 'em ! */ + for (listp = list; *listp != NULL; ++listp) + { + sprintf(filename,"%s/%s",prefix,*listp); + if (priv->dblayer_circular_logging) { + checkpoint_debug_message(debug_checkpointing, + "Deleting %s\n",filename, 0, 0); + unlink(filename); + } else { + char new_filename[MAXPATHLEN]; + sprintf(new_filename,"%s/old.%s", + prefix,*listp); + checkpoint_debug_message(debug_checkpointing, + "Renaming %s\n",filename,0, 0); + rename(filename,new_filename); + } + } + slapi_ch_free((void**)&list); + } + } + } + dblayer_force_checkpoint(li); +diskfull_return: + PR_AtomicDecrement(&(priv->dblayer_thread_count)); + return 0; +} + +/* + * create a thread for trickle_threadmain + */ +static int +dblayer_start_trickle_thread(struct ldbminfo *li) +{ + int return_value = 0; + if (NULL == PR_CreateThread (PR_USER_THREAD, + (VFP) (void *) trickle_threadmain, li, + PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE) ) + { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, "failed to create database trickle thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + return_value = -1; + } + return return_value; +} + +static int trickle_threadmain(void *param) +{ + PRIntervalTime interval; /*NSPR timeout stuffy*/ + int rval = -1; + dblayer_private *priv = NULL; + struct ldbminfo *li = NULL; + int debug_checkpointing = 0; + + PR_ASSERT(NULL != param); + li = (struct ldbminfo*)param; + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + interval = PR_MillisecondsToInterval(DBLAYER_SLEEP_INTERVAL); + debug_checkpointing = priv->db_debug_checkpointing; + PR_AtomicIncrement(&(priv->dblayer_thread_count)); + while (!priv->dblayer_stop_threads) + { + DS_Sleep(interval); /* 622855: wait for other threads fully started */ + if (priv->dblayer_enable_transactions) + { + if ( (NULL != priv->dblayer_env->dblayer_DB_ENV->mp_handle) && + (0 != priv->dblayer_trickle_percentage) ) + { + int pages_written = 0; + if ((rval = MEMP_TRICKLE( + priv->dblayer_env->dblayer_DB_ENV, + priv->dblayer_trickle_percentage, + &pages_written)) != 0) + { + LDAPDebug(LDAP_DEBUG_ANY,"Serious Error---Failed to trickle, err=%d (%s)\n",rval,dblayer_strerror(rval), 0); + } + if (pages_written > 0) + { + checkpoint_debug_message(debug_checkpointing,"Trickle thread wrote %d pages\n",pages_written,0, 0); + } + } + } + } + PR_AtomicDecrement(&(priv->dblayer_thread_count)); + return 0; +} + + +/* better atol -- it understands a trailing multiplier k/m/g + * for example, "32k" will be returned as 32768 + * richm: added better error checking and support for 64 bit values. + * The err parameter is used by the caller to tell if there was an error + * during the a to i conversion - if 0, the value was successfully + * converted - if non-zero, there was some error (e.g. not a number) + */ +PRInt64 db_atol(char *str, int *err) +{ + PRInt64 mres1 = LL_INIT(0, 1); + PRInt64 mres2 = LL_INIT(0, 1); + PRInt64 mres3 = LL_INIT(0, 1); + PRInt64 onek = LL_INIT(0, 1024); + PRInt64 multiplier = LL_INIT(0, 1); + PRInt64 val = LL_INIT(0, 0); + PRInt64 result = LL_INIT(0, 0); + char x = 0; + int num = PR_sscanf(str, "%lld%c", &val, &x); + if (num < 1) { /* e.g. not a number */ + if (err) + *err = 1; + return result; /* return 0 */ + } + + switch (x) { + case 'g': + case 'G': + LL_MUL(mres1, onek, multiplier); +/* multiplier *= 1024;*/ + case 'm': + case 'M': + LL_MUL(mres2, onek, mres1); +/* multiplier *= 1024;*/ + case 'k': + case 'K': + LL_MUL(mres3, onek, mres2); +/* multiplier *= 1024;*/ + } + LL_MUL(result, val, mres3); +/* result = val * multiplier;*/ + if (err) + *err = 0; + return result; +} + +PRInt64 db_atoi(char *str, int *err) +{ + return db_atol(str, err); +} + +unsigned long db_strtoul(const char *str, int *err) +{ + unsigned long val, result, multiplier = 1; + char *p; + errno = 0; + + val = strtoul(str, &p, 10); + if (errno != 0) { + if (err) *err = errno; + return val; + } + + switch (*p) { + case 'g': + case 'G': + multiplier *= 1024; + case 'm': + case 'M': + multiplier *= 1024; + case 'k': + case 'K': + multiplier *= 1024; + p++; + if (*p == 'b' || *p == 'B') p++; + if (err) { + /* extra chars? */ + *err = (*p != '\0') ? EINVAL : 0; + } + break; + case '\0': + if (err) *err = 0; + break; + default: + if (err) *err = EINVAL; + return val; + } + + result = val * multiplier; + + return result; +} + +/* functions called directly by the plugin interface from the front-end */ + +/* Begin transaction */ +int dblayer_plugin_begin(Slapi_PBlock *pb) +{ + int return_value = -1; + struct ldbminfo *li = NULL; + back_txnid parent; + back_txn current; + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_PARENT_TXN, (void**)&parent ); + + /* call begin, and put the result in the txnid parameter */ + return_value = dblayer_txn_begin(li,parent,¤t); + + if (0 == return_value) + { + slapi_pblock_set( pb, SLAPI_TXN, (void*)current.back_txn_txn ); + } + + return return_value; +} + +/* Commit transaction */ +int dblayer_plugin_commit(Slapi_PBlock *pb) +{ + /* get the txnid and call commit */ + int return_value = -1; + struct ldbminfo *li = NULL; + back_txn current; + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_TXN, (void**)&(current.back_txn_txn) ); + + /* call begin, and put the result in the txnid parameter */ + return_value = dblayer_txn_commit(li,¤t); + + return return_value; +} + +/* Abort Transaction */ +int dblayer_plugin_abort(Slapi_PBlock *pb) +{ + /* get the txnid and call abort */ + return 0; +} + + +/* Helper function for monitor stuff */ +int dblayer_memp_stat(struct ldbminfo *li, DB_MPOOL_STAT **gsp, + DB_MPOOL_FSTAT ***fsp) +{ + dblayer_private *priv = NULL; + DB_ENV *env = NULL; + + PR_ASSERT(NULL != li); + + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + env = priv->dblayer_env->dblayer_DB_ENV; + PR_ASSERT(NULL != env); + + return MEMP_STAT(env, gsp, fsp, 0, malloc); +} + +/* import wants this one */ +int dblayer_memp_stat_instance(ldbm_instance *inst, DB_MPOOL_STAT **gsp, + DB_MPOOL_FSTAT ***fsp) +{ + DB_ENV *env = NULL; + dblayer_private *priv = NULL; + + PR_ASSERT(NULL != inst); + + if (inst->import_env->dblayer_DB_ENV) { + env = inst->import_env->dblayer_DB_ENV; + } else { + priv = (dblayer_private *)inst->inst_li->li_dblayer_private; + PR_ASSERT(NULL != priv); + env = priv->dblayer_env->dblayer_DB_ENV; + } + PR_ASSERT(NULL != env); + + return MEMP_STAT(env, gsp, fsp, 0, malloc); +} + +/* Helper functions for recovery */ + +#define DB_LINE_LENGTH 80 + +static int commit_good_database(dblayer_private *priv) +{ + /* Write out the guard file */ + char filename[MAXPATHLEN]; + char line[DB_LINE_LENGTH * 2]; + PRFileDesc *prfd; + int return_value = 0; + int num_bytes; + + sprintf(filename,"%s/guardian",priv->dblayer_home_directory); + + prfd = PR_Open(filename, PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, + priv->dblayer_file_mode ); + if (NULL == prfd) + { + LDAPDebug( LDAP_DEBUG_ANY,"Fatal Error---Failed to write guardian file, database corruption possible" SLAPI_COMPONENT_NAME_NSPR " %d (%s)\n", + filename, PR_GetError(), slapd_pr_strerror(PR_GetError()) ); + return -1; + } + sprintf(line,"cachesize:%lu\nncache:%d\nversion:%d\n", + priv->dblayer_cachesize, priv->dblayer_ncache, 3); + num_bytes = strlen(line); + return_value = slapi_write_buffer(prfd, line, num_bytes); + if (return_value != num_bytes) + { + goto error; + } + return_value = PR_Close(prfd); + if (PR_SUCCESS == return_value) + { + return 0; + } else + { + LDAPDebug( LDAP_DEBUG_ANY,"Fatal Error---Failed to write guardian file, database corruption possible\n", 0,0, 0 ); + (void)PR_Delete(filename); + return -1; + } +error: + (void)PR_Close(prfd); + (void)PR_Delete(filename); + return -1; +} + +/* read the guardian file from db/ and possibly recover the database */ +static int read_metadata(struct ldbminfo *li) +{ + char filename[MAXPATHLEN]; + char *buf; + char *thisline; + char *nextline; + char **dirp; + PRFileDesc *prfd; + PRFileInfo prfinfo; + int return_value = 0; + PRInt32 byte_count = 0; + char attribute[512]; + char value[128], delimiter; + int number = 0; + dblayer_private *priv = (dblayer_private *)li->li_dblayer_private; + + priv->dblayer_previous_cachesize = 0; + priv->dblayer_previous_ncache = 0; + /* Open the guard file and read stuff, then delete it */ + sprintf(filename,"%s/guardian",priv->dblayer_home_directory); + + memset(&prfinfo, '\0', sizeof(PRFileInfo)); + (void)PR_GetFileInfo(filename, &prfinfo); + + priv->dblayer_recovery_required = 0; + prfd = PR_Open(filename,PR_RDONLY,priv->dblayer_file_mode); + if (NULL == prfd || 0 == prfinfo.size) { + /* file empty or not present--means the database needs recovered */ + int count = 0; + priv->dblayer_recovery_required = 0; + for (dirp = priv->dblayer_data_directories; dirp && *dirp; dirp++) + { + count_dbfiles_in_dir(*dirp, &count, 1 /* recurse */); + if (count > 0) { +#if 0 + char *home_dir; + /* This code used to check for a broken import by looking + * for a dbversion file. If it wasn't there, then an import + * failed. Now each instance has its own dbversion file. + * If this check is done at all, it ought to be done when + * bringing up individual backend instances. + */ + /* While we're here, let's check for a broken import. + This would be indicated by the following conditions: + 1. db files in the directory. + 2. No guardian file. + 3. No DBVERSION file. + If we're here we have confitions 1 and 2, + so we should check for condition 3. + */ + if (!dbversion_exists(li, home_dir)) { + LDAPDebug( LDAP_DEBUG_ANY,"Fatal Error---database is corrupt. Server can't start. Most likely cause is a previously aborted import. Either re-import or delete the database and re-start the server.\n", 0,0, 0 ); + return -1; + } else { + priv->dblayer_recovery_required = 1; + } +#endif + priv->dblayer_recovery_required = 1; + return 0; + } + } + return 0; /* no files found; no need to run recover start */ + } + /* dblayer_recovery_required is initialized in dblayer_init; + * and might be set 1 in check_db_version; + * we don't want to override it + * priv->dblayer_recovery_required = 0; */ + /* So, we opened the file, now let's read the cache size and version stuff + */ + buf = slapi_ch_calloc(1, prfinfo.size + 1); + byte_count = slapi_read_buffer(prfd, buf, prfinfo.size); + if (byte_count < 0) { + /* something bad happened while reading */ + priv->dblayer_recovery_required = 1; + } else { + buf[ byte_count ] = '\0'; + thisline = buf; + while (1) { + /* Find the end of the line */ + nextline = strchr( thisline, '\n' ); + if (NULL != nextline) { + *nextline++ = '\0'; + while ('\n' == *nextline) { + nextline++; + } + } + sscanf(thisline,"%[a-z]%c%s",attribute,&delimiter,value); + if (0 == strcmp("cachesize",attribute)) { + priv->dblayer_previous_cachesize = strtoul(value, NULL, 10); + } else if (0 == strcmp("ncache",attribute)) { + number = atoi(value); + priv->dblayer_previous_ncache = number; + } else if (0 == strcmp("version",attribute)) { + } + if (NULL == nextline || '\0' == *nextline) { + /* Nothing more to read */ + break; + } + thisline = nextline; + } + } + slapi_ch_free((void **)&buf); + (void)PR_Close(prfd); + return_value = PR_Delete(filename); /* very important that this happen ! */ + if (PR_SUCCESS != return_value) { + LDAPDebug(LDAP_DEBUG_ANY, + "Fatal Error---Failed to delete guardian file, " + "database corruption possible\n", 0, 0, 0 ); + } + return return_value; +} + +/* handy routine for checkpointing the db */ +static int dblayer_force_checkpoint(struct ldbminfo *li) +{ + int ret = 0, i; + dblayer_private *priv = (dblayer_private *)li->li_dblayer_private; + struct dblayer_private_env *pEnv; + + if (NULL == priv){ + /* already terminated. nothing to do */ + return -1; + } + + pEnv= priv->dblayer_env; + + + PR_ASSERT(pEnv != NULL); + + if (priv->dblayer_enable_transactions) { + + LDAPDebug(LDAP_DEBUG_TRACE, "Checkpointing database ...\n", 0, 0, 0); + + /* + * DB workaround. Newly created environments do not know what the + * previous checkpoint LSN is. The default LSN of [0][0] would + * cause us to read all log files from very beginning during a + * later recovery. Taking two checkpoints solves the problem. + */ + + for (i = 0; i < 2; i++) { + ret = dblayer_txn_checkpoint(li, pEnv, PR_TRUE, PR_FALSE); + if (ret == 0) continue; +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR < 4100 + if (ret != DB_INCOMPLETE) +#endif + { + LDAPDebug(LDAP_DEBUG_ANY, "Checkpoint FAILED, error %s (%d)\n", + dblayer_strerror(ret), ret, 0); + break; + } +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR < 4100 + + LDAPDebug(LDAP_DEBUG_ANY, "Busy: retrying checkpoint\n", 0, 0, 0); + + /* teletubbies: "again! again!" */ + ret = dblayer_txn_checkpoint(li, pEnv, PR_TRUE, PR_FALSE); + if (ret == DB_INCOMPLETE) { + LDAPDebug(LDAP_DEBUG_ANY, "Busy: giving up on checkpoint\n", 0, 0, 0); + break; + } else if (ret != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "Checkpoint FAILED, error %s (%d)\n", + dblayer_strerror(ret), ret, 0); + break; + } +#endif + } + } + + return ret; +} + + +static int _dblayer_delete_instance_dir(ldbm_instance *inst) +{ + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + char filename[MAXPATHLEN]; + struct ldbminfo *li = inst->inst_li; + dblayer_private *priv = NULL; + struct dblayer_private_env *pEnv = NULL; + char inst_dir[MAXPATHLEN]; + char *inst_dirp = NULL; + int rval = 0; + + if (NULL != li) + { + priv = (dblayer_private*)li->li_dblayer_private; + if (NULL != priv) + { + pEnv = priv->dblayer_env; + } + } + + if (inst->inst_dir_name == NULL) + dblayer_get_instance_data_dir(inst->inst_be); + + inst_dirp = dblayer_get_full_inst_dir(li, inst, inst_dir, MAXPATHLEN); + dirhandle = PR_OpenDir(inst_dirp); + if (! dirhandle) { + if ( PR_GetError() == PR_FILE_NOT_FOUND_ERROR ) { + /* the directory does not exist... that's not an error */ + rval = 0; + goto done; + } + LDAPDebug(LDAP_DEBUG_ANY, + "_dblayer_delete_instance_dir: PR_OpenDir(%s) failed (%d): %s\n", + inst_dirp, PR_GetError(),slapd_pr_strerror(PR_GetError())); + rval = -1; + goto done; + } + + /* + Note the use of PR_Delete here as opposed to using + sleepycat to "remove" the file. Reason: One should + not expect logging to be able to recover the wholesale + removal of a complete directory... a directory that includes + files outside the scope of sleepycat's logging. rwagner + + ADDITIONAL COMMENT: + libdb41 is more strict on the transaction log control. + Even if checkpoint is forced before this delete function, + no log regarding the file deleted found in the log file, + following checkpoint repeatedly complains with these error messages: + libdb: <path>/mail.db4: cannot sync: No such file or directory + libdb: txn_checkpoint: failed to flush the buffer cache + No such file or directory + */ + + while (NULL != (direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | + PR_SKIP_DOT_DOT))) { + if (! direntry->name) + break; + sprintf(filename, "%s/%s", inst_dirp, direntry->name); + if (pEnv && + strcmp(LDBM_FILENAME_SUFFIX , last_four_chars(direntry->name)) == 0) + { + rval = dblayer_db_remove_ex(pEnv, filename, 0, PR_TRUE); + } + else + { + rval = PR_Delete(filename); + } + } + PR_CloseDir(dirhandle); +done: + /* remove the directory itself too */ + if (0 == rval) + PR_RmDir(inst_dirp); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return rval; +} + +/* delete the db3 files in a specific backend instance -- + * this is probably only used for import. + * assumption: dblayer is open, but the instance has been closed. + */ +int dblayer_delete_instance_dir(backend *be) +{ + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + int ret = dblayer_force_checkpoint(li); + + if (ret != 0) { + return ret; + } else { + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; + return _dblayer_delete_instance_dir(inst); + } +} + +/* delete an entire db/ directory, including all instances under it! + * this is used mostly for restores. + * dblayer is assumed to be closed. + */ +int dblayer_delete_database(struct ldbminfo *li) +{ + dblayer_private *priv = NULL; + Object *inst_obj; + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + char filename[MAXPATHLEN]; + char *log_dir; + int ret; + + PR_ASSERT(NULL != li); + priv = (dblayer_private *)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + /* delete each instance */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + ldbm_instance *inst = (ldbm_instance *)object_get_data(inst_obj); + + if (inst->inst_be->be_instance_info != NULL) { + ret = _dblayer_delete_instance_dir(inst); + if (ret != 0) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dblayer_delete_database: WARNING _dblayer_delete_instance_dir failed (%d)\n", ret, 0, 0); + return ret; + } + } + } + + /* now smash everything else in the db/ dir */ + dirhandle = PR_OpenDir(priv->dblayer_home_directory); + if (! dirhandle) + { + LDAPDebug(LDAP_DEBUG_ANY, "PR_OpenDir (%s) failed (%d): %s\n", + priv->dblayer_home_directory, + PR_GetError(),slapd_pr_strerror(PR_GetError())); + return -1; + } + while (NULL != (direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | + PR_SKIP_DOT_DOT))) { + if (! direntry->name) + break; + sprintf(filename, "%s/%s", priv->dblayer_home_directory, + direntry->name); + PR_Delete(filename); + } + + PR_CloseDir(dirhandle); + /* remove transaction logs */ + if ((NULL != priv->dblayer_log_directory) && + (0 != strlen(priv->dblayer_log_directory) )) + { + log_dir = priv->dblayer_log_directory; + } + else + { + log_dir = dblayer_get_home_dir(li, NULL); + } + ret = dblayer_delete_transaction_logs(log_dir); + if(ret) { + LDAPDebug(LDAP_DEBUG_ANY, + "dblayer_delete_database: dblayer_delete_transaction_logs failed (%d)\n", + ret, 0, 0); + return -1; + } + return 0; +} + + +/* + * Return the size of the database (in kilobytes). XXXggood returning + * the size in units of kb is really a hack, and is done because we + * don't have NSPR support for 64-bit file offsets. + * Caveats: + * - We can still return incorrect results if an individual file is + * larger than fit in a PRUint32. + * - PR_GetFileInfo doesn't do any special processing for symlinks, + * nor does it inform us if the file is a symlink. Nice. So if + * a file in the db directory is a symlink, the size we return + * will probably be way too small. + */ +int dblayer_database_size(struct ldbminfo *li, unsigned int *size) +{ + dblayer_private *priv = NULL; + int return_value = 0; + char filename[MAXPATHLEN]; + PRDir *dirhandle = NULL; + /* + * XXXggood - NSPR will only give us an unsigned 32-bit quantity for + * file sizes. This is bad. Files can be bigger than that these days. + */ + unsigned int cumulative_size = 0; + unsigned int remainder = 0; + PRFileInfo info; + + PR_ASSERT(NULL != li); + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + dirhandle = PR_OpenDir(priv->dblayer_home_directory); + if (NULL != dirhandle) + { + PRDirEntry *direntry = NULL; + while (NULL != (direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { + if (NULL == direntry->name) + { + break; + } + sprintf(filename,"%s/%s",priv->dblayer_home_directory,direntry->name); + return_value = PR_GetFileInfo(filename, &info); + if (PR_SUCCESS == return_value) + { + cumulative_size += (info.size / 1024); + remainder += (info.size % 1024); + } else + { + cumulative_size = (PRUint32) 0; + return_value = -1; + break; + } + } + PR_CloseDir(dirhandle); + } else + { + return_value = -1; + } + + *size = cumulative_size + (remainder / 1024); + return return_value; +} + +static char* last_four_chars(const char* s) +{ + size_t l = strlen(s); + return ((char*)s + (l - 4)); +} + +static int count_dbfiles_in_dir(char *directory, int *count, int recurse) +{ + /* The new recurse argument was added to help with multiple backend + * instances. When recurse is true, this function will also look through + * the directories in the given directory for .db3 files. */ + int return_value = 0; + PRDir *dirhandle = NULL; + + if (!recurse) { + /* It is really the callers responsibility to set count to 0 before + * calling. However, if recurse isn't true, we can make sure it is + * set to 0. */ + *count = 0; + } + dirhandle = PR_OpenDir(directory); + if (NULL != dirhandle) { + PRDirEntry *direntry = NULL; + char *direntry_name; + PRFileInfo info; + + while (NULL != (direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { + if (NULL == direntry->name) { + break; + } + direntry_name = slapi_ch_malloc(strlen(directory) + + strlen(direntry->name) + 2); + sprintf(direntry_name, "%s/%s", directory, direntry->name); + if ((PR_GetFileInfo(direntry_name, &info) == PR_SUCCESS) && + (PR_FILE_DIRECTORY == info.type) && recurse) { + /* Recurse into this directory but not any further. This is + * because each instance gets its own directory, but in those + * directories there should be only .db3 files. There should + * not be any more directories in an instance directory. */ + count_dbfiles_in_dir(direntry_name, count, 0 /* don't recurse */); + } + slapi_ch_free((void**)&direntry_name); + if (strcmp( LDBM_FILENAME_SUFFIX , last_four_chars(direntry->name)) == 0) { + (*count)++; + } + } + PR_CloseDir(dirhandle); + } else { + return_value = -1; + } + + return return_value; +} + +/* And finally... Tubular Bells. + * Well, no, actually backup and restore... + */ + +/* Backup works like this: + * the slapd executable is run like for ldif2ldbm and so on. + * this means that the front-end gets the back-end loaded, and then calls + * into the back-end backup entry point. This then gets us down to here. + * + * So, we need to copy the data files to the backup point. + * While we are doing that, we need to make sure that the logfile + * truncator in slapd doesn't delete our files. To do this we need + * some way to signal to it that it should cease its work, or we need + * to do something like start a long-lived transaction so that the + * log files look like they're needed. + * + * When we've copied the data files, we can then copy the log files + * too. + * + * Finally, we tell the log file truncator to go back about its business in peace + * + */ + +int +dblayer_copyfile(char *source, char *destination, int overwrite, int mode) +{ +#if defined _WIN32 + return (0 == CopyFile(source,destination,overwrite ? FALSE : TRUE)); +#else +#ifdef DB_USE_64LFS +#define OPEN_FUNCTION dblayer_open_large +#else +#define OPEN_FUNCTION open +#endif + int source_fd = -1; + int dest_fd = -1; + char *buffer = NULL; + int return_value = -1; + int bytes_to_write = 0; + + /* malloc the buffer */ + buffer = slapi_ch_malloc(64*1024); + if (NULL == buffer) + { + goto error; + } + /* Open source file */ + source_fd = OPEN_FUNCTION(source,O_RDONLY,0); + if (-1 == source_fd) + { + goto error; + } + /* Open destination file */ + dest_fd = OPEN_FUNCTION(destination,O_CREAT | O_WRONLY, mode); + if (-1 == dest_fd) + { + goto error; + } + /* Loop round reading data and writing it */ + while (1) + { + return_value = read(source_fd,buffer,64*1024); + if (return_value <= 0) + { + /* means error or EOF */ + break; + } + bytes_to_write = return_value; + return_value = write(dest_fd,buffer,bytes_to_write); + if (return_value != bytes_to_write) + { + /* means error */ + return_value = -1; + break; + } + } +error: + if (source_fd != -1) + { + close(source_fd); + } + if (dest_fd != -1) + { + close(dest_fd); + } + slapi_ch_free((void**)&buffer); + return return_value; +#endif +} + +/* + * Copies all the .db# files in instance_dir to a directory with the same name + * in destination_dir. Both instance_dir and destination_dir are absolute + * paths. + * (#604921: added indexonly flag for the use in convindices + * -- backup/restore indices) + * + * If the argument restore is true, + * logging messages will be about "Restoring" files. + * If the argument restore is false, + * logging messages will be about "Backing up" files. + * The argument cnt is used to count the number of files that were copied. + * + * This function is used during db2bak and bak2db. + */ +int dblayer_copy_directory(struct ldbminfo *li, + Slapi_Task *task, + char *src_dir, + char *dest_dir, + int restore, + int *cnt, + int instance_dir_flag, + int indexonly) +{ + dblayer_private *priv = NULL; + char *new_src_dir = NULL; + char *new_dest_dir = NULL; + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + size_t filename_length = 0; + size_t offset = 0; + char *compare_piece = NULL; + char *filename1; + char *filename2; + int return_value = -1; + char *relative_instance_name = NULL; + char *inst_dirp = NULL; + char inst_dir[MAXPATHLEN]; + char sep; + ldbm_instance *inst; + + if (!src_dir || '\0' == *src_dir || !dest_dir || '\0' == *dest_dir) + return return_value; + + priv = (dblayer_private *) li->li_dblayer_private; + + /* get the backend instance name */ + sep = get_sep(src_dir); + if ((relative_instance_name = strrchr(src_dir, sep)) == NULL) + relative_instance_name = src_dir; + else + relative_instance_name++; + + inst = ldbm_instance_find_by_name(li, relative_instance_name); + if (NULL == inst) + { + LDAPDebug(LDAP_DEBUG_ANY, "Backend instance \"%s\" does not exist; " + "Instance path %s could be invalid.\n", + relative_instance_name, src_dir, 0); + return return_value; + } + + inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN); + if (is_fullpath(src_dir)) + new_src_dir = src_dir; + else + { + int len = strlen(inst_dirp); + sep = get_sep(inst_dirp); + if (*(inst_dirp+len-1) == sep) + sep = '\0'; + new_src_dir = (char *)slapi_ch_malloc(strlen(src_dir) + len + 2); + sprintf(new_src_dir, "%s%c%s", inst_dirp, sep, src_dir); + } + + dirhandle = PR_OpenDir(new_src_dir); + if (NULL == dirhandle) + return return_value; + + while (NULL != (direntry = + PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) + { + if (NULL == direntry->name) { + /* NSPR doesn't behave like the docs say it should */ + break; + } + if (indexonly && + 0 == strcmp(direntry->name, ID2ENTRY LDBM_FILENAME_SUFFIX)) + { + continue; + } + + /* Look at the last three characters in the filename */ + filename_length = strlen(direntry->name); + if (filename_length > 4) { + offset = filename_length - 4; + } else { + offset = 0; + } + compare_piece = (char *)direntry->name + offset; + + if (0 == strcmp(compare_piece, LDBM_FILENAME_SUFFIX) || /* .db4 */ + 0 == strcmp(compare_piece, LDBM_SUFFIX_OLD) || /* support .db3 */ + 0 == strcmp(direntry->name, DBVERSION_FILENAME)) { + /* Found a database file. Copy it. */ + + if (NULL == new_dest_dir) { + /* Need to create the new directory where the files will be + * copied to. */ + PRFileInfo info; + char *prefix = ""; + char mysep = 0; + + if (!is_fullpath(dest_dir)) + { + prefix = dblayer_get_home_dir(li, NULL); + mysep = get_sep(prefix); + } + + new_dest_dir = slapi_ch_malloc(strlen(dest_dir) + + strlen(relative_instance_name) + + strlen(prefix) + 3); + if (mysep) + sprintf(new_dest_dir, "%s%c%s%c%s", + prefix, mysep, dest_dir, mysep, relative_instance_name); + else + sprintf(new_dest_dir, "%s/%s", + dest_dir, relative_instance_name); + /* } */ + if (PR_SUCCESS == PR_GetFileInfo(new_dest_dir, &info)) + { + ldbm_delete_dirs(new_dest_dir); + } + if (mkdir_p(new_dest_dir, 0700) != PR_SUCCESS) + { + LDAPDebug(LDAP_DEBUG_ANY, "Can't create new directory %s, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + new_dest_dir, PR_GetError(), + slapd_pr_strerror(PR_GetError())); + goto out; + } + } + + filename1 = slapi_ch_malloc(strlen(new_src_dir) + + strlen(direntry->name) + 2); + sprintf(filename1, "%s/%s", new_src_dir, direntry->name); + filename2 = slapi_ch_malloc(strlen(new_dest_dir) + + strlen(direntry->name) + 2); + sprintf(filename2, "%s/%s", new_dest_dir, direntry->name); + + if (restore) { + LDAPDebug(LDAP_DEBUG_ANY, "Restoring file %d (%s)\n", + *cnt, filename2, 0); + if (task) { + slapi_task_log_notice(task, + "Restoring file %d (%s)", *cnt, filename2); + slapi_task_log_status(task, + "Restoring file %d (%s)", *cnt, filename2); + } + } else { + LDAPDebug(LDAP_DEBUG_ANY, "Backing up file %d (%s)\n", + *cnt, filename2, 0); + if (task) { + slapi_task_log_notice(task, + "Backing up file %d (%s)", *cnt, filename2); + slapi_task_log_status(task, + "Backing up file %d (%s)", *cnt, filename2); + } + } + + /* copy filename1 to filename2 */ + return_value = dblayer_copyfile(filename1, filename2, + 0, priv->dblayer_file_mode); + slapi_ch_free((void**)&filename1); + slapi_ch_free((void**)&filename2); + if (0 > return_value) + break; + + (*cnt)++; + } + } +out: + PR_CloseDir(dirhandle); + slapi_ch_free((void**)&new_dest_dir); + if (new_src_dir != src_dir) + slapi_ch_free((void**)&new_src_dir); + return return_value; +} + + + +/* Destination Directory is an absolute pathname */ +int dblayer_backup(struct ldbminfo *li, char *dest_dir, Slapi_Task *task) +{ + dblayer_private *priv = NULL; + char **listA = NULL, **listB = NULL, **listi, **listj, *prefix; + char *home_dir = NULL; + int return_value = 0; + char *pathname1; + char *pathname2; + back_txn txn; + int cnt = 1, ok = 0; + ldbm_instance *inst; + Object *inst_obj; + + PR_ASSERT(NULL != li); + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + home_dir = dblayer_get_home_dir(li, NULL); + if (NULL == home_dir) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Backup failed due to missing db home directory info\n", 0, 0, 0); + return -1; + } + + /* + * What are we doing here ? + * We want to copy into the backup directory: + * All the backend instance dir / database files; + * All the logfiles + * The version file + */ + + /* changed in may 1999 for political correctness. + * 1. take checkpoint + * 2. open transaction + * 3. get list of logfiles (A) + * 4. copy the db# files + * 5. get list of logfiles (B) + * 6. if !(A in B), goto 3 + * (logfiles were flushed during our backup) + * 7. copy logfiles from list B + * 8. abort transaction + * 9. backup index config info + */ + + /* Order of checkpointing and txn creation reversed to work + * around DB problem. If we don't do it this way around DB + * thinks all old transaction logs are required for recovery + * when the DB environment has been newly created (such as + * after an import). + */ + + /* do a quick checkpoint */ + dblayer_force_checkpoint(li); + dblayer_txn_init(li,&txn); + return_value=dblayer_txn_begin(li,NULL,&txn); + if (0 != return_value) { + LDAPDebug(LDAP_DEBUG_ANY, + "Backup failed due to transaction failure\n", 0, 0, 0); + return -1; + } + + if ( g_get_shutdown() || c_get_shutdown() ) { + dblayer_txn_abort(li,&txn); + return -1; + } + + /* repeat this until the logfile sets match... */ + do { + /* get the list of logfiles currently existing */ + if (priv->dblayer_enable_transactions) { + return_value = LOG_ARCHIVE(priv->dblayer_env->dblayer_DB_ENV, + &listA, DB_ARCH_LOG, malloc); + if ((return_value != 0) || (listA == NULL)) { + LDAPDebug(LDAP_DEBUG_ANY, "BAD: can't get list of logs\n", + 0, 0, 0); + dblayer_txn_abort(li,&txn); + return return_value; + } + } else { + ok=1; + } + if ( g_get_shutdown() || c_get_shutdown() ) { + dblayer_txn_abort(li,&txn); + return -1; + } + + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) + { + char inst_dir[MAXPATHLEN]; + char *inst_dirp = NULL; + inst = (ldbm_instance *)object_get_data(inst_obj); + inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN); + return_value = dblayer_copy_directory(li, task, inst_dirp, + dest_dir, 0 /* backup */, &cnt, 0, 0); + if (return_value != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR: error copying directory (%s -> %s): err=%d\n", + inst_dirp, dest_dir, return_value); + if (task) { + slapi_task_log_notice(task, + "ERROR: error copying directory (%s -> %s): err=%d", + inst_dirp, dest_dir, return_value); + } + if (listA) { + free(listA); + } + dblayer_txn_abort(li,&txn); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return return_value; + } + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + } + if (priv->dblayer_enable_transactions) { + /* now, get the list of logfiles that still exist */ + return_value = LOG_ARCHIVE(priv->dblayer_env->dblayer_DB_ENV, + &listB, DB_ARCH_LOG, malloc); + if ((return_value != 0) || (listB == NULL)) { + LDAPDebug(LDAP_DEBUG_ANY, "ERROR: can't get list of logs\n", + 0, 0, 0); + free(listA); + dblayer_txn_abort(li,&txn); + return return_value; + } + + /* compare: make sure everything in list A is still in list B */ + ok = 1; + for (listi = listA; *listi && ok; listi++) { + int found = 0; + for (listj = listB; *listj && !found; listj++) { + if (strcmp(*listi, *listj) == 0) + found = 1; + } + if (! found) { + ok = 0; /* missing log: start over */ + LDAPDebug(LDAP_DEBUG_ANY, "WARNING: Log %s has been swiped " + "out from under me! (retrying)\n", *listi, 0, 0); + if (task) { + slapi_task_log_notice(task, + "WARNING: Log %s has been swiped out from under me! " + "(retrying)", *listi); + } + } + } + + if ( g_get_shutdown() || c_get_shutdown() ) { + dblayer_txn_abort(li,&txn); + return -1; + } + + if (ok) { + char **listptr; + + prefix = NULL; + if ((NULL != priv->dblayer_log_directory) && + (0 != strlen(priv->dblayer_log_directory))) { + prefix = priv->dblayer_log_directory; + } else { + prefix = home_dir; + } + /* log files have the same filename len(100 is a safety net:) */ + pathname1 = (char *)slapi_ch_malloc(strlen(prefix) + + strlen(*listB) + 100); + pathname2 = (char *)slapi_ch_malloc(strlen(dest_dir) + + strlen(*listB) + 100); + /* We copy those over */ + for (listptr = listB; (*listptr) && ok; ++listptr) { + sprintf(pathname1, "%s/%s", prefix, *listptr); + sprintf(pathname2, "%s/%s", dest_dir, *listptr); + LDAPDebug(LDAP_DEBUG_ANY, "Backing up file %d (%s)\n", + cnt, pathname2, 0); + if (task) + { + slapi_task_log_notice(task, + "Backing up file %d (%s)", cnt, pathname2); + slapi_task_log_status(task, + "Backing up file %d (%s)", cnt, pathname2); + } + return_value = dblayer_copyfile(pathname1, pathname2, + 0, priv->dblayer_file_mode); + if (0 > return_value) { + LDAPDebug(LDAP_DEBUG_ANY, "Error copying file '%s' " + "(err=%d) -- Starting over...\n", + pathname1, return_value, 0); + if (task) { + slapi_task_log_notice(task, + "Error copying file '%s' (err=%d) -- Starting " + "over...", pathname1, return_value); + } + ok = 0; + } + if ( g_get_shutdown() || c_get_shutdown() ) { + dblayer_txn_abort(li,&txn); + return -1; + } + cnt++; + } + slapi_ch_free((void **)&pathname1); + slapi_ch_free((void **)&pathname2); + } + + if (listA) { + free(listA); + listA = NULL; + } + if (listB) { + free(listB); + listB = NULL; + } + } + } while (!ok); + + + if ( g_get_shutdown() || c_get_shutdown() ) { + dblayer_txn_abort(li,&txn); + return -1; + } + + /* now copy the version file */ + pathname1 = (char *)slapi_ch_malloc(strlen(home_dir) + + strlen(DBVERSION_FILENAME) + 2); + pathname2 = (char *)slapi_ch_malloc(strlen(dest_dir) + + strlen(DBVERSION_FILENAME) + 2); + sprintf(pathname1, "%s/%s", home_dir, DBVERSION_FILENAME); + sprintf(pathname2, "%s/%s", dest_dir, DBVERSION_FILENAME); + LDAPDebug(LDAP_DEBUG_ANY, "Backing up file %d (%s)\n", + cnt, pathname2, 0); + if (task) { + slapi_task_log_notice(task, "Backing up file %d (%s)", cnt, pathname2); + slapi_task_log_status(task, "Backing up file %d (%s)", cnt, pathname2); + } + return_value = dblayer_copyfile(pathname1,pathname2,0,priv->dblayer_file_mode); + slapi_ch_free((void **)&pathname1); + slapi_ch_free((void **)&pathname2); + + /* Lastly we tell log file truncation to start again */ + + if (0 == return_value) /* if everything went well, backup the index conf */ + return_value = dse_conf_backup(li, dest_dir); + + return_value = dblayer_txn_abort(li,&txn); + return return_value; +} + + +/* + * Restore is pretty easy. + * We delete the current database. + * We then copy all the files over from the backup point. + * We then leave them there for the slapd process to pick up and do the recovery + * (which it will do as it sees no guard file). + */ + +/* Helper function first */ + +static int dblayer_is_logfilename(const char* path) +{ + int ret = 0; + /* Is the filename at least 4 characters long ? */ + if (strlen(path) < 4) + { + return 0; /* Not a log file then */ + } + /* Are the first 4 characters "log." ? */ + ret = strncmp(path,"log.",4); + if (0 == ret) + { + /* Now, are the last 4 characters _not_ .db# ? */ + const char *piece = path + (strlen(path) - 4); + ret = strcmp(piece,LDBM_FILENAME_SUFFIX); + if (0 != ret) + { + /* Is */ + return 1; + } + } + return 0; /* Is not */ +} + +/* remove log.xxx from log directory*/ +static +int dblayer_delete_transaction_logs(const char * log_dir) +{ + int rc=0; + char filename1[MAXPATHLEN]; + PRDir *dirhandle = NULL; + dirhandle = PR_OpenDir(log_dir); + if (NULL != dirhandle) { + PRDirEntry *direntry = NULL; + int is_a_logfile = 0; + int pre=0; + PRFileInfo info ; + + while (NULL != (direntry = + PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) + { + if (NULL == direntry->name) { + /* NSPR doesn't behave like the docs say it should */ + LDAPDebug(LDAP_DEBUG_ANY, "PR_ReadDir failed (%d): %s\n", + PR_GetError(),slapd_pr_strerror(PR_GetError()), 0); + break; + } + sprintf(filename1, "%s/%s", log_dir, direntry->name); + pre = PR_GetFileInfo(filename1, &info); + if (pre == PR_SUCCESS && PR_FILE_DIRECTORY == info.type) { + continue; + } + is_a_logfile = dblayer_is_logfilename(direntry->name); + if (is_a_logfile && (NULL != log_dir) && (0 != strlen(log_dir)) ) + { + LDAPDebug(LDAP_DEBUG_ANY, "Deleting log file: (%s)\n", + filename1, 0, 0); + unlink(filename1); + } + } + PR_CloseDir(dirhandle); + } + else if (PR_FILE_NOT_FOUND_ERROR != PR_GetError()) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dblayer_delete_transaction_logs: PR_OpenDir(%s) failed (%d): %s\n", + log_dir, PR_GetError(),slapd_pr_strerror(PR_GetError())); + rc=1; + } + return rc; +} + +const char *skip_list[] = +{ + ".ldif", + NULL +}; + +static int doskip(const char *filename) +{ + const char **p; + int len = strlen(filename); + + for (p = skip_list; p && *p; p++) + { + int n = strlen(*p); + if (0 == strncmp(filename + len - n, *p, n)) + return 1; + } + return 0; +} + +/* Destination Directory is an absolute pathname */ + +int dblayer_restore(struct ldbminfo *li, char *src_dir, Slapi_Task *task) +{ + dblayer_private *priv = NULL; + int return_value = 0; + int tmp_rval; + char filename1[MAXPATHLEN]; + char filename2[MAXPATHLEN]; + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + PRFileInfo info; + ldbm_instance *inst; + int seen_logfiles = 0; /* Tells us if we restored any logfiles */ + int is_a_logfile = 0; + int dbmode; + int action = 0; + char *home_dir = NULL; + + PR_ASSERT(NULL != li); + priv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != priv); + + /* DBDB this is a hack, take out later */ + PR_Lock(li->li_config_mutex); + priv->dblayer_home_directory = li->li_directory; + priv->dblayer_cachesize = li->li_dbcachesize; + priv->dblayer_ncache = li->li_dbncache; + priv->dblayer_file_mode = li->li_mode; + PR_Unlock(li->li_config_mutex); + + home_dir = dblayer_get_home_dir(li, NULL); + if (NULL == home_dir) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Restore failed due to missing db home directory info\n", 0, 0, 0); + return -1; + } + + /* We find out if slapd is running */ + /* If it is, we fail */ + /* We check on the source staging area, no point in going further if it + * isn't there */ + if (!dbversion_exists(li, src_dir)) { + LDAPDebug(LDAP_DEBUG_ANY, "restore: source directory %s does not " + "contain a complete backup\n", src_dir, 0, 0); + + + if (task) { + slapi_task_log_notice(task, "Source directory %s does not " + "contain a complete backup", src_dir ); + } + } + + /* + * Check if the target is a superset of the backup. + * If not don't restore any db at all, otherwise + * the target will be crippled. + */ + dirhandle = PR_OpenDir(src_dir); + if (NULL != dirhandle) + { + while ((direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT)) + && direntry->name) + { + sprintf(filename1, "%s/%s", src_dir, direntry->name); + tmp_rval = PR_GetFileInfo(filename1, &info); + if (tmp_rval == PR_SUCCESS && PR_FILE_DIRECTORY == info.type) { + inst = ldbm_instance_find_by_name(li, (char *)direntry->name); + if ( inst == NULL) + { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR: target server has no %s configured\n", direntry->name, 0, 0); + if (task) { + slapi_task_log_notice(task, + "ERROR: target server has no %s configured\n", direntry->name); + } + PR_CloseDir(dirhandle); + return LDAP_UNWILLING_TO_PERFORM; + } + } + } + PR_CloseDir(dirhandle); + } + + /* We delete the existing database */ + return_value = dblayer_delete_database(li); + if (return_value) { + return return_value; + } + + /* We copy the files over from the staging area */ + /* We want to treat the logfiles specially: if there's + * a log file directory configured, copy the logfiles there + * rather than to the db dirctory */ + if (0 == return_value) { + dirhandle = PR_OpenDir(src_dir); + if (NULL != dirhandle) { + char *restore_dir; + char *prefix = NULL; + int cnt = 1; + + + while (NULL != (direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) { + if (NULL == direntry->name) { + /* NSPR doesn't behave like the docs say it should */ + break; + } + + /* Is this entry a directory? */ + sprintf(filename1, "%s/%s", src_dir, direntry->name); + tmp_rval = PR_GetFileInfo(filename1, &info); + if (tmp_rval == PR_SUCCESS && PR_FILE_DIRECTORY == info.type) { + /* This is an instance directory. It contains the *.db# + * files for the backend instance. + * restore directory is supposed to be where the backend + * directory is located. + */ + + inst = ldbm_instance_find_by_name(li, (char *)direntry->name); + if (inst == NULL) + continue; + + restore_dir = inst->inst_parent_dir_name; + + if (dblayer_copy_directory(li, task, filename1, + restore_dir, 1 /* restore */, &cnt, 0, 0) == 0) + continue; + else + { + LDAPDebug(LDAP_DEBUG_ANY, + "restore: failed to copy directory %s\n", + filename1, 0, 0); + if (task) { + slapi_task_log_notice(task, + "Failed to copy directory %s", filename1); + } + break; + } + } + + if (doskip(direntry->name)) + continue; + + /* Is this a log file ? */ + /* Log files have names of the form "log.xxxxx" */ + /* We detect these by looking for the prefix "log." and + * the lack of the ".db#" suffix */ + is_a_logfile = dblayer_is_logfilename(direntry->name); + if (is_a_logfile) { + seen_logfiles = 1; + } + if (is_a_logfile && (NULL != priv->dblayer_log_directory) && + (0 != strlen(priv->dblayer_log_directory)) ) { + prefix = priv->dblayer_log_directory; + } else { + prefix = home_dir; + } + mkdir_p(prefix, 0700); + sprintf(filename1, "%s/%s", src_dir, direntry->name); + sprintf(filename2, "%s/%s", prefix, direntry->name); + LDAPDebug(LDAP_DEBUG_ANY, "Restoring file %d (%s)\n", + cnt, filename2, 0); + if (task) { + slapi_task_log_notice(task, "Restoring file %d (%s)", + cnt, filename2); + slapi_task_log_status(task, "Restoring file %d (%s)", + cnt, filename2); + } + return_value = dblayer_copyfile(filename1, filename2, 0, + priv->dblayer_file_mode); + if (0 > return_value) + break; + + cnt++; + } + PR_CloseDir(dirhandle); + } + } + /* We're done ! */ + +#if defined(UPGRADEDB) + /* [605024] check the DBVERSION and reset idl-switch if needed */ + if (dbversion_exists(li, home_dir)) + { + char ldbmversion[LDBM_VERSION_MAXBUF]; + char dataversion[LDBM_VERSION_MAXBUF]; + + if (dbversion_read(li, home_dir, ldbmversion, dataversion) != 0) + { + LDAPDebug(LDAP_DEBUG_ANY, "Warning: Unable to read dbversion " + "file in %s\n", home_dir, 0, 0); + } + else + { + adjust_idl_switch(ldbmversion, li); + } + } +#endif + + return_value = check_db_version(li, &action); + if (action & DBVERSION_UPGRADE_3_4) + { + dbmode = DBLAYER_CLEAN_RECOVER_MODE;/* upgrade: remove logs & recover */ + } + else if (seen_logfiles) + { + dbmode = DBLAYER_RESTORE_MODE; + } + else + { + dbmode = DBLAYER_RESTORE_NO_RECOVERY_MODE; + } + + /* now start the database code up, to prevent recovery next time the + * server starts; + * dse_conf_verify may need to have db started, as well. */ + /* If no logfiles were stored, then fatal recovery isn't required */ + + if (li->li_flags & TASK_RUNNING_FROM_COMMANDLINE) + { + dbmode |= DBLAYER_CMDLINE_MODE; + } + else /* on-line mode */ + { + allinstance_set_not_busy(li); + } + + tmp_rval = dblayer_start(li, dbmode); + if (0 != tmp_rval) { + LDAPDebug(LDAP_DEBUG_ANY, + "dblayer_restore: Failed to init database\n", 0, 0, 0); + if (task) { + slapi_task_log_notice(task, "Failed to init database"); + } + return tmp_rval; + } + + if (0 == return_value) { /* only when the copyfile succeeded */ + /* check the DSE_* files, if any */ + tmp_rval = dse_conf_verify(li, src_dir); + if (0 != tmp_rval) + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: Unable to verify the index configuration\n", 0, 0, 0); + } + + if (li->li_flags & TASK_RUNNING_FROM_COMMANDLINE) { + /* command line: close the database down again */ + tmp_rval = dblayer_close(li, dbmode); + if (0 != tmp_rval) { + LDAPDebug(LDAP_DEBUG_ANY, + "dblayer_restore: Failed to close database\n", 0, 0, 0); + } + } else { + allinstance_set_busy(li); /* on-line mode */ + } + + return_value = tmp_rval?tmp_rval:return_value; + + return return_value; +} + + +static char *dblayer_make_friendly_instance_name(ldbm_instance *inst) +{ + char *name = slapi_ch_strdup(inst->inst_name); + int x; + + if (name == NULL) + return NULL; + for (x = 0; name[x]; x++) + if (name[x] == ' ') + name[x] = '_'; + return name; +} + +/* + * inst_dir_name is a relative path (from 6.21) + * ==> txn log stores relative paths and becomes relocatable + * if full path is given, parent dir is inst_parent_dir_name; + * otherwise, inst_dir in home_dir + * + * Set an appropriate path to inst_dir_name, if not yet. + * Create the specified directory, if not exists. + */ +int dblayer_get_instance_data_dir(backend *be) +{ + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; + char *full_namep = NULL; + char full_name[MAXPATHLEN]; + PRDir *db_dir = NULL; + int ret = -1; + + /* if a specific directory name was specified for this particular + * instance use it othewise use the ldbm-wide one + */ + full_namep = dblayer_get_full_inst_dir(inst->inst_li, inst, + full_name, MAXPATHLEN); + /* Does this directory already exist? */ + if ((db_dir = PR_OpenDir(full_namep)) != NULL) { + /* yep. */ + PR_CloseDir(db_dir); + ret = 0; + } else { + /* nope -- create it. */ + ret = mkdir_p(full_namep, 0700); + } + + if (full_name != full_namep) + slapi_ch_free_string(&full_namep); + + return ret; +} + +char * +dblayer_strerror(int error) +{ + return db_strerror(error); +} + +/* [605974] check a db region file's existence to know whether import is executed by other process or not */ +#define DB_REGION_PREFIX "__db." + +int +dblayer_in_import(ldbm_instance *inst) +{ + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + char inst_dir[MAXPATHLEN]; + char *inst_dirp = NULL; + int rval = 0; + + inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN); + dirhandle = PR_OpenDir(inst_dirp); + + if (NULL == dirhandle) + goto done; + + while (NULL != (direntry = PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) + { + if (NULL == direntry->name) + { + break; + } + if (0 ==strncmp(direntry->name, DB_REGION_PREFIX, 5)) + { + rval = 1; + break; + } + } + PR_CloseDir(dirhandle); +done: + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return rval; +} + +/* + * to change the db extention (e.g., .db3 -> .db4) + */ +int dblayer_update_db_ext(ldbm_instance *inst, char *oldext, char *newext) +{ + struct attrinfo *a = NULL; + backend *be = NULL; + struct ldbminfo *li = NULL; + dblayer_private *priv = NULL; + DB *thisdb = NULL; + int rval = 0; + char *ofile = NULL; + char *nfile = NULL; + char inst_dir[MAXPATHLEN]; + char *inst_dirp; + + if (NULL == inst) + { + LDAPDebug(LDAP_DEBUG_ANY, + "update_db_ext: Null instance is passed\n", 0, 0, 0); + return -1; /* non zero */ + } + be = inst->inst_be; + li = inst->inst_li; + priv = (dblayer_private*)li->li_dblayer_private; + inst_dirp = dblayer_get_full_inst_dir(li, inst, inst_dir, MAXPATHLEN); + for (a = (struct attrinfo *)avl_getfirst(inst->inst_attrs); + NULL != a; + a = (struct attrinfo *)avl_getnext()) + { + PRFileInfo info; + ofile = slapi_ch_malloc(strlen(inst_dirp) + + strlen(a->ai_type) + strlen(oldext) + 2); + sprintf(ofile, "%s/%s%s", inst_dirp, a->ai_type, oldext); + + if (PR_GetFileInfo(ofile, &info) != PR_SUCCESS) + { + slapi_ch_free_string(&ofile); + continue; + } + + /* db->rename disable DB in it; we need to create for each */ + rval = db_create(&thisdb, priv->dblayer_env->dblayer_DB_ENV, 0); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, "db_create returned %d (%s)\n", + rval, dblayer_strerror(rval), 0); + goto done; + } + nfile = slapi_ch_malloc(strlen(inst_dirp) + + strlen(a->ai_type) + strlen(newext) + 2); + sprintf(nfile, "%s/%s%s", inst_dirp, a->ai_type, newext); + LDAPDebug(LDAP_DEBUG_TRACE, "update_db_ext: rename %s -> %s\n", + ofile, nfile, 0); + + rval = thisdb->rename(thisdb, (const char *)ofile, NULL /* subdb */, + (const char *)nfile, 0); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, "rename returned %d (%s)\n", + rval, dblayer_strerror(rval), 0); + LDAPDebug(LDAP_DEBUG_ANY, + "update_db_ext: index (%s) Failed to update index %s -> %s\n", + inst->inst_name, ofile, nfile); + goto done; + } + slapi_ch_free_string(&ofile); + slapi_ch_free_string(&nfile); + } + + rval = db_create(&thisdb, priv->dblayer_env->dblayer_DB_ENV, 0); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, "db_create returned %d (%s)\n", + rval, dblayer_strerror(rval), 0); + goto done; + } + ofile = slapi_ch_malloc(strlen(inst_dirp) + + strlen(ID2ENTRY) + strlen(oldext) + 2); + nfile = slapi_ch_malloc(strlen(inst_dirp) + + strlen(ID2ENTRY) + strlen(newext) + 2); + sprintf(ofile, "%s/%s%s", inst_dirp, ID2ENTRY, oldext); + sprintf(nfile, "%s/%s%s", inst_dirp, ID2ENTRY, newext); + LDAPDebug(LDAP_DEBUG_TRACE, "update_db_ext: rename %s -> %s\n", + ofile, nfile, 0); + rval = thisdb->rename(thisdb, (const char *)ofile, NULL /* subdb */, + (const char *)nfile, 0); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, "rename returned %d (%s)\n", + rval, dblayer_strerror(rval), 0); + LDAPDebug(LDAP_DEBUG_ANY, + "update_db_ext: index (%s) Failed to update index %s -> %s\n", + inst->inst_name, ofile, nfile); + } +done: + slapi_ch_free_string(&ofile); + slapi_ch_free_string(&nfile); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + + return rval; +} + +/* + * delete the index files belonging to the instance + */ +int dblayer_delete_indices(ldbm_instance *inst) +{ + int rval = -1; + struct attrinfo *a = NULL; + int i; + + if (NULL == inst) + { + LDAPDebug(LDAP_DEBUG_ANY, + "update_index_ext: Null instance is passed\n", 0, 0, 0); + return rval; + } + rval = 0; + for (a = (struct attrinfo *)avl_getfirst(inst->inst_attrs), i = 0; + NULL != a; + a = (struct attrinfo *)avl_getnext(), i++) + { + rval += dblayer_erase_index_file(inst->inst_be, a, i/* chkpt; 1st time only */); + } + return rval; +} + +void dblayer_set_recovery_required(struct ldbminfo *li) +{ + if (NULL == li || NULL == li->li_dblayer_private) + { + LDAPDebug(LDAP_DEBUG_ANY,"set_recovery_required: no dblayer info\n", + 0, 0, 0); + return; + } + li->li_dblayer_private->dblayer_recovery_required = 1; +} diff --git a/ldap/servers/slapd/back-ldbm/dblayer.h b/ldap/servers/slapd/back-ldbm/dblayer.h new file mode 100644 index 00000000..78274ec6 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/dblayer.h @@ -0,0 +1,140 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* Structures and #defines used in the dblayer. */ + +#ifndef _DBLAYER_H_ +#define _DBLAYER_H_ + +#ifdef DB_USE_64LFS +#ifdef OS_solaris +#include <dlfcn.h> /* needed for dlopen and dlsym */ +#endif /* solaris: dlopen */ +#ifdef OS_solaris +#include <sys/mman.h> /* needed for mmap/mmap64 */ +#ifndef MAP_FAILED +#define MAP_FAILED (-1) +#endif +#endif /* solaris: mmap */ +#endif /* DB_USE_64LFS */ + +#define DBLAYER_PAGESIZE (size_t)8*1024 +#define DBLAYER_INDEX_PAGESIZE (size_t)8*1024 /* With the new idl design, + the large 8Kbyte pages we use are not optimal. The page pool churns very + quickly as we add new IDs under a sustained add load. Smaller pages stop + this happening so much and consequently make us spend less time flushing + dirty pages on checkpoints. But 8K is still a good page size for id2entry. + So we now allow different page sizes for the primary and secondary indices. + */ + +/* Interval, in ms, that threads sleep when they are wanting to + * wait for a while withouth spinning. If this time is too long, + * the server takes too long to shut down. If this interval is too + * short, then CPU time gets burned by threads doing nothing. + * As CPU speed increases over time, we reduce this interval + * to allow the server to be more responsive to shutdown. + * (Why is this important ? : A: because the TET tests start up + * and shut down the server a gazillion times, so the server + * shut down delay has a significant impact on the overall test + * run time (which is very very very looooonnnnnggggg....).) +*/ +#define DBLAYER_SLEEP_INTERVAL 250 + +#define DB_EXTN_PAGE_HEADER_SIZE 64 /* DBDB this is a guess */ + +#define DBLAYER_CACHE_FORCE_FILE 1 + +#define DBLAYER_LIB_VERSION_PRE_24 1 +#define DBLAYER_LIB_VERSION_POST_24 2 + +/* Define constants from DB2.4 when using DB2.3 header file */ +#ifndef DB_TSL_SPINS +#define DB_TSL_SPINS 21 /* DB: initialize spin count. */ +#endif +#ifndef DB_REGION_INIT +#define DB_REGION_INIT 24 /* DB: page-fault regions in create. */ +#endif +#ifndef DB_REGION_NAME +#define DB_REGION_NAME 25 /* DB: named regions, no backing file. */ +#endif + +struct dblayer_private_env { + DB_ENV *dblayer_DB_ENV; + PRRWLock * dblayer_env_lock; + int dblayer_openflags; + int dblayer_priv_flags; +}; + +#define DBLAYER_PRIV_SET_DATA_DIR 0x1 + +/* structure which holds our stuff */ +struct dblayer_private +{ + struct dblayer_private_env * dblayer_env; + char *dblayer_home_directory; + char *dblayer_log_directory; + char *dblayer_dbhome_directory; /* default path for relative inst paths */ + char **dblayer_data_directories; /* passed to set_data_dir + * including dblayer_dbhome_directory */ + char **dblayer_db_config; + int dblayer_ncache; + int dblayer_previous_ncache; + int dblayer_tx_max; + size_t dblayer_cachesize; + size_t dblayer_previous_cachesize; /* Cache size when we last shut down-- + * used to determine if we delete + * the mpool */ + int dblayer_recovery_required; + int dblayer_enable_transactions; + int dblayer_durable_transactions; + int dblayer_checkpoint_interval; + int dblayer_circular_logging; + size_t dblayer_page_size; /* db page size if configured, + * otherwise default to DBLAYER_PAGESIZE */ + size_t dblayer_index_page_size; /* db index page size if configured, + * otherwise default to + * DBLAYER_INDEX_PAGESIZE */ + int dblayer_idl_divisor; /* divide page size by this to get IDL + * size */ + size_t dblayer_logfile_size; /* How large can one logfile be ? */ + size_t dblayer_logbuf_size; /* how large log buffer can be */ + int dblayer_file_mode; /* pmode for files we create */ + int dblayer_verbose; /* Get libdb to exhale debugging info */ + int dblayer_debug; /* Will libdb emit debugging info into + * our log ? */ + int dblayer_trickle_percentage; + int dblayer_cache_config; /* Special cache configurations + * e.g. force file-based mpool */ + int dblayer_lib_version; + int dblayer_spin_count; /* DB Mutex spin count, 0 == use default */ + int dblayer_named_regions; /* Should the regions be named sections, + * or backed by files ? */ + int dblayer_private_mem; /* private memory will be used for + * allocation of regions and mutexes */ + int dblayer_private_import_mem; /* private memory will be used for + * allocation of regions and mutexes for + * import */ + long dblayer_shm_key; /* base segment ID for named regions */ + int db_debug_checkpointing; /* Enable debugging messages from + * checkpointing */ + int dblayer_bad_stuff_happened; /* Means that something happened (e.g. out + * of disk space) such that the guardian + * file must not be written on shutdown */ + perfctrs_private *perf_private; /* Private data for performance counters + * code */ + int dblayer_stop_threads; /* Used to signal to threads that they + * should stop ASAP */ + PRInt32 dblayer_thread_count; /* Tells us how many threads are running, + * used to figure out when they're all + * stopped */ + int dblayer_lockdown; /* use DB_LOCKDOWN */ + int dblayer_lock_config; +}; + +int dblayer_db_remove(dblayer_private_env * env, char const path[], char const dbName[]); + +int dblayer_delete_indices(ldbm_instance *inst); + +#endif /* _DBLAYER_H_ */ diff --git a/ldap/servers/slapd/back-ldbm/dbsize.c b/ldap/servers/slapd/back-ldbm/dbsize.c new file mode 100644 index 00000000..891310fd --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/dbsize.c @@ -0,0 +1,25 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* + * dbsize.c - ldbm backend routine which returns the size (in bytes) + * that the database occupies on disk. + */ + +#include "back-ldbm.h" + +int +ldbm_db_size( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + unsigned int size; + int rc; + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + rc = dblayer_database_size(li, &size); + slapi_pblock_set( pb, SLAPI_DBSIZE, &size ); + + return rc; +} diff --git a/ldap/servers/slapd/back-ldbm/dbtest.c b/ldap/servers/slapd/back-ldbm/dbtest.c new file mode 100644 index 00000000..e2ba3dd0 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/dbtest.c @@ -0,0 +1,312 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* dbtest.c - ldbm database test program */ + +#include "back-ldbm.h" + +#define SLAPI_LDBM_DBTEST_OPT_DUMPDATA 0x0001 +#define SLAPI_LDBM_DBTEST_OPT_KEY_IS_BINARY 0x0002 +#define SLAPI_LDBM_DBTEST_OPT_DATA_IS_BINARY 0x0004 +#define SLAPI_LDBM_DBTEST_OPT_DATA_IS_IDLIST 0x0008 +#define SLAPI_LDBM_DBTEST_OPT_KEY_IS_ID 0x0010 + +static void dbtest_help( void ); +static void dbtest_traverse( DB *db, char *filename, unsigned int options, + FILE *outfp ); +static void dbtest_print_idlist( char *keystr, void *p, u_int32_t size, + FILE *outfp ); +static void dbtest_bprint( char *data, int len, char *lineprefix, + FILE *outfp ); + +int ldbm_back_db_test( Slapi_PBlock *pb ) +{ + char buf[256], *instance_name; + backend *be; + struct ldbminfo *li; + ldbm_instance *inst; + struct attrinfo *ai; + DB *db; + int err, traversal_options; + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + + /* essential initialization */ + mapping_tree_init(); + ldbm_config_load_dse_info(li); + /* Turn off transactions */ + ldbm_config_internal_set(li, CONFIG_DB_TRANSACTION_LOGGING, "off"); + + /* Find the instance */ + slapi_pblock_get( pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name ); + inst = ldbm_instance_find_by_name(li, instance_name); + if (NULL == inst) { + LDAPDebug(LDAP_DEBUG_ANY, "dbtest: unknown ldbm instance %s\n", + instance_name, 0, 0); + return -1; + } + + /* store the be in the pb */ + be = inst->inst_be; + slapi_pblock_set(pb, SLAPI_BACKEND, be); + + /***** prepare & init libdb, dblayer, and dbinstance *****/ + if (0 != dblayer_start(li, DBLAYER_TEST_MODE)) { + LDAPDebug( LDAP_DEBUG_ANY, + "dbtest: Failed to init database\n", 0, 0, 0 ); + return( -1 ); + } + if ( 0 != dblayer_instance_start(inst->inst_be, DBLAYER_NORMAL_MODE)) { + LDAPDebug( LDAP_DEBUG_ANY, + "dbtest: failed to start instance\n", 0, 0, 0 ); + return( -1 ); + } + + /* display commands help test */ + dbtest_help(); + + while ( 1 ) { + traversal_options = 0; + fputs( "dbtest: ", stdout ); + + if ( fgets( buf, sizeof(buf), stdin ) == NULL ) + break; + + switch ( buf[0] ) { + case 'i': + traversal_options |= SLAPI_LDBM_DBTEST_OPT_DATA_IS_IDLIST; + /*FALLTHRU*/ + + case 't': + traversal_options |= SLAPI_LDBM_DBTEST_OPT_DUMPDATA; + /*FALLTHRU*/ + + case 'T': + /* read the index to traverse */ + fputs( " attr: ", stdout ); + if ( fgets( buf, sizeof(buf), stdin ) == NULL ) { + exit( 0 ); + } + buf[strlen( buf ) - 1] = '\0'; + ai = NULL; + ainfo_get( be, buf, &ai ); + if ( ai == NULL ) { + fprintf( stderr, "no index for %s\n", buf ); + continue; + } + + /* open the index file */ + if ( (err = dblayer_get_index_file( be, ai, &db, 0 /* no create */ )) + != 0 ) { + fprintf( stderr, "could not get index for %s (error %d - %s)\n", + buf, err, slapd_system_strerror( err )); + continue; + } + + /* traverse the file */ + traversal_options |= SLAPI_LDBM_DBTEST_OPT_DATA_IS_BINARY; + dbtest_traverse( db, buf, traversal_options, stdout ); + + /* clean up */ + dblayer_release_index_file( be, ai, db ); + break; + + case 'u': + traversal_options |= SLAPI_LDBM_DBTEST_OPT_DUMPDATA; + /*FALLTHRU*/ + + case 'U': + /* open the id2entry file */ + if ( (err = dblayer_get_id2entry( be, &db )) != 0 ) { + fprintf( stderr, "could not get i2entry\n" ); + continue; + } + + /* traverse the file */ + traversal_options |= SLAPI_LDBM_DBTEST_OPT_KEY_IS_ID; + dbtest_traverse( db, "id2entry", traversal_options, stdout ); + + /* clean up */ + dblayer_release_id2entry( be, db ); + break; + + default: + dbtest_help(); + break; + } + } + + return( 0 ); +} + + +static void +dbtest_help() +{ + puts( LDBM_DATABASE_TYPE_NAME " test mode" ); + puts( "\nindex key prefixes:" ); + printf( " %c presence (sn=*)\n", PRES_PREFIX ); + printf( " %c equality (sn=jensen)\n", EQ_PREFIX ); + printf( " %c approximate (sn~=jensin)\n", APPROX_PREFIX ); + printf( " %c substring (sn=jen*)\n", SUB_PREFIX ); + printf( " %c matching rule (sn:1.2.3.4.5:=Jensen)\n", RULE_PREFIX ); + printf( " %c continuation\n", CONT_PREFIX ); + + puts( "\ncommands: i => traverse index keys and ID list values" ); + puts( " t => traverse index keys and values" ); + puts( " T => traverse index keys" ); + puts( " u => traverse id2entry keys and values" ); + puts( " U => traverse id2entry keys" ); +#if 0 + puts( " l<c> => lookup index" ); + puts( " L<c> => lookup index (all)" ); + puts( " t<c> => traverse index keys and values" ); + puts( " T<c> => traverse index keys" ); + puts( " x<c> => delete from index" ); + puts( " e<c> => edit index entry" ); + puts( " a<c> => add index entry" ); + puts( " c<c> => create index" ); + puts( " i<c> => insert ids into index" ); + puts( " b => change default backend" ); + puts( " B => print default backend" ); + puts( " d<n> => set slapd_ldap_debug to n" ); + puts( "where <c> is a char selecting the index:" ); + puts( " c => id2children" ); + puts( " d => dn2id" ); + puts( " e => id2entry" ); + puts( " f => arbitrary file" ); + puts( " i => attribute index" ); +#endif /* 0 */ +} + + +/* + * get a cursor and walk over the databasea + */ +static void +dbtest_traverse( DB *db, char *filename, unsigned int options, FILE *outfp ) +{ + DBC *dbc; + DBT key, data; + + dbc = NULL; + if ( db->cursor( db, NULL, &dbc, 0 ) != 0 ) { + fprintf( stderr, "could not get cursor for %s\n", filename ); + return; + } + + memset( &key, 0, sizeof(key) ); + memset( &data, 0, sizeof(data) ); + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + while ( dbc->c_get( dbc, &key, &data, DB_NEXT ) == 0 ) { + if (( options & SLAPI_LDBM_DBTEST_OPT_KEY_IS_BINARY ) != 0 ) { + fputs( "\tkey: ", outfp ); + dbtest_bprint( key.data, key.size, "\t ", outfp ); + } else if (( options & SLAPI_LDBM_DBTEST_OPT_KEY_IS_ID ) != 0 ) { + fprintf( outfp, "\tkey: %ld\n", + (u_long)id_stored_to_internal( (char *)key.data )); + } else { + fprintf( outfp, "\tkey: %s\n", (char *)key.data ); + } + if (( options & SLAPI_LDBM_DBTEST_OPT_DUMPDATA ) != 0 ) { + if (( options & SLAPI_LDBM_DBTEST_OPT_DATA_IS_IDLIST ) != 0 ) { + fputs( "\tdata: ", outfp ); + dbtest_print_idlist( (char *)key.dptr, data.data, data.size, + outfp ); + } else if (( options & SLAPI_LDBM_DBTEST_OPT_DATA_IS_BINARY ) != 0 ) { + fputs( "\tdata: ", outfp ); + dbtest_bprint( data.data, data.size, "\t ", outfp ); + } else { + fprintf( outfp, "\tdata: %s\n", (char *)data.data ); + } + } + free( key.data ); + free( data.data ); + } + dbc->c_close(dbc); +} + +static void +dbtest_print_idlist( char *keystr, void *p, u_int32_t size, FILE *outfp ) +{ + IDList *idl; + ID i; + + idl = (IDList *)p; + if ( ALLIDS( idl )) { + fputs( "ALLIDS block\n", outfp ); + } else if ( INDIRECT_BLOCK( idl )) { + fputs( "Indirect block)\n", outfp ); + for ( i = 0; idl->b_ids[i] != NOID; ++i ) { + fprintf( outfp, "\t\tkey: %c%s%lu\n", CONT_PREFIX, keystr, + (u_long)idl->b_ids[i] ); + } + } else { + const char *block_type; + + if ( NULL != keystr && *keystr == CONT_PREFIX ) { + block_type = "Continued"; + } else { + block_type = "Regular"; + } + fprintf( outfp, "%s block (count=%lu, max=%lu)\n", + block_type, (u_long)idl->b_nids, (u_long)idl->b_nmax ); + for ( i = 0; i < idl->b_nids; ++i ) { + fprintf( outfp, "\t\tid: %lu\n", (u_long)idl->b_ids[i] ); + } + } +} + + + +#define BPLEN 48 + +static void +dbtest_bprint( char *data, int len, char *lineprefix, FILE *outfp ) +{ + static char hexdig[] = "0123456789abcdef"; + char out[ BPLEN ], *curprefix; + int i = 0; + + if ( NULL == lineprefix ) { + lineprefix = ""; + } + curprefix = ""; + + memset( out, 0, BPLEN ); + for ( ;; ) { + if ( len < 1 ) { + if ( i > 0 ) { + fprintf( outfp, "%s%s\n", curprefix, out ); + } + break; + } + +#ifndef HEX + if ( isgraph( (unsigned char)*data )) { + out[ i ] = ' '; + out[ i+1 ] = *data; + } else { +#endif + out[ i ] = hexdig[ ( *data & 0xf0 ) >> 4 ]; + out[ i+1 ] = hexdig[ *data & 0x0f ]; +#ifndef HEX + } +#endif + i += 2; + len--; + data++; + + if ( i > BPLEN - 2 ) { + fprintf( outfp, "%s%s\n", curprefix, out ); + curprefix = lineprefix; + memset( out, 0, BPLEN ); + i = 0; + continue; + } + out[ i++ ] = ' '; + } +} diff --git a/ldap/servers/slapd/back-ldbm/dbversion.c b/ldap/servers/slapd/back-ldbm/dbversion.c new file mode 100644 index 00000000..4c1a56da --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/dbversion.c @@ -0,0 +1,181 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +#include "back-ldbm.h" + +static void +mk_dbversion_fullpath(struct ldbminfo *li, const char *directory, char *filename) +{ + if (li) + { + if (is_fullpath((char *)directory)) + { + sprintf(filename, "%s/%s", directory, DBVERSION_FILENAME); + } + else + { + char *home_dir = dblayer_get_home_dir(li, NULL); + /* if relpath, nsslapd-dbhome_directory should be set */ + sprintf(filename,"%s/%s/%s", home_dir,directory,DBVERSION_FILENAME); + } + } + else + { + sprintf(filename, "%s/%s", directory, DBVERSION_FILENAME); + } +} + +/* + * Function: dbversion_write + * + * Returns: returns 0 on success, -1 on failure + * + * Description: This function writes the DB version file. + */ +int +dbversion_write(struct ldbminfo *li, const char *directory, + const char *dataversion) +{ + char filename[ MAXPATHLEN*2 ]; + PRFileDesc *prfd; + int rc = 0; + + PR_ASSERT(is_fullpath((char *)directory)); + mk_dbversion_fullpath(li, directory, filename); + + /* Open the file */ + if (( prfd = PR_Open( filename, PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, + SLAPD_DEFAULT_FILE_MODE )) == NULL ) + { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open file \"%s\" for writing " + SLAPI_COMPONENT_NAME_NSPR " %d (%s)\n", + filename, PR_GetError(), slapd_pr_strerror(PR_GetError()) ); + rc= -1; + } + else + { + /* Write the file */ + PRInt32 len; + char buf[ LDBM_VERSION_MAXBUF ]; + /* recognize the difference between an old/new database regarding idl + * (406922) */ + if (idl_get_idl_new()) + { +#if defined(USE_NEW_IDL) + sprintf( buf, "%s\n", LDBM_VERSION ); +#else + sprintf( buf, "%s\n", LDBM_VERSION_NEW ); +#endif + } + else + { +#if defined(USE_NEW_IDL) + sprintf( buf, "%s\n", LDBM_VERSION_OLD ); +#else + sprintf( buf, "%s\n", LDBM_VERSION ); +#endif + } + len = strlen( buf ); + if ( slapi_write_buffer( prfd, buf, len ) != len ) + { + LDAPDebug( LDAP_DEBUG_ANY, "Could not write to file \"%s\"\n", filename, 0, 0 ); + rc= -1; + } + if(rc==0 && dataversion!=NULL) + { + sprintf( buf, "%s\n", dataversion ); + len = strlen( buf ); + if ( slapi_write_buffer( prfd, buf, len ) != len ) + { + LDAPDebug( LDAP_DEBUG_ANY, "Could not write to file \"%s\"\n", filename, 0, 0 ); + rc= -1; + } + } + (void)PR_Close( prfd ); + } + return rc; +} + +/* + * Function: dbversion_read + * + * Returns: returns 0 on success, -1 on failure + * + * Description: This function reads the DB version file. + */ +int +dbversion_read(struct ldbminfo *li, const char *directory, + char *ldbmversion, char *dataversion) +{ + char filename[ MAXPATHLEN*2 ]; + PRFileDesc *prfd; + int rc = -1; + char * iter = NULL; + + PR_ASSERT(is_fullpath((char *)directory)); + mk_dbversion_fullpath(li, directory, filename); + + ldbmversion[0]= '\0'; + dataversion[0]= '\0'; + + /* Open the file */ + if (( prfd = PR_Open( filename, PR_RDONLY, SLAPD_DEFAULT_FILE_MODE )) == + NULL ) + { + /* File missing... we are probably creating a new database. */ + } + else + { + char buf[LDBM_VERSION_MAXBUF]; + PRInt32 nr = slapi_read_buffer( prfd, buf, + (PRInt32)LDBM_VERSION_MAXBUF-1 ); + if ( nr > 0 && nr != (PRInt32)LDBM_VERSION_MAXBUF-1 ) + { + char *t; + buf[nr]= '\0'; + t= ldap_utf8strtok_r(buf,"\n", &iter); + if(t!=NULL) + { + strcpy(ldbmversion,t); + t= ldap_utf8strtok_r(NULL,"\n", &iter); + if(t!=NULL && t[0]!='\0') + { + strcpy(dataversion,t); + } + } + } + (void)PR_Close( prfd ); + rc= 0; + } + return rc; +} + + +/* + * Function: dbversion_exists + * + * Returns: 1 for exists, 0 for not. + * + * Description: This function checks if the DB version file exists. + */ +int +dbversion_exists(struct ldbminfo *li, const char *directory) +{ + char filename[ MAXPATHLEN*2 ]; + PRFileDesc *prfd; + + PR_ASSERT(is_fullpath((char *)directory)); + mk_dbversion_fullpath(li, directory, filename); + + if (( prfd = PR_Open( filename, PR_RDONLY, SLAPD_DEFAULT_FILE_MODE )) == + NULL ) + { + return 0; + } + (void)PR_Close( prfd ); + return 1; +} + diff --git a/ldap/servers/slapd/back-ldbm/dllmain.c b/ldap/servers/slapd/back-ldbm/dllmain.c new file mode 100644 index 00000000..d473ca49 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/dllmain.c @@ -0,0 +1,128 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* + * Microsoft Windows specifics for BACK-LDBM DLL + */ +#include "back-ldbm.h" + + +#ifdef _WIN32 +/* Lifted from Q125688 + * How to Port a 16-bit DLL to a Win32 DLL + * on the MSVC 4.0 CD + */ +BOOL WINAPI DllMain (HANDLE hModule, DWORD fdwReason, LPVOID lpReserved) +{ + + switch (fdwReason) + { + case DLL_PROCESS_ATTACH: + /* Code from LibMain inserted here. Return TRUE to keep the + DLL loaded or return FALSE to fail loading the DLL. + + You may have to modify the code in your original LibMain to + account for the fact that it may be called more than once. + You will get one DLL_PROCESS_ATTACH for each process that + loads the DLL. This is different from LibMain which gets + called only once when the DLL is loaded. The only time this + is critical is when you are using shared data sections. + If you are using shared data sections for statically + allocated data, you will need to be careful to initialize it + only once. Check your code carefully. + + Certain one-time initializations may now need to be done for + each process that attaches. You may also not need code from + your original LibMain because the operating system may now + be doing it for you. + */ + /* + * 16 bit code calls UnlockData() + * which is mapped to UnlockSegment in windows.h + * in 32 bit world UnlockData is not defined anywhere + * UnlockSegment is mapped to GlobalUnfix in winbase.h + * and the docs for both UnlockSegment and GlobalUnfix say + * ".. function is oboslete. Segments have no meaning + * in the 32-bit environment". So we do nothing here. + */ + + + break; + + case DLL_THREAD_ATTACH: + /* Called each time a thread is created in a process that has + already loaded (attached to) this DLL. Does not get called + for each thread that exists in the process before it loaded + the DLL. + + Do thread-specific initialization here. + */ + break; + + case DLL_THREAD_DETACH: + /* Same as above, but called when a thread in the process + exits. + + Do thread-specific cleanup here. + */ + break; + + case DLL_PROCESS_DETACH: + /* Code from _WEP inserted here. This code may (like the + LibMain) not be necessary. Check to make certain that the + operating system is not doing it for you. + */ + + break; + } + /* The return value is only used for DLL_PROCESS_ATTACH; all other + conditions are ignored. */ + return TRUE; // successful DLL_PROCESS_ATTACH +} +#else +int CALLBACK +LibMain( HINSTANCE hinst, WORD wDataSeg, WORD cbHeapSize, LPSTR lpszCmdLine ) +{ + /*UnlockData( 0 );*/ + return( 1 ); +} +#endif + +#ifdef LDAP_DEBUG +#ifndef _WIN32 +#include <stdarg.h> +#include <stdio.h> + +void LDAPDebug( int level, char* fmt, ... ) +{ + static char debugBuf[1024]; + + if (slapd_ldap_debug & level) + { + va_list ap; + va_start (ap, fmt); + _snprintf (debugBuf, sizeof(debugBuf), fmt, ap); + va_end (ap); + + OutputDebugString (debugBuf); + } +} +#endif +#endif + +#ifndef _WIN32 + +/* The 16-bit version of the RTL does not implement perror() */ + +#include <stdio.h> + +void perror( const char *msg ) +{ + char buf[128]; + wsprintf( buf, "%s: error %d\n", msg, WSAGetLastError()) ; + OutputDebugString( buf ); +} + +#endif diff --git a/ldap/servers/slapd/back-ldbm/dn2entry.c b/ldap/servers/slapd/back-ldbm/dn2entry.c new file mode 100644 index 00000000..f34e7f77 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/dn2entry.c @@ -0,0 +1,230 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* dn2entry.c - given a dn return an entry */ + +#include "back-ldbm.h" + +/* + * Fetch the entry for this DN. + * + * Retuns NULL of the entry doesn't exist + */ +struct backentry * +dn2entry( + Slapi_Backend *be, + const Slapi_DN *sdn, + back_txn *txn, + int *err +) +{ + ldbm_instance *inst; + struct berval ndnv; + struct backentry *e = NULL; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> dn2entry \"%s\"\n", slapi_sdn_get_dn(sdn), 0, 0 ); + + inst = (ldbm_instance *) be->be_instance_info; + + *err = 0; + ndnv.bv_val = (void*)slapi_sdn_get_ndn(sdn); /* jcm - Had to cast away const */ + ndnv.bv_len = slapi_sdn_get_ndn_len(sdn); + + e = cache_find_dn(&inst->inst_cache, ndnv.bv_val, ndnv.bv_len); + if (e == NULL) + { + /* convert dn to entry id */ + IDList *idl = NULL; + if ( (idl = index_read( be, "entrydn", indextype_EQUALITY, &ndnv, txn, err )) == NULL ) + { + /* There's no entry with this DN. */ + } + else + { + /* convert entry id to entry */ + if ( (e = id2entry( be, idl_firstid( idl ), txn, err )) != NULL ) + { + /* Means that we found the entry OK */ + } + else + { + /* Hmm. The DN mapped onto an EntryID, but that didn't map onto an Entry. */ + if ( *err != 0 && *err != DB_NOTFOUND ) + { + /* JCM - Not sure if this is ever OK or not. */ + } + else + { + /* + * this is pretty bad anyway. the dn was in the + * entrydn index, but we could not read the entry + * from the id2entry index. what should we do? + */ + LDAPDebug( LDAP_DEBUG_ANY, + "dn2entry: the dn was in the entrydn index (id %lu), " + "but it did not exist in id2entry.\n", + (u_long)idl_firstid( idl ), 0, 0 ); + } + } + slapi_ch_free((void**)&idl); + } + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= dn2entry %p\n", e, 0, 0 ); + return( e ); +} + +/* + * dn2entry_or_ancestor - look up dn in the cache/indexes and return the + * corresponding entry. If the entry is not found, this function returns NULL + * and sets ancestordn to the DN of highest entry in the tree matched. + * + * ancestordn should be initialized before calling this function. + * + * When the caller is finished with the entry returned, it should return it + * to the cache: + * e = dn2entry_or_ancestor( ... ); + * if ( NULL != e ) { + * cache_return( &inst->inst_cache, &e ); + * } + */ +struct backentry * +dn2entry_or_ancestor( + Slapi_Backend *be, + const Slapi_DN *sdn, + Slapi_DN *ancestordn, + back_txn *txn, + int *err +) +{ + struct backentry *e; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> dn2entry_or_ancestor \"%s\"\n", slapi_sdn_get_dn(sdn), 0, 0 ); + + /* + * Fetch the entry asked for. + */ + + e= dn2entry(be,sdn,txn,err); + + if(e==NULL) + { + /* + * could not find the entry named. crawl back up the dn and + * stop at the first ancestor that does exist, or when we get + * to the suffix. + */ + e= dn2ancestor(be,sdn,ancestordn,txn,err); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= dn2entry_or_ancestor %p\n", e, 0, 0 ); + return( e ); +} + +/* + * Use the DN to fetch the parent of the entry. + * If the parent entry doesn't exist, keep working + * up the DN until we hit "" or an backend suffix. + * + * ancestordn should be initialized before calling this function. + * + * Returns NULL for no entry found. + * + * When the caller is finished with the entry returned, it should return it + * to the cache: + * e = dn2ancestor( ... ); + * if ( NULL != e ) { + * cache_return( &inst->inst_cache, &e ); + * } + */ +struct backentry * +dn2ancestor( + Slapi_Backend *be, + const Slapi_DN *sdn, + Slapi_DN *ancestordn, + back_txn *txn, + int *err +) +{ + struct backentry *e = NULL; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> dn2ancestor \"%s\"\n", slapi_sdn_get_dn(sdn), 0, 0 ); + + /* stop when we get to "", or a backend suffix point */ + slapi_sdn_done(ancestordn); /* free any previous contents */ + slapi_sdn_get_backend_parent(sdn,ancestordn,be); + if ( !slapi_sdn_isempty(ancestordn) ) + { + Slapi_DN *newsdn = slapi_sdn_dup(ancestordn); + e = dn2entry_or_ancestor( be, newsdn, ancestordn, txn, err ); + slapi_sdn_free(&newsdn); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= dn2ancestor %p\n", e, 0, 0 ); + return( e ); +} + +/* + * Use uniqueid2entry or dn2entry to fetch an entry from the cache, + * make a copy of it, and stash it in the pblock. + */ +int +get_copy_of_entry(Slapi_PBlock *pb, const entry_address *addr, back_txn *txn, int plock_parameter, int must_exist) /* JCM - Move somewhere more appropriate */ +{ + int err= 0; + int rc= LDAP_SUCCESS; + backend *be; + struct backentry *entry; + + slapi_pblock_get( pb, SLAPI_BACKEND, &be); + + if( addr->uniqueid!=NULL) + { + entry = uniqueid2entry(be, addr->uniqueid, txn, &err ); + } + else + { + Slapi_DN sdn; + slapi_sdn_init_dn_byref (&sdn, addr->dn); /* We assume that the DN is not normalized */ + entry = dn2entry( be, &sdn, txn, &err ); + slapi_sdn_done (&sdn); + } + if ( 0 != err && DB_NOTFOUND != err ) + { + if(must_exist) + { + LDAPDebug( LDAP_DEBUG_ANY, "Operation error fetching %s (%s), error %d.\n", + addr->dn, (addr->uniqueid==NULL?"null":addr->uniqueid), err ); + } + rc= LDAP_OPERATIONS_ERROR; + } + else + { + /* If an entry is found, copy it into the PBlock. */ + if(entry!=NULL) + { + ldbm_instance *inst; + slapi_pblock_set( pb, plock_parameter, slapi_entry_dup(entry->ep_entry)); + inst = (ldbm_instance *) be->be_instance_info; + cache_return( &inst->inst_cache, &entry ); + } + } + /* JCMREPL - Free the backentry? */ + return rc; +} + +void +done_with_pblock_entry(Slapi_PBlock *pb, int plock_parameter) /* JCM - Move somewhere more appropriate */ +{ + Slapi_Entry *entry; + slapi_pblock_get( pb, plock_parameter, &entry); + if(entry!=NULL) + { + slapi_entry_free(entry); + entry= NULL; + slapi_pblock_set( pb, plock_parameter, entry); + } +} + diff --git a/ldap/servers/slapd/back-ldbm/entrystore.c b/ldap/servers/slapd/back-ldbm/entrystore.c new file mode 100644 index 00000000..46c011ae --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/entrystore.c @@ -0,0 +1,12 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +/* entrystore.c --- functions dealing with entries and their storage. + Put computed attributes, compression etc here */ + +#include "back-ldbm.h" + +/* Nothing here yet */ diff --git a/ldap/servers/slapd/back-ldbm/filterindex.c b/ldap/servers/slapd/back-ldbm/filterindex.c new file mode 100644 index 00000000..445f3195 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/filterindex.c @@ -0,0 +1,830 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* filterindex.c - generate the list of candidate entries from a filter */ + +#include "back-ldbm.h" +#include "../index_subsys.h" + +extern const char *indextype_PRESENCE; +extern const char *indextype_EQUALITY; +extern const char *indextype_APPROX; +extern const char *indextype_SUB; + +static IDList *ava_candidates(Slapi_PBlock *pb, backend *be, Slapi_Filter *f, int ftype, Slapi_Filter *nextf, int range, int *err); +static IDList *presence_candidates(Slapi_PBlock *pb, backend *be, Slapi_Filter *f, int *err); +static IDList *extensible_candidates(backend *be, Slapi_Filter *f, int *err); +static IDList *list_candidates(Slapi_PBlock *pb, backend *be, const char *base, Slapi_Filter *flist, int ftype, int *err); +static IDList *substring_candidates(Slapi_PBlock *pb, backend *be, Slapi_Filter *f, int *err); +static IDList * range_candidates( + Slapi_PBlock *pb, + backend *be, + char *type, + struct berval *low_val, + struct berval *high_val, + int *err +); +static IDList * +keys2idl( + backend *be, + char *type, + const char *indextype, + Slapi_Value **ivals, + int *err +); + +IDList * +filter_candidates( + Slapi_PBlock *pb, + backend *be, + const char *base, + Slapi_Filter *f, + Slapi_Filter *nextf, + int range, + int *err +) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + IDList *result; + int ftype; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> filter_candidates\n", 0, 0, 0 ); + + /* check if this is to be serviced by a virtual index */ + if(INDEX_FILTER_EVALUTED == index_subsys_evaluate_filter(f, (Slapi_DN*)slapi_be_getsuffix(be, 0), (IndexEntryList**)&result)) + { + LDAPDebug( LDAP_DEBUG_TRACE, "<= filter_candidates %lu (vattr)\n", + (u_long)IDL_NIDS(result), 0, 0 ); + return result; + } + + if (li->li_use_vlv) { + /* first, check to see if this particular filter node matches any + * vlv indexes we're keeping. if so, we can use that index + * instead. + */ + result = vlv_find_index_by_filter(be, base, f); + if (result) { + LDAPDebug( LDAP_DEBUG_TRACE, "<= filter_candidates %lu (vlv)\n", + (u_long)IDL_NIDS(result), 0, 0 ); + return result; + } + } + + result = NULL; + switch ( (ftype = slapi_filter_get_choice( f )) ) { + case LDAP_FILTER_EQUALITY: + LDAPDebug( LDAP_DEBUG_FILTER, "\tEQUALITY\n", 0, 0, 0 ); + result = ava_candidates( pb, be, f, LDAP_FILTER_EQUALITY, nextf, range, err ); + break; + + case LDAP_FILTER_SUBSTRINGS: + LDAPDebug( LDAP_DEBUG_FILTER, "\tSUBSTRINGS\n", 0, 0, 0 ); + result = substring_candidates( pb, be, f, err ); + break; + + case LDAP_FILTER_GE: + LDAPDebug( LDAP_DEBUG_FILTER, "\tGE\n", 0, 0, 0 ); + result = ava_candidates( pb, be, f, LDAP_FILTER_GE, nextf, range, + err ); + break; + + case LDAP_FILTER_LE: + LDAPDebug( LDAP_DEBUG_FILTER, "\tLE\n", 0, 0, 0 ); + result = ava_candidates( pb, be, f, LDAP_FILTER_LE, nextf, range, + err ); + break; + + case LDAP_FILTER_PRESENT: + LDAPDebug( LDAP_DEBUG_FILTER, "\tPRESENT\n", 0, 0, 0 ); + result = presence_candidates( pb, be, f, err ); + break; + + case LDAP_FILTER_APPROX: + LDAPDebug( LDAP_DEBUG_FILTER, "\tAPPROX\n", 0, 0, 0 ); + result = ava_candidates( pb, be, f, LDAP_FILTER_APPROX, nextf, + range, err ); + break; + + case LDAP_FILTER_EXTENDED: + LDAPDebug( LDAP_DEBUG_FILTER, "\tEXTENSIBLE\n", 0, 0, 0 ); + result = extensible_candidates( be, f, err ); + break; + + case LDAP_FILTER_AND: + LDAPDebug( LDAP_DEBUG_FILTER, "\tAND\n", 0, 0, 0 ); + result = list_candidates( pb, be, base, f, LDAP_FILTER_AND, err ); + break; + + case LDAP_FILTER_OR: + LDAPDebug( LDAP_DEBUG_FILTER, "\tOR\n", 0, 0, 0 ); + result = list_candidates( pb, be, base, f, LDAP_FILTER_OR, err ); + break; + + case LDAP_FILTER_NOT: + LDAPDebug( LDAP_DEBUG_FILTER, "\tNOT\n", 0, 0, 0 ); + result = idl_allids( be ); + break; + + default: + LDAPDebug( LDAP_DEBUG_FILTER, + "filter_candidates: unknown type 0x%X\n", + ftype, 0, 0 ); + break; + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= filter_candidates %lu\n", + (u_long)IDL_NIDS(result), 0, 0 ); + return( result ); +} + +static IDList * +ava_candidates( + Slapi_PBlock *pb, + backend *be, + Slapi_Filter *f, + int ftype, + Slapi_Filter *nextf, + int range, + int *err +) +{ + char *type, *indextype = NULL; + Slapi_Value sv; + struct berval *bval; + Slapi_Value **ivals; + IDList *idl; + void *pi; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> ava_candidates\n", 0, 0, 0 ); + + if ( slapi_filter_get_ava( f, &type, &bval ) != 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, " slapi_filter_get_ava failed\n", + 0, 0, 0 ); + return( NULL ); + } + +#ifdef LDAP_DEBUG + if ( LDAPDebugLevelIsSet( LDAP_DEBUG_TRACE )) { + char *op = NULL; + char buf[BUFSIZ]; + + switch ( ftype ) { + case LDAP_FILTER_GE: + op = ">="; + break; + case LDAP_FILTER_LE: + op = "<="; + break; + case LDAP_FILTER_EQUALITY: + op = "="; + break; + case LDAP_FILTER_APPROX: + op = "~="; + break; + } + LDAPDebug( LDAP_DEBUG_TRACE, " %s%s%s\n", type, op, + encode( bval, buf ) ); + } +#endif + + switch ( ftype ) { + case LDAP_FILTER_GE: + idl = range_candidates(pb, be, type, bval, NULL, err); + LDAPDebug( LDAP_DEBUG_TRACE, "<= ava_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0 ); + return( idl ); + case LDAP_FILTER_LE: + idl = range_candidates(pb, be, type, NULL, bval, err); + LDAPDebug( LDAP_DEBUG_TRACE, "<= ava_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0 ); + return( idl ); + case LDAP_FILTER_EQUALITY: + indextype = (char*)indextype_EQUALITY; + break; + case LDAP_FILTER_APPROX: + indextype = (char*)indextype_APPROX; + break; + } + + /* + * get the keys corresponding to this assertion value + */ + if ( slapi_attr_type2plugin( type, &pi ) != 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, " slapi_filter_get_ava no plugin\n", + 0, 0, 0 ); + return( NULL ); + } + + /* This code is result of performance anlysis; we are trying to + * optimize our equality filter processing -- mainly by limiting + * malloc/free calls. + * + * When the filter type is LDAP_FILTER_EQUALITY_FAST, the + * syntax_assertion2keys functions are passed a stack-based + * destination Slapi_Value array (ivals) that contains room + * for one key value with a fixed size buffer (also stack-based). + * If the buffer provided is not large enough, the + * syntax_assertion2keys function can alloc a new buffer (and + * reset ivals[0]->bv.bv_val) or alloc an entirely new ivals array. + */ + + if(ftype==LDAP_FILTER_EQUALITY) { + Slapi_Value tmp, *ptr[2], fake; + char buf[1024]; + + tmp.bv = *bval; + tmp.v_csnset=NULL; + fake.bv.bv_val=buf; + fake.bv.bv_len=sizeof(buf); + ptr[0]=&fake; + ptr[1]=NULL; + ivals=ptr; + + slapi_call_syntax_assertion2keys_ava_sv( pi, &tmp, (Slapi_Value ***)&ivals, LDAP_FILTER_EQUALITY_FAST); + idl = keys2idl( be, type, indextype, ivals, err ); + + /* We don't use valuearray_free here since the valueset, berval + * and value was all allocated at once in one big chunk for + * performance reasons + */ + if (fake.bv.bv_val != buf) { + slapi_ch_free((void**)&fake.bv.bv_val); + } + + /* Some syntax_assertion2keys functions may allocate a whole new + * ivals array. Free it if so. + */ + if (ivals != ptr) { + slapi_ch_free((void**)&ivals); + } + } else { + slapi_value_init_berval(&sv, bval); + ivals=NULL; + slapi_call_syntax_assertion2keys_ava_sv( pi, &sv, &ivals, ftype ); + value_done(&sv); + if ( ivals == NULL || *ivals == NULL ) { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= ava_candidates ALLIDS (no keys)\n", 0, 0, 0 ); + return( idl_allids( be ) ); + } + idl = keys2idl( be, type, indextype, ivals, err ); + valuearray_free( &ivals ); + LDAPDebug( LDAP_DEBUG_TRACE, "<= ava_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0 ); + } + return( idl ); +} + +static IDList * +presence_candidates( + Slapi_PBlock *pb, + backend *be, + Slapi_Filter *f, + int *err +) +{ + char *type; + IDList *idl; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> presence_candidates\n", 0, 0, 0 ); + + if ( slapi_filter_get_type( f, &type ) != 0 ) { + LDAPDebug( LDAP_DEBUG_ANY, " slapi_filter_get_type failed\n", + 0, 0, 0 ); + return( NULL ); + } + idl = index_read( be, type, indextype_PRESENCE, NULL, NULL, err ); + + if (idl != NULL && ALLIDS(idl) && strcasecmp(type, "nscpentrydn") == 0) { + /* try the equality index instead */ + LDAPDebug(LDAP_DEBUG_TRACE, + "fallback to eq index as pres index gave allids\n", + 0, 0, 0); + idl_free(idl); + idl = index_range_read(pb, be, type, indextype_EQUALITY, + SLAPI_OP_GREATER_OR_EQUAL, + NULL, NULL, 0, NULL, err); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= presence_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0 ); + return( idl ); +} + +static IDList * +extensible_candidates( + backend *be, + Slapi_Filter *f, + int *err +) +{ + IDList* idl = NULL; + Slapi_PBlock* pb = slapi_pblock_new(); + int mrOP = 0; + LDAPDebug (LDAP_DEBUG_TRACE, "=> extensible_candidates\n", 0, 0, 0); + if ( ! slapi_mr_filter_index (f, pb) && !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_QUERY_OPERATOR, &mrOP)) + { + switch (mrOP) + { + case SLAPI_OP_LESS: + case SLAPI_OP_LESS_OR_EQUAL: + case SLAPI_OP_EQUAL: + case SLAPI_OP_GREATER_OR_EQUAL: + case SLAPI_OP_GREATER: + { + IFP mrINDEX = NULL; + void* mrOBJECT = NULL; + struct berval** mrVALUES = NULL; + char* mrOID = NULL; + char* mrTYPE = NULL; + + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX); + slapi_pblock_get (pb, SLAPI_PLUGIN_OBJECT, &mrOBJECT); + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_VALUES, &mrVALUES); + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_OID, &mrOID); + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_TYPE, &mrTYPE); + + if (mrVALUES != NULL && *mrVALUES != NULL) + { + /* + * Compute keys for each of the values, individually. + * Search the index, for the computed keys. + * Collect the resulting IDs in idl. + */ + size_t n; + struct berval** val; + mrTYPE = slapi_attr_basetype (mrTYPE, NULL, 0); + for (n=0,val=mrVALUES; *val; ++n,++val) + { + struct berval** keys = NULL; + /* keys = mrINDEX (*val), conceptually. In detail: */ + struct berval* bvec[2]; + bvec[0] = *val; + bvec[1] = NULL; + if (slapi_pblock_set (pb, SLAPI_PLUGIN_OBJECT, mrOBJECT) || + slapi_pblock_set (pb, SLAPI_PLUGIN_MR_VALUES, bvec) || + mrINDEX (pb) || + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_KEYS, &keys)) + { + /* something went wrong. bail. */ + break; + } + else if (keys == NULL || keys[0] == NULL) + { + /* no keys */ + idl_free (idl); + idl = idl_allids (be); + } + else + { + IDList* idl2= NULL; + struct berval** key; + for (key = keys; *key != NULL; ++key) + { + IDList* idl3 = (mrOP == SLAPI_OP_EQUAL) ? + index_read (be, mrTYPE, mrOID, *key, NULL, err) : + index_range_read (pb, be, mrTYPE, mrOID, mrOP, *key, NULL, 0, NULL, err); + if (idl2 == NULL) + { + /* first iteration */ + idl2 = idl3; + } + else + { + IDList* tmp = idl_intersection (be, idl2, idl3); + idl_free (idl2); + idl_free (idl3); + idl2 = tmp; + } + if (idl2 == NULL) break; /* look no further */ + } + if (idl == NULL) + { + idl = idl2; + } + else if (idl2 != NULL) + { + IDList* tmp = idl_union (be, idl, idl2); + idl_free (idl); + idl_free (idl2); + idl = tmp; + } + } + } + slapi_ch_free((void**)&mrTYPE); + goto return_idl; /* possibly no matches */ + } + } + break; + default: + /* unsupported query operator */ + break; + } + } + if (idl == NULL) + { + /* this filter isn't indexed */ + idl = idl_allids (be); /* all entries are candidates */ + } +return_idl: + slapi_pblock_destroy (pb); + LDAPDebug (LDAP_DEBUG_TRACE, "<= extensible_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0); + return idl; +} + +static int +slapi_berval_reverse_cmp(const struct berval *a, const struct berval *b) +{ + return slapi_berval_cmp(b, a); +} + +static IDList * +range_candidates( + Slapi_PBlock *pb, + backend *be, + char *type, + struct berval *low_val, + struct berval *high_val, + int *err +) +{ + IDList *idl; + struct berval *low = NULL, *high = NULL; + struct berval **lows = NULL, **highs = NULL; + void *pi; + + LDAPDebug(LDAP_DEBUG_TRACE, "=> range_candidates attr=%s\n", type, 0, 0); + + /* + * get the keys corresponding to the assertion values + */ + + if ( slapi_attr_type2plugin( type, &pi ) != 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, " slapi_filter_get_ava no plugin\n", + 0, 0, 0 ); + return( NULL ); + } + + if (low_val != NULL) { + slapi_call_syntax_assertion2keys_ava(pi, low_val, &lows, LDAP_FILTER_EQUALITY); + if (lows == NULL || *lows == NULL) { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= range_candidates ALLIDS (no keys)\n", 0, 0, 0 ); + return( idl_allids( be ) ); + } + low = attr_value_lowest(lows, slapi_berval_reverse_cmp); + } + + if (high_val != NULL) { + slapi_call_syntax_assertion2keys_ava(pi, high_val, &highs, LDAP_FILTER_EQUALITY); + if (highs == NULL || *highs == NULL) { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= range_candidates ALLIDS (no keys)\n", 0, 0, 0 ); + if (lows) ber_bvecfree(lows); + return( idl_allids( be ) ); + } + high = attr_value_lowest(highs, slapi_berval_cmp); + } + + if (low == NULL) { + idl = index_range_read(pb, be, type, (char*)indextype_EQUALITY, + SLAPI_OP_LESS_OR_EQUAL, + high, NULL, 0, NULL, err); + } else if (high == NULL) { + idl = index_range_read(pb, be, type, (char*)indextype_EQUALITY, + SLAPI_OP_GREATER_OR_EQUAL, + low, NULL, 0, NULL, err); + } else { + idl = index_range_read(pb, be, type, (char*)indextype_EQUALITY, + SLAPI_OP_GREATER_OR_EQUAL, + low, high, 1, NULL, err); + } + + if (lows) ber_bvecfree(lows); + if (highs) ber_bvecfree(highs); + + LDAPDebug( LDAP_DEBUG_TRACE, "<= range_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0 ); + + return idl; +} + +static IDList * +list_candidates( + Slapi_PBlock *pb, + backend *be, + const char *base, + Slapi_Filter *flist, + int ftype, + int *err +) +{ + IDList *idl, *tmp, *tmp2; + Slapi_Filter *f, *nextf, *f_head; + int range = 0; + int isnot; + int f_count = 0, le_count = 0, ge_count = 0, is_bounded_range = 1; + struct berval *low_val = NULL, *high_val = NULL; + char *t1; + Slapi_Filter *fpairs[2] = {NULL, NULL}; /* low, high */ + char *tpairs[2] = {NULL, NULL}; + struct berval *vpairs[2] = {NULL, NULL}; + int is_and = 0; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> list_candidates 0x%x\n", ftype, 0, 0 ); + + /* + * Optimize bounded range queries such as (&(cn>=A)(cn<=B)). + * Could be better by matching pairs in a longer list + * but for now support only a single pair. + */ + if (ftype != LDAP_FILTER_AND) + { + is_bounded_range = 0; + } + for ( f = slapi_filter_list_first( flist ); + f != NULL && is_bounded_range; + f = slapi_filter_list_next( flist, f ) ) { + f_count++; + switch (slapi_filter_get_choice(f)) { + case LDAP_FILTER_GE: + if ( slapi_filter_get_ava(f, &t1, &low_val) != 0 ) { + is_bounded_range = 0; + continue; + } + ge_count++; + if (NULL == fpairs[0]) + { + fpairs[0] = f; + tpairs[0] = slapi_ch_strdup(t1); + vpairs[0] = slapi_ch_bvdup(low_val); + } + else if (NULL != fpairs[1] && + slapi_attr_type_cmp(tpairs[1], t1, SLAPI_TYPE_CMP_EXACT) != 0) + { + fpairs[0] = f; + slapi_ch_free_string(&tpairs[0]); + tpairs[0] = slapi_ch_strdup(t1); + slapi_ch_bvfree(&vpairs[0]); + vpairs[0] = slapi_ch_bvdup(low_val); + } + break; + case LDAP_FILTER_LE: + if ( slapi_filter_get_ava(f, &t1, &high_val) != 0 ) { + is_bounded_range = 0; + continue; + } + le_count++; + if (NULL == fpairs[1]) + { + fpairs[1] = f; + tpairs[1] = slapi_ch_strdup(t1); + vpairs[1] = slapi_ch_bvdup(high_val); + } + else if (NULL != fpairs[0] && + slapi_attr_type_cmp(tpairs[0], t1, SLAPI_TYPE_CMP_EXACT) != 0) + { + fpairs[1] = f; + slapi_ch_free_string(&tpairs[1]); + tpairs[1] = slapi_ch_strdup(t1); + slapi_ch_bvfree(&vpairs[1]); + vpairs[1] = slapi_ch_bvdup(high_val); + } + break; + default: + continue; + } + } + if (ftype == LDAP_FILTER_AND && f_count > 1) + { + is_and = 1; + } + slapi_pblock_set(pb, SLAPI_SEARCH_IS_AND, &is_and); + if (le_count != 1 || ge_count != 1 || f_count != 2) + { + is_bounded_range = 0; + } + if (NULL == fpairs[0] || NULL == fpairs[1]) + { + fpairs[0] = fpairs[1] = NULL; + slapi_ch_free_string(&tpairs[0]); + slapi_ch_bvfree(&vpairs[0]); + slapi_ch_free_string(&tpairs[1]); + slapi_ch_bvfree(&vpairs[1]); + is_bounded_range = 0; + } + if (is_bounded_range) { + idl = range_candidates(pb, be, tpairs[0], vpairs[0], vpairs[1], err); + LDAPDebug( LDAP_DEBUG_TRACE, "<= list_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0 ); + goto out; + } + + idl = NULL; + nextf = NULL; + isnot = 0; + for ( f_head = f = slapi_filter_list_first( flist ); f != NULL; + f = slapi_filter_list_next( flist, f ) ) { + + /* Look for NOT foo type filter elements where foo is simple equality */ + isnot = (LDAP_FILTER_NOT == slapi_filter_get_choice( f )) && + (LDAP_FILTER_AND == ftype && + (LDAP_FILTER_EQUALITY == slapi_filter_get_choice(slapi_filter_list_first(f)))); + + if (isnot) { + /* if this is the first filter we have an allid search anyway, so bail */ + if(f == f_head) + { + idl = idl_allids( be ); + break; + } + + /* Fetch the IDL for foo */ + /* Later we'll remember to call idl_notin() */ + LDAPDebug( LDAP_DEBUG_TRACE,"NOT filter\n", 0, 0, 0 ); + tmp = ava_candidates( pb, be, slapi_filter_list_first(f), LDAP_FILTER_EQUALITY, nextf, range, err ); + } else { + if (fpairs[0] == f) + { + continue; + } + else if (fpairs[1] == f) + { + tmp = range_candidates(pb, be, tpairs[0], + vpairs[0], vpairs[1], err); + if (tmp == NULL && ftype == LDAP_FILTER_AND) + { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= list_candidates NULL\n", 0, 0, 0 ); + idl_free( idl ); + idl = NULL; + goto out; + } + } + /* Proceed as normal */ + else if ( (tmp = filter_candidates( pb, be, base, f, nextf, range, err )) + == NULL && ftype == LDAP_FILTER_AND ) { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= list_candidates NULL\n", 0, 0, 0 ); + idl_free( idl ); + idl = NULL; + goto out; + } + } + + tmp2 = idl; + if ( idl == NULL ) { + idl = tmp; + if ( (ftype == LDAP_FILTER_AND) && ((idl == NULL) || + (idl_length(idl) <= FILTER_TEST_THRESHOLD))) + break; /* We can exit the loop now, since the candidate list is small already */ + } else if ( ftype == LDAP_FILTER_AND ) { + if (isnot) { + IDList *new_idl = NULL; + int notin_result = 0; + notin_result = idl_notin( be, idl, tmp, &new_idl ); + if (notin_result) { + idl_free(idl); + idl = new_idl; + } + } else { + idl = idl_intersection(be, idl, tmp); + idl_free( tmp2 ); + } + idl_free( tmp ); + /* stop if the list has gotten too small */ + if ((idl == NULL) || + (idl_length(idl) <= FILTER_TEST_THRESHOLD)) + break; + } else { + idl = idl_union( be, idl, tmp ); + idl_free( tmp ); + idl_free( tmp2 ); + /* stop if we're already committed to an exhaustive + * search. :( + */ + if (idl_is_allids(idl)) + break; + } + + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= list_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0 ); +out: + is_and = 0; + slapi_pblock_set(pb, SLAPI_SEARCH_IS_AND, &is_and); + slapi_ch_free_string(&tpairs[0]); + slapi_ch_bvfree(&vpairs[0]); + slapi_ch_free_string(&tpairs[1]); + slapi_ch_bvfree(&vpairs[1]); + return( idl ); +} + +static IDList * +substring_candidates( + Slapi_PBlock *pb, + backend *be, + Slapi_Filter *f, + int *err +) +{ + char *type, *initial, *final; + char **any; + IDList *idl; + void *pi; + Slapi_Value **ivals; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> sub_candidates\n", 0, 0, 0 ); + + if (slapi_filter_get_subfilt( f, &type, &initial, &any, &final ) != 0) { + LDAPDebug( LDAP_DEBUG_ANY, " slapi_filter_get_subfilt fails\n", + 0, 0, 0 ); + return( NULL ); + } + + /* + * get the index keys corresponding to the substring + * assertion values + */ + if ( slapi_attr_type2plugin( type, &pi ) != 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, " sub_candidates no plugin\n", + 0, 0, 0 ); + return( NULL ); + } + slapi_call_syntax_assertion2keys_sub_sv( pi, initial, any, final, &ivals ); + if ( ivals == NULL || *ivals == NULL ) { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= sub_candidates ALLIDS (no keys)\n", 0, 0, 0 ); + return( idl_allids( be ) ); + } + + /* + * look up each key in the index, ANDing the resulting + * IDLists together. + */ + idl = keys2idl( be, type, indextype_SUB, ivals, err ); + valuearray_free( &ivals ); + + LDAPDebug( LDAP_DEBUG_TRACE, "<= sub_candidates %lu\n", + (u_long)IDL_NIDS(idl), 0, 0 ); + return( idl ); +} + +static IDList * +keys2idl( + backend *be, + char *type, + const char *indextype, + Slapi_Value **ivals, + int *err +) +{ + IDList *idl; + int i; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> keys2idl type %s indextype %s\n", + type, indextype, 0 ); + idl = NULL; + for ( i = 0; ivals[i] != NULL; i++ ) { + IDList *idl2; + + idl2 = index_read( be, type, indextype, slapi_value_get_berval(ivals[i]), NULL, err ); + +#ifdef LDAP_DEBUG + /* XXX if ( slapd_ldap_debug & LDAP_DEBUG_TRACE ) { XXX */ + { + char buf[BUFSIZ]; + + LDAPDebug( LDAP_DEBUG_TRACE, + " ival[%d] = \"%s\" => %lu IDs\n", i, + encode( slapi_value_get_berval(ivals[i]), buf ), (u_long)IDL_NIDS(idl2) ); + } +#endif + if ( idl2 == NULL ) { + idl_free( idl ); + idl = NULL; + break; + } + + if (idl == NULL) { + idl = idl2; + } else { + IDList *tmp; + + tmp = idl; + idl = idl_intersection(be, idl, idl2); + idl_free( idl2 ); + idl_free( tmp ); + if ( idl == NULL ) { + break; + } + } + } + + return( idl ); +} diff --git a/ldap/servers/slapd/back-ldbm/findentry.c b/ldap/servers/slapd/back-ldbm/findentry.c new file mode 100644 index 00000000..6565b279 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/findentry.c @@ -0,0 +1,284 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* findentry.c - find a database entry, obeying referrals (& aliases?) */ + +#include "back-ldbm.h" + +int +check_entry_for_referral(Slapi_PBlock *pb, Slapi_Entry *entry, char *matched, const char *callingfn) /* JCM - Move somewhere more appropriate */ +{ + int rc=0, i=0, numValues=0; + Slapi_Attr *attr; + + /* if the entry is a referral send the referral */ + if ( slapi_entry_attr_find( entry, "ref", &attr ) == 0 ) + { + Slapi_Value *val=NULL; + struct berval **refscopy=NULL; + struct berval **url=NULL; + slapi_attr_get_numvalues(attr, &numValues ); + if(numValues > 0) { + url=(struct berval **) slapi_ch_malloc((numValues + 1) * sizeof(struct berval*)); + } + for (i = slapi_attr_first_value(attr, &val); i != -1; + i = slapi_attr_next_value(attr, i, &val)) { + url[i]=(struct berval*)slapi_value_get_berval(val); + } + url[numValues]=NULL; + refscopy = ref_adjust( pb, url, slapi_entry_get_sdn(entry), 0 ); /* JCM - What's this PBlock* for? */ + slapi_send_ldap_result( pb, LDAP_REFERRAL, matched, NULL, 0, refscopy ); + LDAPDebug( LDAP_DEBUG_TRACE, + "<= %s sent referral to (%s) for (%s)\n", + callingfn, + refscopy ? refscopy[0]->bv_val : "", + slapi_entry_get_dn(entry)); + if ( refscopy != NULL ) + { + ber_bvecfree( refscopy ); + } + if( url != NULL) { + slapi_ch_free( (void **)&url ); + } + rc= 1; + } + return rc; +} + +static struct backentry * +find_entry_internal_dn( + Slapi_PBlock *pb, + backend *be, + const Slapi_DN *sdn, + int lock, + back_txn *txn, + int really_internal +) +{ + struct backentry *e; + int managedsait = 0; + int err; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + size_t tries = 0; + + /* get the managedsait ldap message control */ + slapi_pblock_get( pb, SLAPI_MANAGEDSAIT, &managedsait ); + + while ( (tries < LDBM_CACHE_RETRY_COUNT) && + (e = dn2entry( be, sdn, txn, &err )) != NULL ) + { + /* + * we found the entry. if the managedsait control is set, + * we return the entry. if managedsait is not set, we check + * for the presence of a ref attribute, returning to the + * client a referral to the ref'ed entry if a ref is present, + * returning the entry to the caller if not. + */ + if ( !managedsait && !really_internal) { + /* see if the entry is a referral */ + if(check_entry_for_referral(pb, e->ep_entry, NULL, "find_entry_internal_dn")) + { + cache_return( &inst->inst_cache, &e ); + return( NULL ); + } + } + + /* + * we'd like to return the entry. lock it if requested, + * retrying if necessary. + */ + + /* wait for entry modify lock */ + if ( !lock || cache_lock_entry( &inst->inst_cache, e ) == 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= find_entry_internal_dn found (%s)\n", slapi_sdn_get_dn(sdn), 0, 0 ); + return( e ); + } + /* + * this entry has been deleted - see if it was actually + * replaced with a new copy, and try the whole thing again. + */ + LDAPDebug( LDAP_DEBUG_ARGS, + " find_entry_internal_dn retrying (%s)\n", slapi_sdn_get_dn(sdn), 0, 0 ); + cache_return( &inst->inst_cache, &e ); + tries++; + } + if (tries >= LDBM_CACHE_RETRY_COUNT) { + LDAPDebug( LDAP_DEBUG_ANY,"find_entry_internal_dn retry count exceeded (%s)\n", slapi_sdn_get_dn(sdn), 0, 0 ); + } + /* + * there is no such entry in this server. see how far we + * can match, and check if that entry contains a referral. + * if it does and managedsait is not set, we return the + * referral to the client. if it doesn't, or managedsait + * is set, we return no such object. + */ + if (!really_internal) { + struct backentry *me; + Slapi_DN ancestordn= {0}; + me= dn2ancestor(pb->pb_backend,sdn,&ancestordn,txn,&err); + if ( !managedsait && me != NULL ) { + /* if the entry is a referral send the referral */ + if(check_entry_for_referral(pb, me->ep_entry, (char*)slapi_sdn_get_dn(&ancestordn), "find_entry_internal_dn")) + { + cache_return( &inst->inst_cache, &me ); + slapi_sdn_done(&ancestordn); + return( NULL ); + } + /* else fall through to no such object */ + } + + /* entry not found */ + slapi_send_ldap_result( pb, ( 0 == err || DB_NOTFOUND == err ) ? + LDAP_NO_SUCH_OBJECT : LDAP_OPERATIONS_ERROR, (char*)slapi_sdn_get_dn(&ancestordn), NULL, + 0, NULL ); + slapi_sdn_done(&ancestordn); + cache_return( &inst->inst_cache, &me ); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= find_entry_internal_dn not found (%s)\n", + slapi_sdn_get_dn(sdn), 0, 0 ); + return( NULL ); +} + +/* Note that this function does not issue any referals. + It should only be called in case of 5.0 replicated operation + which should not be referred. + */ +static struct backentry * +find_entry_internal_uniqueid( + Slapi_PBlock *pb, + backend *be, + const char *uniqueid, + int lock, + back_txn *txn +) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + struct backentry *e; + int err; + size_t tries = 0; + + while ( (tries < LDBM_CACHE_RETRY_COUNT) && + (e = uniqueid2entry(be, uniqueid, txn, &err )) + != NULL ) { + + /* + * we'd like to return the entry. lock it if requested, + * retrying if necessary. + */ + + /* wait for entry modify lock */ + if ( !lock || cache_lock_entry( &inst->inst_cache, e ) == 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= find_entry_internal_uniqueid found; uniqueid = (%s)\n", + uniqueid, 0, 0 ); + return( e ); + } + /* + * this entry has been deleted - see if it was actually + * replaced with a new copy, and try the whole thing again. + */ + LDAPDebug( LDAP_DEBUG_ARGS, + " find_entry_internal_uniqueid retrying; uniqueid = (%s)\n", + uniqueid, 0, 0 ); + cache_return( &inst->inst_cache, &e ); + tries++; + } + if (tries >= LDBM_CACHE_RETRY_COUNT) { + LDAPDebug( LDAP_DEBUG_ANY, + "find_entry_internal_uniqueid retry count exceeded; uniqueid = (%s)\n", + uniqueid , 0, 0 ); + } + + /* entry not found */ + slapi_send_ldap_result( pb, ( 0 == err || DB_NOTFOUND == err ) ? + LDAP_NO_SUCH_OBJECT : LDAP_OPERATIONS_ERROR, NULL /* matched */, NULL, + 0, NULL ); + LDAPDebug( LDAP_DEBUG_TRACE, + "<= find_entry_internal not found; uniqueid = (%s)\n", + uniqueid, 0, 0 ); + return( NULL ); +} + +static struct backentry * +find_entry_internal( + Slapi_PBlock *pb, + Slapi_Backend *be, + const entry_address *addr, + int lock, + back_txn *txn, + int really_internal +) +{ + /* check if we should search based on uniqueid or dn */ + if (addr->uniqueid!=NULL) + { + LDAPDebug( LDAP_DEBUG_TRACE, "=> find_entry_internal (uniqueid=%s) lock %d\n", + addr->uniqueid, lock, 0 ); + return (find_entry_internal_uniqueid (pb, be, addr->uniqueid, lock, txn)); + } + else + { + Slapi_DN sdn; + struct backentry *entry; + + slapi_sdn_init_dn_ndn_byref (&sdn, addr->dn); /* normalized by front end */ + LDAPDebug( LDAP_DEBUG_TRACE, "=> find_entry_internal (dn=%s) lock %d\n", + addr->dn, lock, 0 ); + entry = find_entry_internal_dn (pb, be, &sdn, lock, txn, really_internal); + slapi_sdn_done (&sdn); + return entry; + } + +} + +struct backentry * +find_entry( + Slapi_PBlock *pb, + Slapi_Backend *be, + const entry_address *addr, + back_txn *txn +) +{ + return( find_entry_internal( pb, be, addr, 0/*!lock*/, txn, 0/*!really_internal*/ ) ); +} + +struct backentry * +find_entry2modify( + Slapi_PBlock *pb, + Slapi_Backend *be, + const entry_address *addr, + back_txn *txn +) +{ + return( find_entry_internal( pb, be, addr, 1/*lock*/, txn, 0/*!really_internal*/ ) ); +} + +/* New routines which do not do any referral stuff. + Call these if all you want to do is get pointer to an entry + and certainly do not want any side-effects relating to client ! */ + +struct backentry * +find_entry_only( + Slapi_PBlock *pb, + Slapi_Backend *be, + const entry_address *addr, + back_txn *txn +) +{ + return( find_entry_internal( pb, be, addr, 0/*!lock*/, txn, 1/*really_internal*/ ) ); +} + +struct backentry * +find_entry2modify_only( + Slapi_PBlock *pb, + Slapi_Backend *be, + const entry_address *addr, + back_txn *txn +) +{ + return( find_entry_internal( pb, be, addr, 1/*lock*/, txn, 1/*really_internal*/ ) ); +} diff --git a/ldap/servers/slapd/back-ldbm/haschildren.c b/ldap/servers/slapd/back-ldbm/haschildren.c new file mode 100644 index 00000000..6e2fa9d9 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/haschildren.c @@ -0,0 +1,8 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* haschildren.c - tell if an entry has kids or not */ + + diff --git a/ldap/servers/slapd/back-ldbm/id2entry.c b/ldap/servers/slapd/back-ldbm/id2entry.c new file mode 100644 index 00000000..7c5a00c2 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/id2entry.c @@ -0,0 +1,250 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* id2entry.c - routines to deal with the id2entry index */ + +#include "back-ldbm.h" + +/* + * The caller MUST check for DB_LOCK_DEADLOCK and DB_RUNRECOVERY returned + */ +int +id2entry_add_ext( backend *be, struct backentry *e, back_txn *txn, int encrypt ) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + DB *db = NULL; + DB_TXN *db_txn = NULL; + DBT data = {0}; + DBT key = {0}; + int len, rc; + char temp_id[sizeof(ID)]; + struct backentry *encrypted_entry = NULL; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> id2entry_add( %lu, \"%s\" )\n", + (u_long)e->ep_id, backentry_get_ndn(e), 0 ); + + if ( (rc = dblayer_get_id2entry( be, &db )) != 0 ) { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open/create id2entry\n", + 0, 0, 0 ); + return( -1 ); + } + + id_internal_to_stored(e->ep_id,temp_id); + + key.dptr = temp_id; + key.dsize = sizeof(temp_id); + + /* Encrypt attributes in this entry if necessary */ + if (encrypt) { + rc = attrcrypt_encrypt_entry(be, e, &encrypted_entry); + if (rc) { + LDAPDebug( LDAP_DEBUG_ANY, "attrcrypt_encrypt_entry failed in id2entry_add\n", + 0, 0, 0 ); + return ( -1 ); + } + } + + { + Slapi_Entry *entry_to_use = encrypted_entry ? encrypted_entry->ep_entry : e->ep_entry; + data.dptr = slapi_entry2str_with_options( entry_to_use, &len, SLAPI_DUMP_STATEINFO | SLAPI_DUMP_UNIQUEID); + data.dsize = len + 1; + /* If we had an encrypted entry, we no longer need it */ + if (encrypted_entry) { + backentry_free(&encrypted_entry); + } + } + + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + + /* call pre-entry-store plugin */ + plugin_call_entrystore_plugins( (char **) &data.dptr, &data.dsize ); + + /* store it */ + rc = db->put( db, db_txn, &key, &data, 0); + /* DBDB looks like we're freeing memory allocated by another DLL, which is bad */ + free( data.dptr ); + + dblayer_release_id2entry( be, db ); + + if (0 == rc) + { + /* DBDB the fact that we don't check the return code here is + * indicitive that there may be a latent race condition lurking + * ---what happens if the entry is already in the cache by this point? + */ + /* + * For ldbm_back_add and ldbm_back_modify, this entry had been already + * reserved as a tentative entry. So, it should be safe. + * For ldbm_back_modify, the original entry having the same dn/id + * should be in the cache. Thus, this entry e won't be put into the + * entry cache. It'll be added by cache_replace. + */ + (void) cache_add( &inst->inst_cache, e, NULL ); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= id2entry_add %d\n", rc, 0, 0 ); + return( rc ); +} + +int +id2entry_add( backend *be, struct backentry *e, back_txn *txn ) +{ + return id2entry_add_ext(be,e,txn,1); +} + +/* + * The caller MUST check for DB_LOCK_DEADLOCK and DB_RUNRECOVERY returned + */ +int +id2entry_delete( backend *be, struct backentry *e, back_txn *txn ) +{ + DB *db = NULL; + DB_TXN *db_txn = NULL; + DBT key = {0}; + int rc; + char temp_id[sizeof(ID)]; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> id2entry_delete( %lu, \"%s\" )\n", + (u_long)e->ep_id, backentry_get_ndn(e), 0 ); + + if ( (rc = dblayer_get_id2entry( be, &db )) != 0 ) { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open/create id2entry\n", + 0, 0, 0 ); + return( -1 ); + } + + id_internal_to_stored(e->ep_id,temp_id); + + key.dptr = temp_id; + key.dsize = sizeof(temp_id); + + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + + rc = db->del( db,db_txn,&key,0 ); + dblayer_release_id2entry( be, db ); + + LDAPDebug( LDAP_DEBUG_TRACE, "<= id2entry_delete %d\n", rc, 0, 0 ); + return( rc ); +} + +struct backentry * +id2entry( backend *be, ID id, back_txn *txn, int *err ) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + DB *db = NULL; + DB_TXN *db_txn = NULL; + DBT key = {0}; + DBT data = {0}; + struct backentry *e; + Slapi_Entry *ee; + char temp_id[sizeof(ID)]; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> id2entry( %lu )\n", (u_long)id, 0, 0 ); + + if ( (e = cache_find_id( &inst->inst_cache, id )) != NULL ) { + LDAPDebug( LDAP_DEBUG_TRACE, "<= id2entry %p (cache)\n", e, 0, + 0 ); + return( e ); + } + + if ( (*err = dblayer_get_id2entry( be, &db )) != 0 ) { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open id2entry err %d\n", + *err, 0, 0 ); + return( NULL ); + } + + + id_internal_to_stored(id,temp_id); + + key.data = temp_id; + key.size = sizeof(temp_id); + + /* DBDB need to improve this, we're mallocing, freeing, all over the place here */ + data.flags = DB_DBT_MALLOC; + + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + do { + *err = db->get( db, db_txn, &key, &data, 0 ); + if ( 0 != *err && + DB_NOTFOUND != *err && DB_LOCK_DEADLOCK != *err ) + { + LDAPDebug( LDAP_DEBUG_ANY, "id2entry error %d\n", + *err, 0, 0 ); + } + } + while ( DB_LOCK_DEADLOCK == *err && txn == NULL ); + + if ( 0 != *err && DB_NOTFOUND != *err && DB_LOCK_DEADLOCK != *err ) + { + LDAPDebug( LDAP_DEBUG_ANY, "id2entry get error %d\n", + *err, 0, 0 ); + dblayer_release_id2entry( be, db ); + return( NULL ); + } + + if ( data.dptr == NULL ) { + LDAPDebug( LDAP_DEBUG_TRACE, "<= id2entry( %lu ) not found\n", + (u_long)id, 0, 0 ); + dblayer_release_id2entry( be, db ); + return( NULL ); + } + + /* call post-entry plugin */ + plugin_call_entryfetch_plugins( (char **) &data.dptr, &data.dsize ); + + if ( (ee = slapi_str2entry( data.dptr, 0 )) != NULL ) { + int retval = 0; + struct backentry *imposter = NULL; + + PR_ASSERT(slapi_entry_get_uniqueid(ee) != NULL); /* All entries should have uniqueids */ + e = backentry_init( ee ); /* ownership of the entry is passed into the backentry */ + e->ep_id = id; + + /* Decrypt any encrypted attributes in this entry, before adding it to the cache */ + retval = attrcrypt_decrypt_entry(be, e); + if (retval) { + LDAPDebug( LDAP_DEBUG_ANY, "attrcrypt_decrypt_entry failed in id2entry\n", + 0, 0, 0 ); + } + + retval = cache_add( &inst->inst_cache, e, &imposter ); + if (1 == retval) { + /* This means that someone else put the entry in the cache + while we weren't looking ! So, we need to use the pointer + returned and free the one we made earlier */ + if (imposter) + { + backentry_free(&e); + e = imposter; + } + } else if (-1 == retval) { + /* the entry is in idtable but not in dntable, i.e., the entry + * could have been renamed */ + LDAPDebug( LDAP_DEBUG_TRACE, + "id2entry: failed to put entry (id %lu, dn %s) into entry cache\n", + (u_long)id, backentry_get_ndn(e), 0 ); + } + } else { + LDAPDebug( LDAP_DEBUG_ANY, "str2entry returned NULL for id %lu, string=\"%s\"\n", (u_long)id, (char*)data.data, 0); + e = NULL; + } + + free( data.data ); + + dblayer_release_id2entry( be, db ); + + LDAPDebug( LDAP_DEBUG_TRACE, "<= id2entry( %lu ) %p (disk)\n", (u_long)id, e, + 0 ); + return( e ); +} + diff --git a/ldap/servers/slapd/back-ldbm/idl.c b/ldap/servers/slapd/back-ldbm/idl.c new file mode 100644 index 00000000..df97a979 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/idl.c @@ -0,0 +1,1599 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* idl.c - ldap id list handling routines */ + +#include "back-ldbm.h" + +/* + * Disable idl locking since it causes unbreakable deadlock. + */ +#undef IDL_LOCKING_ENABLE + +static int idl_delete( IDList **idl, ID id ) ; +static void make_cont_key( DBT *contkey, DBT *key, ID id ); +static int idl_insert_maxids( IDList **idl, ID id, int maxids ); + +/* for the cache of open index files */ +struct idl_private { + int idl_maxids; /* Number of IDS in a block */ + int idl_maxindirect; /* Number of blocks allowed */ + size_t idl_allidslimit; /* Max number of IDs before it turns to allids */ +#ifdef IDL_LOCKING_ENABLE + PRRWLock *idl_rwlock; +#endif +}; + +static int idl_tune = DEFAULT_IDL_TUNE; /* tuning parameters for IDL code */ +#define IDL_TUNE_BSEARCH 1 /* do a binary search when inserting into an IDL */ +#define IDL_TUNE_NOPAD 2 /* Don't pad IDLs with space at the end */ + +void idl_old_set_tune(int val) +{ + idl_tune = val; +} + +int idl_old_get_tune() { + return idl_tune; +} + +size_t idl_old_get_allidslimit(struct attrinfo *a) +{ + idl_private *priv = NULL; + + PR_ASSERT(NULL != a); + PR_ASSERT(NULL != a->ai_idl); + + priv = a->ai_idl; + + return priv->idl_allidslimit; +} + +static void idl_init_maxids(struct ldbminfo *li,idl_private *priv) +{ + const size_t blksize = dblayer_get_optimal_block_size(li); + + if (0 == li->li_allidsthreshold) { + li->li_allidsthreshold = DEFAULT_ALLIDSTHRESHOLD; + } + priv->idl_maxids = (blksize / sizeof(ID)) - 2; + priv->idl_maxindirect = (li->li_allidsthreshold / priv->idl_maxids) + 1; + priv->idl_allidslimit = (priv->idl_maxids * priv->idl_maxindirect); + LDAPDebug (LDAP_DEBUG_ARGS, + "idl_init_private: blksize %lu, maxids %i, maxindirect %i\n", + (unsigned long)blksize, priv->idl_maxids, priv->idl_maxindirect); +} + +/* routine to initialize the private data used by the IDL code per-attribute */ +int idl_old_init_private(backend *be,struct attrinfo *a) +{ + idl_private *priv = NULL; + + PR_ASSERT(NULL != a); + PR_ASSERT(NULL == a->ai_idl); + + priv = (idl_private*) slapi_ch_malloc(sizeof(idl_private)); + if (NULL == priv) { + return -1; /* Memory allocation failure */ + } + { + priv->idl_maxids = 0; + priv->idl_maxindirect = 0; + } +#ifdef IDL_LOCKING_ENABLE + priv->idl_rwlock = PR_NewRWLock(PR_RWLOCK_RANK_NONE, "idl lock"); + + if (NULL == priv->idl_rwlock) { + slapi_ch_free((void**)&priv); + return -1; + } +#endif + a->ai_idl = (void*)priv; + return 0; +} + +/* routine to release resources used by IDL private data structure */ +int idl_old_release_private(struct attrinfo *a) +{ + PR_ASSERT(NULL != a); + if (NULL != a->ai_idl) + { +#ifdef IDL_LOCKING_ENABLE + idl_private *priv = a->ai_idl; + PR_ASSERT(NULL != priv->idl_rwlock); + PR_DestroyRWLock(priv->idl_rwlock); +#endif + free( a->ai_idl ); + } + return 0; +} + +/* Locks one IDL so we can modify it knowing that + * nobody else is trying to do so at the same time + * also called by readers, since they need to be blocked + * when they read to avoid them seeing inconsistent data + * This is not really necessary for update operations + * today because they are already serialized by a lock + * at the backend level but is still necessary to + * stop concurrent access by one update thread and + * some other search threads + */ + +#ifdef IDL_LOCKING_ENABLE +static void idl_Wlock_list(idl_private *priv, DBT *key) +{ + PRRWLock *lock = NULL; + + PR_ASSERT(NULL != priv); + lock = priv->idl_rwlock; + PR_ASSERT(NULL != lock); + + PR_RWLock_Wlock(lock); +} + +static void idl_Rlock_list(idl_private *priv, DBT *key) +{ + PRRWLock *lock = NULL; + + PR_ASSERT(NULL != priv); + lock = priv->idl_rwlock; + PR_ASSERT(NULL != lock); + + PR_RWLock_Rlock(lock); +} + +static void idl_unlock_list(idl_private *priv, DBT *key) +{ + PRRWLock *lock = NULL; + + PR_ASSERT(NULL != priv); + lock = priv->idl_rwlock; + PR_ASSERT(NULL != lock); + + PR_RWLock_Unlock(lock); +} +#endif + +#ifndef IDL_LOCKING_ENABLE +#define idl_Wlock_list(idl,dbt) +#define idl_Rlock_list(idl,dbt) +#define idl_unlock_list(idl,dbt) +#endif + +/* + * idl_fetch_one - fetch a single IDList from the database and return a + * pointer to it. + * + * this routine always propagates errors other than DB_LOCK_DEADLOCK. + * for DB_LOCK_DEADLOCK, it propagates the error if called inside a + * transaction. if called not inside a transaction, it loops on + * DB_LOCK_DEADLOCK, retrying the fetch. + * + */ +static IDList * +idl_fetch_one( + struct ldbminfo *li, + DB *db, + DBT *key, + DB_TXN *txn, + int *err +) +{ + DBT data = {0}; + IDList *idl = NULL; + + /* LDAPDebug( LDAP_DEBUG_TRACE, "=> idl_fetch_one\n", 0, 0, 0 ); */ + + data.flags = DB_DBT_MALLOC; + + do { + *err = db->get( db, txn, key, &data, 0 ); + if ( 0 != *err && DB_NOTFOUND != *err && DB_LOCK_DEADLOCK != *err ) + { + char *msg; + if ( EPERM == *err && *err != errno ) { + LDAPDebug( LDAP_DEBUG_ANY, + "idl_fetch_one(%s): Database failed to run, " + "There is either insufficient disk space or " + "insufficient memory available for database.\n", + ((char*)key->dptr)[ key->dsize - 1 ] ? + "" : (char*)key->dptr, 0, 0 ); + } else { + LDAPDebug( LDAP_DEBUG_ANY, + "idl_fetch_one error %d %s\n", + *err, (msg = dblayer_strerror( *err )) ? msg : "", 0 ); + } + } + } + while ( DB_LOCK_DEADLOCK == *err && NULL == txn ); + + if (0 == *err) { + idl = (IDList *) data.data; + } + + return( idl ); +} + +IDList * +idl_old_fetch( + backend *be, + DB *db, + DBT *key, + DB_TXN *txn, + struct attrinfo *a, + int *err +) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + DBT k2 = {0}; + IDList *idl; + IDList **tmp; + back_txn s_txn; + char *kstr; + int i; + unsigned long nids; + + /* LDAPDebug( LDAP_DEBUG_TRACE, "=> idl_fetch\n", 0, 0, 0 ); */ + if ( (idl = idl_fetch_one( li, db, key, txn, err )) == NULL ) { + return( NULL ); + } + + /* regular block */ + if ( ! INDIRECT_BLOCK( idl ) ) { + /* make sure we have the current value of highest id */ + if ( ALLIDS(idl) ) { + idl_free( idl ); + idl = idl_allids( be ); + } + return( idl ); + } + idl_free( idl ); + + /* Taking a transaction is expensive; so we try and optimize for the common case by not + taking one above. If we have a indirect block; we need to take a transaction and re-read + the idl since they could have been changed by another thread after we read the first block + above */ + + dblayer_txn_init(li,&s_txn); + if (NULL != txn) + { + dblayer_read_txn_begin(li,txn,&s_txn); + } + if ( (idl = idl_fetch_one( li, db, key, s_txn.back_txn_txn, err )) == NULL ) { + dblayer_read_txn_commit(li,&s_txn); + return( NULL ); + } + + /* regular block */ + if ( ! INDIRECT_BLOCK( idl ) ) { + dblayer_read_txn_commit(li,&s_txn); + /* make sure we have the current value of highest id */ + if ( ALLIDS(idl) ) { + idl_free( idl ); + idl = idl_allids( be ); + } + return( idl ); + } + /* + * this is an indirect block which points to other blocks. + * we need to read in all the blocks it points to and construct + * a big id list containing all the ids, which we will return. + */ + + /* count the number of blocks & allocate space for pointers to them */ + for ( i = 0; idl->b_ids[i] != NOID; i++ ) + ; /* NULL */ + tmp = (IDList **) slapi_ch_malloc( (i + 1) * sizeof(IDList *) ); + + /* read in all the blocks */ + kstr = (char *) slapi_ch_malloc( key->dsize + 20 ); + nids = 0; + for ( i = 0; idl->b_ids[i] != NOID; i++ ) { + ID thisID = idl->b_ids[i]; + ID nextID = idl->b_ids[i+1]; + + sprintf( kstr, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, (u_long)thisID ); + k2.dptr = kstr; + k2.dsize = strlen( kstr ) + 1; + + if ( (tmp[i] = idl_fetch_one( li, db, &k2, s_txn.back_txn_txn, err )) == NULL ) { + if(*err == DB_LOCK_DEADLOCK) { + dblayer_read_txn_abort(li,&s_txn); + } else { + dblayer_read_txn_commit(li,&s_txn); + } + slapi_ch_free((void**)&kstr ); + slapi_ch_free((void**)&tmp ); + return( NULL ); + } + + nids += tmp[i]->b_nids; + + /* Check for inconsistencies: */ + if ( tmp[i]->b_ids[0] != thisID ) { + LDAPDebug (LDAP_DEBUG_ANY, "idl_fetch_one(%s)->b_ids[0] == %lu\n", + k2.dptr, (u_long)tmp[i]->b_ids[0], 0); + } + if ( nextID != NOID ) { + if ( nextID <= thisID ) { + LDAPDebug (LDAP_DEBUG_ANY, "indirect block (%s) contains %lu, %lu\n", + key->dptr, (u_long)thisID, (u_long)nextID); + } + if ( nextID <= tmp[i]->b_ids[(tmp[i]->b_nids)-1] ) { + LDAPDebug (LDAP_DEBUG_ANY, "idl_fetch_one(%s)->b_ids[last] == %lu" + " >= %lu (next indirect ID)\n", + k2.dptr, (u_long)tmp[i]->b_ids[(tmp[i]->b_nids)-1], (u_long)nextID); + } + } + } + dblayer_read_txn_commit(li,&s_txn); + tmp[i] = NULL; + slapi_ch_free((void**)&kstr ); + idl_free( idl ); + + /* allocate space for the big block */ + idl = idl_alloc( nids ); + idl->b_nids = nids; + nids = 0; + + /* copy in all the ids from the component blocks */ + for ( i = 0; tmp[i] != NULL; i++ ) { + if ( tmp[i] == NULL ) { + continue; + } + + SAFEMEMCPY( (char *) &idl->b_ids[nids], (char *) tmp[i]->b_ids, + tmp[i]->b_nids * sizeof(ID) ); + nids += tmp[i]->b_nids; + + idl_free( tmp[i] ); + } + slapi_ch_free((void**)&tmp ); + + LDAPDebug( LDAP_DEBUG_TRACE, "<= idl_fetch %lu ids (%lu max)\n", (u_long)idl->b_nids, + (u_long)idl->b_nmax, 0 ); + return( idl ); +} + +static int +idl_store( + backend *be, + DB *db, + DBT *key, + IDList *idl, + DB_TXN *txn +) +{ + int rc; + DBT data = {0}; + + /* LDAPDebug( LDAP_DEBUG_TRACE, "=> idl_store\n", 0, 0, 0 ); */ + + data.dptr = (char *) idl; + data.dsize = (2 + idl->b_nmax) * sizeof(ID); + + rc = db->put( db, txn, key, &data, 0 ); + if ( 0 != rc ) { + char *msg; + if ( EPERM == rc && rc != errno ) { + LDAPDebug( LDAP_DEBUG_ANY, + "idl_store(%s): Database failed to run, " + "There is insufficient memory available for database.\n", + ((char*)key->dptr)[ key->dsize - 1 ] ? "" : (char*)key->dptr, 0, 0 ); + } else { + if (LDBM_OS_ERR_IS_DISKFULL(rc)) { + operation_out_of_disk_space(); + } + LDAPDebug( ((DB_LOCK_DEADLOCK == rc) ? LDAP_DEBUG_TRACE : LDAP_DEBUG_ANY), + "idl_store(%s) returns %d %s\n", + ((char*)key->dptr)[ key->dsize - 1 ] ? "" : (char*)key->dptr, + rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + if (rc == DB_RUNRECOVERY) { + LDAPDebug(LDAP_DEBUG_ANY, "%s\n", "Note: idl_store failures can be an indication of insufficient disk space.", 0, 0); + ldbm_nasty("idl_store",71,rc); + } + } + } + + /* LDAPDebug( LDAP_DEBUG_TRACE, "<= idl_store %d\n", rc, 0, 0 ); */ + return( rc ); +} + +static void +idl_split_block( + IDList *b, + ID id, + IDList **n1, + IDList **n2 +) +{ + ID i; + + /* find where to split the block */ + for ( i = 0; i < b->b_nids && id > b->b_ids[i]; i++ ) + ; /* NULL */ + + *n1 = idl_alloc( i == 0 ? 1 : i ); + *n2 = idl_alloc( b->b_nids - i + (i == 0 ? 0 : 1)); + + /* + * everything before the id being inserted in the first block + * unless there is nothing, in which case the id being inserted + * goes there. + */ + SAFEMEMCPY( (char *) &(*n1)->b_ids[0], (char *) &b->b_ids[0], + i * sizeof(ID) ); + (*n1)->b_nids = (i == 0 ? 1 : i); + + if ( i == 0 ) { + (*n1)->b_ids[0] = id; + } else { + (*n2)->b_ids[0] = id; + } + + /* the id being inserted & everything after in the second block */ + SAFEMEMCPY( (char *) &(*n2)->b_ids[i == 0 ? 0 : 1], + (char *) &b->b_ids[i], (b->b_nids - i) * sizeof(ID) ); + (*n2)->b_nids = b->b_nids - i + (i == 0 ? 0 : 1); +} + +/* + * idl_change_first - called when an indirect block's first key has + * changed, meaning it needs to be stored under a new key, and the + * header block pointing to it needs updating. + */ + +static int +idl_change_first( + backend *be, + DB *db, + DBT *hkey, /* header block key */ + IDList *h, /* header block */ + int pos, /* pos in h to update */ + DBT *bkey, /* data block key */ + IDList *b, /* data block */ + DB_TXN *txn +) +{ + int rc; + char *msg; + + /* LDAPDebug( LDAP_DEBUG_TRACE, "=> idl_change_first\n", 0, 0, 0 ); */ + + /* delete old key block */ + rc = db->del( db, txn, bkey, 0 ); + if ( (rc != 0) && (DB_LOCK_DEADLOCK != rc) ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_change_first del (%s) err %d %s\n", + bkey->dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + if (rc == DB_RUNRECOVERY) { + ldbm_nasty("idl_store",72,rc); + } + return( rc ); + } + + /* write block with new key */ + sprintf( bkey->dptr, "%c%s%lu", CONT_PREFIX, (char *)hkey->dptr, (u_long)b->b_ids[0] ); + bkey->dsize = strlen( bkey->dptr ) + 1; + if ( (rc = idl_store( be, db, bkey, b, txn )) != 0 ) { + return( rc ); + } + + /* update + write indirect header block */ + h->b_ids[pos] = b->b_ids[0]; + if ( (rc = idl_store( be, db, hkey, h, txn )) != 0 ) { + return( rc ); + } + + return( 0 ); +} + + +#define IDL_CHECK_FAILED(FORMAT, ARG1, ARG2) \ +do { \ + char* fmt = slapi_ch_malloc (strlen(func) + strlen(note) + strlen(FORMAT) + 30); \ + if (fmt != NULL) { \ + sprintf (fmt, "%s(%%s,%lu) %s: %s\n", func, (u_long)id, note, FORMAT); \ + LDAPDebug (LDAP_DEBUG_ANY, fmt, key->dptr, ARG1, ARG2); \ + slapi_ch_free((void**)&fmt); \ + } \ +} while(0) + + +static void +idl_check_indirect (IDList* idl, int i, IDList* tmp, IDList* tmp2, + char* func, char* note, DBT* key, ID id) + /* Check for inconsistencies; report any via LDAPDebug(LDAP_DEBUG_ANY). + The caller alleges that *idl is a header block, in which the + i'th item points to the indirect block *tmp, and either tmp2 == NULL + or *tmp2 is the indirect block to which the i+1'th item in *idl points. + The other parameters are merely output in each error message, like: + printf ("%s(%s,%lu) %s: ...", func, key->dptr, (u_long)id, note, ...) + */ +{ + /* The implementation is optimized for no inconsistencies. */ + const ID thisID = idl->b_ids[i]; + const ID nextID = idl->b_ids[i+1]; + const ID tmp0 = tmp->b_ids[0]; + const ID tmpLast = tmp->b_ids[tmp->b_nids-1]; + + if (tmp0 != thisID) { + IDL_CHECK_FAILED ("tmp->b_ids[0] == %lu, not %lu\n", + (u_long)tmp0, (u_long)thisID); + } + if (tmp0 > tmpLast) { + IDL_CHECK_FAILED ("tmp->b_ids[0] == %lu > %lu [last]\n", + (u_long)tmp0, (u_long)tmpLast); + } + if (nextID == NOID) { + if (tmp2 != NULL) { + IDL_CHECK_FAILED ("idl->b_ids[%i+1] == NOID, but tmp2 != NULL\n", i, 0); + } + } else { + if (nextID <= thisID) { + IDL_CHECK_FAILED ("idl->b_ids contains %lu, %lu\n", (u_long)thisID, (u_long)nextID); + } + if (nextID <= tmpLast) { + IDL_CHECK_FAILED ("idl->b_ids[i+1] == %lu <= %lu (last of idl->b_ids[i])\n", + (u_long)nextID, (u_long)tmpLast); + } + if (tmp2 != NULL && tmp2->b_ids[0] != nextID) { + IDL_CHECK_FAILED ("tmp2->b_ids[0] == %lu, not %lu\n", + (u_long)tmp2->b_ids[0], (u_long)nextID); + } + } +} + + +int +idl_old_insert_key( + backend *be, + DB *db, + DBT *key, + ID id, + DB_TXN *txn, + struct attrinfo *a, + int *disposition +) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + int i, j, rc = 0; + char *msg; + IDList *idl, *tmp, *tmp2, *tmp3; + char *kstr; + DBT k2 = {0}; + DBT k3 = {0}; + + if (NULL != disposition) { + *disposition = IDL_INSERT_NORMAL; + } + + if (0 == a->ai_idl->idl_maxids) { + idl_init_maxids(li,a->ai_idl); + } + + idl_Wlock_list(a->ai_idl,key); + if ( (idl = idl_fetch_one( li, db, key, txn, &rc )) == NULL ) { + if ( rc != 0 && rc != DB_NOTFOUND ) { + if ( rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_insert_key 0 BAD %d %s\n", + rc, (msg = dblayer_strerror( rc )) ? msg : "", 0 ); + } + return( rc ); + } + idl = idl_alloc( 1 ); + idl->b_ids[idl->b_nids++] = id; + rc = idl_store( be, db, key, idl, txn ); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_insert_key 1 BAD %d %s\n", + rc, (msg = dblayer_strerror( rc )) ? msg : "", 0 ); + } + + idl_free( idl ); + idl_unlock_list(a->ai_idl,key); + return( rc ); + } + + /* regular block */ + if ( ! INDIRECT_BLOCK( idl ) ) { + switch ( idl_insert_maxids( &idl, id, a->ai_idl->idl_maxids ) ) { + case 0: /* id inserted - store the updated block */ + case 1: + rc = idl_store( be, db, key, idl, txn ); + break; + + case 2: /* id already there - nothing to do */ + rc = 0; + /* Could be an ALLID block, let's check */ + if (ALLIDS(idl)) { + if (NULL != disposition) { + *disposition = IDL_INSERT_ALLIDS; + } + } + break; + + case 3: /* id not inserted - block must be split */ + /* check threshold for marking this an all-id block */ + if ( a->ai_idl->idl_maxindirect < 2 ) { + idl_free( idl ); + idl = idl_allids( be ); + rc = idl_store( be, db, key, idl, txn ); + idl_free( idl ); + + idl_unlock_list(a->ai_idl,key); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_insert_key 2 BAD %d %s\n", + rc, (msg = dblayer_strerror( rc )) ? msg : "", 0 ); + } + if (NULL != disposition) { + *disposition = IDL_INSERT_NOW_ALLIDS; + } + return( rc ); + } + + idl_split_block( idl, id, &tmp, &tmp2 ); + idl_free( idl ); + + /* create the header indirect block */ + idl = idl_alloc( 3 ); + idl->b_nmax = 3; + idl->b_nids = INDBLOCK; + idl->b_ids[0] = tmp->b_ids[0]; + idl->b_ids[1] = tmp2->b_ids[0]; + idl->b_ids[2] = NOID; + + /* store it */ + rc = idl_store( be, db, key, idl, txn ); + if ( rc != 0 ) { + idl_free( idl ); + idl_free( tmp ); + idl_free( tmp2 ); + if ( rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_insert_key 3 BAD %d %s\n", + rc, (msg = dblayer_strerror( rc )) ? msg : "", 0 ); + } + return( rc ); + } + + /* store the first id block */ + kstr = (char *) slapi_ch_malloc( key->dsize + 20 ); + sprintf( kstr, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, + (u_long)tmp->b_ids[0] ); + k2.dptr = kstr; + k2.dsize = strlen( kstr ) + 1; + rc = idl_store( be, db, &k2, tmp, txn ); + + /* store the second id block */ + sprintf( kstr, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, + (u_long)tmp2->b_ids[0] ); + k2.dptr = kstr; + k2.dsize = strlen( kstr ) + 1; + rc = idl_store( be, db, &k2, tmp2, txn ); + if ( rc != 0 ) { + idl_free( idl ); + idl_free( tmp ); + idl_free( tmp2 ); + if ( rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_insert_key 4 BAD %d %s\n", + rc, (msg = dblayer_strerror( rc )) ? msg : "", 0 ); + } + return( rc ); + } + idl_check_indirect (idl, 0, tmp, tmp2, + "idl_insert_key", "split", key, id); + + slapi_ch_free((void**)&kstr ); + idl_free( tmp ); + idl_free( tmp2 ); + break; + } + + idl_free( idl ); + idl_unlock_list(a->ai_idl,key); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_insert_key 5 BAD %d %s\n", + rc, (msg = dblayer_strerror( rc )) ? msg : "", 0 ); + } + return( rc ); + } + + /* + * this is an indirect block which points to other blocks. + * we need to read in the block into which the id should be + * inserted, then insert the id and store the block. we might + * have to split the block if it is full, which means we also + * need to write a new "header" block. + */ + + /* select the block to try inserting into */ + for ( i = 0; idl->b_ids[i] != NOID && id > idl->b_ids[i]; i++ ) + ; /* NULL */ + if ( id == idl->b_ids[i] ) { /* already in a block */ +#ifdef _DEBUG_LARGE_BLOCKS + LDAPDebug( LDAP_DEBUG_ANY, + "id %lu for key (%s) is already in block %d\n", + (u_long)id, key.dptr, i); +#endif + idl_unlock_list(a->ai_idl,key); + idl_free( idl ); + return( 0 ); + } + if ( i != 0 ) { + i--; + } + + /* get the block */ + kstr = (char *) slapi_ch_malloc( key->dsize + 20 ); + sprintf( kstr, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, (u_long)idl->b_ids[i] ); + k2.dptr = kstr; + k2.dsize = strlen( kstr ) + 1; + if ( (tmp = idl_fetch_one( li, db, &k2, txn, &rc )) == NULL ) { + if ( rc != 0 ) { + if ( rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_insert_key 5.5 BAD %d %s\n", + rc, (msg = dblayer_strerror( rc )) ? msg : "", 0 ); + } + return( rc ); + } + LDAPDebug( LDAP_DEBUG_ANY, + "nonexistent continuation block (%s)\n", k2.dptr, 0, 0 ); + idl_unlock_list(a->ai_idl,key); + idl_free( idl ); + slapi_ch_free((void**)&kstr ); + return( -1 ); + } + + /* insert the id */ + switch ( idl_insert_maxids( &tmp, id, a->ai_idl->idl_maxids ) ) { + case 0: /* id inserted ok */ + rc = idl_store( be, db, &k2, tmp, txn ); + if (0 != rc) { + idl_check_indirect (idl, i, tmp, NULL, + "idl_insert_key", "indirect", key, id); + } + break; + + case 1: /* id inserted - first id in block has changed */ + /* + * key for this block has changed, so we have to + * write the block under the new key, delete the + * old key block + update and write the indirect + * header block. + */ + + rc = idl_change_first( be, db, key, idl, i, &k2, tmp, txn ); + if ( rc != 0 ) { + break; /* return error in rc */ + } + idl_check_indirect (idl, i, tmp, NULL, + "idl_insert_key", "indirect 1", key, id); + break; + + case 2: /* id not inserted - already there */ + idl_check_indirect (idl, i, tmp, NULL, + "idl_insert_key", "indirect no change", key, id); + break; + + case 3: /* id not inserted - block is full */ + /* + * first, see if we can shift ids down one, moving + * the last id in the current block to the next + * block, and then adding the id we are inserting to + * the current block. we'll need to split the block + * otherwise. + */ + + /* is there a next block? */ + if ( idl->b_ids[i + 1] != NOID ) { + char *kstr3 = (char *) slapi_ch_malloc( key->dsize + 20 ); + /* yes - read it in */ + sprintf( kstr3, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, + (u_long)idl->b_ids[i + 1] ); + k3.dptr = kstr3; + k3.dsize = strlen( kstr3 ) + 1; + if ( (tmp2 = idl_fetch_one( li, db, &k3, txn, &rc )) + == NULL ) { + if ( rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, + "idl_fetch_one (%s) returns NULL\n", + k3.dptr, 0, 0 ); + } + if (0 != rc) { + idl_check_indirect (idl, i, tmp, NULL, + "idl_insert_key", "indirect missing", key, id); + } + break; + } + + /* + * insert the last key in the previous block in + * the next block. it should go at the beginning + * always, if it fits at all. + */ + rc = idl_insert_maxids (&tmp2, + id > tmp->b_ids[tmp->b_nids-1] ? + id : tmp->b_ids[tmp->b_nids-1], + a->ai_idl->idl_maxids); + switch ( rc ) { + case 1: /* id inserted first in block */ + rc = idl_change_first( be, db, key, idl, + i + 1, &k3, tmp2, txn ); + if ( rc != 0 ) { + break; /* return error in rc */ + } + + if (id < tmp->b_ids[tmp->b_nids-1]) { + /* + * we inserted the last id in the previous + * block in this block. we need to "remove" + * it from the previous block and insert the + * new id. decrementing the b_nids count + * in the previous block has the effect + * of removing the last id. + */ + + /* remove last id in previous block */ + tmp->b_nids--; + + /* insert new id in previous block */ + switch ( (rc = idl_insert_maxids( &tmp, id, + a->ai_idl->idl_maxids )) ) { + case 0: /* id inserted */ + rc = idl_store( be, db, &k2, tmp, txn ); + break; + case 1: /* first in block */ + rc = idl_change_first( be, db, key, idl, + i, &k2, tmp, txn ); + break; + case 2: /* already there - how? */ + case 3: /* split block - how? */ + LDAPDebug( LDAP_DEBUG_ANY, + "not expecting (%d) from idl_insert_maxids of %lu in (%s)\n", + rc, (u_long)id, k2.dptr ); + LDAPDebug( LDAP_DEBUG_ANY, + "likely database corruption\n", + 0, 0, 0 ); + rc = 0; + break; + } + } + if ( rc != 0 ) { + break; /* return error in rc */ + } + idl_check_indirect (idl, i, tmp, tmp2, + "idl_insert_key", "overflow", key, id); + + if ( k2.dptr != NULL ) { + free( k2.dptr ); + } + if ( k3.dptr != NULL ) { + free( k3.dptr ); + } + idl_free( tmp ); + idl_free( tmp2 ); + idl_free( idl ); + idl_unlock_list(a->ai_idl,key); + return( rc ); + + case 0: /* id inserted not at start - how? */ + case 2: /* id already there - how? */ + /* + * if either of these cases happen, this + * index entry must have been corrupt when + * we started this insert. what can we do + * aside from log a warning? + */ + LDAPDebug( LDAP_DEBUG_ANY, + "not expecting return %d from idl_insert_maxids of id %lu in block with key (%s)\n", + rc, (u_long)tmp->b_ids[tmp->b_nids-1], k3.dptr ); + LDAPDebug( LDAP_DEBUG_ANY, + "likely database corruption\n", 0, 0, 0 ); + /* FALL */ + case 3: /* block is full */ + /* + * if this case happens, we fall back to + * splitting the original block. + * This is not an error condition. So set + * rc = 0 to continue. Otherwise, it will break + * from the case statement and return rc=3, + * which is not correct. + */ + rc = 0; + idl_free( tmp2 ); + break; + } + if ( rc != 0 ) { + break; /* return error in rc */ + } + } + + /* + * must split the block, write both new blocks + update + * and write the indirect header block. + */ + + /* count how many indirect blocks */ + for ( j = 0; idl->b_ids[j] != NOID; j++ ) + ; /* NULL */ + + /* check it against all-id thresholed */ + if ( j + 1 > a->ai_idl->idl_maxindirect ) { + /* + * we've passed the all-id threshold, meaning + * that this set of blocks should be replaced + * by a single "all-id" block. our job: delete + * all the indirect blocks, and replace the header + * block by an all-id block. + */ + + /* delete all indirect blocks */ + for ( j = 0; idl->b_ids[j] != NOID; j++ ) { + sprintf( kstr, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, + (u_long)idl->b_ids[j] ); + k2.dptr = kstr; + k2.dsize = strlen( kstr ) + 1; + + rc = db->del( db, txn, &k2, 0 ); + if ( rc != 0 ) { + if (rc == DB_RUNRECOVERY) { + ldbm_nasty("",73,rc); + } + break; + } + } + + /* store allid block in place of header block */ + if ( 0 == rc ) { + idl_free( idl ); + idl = idl_allids( be ); + rc = idl_store( be, db, key, idl, txn ); + if (NULL != disposition) { + *disposition = IDL_INSERT_NOW_ALLIDS; + } + } + + if ( k2.dptr != NULL ) { + free( k2.dptr ); + } + if ( k3.dptr != NULL ) { + free( k3.dptr ); + } + idl_free( idl ); + idl_free( tmp ); + idl_unlock_list(a->ai_idl,key); + return( rc ); + } + + idl_split_block( tmp, id, &tmp2, &tmp3 ); + idl_free( tmp ); + + /* create a new updated indirect header block */ + tmp = idl_alloc( idl->b_nmax + 1 ); + tmp->b_nids = INDBLOCK; + /* everything up to the split block */ + SAFEMEMCPY( (char *) tmp->b_ids, (char *) idl->b_ids, + i * sizeof(ID) ); + /* the two new blocks */ + tmp->b_ids[i] = tmp2->b_ids[0]; + tmp->b_ids[i + 1] = tmp3->b_ids[0]; + /* everything after the split block */ + SAFEMEMCPY( (char *) &tmp->b_ids[i + 2], (char *) + &idl->b_ids[i + 1], (idl->b_nmax - i - 1) * sizeof(ID) ); + + /* store the header block */ + rc = idl_store( be, db, key, tmp, txn ); + if ( rc != 0 ) { + idl_free( tmp2 ); + idl_free( tmp3 ); + break; + } + + /* store the first id block */ + sprintf( kstr, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, + (u_long)tmp2->b_ids[0] ); + k2.dptr = kstr; + k2.dsize = strlen( kstr ) + 1; + rc = idl_store( be, db, &k2, tmp2, txn ); + if ( rc != 0 ) { + idl_free( tmp2 ); + idl_free( tmp3 ); + break; + } + + /* store the second id block */ + sprintf( kstr, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, + (u_long)tmp3->b_ids[0] ); + k2.dptr = kstr; + k2.dsize = strlen( kstr ) + 1; + rc = idl_store( be, db, &k2, tmp3, txn ); + if ( rc != 0 ) { + idl_free( tmp2 ); + idl_free( tmp3 ); + break; + } + + idl_check_indirect (tmp, i, tmp2, tmp3, + "idl_insert_key", "indirect split", key, id); + idl_free( tmp2 ); + idl_free( tmp3 ); + break; + } + + if ( k2.dptr != NULL ) { + free( k2.dptr ); + } + if ( k3.dptr != NULL ) { + free( k3.dptr ); + } + idl_free( tmp ); + idl_free( idl ); + idl_unlock_list(a->ai_idl,key); + return( rc ); +} + + +/* Store a complete IDL all in one go, there must not be an existing key with the same value */ +/* Routine used by merging import code */ +int idl_old_store_block( + backend *be, + DB *db, + DBT *key, + IDList *idl, + DB_TXN *txn, + struct attrinfo *a + ) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + int ret = 0; + idl_private *priv = a->ai_idl; + + if (0 == a->ai_idl->idl_maxids) { + idl_init_maxids(li,a->ai_idl); + } + + /* First, is it an ALLIDS block ? */ + if (ALLIDS(idl)) { + /* If so, we can store it as-is */ + ret = idl_store(be,db,key,idl,txn); + } else { + /* Next, is it a block with so many IDs in it that it _should_ be an ALLIDS block ? */ + if (idl->b_nids > (ID)li->li_allidsthreshold) { + /* If so, store an ALLIDS block */ + IDList *all = idl_allids(be); + ret = idl_store(be,db,key,all,txn); + idl_free(all); + } else { + /* Then , is it a block which is smaller than the size at which it needs splitting ? */ + if (idl->b_nids <= (ID)priv->idl_maxids) { + /* If so, store as-is */ + ret = idl_store(be,db,key,idl,txn); + } else { + size_t number_of_ids = 0; + size_t max_ids_in_block = 0; + size_t number_of_cont_blks = 0; + size_t i = 0; + size_t number_of_ids_left = 0; + IDList *master_block = NULL; + size_t index = 0; + DBT cont_key = {0}; + + number_of_ids = idl->b_nids; + max_ids_in_block = priv->idl_maxids; + number_of_cont_blks = number_of_ids / max_ids_in_block; + if (0 != number_of_ids % max_ids_in_block) { + number_of_cont_blks++; + } + number_of_ids_left = number_of_ids; + /* Block needs splitting into continuation blocks */ + /* We need to make up a master block and n continuation blocks */ + /* Alloc master block */ + master_block = idl_alloc(number_of_cont_blks + 1); + if (NULL == master_block) { + return -1; + } + master_block->b_nids = INDBLOCK; + master_block->b_ids[number_of_cont_blks] = NOID; + /* Iterate over ids making the continuation blocks */ + for (i = 0 ; i < number_of_cont_blks; i++) { + IDList *this_cont_block = NULL; + size_t size_of_this_block = 0; + ID lead_id = NOID; + size_t j = 0; + + lead_id = idl->b_ids[index]; + if (number_of_ids_left >= max_ids_in_block) { + size_of_this_block = max_ids_in_block; + } else { + size_of_this_block = number_of_ids_left; + } + this_cont_block = idl_alloc(size_of_this_block); + if (NULL == this_cont_block) { + return -1; + } + this_cont_block->b_nids = size_of_this_block; + /* Copy over the ids to the cont block we're making */ + for (j = 0; j < size_of_this_block; j++) { + this_cont_block->b_ids[j] = idl->b_ids[index + j]; + } + /* Make the continuation key */ + make_cont_key(&cont_key,key,lead_id); + /* Now store the continuation block */ + ret = idl_store(be,db,&cont_key,this_cont_block,txn); + idl_free(this_cont_block); + free(cont_key.data); + if ( ret != 0 && ret != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_store_block(%s) 1 BAD %d %s\n",key->data, ret, dblayer_strerror( ret )); + return ret; + } + /* Put the lead ID number in the header block */ + master_block->b_ids[i] = lead_id; + + /* Make our loop invariants correct */ + number_of_ids_left -= size_of_this_block; + index += size_of_this_block; + } + PR_ASSERT(0 == number_of_ids_left); + /* Now store the master block */ + ret = idl_store(be,db,key,master_block,txn); + /* And free it */ + idl_free(master_block); + } + } + } + return ret; +} + +/* + * idl_insert - insert an id into an id list. + */ +void idl_insert(IDList **idl, ID id) +{ + ID i, j; + NIDS nids; + + if ((*idl) == NULL) { + (*idl) = idl_alloc(1); + idl_append((*idl), id); + return; + } + + if (ALLIDS(*idl)) { + return; + } + + i = nids = (*idl)->b_nids; + + if (nids > 0) { + /* optimize for a simple append */ + if (id == (*idl)->b_ids[nids-1]) { + return; + } else if (id > (*idl)->b_ids[nids-1]) { + if (nids < (*idl)->b_nmax) { + (*idl)->b_ids[nids] = id; + (*idl)->b_nids++; + return; + } + + i = nids; + + } else if (id < (*idl)->b_ids[0]) { + /* prepend */ + i = 0; + } else { + int lo = 0; + int hi = (*idl)->b_nids - 1; + int mid = 0; + ID *ids = (*idl)->b_ids; + + if (0 != (*idl)->b_nids) { + while (lo <= hi) { + mid = (hi + lo) >> 1; + if (ids[mid] > id) { + hi = mid - 1; + } else { + if (ids[mid] < id) { + lo = mid + 1; + } else { + /* Found it ! */ + return; + } + } + } + } + i = lo; + } + } + + /* do we need to make room for it? */ + if ( (*idl)->b_nids == (*idl)->b_nmax ) { + (*idl)->b_nmax *= 2; + + (*idl) = (IDList *) slapi_ch_realloc( (char *) (*idl), + ((*idl)->b_nmax + 2) * sizeof(ID) ); + } + + /* make a slot for the new id */ + for ( j = (*idl)->b_nids; j != i; j-- ) { + (*idl)->b_ids[j] = (*idl)->b_ids[j-1]; + } + + (*idl)->b_ids[i] = id; + (*idl)->b_nids++; + + memset( (char *) &(*idl)->b_ids[(*idl)->b_nids], '\0', + ((*idl)->b_nmax - (*idl)->b_nids) * sizeof(ID) ); + + return; +} + +/* + * idl_insert_maxids - insert an id into an id list. + * returns 0 id inserted + * 1 id inserted, first id in block has changed + * 2 id not inserted, already there + * 3 id not inserted, block must be split + */ + +static int +idl_insert_maxids( IDList **idl, ID id, int maxids ) +{ + ID i, j; + NIDS nids; + + if ( ALLIDS( *idl ) ) { + return( 2 ); /* already there */ + } + + nids = (*idl)->b_nids; + + if (nids > 0) { + /* optimize for a simple append */ + if (id == (*idl)->b_ids[nids-1]) { + return (2); + } else if (id > (*idl)->b_ids[nids-1]) { + if (nids < (*idl)->b_nmax) { + (*idl)->b_ids[nids] = id; + (*idl)->b_nids++; + return 0; + } + + i = nids; + + } else if (idl_tune & IDL_TUNE_BSEARCH) { + int lo = 0; + int hi = (*idl)->b_nids - 1; + int mid = 0; + ID *ids = (*idl)->b_ids; + if (0 != (*idl)->b_nids) { + while (lo <= hi) { + mid = (hi + lo) >> 1; + if (ids[mid] > id) { + hi = mid - 1; + } else { + if (ids[mid] < id) { + lo = mid + 1; + } else { + /* Found it ! */ + return(2); + } + } + } + } + i = lo; + } else { + /* is it already there? linear search */ + for ( i = 0; i < (*idl)->b_nids && id > (*idl)->b_ids[i]; i++ ) { + ; /* NULL */ + } + if ( i < (*idl)->b_nids && (*idl)->b_ids[i] == id ) { + return( 2 ); /* already there */ + } + } + } + + /* do we need to make room for it? */ + if ( (*idl)->b_nids == (*idl)->b_nmax ) { + /* make room or indicate block needs splitting */ + if ( (*idl)->b_nmax == (ID) maxids ) { + return( 3 ); /* block needs splitting */ + } + + if (idl_tune & IDL_TUNE_NOPAD) { + (*idl)->b_nmax++; + } else { + (*idl)->b_nmax *= 2; + } + if ( (*idl)->b_nmax > (ID)maxids ) { + (*idl)->b_nmax = maxids; + } + *idl = (IDList *) slapi_ch_realloc( (char *) *idl, + ((*idl)->b_nmax + 2) * sizeof(ID) ); + } + + /* make a slot for the new id */ + for ( j = (*idl)->b_nids; j != i; j-- ) { + (*idl)->b_ids[j] = (*idl)->b_ids[j-1]; + } + (*idl)->b_ids[i] = id; + (*idl)->b_nids++; + (void) memset( (char *) &(*idl)->b_ids[(*idl)->b_nids], '\0', + ((*idl)->b_nmax - (*idl)->b_nids) * sizeof(ID) ); + + return( i == 0 ? 1 : 0 ); /* inserted - first id changed or not */ +} + +/* + * idl_delete_key - delete an id from the index entry identified by key + * returns 0 id was deleted + * -666 no such index entry or id in index entry + * other an error code from db + */ + +int +idl_old_delete_key( + backend *be, + DB *db, + DBT *key, + ID id, + DB_TXN *txn, + struct attrinfo *a +) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + int i, j, rc; + char *msg; + IDList *idl, *didl; + DBT contkey = {0}; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> idl_delete_key(%s,%lu)\n", + key->dptr, (u_long)id, 0 ); + + idl_Wlock_list(a->ai_idl,key); + + if ( (idl = idl_fetch_one( li, db, key, txn, &rc )) == NULL ) { + idl_unlock_list(a->ai_idl,key); + if ( rc != 0 && rc != DB_NOTFOUND && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key(%s) 0 BAD %d %s\n", + key->dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + } + if ( 0 == rc || DB_NOTFOUND == rc ) rc = -666; + LDAPDebug( LDAP_DEBUG_TRACE, "<= idl_delete_key(%s,%lu) %d !idl_fetch_one\n", + key->dptr, (u_long)id, rc ); + return rc; + } + + /* regular block */ + if ( ! INDIRECT_BLOCK( idl ) ) { + switch ( idl_delete( &idl, id ) ) { + case 0: /* id deleted, store the updated block */ + case 1: /* first id changed - ok in direct block */ + rc = idl_store( be, db, key, idl, txn ); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key(%s) 1 BAD %d %s\n", + key->dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + } + break; + + case 2: /* id deleted, block empty - delete it */ + rc = db->del( db, txn, key, 0 ); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key(%s) 2 BAD %d %s\n", + key->dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + if (rc == DB_RUNRECOVERY) { + ldbm_nasty("",74,rc); + } + + } + break; + + case 3: /* not there - previously deleted */ + case 4: /* all ids block */ + rc = 0; + break; + + default: + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key(%s) 3 BAD idl_delete\n", + key->dptr, 0, 0 ); + break; + } + + idl_free( idl ); + idl_unlock_list(a->ai_idl,key); + LDAPDebug( LDAP_DEBUG_TRACE, "<= idl_delete_key(%s,%lu) %d (not indirect)\n", + key->dptr, (u_long)id, rc ); + return( rc ); + } + + /* + * this is an indirect block that points to other blocks. we + * need to read the block containing the id to delete, delete + * the id, and store the changed block. if the first id in the + * block changes, or the block becomes empty, we need to rewrite + * the header block too. + */ + + /* select the block the id is in */ + for ( i = 0; idl->b_ids[i] != NOID && id > idl->b_ids[i]; i++ ) { + ; /* NULL */ + } + /* id smaller than smallest id - not there */ + if ( i == 0 && id < idl->b_ids[i] ) { + idl_free( idl ); + idl_unlock_list(a->ai_idl,key); + LDAPDebug( LDAP_DEBUG_TRACE, "<= idl_delete_key(%s,%lu) -666 (id not found)\n", + key->dptr, (u_long)id, 0 ); + return( -666 ); + } + if ( id != idl->b_ids[i] ) { + i--; + } + + /* get the block to delete from */ + make_cont_key( &contkey, key, idl->b_ids[i] ); + if ( (didl = idl_fetch_one( li, db, &contkey, txn, &rc )) == NULL ) { + idl_free( idl ); + idl_unlock_list(a->ai_idl,key); + if ( rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key(%s) 5 BAD %d %s\n", + contkey.dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + } + LDAPDebug( LDAP_DEBUG_TRACE, "<= idl_delete_key(%s,%lu) %d idl_fetch_one(contkey)\n", + contkey.dptr, (u_long)id, rc ); + if ( contkey.dptr != NULL ) { + free( contkey.dptr ); + } + return( rc ); + } + + rc = 0; + switch ( idl_delete( &didl, id ) ) { + case 0: /* id deleted - rewrite block */ + if ( (rc = idl_store( be, db, &contkey, didl, txn )) != 0 ) { + if ( rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key(%s) BAD %d %s\n", + contkey.dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + } + } + if (0 != rc) { + idl_check_indirect( idl, i, didl, NULL, "idl_delete_key", "0", key, id ); + } + break; + + case 1: /* id deleted, first id changed, - write hdr, block */ + rc = idl_change_first( be, db, key, idl, i, &contkey, didl, txn ); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key(%s) 7 BAD %d %s\n", + contkey.dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + } + if (0 != rc) { + idl_check_indirect( idl, i, didl, NULL, "idl_delete_key", "1", key, id ); + } + break; + + case 2: /* id deleted, block empty - write hdr, del block */ + for ( j = i; idl->b_ids[j] != NOID; j++ ) { + idl->b_ids[j] = idl->b_ids[j+1]; + } + if ( idl->b_ids[0] != NOID ) { /* Write the header, first: */ + rc = idl_store( be, db, key, idl, txn ); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key: idl_store(%s) BAD %d %s\n", + key->dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + } + } else { /* This index is entirely empty. Delete the header: */ + rc = db->del( db, txn, key, 0 ); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key: db->del(%s) BAD %d %s\n", + key->dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + if (rc == DB_RUNRECOVERY) { + ldbm_nasty("",75,rc); + } + + } + } + if ( rc == 0 ) { /* Delete the indirect block: */ + rc = db->del( db, txn, &contkey, 0 ); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key: db->del(%s) BAD %d %s\n", + contkey.dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + if (rc == DB_RUNRECOVERY) { + ldbm_nasty("",76,rc); + } + + } + } + break; + + case 3: /* id not found - previously deleted */ + rc = 0; + idl_check_indirect( idl, i, didl, NULL, "idl_delete_key", "3", key, id ); + break; + case 4: /* all ids block - should not happen */ + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key: cont block (%s) is allids\n", + contkey.dptr, 0, 0 ); + rc = 0; + break; + } + idl_free( idl ); + idl_free( didl ); + if ( contkey.dptr != NULL ) { + free( contkey.dptr ); + } + idl_unlock_list(a->ai_idl,key); + if ( rc != 0 && rc != DB_LOCK_DEADLOCK ) + { + LDAPDebug( LDAP_DEBUG_ANY, "idl_delete_key(%s) 9 BAD %d %s\n", + key->dptr, rc, (msg = dblayer_strerror( rc )) ? msg : "" ); + } + LDAPDebug( LDAP_DEBUG_TRACE, "<= idl_delete_key(%s,%lu) %d (indirect)\n", + key->dptr, (u_long)id, rc ); + return( rc ); +} + +/* + * idl_delete - delete an id from an id list. + * returns 0 id deleted + * 1 id deleted, first id in block has changed + * 2 id deleted, block is empty + * 3 id not there + * 4 cannot delete from allids block + */ + +static int +idl_delete( IDList **idl, ID id ) +{ + ID i, delpos; + + if ( ALLIDS( *idl ) ) { + return( 4 ); /* cannot delete from allids block */ + } + + /* find the id to delete */ + for ( i = 0; i < (*idl)->b_nids && id > (*idl)->b_ids[i]; i++ ) { + ; /* NULL */ + } + if ( i == (*idl)->b_nids || (*idl)->b_ids[i] != id ) { + return( 3 ); /* id not there */ + } + + if ( --((*idl)->b_nids) == 0 ) { + return( 2 ); /* id deleted, block empty */ + } + + /* delete it */ + delpos = i; + for ( ; i < (*idl)->b_nids; i++ ) { + (*idl)->b_ids[i] = (*idl)->b_ids[i+1]; + } + + return( delpos == 0 ? 1 : 0 ); /* first id changed : id deleted */ +} + + +static void +make_cont_key( DBT *contkey, DBT *key, ID id ) +{ + contkey->dptr = (char *) slapi_ch_malloc( key->dsize + 20 ); + sprintf( contkey->dptr, "%c%s%lu", CONT_PREFIX, (char *)key->dptr, (u_long)id ); + contkey->dsize = strlen( contkey->dptr ) + 1; +} diff --git a/ldap/servers/slapd/back-ldbm/idl_common.c b/ldap/servers/slapd/back-ldbm/idl_common.c new file mode 100644 index 00000000..2cea183e --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/idl_common.c @@ -0,0 +1,402 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* Common IDL code, used in both old and new indexing schemes */ + +#include "back-ldbm.h" + +size_t idl_sizeof(IDList *idl) +{ + return (2 + idl->b_nmax) * sizeof(ID); +} + +NIDS idl_length(IDList *idl) +{ + return (idl->b_nmax == ALLIDSBLOCK) ? UINT_MAX : idl->b_nids; +} + +int idl_is_allids(IDList *idl) +{ + return (idl->b_nmax == ALLIDSBLOCK); +} + +IDList * +idl_alloc( NIDS nids ) +{ + IDList *new; + + /* nmax + nids + space for the ids */ + new = (IDList *) slapi_ch_calloc( (2 + nids), sizeof(ID) ); + new->b_nmax = nids; + new->b_nids = 0; + + return( new ); +} + +IDList * +idl_allids( backend *be ) +{ + IDList *idl; + + idl = idl_alloc( 0 ); + idl->b_nmax = ALLIDSBLOCK; + idl->b_nids = next_id_get( be ); + + return( idl ); +} + +void +idl_free( IDList *idl ) /* JCM - pass in ** */ +{ + if ( idl == NULL ) { + return; + } + + slapi_ch_free((void**)&idl ); +} + + +/* + * idl_append - append an id to an id list. + * + * Warning: The ID List must be maintained in order. + * Use idl_insert if the id may not + * + * returns + * 0 - appended + * 1 - already in there + * 2 - not enough room + */ + +int +idl_append( IDList *idl, ID id) +{ + if ( ALLIDS( idl ) || ( (idl->b_nids) && (idl->b_ids[idl->b_nids - 1] == id)) ) { + return( 1 ); /* already there */ + } + + if ( idl->b_nids == idl->b_nmax ) { + return( 2 ); /* not enough room */ + } + + idl->b_ids[idl->b_nids] = id; + idl->b_nids++; + + return( 0 ); +} + +static IDList * +idl_dup( IDList *idl ) +{ + IDList *new; + + if ( idl == NULL ) { + return( NULL ); + } + + new = idl_alloc( idl->b_nmax ); + SAFEMEMCPY( (char *) new, (char *) idl, (idl->b_nmax + 2) + * sizeof(ID) ); + + return( new ); +} + +static IDList * +idl_min( IDList *a, IDList *b ) +{ + return( a->b_nids > b->b_nids ? b : a ); +} + +/* + * idl_intersection - return a intersection b + */ + +IDList * +idl_intersection( + backend *be, + IDList *a, + IDList *b +) +{ + NIDS ai, bi, ni; + IDList *n; + + if ( a == NULL || b == NULL ) { + return( NULL ); + } + if ( ALLIDS( a ) ) { + slapi_be_set_flag(be, SLAPI_BE_FLAG_DONT_BYPASS_FILTERTEST); + return( idl_dup( b ) ); + } + if ( ALLIDS( b ) ) { + slapi_be_set_flag(be, SLAPI_BE_FLAG_DONT_BYPASS_FILTERTEST); + return( idl_dup( a ) ); + } + + n = idl_dup( idl_min( a, b ) ); + + for ( ni = 0, ai = 0, bi = 0; ai < a->b_nids; ai++ ) { + for ( ; bi < b->b_nids && b->b_ids[bi] < a->b_ids[ai]; bi++ ) + ; /* NULL */ + + if ( bi == b->b_nids ) { + break; + } + + if ( b->b_ids[bi] == a->b_ids[ai] ) { + n->b_ids[ni++] = a->b_ids[ai]; + } + } + + if ( ni == 0 ) { + idl_free( n ); + return( NULL ); + } + n->b_nids = ni; + + return( n ); +} + +/* + * idl_union - return a union b + */ + +IDList * +idl_union( + backend *be, + IDList *a, + IDList *b +) +{ + NIDS ai, bi, ni; + IDList *n; + + if ( a == NULL ) { + return( idl_dup( b ) ); + } + if ( b == NULL ) { + return( idl_dup( a ) ); + } + if ( ALLIDS( a ) || ALLIDS( b ) ) { + return( idl_allids( be ) ); + } + + if ( b->b_nids < a->b_nids ) { + n = a; + a = b; + b = n; + } + + n = idl_alloc( a->b_nids + b->b_nids ); + + for ( ni = 0, ai = 0, bi = 0; ai < a->b_nids && bi < b->b_nids; ) { + if ( a->b_ids[ai] < b->b_ids[bi] ) { + n->b_ids[ni++] = a->b_ids[ai++]; + } else if ( b->b_ids[bi] < a->b_ids[ai] ) { + n->b_ids[ni++] = b->b_ids[bi++]; + } else { + n->b_ids[ni++] = a->b_ids[ai]; + ai++, bi++; + } + } + + for ( ; ai < a->b_nids; ai++ ) { + n->b_ids[ni++] = a->b_ids[ai]; + } + for ( ; bi < b->b_nids; bi++ ) { + n->b_ids[ni++] = b->b_ids[bi]; + } + n->b_nids = ni; + + return( n ); +} + +/* + * idl_notin - return a intersection ~b (or a minus b) + * DB --- changed the interface of this function (no code called it), + * such that it can modify IDL a in place (it'll always be the same + * or smaller than the a passed in if not allids). + * If a new list is generated, it's returned in new_result and the function + * returns 1. Otherwise the result remains in a, and the function returns 0. + * The intention is to optimize for the interesting case in filterindex.c + * where we are computing foo AND NOT bar, and both foo and bar are not allids. + */ + +int +idl_notin( + backend *be, + IDList *a, + IDList *b, + IDList **new_result +) +{ + NIDS ni, ai, bi; + IDList *n; + *new_result = NULL; + + if ( a == NULL ) { + return( 0 ); + } + if ( b == NULL || ALLIDS( b ) ) { + *new_result = idl_dup( a ); + return( 1 ); + } + + if ( ALLIDS( a ) ) { /* Not convinced that this code is really worth it */ + /* It's trying to do allids notin b, where maxid is smaller than some size */ + n = idl_alloc( SLAPD_LDBM_MIN_MAXIDS ); + ni = 0; + + for ( ai = 1, bi = 0; ai < a->b_nids && ni < n->b_nmax && + bi < b->b_nmax; ai++ ) { + if ( b->b_ids[bi] == ai ) { + bi++; + } else { + n->b_ids[ni++] = ai; + } + } + + for ( ; ai < a->b_nids && ni < n->b_nmax; ai++ ) { + n->b_ids[ni++] = ai; + } + + if ( ni == n->b_nmax ) { + idl_free( n ); + *new_result = idl_allids( be ); + } else { + n->b_nids = ni; + *new_result = n; + } + return( 1 ); + } + + /* This is the case we're interested in, we want to detect where a and b don't overlap */ + { + size_t ahii, aloi, bhii, bloi; + size_t ahi, alo, bhi, blo; + int aloblo, ahiblo, alobhi, ahibhi; + + aloi = bloi = 0; + ahii = a->b_nids - 1; + bhii = b->b_nids - 1; + + ahi = a->b_ids[ahii]; + alo = a->b_ids[aloi]; + bhi = b->b_ids[bhii]; + blo = b->b_ids[bloi]; + /* if the ranges don't overlap, we're done, current a is the result */ + aloblo = alo < blo; + ahiblo = ahi < blo; + alobhi = ahi > bhi; + ahibhi = alo > bhi; + if ( (aloblo & ahiblo) || (alobhi & ahibhi) ) { + return 0; + } else { + /* Do what we did before */ + n = idl_dup( a ); + + ni = 0; + for ( ai = 0, bi = 0; ai < a->b_nids; ai++ ) { + for ( ; bi < b->b_nids && b->b_ids[bi] < a->b_ids[ai]; + bi++ ) { + ; /* NULL */ + } + + if ( bi == b->b_nids ) { + break; + } + + if ( b->b_ids[bi] != a->b_ids[ai] ) { + n->b_ids[ni++] = a->b_ids[ai]; + } + } + + for ( ; ai < a->b_nids; ai++ ) { + n->b_ids[ni++] = a->b_ids[ai]; + } + n->b_nids = ni; + + *new_result = n; + return( 1 ); + } + } +} + +ID +idl_firstid( IDList *idl ) +{ + if ( idl == NULL || idl->b_nids == 0 ) { + return( NOID ); + } + + if ( ALLIDS( idl ) ) { + return( idl->b_nids == 1 ? NOID : 1 ); + } + + return( idl->b_ids[0] ); +} + +ID +idl_nextid( IDList *idl, ID id ) +{ + NIDS i; + + if ( ALLIDS( idl ) ) { + return( ++id < idl->b_nids ? id : NOID ); + } + + for ( i = 0; i < idl->b_nids && idl->b_ids[i] < id; i++ ) { + ; /* NULL */ + } + i++; + + if ( i >= idl->b_nids ) { + return( NOID ); + } else { + return( idl->b_ids[i] ); + } +} + +/* Make an ID list iterator */ +idl_iterator idl_iterator_init(const IDList *idl) +{ + return (idl_iterator) 0; +} + +idl_iterator idl_iterator_increment(idl_iterator *i) +{ + size_t t = (size_t) *i; + t += 1; + *i = (idl_iterator) t; + return *i; +} + +idl_iterator idl_iterator_decrement(idl_iterator *i) +{ + size_t t = (size_t) *i; + t -= 1; + *i = (idl_iterator) t; + return *i; +} + +ID idl_iterator_dereference(idl_iterator i, const IDList *idl) +{ + if ( (NULL == idl) || (i >= idl->b_nids)) { + return NOID; + } + if (ALLIDS(idl)) { + return (ID) i + 1; + } else { + return idl->b_ids[i]; + } +} + +ID idl_iterator_dereference_increment(idl_iterator *i, const IDList *idl) +{ + ID t = idl_iterator_dereference(*i,idl); + idl_iterator_increment(i); + return t; +} + diff --git a/ldap/servers/slapd/back-ldbm/idl_new.c b/ldap/servers/slapd/back-ldbm/idl_new.c new file mode 100644 index 00000000..d2038261 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/idl_new.c @@ -0,0 +1,671 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +/* New IDL code for new indexing scheme */ + +/* Note to future editors: + This file is now full of redundant code + (the DB_ALLIDS_ON_WRITE==true and DB_USE_BULK_FETCH==false code). + It should be stripped out at the beginning of a + major release cycle. + */ + +#include "back-ldbm.h" + +static char* filename = "idl_new.c"; + +/* Bulk fetch feature first in DB 3.3 */ +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 3300 +#define DB_USE_BULK_FETCH 1 +#define BULK_FETCH_BUFFER_SIZE (8*1024) +#else +#undef DB_USE_BULK_FETCH +#endif + +/* We used to implement allids for inserts, but that's a bad idea. + Why ? Because: + 1) Allids results in hard to understand query behavior. + 2) The get() calls needed to check for allids on insert cost performance. + 3) Tests show that there is no significant performance benefit to having allids on writes, + either for updates or searches. + Set this to revert to that code */ +/* #undef DB_ALLIDS_ON_WRITE */ +/* We still enforce allids threshold on reads, to save time and space fetching vast id lists */ +#define DB_ALLIDS_ON_READ 1 + +#if !defined(DB_NEXT_DUP) +#define DB_NEXT_DUP 0 +#endif +#if !defined(DB_GET_BOTH) +#define DB_GET_BOTH 0 +#endif + +/* Structure used to hide private idl-specific data in the attrinfo object */ +struct idl_private { + size_t idl_allidslimit; + int dummy; +}; + +static int idl_tune = DEFAULT_IDL_TUNE; /* tuning parameters for IDL code */ +/* Currently none for new IDL code */ + +#if defined(DB_ALLIDS_ON_WRITE) +static int idl_new_store_allids(backend *be, DB *db, DBT *key, DB_TXN *txn); +#endif + +void idl_new_set_tune(int val) +{ + idl_tune = val; +} + +int idl_new_get_tune() { + return idl_tune; +} + +/* Append an ID to an IDL, realloc-ing the space if needs be */ +/* ID presented is not to be already in the IDL. */ +static int +idl_append_extend( IDList **orig_idl, ID id) +{ + IDList *idl = *orig_idl; + + if (idl == NULL) { + idl = idl_alloc(1); + idl_append(idl, id); + + *orig_idl = idl; + return 0; + } + + if ( idl->b_nids == idl->b_nmax ) { + size_t x = 0; + /* No more room, need to extend */ + /* Allocate new IDL with twice the space of this one */ + IDList *idl_new = NULL; + idl_new = idl_alloc(idl->b_nmax * 2); + if (NULL == idl_new) { + return ENOMEM; + } + /* copy over the existing contents */ + idl_new->b_nids = idl->b_nids; + for (x = 0; x < idl->b_nids;x++) { + idl_new->b_ids[x] = idl->b_ids[x]; + } + idl_free(idl); + idl = idl_new; + } + + idl->b_ids[idl->b_nids] = id; + idl->b_nids++; + *orig_idl = idl; + + return 0; +} + +size_t idl_new_get_allidslimit(struct attrinfo *a) +{ + idl_private *priv = NULL; + + PR_ASSERT(NULL != a); + PR_ASSERT(NULL != a->ai_idl); + + priv = a->ai_idl; + + return priv->idl_allidslimit; +} + +/* routine to initialize the private data used by the IDL code per-attribute */ +int idl_new_init_private(backend *be,struct attrinfo *a) +{ + idl_private *priv = NULL; + struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private; + + PR_ASSERT(NULL != a); + PR_ASSERT(NULL == a->ai_idl); + + priv = (idl_private*) slapi_ch_calloc(sizeof(idl_private),1); + if (NULL == priv) { + return -1; /* Memory allocation failure */ + } + priv->idl_allidslimit = li->li_allidsthreshold; + /* Initialize the structure */ + a->ai_idl = (void*)priv; + return 0; +} + +/* routine to release resources used by IDL private data structure */ +int idl_new_release_private(struct attrinfo *a) +{ + PR_ASSERT(NULL != a); + if (NULL != a->ai_idl) + { + slapi_ch_free((void**)&(a->ai_idl) ); + } + return 0; +} + +IDList * idl_new_fetch( + backend *be, + DB* db, + DBT *inkey, + DB_TXN *txn, + struct attrinfo *a, + int *flag_err +) +{ + int ret = 0; + DBC *cursor = NULL; + IDList *idl = NULL; + DBT key = {0}; + DBT data = {0}; + ID id = 0; + size_t count = 0; +#ifdef DB_USE_BULK_FETCH + /* beware that a large buffer on the stack might cause a stack overflow on some platforms */ + char buffer[BULK_FETCH_BUFFER_SIZE]; + void *ptr; + DBT dataret = {0}; +#endif + + if (NEW_IDL_NOOP == *flag_err) + { + *flag_err = 0; + return NULL; + } + + /* Make a cursor */ + ret = db->cursor(db,txn,&cursor,0); + if (0 != ret) { + ldbm_nasty(filename,1,ret); + cursor = NULL; + goto error; + } +#ifdef DB_USE_BULK_FETCH + data.ulen = sizeof(buffer); + data.size = sizeof(buffer); + data.data = buffer; + data.flags = DB_DBT_USERMEM; +#else + data.ulen = sizeof(id); + data.size = sizeof(id); + data.data = &id; + data.flags = DB_DBT_USERMEM; +#endif + + /* + * We're not expecting the key to change in value + * so we can just use the input key as a buffer. + * This avoids memory management of the key. + */ + key.ulen = inkey->size; + key.size = inkey->size; + key.data = inkey->data; + key.flags = DB_DBT_USERMEM; + + /* Position cursor at the first matching key */ +#ifdef DB_USE_BULK_FETCH + ret = cursor->c_get(cursor,&key,&data,DB_SET|DB_MULTIPLE); +#else + ret = cursor->c_get(cursor,&key,&data,DB_SET); +#endif + if (0 != ret) { + if (DB_NOTFOUND == ret) { + ret = 0; + } else { +#ifdef DB_USE_BULK_FETCH + if (ret == ENOMEM) { + LDAPDebug(LDAP_DEBUG_ANY, "database index is corrupt; " + "data item for key %s is too large for our buffer " + "(need=%d actual=%d)\n", + key.data, data.size, data.ulen); + } +#endif + ldbm_nasty(filename,2,ret); + } + goto error; /* Not found is OK, return NULL IDL */ + } + + /* Iterate over the duplicates, amassing them into an IDL */ +#ifdef DB_USE_BULK_FETCH + for (;;) { + + DB_MULTIPLE_INIT(ptr, &data); + + for (;;) { + DB_MULTIPLE_NEXT(ptr, &data, dataret.data, dataret.size); + if (dataret.data == NULL) break; + if (ptr == NULL) break; + + if (dataret.size != sizeof(ID)) { + LDAPDebug(LDAP_DEBUG_ANY, "database index is corrupt; " + "key %s has a data item with the wrong size (%d)\n", + key.data, dataret.size, 0); + goto error; + } + memcpy(&id, dataret.data, sizeof(ID)); + + /* we got another ID, add it to our IDL */ + idl_append_extend(&idl, id); + + count++; + } + + LDAPDebug(LDAP_DEBUG_TRACE, "bulk fetch buffer nids=%d\n", count, 0, 0); + + ret = cursor->c_get(cursor,&key,&data,DB_NEXT_DUP|DB_MULTIPLE); + if (0 != ret) { + break; + } +#if defined(DB_ALLIDS_ON_READ) + /* enforce the allids read limit */ + if (NEW_IDL_NO_ALLID != *flag_err && + NULL != a && count > idl_new_get_allidslimit(a)) { + idl->b_nids = 1; + idl->b_ids[0] = ALLID; + ret = DB_NOTFOUND; /* fool the code below into thinking that we finished the dups */ + break; + } +#endif + } +#else + for (;;) { + ret = cursor->c_get(cursor,&key,&data,DB_NEXT_DUP); + count++; + if (0 != ret) { + break; + } + /* we got another ID, add it to our IDL */ + idl_append_extend(&idl, id); +#if defined(DB_ALLIDS_ON_READ) + /* enforce the allids read limit */ + if (count > idl_new_get_allidslimit(a)) { + idl->b_nids = 1; + idl->b_ids[0] = ALLID; + ret = DB_NOTFOUND; /* fool the code below into thinking that we finished the dups */ + break; + } +#endif + } +#endif + + if (ret != DB_NOTFOUND) { + idl_free(idl); idl = NULL; + ldbm_nasty(filename,59,ret); + goto error; + } + + ret = 0; + + /* check for allids value */ + if (idl != NULL && idl->b_nids == 1 && idl->b_ids[0] == ALLID) { + idl_free(idl); + idl = idl_allids(be); + LDAPDebug(LDAP_DEBUG_TRACE, "idl_new_fetch %s returns allids\n", + key.data, 0, 0); + } else { + LDAPDebug(LDAP_DEBUG_TRACE, "idl_new_fetch %s returns nids=%lu\n", + key.data, (u_long)IDL_NIDS(idl), 0); + } + +error: + /* Close the cursor */ + if (NULL != cursor) { + if (0 != cursor->c_close(cursor)) { + ldbm_nasty(filename,3,ret); + } + } + *flag_err = ret; + return idl; +} + +int idl_new_insert_key( + backend *be, + DB* db, + DBT *key, + ID id, + DB_TXN *txn, + struct attrinfo *a, + int *disposition +) +{ + int ret = 0; + DBT data = {0}; + +#if defined(DB_ALLIDS_ON_WRITE) + DBC *cursor = NULL; + db_recno_t count; + ID tmpid = 0; + /* Make a cursor */ + ret = db->cursor(db,txn,&cursor,0); + if (0 != ret) { + ldbm_nasty(filename,58,ret); + cursor = NULL; + goto error; + } + data.ulen = sizeof(id); + data.size = sizeof(id); + data.flags = DB_DBT_USERMEM; + data.data = &tmpid; + ret = cursor->c_get(cursor,key,&data,DB_SET); + if (0 == ret) { + if (tmpid == ALLID) { + if (NULL != disposition) { + *disposition = IDL_INSERT_ALLIDS; + } + goto error; /* allid: don't bother inserting any more */ + } + } else if (DB_NOTFOUND != ret) { + ldbm_nasty(filename,12,ret); + goto error; + } + if (NULL != disposition) { + *disposition = IDL_INSERT_NORMAL; + } + + data.data = &id; + + /* insert it */ + ret = cursor->c_put(cursor, key, &data, DB_NODUPDATA); + if (0 != ret) { + if (DB_KEYEXIST == ret) { + /* this is okay */ + ret = 0; + } else { + ldbm_nasty(filename,50,ret); + } + } else { + /* check for allidslimit exceeded in database */ + if (cursor->c_count(cursor, &count, 0) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "could not obtain count for key %s\n", + key->data, 0, 0); + goto error; + } + if ((size_t)count > idl_new_get_allidslimit(a)) { + LDAPDebug(LDAP_DEBUG_TRACE, "allidslimit exceeded for key %s\n", + key->data, 0, 0); + cursor->c_close(cursor); + cursor = NULL; + if ((ret = idl_new_store_allids(be, db, key, txn)) == 0) { + if (NULL != disposition) { + *disposition = IDL_INSERT_NOW_ALLIDS; + } + } + } +error: + /* Close the cursor */ + if (NULL != cursor) { + if (0 != cursor->c_close(cursor)) { + ldbm_nasty(filename,56,ret); + } + } +#else + data.ulen = sizeof(id); + data.size = sizeof(id); + data.flags = DB_DBT_USERMEM; + data.data = &id; + + if (NULL != disposition) { + *disposition = IDL_INSERT_NORMAL; + } + + ret = db->put(db, txn, key, &data, DB_NODUPDATA); + if (0 != ret) { + if (DB_KEYEXIST == ret) { + /* this is okay */ + ret = 0; + } else { + ldbm_nasty(filename,50,ret); + } + } +#endif + + + return ret; +} + +int idl_new_delete_key( + backend *be, + DB *db, + DBT *key, + ID id, + DB_TXN *txn, + struct attrinfo *a +) +{ + int ret = 0; + DBC *cursor = NULL; + DBT data = {0}; + ID tmpid = 0; + + /* Make a cursor */ + ret = db->cursor(db,txn,&cursor,0); + if (0 != ret) { + ldbm_nasty(filename,21,ret); + cursor = NULL; + goto error; + } + data.ulen = sizeof(id); + data.size = sizeof(id); + data.flags = DB_DBT_USERMEM; + data.data = &tmpid; + ret = cursor->c_get(cursor,key,&data,DB_SET); + if (0 == ret) { + if (tmpid == ALLID) { + goto error; /* allid: never delete it */ + } + } else if (DB_NOTFOUND != ret) { + ldbm_nasty(filename,22,ret); + goto error; + } + + /* Position cursor at the key, value pair */ + data.data = &id; + ret = cursor->c_get(cursor,key,&data,DB_GET_BOTH); + if (0 != ret) { + if (DB_NOTFOUND == ret) { + ret = 0; /* Not Found is OK, return immediately */ + } else { + ldbm_nasty(filename,23,ret); + } + goto error; + } + /* We found it, so delete it */ + ret = cursor->c_del(cursor,0); +error: + /* Close the cursor */ + if (NULL != cursor) { + if (0 != cursor->c_close(cursor)) { + ldbm_nasty(filename,24,ret); + } + } + return ret; +} + +#if defined(DB_ALLIDS_ON_WRITE) +static int idl_new_store_allids(backend *be, DB *db, DBT *key, DB_TXN *txn) +{ + int ret = 0; + DBC *cursor = NULL; + DBT data = {0}; + ID id = 0; + + /* Make a cursor */ + ret = db->cursor(db,txn,&cursor,0); + if (0 != ret) { + ldbm_nasty(filename,31,ret); + cursor = NULL; + goto error; + } + data.ulen = sizeof(ID); + data.size = sizeof(ID); + data.data = &id; + data.flags = DB_DBT_USERMEM; + + /* Position cursor at the key */ + ret = cursor->c_get(cursor,key,&data,DB_SET); + if (ret == 0) { + /* We found it, so delete all duplicates */ + ret = cursor->c_del(cursor,0); + while (0 == ret) { + ret = cursor->c_get(cursor,key,&data,DB_NEXT_DUP); + if (0 != ret) { + break; + } + ret = cursor->c_del(cursor,0); + } + if (0 != ret && DB_NOTFOUND != ret) { + ldbm_nasty(filename,54,ret); + goto error; + } else { + ret = 0; + } + } else { + if (DB_NOTFOUND == ret) { + ret = 0; /* Not Found is OK */ + } else { + ldbm_nasty(filename,32,ret); + goto error; + } + } + + /* store the ALLID value */ + id = ALLID; + ret = cursor->c_put(cursor, key, &data, DB_NODUPDATA); + if (0 != ret) { + ldbm_nasty(filename,53,ret); + goto error; + } + + LDAPDebug(LDAP_DEBUG_TRACE, "key %s has been set to allids\n", + key->data, 0, 0); + +error: + /* Close the cursor */ + if (NULL != cursor) { + if (0 != cursor->c_close(cursor)) { + ldbm_nasty(filename,33,ret); + } + } + return ret; + /* If this function is called in "no-allids" mode, then it's a bug */ + ldbm_nasty(filename,63,0); + return -1; +} +#endif + +int idl_new_store_block( + backend *be, + DB *db, + DBT *key, + IDList *idl, + DB_TXN *txn, + struct attrinfo *a +) +{ + int ret = 0; + DBC *cursor = NULL; + DBT data = {0}; + ID id = 0; + size_t x = 0; +#if defined(DB_ALLIDS_ON_WRITE) + db_recno_t count; +#endif + + if (NULL == idl) + { + return ret; + } + + /* + * Really we need an extra entry point to the DB here, which + * inserts a list of duplicate keys. In the meantime, we'll + * just do it by brute force. + */ + +#if defined(DB_ALLIDS_ON_WRITE) + /* allids check on input idl */ + if (ALLIDS(idl) || (idl->b_nids > (ID)idl_new_get_allidslimit(a))) { + return idl_new_store_allids(be, db, key, txn); + } +#endif + + /* Make a cursor */ + ret = db->cursor(db,txn,&cursor,0); + if (0 != ret) { + ldbm_nasty(filename,41,ret); + cursor = NULL; + goto error; + } + + /* initialize data DBT */ + data.data = &id; + data.ulen = sizeof(id); + data.size = sizeof(id); + data.flags = DB_DBT_USERMEM; + + /* Position cursor at the key, value pair */ + ret = cursor->c_get(cursor,key,&data,DB_GET_BOTH); + if (ret == DB_NOTFOUND) { + ret = 0; + } else if (ret != 0) { + ldbm_nasty(filename,47,ret); + goto error; + } + + /* Iterate over the IDs in the idl */ + for (x = 0; x < idl->b_nids; x++) { + /* insert an id */ + id = idl->b_ids[x]; + ret = cursor->c_put(cursor, key, &data, DB_NODUPDATA); + if (0 != ret) { + if (DB_KEYEXIST == ret) { + ret = 0; /* exist is okay */ + } else { + ldbm_nasty(filename,48,ret); + goto error; + } + } + } +#if defined(DB_ALLIDS_ON_WRITE) + /* check for allidslimit exceeded in database */ + if (cursor->c_count(cursor, &count, 0) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "could not obtain count for key %s\n", + key->data, 0, 0); + goto error; + } + if ((size_t)count > idl_new_get_allidslimit(a)) { + LDAPDebug(LDAP_DEBUG_TRACE, "allidslimit exceeded for key %s\n", + key->data, 0, 0); + cursor->c_close(cursor); + cursor = NULL; + ret = idl_new_store_allids(be, db, key, txn); + } +#endif + +error: + /* Close the cursor */ + if (NULL != cursor) { + if (0 != cursor->c_close(cursor)) { + ldbm_nasty(filename,49,ret); + } + } + return ret; +} + +/* idl_new_compare_dups: comparing ID, pass to libdb for callback */ +int idl_new_compare_dups( +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 3200 + DB *db, +#endif + const DBT *a, + const DBT *b +) +{ + ID a_copy, b_copy; + memmove(&a_copy, a->data, sizeof(ID)); + memmove(&b_copy, b->data, sizeof(ID)); + return a_copy - b_copy; +} diff --git a/ldap/servers/slapd/back-ldbm/idl_shim.c b/ldap/servers/slapd/back-ldbm/idl_shim.c new file mode 100644 index 00000000..2332ce1b --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/idl_shim.c @@ -0,0 +1,124 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* Shim which forwards IDL calls to the appropriate implementation */ + +#include "back-ldbm.h" + +static int idl_new = 0; /* non-zero if we're doing new IDL style */ + + +void idl_old_set_tune(int val); +int idl_old_get_tune(); +int idl_old_init_private(backend *be, struct attrinfo *a); +int idl_old_release_private(struct attrinfo *a); +size_t idl_old_get_allidslimit(struct attrinfo *a); +IDList * idl_old_fetch( backend *be, DB* db, DBT *key, DB_TXN *txn, struct attrinfo *a, int *err ); +int idl_old_insert_key( backend *be, DB* db, DBT *key, ID id, DB_TXN *txn, struct attrinfo *a,int *disposition ); +int idl_old_delete_key( backend *be, DB *db, DBT *key, ID id, DB_TXN *txn, struct attrinfo *a ); +int idl_old_store_block( backend *be,DB *db,DBT *key,IDList *idl,DB_TXN *txn,struct attrinfo *a); + + +void idl_new_set_tune(int val); +int idl_new_get_tune(); +int idl_new_init_private(backend *be, struct attrinfo *a); +int idl_new_release_private(struct attrinfo *a); +size_t idl_new_get_allidslimit(struct attrinfo *a); +IDList * idl_new_fetch( backend *be, DB* db, DBT *key, DB_TXN *txn, struct attrinfo *a, int *err ); +int idl_new_insert_key( backend *be, DB* db, DBT *key, ID id, DB_TXN *txn, struct attrinfo *a,int *disposition ); +int idl_new_delete_key( backend *be, DB *db, DBT *key, ID id, DB_TXN *txn, struct attrinfo *a ); +int idl_new_store_block( backend *be,DB *db,DBT *key,IDList *idl,DB_TXN *txn,struct attrinfo *a); + +int idl_get_idl_new() +{ + return idl_new; +} + +void idl_set_tune(int val) +{ + /* Catch idl_tune requests to use new idl code */ + if (4096 == val) { + idl_new = 1; + } else { + idl_new = 0; + } + if (idl_new) { + idl_new_set_tune(val); + } else { + idl_old_set_tune(val); + } +} + +int idl_get_tune() +{ + if (idl_new) { + return idl_new_get_tune(); + } else { + return idl_old_get_tune(); + } +} + +int idl_init_private(backend *be, struct attrinfo *a) +{ + if (idl_new) { + return idl_new_init_private(be,a); + } else { + return idl_old_init_private(be,a); + } +} + +int idl_release_private(struct attrinfo *a) +{ + if (idl_new) { + return idl_new_release_private(a); + } else { + return idl_old_release_private(a); + } +} + +size_t idl_get_allidslimit(struct attrinfo *a) +{ + if (idl_new) { + return idl_new_get_allidslimit(a); + } else { + return idl_old_get_allidslimit(a); + } +} + +IDList * idl_fetch( backend *be, DB* db, DBT *key, DB_TXN *txn, struct attrinfo *a, int *err ) +{ + if (idl_new) { + return idl_new_fetch(be,db,key,txn,a,err); + } else { + return idl_old_fetch(be,db,key,txn,a,err); + } +} + +int idl_insert_key( backend *be, DB* db, DBT *key, ID id, DB_TXN *txn, struct attrinfo *a,int *disposition ) +{ + if (idl_new) { + return idl_new_insert_key(be,db,key,id,txn,a,disposition); + } else { + return idl_old_insert_key(be,db,key,id,txn,a,disposition); + } +} + +int idl_delete_key(backend *be, DB *db, DBT *key, ID id, DB_TXN *txn, struct attrinfo *a ) +{ + if (idl_new) { + return idl_new_delete_key(be,db,key,id,txn,a); + } else { + return idl_old_delete_key(be,db,key,id,txn,a); + } +} + +int idl_store_block(backend *be,DB *db,DBT *key,IDList *idl,DB_TXN *txn,struct attrinfo *a) +{ + if (idl_new) { + return idl_new_store_block(be,db,key,idl,txn,a); + } else { + return idl_old_store_block(be,db,key,idl,txn,a); + } +} diff --git a/ldap/servers/slapd/back-ldbm/idlapi.h b/ldap/servers/slapd/back-ldbm/idlapi.h new file mode 100644 index 00000000..39fe9c15 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/idlapi.h @@ -0,0 +1,30 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +#ifndef _IDL_API_H_ +#define _IDL_API_H_ + +/* mechanics */ + +typedef IDList *(*api_idl_alloc)( NIDS nids ); +typedef void (*api_idl_insert)(IDList **idl, ID id); + +/* API ID for slapi_apib_get_interface */ + +#define IDL_v1_0_GUID "ec228d97-971d-4b9e-91b5-4f90e1841f24" + +/* API */ + +/* the api broker reserves api[0] for its use */ + +#define IDList_alloc(api, nids) \ + ((api_idl_alloc*)(api))[1](nids) + +#define IDList_insert(api, idl, id) \ + ((api_idl_insert*)(api))[2](idl, id) + + +#endif /*_IDL_API_H_*/ diff --git a/ldap/servers/slapd/back-ldbm/import-merge.c b/ldap/servers/slapd/back-ldbm/import-merge.c new file mode 100644 index 00000000..b50aaaec --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/import-merge.c @@ -0,0 +1,680 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* + * this is a bunch of routines for merging groups of db files together -- + * currently it's only used for imports (when we import into several small + * db sets for speed, then merge them). + */ + +#include "back-ldbm.h" +#include "import.h" + +struct _import_merge_thang +{ + int type; +#define IMPORT_MERGE_THANG_IDL 1 /* Values for type */ +#define IMPORT_MERGE_THANG_VLV 2 + union { + IDList *idl; /* if type == IMPORT_MERGE_THANG_IDL */ + DBT vlv_data; /* if type == IMPORT_MERGE_THANG_VLV */ + } payload; +}; +typedef struct _import_merge_thang import_merge_thang; + +struct _import_merge_queue_entry +{ + int *file_referenced_list; + import_merge_thang thang; + DBT key; + struct _import_merge_queue_entry *next; +}; +typedef struct _import_merge_queue_entry import_merge_queue_entry; + +static int import_merge_get_next_thang(backend *be, DBC *cursor, DB *db, import_merge_thang *thang, DBT *key, int type) +{ + int ret = 0; + DBT value = {0}; + + value.flags = DB_DBT_MALLOC; + key->flags = DB_DBT_MALLOC; + + thang->type = type; + if (IMPORT_MERGE_THANG_IDL == type) { + /* IDL case */ + around: + ret = cursor->c_get(cursor, key, &value, DB_NEXT_NODUP); + if (0 == ret) { + /* Check that we've not reached the beginning of continuation + * blocks */ + if (CONT_PREFIX != ((char*)key->data)[0]) { + /* If not, read the IDL using idl_fetch() */ + key->flags = DB_DBT_REALLOC; + ret = NEW_IDL_NO_ALLID; + thang->payload.idl = idl_fetch(be, db, key, NULL, NULL, &ret); + PR_ASSERT(NULL != thang->payload.idl); + } else { + free(value.data); + free(key->data); + key->flags = DB_DBT_MALLOC; + goto around; /* Just skip these */ + } + free(value.data); + } else { + if (DB_NOTFOUND == ret) { + /* This means that we're at the end of the file */ + ret = EOF; + } + } + } else { + /* VLV case */ + ret = cursor->c_get(cursor,key,&value,DB_NEXT); + if (0 == ret) { + thang->payload.vlv_data = value; + thang->payload.vlv_data.flags = 0; + key->flags = 0; + } else { + if (DB_NOTFOUND == ret) { + /* This means that we're at the end of the file */ + ret = EOF; + } + } + } + + return ret; +} + +static import_merge_queue_entry *import_merge_make_new_queue_entry(import_merge_thang *thang, DBT *key, int fileno, int passes) +{ + /* Make a new entry */ + import_merge_queue_entry *new_entry = (import_merge_queue_entry *)slapi_ch_calloc(1, sizeof(import_merge_queue_entry)); + + if (NULL == new_entry) { + return NULL; + } + new_entry->key = *key; + new_entry->thang = *thang; + new_entry->file_referenced_list = + (int *)slapi_ch_calloc(passes, sizeof(fileno)); + + if (NULL == new_entry->file_referenced_list) { + return NULL; + } + (new_entry->file_referenced_list)[fileno] = 1; + return new_entry; +} + +/* Put an IDL onto the priority queue */ +static int import_merge_insert_input_queue(backend *be, import_merge_queue_entry **queue,int fileno, DBT *key, import_merge_thang *thang,int passes) +{ + /* Walk the list, looking for a key value which is greater than or equal + * to the presented key */ + /* If an equal key is found, compute the union of the IDLs and store that + * back in the queue entry */ + /* If a key greater than is found, or no key greater than is found, insert + * a new queue entry */ + import_merge_queue_entry *current_entry = NULL; + import_merge_queue_entry *previous_entry = NULL; + + PR_ASSERT(NULL != thang); + if (NULL == *queue) { + /* Queue was empty--- put ourselves at the head */ + *queue = import_merge_make_new_queue_entry(thang,key,fileno,passes); + if (NULL == *queue) { + return -1; + } + } else { + for (current_entry = *queue; current_entry != NULL; + current_entry = current_entry->next) { + int cmp = strcmp(key->data,current_entry->key.data); + + if (0 == cmp) { + if (IMPORT_MERGE_THANG_IDL == thang->type) { /* IDL case */ + IDList *idl = thang->payload.idl; + /* Equal --- merge into the stored IDL, add file ID + * to the list */ + IDList *new_idl = + idl_union(be, current_entry->thang.payload.idl, idl); + + idl_free(current_entry->thang.payload.idl); + idl_free(idl); + current_entry->thang.payload.idl = new_idl; + /* Add this file id into the entry's referenced list */ + (current_entry->file_referenced_list)[fileno] = 1; + /* Because we merged the entries, we no longer need the + * key, so free it */ + free(key->data); + goto done; + } else { + /* VLV case, we can see exact keys, this is not a bug ! */ + /* We want to ensure that they key read most recently is + * put later in the queue than any others though */ + } + } else { + if (cmp < 0) { + /* We compare smaller than the stored key, so we should + * insert ourselves before this entry */ + break; + } else { + /* We compare greater than this entry, so we should keep + * going */ ; + } + } + previous_entry = current_entry; + } + + /* Now insert */ + { + import_merge_queue_entry *new_entry = + import_merge_make_new_queue_entry(thang, key, fileno, passes); + + if (NULL == new_entry) { + return -1; + } + + /* If not, then we must need to insert ourselves after the last + * entry */ + new_entry->next = current_entry; + if (NULL == previous_entry) { + *queue = new_entry; + } else { + previous_entry->next = new_entry; + } + } + } + +done: + return 0; +} + +static int import_merge_remove_input_queue(backend *be, import_merge_queue_entry **queue, import_merge_thang *thang,DBT *key,DBC **input_cursors, DB **input_files,int passes) +{ + import_merge_queue_entry *head = NULL; + int file_referenced = 0; + int i = 0; + int ret = 0; + + PR_ASSERT(NULL != queue); + head = *queue; + if (head == NULL) { + /* Means we've exhausted the queue---we're done */ + return EOF; + } + /* Remove the head of the queue */ + *queue = head->next; + /* Get the IDL */ + *thang = head->thang; + *key = head->key; + PR_ASSERT(NULL != thang); + /* Walk the list of referenced files, reading in the next IDL from each + * one to the queue */ + for (i = 0 ; i < passes; i++) { + import_merge_thang new_thang = {0}; + DBT new_key = {0}; + + file_referenced = (head->file_referenced_list)[i]; + if (file_referenced) { + ret = import_merge_get_next_thang(be, input_cursors[i], + input_files[i], &new_thang, &new_key, thang->type); + if (0 != ret) { + if (EOF == ret) { + /* Means that we walked off the end of the list, + * do nothing */ + ret = 0; + } else { + /* Some other error */ + break; + } + } else { + /* This function is responsible for any freeing needed */ + import_merge_insert_input_queue(be, queue, i, &new_key, + &new_thang, passes); + } + } + } + slapi_ch_free( (void**)&(head->file_referenced_list)); + slapi_ch_free( (void**)&head); + + return ret; +} + +static int import_merge_open_input_cursors(DB**files, int passes, DBC ***cursors) +{ + int i = 0; + int ret = 0; + *cursors = (DBC**)slapi_ch_calloc(passes,sizeof(DBC*)); + if (NULL == *cursors) { + return -1; + } + + for (i = 0; i < passes; i++) { + DB *pDB = files[i]; + DBC *pDBC = NULL; + if (NULL != pDB) { + /* Try to open a cursor onto the file */ + ret = pDB->cursor(pDB,NULL,&pDBC,0); + if (0 != ret) { + break; + } else { + (*cursors)[i] = pDBC; + } + } + } + + return ret; +} + +static int import_count_merge_input_files(ldbm_instance *inst, + char *indexname, int passes, int *number_found, int *pass_number) +{ + int i = 0; + int found_one = 0; + + *number_found = 0; + *pass_number = 0; + + for (i = 0; i < passes; i++) { + int fd; + char *filename = NULL; + size_t filename_length = strlen(inst->inst_dir_name) + 1 + + strlen(indexname) + 10 ; + + filename = slapi_ch_malloc(filename_length); + if (NULL == filename) { + return -1; + } + sprintf(filename, "%s/%s.%d%s", inst->inst_dir_name, indexname, i+1, + LDBM_FILENAME_SUFFIX); + fd = dblayer_open_huge_file(filename, O_RDONLY, 0); + slapi_ch_free( (void**)&filename); + if (fd >= 0) { + close(fd); + if (found_one == 0) { + *pass_number = i+1; + } + found_one = 1; + (*number_found)++; + } else { + ; /* Not finding a file is OK */ + } + } + + return 0; +} + +static int import_open_merge_input_files(backend *be, char *indexname, + int passes, DB ***input_files, int *number_found, int *pass_number) +{ + int i = 0; + int ret = 0; + int found_one = 0; + + *number_found = 0; + *pass_number = 0; + *input_files = (DB**)slapi_ch_calloc(passes,sizeof(DB*)); + if (NULL == *input_files) { + /* Memory allocation error */ + return -1; + } + for (i = 0; i < passes; i++) { + DB *pDB = NULL; + char *filename = NULL; + size_t filename_length = strlen(indexname) + 10 ; + + filename = slapi_ch_malloc(filename_length); + if (NULL == filename) { + return -1; + } + sprintf(filename,"%s.%d", indexname, i+1); + + if (vlv_isvlv(filename)) { + ret = dblayer_open_file(be, filename, 0, INDEX_VLV, &pDB); + } else { + ret = dblayer_open_file(be, filename, 0, 0, &pDB); + } + + slapi_ch_free( (void**)&filename); + if (0 == ret) { + if (found_one == 0) { + *pass_number = i+1; + } + found_one = 1; + (*number_found)++; + (*input_files)[i] = pDB; + } else { + if (ENOENT == ret) { + ret = 0; /* Not finding a file is OK */ + } else { + break; + } + } + } + + return ret; +} + +/* Performs the n-way merge on one file */ +static int import_merge_one_file(ImportWorkerInfo *worker, int passes, + int *key_count) +{ + ldbm_instance *inst = worker->job->inst; + backend *be = inst->inst_be; + DB *output_file = NULL; + int ret = 0; + int preclose_ret = 0; + int number_found = 0; + int pass_number = 0; + + PR_ASSERT(NULL != inst); + + /* Try to open all the input files. + If we can't open file a file, we assume that is + because there was no data in it. */ + ret = import_count_merge_input_files(inst, worker->index_info->name, + passes, &number_found, &pass_number); + if (0 != ret) { + goto error; + } + /* If there were no input files, then we're finished ! */ + if (0 == number_found) { + ret = 0; + goto error; + } + /* Special-case where there's only one input file---just rename it */ + if (1 == number_found) { + char *newname = NULL; + char *oldname = NULL; + + ret = import_make_merge_filenames(inst->inst_dir_name, + worker->index_info->name, pass_number, &oldname, &newname); + if (0 != ret) { + import_log_notice(worker->job, "Failed making filename in merge"); + goto error; + } + ret = PR_Rename(newname,oldname); + if (0 != ret) { + PRErrorCode prerr = PR_GetError(); + import_log_notice(worker->job, "Failed to rename file \"%s\" to \"%s\" " + "in merge, " SLAPI_COMPONENT_NAME_NSPR " error %d (%s)", + oldname, newname, prerr, slapd_pr_strerror(prerr)); + slapi_ch_free( (void**)&newname); + slapi_ch_free( (void**)&oldname); + goto error; + } + slapi_ch_free( (void**)&newname); + slapi_ch_free( (void**)&oldname); + *key_count = -1; + } else { + /* We really need to merge */ + import_merge_queue_entry *merge_queue = NULL; + DB **input_files = NULL; + DBC **input_cursors = NULL; + DBT key = {0}; + import_merge_thang thang = {0}; + int i = 0; + int not_finished = 1; + int vlv_index = (INDEX_VLV == worker->index_info->ai->ai_indexmask); + +#if 0 + /* Close and re-open regions, bugs otherwise */ + ret = dblayer_close(inst->inst_li, DBLAYER_IMPORT_MODE); + if (0 != ret) { + if (ENOSPC == ret) { + import_log_notice(worker->job, "FAILED: NO DISK SPACE LEFT"); + } else { + import_log_notice(worker->job, "MERGE FAIL 8 %d", ret); + } + return ret; + } + ret = dblayer_start(inst->inst_li, DBLAYER_IMPORT_MODE); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 9"); + return ret; + } + ret = dblayer_instance_start(be, DBLAYER_IMPORT_MODE); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 9A"); + return ret; + } +#else + /* we have reason to believe that it's okay to leave the region files + * open in db3.x, since they track which files are opened and closed. + * if we had to close the region files, we'd have to take down the + * whole backend and defeat the purpose of an online import --- + * baaad medicine. + */ + ret = dblayer_instance_close(be); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 8i %d\n", ret); + return ret; + } + ret = dblayer_instance_start(be, DBLAYER_IMPORT_MODE); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 8j %d\n", ret); + return ret; + } +#endif + + ret = import_open_merge_input_files(be, worker->index_info->name, + passes, &input_files, &number_found, &pass_number); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 10"); + return ret; + } + + ret = dblayer_open_file(be, worker->index_info->name, 1, + vlv_index ? INDEX_VLV : 0, &output_file); + if (0 != ret) { + import_log_notice(worker->job, "Failed to open output file for " + "index %s in merge", worker->index_info->name); + goto error; + } + + /* OK, so we now have input and output files open and can proceed to + * merge */ + /* We want to pre-fill the input IDL queue */ + /* Open cursors onto the input files */ + ret = import_merge_open_input_cursors(input_files, passes, + &input_cursors); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 2 %s %d", + worker->index_info->name, ret); + goto error; + } + + /* Now read from the first location in each file and insert into the + * queue */ + for (i = 0; i < passes; i++) if (input_files[i]) { + import_merge_thang prime_thang = {0}; + + /* Read an IDL from the file */ + ret = import_merge_get_next_thang(be, input_cursors[i], + input_files[i], &prime_thang, &key, + vlv_index ? IMPORT_MERGE_THANG_VLV : IMPORT_MERGE_THANG_IDL); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 1 %s %d", + worker->index_info->name, ret); + goto error; + } + /* Put it on the queue */ + ret = import_merge_insert_input_queue(be, &merge_queue, i,& key, + &prime_thang, passes); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 0 %s", + worker->index_info->name); + goto error; + } + } + + /* We now have a pre-filled queue, so we may now proceed to remove the + head entry and write it to the output file, and repeat this process + until we've finished reading all the input data */ + while (not_finished && (0 == ret) ) { + ret = import_merge_remove_input_queue(be, &merge_queue, &thang, + &key, input_cursors, input_files, passes); + if (0 != ret) { + /* Have we finished cleanly ? */ + if (EOF == ret) { + not_finished = 0; + } else { + import_log_notice(worker->job, "MERGE FAIL 3 %s, %d", + worker->index_info->name, ret); + } + } else { + /* Write it out */ + (*key_count)++; + if (vlv_index) { + /* Write the vlv index */ + ret = output_file->put(output_file, NULL, &key, + &(thang.payload.vlv_data),0); + free(thang.payload.vlv_data.data); + thang.payload.vlv_data.data = NULL; + } else { + /* Write the IDL index */ + ret = idl_store_block(be, output_file, &key, + thang.payload.idl, NULL, worker->index_info->ai); + /* Free the key we got back from the queue */ + idl_free(thang.payload.idl); + thang.payload.idl = NULL; + } + free(key.data); + key.data = NULL; + if (0 != ret) { + /* Failed to write--- most obvious cause being out of + disk space, let's make sure that we at least print a + sensible error message right here. The caller should + really handle this properly, but we're always bad at + this. */ + if (ret == DB_RUNRECOVERY || ret == ENOSPC) { + import_log_notice(worker->job, "OUT OF SPACE ON DISK, " + "failed writing index file %s", + worker->index_info->name); + } else { + import_log_notice(worker->job, "Failed to write " + "index file %s, errno=%d (%s)\n", + worker->index_info->name, errno, + dblayer_strerror(errno)); + } + } + } + } + preclose_ret = ret; + /* Now close the files */ + dblayer_close_file(output_file); + /* Close the cursors */ + /* Close and delete the files */ + for (i = 0; i < passes; i++) { + DBC *cursor = input_cursors[i]; + DB *db = input_files[i]; + if (NULL != db) { + PR_ASSERT(NULL != cursor); + ret = cursor->c_close(cursor); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 4"); + } + ret = dblayer_close_file(db); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 5"); + } + /* Now make the filename and delete the file */ + { + char *newname = NULL; + char *oldname = NULL; + ret = import_make_merge_filenames(inst->inst_dir_name, + worker->index_info->name, i+1, &oldname, &newname); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 6"); + } else { + ret = PR_Delete(newname); + if (0 != ret) { + import_log_notice(worker->job, "MERGE FAIL 7"); + } + slapi_ch_free( (void**)&newname); + slapi_ch_free( (void**)&oldname); + } + } + } + } + if (preclose_ret != 0) ret = preclose_ret; + slapi_ch_free( (void**)&input_files); + slapi_ch_free( (void**)&input_cursors); + } + if (EOF == ret) { + ret = 0; + } + +error: + return ret; +} + +/********** the real deal here: **********/ + +/* Our mission here is as follows: + * for each index job except entrydn and id2entry: + * open all the pass files + * open a new output file + * iterate cursors over all of the input files picking each distinct + * key and combining the input IDLs into a merged IDL. Put that + * IDL to the output file. + */ +int import_mega_merge(ImportJob *job) +{ + ImportWorkerInfo *current_worker = NULL; + int ret = 0; + time_t beginning = 0; + time_t end = 0; + int passes = job->current_pass; + + if (1 == job->number_indexers) { + import_log_notice(job, "Beginning %d-way merge of one file...", passes, + job->number_indexers); + } else { + import_log_notice(job, "Beginning %d-way merge of up to %lu files...", + passes, job->number_indexers); + } + + time(&beginning); + /* Iterate over the files */ + for (current_worker = job->worker_list; + (ret == 0) && (current_worker != NULL); + current_worker = current_worker->next) { + /* We need to ignore the primary index */ + if ((current_worker->work_type != FOREMAN) && + (current_worker->work_type != PRODUCER)) { + time_t file_beginning = 0; + time_t file_end = 0; + int key_count = 0; + + time(&file_beginning); + ret = import_merge_one_file(current_worker,passes,&key_count); + time(&file_end); + if (key_count == 0) { + import_log_notice(job, "No files to merge for \"%s\".", + current_worker->index_info->name); + } else { + if (-1 == key_count) { + import_log_notice(job, "Merged \"%s\": Simple merge - " + "file renamed.", + current_worker->index_info->name); + } else { + import_log_notice(job, "Merged \"%s\": %d keys merged " + "in %ld seconds.", + current_worker->index_info->name, + key_count, file_end-file_beginning); + } + } + } + } + + time(&end); + if (0 == ret) { + int seconds_to_merge = end - beginning; + + import_log_notice(job, "Merging completed in %d seconds.", + seconds_to_merge); + } + + return ret; +} diff --git a/ldap/servers/slapd/back-ldbm/import-threads.c b/ldap/servers/slapd/back-ldbm/import-threads.c new file mode 100644 index 00000000..413eaca6 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/import-threads.c @@ -0,0 +1,1992 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* + * the threads that make up an import: + * producer (1) + * foreman (1) + * worker (N: 1 for each index) + * + * a wire import (aka "fast replica" import) won't have a producer thread. + */ + +#include "back-ldbm.h" +#include "vlv_srch.h" +#include "import.h" +#ifdef XP_WIN32 +#define STDIN_FILENO 0 +#endif + + +static struct backentry *import_make_backentry(Slapi_Entry *e, ID id) +{ + struct backentry *ep = backentry_alloc(); + + if (NULL != ep) { + ep->ep_entry = e; + ep->ep_id = id; + } + return ep; +} + +static void import_decref_entry(struct backentry *ep) +{ + PR_AtomicDecrement(&(ep->ep_refcnt)); + PR_ASSERT(ep->ep_refcnt >= 0); +} + +/* generate uniqueid if requested */ +static void import_generate_uniqueid(ImportJob *job, Slapi_Entry *e) +{ + const char *uniqueid = slapi_entry_get_uniqueid(e); + int rc; + + if (!uniqueid && (job->uuid_gen_type != SLAPI_UNIQUEID_GENERATE_NONE)) { + char *newuniqueid; + + /* generate id based on dn */ + if (job->uuid_gen_type == SLAPI_UNIQUEID_GENERATE_NAME_BASED) { + char *dn = slapi_entry_get_dn(e); + + rc = slapi_uniqueIDGenerateFromNameString(&newuniqueid, + job->uuid_namespace, dn, strlen(dn)); + } else { + /* time based */ + rc = slapi_uniqueIDGenerateString(&newuniqueid); + } + + if (rc == UID_SUCCESS) { + slapi_entry_set_uniqueid (e, newuniqueid); + } else { + char ebuf[BUFSIZ]; + LDAPDebug( LDAP_DEBUG_ANY, + "import_generate_uniqueid: failed to generate " + "uniqueid for %s; error=%d.\n", + escape_string(slapi_entry_get_dn_const(e), ebuf), rc, 0 ); + } + } +} + + +/********** BETTER LDIF PARSER **********/ + + +/* like the function in libldif, except this one doesn't need to use + * FILE (which breaks on various platforms for >4G files or large numbers + * of open files) + */ +#define LDIF_BUFFER_SIZE 8192 + +typedef struct { + char *b; /* buffer */ + size_t size; /* how full the buffer is */ + size_t offset; /* where the current entry starts */ +} ldif_context; + +static void import_init_ldif(ldif_context *c) +{ + c->size = c->offset = 0; + c->b = NULL; +} + +static void import_free_ldif(ldif_context *c) +{ + if (c->b) + FREE(c->b); + import_init_ldif(c); +} + +static char *import_get_entry(ldif_context *c, int fd, int *lineno) +{ + int ret; + int done = 0, got_lf = 0; + size_t bufSize = 0, bufOffset = 0, i; + char *buf = NULL; + + while (!done) { + + /* If there's no data in the buffer, get some */ + if ((c->size == 0) || (c->offset == c->size)) { + /* Do we even have a buffer ? */ + if (! c->b) { + c->b = slapi_ch_malloc(LDIF_BUFFER_SIZE); + if (! c->b) + return NULL; + } + ret = read(fd, c->b, LDIF_BUFFER_SIZE); + if (ret < 0) { + /* Must be error */ + goto error; + } else if (ret == 0) { + /* eof */ + if (buf) { + /* last entry */ + buf[bufOffset] = 0; + return buf; + } + return NULL; + } else { + /* read completed OK */ + c->size = ret; + c->offset = 0; + } + } + + /* skip blank lines at start of entry */ + if (bufOffset == 0) { + size_t n; + char *p; + + for (n = c->offset, p = c->b + n; n < c->size; n++, p++) { + if (!(*p == '\r' || *p == '\n' || *p == ' '|| *p == '\t')) + break; + } + c->offset = n; + if (c->offset == c->size) continue; + } + + i = c->offset; + while (!done && (i < c->size)) { + /* scan forward in the buffer, looking for the end of the entry */ + while ((i < c->size) && (c->b[i] != '\n')) + i++; + + if ((i < c->size) && (c->b[i] == '\n')) { + if (got_lf && ((i == 0) || ((i == 1) && (c->b[0] == '\r')))) { + /* saw an lf at the end of the last buffer */ + i++, (*lineno)++; + done = 1; + got_lf = 0; + break; + } + got_lf = 0; + (*lineno)++; + /* is this the end? (need another linefeed) */ + if (++i < c->size) { + if (c->b[i] == '\n') { + /* gotcha! */ + i++, (*lineno)++; + done = 1; + } else if (c->b[i] == '\r') { + if (++i < c->size) { + if (c->b[i] == '\n') { + /* gotcha! (nt) */ + i++, (*lineno)++; + done = 1; + } + } else { + got_lf = 1; + } + } + } else { + /* lf at the very end of the buffer */ + got_lf = 1; + } + } + } + + /* copy what we did so far into the output buffer */ + /* (first, make sure the output buffer is large enough) */ + if (bufSize - bufOffset < i - c->offset + 1) { + char *newbuf = NULL; + size_t newsize = (buf ? bufSize*2 : LDIF_BUFFER_SIZE); + + newbuf = slapi_ch_malloc(newsize); + if (! newbuf) + goto error; + /* copy over the old data (if there was any) */ + if (buf) { + memmove(newbuf, buf, bufOffset); + slapi_ch_free((void **)&buf); + } + buf = newbuf; + bufSize = newsize; + } + memmove(buf + bufOffset, c->b + c->offset, i - c->offset); + bufOffset += (i - c->offset); + c->offset = i; + } + + /* add terminating NUL char */ + buf[bufOffset] = 0; + return buf; + +error: + if (buf) + slapi_ch_free((void **)&buf); + return NULL; +} + + +/********** THREADS **********/ + +/* + * Description: + * 1) return the ldif version # + * 2) replace "version: 1" with "#ersion: 1" + * to pretend like a comment for the str2entry + */ +static int +import_get_version(char *str) +{ + char *s; + char *type; + char *valuecharptr; + char *mystr, *ms; + int offset; + int valuelen; + int my_version = 0; + int retmalloc = 0; + + if ((s = strstr(str, "version:")) == NULL) + return 0; + + offset = s - str; + mystr = ms = slapi_ch_strdup(str); + while ( (s = ldif_getline( &ms )) != NULL ) { + char *errmsg = NULL; + if ( (retmalloc = ldif_parse_line( s, &type, &valuecharptr, &valuelen, &errmsg )) >= 0 ) { + if (!strcasecmp(type, "version")) { + my_version = atoi(valuecharptr); + *(str + offset) = '#'; + /* the memory below was not allocated by the slapi_ch_ functions */ + if (errmsg) slapi_ch_free((void **) &errmsg); + if (retmalloc) slapi_ch_free((void **) &valuecharptr); + break; + } + } else if ( errmsg != NULL ) { + LDAPDebug( LDAP_DEBUG_PARSE, "%s", errmsg, 0, 0 ); + } + /* the memory below was not allocated by the slapi_ch_ functions */ + if (errmsg) slapi_ch_free((void **) &errmsg); + if (retmalloc) slapi_ch_free((void **) &valuecharptr); + } + + slapi_ch_free((void **)&mystr); + return my_version; +} + +/* producer thread: + * read through the given file list, parsing entries (str2entry), assigning + * them IDs and queueing them on the entry FIFO. other threads will do + * the indexing. + */ +void import_producer(void *param) +{ + ImportWorkerInfo *info = (ImportWorkerInfo *)param; + ImportJob *job = info->job; + ID id = job->first_ID, id_filestart = id; + Slapi_Entry *e = NULL; + struct backentry *ep = NULL, *old_ep = NULL; + ldbm_instance *inst = job->inst; + PRIntervalTime sleeptime; + char *estr = NULL; + int str2entry_flags = + SLAPI_STR2ENTRY_TOMBSTONE_CHECK | + SLAPI_STR2ENTRY_REMOVEDUPVALS | + SLAPI_STR2ENTRY_EXPAND_OBJECTCLASSES | + SLAPI_STR2ENTRY_ADDRDNVALS | + SLAPI_STR2ENTRY_NOT_WELL_FORMED_LDIF; + int finished = 0; + int detected_eof = 0; + int fd, curr_file, curr_lineno; + char *curr_filename = NULL; + int idx; + ldif_context c; + int my_version = 0; + size_t newesize = 0; + + PR_ASSERT(info != NULL); + PR_ASSERT(inst != NULL); + + if ( job->flags & FLAG_ABORT ) { + goto error; + } + + sleeptime = PR_MillisecondsToInterval(import_sleep_time); + + /* pause until we're told to run */ + while ((info->command == PAUSE) && !(job->flags & FLAG_ABORT)) { + info->state = WAITING; + DS_Sleep(sleeptime); + } + info->state = RUNNING; + import_init_ldif(&c); + + /* jumpstart by opening the first file */ + curr_file = 0; + fd = -1; + detected_eof = finished = 0; + + /* we loop around reading the input files and processing each entry + * as we read it. + */ + while (! finished) { + Slapi_Attr *attr = NULL; + int flags = 0; + int prev_lineno = 0; + int lines_in_entry = 0; + + if (job->flags & FLAG_ABORT) { + goto error; + } + + /* move on to next file? */ + if (detected_eof) { + /* check if the file can still be read, whine if so... */ + if (read(fd, (void *)&idx, 1) > 0) { + import_log_notice(job, "WARNING: Unexpected end of file found " + "at line %d of file \"%s\"", curr_lineno, + curr_filename); + } + + if (fd == STDIN_FILENO) { + import_log_notice(job, "Finished scanning file stdin (%lu " + "entries)", (u_long)(id-id_filestart)); + } else { + import_log_notice(job, "Finished scanning file \"%s\" (%lu " + "entries)", curr_filename, (u_long)(id-id_filestart)); + } + close(fd); + fd = -1; + detected_eof = 0; + id_filestart = id; + curr_file++; + if (job->task) { + job->task->task_progress++; + slapi_task_status_changed(job->task); + } + if (job->input_filenames[curr_file] == NULL) { + /* done! */ + finished = 1; + break; + } + } + + /* separate from above, because this is also triggered when we + * start (to open the first file) + */ + if (fd < 0) { + curr_lineno = 0; + curr_filename = job->input_filenames[curr_file]; + if (strcmp(curr_filename, "-") == 0) { + fd = STDIN_FILENO; + } else { + int o_flag = O_RDONLY; +#ifdef XP_WIN32 + /* 613041 Somehow the windows low level io lose "\n" + at a very particular situation using O_TEXT mode read. + I think it is a windows bug for O_TEXT mode read. + Use O_BINARY instead, which honestly returns chars + without any translation. + */ + o_flag |= O_BINARY; +#endif + fd = dblayer_open_huge_file(curr_filename, o_flag, 0); + } + if (fd < 0) { + import_log_notice(job, "Could not open LDIF file \"%s\"", + curr_filename); + goto error; + } + if (fd == STDIN_FILENO) { + import_log_notice(job, "Processing file stdin"); + } else { + import_log_notice(job, "Processing file \"%s\"", curr_filename); + } + } + if (job->flags & FLAG_ABORT) { + goto error; + } + + + while ((info->command == PAUSE) && !(job->flags & FLAG_ABORT)){ + info->state = WAITING; + DS_Sleep(sleeptime); + } + info->state = RUNNING; + + prev_lineno = curr_lineno; + estr = import_get_entry(&c, fd, &curr_lineno); + + lines_in_entry = curr_lineno - prev_lineno; + if (!estr) { + /* error reading entry, or end of file */ + detected_eof = 1; + continue; + } + + if (0 == my_version && strstr(estr, "version:")) { + my_version = import_get_version(estr); + str2entry_flags |= SLAPI_STR2ENTRY_INCLUDE_VERSION_STR; + } + + /* If there are more than so many lines in the entry, we tell + * str2entry to optimize for a large entry. + */ + if (lines_in_entry > STR2ENTRY_ATTRIBUTE_PRESENCE_CHECK_THRESHOLD) { + flags = str2entry_flags | SLAPI_STR2ENTRY_BIGENTRY; + } else { + flags = str2entry_flags; + } + e = slapi_str2entry(estr, flags); + FREE(estr); + if (! e) { + if (!(str2entry_flags & SLAPI_STR2ENTRY_INCLUDE_VERSION_STR)) + import_log_notice(job, "WARNING: skipping bad LDIF entry " + "ending line %d of file \"%s\"", curr_lineno, + curr_filename); + continue; + } + if (0 == my_version) { + /* after the first entry version string won't be given */ + my_version = -1; + } + + if (! import_entry_belongs_here(e, inst->inst_be)) { + /* silently skip */ + if (e) { + job->not_here_skipped++; + slapi_entry_free(e); + } + continue; + } + + if (slapi_entry_schema_check(NULL, e) != 0) { + char ebuf[BUFSIZ]; + import_log_notice(job, "WARNING: skipping entry \"%s\" which " + "violates schema, ending line %d of file " + "\"%s\"", escape_string(slapi_entry_get_dn(e), ebuf), + curr_lineno, curr_filename); + if (e) + slapi_entry_free(e); + job->skipped++; + continue; + } + + /* generate uniqueid if necessary */ + import_generate_uniqueid(job, e); + + ep = import_make_backentry(e, id); + if (!ep) + goto error; + + /* check for include/exclude subtree lists */ + if (! ldbm_back_ok_to_dump(backentry_get_ndn(ep), + job->include_subtrees, + job->exclude_subtrees)) { + backentry_free(&ep); + continue; + } + + /* not sure what this does, but it looked like it could be + * simplified. if it's broken, it's my fault. -robey + */ + if (slapi_entry_attr_find(ep->ep_entry, "userpassword", &attr) == 0) { + Slapi_Value **va = attr_get_present_values(attr); + + pw_encodevals( (Slapi_Value **)va ); /* jcm - cast away const */ + } + + if (job->flags & FLAG_ABORT) { + goto error; + } + + + /* Now we have this new entry, all decoded + * Next thing we need to do is: + * (1) see if the appropriate fifo location contains an + * entry which had been processed by the indexers. + * If so, proceed. + * If not, spin waiting for it to become free. + * (2) free the old entry and store the new one there. + * (3) Update the job progress indicators so the indexers + * can use the new entry. + */ + idx = id % job->fifo.size; + old_ep = job->fifo.item[idx].entry; + if (old_ep) { + /* for the slot to be recycled, it needs to be already absorbed + * by the foreman (id >= ready_ID), and all the workers need to + * be finished with it (refcount = 0). + */ + while (((old_ep->ep_refcnt > 0) || + (old_ep->ep_id >= job->ready_ID)) + && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + info->state = WAITING; + DS_Sleep(sleeptime); + } + if (job->flags & FLAG_ABORT){ + goto error; + } + info->state = RUNNING; + PR_ASSERT(old_ep == job->fifo.item[idx].entry); + job->fifo.item[idx].entry = NULL; + if (job->fifo.c_bsize > job->fifo.item[idx].esize) + job->fifo.c_bsize -= job->fifo.item[idx].esize; + else + job->fifo.c_bsize = 0; + backentry_free(&old_ep); + } + + newesize = (slapi_entry_size(ep->ep_entry) + sizeof(struct backentry)); + if (newesize > job->fifo.bsize) { /* entry too big */ + char ebuf[BUFSIZ]; + import_log_notice(job, "WARNING: skipping entry \"%s\" " + "ending line %d of file \"%s\"", + escape_string(slapi_entry_get_dn(e), ebuf), + curr_lineno, curr_filename); + import_log_notice(job, "REASON: entry too large (%d bytes) for " + "the buffer size (%d bytes)", newesize, job->fifo.bsize); + backentry_free(&ep); + job->skipped++; + continue; + } + /* Now check if fifo has enough space for the new entry */ + if ((job->fifo.c_bsize + newesize) > job->fifo.bsize) { + import_wait_for_space_in_fifo( job, newesize ); + } + + /* We have enough space */ + job->fifo.item[idx].filename = curr_filename; + job->fifo.item[idx].line = curr_lineno; + job->fifo.item[idx].entry = ep; + job->fifo.item[idx].bad = 0; + job->fifo.item[idx].esize = newesize; + + /* Add the entry size to total fifo size */ + job->fifo.c_bsize += ep->ep_entry? job->fifo.item[idx].esize : 0; + + /* Update the job to show our progress */ + job->lead_ID = id; + if ((id - info->first_ID) <= job->fifo.size) { + job->trailing_ID = info->first_ID; + } else { + job->trailing_ID = id - job->fifo.size; + } + + /* Update our progress meter too */ + info->last_ID_processed = id; + id++; + if (job->flags & FLAG_ABORT){ + goto error; + } + if (info->command == STOP) { + if (fd >= 0) + close(fd); + finished = 1; + } + } + + import_free_ldif(&c); + info->state = FINISHED; + return; + +error: + info->state = ABORTED; +} + +#if defined(UPGRADEDB) +/* producer thread for re-indexing: + * read id2entry, parsing entries (str2entry) (needed???), assigning + * them IDs (again, needed???) and queueing them on the entry FIFO. + * other threads will do the indexing -- same as in import. + */ +void index_producer(void *param) +{ + ImportWorkerInfo *info = (ImportWorkerInfo *)param; + ImportJob *job = info->job; + ID id = job->first_ID; + Slapi_Entry *e = NULL; + struct backentry *ep = NULL, *old_ep = NULL; + ldbm_instance *inst = job->inst; + PRIntervalTime sleeptime; + int finished = 0; + int idx; + + /* vars for Berkeley DB */ + DB_ENV *env = NULL; + DB *db = NULL; + DBC *dbc = NULL; + DBT key = {0}; + DBT data = {0}; + int db_rval = -1; + backend *be = inst->inst_be; + int isfirst = 1; + int curr_entry = 0; + size_t newesize = 0; + + PR_ASSERT(info != NULL); + PR_ASSERT(inst != NULL); + PR_ASSERT(be != NULL); + + if ( job->flags & FLAG_ABORT ) + goto error; + + sleeptime = PR_MillisecondsToInterval(import_sleep_time); + + /* pause until we're told to run */ + while ((info->command == PAUSE) && !(job->flags & FLAG_ABORT)) { + info->state = WAITING; + DS_Sleep(sleeptime); + } + info->state = RUNNING; + + /* open id2entry with dedicated db env and db handler */ + if ( dblayer_get_aux_id2entry( be, &db, &env ) != 0 || db == NULL || + env == NULL) { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open id2entry\n", 0, 0, 0 ); + goto error; + } + + /* get a cursor to we can walk over the table */ + db_rval = db->cursor(db, NULL, &dbc, 0); + if ( 0 != db_rval ) { + LDAPDebug( LDAP_DEBUG_ANY, + "Failed to get cursor for reindexing\n", 0, 0, 0 ); + dblayer_release_id2entry(be, db); + goto error; + } + + /* we loop around reading the input files and processing each entry + * as we read it. + */ + finished = 0; + while (!finished) { + Slapi_Attr *attr = NULL; + ID temp_id; + + if (job->flags & FLAG_ABORT) { + goto error; + } + while ((info->command == PAUSE) && !(job->flags & FLAG_ABORT)){ + info->state = WAITING; + DS_Sleep(sleeptime); + } + info->state = RUNNING; + + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + if (isfirst) + { + db_rval = dbc->c_get(dbc, &key, &data, DB_FIRST); + isfirst = 0; + } + else + { + db_rval = dbc->c_get(dbc, &key, &data, DB_NEXT); + } + + if (0 != db_rval) { + if (DB_NOTFOUND != db_rval) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed to read database, " + "errno=%d (%s)\n", inst->inst_name, db_rval, + dblayer_strerror(db_rval)); + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Failed to read database, err %d (%s)", + inst->inst_name, db_rval, + dblayer_strerror(db_rval)); + } + } + break; + } + curr_entry++; + temp_id = id_stored_to_internal((char *)key.data); + free(key.data); + + /* call post-entry plugin */ + plugin_call_entryfetch_plugins((char **) &data.dptr, &data.dsize); + e = slapi_str2entry(data.data, 0); + if ( NULL == e ) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: WARNING: skipping badly formatted entry (id %lu)", + inst->inst_name, (u_long)temp_id); + } + LDAPDebug(LDAP_DEBUG_ANY, + "%s: WARNING: skipping badly formatted entry (id %lu)\n", + inst->inst_name, (u_long)temp_id, 0); + continue; + } + free(data.data); + + /* generate uniqueid if necessary */ + import_generate_uniqueid(job, e); + + ep = import_make_backentry(e, temp_id); + if (!ep) + goto error; + + /* not sure what this does, but it looked like it could be + * simplified. if it's broken, it's my fault. -robey + */ + if (slapi_entry_attr_find(ep->ep_entry, "userpassword", &attr) == 0) { + Slapi_Value **va = attr_get_present_values(attr); + + pw_encodevals( (Slapi_Value **)va ); /* jcm - cast away const */ + } + + if (job->flags & FLAG_ABORT) + goto error; + + /* Now we have this new entry, all decoded + * Next thing we need to do is: + * (1) see if the appropriate fifo location contains an + * entry which had been processed by the indexers. + * If so, proceed. + * If not, spin waiting for it to become free. + * (2) free the old entry and store the new one there. + * (3) Update the job progress indicators so the indexers + * can use the new entry. + */ + idx = id % job->fifo.size; + old_ep = job->fifo.item[idx].entry; + if (old_ep) { + /* for the slot to be recycled, it needs to be already absorbed + * by the foreman (id >= ready_ID), and all the workers need to + * be finished with it (refcount = 0). + */ + while (((old_ep->ep_refcnt > 0) || + (old_ep->ep_id >= job->ready_ID)) + && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + info->state = WAITING; + DS_Sleep(sleeptime); + } + if (job->flags & FLAG_ABORT) + goto error; + + info->state = RUNNING; + PR_ASSERT(old_ep == job->fifo.item[idx].entry); + job->fifo.item[idx].entry = NULL; + if (job->fifo.c_bsize > job->fifo.item[idx].esize) + job->fifo.c_bsize -= job->fifo.item[idx].esize; + else + job->fifo.c_bsize = 0; + backentry_free(&old_ep); + } + + newesize = (slapi_entry_size(ep->ep_entry) + sizeof(struct backentry)); + if (newesize > job->fifo.bsize) { /* entry too big */ + char ebuf[BUFSIZ]; + import_log_notice(job, "WARNING: skipping entry \"%s\"", + escape_string(slapi_entry_get_dn(e), ebuf)); + import_log_notice(job, "REASON: entry too large (%d bytes) for " + "the buffer size (%d bytes)", newesize, job->fifo.bsize); + backentry_free(&ep); + job->skipped++; + continue; + } + /* Now check if fifo has enough space for the new entry */ + if ((job->fifo.c_bsize + newesize) > job->fifo.bsize) { + import_wait_for_space_in_fifo( job, newesize ); + } + + /* We have enough space */ + job->fifo.item[idx].filename = ID2ENTRY LDBM_FILENAME_SUFFIX; + job->fifo.item[idx].line = curr_entry; + job->fifo.item[idx].entry = ep; + job->fifo.item[idx].bad = 0; + job->fifo.item[idx].esize = newesize; + + /* Add the entry size to total fifo size */ + job->fifo.c_bsize += ep->ep_entry? job->fifo.item[idx].esize : 0; + + /* Update the job to show our progress */ + job->lead_ID = id; + if ((id - info->first_ID) <= job->fifo.size) { + job->trailing_ID = info->first_ID; + } else { + job->trailing_ID = id - job->fifo.size; + } + + /* Update our progress meter too */ + info->last_ID_processed = id; + id++; + if (job->flags & FLAG_ABORT) + goto error; + if (info->command == STOP) + { + finished = 1; + } + } + + dbc->c_close(dbc); + dblayer_release_aux_id2entry( be, db, env ); + info->state = FINISHED; + return; + +error: + dbc->c_close(dbc); + dblayer_release_aux_id2entry( be, db, env ); + info->state = ABORTED; +} +#endif + +static void +import_wait_for_space_in_fifo(ImportJob *job, size_t new_esize) +{ + struct backentry *temp_ep = NULL; + size_t i; + int slot_found; + PRIntervalTime sleeptime; + + sleeptime = PR_MillisecondsToInterval(import_sleep_time); + + /* Now check if fifo has enough space for the new entry */ + while ((job->fifo.c_bsize + new_esize) > job->fifo.bsize) { + for ( i = 0, slot_found = 0 ; i < job->fifo.size ; i++ ) { + temp_ep = job->fifo.item[i].entry; + if (temp_ep) { + if (temp_ep->ep_refcnt == 0 && temp_ep->ep_id < job->ready_ID) { + job->fifo.item[i].entry = NULL; + if (job->fifo.c_bsize > job->fifo.item[i].esize) + job->fifo.c_bsize -= job->fifo.item[i].esize; + else + job->fifo.c_bsize = 0; + backentry_free(&temp_ep); + slot_found = 1; + } + } + } + if ( slot_found == 0 ) + DS_Sleep(sleeptime); + } +} + +/* helper function for the foreman: */ +static int foreman_do_parentid(ImportJob *job, struct backentry *entry, + struct attrinfo *parentid_ai) +{ + backend *be = job->inst->inst_be; + Slapi_Value **svals = NULL; + Slapi_Attr *attr = NULL; + int idl_disposition = 0; + int ret = 0; + + if (slapi_entry_attr_find(entry->ep_entry, "parentid", &attr) == 0) { + svals = attr_get_present_values(attr); + ret = index_addordel_values_ext_sv(be, "parentid", svals, NULL, entry->ep_id, + BE_INDEX_ADD, NULL, &idl_disposition, NULL); + if (idl_disposition != IDL_INSERT_NORMAL) { + char *attr_value = slapi_value_get_berval(svals[0])->bv_val; + ID parent_id = atol(attr_value); + + if (idl_disposition == IDL_INSERT_NOW_ALLIDS) { + import_subcount_mother_init(job->mothers, parent_id, + idl_get_allidslimit(parentid_ai)+1); + } else if (idl_disposition == IDL_INSERT_ALLIDS) { + import_subcount_mother_count(job->mothers, parent_id); + } + } + if (ret != 0) { + import_log_notice(job, "ERROR: Can't update parentid index " + "(error %d)", ret); + return ret; + } + } + + return 0; +} + +/* helper function for the foreman: */ +static int foreman_do_entrydn(ImportJob *job, FifoItem *fi) +{ + backend *be = job->inst->inst_be; + struct berval bv; + int err = 0, ret = 0; + IDList *IDL; + + /* insert into the entrydn index */ + bv.bv_val = (void*)backentry_get_ndn(fi->entry); /* jcm - Had to cast away const */ + bv.bv_len = strlen(backentry_get_ndn(fi->entry)); + + /* We need to check here whether the DN is already present in + * the entrydn index. If it is then the input ldif + * contained a duplicate entry, which it isn't allowed to */ + /* Due to popular demand, we only warn on this, given the + * tendency for customers to want to import dirty data */ + /* So, we do an index read first */ + err = 0; + IDL = index_read(be, "entrydn", indextype_EQUALITY, &bv, NULL, &err); + + /* Did this work ? */ + if (NULL != IDL) { + /* IMPOSTER ! Get thee hence... */ + import_log_notice(job, "WARNING: Skipping duplicate entry " + "\"%s\" found at line %d of file \"%s\"", + slapi_entry_get_dn(fi->entry->ep_entry), + fi->line, fi->filename); + idl_free(IDL); + /* skip this one */ + fi->bad = 1; + job->skipped++; + return -1; /* skip to next entry */ + } + if ((ret = index_addordel_string(be, "entrydn", + bv.bv_val, + fi->entry->ep_id, + BE_INDEX_ADD, NULL)) != 0) { + import_log_notice(job, "Error writing entrydn index " + "(error %d: %s)", + ret, dblayer_strerror(ret)); + return ret; + } + return 0; +} + +/* foreman thread: + * i go through the FIFO just like the other worker threads, but i'm + * responsible for the interrelated indexes: entrydn, id2entry, and the + * operational attributes (plus the parentid index). + */ +void import_foreman(void *param) +{ + ImportWorkerInfo *info = (ImportWorkerInfo *)param; + ImportJob *job = info->job; + ldbm_instance *inst = job->inst; + backend *be = inst->inst_be; + PRIntervalTime sleeptime; + int finished = 0; + ID id = info->first_ID; + int ret = 0; + struct attrinfo *parentid_ai; + Slapi_PBlock *pb = slapi_pblock_new(); + + PR_ASSERT(info != NULL); + PR_ASSERT(inst != NULL); + + if (job->flags & FLAG_ABORT) { + goto error; + } + + /* the pblock is used only by add_op_attrs */ + slapi_pblock_set(pb, SLAPI_BACKEND, be); + sleeptime = PR_MillisecondsToInterval(import_sleep_time); + info->state = RUNNING; + + ainfo_get(be, "parentid", &parentid_ai); + + while (! finished) { + FifoItem *fi = NULL; + int parent_status = 0; + + + if (job->flags & FLAG_ABORT) { + goto error; + } + + while ( ((info->command == PAUSE) || (id > job->lead_ID)) && + (info->command != STOP) && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + /* Check to see if we've been told to stop */ + info->state = WAITING; + DS_Sleep(sleeptime); + } + if (info->command == STOP) { + finished = 1; + continue; + } + if (job->flags & FLAG_ABORT) { + goto error; + } + + info->state = RUNNING; + + /* Read that entry from the cache */ + fi = import_fifo_fetch(job, id, 0); + if (! fi) { + import_log_notice(job, "ERROR: foreman fifo error"); + goto error; + } + + /* first, fill in any operational attributes */ + /* add_op_attrs wants a pblock for some reason. */ + if (add_op_attrs(pb, inst->inst_li, fi->entry, &parent_status) != 0) { + import_log_notice(job, "ERROR: Could not add op attrs to " + "entry ending at line %d of file \"%s\"", + fi->line, fi->filename); + goto error; + } + + if (! slapi_entry_flag_is_set(fi->entry->ep_entry, + SLAPI_ENTRY_FLAG_TOMBSTONE)) { + /* + * Only check for a parent and add to the entry2dn index if + * the entry is not a tombstone. + */ + if (job->flags & FLAG_ABORT) { + goto error; + } + + if (parent_status == IMPORT_ADD_OP_ATTRS_NO_PARENT) { + /* If this entry is a suffix entry, this is not a problem */ + /* However, if it is not, this is an error---it means that + * someone tried to import an entry before importing its parent + * we reject the entry but carry on since we've not stored + * anything related to this entry. + */ + if (! slapi_be_issuffix(inst->inst_be, backentry_get_sdn(fi->entry))) { + import_log_notice(job, "WARNING: Skipping entry \"%s\" " + "which has no parent, ending at line %d " + "of file \"%s\"", + slapi_entry_get_dn(fi->entry->ep_entry), + fi->line, fi->filename); + /* skip this one */ + fi->bad = 1; + job->skipped++; + goto cont; /* below */ + } + } + if (job->flags & FLAG_ABORT) { + goto error; + } + + + /* insert into the entrydn index */ + ret = foreman_do_entrydn(job, fi); + if (ret == -1) + goto cont; /* skip entry */ + if (ret != 0) + goto error; + } + + if (job->flags & FLAG_ABORT) { + goto error; + } + +#if defined (UPGRADEDB) + if (!(job->flags & FLAG_REINDEXING))/* reindex reads data from id2entry */ +#endif + { + /* insert into the id2entry index + * (that isn't really an index -- it's the storehouse of the entries + * themselves.) + */ + if ((ret = id2entry_add_ext(be, fi->entry, NULL, job->encrypt)) != 0) { + /* DB_RUNRECOVERY usually occurs if disk fills */ + if (LDBM_OS_ERR_IS_DISKFULL(ret)) { + import_log_notice(job, "ERROR: OUT OF SPACE ON DISK or FILE TOO LARGE -- " + "Could not store the entry ending at line " + "%d of file \"%s\"", + fi->line, fi->filename); + } else if (ret == DB_RUNRECOVERY) { + import_log_notice(job, "FATAL ERROR: (LARGEFILE SUPPORT NOT ENABLED? OUT OF SPACE ON DISK?) -- " + "Could not store the entry ending at line " + "%d of file \"%s\"", + fi->line, fi->filename); + } else { + import_log_notice(job, "ERROR: Could not store the entry " + "ending at line %d of file \"%s\" -- " + "error %d", fi->line, fi->filename, ret); + } + goto error; + } + } + + if (job->flags & FLAG_ABORT) { + goto error; + } + + if (! slapi_entry_flag_is_set(fi->entry->ep_entry, + SLAPI_ENTRY_FLAG_TOMBSTONE)) { + /* parentid index + * (we have to do this here, because the parentID is dependent on + * looking up by entrydn.) + * Only add to the parent index if the entry is not a tombstone. + */ + ret = foreman_do_parentid(job, fi->entry, parentid_ai); + if (ret != 0) + goto error; + + /* Lastly, before we're finished with the entry, pass it to the + vlv code to see whether it's within the scope a VLV index. */ + vlv_grok_new_import_entry(fi->entry, be); + } + if (job->flags & FLAG_ABORT) { + goto error; + } + + + /* Remove the entry from the cache (caused by id2entry_add) */ +#if defined (UPGRADEDB) + if (!(job->flags & FLAG_REINDEXING))/* reindex reads data from id2entry */ +#endif + cache_remove(&inst->inst_cache, fi->entry); + fi->entry->ep_refcnt = job->number_indexers; + + cont: + if (job->flags & FLAG_ABORT) { + goto error; + } + + job->ready_ID = id; + info->last_ID_processed = id; + id++; + + if (job->flags & FLAG_ABORT){ + goto error; + } + } + + slapi_pblock_destroy(pb); + info->state = FINISHED; + return; + +error: + slapi_pblock_destroy(pb); + info->state = ABORTED; +} + + +/* worker thread: + * given an attribute, this worker plows through the entry FIFO, building + * up the attribute index. + */ +void import_worker(void *param) +{ + ImportWorkerInfo *info = (ImportWorkerInfo *)param; + ImportJob *job = info->job; + ldbm_instance *inst = job->inst; + backend *be = inst->inst_be; + PRIntervalTime sleeptime; + int finished = 0; + ID id = info->first_ID; + int ret = 0; + int idl_disposition = 0; + struct vlvIndex* vlv_index = NULL; + void *substring_key_buffer = NULL; + FifoItem *fi; + int is_objectclass_attribute; + int is_nsuniqueid_attribute; + void *attrlist_cursor; + + PR_ASSERT(NULL != info); + PR_ASSERT(NULL != inst); + + if (job->flags & FLAG_ABORT) { + goto error; + } + + if (INDEX_VLV == info->index_info->ai->ai_indexmask) { + vlv_index = vlv_find_indexname(info->index_info->name, be); + if (NULL == vlv_index) { + goto error; + } + } + + /* + * If the entry is a Tombstone, then we only add it to the nsuniqeid index + * and the idlist for (objectclass=tombstone). These two flags are just + * handy for working out what to do in this case. + */ + is_objectclass_attribute = + (strcasecmp(info->index_info->name, "objectclass") == 0); + is_nsuniqueid_attribute = + (strcasecmp(info->index_info->name, SLAPI_ATTR_UNIQUEID) == 0); + + if (1 != idl_get_idl_new()) { + /* Is there substring indexing going on here ? */ + if ( (INDEX_SUB & info->index_info->ai->ai_indexmask) && + (info->index_buffer_size > 0) ) { + /* Then make a key buffer thing */ + ret = index_buffer_init(info->index_buffer_size, 0, + &substring_key_buffer); + if (0 != ret) { + import_log_notice(job, "IMPORT FAIL 1 (error %d)", ret); + } + } + } + + sleeptime = PR_MillisecondsToInterval(import_sleep_time); + info->state = RUNNING; + info->last_ID_processed = id-1; + + while (! finished) { + struct backentry *ep = NULL; + Slapi_Value **svals = NULL; + Slapi_Attr *attr = NULL; + + if (job->flags & FLAG_ABORT) { + goto error; + } + + /* entry can be NULL if it turned out to be bogus */ + while (!finished && !ep) { + /* This worker thread must wait if the command flag is "PAUSE" or + * the entry corresponds to the current entry treated by the foreman + * thread, and the state is neither STOP nor ABORT + */ + while (((info->command == PAUSE) || (id > job->ready_ID)) && + (info->command != STOP) && (info->command != ABORT) && !(job->flags & FLAG_ABORT)) { + /* Check to see if we've been told to stop */ + info->state = WAITING; + DS_Sleep(sleeptime); + } + + if (info->command == STOP) { + finished = 1; + continue; + } + if (job->flags & FLAG_ABORT) { + goto error; + } + + info->state = RUNNING; + + /* Read that entry from the cache */ + fi = import_fifo_fetch(job, id, 1); + ep = fi ? fi->entry : NULL; + if (!ep) { + /* skipping an entry that turned out to be bad */ + info->last_ID_processed = id; + id++; + } + } + if (finished) + continue; + + if (! slapi_entry_flag_is_set(fi->entry->ep_entry, + SLAPI_ENTRY_FLAG_TOMBSTONE)) { + /* This is not a tombstone entry. */ + /* Is this a VLV index ? */ + + if (job->flags & FLAG_ABORT) { + goto error; + } + + if (INDEX_VLV == info->index_info->ai->ai_indexmask) { + /* Yes, call VLV code -- needs pblock to find backend */ + Slapi_PBlock *pb = slapi_pblock_new(); + + PR_ASSERT(NULL != vlv_index); + slapi_pblock_set(pb, SLAPI_BACKEND, be); + vlv_update_index(vlv_index, NULL, inst->inst_li, pb, NULL, ep); + slapi_pblock_destroy(pb); + } else { + /* No, process regular index */ + /* Look for the attribute we're indexing and its subtypes */ + /* For each attr write to the index */ + attrlist_cursor = NULL; + while ((attr = attrlist_find_ex(ep->ep_entry->e_attrs, + info->index_info->name, + NULL, + NULL, + &attrlist_cursor)) != NULL) { + + if (job->flags & FLAG_ABORT) { + goto error; + } + if(valueset_isempty(&(attr->a_present_values))) continue; + svals = attr_get_present_values(attr); + ret = index_addordel_values_ext_sv(be, info->index_info->name, + svals, NULL, ep->ep_id, BE_INDEX_ADD | (job->encrypt ? 0 : BE_INDEX_DONT_ENCRYPT), NULL, &idl_disposition, + substring_key_buffer); + + if (0 != ret) { + /* Something went wrong, eg disk filled up */ + goto error; + } + } + } + } else { + /* This is a Tombstone entry... we only add it to the nsuniqeid + * index and the idlist for (objectclass=nstombstone). + */ + if (job->flags & FLAG_ABORT) { + goto error; + } + if (is_nsuniqueid_attribute) { + ret = index_addordel_string(be, SLAPI_ATTR_UNIQUEID, + slapi_entry_get_uniqueid(ep->ep_entry), ep->ep_id, + BE_INDEX_ADD, NULL); + if (0 != ret) { + /* Something went wrong, eg disk filled up */ + goto error; + } + } + if (is_objectclass_attribute) { + ret = index_addordel_string(be, SLAPI_ATTR_OBJECTCLASS, + SLAPI_ATTR_VALUE_TOMBSTONE, ep->ep_id, BE_INDEX_ADD, NULL); + if (0 != ret) { + /* Something went wrong, eg disk filled up */ + goto error; + } + } + } + import_decref_entry(ep); + info->last_ID_processed = id; + id++; + + if (job->flags & FLAG_ABORT) { + goto error; + } + } + + if (job->flags & FLAG_ABORT) { + goto error; + } + + + /* If we were buffering index keys, now flush them */ + if (substring_key_buffer) { + ret = index_buffer_flush(substring_key_buffer, + inst->inst_be, NULL, + info->index_info->ai); + if (0 != ret) { + goto error; + } + index_buffer_terminate(substring_key_buffer); + } + info->state = FINISHED; + return; + +error: + if (ret == DB_RUNRECOVERY) { + LDAPDebug(LDAP_DEBUG_ANY,"cannot import; database recovery needed\n", + 0,0,0); + } else if (ret == DB_LOCK_DEADLOCK) { + /* can this occur? */ + } + + info->state = ABORTED; +} + + + +/* + * import entries to a backend, over the wire -- entries will arrive + * asynchronously, so this method has no "producer" thread. instead, the + * front-end drops new entries in as they arrive. + * + * this is sometimes called "fast replica initialization". + * + * some of this code is duplicated from ldif2ldbm, but i don't think we + * can avoid it. + */ +static int bulk_import_start(Slapi_PBlock *pb) +{ + struct ldbminfo *li = NULL; + ImportJob *job = NULL; + backend *be = NULL; + PRThread *thread = NULL; + int ret = 0; + + job = CALLOC(ImportJob); + if (job == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "not enough memory to do import job\n", + 0, 0, 0); + return -1; + } + + slapi_pblock_get(pb, SLAPI_BACKEND, &be); + PR_ASSERT(be != NULL); + li = (struct ldbminfo *)(be->be_database->plg_private); + job->inst = (ldbm_instance *)be->be_instance_info; + + /* check if an import/restore is already ongoing... */ + PR_Lock(job->inst->inst_config_mutex); + if (job->inst->inst_flags & INST_FLAG_BUSY) { + PR_Unlock(job->inst->inst_config_mutex); + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + job->inst->inst_name, 0, 0); + FREE(job); + return SLAPI_BI_ERR_BUSY; + } + job->inst->inst_flags |= INST_FLAG_BUSY; + PR_Unlock(job->inst->inst_config_mutex); + + /* take backend offline */ + slapi_mtn_be_disable(be); + + /* get uniqueid info */ + slapi_pblock_get(pb, SLAPI_LDIF2DB_GENERATE_UNIQUEID, &job->uuid_gen_type); + if (job->uuid_gen_type == SLAPI_UNIQUEID_GENERATE_NAME_BASED) { + char *namespaceid; + + slapi_pblock_get(pb, SLAPI_LDIF2DB_NAMESPACEID, &namespaceid); + job->uuid_namespace = slapi_ch_strdup(namespaceid); + } + + job->flags = 0; /* don't use files */ + job->flags |= FLAG_INDEX_ATTRS; + job->flags |= FLAG_ONLINE; + job->starting_ID = 1; + job->first_ID = 1; + + job->mothers = CALLOC(import_subcount_stuff); + /* how much space should we allocate to index buffering? */ + job->job_index_buffer_size = import_get_index_buffer_size(); + if (job->job_index_buffer_size == 0) { + /* 10% of the allocated cache size + one meg */ + job->job_index_buffer_size = (job->inst->inst_li->li_dbcachesize/10) + + (1024*1024); + } + import_subcount_stuff_init(job->mothers); + job->wire_lock = PR_NewLock(); + job->wire_cv = PR_NewCondVar(job->wire_lock); + + /* COPIED from ldif2ldbm.c : */ + + /* shutdown this instance of the db */ + cache_clear(&job->inst->inst_cache); + dblayer_instance_close(be); + + /* Delete old database files */ + dblayer_delete_instance_dir(be); + /* it's okay to fail -- it might already be gone */ + + /* dblayer_instance_start will init the id2entry index. */ + /* it also (finally) fills in inst_dir_name */ + ret = dblayer_instance_start(be, DBLAYER_IMPORT_MODE); + if (ret != 0) + goto fail; + + /* END OF COPIED SECTION */ + + PR_Lock(job->wire_lock); + vlv_init(job->inst); + + /* create thread for import_main, so we can return */ + thread = PR_CreateThread(PR_USER_THREAD, import_main, (void *)job, + PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_JOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE); + if (thread == NULL) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, "unable to spawn import thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + PR_Unlock(job->wire_lock); + ret = -2; + goto fail; + } + + job->main_thread = thread; + slapi_set_object_extension(li->li_bulk_import_object, pb->pb_conn, + li->li_bulk_import_handle, job); + + /* wait for the import_main to signal that it's ready for entries */ + /* (don't want to send the success code back to the LDAP client until + * we're ready for the adds to start rolling in) + */ + PR_WaitCondVar(job->wire_cv, PR_INTERVAL_NO_TIMEOUT); + PR_Unlock(job->wire_lock); + + return 0; + +fail: + PR_Lock(job->inst->inst_config_mutex); + job->inst->inst_flags &= ~INST_FLAG_BUSY; + PR_Unlock(job->inst->inst_config_mutex); + import_free_job(job); + FREE(job); + return ret; +} + +/* returns 0 on success, or < 0 on error + * + * on error, the import process is aborted -- so if this returns an error, + * don't try to queue any more entries or you'll be sorry. + * + * flag_block in used to know if this thread should block when + * the fifo is full or return an error LDAP_BUSY + * Typically, import done on from the GUI or the command line will + * block while online import as used by the replication total update + * will not block + */ +static int bulk_import_queue(ImportJob *job, Slapi_Entry *entry, int flag_block) +{ + struct backentry *ep = NULL, *old_ep = NULL; + int idx; + ID id = job->lead_ID + 1; + Slapi_Attr *attr = NULL; + size_t newesize = 0; + + PR_Lock(job->wire_lock); + + /* generate uniqueid if necessary */ + import_generate_uniqueid(job, entry); + + /* make into backentry */ + ep = import_make_backentry(entry, id); + if (!ep) { + import_abort_all(job, 1); + PR_Unlock(job->wire_lock); + return -1; + } + + /* encode the password */ + if (slapi_entry_attr_find(ep->ep_entry, "userpassword", &attr) == 0) { + Slapi_Value **va = attr_get_present_values(attr); + + pw_encodevals( (Slapi_Value **)va ); /* jcm - had to cast away const */ + } + + /* Now we have this new entry, all decoded + * Next thing we need to do is: + * (1) see if the appropriate fifo location contains an + * entry which had been processed by the indexers. + * If so, proceed. + * If not, spin waiting for it to become free. + * (2) free the old entry and store the new one there. + * (3) Update the job progress indicators so the indexers + * can use the new entry. + */ + idx = id % job->fifo.size; + old_ep = job->fifo.item[idx].entry; + if (old_ep) { + while ((old_ep->ep_refcnt > 0) && !(job->flags & FLAG_ABORT)) + { + if (flag_block) + DS_Sleep(PR_MillisecondsToInterval(import_sleep_time)); + else + { + PR_Unlock(job->wire_lock); + return LDAP_BUSY; + } + } + + /* the producer could be running thru the fifo while + * everyone else is cycling to a new pass... + * double-check that this entry is < ready_ID + */ + while ((old_ep->ep_id >= job->ready_ID) && !(job->flags & FLAG_ABORT)) + { + if (flag_block) + DS_Sleep(PR_MillisecondsToInterval(import_sleep_time)); + else + { + PR_Unlock(job->wire_lock); + return LDAP_BUSY; + } + } + + if (job->flags & FLAG_ABORT) { + PR_Unlock(job->wire_lock); + return -2; + } + + PR_ASSERT(old_ep == job->fifo.item[idx].entry); + job->fifo.item[idx].entry = NULL; + if (job->fifo.c_bsize > job->fifo.item[idx].esize) + job->fifo.c_bsize -= job->fifo.item[idx].esize; + else + job->fifo.c_bsize = 0; + backentry_free(&old_ep); + } + + newesize = (slapi_entry_size(ep->ep_entry) + sizeof(struct backentry)); + if (newesize > job->fifo.bsize) { /* entry too big */ + char ebuf[BUFSIZ]; + import_log_notice(job, "WARNING: skipping entry \"%s\"", + escape_string(slapi_entry_get_dn(ep->ep_entry), ebuf)); + import_log_notice(job, "REASON: entry too large (%d bytes) for " + "the buffer size (%d bytes)", newesize, job->fifo.bsize); + backentry_free(&ep); + PR_Unlock(job->wire_lock); + return -1; + } + /* Now check if fifo has enough space for the new entry */ + if ((job->fifo.c_bsize + newesize) > job->fifo.bsize) { + import_wait_for_space_in_fifo( job, newesize ); + } + + /* We have enough space */ + job->fifo.item[idx].filename = "(bulk import)"; + job->fifo.item[idx].line = 0; + job->fifo.item[idx].entry = ep; + job->fifo.item[idx].bad = 0; + job->fifo.item[idx].esize = newesize; + + /* Add the entry size to total fifo size */ + job->fifo.c_bsize += ep->ep_entry? job->fifo.item[idx].esize : 0; + + /* Update the job to show our progress */ + job->lead_ID = id; + if ((id - job->starting_ID) <= job->fifo.size) { + job->trailing_ID = job->starting_ID; + } else { + job->trailing_ID = id - job->fifo.size; + } + + PR_Unlock(job->wire_lock); + return 0; +} + +void *factory_constructor(void *object, void *parent) +{ + return NULL; +} + +void factory_destructor(void *extension, void *object, void *parent) +{ + ImportJob *job = (ImportJob *)extension; + PRThread *thread; + + if (extension == NULL) + return; + + /* connection was destroyed while we were still storing the extension -- + * this is bad news and means we have a bulk import that needs to be + * aborted! + */ + thread = job->main_thread; + LDAPDebug(LDAP_DEBUG_ANY, "ERROR bulk import abandoned\n", + 0, 0, 0); + import_abort_all(job, 1); + /* wait for import_main to finish... */ + PR_JoinThread(thread); + /* extension object is free'd by import_main */ + return; +} + +/* plugin entry function for replica init */ +int ldbm_back_wire_import(Slapi_PBlock *pb) +{ + struct ldbminfo *li; + backend *be = NULL; + ImportJob *job; + PRThread *thread; + int state; + + slapi_pblock_get(pb, SLAPI_BACKEND, &be); + PR_ASSERT(be != NULL); + li = (struct ldbminfo *)(be->be_database->plg_private); + slapi_pblock_get(pb, SLAPI_BULK_IMPORT_STATE, &state); + if (state == SLAPI_BI_STATE_START) { + /* starting a new import */ + return bulk_import_start(pb); + } + + PR_ASSERT(pb->pb_conn != NULL); + if (pb->pb_conn != NULL) { + job = (ImportJob *)slapi_get_object_extension(li->li_bulk_import_object, pb->pb_conn, li->li_bulk_import_handle); + } + + if ((job == NULL) || (pb->pb_conn == NULL)) { + /* import might be aborting */ + return -1; + } + + if (state == SLAPI_BI_STATE_ADD) { + /* continuing previous import */ + if (! import_entry_belongs_here(pb->pb_import_entry, + job->inst->inst_be)) { + /* silently skip */ + return 0; + } + return bulk_import_queue(job, pb->pb_import_entry, + job->flags & FLAG_USE_FILES); + } + + thread = job->main_thread; + + if (state == SLAPI_BI_STATE_DONE) { + /* finished with an import */ + job->flags |= FLAG_PRODUCER_DONE; + /* "job" struct may vanish at any moment after we set the DONE + * flag, so keep a copy of the thread id in 'thread' for safekeeping. + */ + /* wait for import_main to finish... */ + PR_JoinThread(thread); + slapi_set_object_extension(li->li_bulk_import_object, pb->pb_conn, + li->li_bulk_import_handle, NULL); + return 0; + } + + /* ??? unknown state */ + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR: ldbm_back_wire_import: unknown state %d\n", + state, 0, 0); + return -1; +} + +/* + * backup index configuration + * this function is called from dblayer_backup (ldbm2archive) + * [547427] index config must not change between backup and restore + */ +#define DSE_INDEX "dse_index.ldif" +#define DSE_INSTANCE "dse_instance.ldif" +#define DSE_INDEX_FILTER "(objectclass=nsIndex)" +#define DSE_INSTANCE_FILTER "(objectclass=nsBackendInstance)" +static int +dse_conf_backup_core(struct ldbminfo *li, char *dest_dir, char *file_name, char *filter) +{ + Slapi_PBlock *srch_pb = NULL; + Slapi_Entry **entries = NULL; + Slapi_Entry **ep = NULL; + Slapi_Attr *attr = NULL; + char *attr_name; + char *filename = NULL; + PRFileDesc *prfd = NULL; + int rval = 0; + int dlen = 0; + PRInt32 prrval; + char tmpbuf[BUFSIZ]; + char *tp = NULL; + + dlen = strlen(dest_dir); + if (0 == dlen) + { + filename = file_name; + } + else + { + filename = (char *)slapi_ch_malloc(strlen(file_name) + dlen + 2); + sprintf(filename, "%s/%s", dest_dir, file_name); + } + LDAPDebug(LDAP_DEBUG_TRACE, "dse_conf_backup(%s): backup file %s\n", + filter, filename, 0); + + /* Open the file to write */ + if ((prfd = PR_Open(filename, PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, + SLAPD_DEFAULT_FILE_MODE)) == NULL) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dse_conf_backup(%s): open %s failed: (%s)\n", + filter, filename, slapd_pr_strerror(PR_GetError())); + rval = -1; + goto out; + } + + srch_pb = slapi_pblock_new(); + slapi_search_internal_set_pb(srch_pb, li->li_plugin->plg_dn, + LDAP_SCOPE_SUBTREE, filter, NULL, 0, NULL, NULL, li->li_identity, 0); + slapi_search_internal_pb(srch_pb); + slapi_pblock_get(srch_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries); + for (ep = entries; ep != NULL && *ep != NULL; ep++) + { + size_t l = strlen(slapi_entry_get_dn_const(*ep)) + 5 /* "dn: \n" */; + LDAPDebug(LDAP_DEBUG_TRACE, "\ndn: %s\n", + slapi_entry_get_dn_const(*ep), 0, 0); + + if (l <= BUFSIZ) + tp = tmpbuf; + else + tp = (char *)slapi_ch_malloc(l); /* should be very rare ... */ + sprintf(tp, "dn: %s\n", slapi_entry_get_dn_const(*ep)); + prrval = PR_Write(prfd, tp, l); + if ((size_t)prrval != l) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dse_conf_backup(%s): write %s failed: %d (%s)\n", + filter, PR_GetError(), slapd_pr_strerror(PR_GetError())); + rval = -1; + if (l > BUFSIZ) + slapi_ch_free_string(&tp); + goto out; + } + if (l > BUFSIZ) + slapi_ch_free_string(&tp); + + for (slapi_entry_first_attr(*ep, &attr); attr; + slapi_entry_next_attr(*ep, attr, &attr)) + { + int i; + Slapi_Value *sval = NULL; + const struct berval *attr_val; + int attr_name_len; + + slapi_attr_get_type(attr, &attr_name); + /* numsubordinates should not be backed up */ + if (!strcasecmp("numsubordinates", attr_name)) + continue; + attr_name_len = strlen(attr_name); + for (i = slapi_attr_first_value(attr, &sval); i != -1; + i = slapi_attr_next_value(attr, i, &sval)) + { + attr_val = slapi_value_get_berval(sval); + l = strlen(attr_val->bv_val) + attr_name_len + 3; /* : \n" */ + LDAPDebug(LDAP_DEBUG_TRACE, "%s: %s\n", attr_name, + attr_val->bv_val, 0); + if (l <= BUFSIZ) + tp = tmpbuf; + else + tp = (char *)slapi_ch_malloc(l); + sprintf(tp, "%s: %s\n", attr_name, attr_val->bv_val); + prrval = PR_Write(prfd, tp, l); + if ((size_t)prrval != l) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dse_conf_backup(%s): write %s failed: %d (%s)\n", + filter, PR_GetError(), slapd_pr_strerror(PR_GetError())); + rval = -1; + if (l > BUFSIZ) + slapi_ch_free_string(&tp); + goto out; + } + if (l > BUFSIZ) + slapi_ch_free_string(&tp); + } + } + if (ep+1 != NULL && *(ep+1) != NULL) + { + prrval = PR_Write(prfd, "\n", 1); + if ((int)prrval != 1) + { + LDAPDebug(LDAP_DEBUG_ANY, + "dse_conf_backup(%s): write %s failed: %d (%s)\n", + filter, PR_GetError(), slapd_pr_strerror(PR_GetError())); + rval = -1; + goto out; + } + } + } + +out: + slapi_free_search_results_internal(srch_pb); + if (srch_pb) + { + slapi_pblock_destroy(srch_pb); + } + + if (0 != dlen) + { + slapi_ch_free_string(&filename); + } + + if (prfd) + { + prrval = PR_Close(prfd); + if (PR_SUCCESS != prrval) + { + LDAPDebug( LDAP_DEBUG_ANY, + "Fatal Error---Failed to back up dse indexes %d (%s)\n", + PR_GetError(), slapd_pr_strerror(PR_GetError()), 0); + rval = -1; + } + } + + return rval; +} + +int +dse_conf_backup(struct ldbminfo *li, char *dest_dir) +{ + int rval = 0; + rval = dse_conf_backup_core(li, dest_dir, DSE_INSTANCE, DSE_INSTANCE_FILTER); + rval += dse_conf_backup_core(li, dest_dir, DSE_INDEX, DSE_INDEX_FILTER); + return rval; +} + +/* + * read the backed up index configuration + * adjust them if the current configuration is different from it. + * this function is called from dblayer_restore (archive2ldbm) + * these functions are placed here to borrow import_get_entry + * [547427] index config must not change between backup and restore + */ +int +dse_conf_verify_core(struct ldbminfo *li, char *src_dir, char *file_name, char *filter, char *log_str) +{ + char *filename = NULL; + int rval = 0; + ldif_context c; + int fd = -1; + int curr_lineno = 0; + int finished = 0; + int backup_entry_len = 256; + Slapi_Entry **backup_entries = NULL; + Slapi_Entry **bep = NULL; + Slapi_Entry **curr_entries = NULL; + Slapi_PBlock srch_pb; + + filename = (char *)slapi_ch_malloc(strlen(file_name) + strlen(src_dir) + 2); + sprintf(filename, "%s/%s", src_dir, file_name); + + if (PR_SUCCESS != PR_Access(filename, PR_ACCESS_READ_OK)) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: config backup file %s not found in backup\n", + file_name, 0, 0); + rval = 0; + goto out; + } + + fd = dblayer_open_huge_file(filename, O_RDONLY, 0); + if (fd < 0) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: can't open config backup file: %s\n", filename, 0, 0); + rval = -1; + goto out; + } + + import_init_ldif(&c); + bep = backup_entries = (Slapi_Entry **)slapi_ch_calloc(1, + backup_entry_len * sizeof(Slapi_Entry *)); + + while (!finished) + { + char *estr = NULL; + Slapi_Entry *e = NULL; + estr = import_get_entry(&c, fd, &curr_lineno); + + if (!estr) + break; + + e = slapi_str2entry(estr, 0); + slapi_ch_free_string(&estr); + if (!e) { + LDAPDebug(LDAP_DEBUG_ANY, "WARNING: skipping bad LDIF entry " + "ending line %d of file \"%s\"", curr_lineno, filename, 0); + continue; + } + if (bep - backup_entries >= backup_entry_len) + { + backup_entries = (Slapi_Entry **)slapi_ch_realloc((char *)backup_entries, + 2 * backup_entry_len * sizeof(Slapi_Entry *)); + bep = backup_entries + backup_entry_len; + backup_entry_len *= 2; + } + *bep = e; + bep++; + } + // 623986: terminate the list if we reallocated backup_entries + if (backup_entry_len > 256) + *bep = NULL; + + pblock_init(&srch_pb); + slapi_search_internal_set_pb(&srch_pb, li->li_plugin->plg_dn, + LDAP_SCOPE_SUBTREE, filter, NULL, 0, NULL, NULL, li->li_identity, 0); + slapi_search_internal_pb(&srch_pb); + slapi_pblock_get(&srch_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &curr_entries); + + if (0 != slapi_entries_diff(backup_entries, curr_entries, 1 /* test_all */, + log_str, 1 /* force_update */, li->li_identity)) + { + LDAPDebug(LDAP_DEBUG_ANY, "WARNING!!: current %s is " + "different from backed up configuration; " + "The backup is restored.\n", log_str, 0, 0); + } + + slapi_free_search_results_internal(&srch_pb); + pblock_done(&srch_pb); + import_free_ldif(&c); +out: + for (bep = backup_entries; bep && *bep; bep++) + slapi_entry_free(*bep); + slapi_ch_free((void **)&backup_entries); + + slapi_ch_free_string(&filename); + + if (fd > 0) + close(fd); + + return rval; +} + +int +dse_conf_verify(struct ldbminfo *li, char *src_dir) +{ + int rval; + rval = dse_conf_verify_core(li, src_dir, DSE_INSTANCE, DSE_INSTANCE_FILTER, + "Instance Config"); + rval += dse_conf_verify_core(li, src_dir, DSE_INDEX, DSE_INDEX_FILTER, + "Index Config"); + return rval; +} diff --git a/ldap/servers/slapd/back-ldbm/import.c b/ldap/servers/slapd/back-ldbm/import.c new file mode 100644 index 00000000..47e05fa6 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/import.c @@ -0,0 +1,1465 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* + * the "new" ("deluxe") backend import code + * + * please make sure you use 4-space indentation on this file. + */ + + +#include "back-ldbm.h" +#include "vlv_srch.h" +#include "import.h" + +#define ERR_IMPORT_ABORTED -23 + + +/********** routines to manipulate the entry fifo **********/ + +/* this is pretty bogus -- could be a HUGE amount of memory */ +/* Not anymore with the Import Queue Adaptative Algorithm (Regulation) */ +#define MAX_FIFO_SIZE 8000 + +static int import_fifo_init(ImportJob *job) +{ + ldbm_instance *inst = job->inst; + + /* Work out how big the entry fifo can be */ + if (inst->inst_cache.c_maxentries > 0) + job->fifo.size = inst->inst_cache.c_maxentries; + else + job->fifo.size = inst->inst_cache.c_maxsize / 1024; /* guess */ + + /* byte limit that should be respected to avoid memory starvation */ + /* conservative computing: multiply by .8 to allow for reasonable overflow */ + job->fifo.bsize = (inst->inst_cache.c_maxsize/10) << 3; + + job->fifo.c_bsize = 0; + + if (job->fifo.size > MAX_FIFO_SIZE) + job->fifo.size = MAX_FIFO_SIZE; + /* has to be at least 1 or 2, and anything less than about 100 destroys + * the point of doing all this optimization in the first place. */ + if (job->fifo.size < 100) + job->fifo.size = 100; + + /* Get memory for the entry fifo */ + /* This is used to keep a ref'ed pointer to the last <cachesize> + * processed entries */ + PR_ASSERT(NULL == job->fifo.item); + job->fifo.item = (FifoItem *)slapi_ch_calloc(job->fifo.size, + sizeof(FifoItem)); + if (NULL == job->fifo.item) { + /* Memory allocation error */ + return -1; + } + return 0; +} + +FifoItem *import_fifo_fetch(ImportJob *job, ID id, int worker) +{ + int idx = id % job->fifo.size; + FifoItem *fi; + + if (job->fifo.item) { + fi = &(job->fifo.item[idx]); + } else { + return NULL; + } + if (fi->entry) { + if (id != fi->entry->ep_id) + fi = NULL; + else if (worker) { + if (fi->bad) return NULL; + PR_ASSERT(fi->entry->ep_refcnt > 0); + } + } + return fi; +} + +static void import_fifo_destroy(ImportJob *job) +{ + /* Free any entries in the fifo first */ + struct backentry *be = NULL; + size_t i = 0; + + for (i = 0; i < job->fifo.size; i++) { + be = job->fifo.item[i].entry; + backentry_free(&be); + job->fifo.item[i].entry = NULL; + job->fifo.item[i].filename = NULL; + } + free(job->fifo.item); + job->fifo.item = NULL; +} + + +/********** logging stuff **********/ + +#define LOG_BUFFER 256 + +/* this changes the 'nsTaskStatus' value, which is transient (anything logged + * here wipes out any previous status) + */ +static void import_log_status_start(ImportJob *job) +{ + if (! job->task_status) + job->task_status = (char *)slapi_ch_malloc(10 * LOG_BUFFER); + if (! job->task_status) + return; /* out of memory? */ + + job->task_status[0] = 0; +} + +static void import_log_status_add_line(ImportJob *job, char *format, ...) +{ + va_list ap; + int len = 0; + + if (! job->task_status) + return; + len = strlen(job->task_status); + if (len + 5 > (10 * LOG_BUFFER)) + return; /* no room */ + + if (job->task_status[0]) + strcat(job->task_status, "\n"); + + va_start(ap, format); + PR_vsnprintf(job->task_status + len, (10 * LOG_BUFFER) - len, format, ap); + va_end(ap); +} + +static void import_log_status_done(ImportJob *job) +{ + if (job->task) { + int len = 0; + len = strlen(job->task_status); + slapi_task_log_status(job->task, "%s", job->task_status); + } +} + +/* this adds a line to the 'nsTaskLog' value, which is cumulative (anything + * logged here is added to the end) + */ +void import_log_notice(ImportJob *job, char *format, ...) +{ + va_list ap; + char buffer[LOG_BUFFER]; + + va_start(ap, format); + PR_vsnprintf(buffer, LOG_BUFFER, format, ap); + va_end(ap); + + if (job->task) { + slapi_task_log_notice(job->task, "%s", buffer); + } + /* also save it in the logs for posterity */ + LDAPDebug(LDAP_DEBUG_ANY, "import %s: %s\n", job->inst->inst_name, + buffer, 0); +} + +static int import_task_destroy(Slapi_Task *task) +{ + ImportJob *job = (ImportJob *)task->task_private; + + if (task->task_log) { + slapi_ch_free((void **)&task->task_log); + } + + if (task->task_status) { + slapi_ch_free((void **)&task->task_status); + } + + + if (job && job->task_status) { + slapi_ch_free((void **)&job->task_status); + job->task_status = NULL; + } + FREE(job); + task->task_private = NULL; + return 0; +} + +static int import_task_abort(Slapi_Task *task) +{ + ImportJob *job; + + /* don't log anything from here, because we're still holding the + * DSE lock for modify... + */ + + if (task->task_state == SLAPI_TASK_FINISHED) { + /* too late */ + return 0; + } + + /* + * Race condition. + * If the import thread happens to finish right now we're in trouble + * because it will free the job. + */ + + job = (ImportJob *)task->task_private; + + import_abort_all(job, 0); + while (task->task_state != SLAPI_TASK_FINISHED) + DS_Sleep(PR_MillisecondsToInterval(100)); + + + return 0; +} + + +/********** helper functions for importing **********/ + + +/* Function used to gather a list of indexed attrs */ +static int import_attr_callback(void *node, void *param) +{ + ImportJob *job = (ImportJob *)param; + struct attrinfo *a = (struct attrinfo *)node; + + /* OK, so we now have hold of the attribute structure and the job info, + * let's see what we have. Remember that although this function is called + * many times, all these calls are in the context of a single thread, so we + * don't need to worry about protecting the data in the job structure. + */ + + /* We need to specifically exclude the entrydn & parentid indexes because + * we build those in the foreman thread. + */ + if (IS_INDEXED(a->ai_indexmask) && + (strcasecmp(a->ai_type, "entrydn") != 0) && + (strcasecmp(a->ai_type, "parentid") != 0) && + (strcasecmp(a->ai_type, "ancestorid") != 0) && + (strcasecmp(a->ai_type, numsubordinates) != 0)) { + /* Make an import_index_info structure, fill it in and insert into the + * job's list */ + IndexInfo *info = CALLOC(IndexInfo); + + if (NULL == info) { + /* Memory allocation error */ + return -1; + } + info->name = slapi_ch_strdup(a->ai_type); + info->ai = a; + if (NULL == info->name) { + /* Memory allocation error */ + free(info); + return -1; + } + info->next = job->index_list; + job->index_list = info; + job->number_indexers++; + } + return 0; +} + +static void import_set_index_buffer_size(ImportJob *job) +{ + IndexInfo *current_index = NULL; + size_t substring_index_count = 0; + size_t proposed_size = 0; + + /* Count the substring indexes we have */ + for (current_index = job->index_list; current_index != NULL; + current_index = current_index->next) { + if (current_index->ai->ai_indexmask & INDEX_SUB) { + substring_index_count++; + } + } + if (substring_index_count > 0) { + /* Make proposed size such that if all substring indices were + * reasonably full, we'd hit the target space */ + proposed_size = (job->job_index_buffer_size / substring_index_count) / + IMPORT_INDEX_BUFFER_SIZE_CONSTANT; + if (proposed_size > IMPORT_MAX_INDEX_BUFFER_SIZE) { + proposed_size = IMPORT_MAX_INDEX_BUFFER_SIZE; + } + if (proposed_size < IMPORT_MIN_INDEX_BUFFER_SIZE) { + proposed_size = 0; + } + } + + job->job_index_buffer_suggestion = proposed_size; +} + +static void import_free_thread_data(ImportJob *job) +{ + /* DBDB free the lists etc */ + ImportWorkerInfo *worker = job->worker_list; + + while (worker != NULL) { + ImportWorkerInfo *asabird = worker; + worker = worker->next; + if (asabird->work_type != PRODUCER) + slapi_ch_free( (void**)&asabird); + } +} + +void import_free_job(ImportJob *job) +{ + /* DBDB free the lists etc */ + IndexInfo *index = job->index_list; + + import_free_thread_data(job); + while (index != NULL) { + IndexInfo *asabird = index; + index = index->next; + slapi_ch_free( (void**)&asabird->name); + slapi_ch_free( (void**)&asabird); + } + job->index_list = NULL; + if (NULL != job->mothers) { + import_subcount_stuff_term(job->mothers); + slapi_ch_free( (void**)&job->mothers); + } + + ldbm_back_free_incl_excl(job->include_subtrees, job->exclude_subtrees); + charray_free(job->input_filenames); + if (job->fifo.size) + import_fifo_destroy(job); + if (NULL != job->uuid_namespace) + slapi_ch_free((void **)&job->uuid_namespace); + if (job->wire_lock) + PR_DestroyLock(job->wire_lock); + if (job->wire_cv) + PR_DestroyCondVar(job->wire_cv); + slapi_ch_free((void **)&job->task_status); +} + +/* determine if we are the correct backend for this entry + * (in a distributed suffix, some entries may be for other backends). + * if the entry's dn actually matches one of the suffixes of the be, we + * automatically take it as a belonging one, for such entries must be + * present in EVERY backend independently of the distribution applied. + */ +int import_entry_belongs_here(Slapi_Entry *e, backend *be) +{ + Slapi_Backend *retbe; + Slapi_DN *sdn = slapi_entry_get_sdn(e); + + if (slapi_be_issuffix(be, sdn)) + return 1; + + retbe = slapi_mapping_tree_find_backend_for_sdn(sdn); + return (retbe == be); +} + + +/********** starting threads and stuff **********/ + +/* Solaris is weird---we need an LWP per thread but NSPR doesn't give us + * one unless we make this magic belshe-call */ +/* Fixed on Solaris 8; NSPR supports PR_GLOBAL_BOUND_THREAD */ +#define CREATE_THREAD PR_CreateThread + +static void import_init_worker_info(ImportWorkerInfo *info, ImportJob *job) +{ + info->command = PAUSE; + info->job = job; + info->first_ID = job->first_ID; + info->index_buffer_size = job->job_index_buffer_suggestion; +} + +static int import_start_threads(ImportJob *job) +{ + IndexInfo *current_index = NULL; + ImportWorkerInfo *foreman = NULL, *worker = NULL; + + foreman = CALLOC(ImportWorkerInfo); + if (!foreman) + goto error; + + /* start the foreman */ + import_init_worker_info(foreman, job); + foreman->work_type = FOREMAN; + if (! CREATE_THREAD(PR_USER_THREAD, (VFP)import_foreman, foreman, + PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, + PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE)) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, "unable to spawn import foreman thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + FREE(foreman); + goto error; + } + + foreman->next = job->worker_list; + job->worker_list = foreman; + + /* Start follower threads, if we are doing attribute indexing */ + current_index = job->index_list; + if (job->flags & FLAG_INDEX_ATTRS) { + while (current_index) { + /* make a new thread info structure */ + worker = CALLOC(ImportWorkerInfo); + if (! worker) + goto error; + + /* fill it in */ + import_init_worker_info(worker, job); + worker->index_info = current_index; + worker->work_type = WORKER; + + /* Start the thread */ + if (! CREATE_THREAD(PR_USER_THREAD, (VFP)import_worker, worker, + PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE)) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, "unable to spawn import worker thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + FREE(worker); + goto error; + } + + /* link it onto the job's thread list */ + worker->next = job->worker_list; + job->worker_list = worker; + current_index = current_index->next; + } + } + return 0; + +error: + import_log_notice(job, "Import thread creation failed."); + import_log_notice(job, "Aborting all import threads..."); + import_abort_all(job, 1); + import_log_notice(job, "Import threads aborted."); + return -1; +} + + +/********** monitoring the worker threads **********/ + +static void import_clear_progress_history(ImportJob *job) +{ + int i = 0; + + for (i = 0; i < IMPORT_JOB_PROG_HISTORY_SIZE /*- 1*/; i++) { + job->progress_history[i] = job->first_ID; + job->progress_times[i] = job->start_time; + } + /* reset libdb cache stats */ + job->inst->inst_cache_hits = job->inst->inst_cache_misses = 0; +} + +static double import_grok_db_stats(ldbm_instance *inst) +{ + DB_MPOOL_STAT *mpstat = NULL; + DB_MPOOL_FSTAT **mpfstat = NULL; + int return_value = -1; + double cache_hit_ratio = 0.0; + + return_value = dblayer_memp_stat_instance(inst, &mpstat, &mpfstat); + + if (0 == return_value) { + unsigned long current_cache_hits = mpstat->st_cache_hit; + unsigned long current_cache_misses = mpstat->st_cache_miss; + + if (inst->inst_cache_hits) { + unsigned long hit_delta, miss_delta; + + hit_delta = current_cache_hits - inst->inst_cache_hits; + miss_delta = current_cache_misses - inst->inst_cache_misses; + if (hit_delta != 0) { + cache_hit_ratio = (double)hit_delta / + (double)(hit_delta + miss_delta); + } + } + inst->inst_cache_misses = current_cache_misses; + inst->inst_cache_hits = current_cache_hits; + + if (mpstat) + free(mpstat); + if (mpfstat) { +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR + DB_VERSION_PATCH <= 3204 + /* In DB 3.2.4 and earlier, we need to free each element */ + DB_MPOOL_FSTAT **tfsp; + for (tfsp = mpfstat; *tfsp; tfsp++) + free(*tfsp); +#endif + free(mpfstat); + } + } + return cache_hit_ratio; +} + +static char* import_decode_worker_state(int state) +{ + switch (state) { + case WAITING: + return "W"; + case RUNNING: + return "R"; + case FINISHED: + return "F"; + case ABORTED: + return "A"; + default: + return "?"; + } +} + +static void import_print_worker_status(ImportWorkerInfo *info) +{ + char *name = (info->work_type == PRODUCER ? "Producer" : + (info->work_type == FOREMAN ? "Foreman" : + info->index_info->name)); + + import_log_status_add_line(info->job, + "%-25s %s%10ld %7.1f", name, + import_decode_worker_state(info->state), + info->last_ID_processed, info->rate); +} + + +#define IMPORT_CHUNK_TEST_HOLDOFF_TIME (5*60) /* Seconds */ + +/* Got to be lower than this: */ +#define IMPORT_CHUNK_TEST_CACHE_HIT_RATIO (0.99) +/* Less than half as fast as we were doing: */ +#define IMPORT_CHUNK_TEST_SLOWDOWN_RATIO_A (0.5) +/* A lot less fast than we were doing: */ +#define IMPORT_CHUNK_TEST_SLOWDOWN_RATIO_B (0.1) + +static int import_throw_in_towel(ImportJob *job, time_t current_time, + ID trailing_ID) +{ + static int number_of_times_here = 0; + + /* secret -c option allows specific chunk size to be set... */ + if (job->merge_chunk_size != 0) { + if ((0 != job->lead_ID) && + (trailing_ID > job->first_ID) && + (trailing_ID - job->first_ID > job->merge_chunk_size)) { + return 1; + } + return 0; + } + + /* Check stats to decide whether we're getting bogged down and should + * terminate this pass. + */ + + /* Check #1 : are we more than 10 minutes into the chunk ? */ + if (current_time - job->start_time > IMPORT_CHUNK_TEST_HOLDOFF_TIME) { + /* Check #2 : Have we slowed down considerably recently ? */ + if ((job->recent_progress_rate / job->average_progress_rate) < + IMPORT_CHUNK_TEST_SLOWDOWN_RATIO_A) { + /* Check #3: Cache performing poorly---the puported reason + * for the slowdown */ + if (job->cache_hit_ratio < IMPORT_CHUNK_TEST_CACHE_HIT_RATIO) { + /* We have a winner ! */ + import_log_notice(job, "Decided to end this pass because " + "the progress rate has dropped below " + "the %.0f%% threshold.", + IMPORT_CHUNK_TEST_SLOWDOWN_RATIO_A*100.0); + return 1; + } + } else { + if ((job->recent_progress_rate / job->average_progress_rate) < + IMPORT_CHUNK_TEST_SLOWDOWN_RATIO_B) { + /* Alternative check: have we really, really slowed down, + * without the test for cache overflow? */ + /* This is designed to catch the case where the cache has + * been misconfigured too large */ + if (number_of_times_here > 10) { + /* Got to get here ten times at least */ + import_log_notice(job, "Decided to end this pass " + "because the progress rate " + "plummeted below %.0f%%", + IMPORT_CHUNK_TEST_SLOWDOWN_RATIO_B*100.0); + return 1; + } + number_of_times_here++; + } + } + } + + number_of_times_here = 0; + return 0; +} + +static void import_push_progress_history(ImportJob *job, ID current_id, + time_t current_time) +{ + int i = 0; + + for (i = 0; i < IMPORT_JOB_PROG_HISTORY_SIZE - 1; i++) { + job->progress_history[i] = job->progress_history[i+1]; + job->progress_times[i] = job->progress_times[i+1]; + } + job->progress_history[i] = current_id; + job->progress_times[i] = current_time; +} + +static void import_calc_rate(ImportWorkerInfo *info, int time_interval) +{ + size_t ids = info->last_ID_processed - info->previous_ID_counted; + double rate = (double)ids / time_interval; + + if ( (info->previous_ID_counted != 0) && (info->last_ID_processed != 0) ) { + info->rate = rate; + } else { + info->rate = 0; + } + info->previous_ID_counted = info->last_ID_processed; +} + +/* find the rate (ids/time) of work from a worker thread between history + * marks A and B. + */ +#define HISTORY(N) (job->progress_history[N]) +#define TIMES(N) (job->progress_times[N]) +#define PROGRESS(A, B) ((HISTORY(B) > HISTORY(A)) ? \ + ((double)(HISTORY(B) - HISTORY(A)) / \ + (double)(TIMES(B) - TIMES(A))) : \ + (double)0) + +static int import_monitor_threads(ImportJob *job, int *status) +{ + PRIntervalTime tenthsecond = PR_MillisecondsToInterval(100); + ImportWorkerInfo *current_worker = NULL; + ImportWorkerInfo *producer = NULL, *foreman = NULL; + int finished = 0; + int giveup = 0; + int count = 1; /* 1 to prevent premature status report */ + int producer_done = 0; + const int display_interval = 200; + time_t time_now = 0; + time_t last_time = 0; + time_t time_interval = 0; + + + for (current_worker = job->worker_list; current_worker != NULL; + current_worker = current_worker->next) { + current_worker->command = RUN; + if (current_worker->work_type == PRODUCER) + producer = current_worker; + if (current_worker->work_type == FOREMAN) + foreman = current_worker; + } + + + if (job->flags & FLAG_USE_FILES) + PR_ASSERT(producer != NULL); + PR_ASSERT(foreman != NULL); + + time(&last_time); + job->start_time = last_time; + import_clear_progress_history(job); + + while (!finished) { + ID trailing_ID = NOID; + + DS_Sleep(tenthsecond); + finished = 1; + + /* First calculate the time interval since last reported */ + if (0 == (count % display_interval)) { + time(&time_now); + time_interval = time_now - last_time; + last_time = time_now; + /* Now calculate our rate of progress overall for this chunk */ + if (time_now != job->start_time) { + /* log a cute chart of the worker progress */ + import_log_status_start(job); + import_log_status_add_line(job, + "Index status for import of %s:", job->inst->inst_name); + import_log_status_add_line(job, + "-------Index Task-------State---Entry----Rate-"); + + import_push_progress_history(job, foreman->last_ID_processed, + time_now); + job->average_progress_rate = + (double)(HISTORY(IMPORT_JOB_PROG_HISTORY_SIZE-1)+1 - foreman->first_ID) / + (double)(TIMES(IMPORT_JOB_PROG_HISTORY_SIZE-1) - job->start_time); + job->recent_progress_rate = + PROGRESS(0, IMPORT_JOB_PROG_HISTORY_SIZE-1); + job->cache_hit_ratio = import_grok_db_stats(job->inst); + } + } + + for (current_worker = job->worker_list; current_worker != NULL; + current_worker = current_worker->next) { + /* Calculate the ID at which the slowest worker is currently + * processing */ + if ((trailing_ID > current_worker->last_ID_processed) && + (current_worker->work_type == WORKER)) { + trailing_ID = current_worker->last_ID_processed; + } + if (0 == (count % display_interval) && time_interval) { + import_calc_rate(current_worker, time_interval); + import_print_worker_status(current_worker); + } + if (current_worker->state != FINISHED) { + finished = 0; + } + if (current_worker->state == ABORTED) { + goto error_abort; + } + } + + if ((0 == (count % display_interval)) && + (job->start_time != time_now)) { + char buffer[256], *p = buffer; + + import_log_status_done(job); + p += sprintf(p, "Processed %lu entries ", (u_long)job->ready_ID); + if (job->total_pass > 1) + p += sprintf(p, "(pass %d) ", job->total_pass); + + p += sprintf(p, "-- average rate %.1f/sec, ", + job->average_progress_rate); + p += sprintf(p, "recent rate %.1f/sec, ", + job->recent_progress_rate); + p += sprintf(p, "hit ratio %.0f%%", job->cache_hit_ratio * 100.0); + import_log_notice(job, "%s", buffer); + } + + /* Then let's see if it's time to complete this import pass */ + if (!giveup) { + giveup = import_throw_in_towel(job, time_now, trailing_ID); + if (giveup) { + /* If so, signal the lead thread to stop */ + import_log_notice(job, "Ending pass number %d ...", + job->total_pass); + foreman->command = STOP; + while (foreman->state != FINISHED) { + DS_Sleep(tenthsecond); + } + import_log_notice(job, "Foreman is done; waiting for " + "workers to finish..."); + } + } + + /* if the producer is finished, and the foreman has caught up... */ + if (producer) { + producer_done = (producer->state == FINISHED); + } else { + producer_done = (job->flags & FLAG_PRODUCER_DONE); + } + if (producer_done && (job->lead_ID == job->ready_ID)) { + /* tell the foreman to stop if he's still working. */ + if (foreman->state != FINISHED) + foreman->command = STOP; + + /* if all the workers are caught up too, we're done */ + if (trailing_ID == job->lead_ID) + break; + } + + /* if the foreman is done (end of pass) and the worker threads + * have caught up... + */ + if ((foreman->state == FINISHED) && (job->ready_ID == trailing_ID)) { + break; + } + + count++; + } + + import_log_notice(job, "Workers finished; cleaning up..."); + + /* Now tell all the workers to stop */ + for (current_worker = job->worker_list; current_worker != NULL; + current_worker = current_worker->next) { + if (current_worker->work_type != PRODUCER) + current_worker->command = STOP; + } + + /* Having done that, wait for them to say that they've stopped */ + for (current_worker = job->worker_list; current_worker != NULL; ) { + if ((current_worker->state != FINISHED) && + (current_worker->state != ABORTED) && + (current_worker->work_type != PRODUCER)) { + DS_Sleep(tenthsecond); /* Only sleep if we hit a thread that is still not done */ + continue; + } else { + current_worker = current_worker->next; + } + } + import_log_notice(job, "Workers cleaned up."); + + /* If we're here and giveup is true, and the primary hadn't finished + * processing the input files, we need to return IMPORT_INCOMPLETE_PASS */ + if (giveup && (job->input_filenames || (job->flags & FLAG_ONLINE) || + (job->flags & FLAG_REINDEXING /* support multi-pass */))) { + if (producer_done && (job->ready_ID == job->lead_ID)) { + /* foreman caught up with the producer, and the producer is + * done. + */ + *status = IMPORT_COMPLETE_PASS; + } else { + *status = IMPORT_INCOMPLETE_PASS; + } + } else { + *status = IMPORT_COMPLETE_PASS; + } + return 0; + +error_abort: + return ERR_IMPORT_ABORTED; +} + + +/********** running passes **********/ + +static int import_run_pass(ImportJob *job, int *status) +{ + int ret = 0; + + /* Start the threads running */ + ret = import_start_threads(job); + if (ret != 0) { + import_log_notice(job, "Starting threads failed: %d\n", ret); + goto error; + } + + /* Monitor the threads until we're done or fail */ + ret = import_monitor_threads(job, status); + if (ret == ERR_IMPORT_ABORTED) { + goto error; + } else if (ret != 0) { + import_log_notice(job, "Thread monitoring aborted: %d\n", ret); + goto error; + } + +error: + return ret; +} + +static void import_set_abort_flag_all(ImportJob *job, int wait_for_them) +{ + + ImportWorkerInfo *worker; + + /* tell all the worker threads to abort */ + job->flags |= FLAG_ABORT; + + /* setting of the flag in the job will be detected in the worker, foreman + * threads and if there are any threads which have a sleeptime 200 msecs + * = import_sleep_time; after that time, they will examine the condition + * (job->flags & FLAG_ABORT) which will unblock the thread to proceed to + * abort. Hence, we will sleep here for atleast 3 sec to make sure clean + * up occurs */ + /* allow all the aborts to be processed */ + DS_Sleep(PR_MillisecondsToInterval(3000)); + + if (wait_for_them) { + /* Having done that, wait for them to say that they've stopped */ + for (worker = job->worker_list; worker != NULL; ) { + DS_Sleep(PR_MillisecondsToInterval(100)); + if ((worker->state != FINISHED) && + (worker->state != ABORTED)){ + continue; + } + else{ + worker = worker->next; + } + } + } + +} + + +/* tell all the threads to abort */ +void import_abort_all(ImportJob *job, int wait_for_them) +{ + ImportWorkerInfo *worker; + + /* tell all the worker threads to abort */ + job->flags |= FLAG_ABORT; + + for (worker = job->worker_list; worker; worker = worker->next) + worker->command = ABORT; + + if (wait_for_them) { + /* Having done that, wait for them to say that they've stopped */ + for (worker = job->worker_list; worker != NULL; ) { + DS_Sleep(PR_MillisecondsToInterval(100)); + if ((worker->state != FINISHED) && + (worker->state != ABORTED)) + continue; + else + worker = worker->next; + } + } +} + +/* Helper function to make up filenames */ +int import_make_merge_filenames(char *directory, char *indexname, int pass, + char **oldname, char **newname) +{ + /* Filenames look like this: attributename<LDBM_FILENAME_SUFFIX> + and need to be renamed to: attributename<LDBM_FILENAME_SUFFIX>.n + where n is the pass number. + */ + size_t oldname_length = strlen(directory) + 1 + strlen(indexname) + + strlen(LDBM_FILENAME_SUFFIX) + 1 ; + /* Enough space for an 8-digit pass number */ + size_t newname_length = oldname_length + 9; + + *oldname = slapi_ch_malloc(oldname_length); + if (NULL == oldname) + return -1; + *newname = slapi_ch_malloc(newname_length); + if (NULL == newname) + return -1; + sprintf(*oldname, "%s/%s%s", directory, indexname, LDBM_FILENAME_SUFFIX); + sprintf(*newname, "%s/%s.%d%s", directory, indexname, pass, + LDBM_FILENAME_SUFFIX); + return 0; +} + +/* Task here is as follows: + * First, if this is pass #1, check for the presence of a merge + * directory. If it is not present, create it. + * If it is present, delete all the files in it. + * Then, flush the dblayer and close files. + * Now create a numbered subdir of the merge directory for this pass. + * Next, move the index files, except entrydn, parentid and id2entry to + * the merge subdirectory. Important to move if we can, because + * that can be millions of times faster than a copy. + * Finally open the dblayer back up because the caller expects + * us to not muck with it. + */ +static int import_sweep_after_pass(ImportJob *job) +{ + backend *be = job->inst->inst_be; + int ret = 0; + + import_log_notice(job, "Sweeping files for merging later..."); + + ret = dblayer_instance_close(be); + + if (0 == ret) { + /* Walk the list of index jobs */ + ImportWorkerInfo *current_worker = NULL; + + for (current_worker = job->worker_list; current_worker != NULL; + current_worker = current_worker->next) { + /* Foreach job, rename the file to <filename>.n, where n is the + * pass number */ + if ((current_worker->work_type != FOREMAN) && + (current_worker->work_type != PRODUCER) && + (strcasecmp(current_worker->index_info->name, "parentid") != 0)) { + char *newname = NULL; + char *oldname = NULL; + + ret = import_make_merge_filenames(job->inst->inst_dir_name, + current_worker->index_info->name, job->current_pass, + &oldname, &newname); + if (0 != ret) { + break; + } + if (PR_Access(oldname, PR_ACCESS_EXISTS) == PR_SUCCESS) { + ret = PR_Rename(oldname, newname); + if (ret != PR_SUCCESS) { + PRErrorCode prerr = PR_GetError(); + import_log_notice(job, "Failed to rename file \"%s\" to \"%s\", " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)", + oldname, newname, prerr, slapd_pr_strerror(prerr)); + slapi_ch_free( (void**)&newname); + slapi_ch_free( (void**)&oldname); + break; + } + } + slapi_ch_free( (void**)&newname); + slapi_ch_free( (void**)&oldname); + } + } + + ret = dblayer_instance_start(be, DBLAYER_IMPORT_MODE); + } + + if (0 == ret) { + import_log_notice(job, "Sweep done."); + } else { + if (ENOSPC == ret) { + import_log_notice(job, "ERROR: NO DISK SPACE LEFT in sweep phase"); + } else { + import_log_notice(job, "ERROR: Sweep phase error %d (%s)", ret, + dblayer_strerror(ret)); + } + } + + return ret; +} + +/* when the import is done, this function is called to bring stuff back up. + * returns 0 on success; anything else is an error + */ +static int import_all_done(ImportJob *job, int ret) +{ + ldbm_instance *inst = job->inst; + + /* Writing this file indicates to future server startups that + * the db is OK */ + if (ret == 0) { + char inst_dir[MAXPATHLEN*2]; + char *inst_dirp = NULL; + inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN*2); + ret = dbversion_write(inst->inst_li, inst_dirp, NULL); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + } + + if (job->task != NULL && 0 == job->task->task_refcount) { + /* exit code */ + job->task->task_exitcode = ret; + job->task->task_state = SLAPI_TASK_FINISHED; + job->task->task_progress = job->task->task_work; + job->task->task_private = NULL; + slapi_task_status_changed(job->task); + } + + if (job->flags & FLAG_ONLINE) { + /* start up the instance */ + ret = dblayer_instance_start(job->inst->inst_be, DBLAYER_NORMAL_MODE); + if (ret != 0) + return ret; + + /* bring backend online again */ + slapi_mtn_be_enable(inst->inst_be); + } + + return ret; +} + + +int import_main_offline(void *arg) +{ + ImportJob *job = (ImportJob *)arg; + ldbm_instance *inst = job->inst; + backend *be = inst->inst_be; + int ret = 0; + time_t beginning = 0; + time_t end = 0; + int finished = 0; + int status = 0; + int verbose = 1; + ImportWorkerInfo *producer = NULL; + + if (job->task) + job->task->task_refcount++; + + PR_ASSERT(inst != NULL); + time(&beginning); + + /* Decide which indexes are needed */ + if (job->flags & FLAG_INDEX_ATTRS) { + /* Here, we get an AVL tree which contains nodes for all attributes + * in the schema. Given this tree, we need to identify those nodes + * which are marked for indexing. */ + avl_apply(job->inst->inst_attrs, (IFP)import_attr_callback, + (caddr_t)job, -1, AVL_INORDER); + vlv_getindices((IFP)import_attr_callback, (void *)job, be); + } + + /* Determine how much index buffering space to allocate to each index */ + import_set_index_buffer_size(job); + + /* initialize the entry FIFO */ + ret = import_fifo_init(job); + if (ret) { + if (! (job->flags & FLAG_USE_FILES)) { + PR_Lock(job->wire_lock); + PR_NotifyCondVar(job->wire_cv); + PR_Unlock(job->wire_lock); + } + goto error; + } + + if (job->flags & FLAG_USE_FILES) { + /* importing from files: start up a producer thread to read the + * files and queue them + */ + producer = CALLOC(ImportWorkerInfo); + if (! producer) + goto error; + + /* start the producer */ + import_init_worker_info(producer, job); + producer->work_type = PRODUCER; +#if defined(UPGRADEDB) + if (job->flags & FLAG_REINDEXING) + { + if (! CREATE_THREAD(PR_USER_THREAD, (VFP)index_producer, producer, + PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE)) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, + "unable to spawn index producer thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + goto error; + } + } + else +#endif + { + import_log_notice(job, "Beginning import job..."); + if (! CREATE_THREAD(PR_USER_THREAD, (VFP)import_producer, producer, + PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE)) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, + "unable to spawn import producer thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + goto error; + } + } + + if (0 == job->job_index_buffer_suggestion) + import_log_notice(job, "Index buffering is disabled."); + else + import_log_notice(job, + "Index buffering enabled with bucket size %lu", + job->job_index_buffer_suggestion); + + job->worker_list = producer; + } else { + /* release the startup lock and let the entries start queueing up + * in for import */ + PR_Lock(job->wire_lock); + PR_NotifyCondVar(job->wire_cv); + PR_Unlock(job->wire_lock); + } + + /* Run as many passes as we need to complete the job or die honourably in + * the attempt */ + while (! finished) { + job->current_pass++; + job->total_pass++; + ret = import_run_pass(job, &status); + /* The following could have happened: + * (a) Some error happened such that we're hosed. + * This is indicated by a non-zero return code. + * (b) We finished the complete file without needing a second pass + * This is indicated by a zero return code and a status of + * IMPORT_COMPLETE_PASS and current_pass == 1; + * (c) We completed a pass and need at least another one + * This is indicated by a zero return code and a status of + * IMPORT_INCOMPLETE_PASS + * (d) We just completed what turned out to be the last in a + * series of passes + * This is indicated by a zero return code and a status of + * IMPORT_COMPLETE_PASS and current_pass > 1 + */ + if (ret == ERR_IMPORT_ABORTED) { + /* at least one of the threads has aborted -- shut down ALL + * of the threads */ + import_log_notice(job, "Aborting all import threads..."); + /* this abort sets the abort flag on the threads and will block for + * the exit of all threads + */ + import_set_abort_flag_all(job, 1); + import_log_notice(job, "Import threads aborted."); + goto error; + } + + if (0 != ret) { + /* Some horrible fate has befallen the import */ + import_log_notice(job, "Fatal pass error %d", ret); + goto error; + } + + /* No error, but a number of possibilities */ + if ( IMPORT_COMPLETE_PASS == status ) { + if (1 == job->current_pass) { + /* We're done !!!! */ ; + } else { + /* Save the files, then merge */ + ret = import_sweep_after_pass(job); + if (0 != ret) { + goto error; + } + ret = import_mega_merge(job); + if (0 != ret) { + goto error; + } + } + finished = 1; + } else { + if (IMPORT_INCOMPLETE_PASS == status) { + /* Need to go round again */ + /* Time to save the files we've built for later */ + ret = import_sweep_after_pass(job); + if (0 != ret) { + goto error; + } + if ( (inst->inst_li->li_maxpassbeforemerge != 0) && + (job->current_pass > inst->inst_li->li_maxpassbeforemerge) ) + { + ret = import_mega_merge(job); + if (0 != ret) { + goto error; + } + job->current_pass = 1; + ret = import_sweep_after_pass(job); + if (0 != ret) { + goto error; + } + } + + /* Fixup the first_ID value to reflect previous work */ + job->first_ID = job->ready_ID + 1; + import_free_thread_data(job); + job->worker_list = producer; + import_log_notice(job, "Beginning pass number %d", + job->total_pass+1); + } else { + /* Bizarro-slapd */ + goto error; + } + } + } + + /* kill the producer now; we're done */ + if (producer) { + import_log_notice(job, "Cleaning up producer thread..."); + producer->command = STOP; + /* wait for the lead thread to stop */ + while (producer->state != FINISHED) { + DS_Sleep(PR_MillisecondsToInterval(100)); + } + } + + /* Now do the numsubordinates attribute */ + import_log_notice(job, "Indexing complete. Post-processing..."); + /* [610066] reindexed db cannot be used in the following backup/restore */ + if ( !(job->flags & FLAG_REINDEXING) && + (ret = update_subordinatecounts(be, job->mothers, NULL)) != 0) { + import_log_notice(job, "Failed to update numsubordinates attributes"); + goto error; + } + + /* And the ancestorid index */ + if ((ret = ldbm_ancestorid_create_index(be)) != 0) { + import_log_notice(job, "Failed to create ancestorid index"); + goto error; + } + + import_log_notice(job, "Flushing caches..."); + if (0 != (ret = dblayer_flush(job->inst->inst_li)) ) { + import_log_notice(job, "Failed to flush database"); + goto error; + } + + /* New way to exit the routine: check the return code. + * If it's non-zero, delete the database files. + * Otherwise don't, but always close the database layer properly. + * Then return. This ensures that we can't make a half-good/half-bad + * Database. */ + +error: + /* If we fail, the database is now in a mess, so we delete it */ + import_log_notice(job, "Closing files..."); + cache_clear(&job->inst->inst_cache); + if (0 != ret) { + dblayer_delete_instance_dir(be); + dblayer_instance_close(job->inst->inst_be); + } else { + if (0 != (ret = dblayer_instance_close(job->inst->inst_be)) ) { + import_log_notice(job, "Failed to close database"); + } + } + if (!(job->flags & FLAG_ONLINE)) + dblayer_close(job->inst->inst_li, DBLAYER_IMPORT_MODE); + + time(&end); + if (verbose && (0 == ret)) { + int seconds_to_import = end - beginning; + size_t entries_processed = job->lead_ID - (job->starting_ID - 1); + double entries_per_second = (double) entries_processed / + (double) seconds_to_import; + + if (job->not_here_skipped) + { + if (job->skipped) + import_log_notice(job, "Import complete. Processed %lu entries " + "(%d bad entries were skipped, " + "%d entries were skipped because they don't " + "belong to this database) in %d seconds. " + "(%.2f entries/sec)", entries_processed, + job->skipped, job->not_here_skipped, + seconds_to_import, entries_per_second); + else + import_log_notice(job, "Import complete. Processed %lu entries " + "(%d entries were skipped because they don't " + "belong to this database) " + "in %d seconds. (%.2f entries/sec)", + entries_processed, job->not_here_skipped, + seconds_to_import, entries_per_second); + } + else + { + if (job->skipped) + import_log_notice(job, "Import complete. Processed %lu entries " + "(%d were skipped) in %d seconds. " + "(%.2f entries/sec)", entries_processed, + job->skipped, seconds_to_import, + entries_per_second); + else + import_log_notice(job, "Import complete. Processed %lu entries " + "in %d seconds. (%.2f entries/sec)", + entries_processed, seconds_to_import, + entries_per_second); + } + } + + if (0 != ret) { + import_log_notice(job, "Import failed."); + if (job->task != NULL) { + job->task->task_state = SLAPI_TASK_FINISHED; + job->task->task_exitcode = ret; + slapi_task_status_changed(job->task); + } + } else { + if (job->task) + job->task->task_refcount--; + + import_all_done(job, ret); + } + + /* This instance isn't busy anymore */ + instance_set_not_busy(job->inst); + + import_free_job(job); + if (producer) + FREE(producer); + + + return(ret); +} + +/* + * to be called by online import using PR_CreateThread() + * offline import directly calls import_main_offline() + * + */ +void import_main(void *arg) +{ + import_main_offline(arg); +} + +int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) +{ + backend *be = NULL; + int noattrindexes = 0; + ImportJob *job = NULL; + char **name_array = NULL; + int total_files, i; + PRThread *thread = NULL; + + job = CALLOC(ImportJob); + if (job == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "not enough memory to do import job\n", + 0, 0, 0); + return -1; + } + + slapi_pblock_get( pb, SLAPI_BACKEND, &be); + PR_ASSERT(NULL != be); + job->inst = (ldbm_instance *)be->be_instance_info; + slapi_pblock_get( pb, SLAPI_LDIF2DB_NOATTRINDEXES, &noattrindexes ); + slapi_pblock_get( pb, SLAPI_LDIF2DB_FILE, &name_array ); + + /* the removedupvals field is blatantly overloaded here to mean + * the chunk size too. (chunk size = number of entries that should + * be imported before starting a new pass. usually for debugging.) + */ + slapi_pblock_get(pb, SLAPI_LDIF2DB_REMOVEDUPVALS, &job->merge_chunk_size); + if (job->merge_chunk_size == 1) + job->merge_chunk_size = 0; + /* get list of specifically included and/or excluded subtrees from + * the front-end */ + ldbm_back_fetch_incl_excl(pb, &job->include_subtrees, + &job->exclude_subtrees); + /* get cn=tasks info, if any */ + slapi_pblock_get(pb, SLAPI_BACKEND_TASK, &job->task); + slapi_pblock_get(pb, SLAPI_LDIF2DB_ENCRYPT, &job->encrypt); + /* get uniqueid info */ + slapi_pblock_get(pb, SLAPI_LDIF2DB_GENERATE_UNIQUEID, &job->uuid_gen_type); + if (job->uuid_gen_type == SLAPI_UNIQUEID_GENERATE_NAME_BASED) { + char *namespaceid; + + slapi_pblock_get(pb, SLAPI_LDIF2DB_NAMESPACEID, &namespaceid); + job->uuid_namespace = slapi_ch_strdup(namespaceid); + } + + job->flags = FLAG_USE_FILES; +#if defined(UPGRADEDB) + if (NULL == name_array) /* no ldif file is given -> reindexing */ + job->flags |= FLAG_REINDEXING; +#endif + if (!noattrindexes) + job->flags |= FLAG_INDEX_ATTRS; + for (i = 0; name_array && name_array[i] != NULL; i++) + charray_add(&job->input_filenames, slapi_ch_strdup(name_array[i])); + job->starting_ID = 1; + job->first_ID = 1; + job->mothers = CALLOC(import_subcount_stuff); + + /* how much space should we allocate to index buffering? */ + job->job_index_buffer_size = import_get_index_buffer_size(); + if (job->job_index_buffer_size == 0) { + /* 10% of the allocated cache size + one meg */ + PR_Lock(job->inst->inst_li->li_config_mutex); + job->job_index_buffer_size = (job->inst->inst_li->li_import_cachesize/10) + + (1024*1024); + PR_Unlock(job->inst->inst_li->li_config_mutex); + } + import_subcount_stuff_init(job->mothers); + + if (job->task != NULL) { + /* count files, use that to track "progress" in cn=tasks */ + total_files = 0; + while (name_array && name_array[total_files] != NULL) + total_files++; + /* add 1 to account for post-import cleanup (which can take a + * significant amount of time) + */ + if (0 == total_files) /* reindexing */ + job->task->task_work = 2; + else + job->task->task_work = total_files + 1; + job->task->task_progress = 0; + job->task->task_state = SLAPI_TASK_RUNNING; + job->task->task_private = job; + job->task->destructor = import_task_destroy; + job->task->cancel = import_task_abort; + job->flags |= FLAG_ONLINE; + + /* create thread for import_main, so we can return */ + thread = PR_CreateThread(PR_USER_THREAD, import_main, (void *)job, + PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, + SLAPD_DEFAULT_THREAD_STACKSIZE); + if (thread == NULL) { + PRErrorCode prerr = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY, "unable to spawn import thread, " + SLAPI_COMPONENT_NAME_NSPR " error %d (%s)\n", + prerr, slapd_pr_strerror(prerr), 0); + import_free_job(job); + FREE(job); + return -2; + } + return 0; + } + + /* old style -- do it all synchronously (THIS IS GOING AWAY SOON) */ + return import_main_offline((void *)job); +} diff --git a/ldap/servers/slapd/back-ldbm/import.h b/ldap/servers/slapd/back-ldbm/import.h new file mode 100644 index 00000000..866ef9f8 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/import.h @@ -0,0 +1,199 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* + * structures & constants used for the import code + */ + + +/* Number of lines in the entry above which we switch to + using a tree to check for attribute presence in str2entry(). + */ +#define STR2ENTRY_ATTRIBUTE_PRESENCE_CHECK_THRESHOLD 100 + +#define IMPORT_ADD_OP_ATTRS_OK 0 +#define IMPORT_ADD_OP_ATTRS_NO_PARENT 1 + +#define IMPORT_COMPLETE_PASS 1 +#define IMPORT_INCOMPLETE_PASS 2 + +/* Constants for index buffering */ +#define IMPORT_MAX_INDEX_BUFFER_SIZE 100 +#define IMPORT_MIN_INDEX_BUFFER_SIZE 5 +#define IMPORT_INDEX_BUFFER_SIZE_CONSTANT (20*20*20*sizeof(ID)) + +static const int import_sleep_time = 200; /* in millisecs */ + +extern char *numsubordinates; +extern char *hassubordinates; + +typedef struct _import_worker_info ImportWorkerInfo; +typedef struct _import_index_info IndexInfo; + + +/* structure which describes an indexing job */ +struct _import_index_info +{ + char *name; + struct attrinfo *ai; + IndexInfo *next; +}; + +/* item on the entry FIFO */ +typedef struct { + struct backentry *entry; + char *filename; /* or NULL */ + int line; /* filename/line are used to report errors */ + int bad; /* foreman did not like the entry */ + size_t esize; /* entry size */ +} FifoItem; + +typedef struct { + FifoItem *item; + size_t size; /* Queue size in entries (computed in import_fifo_init). */ + size_t bsize; /* Queue limitation in max bytes */ + size_t c_bsize; /* Current queue size in bytes */ +} Fifo; + +/* notes on the import gang: + * 1. producer: reads the file(s), performs str2entry() and assigns IDs. + * job->lead_ID is the last entry in the FIFO it's decoded. as it + * circles the FIFO, it pauses whenever it runs into an entry with a + * non-zero refcount, and waits for the worker threads to finish. + * 2. foreman: reads the FIFO (up to lead_ID), adding operational attrs, + * and creating the entrydn & id2entry indexes. job->ready_ID is the + * last entry in the FIFO it's finished with. (workers can't browse + * the entries it's working on because it's effectively modifying the + * entry.) + * 3. workers (one for each other index): read the FIFO (up to ready_ID), + * creating the index for a particular attribute. + */ + +/* Structure holding stuff about the whole import job */ +#define IMPORT_JOB_PROG_HISTORY_SIZE 3 +typedef struct { + ldbm_instance *inst; /* db instance we're importing to */ + Slapi_Task *task; /* cn=tasks entry ptr */ + int flags; /* (see below) */ + char **input_filenames; /* NULL-terminated list of charz pointers */ + IndexInfo *index_list; /* A list of indexing jobs to do */ + ImportWorkerInfo *worker_list; /* A list of threads to work on the + * indexes */ + size_t number_indexers; /* count of the indexer threads (not including + * the primary) */ + ID starting_ID; /* Import starts work at this ID */ + ID first_ID; /* Import pass starts at this ID */ + ID lead_ID; /* Highest ID available in the cache */ + ID ready_ID; /* Highest ID the foreman is done with */ + ID trailing_ID; /* Lowest ID still available in the cache */ + int current_pass; /* un-merged pass number in a multi-pass import */ + int total_pass; /* total pass number in a multi-pass import */ + int skipped; /* # entries skipped because they were bad */ + int not_here_skipped; /* # entries skipped because they belong + * to another backend */ + size_t merge_chunk_size; /* Allows us to manually override the magic + * voodoo logic for deciding when to begin + * another pass */ + int uuid_gen_type; /* kind of uuid to generate */ + char *uuid_namespace; /* namespace for name-generated uuid */ + import_subcount_stuff *mothers; + double average_progress_rate; + double recent_progress_rate; + double cache_hit_ratio; + time_t start_time; + ID progress_history[IMPORT_JOB_PROG_HISTORY_SIZE]; + time_t progress_times[IMPORT_JOB_PROG_HISTORY_SIZE]; + size_t job_index_buffer_size; /* Suggested size of index buffering + * for all indexes */ + size_t job_index_buffer_suggestion; /* Suggested size of index buffering + * for one index */ + char **include_subtrees; /* list of subtrees to import */ + char **exclude_subtrees; /* list of subtrees to NOT import */ + Fifo fifo; /* entry fifo for indexing */ + char *task_status; /* transient state info for the end-user */ + PRLock *wire_lock; /* lock for serializing wire imports */ + PRCondVar *wire_cv; /* ... and ordering the startup */ + PRThread *main_thread; /* for FRI: import_main() thread id */ + int encrypt; +} ImportJob; + +#define FLAG_INDEX_ATTRS 0x01 /* should we index the attributes? */ +#define FLAG_USE_FILES 0x02 /* import from files */ +#define FLAG_PRODUCER_DONE 0x04 /* frontend is done sending entries + * for replica initialization */ +#define FLAG_ABORT 0x08 /* import has been aborted */ +#define FLAG_ONLINE 0x10 /* bring backend online when done */ +#if defined(UPGRADEDB) +#define FLAG_REINDEXING 0x20 /* read from id2entry and do indexing */ +#endif + + +/* Structure holding stuff about a worker thread and what it's up to */ +struct _import_worker_info { + int work_type; /* What sort of work is this ? */ + int command; /* Used to control the thread */ + int state; /* Thread indicates its state here */ + IndexInfo *index_info; /* info on what we're asked to do */ + ID last_ID_processed; + ID previous_ID_counted; /* Used by the monitor to calculate progress + * rate */ + double rate; /* Number of IDs processed per second */ + ID first_ID; /* Tell the thread to start at this ID */ + ImportJob *job; + ImportWorkerInfo *next; + size_t index_buffer_size; /* Size of index buffering for this index */ +}; + +/* Values for work_type */ +#define WORKER 1 +#define FOREMAN 2 +#define PRODUCER 3 + +/* Values for command */ +#define RUN 1 +#define PAUSE 2 +#define ABORT 3 +#define STOP 4 + +/* Values for state */ +#define WAITING 1 +#define RUNNING 2 +#define FINISHED 3 +#define ABORTED 4 + +/* this is just a convenience, because the slapi_ch_* calls are annoying */ +#define CALLOC(name) (name *)slapi_ch_calloc(1, sizeof(name)) +#define FREE(x) slapi_ch_free((void **)&(x)) + + +/* import.c */ +FifoItem *import_fifo_fetch(ImportJob *job, ID id, int worker); +void import_free_job(ImportJob *job); +void import_log_notice(ImportJob *job, char *format, ...); +void import_abort_all(ImportJob *job, int wait_for_them); +int import_entry_belongs_here(Slapi_Entry *e, backend *be); +int import_make_merge_filenames(char *directory, char *indexname, int pass, + char **oldname, char **newname); +void import_main(void *arg); +int import_main_offline(void *arg); +int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb); + +/* import-merge.c */ +int import_mega_merge(ImportJob *job); + +/* ldif2ldbm.c */ +void reset_progress( void ); +void report_progress( int count, int done ); +int add_op_attrs(Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *ep, + int *status); + +/* import-threads.c */ +void import_producer(void *param); +#if defined(UPGRADEDB) +void index_producer(void *param); +#endif +void import_foreman(void *param); +void import_worker(void *param); +static void import_wait_for_space_in_fifo(ImportJob *job, size_t new_esize); diff --git a/ldap/servers/slapd/back-ldbm/index.c b/ldap/servers/slapd/back-ldbm/index.c new file mode 100644 index 00000000..c742c5fe --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/index.c @@ -0,0 +1,1852 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +/* index.c - routines for dealing with attribute indexes */ + +#include "back-ldbm.h" +#if ( defined ( OSF1 )) +#undef BUFSIZ +#define BUFSIZ 1024 +#endif + +static const char *errmsg = "database index operation failed"; + +static int is_indexed (const char* indextype, int indexmask, char** index_rules); +static char* index2prefix (const char* indextype); +static void free_prefix (char*); +static Slapi_Value ** +valuearray_minus_valuearray( + void *plugin, + Slapi_Value **a, + Slapi_Value **b +); +static int index_addordel_values( backend *be, const char *type, struct berval **vals, struct berval **evals, ID id, int flags, back_txn *txn ); +static int index_addordel_values_ext( backend *be, const char *type, struct berval **vals, struct berval **evals, ID id, int flags, back_txn *txn,int *idl_disposition, void *buffer_handle ); + +const char* indextype_PRESENCE = "pres"; +const char* indextype_EQUALITY = "eq"; +const char* indextype_APPROX = "approx"; +const char* indextype_SUB = "sub"; + +static char prefix_PRESENCE[2] = {PRES_PREFIX, 0}; +static char prefix_EQUALITY[2] = {EQ_PREFIX, 0}; +static char prefix_APPROX [2] = {APPROX_PREFIX, 0}; +static char prefix_SUB [2] = {SUB_PREFIX, 0}; + +/* Yes, prefix_PRESENCE and prefix_SUB are identical. + * It works because SUB is always followed by a key value, + * but PRESENCE never is. Too slick by half. + */ + + +/* Structures for index key buffering magic used by import code */ +struct _index_buffer_bin { + DBT key; + IDList *value; +}; +typedef struct _index_buffer_bin index_buffer_bin; + +struct _index_buffer_handle { + int flags; + size_t buffer_size; + size_t idl_size; + size_t max_key_length; + index_buffer_bin *bins; + unsigned char high_key_byte_range; + unsigned char low_key_byte_range; + unsigned char special_byte_a; + unsigned char special_byte_b; + size_t byte_range; + /* Statistics */ + int inserts; + int keys; +}; +typedef struct _index_buffer_handle index_buffer_handle; +#define INDEX_BUFFER_FLAG_SERIALIZE 1 +#define INDEX_BUFFER_FLAG_STATS 2 + +/* Index buffering functions */ + +static int +index_buffer_init_internal(size_t idl_size, + unsigned char high_key_byte_range, unsigned char low_key_byte_range, + size_t max_key_length,unsigned char special_byte_a, unsigned char special_byte_b, + int flags,void **h) +{ + size_t bin_count = 0; + /* Allocate the handle */ + index_buffer_bin *bins = NULL; + size_t i = 0; + size_t byte_range = 0; + + index_buffer_handle *handle = (index_buffer_handle *) slapi_ch_calloc(1,sizeof(index_buffer_handle)); + if (NULL == handle) { + return -1; + } + handle->idl_size = idl_size; + handle->flags = flags; + handle->high_key_byte_range = high_key_byte_range; + handle->low_key_byte_range = low_key_byte_range; + handle->special_byte_a = special_byte_a; + handle->special_byte_b = special_byte_b; + handle->max_key_length = max_key_length; + byte_range = (high_key_byte_range - low_key_byte_range) + 3 + 10; + handle->byte_range = byte_range; + /* Allocate the bins */ + bin_count = 1; + for (i = 0 ; i < max_key_length - 2; i++) { + bin_count *= byte_range; + } + handle->buffer_size = bin_count; + bins = (index_buffer_bin *)slapi_ch_calloc(bin_count, sizeof(index_buffer_bin)); + if (NULL == bins) { + return -1; + } + handle->bins = bins; + *h = (void*) handle; + return 0; +} + +int index_buffer_init(size_t size,int flags,void **h) +{ + return index_buffer_init_internal(size,'z','a',5,'^','$',flags,h); +} + +static int +index_put_idl(index_buffer_bin *bin,backend *be, DB_TXN *txn,struct attrinfo *a) +{ + int ret = 0; + DB *db = NULL; + int need_to_freed_new_idl = 0; + IDList *old_idl = NULL; + IDList *new_idl = NULL; + + if ( (ret = dblayer_get_index_file( be, a, &db, DBOPEN_CREATE )) != 0 ) { + return ret; + } + if (bin->key.data && bin->value) { + /* Need to read the IDL at the key, if present, and form the union with what we have */ + ret = NEW_IDL_NOOP; /* this flag is for new idl only; + * but this func is called only from index_buffer, + * which is enabled only for old idl. + */ + old_idl = idl_fetch(be,db,&bin->key,txn,a,&ret); + if ( (0 != ret) && (DB_NOTFOUND != ret)) { + goto error; + } + if ( (old_idl != NULL) && !ALLIDS(old_idl)) { + /* We need to merge in our block with what was there */ + new_idl = idl_union(be,old_idl,bin->value); + need_to_freed_new_idl = 1; + } else { + /* Nothing there previously, we store just what we have */ + new_idl = bin->value; + } + /* Then write back the result, but only if the existing idl wasn't ALLIDS */ + if (!old_idl || (old_idl && !ALLIDS(old_idl))) { + ret = idl_store_block(be,db,&bin->key,new_idl,txn,a); + } + if (0 != ret) { + goto error; + } + slapi_ch_free((void**)&bin->key.data ); + idl_free(bin->value); + /* If we're already at allids, store an allids block to prevent needless accumulation of blocks */ + if (old_idl && ALLIDS(old_idl)) { + bin->value = idl_allids(be); + } else { + bin->value = NULL; + } + } +error: + if (old_idl) { + idl_free(old_idl); + } + if (new_idl && need_to_freed_new_idl) { + idl_free(new_idl); + } + dblayer_release_index_file( be, a, db ); + return ret; +} + +/* The caller MUST check for DB_RUNRECOVERY being returned */ + +int +index_buffer_flush(void *h,backend *be, DB_TXN *txn,struct attrinfo *a) +{ + index_buffer_handle *handle = (index_buffer_handle *) h; + index_buffer_bin *bin = NULL; + int ret = 0; + size_t i = 0; + DB *db = NULL; + + PR_ASSERT(h); + + /* Note to the wary: here we do NOT create the index file up front */ + /* This is becuase there may be no buffers to flush, and the goal is to + * never create the index file (merging gets confused by this, among other things */ + + /* Walk along the bins, writing them to the database */ + for (i = 0; i < handle->buffer_size; i++) { + bin = &(handle->bins[i]); + if (bin->key.data && bin->value) { + if (NULL == db) { + if ( (ret = dblayer_get_index_file( be, a, &db, DBOPEN_CREATE )) != 0 ) { + return ret; + } + } + ret = index_put_idl(bin,be,txn,a); + if (0 != ret) { + goto error; + } + } + } +error: + if (NULL != db) { + dblayer_release_index_file( be, a, db ); + } + return ret; +} + +int +index_buffer_terminate(void *h) +{ + index_buffer_handle *handle = (index_buffer_handle *) h; + index_buffer_bin *bin = NULL; + size_t i = 0; + + PR_ASSERT(h); + /* Free all the buffers */ + /* First walk down the bins, freeing the IDLs and the bins they're in */ + for (i = 0; i < handle->buffer_size; i++) { + bin = &(handle->bins[i]); + if (bin->value) { + idl_free(bin->value); + bin->value = NULL; + } + if (bin->key.data) { + free(bin->key.data); + } + } + free(handle->bins); + /* Now free the handle */ + free(handle); + return 0; +} + +/* This function returns -1 or -2 for local errors, and DB_ errors as well. */ + +static int +index_buffer_insert(void *h, DBT *key, ID id,backend *be, DB_TXN *txn,struct attrinfo *a) +{ + index_buffer_handle *handle = (index_buffer_handle *) h; + index_buffer_bin *bin = NULL; + size_t index = 0; + int idl_ret = 0; + unsigned char x = 0; + unsigned int i = 0; + int ret = 0; + + PR_ASSERT(h); + + /* Check key length for validity */ + if (key->size > handle->max_key_length) { + return -2; + } + /* discard the first character, as long as its the substring prefix */ + if ((unsigned char)((char*)key->data)[0] != SUB_PREFIX) { + return -2; + } + /* Compute the bin index from the key */ + /* Walk along the key data, byte by byte */ + for (i = 1; i < (key->size - 1); i++) { + /* foreach byte, normalize to the range we accept */ + x = (unsigned char) ((char*)key->data)[i]; + if ( (x == handle->special_byte_a) || (x == handle->special_byte_b) ) { + if (x == handle->special_byte_a) { + x = handle->high_key_byte_range + 1; + } + if (x == handle->special_byte_b) { + x = handle->high_key_byte_range + 2; + } + } else { + if ( x >= '0' && x <= '9' ) { + x = (x - '0') + handle->high_key_byte_range + 3; + } else { + if (x > handle->high_key_byte_range) { + return -2; /* Out of range */ + } + if (x < handle->low_key_byte_range) { + return -2; /* Out of range */ + } + } + } + x = x - handle->low_key_byte_range; + index *= handle->byte_range; + index += x; + } + /* Check that the last byte in the key is zero */ + if (0 != (unsigned char)((char*)key->data)[i]) { + return -2; + } + PR_ASSERT(index < handle->buffer_size); + /* Get the bin */ + bin = &(handle->bins[index]); + /* Is the key already there ? */ +retry: + if (!(bin->key).data) { + (bin->key).size = key->size; + (bin->key).data = malloc(key->size); + if (NULL == bin->key.data) { + return -1; + } + memcpy(bin->key.data,key->data,key->size); + /* Make the IDL */ + bin->value = idl_alloc(handle->idl_size); + if (!bin->value) { + return -1; + } + } + idl_ret = idl_append(bin->value, id); + if (0 != idl_ret) { + if (1 == idl_ret) { + /* ID already present */ + } else { + /* If we get to here, it means that we've overflowed our IDL */ + /* So, we need to write it out to the DB and zero out the pointers */ + ret = index_put_idl(bin,be,txn,a); + /* Now we need to append the ID we have at hand */ + if (0 == ret) { + goto retry; + } + } + } + return ret; +} + +/* + * Add or Delete an entry from the attribute indexes. + * 'flags' is either BE_INDEX_ADD or BE_INDEX_DEL + */ +int +index_addordel_entry( + backend *be, + struct backentry *e, + int flags, + back_txn *txn +) +{ + char *type; + Slapi_Value **svals; + int rc, result; + Slapi_Attr *attr; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> index_%s_entry( \"%s\", %lu )\n", + (flags & BE_INDEX_ADD) ? "add" : "del", + backentry_get_ndn(e), (u_long)e->ep_id ); + + /* if we are adding a tombstone entry (see ldbm_add.c) */ + if ((flags & BE_INDEX_TOMBSTONE) && (flags & BE_INDEX_ADD)) + { + Slapi_DN parent; + Slapi_DN *sdn = slapi_entry_get_sdn(e->ep_entry); + slapi_sdn_init(&parent); + slapi_sdn_get_parent(sdn, &parent); + /* + * Just index the "nstombstone" attribute value from the objectclass + * attribute, and the nsuniqueid attribute value, and the entrydn value of the deleted entry. + */ + result = index_addordel_string(be, SLAPI_ATTR_OBJECTCLASS, SLAPI_ATTR_VALUE_TOMBSTONE, e->ep_id, flags, txn); + if ( result != 0 ) { + ldbm_nasty(errmsg, 1010, result); + return( result ); + } + result = index_addordel_string(be, SLAPI_ATTR_UNIQUEID, slapi_entry_get_uniqueid(e->ep_entry), e->ep_id, flags, txn); + if ( result != 0 ) { + ldbm_nasty(errmsg, 1020, result); + return( result ); + } + result = index_addordel_string(be, SLAPI_ATTR_NSCP_ENTRYDN, slapi_sdn_get_ndn(&parent), e->ep_id, flags, txn); + if ( result != 0 ) { + ldbm_nasty(errmsg, 1020, result); + return( result ); + } + slapi_sdn_done(&parent); + } + else + { + /* add each attribute to the indexes */ + rc = 0, result = 0; + for ( rc = slapi_entry_first_attr( e->ep_entry, &attr ); rc == 0; + rc = slapi_entry_next_attr( e->ep_entry, attr, &attr ) ) { + slapi_attr_get_type( attr, &type ); + svals = attr_get_present_values(attr); + result = index_addordel_values_sv( be, type, svals, NULL, e->ep_id, + flags, txn ); + if ( result != 0 ) { + ldbm_nasty(errmsg, 1030, result); + return( result ); + } + } + + /* update ancestorid index . . . */ + /* . . . only if we are not deleting a tombstone entry - tombstone entries are not in the ancestor id index - see bug 603279 */ + if (!((flags & BE_INDEX_TOMBSTONE) && (flags & BE_INDEX_DEL))) { + result = ldbm_ancestorid_index_entry(be, e, flags, txn); + if ( result != 0 ) { + return( result ); + } + } + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= index_%s_entry%s %d\n", + (flags & BE_INDEX_ADD) ? "add" : "del", + (flags & BE_INDEX_TOMBSTONE) ? " (tombstone)" : "", result ); + return( result ); +} + +/* + * Add ID to attribute indexes for which Add/Replace/Delete modifications exist + * [olde is the OLD entry, before modifications] + * [newe is the NEW entry, after modifications] + * the old entry is used for REPLACE; the new for DELETE */ +int +index_add_mods( + backend *be, + const LDAPMod **mods, + struct backentry *olde, + struct backentry *newe, + back_txn *txn +) +{ + int rc = 0; + int i; + Slapi_Attr *attr; + ID id = olde->ep_id; + Slapi_Value **svals = NULL; + + for ( i = 0; mods[i] != NULL; i++ ) { + switch ( mods[i]->mod_op & ~LDAP_MOD_BVALUES ) { + case LDAP_MOD_REPLACE: + /* We need to first remove the old values from the + * index. */ + if ( slapi_entry_attr_find( olde->ep_entry, mods[i]->mod_type, &attr ) == 0 && + (svals = attr_get_present_values(attr)) != NULL ) { + index_addordel_values_sv( be, mods[i]->mod_type, + svals, NULL, id, + BE_INDEX_DEL|BE_INDEX_PRESENCE, + txn ); + } + case LDAP_MOD_ADD: + if ( mods[i]->mod_bvalues == NULL ) { + rc = 0; + } else { + Slapi_Value **mods_valueArray = NULL; + valuearray_init_bervalarray(mods[i]->mod_bvalues, + &mods_valueArray); + rc = index_addordel_values_sv( be, + mods[i]->mod_type, + mods_valueArray, NULL, + id, BE_INDEX_ADD, txn ); + valuearray_free(&mods_valueArray); + } + break; + + case LDAP_MOD_DELETE: + if ( (mods[i]->mod_bvalues == NULL) || + (mods[i]->mod_bvalues[0] == NULL) ) { + rc = 0; + /* if no value are specified all the values will + * be suppressed -> remove the presence index + */ + if ( slapi_entry_attr_find( olde->ep_entry, mods[i]->mod_type, &attr ) == 0 && + (svals = attr_get_present_values(attr)) != NULL ) { + index_addordel_values_sv( be, mods[i]->mod_type, + svals, NULL, id, BE_INDEX_DEL|BE_INDEX_PRESENCE, txn); + } + } else { + /* determine if the presence key should be + * removed (are we removing the last value + * for this attribute?) + */ + int flags = BE_INDEX_DEL; + Slapi_Value ** svals = NULL; + Slapi_Value **mods_valueArray = NULL; + + valuearray_init_bervalarray(mods[i]->mod_bvalues, + &mods_valueArray); + + if (slapi_entry_attr_find(newe->ep_entry, + mods[i]->mod_type, &attr) == 0) { + svals = attr_get_present_values(attr); + } + + if (svals == NULL || svals[0] == NULL) { + flags |= BE_INDEX_PRESENCE; + } + + rc = index_addordel_values_sv( be, mods[i]->mod_type, + mods_valueArray, + svals, id, flags, txn ); + valuearray_free(&mods_valueArray); + } + rc = 0; + break; + } + + if ( rc != 0 ) { + ldbm_nasty(errmsg, 1040, rc); + return( rc ); + } + } + + return( 0 ); +} + + +/* + * Convert a 'struct berval' into a displayable ASCII string + */ + +#define SPECIAL(c) (c < 32 || c > 126 || c == '\\' || c == '"') + +const char* +encode (const struct berval* data, char buf[BUFSIZ]) +{ + char* s; + char* last; + if (data == NULL || data->bv_len == 0) return ""; + last = data->bv_val + data->bv_len - 1; + for (s = data->bv_val; s < last; ++s) { + if ( SPECIAL (*s)) { + char* first = data->bv_val; + char* bufNext = buf; + size_t bufSpace = BUFSIZ - 4; + while (1) { +/* printf ("%lu bytes ASCII\n", (unsigned long)(s - first)); */ + if (bufSpace < (size_t)(s - first)) s = first + bufSpace - 1; + if (s != first) { + memcpy (bufNext, first, s - first); + bufNext += (s - first); + bufSpace -= (s - first); + } + do { + *bufNext++ = '\\'; --bufSpace; + if (bufSpace < 2) { + memcpy (bufNext, "..", 2); + bufNext += 2; + goto bail; + } + if (*s == '\\' || *s == '"') { + *bufNext++ = *s; --bufSpace; + } else { + sprintf (bufNext, "%02x", (unsigned)*(unsigned char*)s); + bufNext += 2; bufSpace -= 2; + } + } while (++s <= last && SPECIAL (*s)); + if (s > last) break; + first = s; + while ( ! SPECIAL (*s) && s <= last) ++s; + } + bail: + *bufNext = '\0'; +/* printf ("%lu chars in buffer\n", (unsigned long)(bufNext - buf)); */ + return buf; + } + } +/* printf ("%lu bytes, all ASCII\n", (unsigned long)(s - data->bv_val)); */ + return data->bv_val; +} + +static const char* +encoded (DBT* d, char buf [BUFSIZ]) +{ + struct berval data; + data.bv_len = d->dsize; + data.bv_val = d->dptr; + return encode (&data, buf); +} + +IDList * +index_read( + backend *be, + char *type, + const char *indextype, + const struct berval *val, + back_txn *txn, + int *err +) +{ + return index_read_ext(be, type, indextype, val, txn, err, NULL); +} + +/* + * Extended version of index_read. + * The unindexed flag can be used to distinguish between a + * return of allids due to the attr not being indexed or + * the value really being allids. + */ +IDList * +index_read_ext( + backend *be, + char *type, + const char *indextype, + const struct berval *val, + back_txn *txn, + int *err, + int *unindexed +) +{ + DB *db = NULL; + DB_TXN *db_txn = NULL; + DBT key = {0}; + IDList *idl; + char *prefix; + char *tmpbuf = NULL; + char buf[BUFSIZ]; + char typebuf[ SLAPD_TYPICAL_ATTRIBUTE_NAME_MAX_LENGTH ]; + struct attrinfo *ai = NULL; + char *basetmp, *basetype; + int retry_count = 0; + struct berval *encrypted_val = NULL; + + *err = 0; + + if (unindexed != NULL) *unindexed = 0; + prefix = index2prefix( indextype ); + LDAPDebug( LDAP_DEBUG_TRACE, "=> index_read( \"%s\" %s \"%s\" )\n", + type, prefix, encode (val, buf)); + + basetype = typebuf; + if ( (basetmp = slapi_attr_basetype( type, typebuf, sizeof(typebuf) )) + != NULL ) { + basetype = basetmp; + } + + ainfo_get( be, basetype, &ai ); + if (ai == NULL) { + free_prefix( prefix ); + slapi_ch_free_string( &basetmp ); + return NULL; + } + + LDAPDebug( LDAP_DEBUG_ARGS, " indextype: \"%s\" indexmask: 0x%x\n", + indextype, ai->ai_indexmask, 0 ); + + if ( !is_indexed( indextype, ai->ai_indexmask, ai->ai_index_rules ) ) { + idl = idl_allids( be ); + if (unindexed != NULL) *unindexed = 1; + LDAPDebug( LDAP_DEBUG_TRACE, "<= index_read %lu candidates " + "(allids - not indexed)\n", (u_long)IDL_NIDS(idl), 0, 0 ); + free_prefix( prefix ); + slapi_ch_free_string( &basetmp ); + return( idl ); + } + if ( (*err = dblayer_get_index_file( be, ai, &db, DBOPEN_CREATE )) != 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, + "<= index_read NULL (index file open for attr %s)\n", + basetype, 0, 0 ); + free_prefix (prefix); + slapi_ch_free_string( &basetmp ); + return( NULL ); + } + slapi_ch_free_string( &basetmp ); + + if ( val != NULL ) { + size_t plen, vlen; + char *realbuf; + int ret = 0; + + /* If necessary, encrypt this index key */ + ret = attrcrypt_encrypt_index_key(be, ai, val, &encrypted_val); + if (ret) { + LDAPDebug( LDAP_DEBUG_ANY, + "index_read failed to encrypt index key for %s\n", + basetype, 0, 0 ); + } + if (encrypted_val) { + val = encrypted_val; + } + plen = strlen( prefix ); + vlen = val->bv_len; + realbuf = (plen + vlen < sizeof(buf)) ? + buf : (tmpbuf = slapi_ch_malloc( plen + vlen + 1 )); + memcpy( realbuf, prefix, plen ); + memcpy( realbuf+plen, val->bv_val, vlen ); + realbuf[plen+vlen] = '\0'; + key.data = realbuf; + key.size = key.ulen = plen + vlen + 1; + key.flags = DB_DBT_USERMEM; + } else { + key.data = prefix; + key.size = key.ulen = strlen( prefix ) + 1; /* include 0 terminator */ + key.flags = DB_DBT_USERMEM; + } + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + for (retry_count = 0; retry_count < IDL_FETCH_RETRY_COUNT; retry_count++) { + *err = NEW_IDL_DEFAULT; + idl = idl_fetch( be, db, &key, db_txn, ai, err ); + if(*err == DB_LOCK_DEADLOCK) { + ldbm_nasty("index read retrying transaction", 1045, *err); + continue; + } else { + break; + } + } + if(retry_count == IDL_FETCH_RETRY_COUNT) { + ldbm_nasty("index_read retry count exceeded",1046,*err); + } else if ( *err != 0 && *err != DB_NOTFOUND ) { + ldbm_nasty(errmsg, 1050, *err); + } + slapi_ch_free_string(&tmpbuf); + + dblayer_release_index_file( be, ai, db ); + + free_prefix (prefix); + + if (encrypted_val) { + ber_bvfree(encrypted_val); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= index_read %lu candidates\n", + (u_long)IDL_NIDS(idl), 0, 0 ); + return( idl ); +} + +static int +DBTcmp (DBT* L, DBT* R) +{ + struct berval Lv; + struct berval Rv; + Lv.bv_val = L->dptr; Lv.bv_len = L->dsize; + Rv.bv_val = R->dptr; Rv.bv_len = R->dsize; + return slapi_berval_cmp (&Lv, &Rv); +} + +#define DBT_EQ(L,R) ((L)->dsize == (R)->dsize &&\ + ! memcmp ((L)->dptr, (R)->dptr, (L)->dsize)) + + +#define DBT_FREE_PAYLOAD(d) if ((d).data) {free((d).data);(d).data=NULL;} + +/* Steps to the next key without keeping a cursor open */ +/* Returns the new key value in the DBT */ +static int index_range_next_key(DB *db,DBT *key,DB_TXN *db_txn) +{ + DBC *cursor = NULL; + DBT data = {0}; + int ret = 0; + void *saved_key = key->data; + + /* Make cursor */ +retry: + ret = db->cursor(db,db_txn,&cursor, 0); + if (0 != ret) { + return ret; + } + /* Seek to the last key */ + data.flags = DB_DBT_MALLOC; + ret = cursor->c_get(cursor,key,&data,DB_SET); /* data allocated here, we don't need it */ + DBT_FREE_PAYLOAD(data); + if (DB_NOTFOUND == ret) { + void *old_key_buffer = key->data; + /* If this happens, it means that we tried to seek to a key which has just been deleted */ + /* So, we seek to the nearest one instead */ + ret = cursor->c_get(cursor,key,&data,DB_SET_RANGE); + /* a new key and data are allocated here, need to free them both */ + if (old_key_buffer != key->data) { + DBT_FREE_PAYLOAD(*key); + } + DBT_FREE_PAYLOAD(data); + } + if (0 != ret) { + if (DB_LOCK_DEADLOCK == ret) + { + /* Deadlock detected, retry the operation */ + cursor->c_close(cursor); + cursor = NULL; + key->data = saved_key; + goto retry; + } else + { + goto error; + } + } + /* Seek to the next one + * [612498] NODUP is needed for new idl to get the next non-duplicated key + * No effect on old idl since there's no dup there (i.e., DB_NEXT == DB_NEXT_NODUP) + */ + ret = cursor->c_get(cursor,key,&data,DB_NEXT_NODUP); /* new key and data are allocated, we only need the key */ + DBT_FREE_PAYLOAD(data); + if (DB_LOCK_DEADLOCK == ret) + { + /* Deadlock detected, retry the operation */ + cursor->c_close(cursor); + cursor = NULL; + key->data = saved_key; + goto retry; + } +error: + /* Close the cursor */ + cursor->c_close(cursor); + if (saved_key) { /* Need to free the original key passed in */ + if (saved_key == key->data) { + /* Means that we never allocated a new key */ + ; + } else { + free(saved_key); + } + } + return ret; +} + +IDList * +index_range_read( + Slapi_PBlock *pb, + backend *be, + char *type, + const char *indextype, + int operator, + struct berval *val, + struct berval *nextval, + int range, + back_txn *txn, + int *err +) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + DB *db; + DB_TXN *db_txn = NULL; + DBC *dbc = NULL; + DBT lowerkey = {0}; + DBT upperkey = {0}; + DBT cur_key = {0}; + DBT data = {0} ; + IDList *idl= NULL; + char *prefix; + char *realbuf, *nextrealbuf; + size_t reallen, nextreallen; + size_t plen; + ID i; + struct attrinfo *ai = NULL; + int lookthrough_limit = -1; /* default no limit */ + int retry_count = 0; + int is_and = 0; + int sizelimit = 0; + + *err = 0; + plen = strlen( prefix = index2prefix( indextype )); + slapi_pblock_get(pb, SLAPI_SEARCH_IS_AND, &is_and); + if (!is_and) + { + slapi_pblock_get(pb, SLAPI_SEARCH_SIZELIMIT, &sizelimit); + } + + /* + * Determine the lookthrough_limit from the PBlock. + * No limit if there is no PBlock supplied or if there is no + * search result set and the requestor is root. + */ + if (pb != NULL) { + back_search_result_set *sr = NULL; + + slapi_pblock_get( pb, SLAPI_SEARCH_RESULT_SET, &sr ); + if (sr != NULL) { + /* the normal case */ + lookthrough_limit = sr->sr_lookthroughlimit; + } else { + int isroot = 0; + slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot ); + if (!isroot) { + lookthrough_limit = li->li_lookthroughlimit; + } + } + } + + LDAPDebug(LDAP_DEBUG_TRACE, "index_range_read lookthrough_limit=%d\n", + lookthrough_limit, 0, 0); + + switch( operator ) { + case SLAPI_OP_LESS: + case SLAPI_OP_LESS_OR_EQUAL: + case SLAPI_OP_GREATER_OR_EQUAL: + case SLAPI_OP_GREATER: + break; + default: + LDAPDebug( LDAP_DEBUG_ANY, + "<= index_range_read(%s,%s) NULL (operator %i)\n", + type, prefix, operator ); + return( NULL ); + } + ainfo_get( be, type, &ai ); + if (ai == NULL) return NULL; + LDAPDebug( LDAP_DEBUG_ARGS, " indextype: \"%s\" indexmask: 0x%x\n", + indextype, ai->ai_indexmask, 0 ); + if ( !is_indexed( indextype, ai->ai_indexmask, ai->ai_index_rules )) { + idl = idl_allids( be ); + LDAPDebug( LDAP_DEBUG_TRACE, + "<= index_range_read(%s,%s) %lu candidates (allids)\n", + type, prefix, (u_long)IDL_NIDS(idl) ); + return( idl ); + } + if ( (*err = dblayer_get_index_file( be, ai, &db, DBOPEN_CREATE )) != 0 ) { + LDAPDebug( LDAP_DEBUG_ANY, + "<= index_range_read(%s,%s) NULL (could not open index file)\n", + type, prefix, 0 ); + return( NULL ); /* why not allids? */ + } + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + /* get a cursor so we can walk over the table */ + *err = db->cursor(db,db_txn,&dbc,0); + if (0 != *err ) { + ldbm_nasty(errmsg, 1060, *err); + LDAPDebug( LDAP_DEBUG_ANY, + "<= index_range_read(%s,%s) NULL: db->cursor() == %i\n", + type, prefix, *err ); + dblayer_release_index_file( be, ai, db ); + return( NULL ); /* why not allids? */ + } + + /* set up the starting and ending keys for a range search */ + if ( val != NULL ) { /* compute a key from val */ + const size_t vlen = val->bv_len; + reallen = plen + vlen + 1; + realbuf = slapi_ch_malloc( reallen ); + memcpy( realbuf, prefix, plen ); + memcpy( realbuf+plen, val->bv_val, vlen ); + realbuf[plen+vlen] = '\0'; + } else { + reallen = plen + 1; /* include 0 terminator */ + realbuf = slapi_ch_strdup(prefix); + } + if (range != 1) { + char *tmpbuf = NULL; + /* this is a search with only one boundary value */ + switch( operator ) { + case SLAPI_OP_LESS: + case SLAPI_OP_LESS_OR_EQUAL: + lowerkey.dptr = slapi_ch_strdup(prefix); + lowerkey.dsize = plen; + upperkey.dptr = realbuf; + upperkey.dsize = reallen; + break; + case SLAPI_OP_GREATER_OR_EQUAL: + case SLAPI_OP_GREATER: + lowerkey.dptr = realbuf; + lowerkey.dsize = reallen; + /* upperkey = a value slightly greater than prefix */ + tmpbuf = slapi_ch_malloc (plen + 1); + memcpy (tmpbuf, prefix, plen + 1); + ++(tmpbuf[plen-1]); + upperkey.dptr = tmpbuf; + upperkey.dsize = plen; + tmpbuf = NULL; + /* ... but not greater than the last key in the index */ + cur_key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + *err = dbc->c_get(dbc,&cur_key,&data,DB_LAST); /* key and data allocated here, need to free them */ + DBT_FREE_PAYLOAD(data); + /* Note that cur_key needs to get freed somewhere below */ + if (0 != *err) { + if (DB_NOTFOUND == *err) { + /* There are no keys in the index so we should return no candidates. */ + *err = 0; + idl = NULL; + slapi_ch_free( (void**)&realbuf); + dbc->c_close(dbc); + goto error; + } else { + ldbm_nasty(errmsg, 1070, *err); + LDAPDebug( LDAP_DEBUG_ANY, + "index_range_read(%s,%s) seek to end of index file err %i\n", + type, prefix, *err ); + } + } else if (DBTcmp (&upperkey, &cur_key) > 0) { + tmpbuf = slapi_ch_realloc (tmpbuf, cur_key.dsize); + memcpy (tmpbuf, cur_key.dptr, cur_key.dsize); + DBT_FREE_PAYLOAD(upperkey); + upperkey.dptr = tmpbuf; + upperkey.dsize = cur_key.dsize; + } + break; + } + } else { + /* this is a search with two boundary values (starting and ending) */ + if ( nextval != NULL ) { /* compute a key from nextval */ + const size_t vlen = nextval->bv_len; + nextreallen = plen + vlen + 1; + nextrealbuf = slapi_ch_malloc( plen + vlen + 1 ); + memcpy( nextrealbuf, prefix, plen ); + memcpy( nextrealbuf+plen, nextval->bv_val, vlen ); + nextrealbuf[plen+vlen] = '\0'; + } else { + nextreallen = plen + 1; /* include 0 terminator */ + nextrealbuf = slapi_ch_strdup(prefix); + } + /* set up the starting and ending keys for search */ + switch( operator ) { + case SLAPI_OP_LESS: + case SLAPI_OP_LESS_OR_EQUAL: + lowerkey.dptr = nextrealbuf; + lowerkey.dsize = nextreallen; + upperkey.dptr = realbuf; + upperkey.dsize = reallen; + break; + case SLAPI_OP_GREATER_OR_EQUAL: + case SLAPI_OP_GREATER: + lowerkey.dptr = realbuf; + lowerkey.dsize = reallen; + upperkey.dptr = nextrealbuf; + upperkey.dsize = nextreallen; + break; + } + } + /* if (LDAP_DEBUG_FILTER) { + char encbuf [BUFSIZ]; + LDAPDebug( LDAP_DEBUG_FILTER, " lowerkey=%s(%li bytes)\n", + encoded (&lowerkey, encbuf), (long)lowerkey.dsize, 0 ); + LDAPDebug( LDAP_DEBUG_FILTER, " upperkey=%s(%li bytes)\n", + encoded (&upperkey, encbuf), (long)upperkey.dsize, 0 ); + } */ + data.flags = DB_DBT_MALLOC; + lowerkey.flags = DB_DBT_MALLOC; + { + void *old_lower_key_data = lowerkey.data; + *err = dbc->c_get(dbc,&lowerkey,&data,DB_SET_RANGE); /* lowerkey, if allocated and needs freed */ + DBT_FREE_PAYLOAD(data); + if (old_lower_key_data != lowerkey.data) { + free(old_lower_key_data); + } + } + /* If the seek above fails due to DB_NOTFOUND, this means that there are no keys + which are >= the target key. This means that we should return no candidates */ + if (0 != *err) { + /* Free the key we just read above */ + DBT_FREE_PAYLOAD(lowerkey); + if (DB_NOTFOUND == *err) { + *err = 0; + idl = NULL; + } else { + idl = idl_allids( be ); + ldbm_nasty(errmsg, 1080, *err); + LDAPDebug( LDAP_DEBUG_ANY, + "<= index_range_read(%s,%s) allids (seek to lower key in index file err %i)\n", + type, prefix, *err ); + } + dbc->c_close(dbc); + goto error; + } + /* We now close the cursor, since we're about to iterate over many keys */ + *err = dbc->c_close(dbc); + + /* step through the indexed db to retrive IDs within the search range */ + DBT_FREE_PAYLOAD(cur_key); + cur_key.data = lowerkey.data; + cur_key.size = lowerkey.size; + lowerkey.data = NULL; /* Don't need this any more, since the memory will be freed from cur_key */ + if (operator == SLAPI_OP_GREATER) { + *err = index_range_next_key(db,&cur_key,db_txn); + } + while (*err == 0 && + (operator == SLAPI_OP_LESS) ? + DBTcmp(&cur_key, &upperkey) < 0 : + DBTcmp(&cur_key, &upperkey) <= 0) { + /* exit the loop when we either run off the end of the table, + * fail to read a key, or read a key that's out of range. + */ + IDList *tmp, *tmp2; + /* + char encbuf [BUFSIZ]; + LDAPDebug( LDAP_DEBUG_FILTER, " cur_key=%s(%li bytes)\n", + encoded (&cur_key, encbuf), (long)cur_key.dsize, 0 ); + */ + /* Check to see if we've already looked too hard */ + if (idl != NULL && lookthrough_limit != -1 && idl->b_nids > (ID)lookthrough_limit) { + if (NULL != idl) { + idl_free(idl); + } + idl = idl_allids( be ); + LDAPDebug(LDAP_DEBUG_TRACE, "index_range_read lookthrough_limit exceeded\n", + 0, 0, 0); + break; + } + if (idl != NULL && sizelimit > 0 && idl->b_nids > (ID)sizelimit) + { + LDAPDebug(LDAP_DEBUG_TRACE, "index_range_read sizelimit exceeded\n", + 0, 0, 0); + break; + } + + /* Check to see if the operation has been abandoned (also happens + * when the connection is closed by the client). + */ + if ( slapi_op_abandoned( pb )) { + if (NULL != idl) { + idl_free(idl); + idl = NULL; + } + LDAPDebug(LDAP_DEBUG_TRACE, + "index_range_read - operation abandoned\n", 0, 0, 0); + break; /* clean up happens outside the while() loop */ + } + + /* the cur_key DBT already has the first entry in it when we enter the loop */ + /* so we process the entry then step to the next one */ + cur_key.flags = 0; + for (retry_count = 0; retry_count < IDL_FETCH_RETRY_COUNT; retry_count++) { + *err = NEW_IDL_DEFAULT; + tmp = idl_fetch( be, db, &cur_key, NULL, ai, err ); + if(*err == DB_LOCK_DEADLOCK) { + ldbm_nasty("index_range_read retrying transaction", 1090, *err); + continue; + } else { + break; + } + } + if(retry_count == IDL_FETCH_RETRY_COUNT) { + ldbm_nasty("index_range_read retry count exceeded",1095,*err); + } + tmp2 = idl_union( be, idl, tmp ); + idl_free( idl ); + idl_free( tmp ); + idl = tmp2; + if (ALLIDS(idl)) { + LDAPDebug(LDAP_DEBUG_TRACE, "index_range_read hit an allids value\n", + 0, 0, 0); + break; + } + if (DBT_EQ (&cur_key, &upperkey)) { /* this is the last key */ + break; + /* Another c_get would return the same key, with no error. */ + } + data.flags = DB_DBT_MALLOC; + cur_key.flags = DB_DBT_MALLOC; + *err = index_range_next_key(db,&cur_key,db_txn); + /* *err = dbc->c_get(dbc,&cur_key,&data,DB_NEXT); */ + if (*err == DB_NOTFOUND) { + *err = 0; + break; + } + } + if (*err) LDAPDebug( LDAP_DEBUG_FILTER, " dbc->c_get(...DB_NEXT) == %i\n", *err, 0, 0); +#ifdef LDAP_DEBUG + /* this is for debugging only */ + if (idl != NULL) + { + if (ALLIDS(idl)) { + LDAPDebug( LDAP_DEBUG_FILTER, + " idl=ALLIDS\n", 0, 0, 0 ); + } else { + LDAPDebug( LDAP_DEBUG_FILTER, + " idl->b_nids=%d\n", idl->b_nids, 0, 0 ); + LDAPDebug( LDAP_DEBUG_FILTER, + " idl->b_nmax=%d\n", idl->b_nmax, 0, 0 ); + + for ( i= 0; i< idl->b_nids; i++) + { + LDAPDebug( LDAP_DEBUG_FILTER, + " idl->b_ids[%d]=%d\n", i, idl->b_ids[i], 0); + } + } + } +#endif +error: + DBT_FREE_PAYLOAD(cur_key); + DBT_FREE_PAYLOAD(upperkey); + + dblayer_release_index_file( be, ai, db ); + + LDAPDebug( LDAP_DEBUG_TRACE, "<= index_range_read(%s,%s) %lu candidates\n", + type, prefix, (u_long)IDL_NIDS(idl) ); + return( idl ); +} + +/* DBDB: this function is never actually called */ +#if 0 +static int +addordel_values( + backend *be, + DB *db, + char *type, + const char *indextype, + struct berval **vals, + ID id, + int flags, /* BE_INDEX_ADD, etc */ + back_txn *txn, + struct attrinfo *a, + int *idl_disposition, + void *buffer_handle +) +{ + int rc = 0; + int i = 0; + DBT key = {0}; + DB_TXN *db_txn = NULL; + size_t plen, vlen, len; + char *tmpbuf = NULL; + size_t tmpbuflen = 0; + char *realbuf; + char *prefix; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> %s_values\n", + (flags & BE_INDEX_ADD) ? "add" : "del", 0, 0); + + prefix = index2prefix( indextype ); + if ( vals == NULL ) { + key.dptr = prefix; + key.dsize = strlen( prefix ) + 1; /* include null terminator */ + key.flags = DB_DBT_MALLOC; + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + + if (flags & BE_INDEX_ADD) { + rc = idl_insert_key( be, db, &key, id, db_txn, a, idl_disposition ); + } else { + rc = idl_delete_key( be, db, &key, id, db_txn, a ); + /* check for no such key/id - ok in some cases */ + if ( rc == DB_NOTFOUND || rc == -666 ) { + rc = 0; + } + } + + if ( rc != 0) + { + ldbm_nasty(errmsg, 1090, rc); + } + free_prefix (prefix); + if (NULL != key.dptr && prefix != key.dptr) + slapi_ch_free( (void**)&key.dptr ); + LDAPDebug( LDAP_DEBUG_TRACE, "<= %s_values %d\n", + (flags & BE_INDEX_ADD) ? "add" : "del", rc, 0 ); + return( rc ); + } + + plen = strlen( prefix ); + for ( i = 0; vals[i] != NULL; i++ ) { + vlen = vals[i]->bv_len; + len = plen + vlen; + + if ( len < tmpbuflen ) { + realbuf = tmpbuf; + } else { + tmpbuf = slapi_ch_realloc( tmpbuf, len + 1 ); + tmpbuflen = len + 1; + realbuf = tmpbuf; + } + + memcpy( realbuf, prefix, plen ); + memcpy( realbuf+plen, vals[i]->bv_val, vlen ); + realbuf[len] = '\0'; + key.dptr = realbuf; + key.size = plen + vlen + 1; + /* should be okay to use USERMEM here because we know what + * the key is and it should never return a different value + * than the one we pass in. + */ + key.flags = DB_DBT_USERMEM; + key.ulen = tmpbuflen; +#ifdef LDAP_DEBUG + /* XXX if ( slapd_ldap_debug & LDAP_DEBUG_TRACE ) XXX */ + { + char encbuf[BUFSIZ]; + + LDAPDebug (LDAP_DEBUG_TRACE, " %s_value(\"%s\")\n", + (flags & BE_INDEX_ADD) ? "add" : "del", + encoded (&key, encbuf), 0); + } +#endif + + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + + if ( flags & BE_INDEX_ADD ) { + if (buffer_handle) { + rc = index_buffer_insert(buffer_handle,&key,id,be,db_txn,a); + if (rc == -2) { + rc = idl_insert_key( be, db, &key, id, db_txn, a, idl_disposition ); + } + } else { + rc = idl_insert_key( be, db, &key, id, db_txn, a, idl_disposition ); + } + } else { + rc = idl_delete_key( be, db, &key, id, db_txn, a ); + /* check for no such key/id - ok in some cases */ + if ( rc == DB_NOTFOUND || rc == -666 ) { + rc = 0; + } + } + if ( rc != 0 ) { + ldbm_nasty(errmsg, 1100, rc); + break; + } + if ( NULL != key.dptr && realbuf != key.dptr) { /* realloc'ed */ + tmpbuf = key.dptr; + tmpbuflen = key.size; + } + } + free_prefix (prefix); + if ( tmpbuf != NULL ) { + slapi_ch_free( (void**)&tmpbuf ); + } + + if ( rc != 0 ) + { + ldbm_nasty(errmsg, 1110, rc); + } + LDAPDebug( LDAP_DEBUG_TRACE, "<= %s_values %d\n", + (flags & BE_INDEX_ADD) ? "add" : "del", rc, 0 ); + return( rc ); +} +#endif + +static int +addordel_values_sv( + backend *be, + DB *db, + char *type, + const char *indextype, + Slapi_Value **vals, + ID id, + int flags, /* BE_INDEX_ADD, etc */ + back_txn *txn, + struct attrinfo *a, + int *idl_disposition, + void *buffer_handle +) +{ + int rc = 0; + int i = 0; + DBT key = {0}; + DB_TXN *db_txn = NULL; + size_t plen, vlen, len; + char *tmpbuf = NULL; + size_t tmpbuflen = 0; + char *realbuf; + char *prefix; + const struct berval *bvp; + struct berval *encrypted_bvp = NULL; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> %s_values\n", + (flags & BE_INDEX_ADD) ? "add" : "del", 0, 0); + + prefix = index2prefix( indextype ); + if ( vals == NULL ) { + key.dptr = prefix; + key.dsize = strlen( prefix ) + 1; /* include null terminator */ + key.flags = DB_DBT_MALLOC; + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + + if (flags & BE_INDEX_ADD) { + rc = idl_insert_key( be, db, &key, id, db_txn, a, idl_disposition ); + } else { + rc = idl_delete_key( be, db, &key, id, db_txn, a ); + /* check for no such key/id - ok in some cases */ + if ( rc == DB_NOTFOUND || rc == -666 ) { + rc = 0; + } + } + + if ( rc != 0 ) + { + ldbm_nasty(errmsg, 1120, rc); + } + free_prefix (prefix); + if (NULL != key.dptr && prefix != key.dptr) + slapi_ch_free( (void**)&key.dptr ); + LDAPDebug( LDAP_DEBUG_TRACE, "<= %s_values %d\n", + (flags & BE_INDEX_ADD) ? "add" : "del", rc, 0 ); + return( rc ); + } + + plen = strlen( prefix ); + for ( i = 0; vals[i] != NULL; i++ ) { + bvp = slapi_value_get_berval(vals[i]); + + /* Encrypt the index key if necessary */ + { + if (a->ai_attrcrypt && (0 == (flags & BE_INDEX_DONT_ENCRYPT))) + { + rc = attrcrypt_encrypt_index_key(be,a,bvp,&encrypted_bvp); + if (rc) + { + LDAPDebug (LDAP_DEBUG_ANY, "Failed to encrypt index key for %s\n", a->ai_type ,0,0); + } else { + bvp = encrypted_bvp; + } + } + } + + vlen = bvp->bv_len; + len = plen + vlen; + + if ( len < tmpbuflen ) { + realbuf = tmpbuf; + } else { + tmpbuf = slapi_ch_realloc( tmpbuf, len + 1 ); + tmpbuflen = len + 1; + realbuf = tmpbuf; + } + + memcpy( realbuf, prefix, plen ); + memcpy( realbuf+plen, bvp->bv_val, vlen ); + realbuf[len] = '\0'; + key.dptr = realbuf; + key.size = plen + vlen + 1; + /* Free the encrypted berval if necessary */ + if (encrypted_bvp) + { + ber_bvfree(encrypted_bvp); + encrypted_bvp = NULL; + } + /* should be okay to use USERMEM here because we know what + * the key is and it should never return a different value + * than the one we pass in. + */ + key.flags = DB_DBT_USERMEM; + key.ulen = tmpbuflen; +#ifdef LDAP_DEBUG + /* XXX if ( slapd_ldap_debug & LDAP_DEBUG_TRACE ) XXX */ + { + char encbuf[BUFSIZ]; + + LDAPDebug (LDAP_DEBUG_TRACE, " %s_value(\"%s\")\n", + (flags & BE_INDEX_ADD) ? "add" : "del", + encoded (&key, encbuf), 0); + } +#endif + + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } + + if ( flags & BE_INDEX_ADD ) { + if (buffer_handle) { + rc = index_buffer_insert(buffer_handle,&key,id,be,db_txn,a); + if (rc == -2) { + rc = idl_insert_key( be, db, &key, id, db_txn, a, idl_disposition ); + } + } else { + rc = idl_insert_key( be, db, &key, id, db_txn, a, idl_disposition ); + } + } else { + rc = idl_delete_key( be, db, &key, id, db_txn, a ); + /* check for no such key/id - ok in some cases */ + if ( rc == DB_NOTFOUND || rc == -666 ) { + rc = 0; + } + } + if ( rc != 0 ) { + ldbm_nasty(errmsg, 1130, rc); + break; + } + if ( NULL != key.dptr && realbuf != key.dptr) { /* realloc'ed */ + tmpbuf = key.dptr; + tmpbuflen = key.size; + } + } + free_prefix (prefix); + if ( tmpbuf != NULL ) { + slapi_ch_free( (void**)&tmpbuf ); + } + + if ( rc != 0 ) + { + ldbm_nasty(errmsg, 1140, rc); + } + LDAPDebug( LDAP_DEBUG_TRACE, "<= %s_values %d\n", + (flags & BE_INDEX_ADD) ? "add" : "del", rc, 0 ); + return( rc ); +} + +int +index_addordel_string(backend *be, const char *type, const char *s, ID id, int flags, back_txn *txn) +{ + Slapi_Value *svp[2]; + Slapi_Value sv; + + memset(&sv,0,sizeof(Slapi_Value)); + sv.bv.bv_len= strlen(s); + sv.bv.bv_val= (void*)s; + svp[0] = &sv; + svp[1] = NULL; + return index_addordel_values_ext_sv(be,type,svp,NULL,id,flags,txn,NULL,NULL); +} + +int +index_addordel_values_sv( + backend *be, + const char *type, + Slapi_Value **vals, + Slapi_Value **evals, /* existing values */ + ID id, + int flags, + back_txn *txn +) +{ + return index_addordel_values_ext_sv(be,type,vals,evals, + id,flags,txn,NULL,NULL); +} + +int +index_addordel_values_ext_sv( + backend *be, + const char *type, + Slapi_Value **vals, + Slapi_Value **evals, + ID id, + int flags, + back_txn *txn, + int *idl_disposition, + void *buffer_handle +) +{ + DB *db; + struct attrinfo *ai = NULL; + int err = -1; + Slapi_Value **ivals; + char buf[SLAPD_TYPICAL_ATTRIBUTE_NAME_MAX_LENGTH]; + char *basetmp, *basetype; + + LDAPDebug( LDAP_DEBUG_TRACE, + "=> index_addordel_values( \"%s\", %lu )\n", type, (u_long)id, 0 ); + + basetype = buf; + if ( (basetmp = slapi_attr_basetype( type, buf, sizeof(buf) )) + != NULL ) { + basetype = basetmp; + } + + ainfo_get( be, basetype, &ai ); + if ( ai == NULL || ai->ai_indexmask == 0 + || ai->ai_indexmask == INDEX_OFFLINE ) { + slapi_ch_free_string( &basetmp ); + return( 0 ); + } + LDAPDebug( LDAP_DEBUG_ARGS, " index_addordel_values indexmask 0x%x\n", + ai->ai_indexmask, 0, 0 ); + if ( (err = dblayer_get_index_file( be, ai, &db, DBOPEN_CREATE )) != 0 ) { + LDAPDebug( LDAP_DEBUG_ANY, + "<= index_read NULL (could not open index attr %s)\n", + basetype, 0, 0 ); + slapi_ch_free_string( &basetmp ); + if ( err != 0 ) { + ldbm_nasty(errmsg, 1210, err); + } + goto bad; + } + + /* + * presence index entry + */ + if (( ai->ai_indexmask & INDEX_PRESENCE ) && + (flags & (BE_INDEX_ADD|BE_INDEX_PRESENCE))) { + /* on delete, only remove the presence index if the + * BE_INDEX_PRESENCE flag is set. + */ + err = addordel_values_sv( be, db, basetype, indextype_PRESENCE, + NULL, id, flags, txn, ai, idl_disposition, NULL ); + if ( err != 0 ) { + ldbm_nasty(errmsg, 1220, err); + goto bad; + } + } + + /* + * equality index entry + */ + if ( ai->ai_indexmask & INDEX_EQUALITY ) { + slapi_call_syntax_values2keys_sv( ai->ai_plugin, vals, &ivals, + LDAP_FILTER_EQUALITY ); + + err = addordel_values_sv( be, db, basetype, indextype_EQUALITY, + ivals != NULL ? ivals : vals, id, flags, txn, ai, idl_disposition, NULL ); + if ( ivals != NULL ) { + valuearray_free( &ivals ); + } + if ( err != 0 ) { + ldbm_nasty(errmsg, 1230, err); + goto bad; + } + } + + /* + * approximate index entry + */ + if ( ai->ai_indexmask & INDEX_APPROX ) { + slapi_call_syntax_values2keys_sv( ai->ai_plugin, vals, &ivals, + LDAP_FILTER_APPROX ); + + if ( ivals != NULL ) { + err = addordel_values_sv( be, db, basetype, + indextype_APPROX, ivals, id, flags, txn, ai, idl_disposition, NULL ); + valuearray_free( &ivals ); + if ( err != 0 ) { + ldbm_nasty(errmsg, 1240, err); + goto bad; + } + } + } + + /* + * substrings index entry + */ + if ( ai->ai_indexmask & INDEX_SUB ) { + Slapi_Value **esubvals = NULL; + Slapi_Value **substresult = NULL; + Slapi_Value **origvals = NULL; + slapi_call_syntax_values2keys_sv( ai->ai_plugin, vals, &ivals, + LDAP_FILTER_SUBSTRINGS ); + + origvals = ivals; + /* delete only: if the attribute has multiple values, + * figure out the substrings that should remain + * by slapi_call_syntax_values2keys, + * then get rid of them from the being deleted values + */ + if ( evals != NULL ) { + slapi_call_syntax_values2keys_sv( ai->ai_plugin, evals, &esubvals, + LDAP_FILTER_SUBSTRINGS ); + substresult = valuearray_minus_valuearray( ai->ai_plugin, ivals, esubvals ); + ivals = substresult; + valuearray_free( &esubvals ); + } + if ( ivals != NULL ) { + err = addordel_values_sv( be, db, basetype, indextype_SUB, + ivals, id, flags, txn, ai, idl_disposition, buffer_handle ); + if ( ivals != origvals ) + valuearray_free( &origvals ); + valuearray_free( &ivals ); + if ( err != 0 ) { + ldbm_nasty(errmsg, 1250, err); + goto bad; + } + + ivals = NULL; + } + } + + /* + * matching rule index entries + */ + if ( ai->ai_indexmask & INDEX_RULES ) + { + Slapi_PBlock* pb = slapi_pblock_new(); + char** oid = ai->ai_index_rules; + for (; *oid != NULL; ++oid) + { + if(create_matchrule_indexer(&pb,*oid,basetype)==0) + { + char* officialOID = NULL; + if (!slapi_pblock_get (pb, SLAPI_PLUGIN_MR_OID, &officialOID) && officialOID != NULL) + { + Slapi_Value** keys = NULL; + matchrule_values_to_keys_sv(pb,vals,&keys); + if(keys != NULL && keys[0] != NULL) + { + /* we've computed keys */ + err = addordel_values_sv (be, db, basetype, officialOID, keys, id, flags, txn, ai, idl_disposition, NULL); + if ( err != 0 ) + { + ldbm_nasty(errmsg, 1260, err); + goto bad; + } + } + /* + * It would improve speed to save the indexer, for future use. + * But, for simplicity, we destroy it now: + */ + destroy_matchrule_indexer(pb); + } + } + } + slapi_pblock_destroy (pb); + } + + dblayer_release_index_file( be, ai, db ); + if ( basetmp != NULL ) { + slapi_ch_free( (void**)&basetmp ); + } + + LDAPDebug (LDAP_DEBUG_TRACE, "<= index_addordel_values\n", 0, 0, 0 ); + return( 0 ); + + bad: + dblayer_release_index_file(be, ai, db); + return err; +} + +int +index_delete_values( + struct ldbminfo *li, + char *type, + struct berval **vals, + ID id +) +{ + return -1; +} + +static int +is_indexed (const char* indextype, int indexmask, char** index_rules) +{ + int indexed; + if (indextype == indextype_PRESENCE) indexed = INDEX_PRESENCE & indexmask; + else if (indextype == indextype_EQUALITY) indexed = INDEX_EQUALITY & indexmask; + else if (indextype == indextype_APPROX) indexed = INDEX_APPROX & indexmask; + else if (indextype == indextype_SUB) indexed = INDEX_SUB & indexmask; + else { /* matching rule */ + indexed = 0; + if (INDEX_RULES & indexmask) { + char** rule; + for (rule = index_rules; *rule; ++rule) { + if ( ! strcmp( *rule, indextype )) { + indexed = INDEX_RULES; + break; + } + } + } + } + + /* if index is currently being generated, pretend it doesn't exist */ + if (indexmask & INDEX_OFFLINE) + indexed = 0; + + return indexed; +} + +static char* +index2prefix (const char* indextype) +{ + char* prefix; + if ( indextype == indextype_PRESENCE ) prefix = prefix_PRESENCE; + else if ( indextype == indextype_EQUALITY ) prefix = prefix_EQUALITY; + else if ( indextype == indextype_APPROX ) prefix = prefix_APPROX; + else if ( indextype == indextype_SUB ) prefix = prefix_SUB; + else { /* indextype is a matching rule name */ + const size_t len = strlen (indextype); + char* p = slapi_ch_malloc (len + 3); + p[0] = RULE_PREFIX; + memcpy( p+1, indextype, len ); + p[len+1] = ':'; + p[len+2] = '\0'; + prefix = p; + } + return( prefix ); +} + +static void +free_prefix (char* prefix) +{ + if (prefix == NULL || + prefix == prefix_PRESENCE || + prefix == prefix_EQUALITY || + prefix == prefix_APPROX || + prefix == prefix_SUB) { + /* do nothing */ + } else { + slapi_ch_free( (void**)&prefix); + } +} + +/* helper stuff for valuearray_minus_valuearray */ + +typedef struct { + value_compare_fn_type cmp_fn; + Slapi_Value *data; +} SVSORT; + +static int +svsort_cmp(const void *x, const void *y) +{ + return ((SVSORT*)x)->cmp_fn(slapi_value_get_berval(((SVSORT*)x)->data), + slapi_value_get_berval(((SVSORT*)y)->data)); +} + +static int +bvals_strcasecmp(const struct berval *a, const struct berval *b) +{ + return strcasecmp(a->bv_val, b->bv_val); +} + +/* a - b = c */ +/* the returned array of Slapi_Value needs to be freed. */ +static Slapi_Value ** +valuearray_minus_valuearray( + void *plugin, + Slapi_Value **a, + Slapi_Value **b +) +{ + int rc, i, j, k, acnt, bcnt; + SVSORT *atmp = NULL, *btmp = NULL; + Slapi_Value **c; + value_compare_fn_type cmp_fn; + + /* get berval comparison function */ + plugin_call_syntax_get_compare_fn(plugin, &cmp_fn); + if (cmp_fn == NULL) { + cmp_fn = (value_compare_fn_type)bvals_strcasecmp; + } + + /* determine length of a */ + for (acnt = 0; a[acnt] != NULL; acnt++); + + /* determine length of b */ + for (bcnt = 0; b[bcnt] != NULL; bcnt++); + + /* allocate return array as big as a */ + c = (Slapi_Value**)calloc(acnt+1, sizeof(Slapi_Value*)); + if (acnt == 0) return c; + + /* sort a */ + atmp = (SVSORT*) slapi_ch_malloc(acnt*sizeof(SVSORT)); + for (i = 0; i < acnt; i++) { + atmp[i].cmp_fn = cmp_fn; + atmp[i].data = a[i]; + } + qsort((void*)atmp, acnt, (size_t)sizeof(SVSORT), svsort_cmp); + + /* sort b */ + if (bcnt > 0) { + btmp = (SVSORT*) slapi_ch_malloc(bcnt*sizeof(SVSORT)); + for (i = 0; i < bcnt; i++) { + btmp[i].cmp_fn = cmp_fn; + btmp[i].data = b[i]; + } + qsort((void*)btmp, bcnt, (size_t)sizeof(SVSORT), svsort_cmp); + } + + /* lock step through a and b */ + for (i = 0, j = 0, k = 0; i < acnt && j < bcnt; ) { + rc = svsort_cmp(&atmp[i], &btmp[j]); + if (rc == 0) { + i++; + } else if (rc < 0) { + c[k++] = slapi_value_new_value(atmp[i++].data); + } else { + j++; + } + } + + /* copy what's left from a */ + while (i < acnt) { + c[k++] = slapi_value_new_value(atmp[i++].data); + } + + /* clean up */ + slapi_ch_free((void**)&atmp); + if (btmp) slapi_ch_free((void**)&btmp); + + return c; +} diff --git a/ldap/servers/slapd/back-ldbm/init.c b/ldap/servers/slapd/back-ldbm/init.c new file mode 100644 index 00000000..a740ce76 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/init.c @@ -0,0 +1,255 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* init.c - initialize ldbm backend */ + +#include "back-ldbm.h" +#include "../slapi-plugin.h" +#include "idlapi.h" + +static void *IDL_api[3]; + +static Slapi_PluginDesc pdesc = { "ldbm-backend", PLUGIN_MAGIC_VENDOR_STR, + PRODUCTTEXT, "high-performance LDAP backend database plugin" }; + +static int add_ldbm_internal_attr_syntax( const char *name, const char *oid, + const char *syntax, const char *mr_equality, unsigned long extraflags ); + +#ifdef _WIN32 +int *module_ldap_debug = 0; + +void +plugin_init_debug_level(int *level_ptr) +{ + module_ldap_debug = level_ptr; +} +#endif + +int +ldbm_back_init( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + int rc; + struct slapdplugin *p; + static int interface_published = 0; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> ldbm_back_init\n", 0, 0, 0 ); + + slapi_pblock_get(pb, SLAPI_PLUGIN, &p); + + /* allocate backend-specific stuff */ + li = (struct ldbminfo *) slapi_ch_calloc( 1, sizeof(struct ldbminfo) ); + + /* Record the identity of the ldbm plugin. The plugin + * identity is used during internal ops. */ + slapi_pblock_get(pb, SLAPI_PLUGIN_IDENTITY, &(li->li_identity)); + + /* keep a pointer back to the plugin */ + li->li_plugin = p; + + /* set shutdown flag to zero.*/ + li->li_shutdown = 0; + + /* Initialize the set of instances. */ + li->li_instance_set = objset_new(&ldbm_back_instance_set_destructor); + + /* initialize dblayer */ + if (dblayer_init(li)) { + LDAPDebug( LDAP_DEBUG_ANY, "ldbm_back_init: dblayer_init failed\n",0, 0, 0 ); + return (-1); + } + + /* Fill in the fields of the ldbminfo and the dblayer_private + * structures with some default values */ + ldbm_config_setup_default(li); + + /* ask the factory to give us space in the Connection object + * (only bulk import uses this) + */ + if (slapi_register_object_extension(p->plg_name, SLAPI_EXT_CONNECTION, + factory_constructor, factory_destructor, + &li->li_bulk_import_object, &li->li_bulk_import_handle) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm_back_init: " + "slapi_register_object_extension failed.\n", 0, 0, 0); + return (-1); + } + + /* add some private attributes */ + rc = add_ldbm_internal_attr_syntax( "entrydn", + LDBM_ENTRYDN_OID, DN_SYNTAX_OID, DNMATCH_NAME, + SLAPI_ATTR_FLAG_SINGLE ); + + rc = add_ldbm_internal_attr_syntax( "dncomp", + LDBM_DNCOMP_OID, DN_SYNTAX_OID, DNMATCH_NAME, + 0 ); + + rc = add_ldbm_internal_attr_syntax( "parentid", + LDBM_PARENTID_OID, DIRSTRING_SYNTAX_OID, CASEIGNOREMATCH_NAME, + SLAPI_ATTR_FLAG_SINGLE ); + + rc = add_ldbm_internal_attr_syntax( "entryid", + LDBM_ENTRYID_OID, DIRSTRING_SYNTAX_OID, CASEIGNOREMATCH_NAME, + SLAPI_ATTR_FLAG_SINGLE ); + + /* set plugin private pointer and initialize locks, etc. */ + rc = slapi_pblock_set( pb, SLAPI_PLUGIN_PRIVATE, (void *) li ); + + if ((li->li_dbcache_mutex = PR_NewLock()) == NULL ) { + LDAPDebug( LDAP_DEBUG_ANY, "ldbm_back_init: PR_NewLock failed\n", + 0, 0, 0 ); + return(-1); + } + + if ((li->li_shutdown_mutex = PR_NewLock()) == NULL ) { + LDAPDebug( LDAP_DEBUG_ANY, "ldbm_back_init: PR_NewLock failed\n", + 0, 0, 0 ); + return(-1); + } + + if ((li->li_config_mutex = PR_NewLock()) == NULL ) { + LDAPDebug( LDAP_DEBUG_ANY, "ldbm_back_init: PR_NewLock failed\n", + 0, 0, 0 ); + return(-1); + } + + if ((li->li_dbcache_cv = PR_NewCondVar( li->li_dbcache_mutex )) == NULL ) { + LDAPDebug( LDAP_DEBUG_ANY, "ldbm_back_init: PR_NewCondVar failed\n", 0, 0, 0 ); + exit(-1); + } + + /* set all of the necessary database plugin callback functions */ + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_VERSION, + (void *) SLAPI_PLUGIN_VERSION_03 ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DESCRIPTION, + (void *)&pdesc ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_BIND_FN, + (void *) ldbm_back_bind ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_UNBIND_FN, + (void *) ldbm_back_unbind ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_SEARCH_FN, + (void *) ldbm_back_search ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_NEXT_SEARCH_ENTRY_FN, + (void *) ldbm_back_next_search_entry ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_NEXT_SEARCH_ENTRY_EXT_FN, + (void *) ldbm_back_next_search_entry_ext ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_ENTRY_RELEASE_FN, + (void *) ldbm_back_entry_release ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_COMPARE_FN, + (void *) ldbm_back_compare ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_MODIFY_FN, + (void *) ldbm_back_modify ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_MODRDN_FN, + (void *) ldbm_back_modrdn ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_ADD_FN, + (void *) ldbm_back_add ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_DELETE_FN, + (void *) ldbm_back_delete ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_ABANDON_FN, + (void *) ldbm_back_abandon ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_CLOSE_FN, + (void *) ldbm_back_close ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_CLEANUP_FN, + (void *) ldbm_back_cleanup ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_FLUSH_FN, + (void *) ldbm_back_flush ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_START_FN, + (void *) ldbm_back_start ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_SEQ_FN, + (void *) ldbm_back_seq ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_RMDB_FN, + (void *) ldbm_back_rmdb ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_LDIF2DB_FN, + (void *) ldbm_back_ldif2ldbm ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_DB2LDIF_FN, + (void *) ldbm_back_ldbm2ldif ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_DB2INDEX_FN, + (void *) ldbm_back_ldbm2index ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_ARCHIVE2DB_FN, + (void *) ldbm_back_archive2ldbm ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_DB2ARCHIVE_FN, + (void *) ldbm_back_ldbm2archive ); +#if defined(UPGRADEDB) + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_UPGRADEDB_FN, + (void *) ldbm_back_upgradedb ); +#endif + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_BEGIN_FN, + (void *) dblayer_plugin_begin ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_COMMIT_FN, + (void *) dblayer_plugin_commit ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_ABORT_FN, + (void *) dblayer_plugin_abort ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_SIZE_FN, + (void *) ldbm_db_size ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_TEST_FN, + (void *) ldbm_back_db_test ); + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_INIT_INSTANCE_FN, + (void *) ldbm_back_init ); /* register itself so that the secon instance + can be initialized */ + rc |= slapi_pblock_set( pb, SLAPI_PLUGIN_DB_WIRE_IMPORT_FN, + (void *) ldbm_back_wire_import); + + if ( rc != 0 ) { + LDAPDebug( LDAP_DEBUG_ANY, "ldbm_back_init failed\n", 0, 0, 0 ); + return( -1 ); + } + + /* register the IDL interface with the API broker */ + if(!interface_published) + { + IDL_api[0] = 0; + IDL_api[1] = (void *)idl_alloc; + IDL_api[2] = (void *)idl_insert; + + if( slapi_apib_register(IDL_v1_0_GUID, IDL_api) ) + { + LDAPDebug( LDAP_DEBUG_ANY, "ldbm_back_init: failed to publish IDL interface\n", 0, 0, 0); + return( -1 ); + } + + interface_published = 1; + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= ldbm_back_init\n", 0, 0, 0 ); + + return( 0 ); +} + + +/* + * Add an attribute syntax using some default flags, etc. + * Returns an LDAP error code (LDAP_SUCCESS if all goes well) + */ +static int +add_ldbm_internal_attr_syntax( const char *name, const char *oid, + const char *syntax, const char *mr_equality, unsigned long extraflags ) +{ + int rc = LDAP_SUCCESS; + struct asyntaxinfo *asip; + char *names[2]; + char *origins[2]; + unsigned long std_flags = SLAPI_ATTR_FLAG_STD_ATTR | SLAPI_ATTR_FLAG_OPATTR + | SLAPI_ATTR_FLAG_NOUSERMOD; + + names[0] = (char *)name; + names[1] = NULL; + + origins[0] = SLAPD_VERSION_STR; + origins[1] = NULL; + + rc = attr_syntax_create( oid, names, 1, + "Netscape defined attribute type", + NULL, /* superior */ + mr_equality, NULL, NULL, /* matching rules */ + origins, syntax, + SLAPI_SYNTAXLENGTH_NONE, + std_flags | extraflags, + &asip ); + + if ( rc == LDAP_SUCCESS ) { + rc = attr_syntax_add( asip ); + } + + return rc; +} diff --git a/ldap/servers/slapd/back-ldbm/instance.c b/ldap/servers/slapd/back-ldbm/instance.c new file mode 100644 index 00000000..aa672000 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/instance.c @@ -0,0 +1,353 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +#include "back-ldbm.h" + +/* Forward declarations */ +static void ldbm_instance_destructor(void **arg); + + + +/* Creates and initializes a new ldbm_instance structure. + * Also sets up some default indexes for the new instance. + */ +int ldbm_instance_create(backend *be, char *name) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + ldbm_instance *inst; + + /* Allocate storage for the ldbm_instance structure. Information specific + * to this instance of the ldbm backend will be held here. */ + inst = (ldbm_instance *) slapi_ch_calloc(1, sizeof(ldbm_instance)); + + /* Record the name of this instance. */ + inst->inst_name = strdup(name); + + /* initialize the entry cache */ + if (! cache_init(&(inst->inst_cache), DEFAULT_CACHE_SIZE, + DEFAULT_CACHE_ENTRIES)) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm_instance_create: cache_init failed\n", + 0, 0, 0); + return -1; + } + + /* Lock for the list of open db handles */ + inst->inst_handle_list_mutex = PR_NewLock(); + if (NULL == inst->inst_handle_list_mutex) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm_instance_create: PR_NewLock failed\n", + 0, 0, 0); + return -1; + } + + /* Lock used to synchronize modify operations. */ + inst->inst_db_mutex = PR_NewLock(); + if (NULL == inst->inst_db_mutex) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm_instance_create: PR_NewLock failed\n", + 0, 0, 0); + return -1; + } + + if ((inst->inst_config_mutex = PR_NewLock()) == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm_instance_create: PR_NewLock failed\n", + 0, 0, 0); + return -1; + } + + if ((inst->inst_nextid_mutex = PR_NewLock()) == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm_instance_create: PR_NewLock failed\n", + 0, 0, 0); + return -1; + } + + if ((inst->inst_indexer_cv = PR_NewCondVar(inst->inst_nextid_mutex)) == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm_instance_create: PR_NewCondVar failed\n", 0, 0, 0 ); + return -1; + } + + inst->inst_be = be; + inst->inst_li = li; + be->be_instance_info = inst; + + /* Initialize the fields with some default values. */ + ldbm_instance_config_setup_default(inst); + + /* Add this new instance to the the set of instances */ + { + Object *instance_obj; + + instance_obj = object_new((void *) inst, &ldbm_instance_destructor); + objset_add_obj(li->li_instance_set, instance_obj); + object_release(instance_obj); + } + + return 0; +} + +/* create the default indexes separately + * (because when we're creating a new backend while the server is running, + * the DSE needs to be pre-seeded first.) + */ +int ldbm_instance_create_default_indexes(backend *be) +{ + char *argv[ 9 ]; + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; + /* write the dse file only on the final index */ + int flags = LDBM_INSTANCE_CONFIG_DONT_WRITE; + + /* + * Always index entrydn, parentid, objectclass, subordinatecount + * copiedFrom, and aci, + * since they are used by some searches, replication and the + * ACL routines. + */ + + argv[ 0 ] = "entrydn"; + argv[ 1 ] = "eq"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, flags); + + argv[ 0 ] = "parentid"; + argv[ 1 ] = "eq"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, flags); + + argv[ 0 ] = "objectclass"; + argv[ 1 ] = "eq"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, flags); + + argv[ 0 ] = "aci"; + argv[ 1 ] = "pres"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, flags); + +#if 0 /* don't need copiedfrom */ + argv[ 0 ] = "copiedfrom"; + argv[ 1 ] = "pres"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, flags); +#endif + + argv[ 0 ] = "numsubordinates"; + argv[ 1 ] = "pres"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, flags); + + argv[ 0 ] = SLAPI_ATTR_UNIQUEID; + argv[ 1 ] = "eq"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, flags); + + /* For MMR, we need this attribute (to replace use of dncomp in delete). */ + argv[ 0 ] = ATTR_NSDS5_REPLCONFLICT; + argv[ 1 ] = "eq"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, flags); + + /* write the dse file only on the final index */ + argv[ 0 ] = SLAPI_ATTR_NSCP_ENTRYDN; + argv[ 1 ] = "eq"; + argv[ 2 ] = NULL; + ldbm_instance_config_add_index_entry(inst, 2, argv, 0); + + argv[ 0 ] = LDBM_PSEUDO_ATTR_DEFAULT; + argv[ 1 ] = "none"; + argv[ 2 ] = NULL; + /* ldbm_instance_config_add_index_entry(inst, 2, argv); */ + attr_index_config( be, "ldbm index init", 0, 2, argv, 1 ); + + /* + * ancestorid is special, there is actually no such attr type + * but we still want to use the attr index file APIs. + */ + argv[ 0 ] = "ancestorid"; + argv[ 1 ] = "eq"; + argv[ 2 ] = NULL; + attr_index_config( be, "ldbm index init", 0, 2, argv, 1 ); + + return 0; +} + + +/* Starts a backend instance */ +int +ldbm_instance_start(backend *be) +{ + int rc; + PR_Lock (be->be_state_lock); + + if (be->be_state != BE_STATE_STOPPED && + be->be_state != BE_STATE_DELETED) { + LDAPDebug( LDAP_DEBUG_TRACE, + "ldbm_instance_start: warning - backend is in a wrong state - %d\n", + be->be_state, 0, 0 ); + PR_Unlock (be->be_state_lock); + return 0; + } + + rc = dblayer_instance_start(be, DBLAYER_NORMAL_MODE); + be->be_state = BE_STATE_STARTED; + + PR_Unlock (be->be_state_lock); + + return rc; +} + + +/* Stops a backend instance */ +int +ldbm_instance_stop(backend *be) +{ + int rc; + ldbm_instance *inst = (ldbm_instance *)be->be_instance_info; + + PR_Lock (be->be_state_lock); + + if (be->be_state != BE_STATE_STARTED) { + LDAPDebug( LDAP_DEBUG_ANY, + "ldbm_back_close: warning - backend %s is in the wrong state - %d\n", + inst ? inst->inst_name : "", be->be_state, 0 ); + PR_Unlock (be->be_state_lock); + return 0; + } + + rc = dblayer_instance_close(be); + + be->be_state = BE_STATE_STOPPED; + PR_Unlock (be->be_state_lock); + + cache_destroy_please(&inst->inst_cache); + + return rc; +} + + +/* Walks down the set of instances, starting each one. */ +int +ldbm_instance_startall(struct ldbminfo *li) +{ + Object *inst_obj; + ldbm_instance *inst; + int rc = 0; + + inst_obj = objset_first_obj(li->li_instance_set); + while (inst_obj != NULL) { + int rc1; + inst = (ldbm_instance *) object_get_data(inst_obj); + rc1 = ldbm_instance_start(inst->inst_be); + if (rc1 != 0) { + rc = rc1; + } else { + vlv_init(inst); + } + inst_obj = objset_next_obj(li->li_instance_set, inst_obj); + } + + return rc; +} + + +/* Walks down the set of instances, stopping each one. */ +int ldbm_instance_stopall(struct ldbminfo *li) +{ + Object *inst_obj; + ldbm_instance *inst; + + inst_obj = objset_first_obj(li->li_instance_set); + while (inst_obj != NULL) { + inst = (ldbm_instance *) object_get_data(inst_obj); + ldbm_instance_stop(inst->inst_be); + inst_obj = objset_next_obj(li->li_instance_set, inst_obj); + } + + return 0; +} + + +/* Walks down the set of instance, looking for one + * with the given name. Returns a pointer to the + * instance if found, and NULL if not found. The + * string compare on the instance name is NOT case + * sensitive. + */ +/* Currently this function doesn't bump + * the ref count of the instance returned. + */ +ldbm_instance * +ldbm_instance_find_by_name(struct ldbminfo *li, char *name) +{ + Object *inst_obj; + ldbm_instance *inst; + + inst_obj = objset_first_obj(li->li_instance_set); + while (inst_obj != NULL) { + inst = (ldbm_instance *) object_get_data(inst_obj); + if (!strcasecmp(inst->inst_name, name)) { + /* Currently we release the object here. There is no + * function for callers of this function to call to + * release the object. + */ + object_release(inst_obj); + return inst; + } + inst_obj = objset_next_obj(li->li_instance_set, inst_obj); + } + return NULL; +} + + +/* Called when all references to the instance are gone. */ +/* (ie, only when an instance is being deleted) */ +static void +ldbm_instance_destructor(void **arg) +{ + ldbm_instance *inst = (ldbm_instance *) *arg; + + LDAPDebug(LDAP_DEBUG_ANY, "Destructor for instance %s called\n", + inst->inst_name, 0, 0); + + slapi_ch_free((void **)&inst->inst_name); + PR_DestroyLock(inst->inst_config_mutex); + slapi_ch_free((void **)&inst->inst_dir_name); + PR_DestroyLock(inst->inst_db_mutex); + PR_DestroyLock(inst->inst_handle_list_mutex); + PR_DestroyLock(inst->inst_nextid_mutex); + PR_DestroyCondVar(inst->inst_indexer_cv); + attrinfo_deletetree(inst); + if (inst->inst_dataversion) { + slapi_ch_free((void **)&inst->inst_dataversion); + } + /* cache has already been destroyed */ + + slapi_ch_free((void **)&inst); +} + + +static int +ldbm_instance_comparator(Object *object, const void *name) +{ + void *data = object_get_data(object); + return (data == name) ? 0 : 1; +} + + +/* find the instance in the objset and remove it */ +int +ldbm_instance_destroy(ldbm_instance *inst) +{ + Object *object = NULL; + struct ldbminfo *li = inst->inst_li; + + object = objset_find(li->li_instance_set, ldbm_instance_comparator, inst); + if (object == NULL) { + return -1; + } + /* decref from objset_find */ + object_release(object); + + /* now remove from the instance set */ + objset_remove_obj(li->li_instance_set, object); + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_abandon.c b/ldap/servers/slapd/back-ldbm/ldbm_abandon.c new file mode 100644 index 00000000..6dd8c087 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_abandon.c @@ -0,0 +1,14 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* abandon.c - ldbm backend abandon routine */ + +#include "back-ldbm.h" + +int ldbm_back_abandon(Slapi_PBlock *pb) +{ + /* DBDB need to implement this */ + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_add.c b/ldap/servers/slapd/back-ldbm/ldbm_add.c new file mode 100644 index 00000000..1c4b8541 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_add.c @@ -0,0 +1,880 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +/* add.c - ldap ldbm back-end add routine */ + +#include "back-ldbm.h" + +extern char *numsubordinates; +extern char *hassubordinates; + +static void delete_update_entry_dn_operational_attributes(struct backentry *ep); + +/* in order to find the parent, we must have either the parent dn or uniqueid + This function will return true if either are set, or false otherwise */ +static int +have_parent_address(const Slapi_DN *parentsdn, const char *parentuniqueid) +{ + if (parentuniqueid && parentuniqueid[0]) { + return 1; /* have parent uniqueid */ + } + + if (parentsdn && !slapi_sdn_isempty(parentsdn)) { + return 1; /* have parent dn */ + } + + return 0; /* have no address */ +} + +int +ldbm_back_add( Slapi_PBlock *pb ) +{ + backend *be; + struct ldbminfo *li; + ldbm_instance *inst; + char *dn = NULL; + Slapi_Entry *e; + struct backentry *tombstoneentry = NULL; + struct backentry *addingentry = NULL; + struct backentry *parententry = NULL; + ID pid; + int isroot; + char *errbuf= NULL; + back_txn txn; + back_txnid parent_txn; + int retval = -1; + char *msg; + int managedsait; + int ldap_result_code = LDAP_SUCCESS; + char *ldap_result_message= NULL; + char *ldap_result_matcheddn= NULL; + int retry_count = 0; + int disk_full = 0; + modify_context parent_modify_c = {0}; + int parent_found = 0; + int rc; + int addingentry_id_assigned= 0; + int addingentry_in_cache= 0; + int tombstone_in_cache= 0; + Slapi_DN sdn; + Slapi_DN parentsdn; + Slapi_Operation *operation; + int dblock_acquired= 0; + int is_replicated_operation= 0; + int is_resurect_operation= 0; + int is_tombstone_operation= 0; + int is_fixup_operation= 0; + CSN *opcsn = NULL; + entry_address addr; + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_ADD_ENTRY, &e ); + slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot ); + slapi_pblock_get( pb, SLAPI_MANAGEDSAIT, &managedsait ); + slapi_pblock_get( pb, SLAPI_PARENT_TXN, (void**)&parent_txn ); + slapi_pblock_get( pb, SLAPI_OPERATION, &operation ); + slapi_pblock_get( pb, SLAPI_IS_REPLICATED_OPERATION, &is_replicated_operation ); + slapi_pblock_get( pb, SLAPI_BACKEND, &be); + + is_resurect_operation= operation_is_flag_set(operation,OP_FLAG_RESURECT_ENTRY); + is_tombstone_operation= operation_is_flag_set(operation,OP_FLAG_TOMBSTONE_ENTRY); + is_fixup_operation = operation_is_flag_set(operation,OP_FLAG_REPL_FIXUP); + + inst = (ldbm_instance *) be->be_instance_info; + + slapi_sdn_init(&sdn); + slapi_sdn_init(&parentsdn); + + /* Get rid of ldbm backend attributes that you are not allowed to specify yourself */ + slapi_entry_delete_values( e, hassubordinates, NULL ); + slapi_entry_delete_values( e, numsubordinates, NULL ); + + dblayer_txn_init(li,&txn); + + /* The dblock serializes writes to the database, + * which reduces deadlocking in the db code, + * which means that we run faster. + * + * But, this lock is re-enterant for the fixup + * operations that the URP code in the Replication + * plugin generates. + */ + if(SERIALLOCK(li) && !is_fixup_operation) + { + dblayer_lock_backend(be); + dblock_acquired= 1; + } + + rc= 0; + + /* + * We are about to pass the last abandon test, so from now on we are + * committed to finish this operation. Set status to "will complete" + * before we make our last abandon check to avoid race conditions in + * the code that processes abandon operations. + */ + if (operation) { + operation->o_status = SLAPI_OP_STATUS_WILL_COMPLETE; + } + if ( slapi_op_abandoned( pb ) ) { + goto error_return; + } + + + if (!is_tombstone_operation && !is_resurect_operation) + { + rc= slapi_setbit_int(rc,SLAPI_RTN_BIT_FETCH_EXISTING_DN_ENTRY); + } + rc= slapi_setbit_int(rc,SLAPI_RTN_BIT_FETCH_EXISTING_UNIQUEID_ENTRY); + rc= slapi_setbit_int(rc,SLAPI_RTN_BIT_FETCH_PARENT_ENTRY); + while(rc!=0) + { + /* JCM - copying entries can be expensive... should optimize */ + /* + * Some present state information is passed through the PBlock to the + * backend pre-op plugin. To ensure a consistent snapshot of this state + * we wrap the reading of the entry with the dblock. + */ + if(slapi_isbitset_int(rc,SLAPI_RTN_BIT_FETCH_EXISTING_UNIQUEID_ENTRY)) + { + /* Check if an entry with the intended uniqueid already exists. */ + done_with_pblock_entry(pb,SLAPI_ADD_EXISTING_UNIQUEID_ENTRY); /* Could be through this multiple times */ + addr.dn = NULL; + addr.uniqueid = (char*)slapi_entry_get_uniqueid(e); /* jcm - cast away const */ + ldap_result_code= get_copy_of_entry(pb, &addr, &txn, SLAPI_ADD_EXISTING_UNIQUEID_ENTRY, !is_replicated_operation); + } + if(slapi_isbitset_int(rc,SLAPI_RTN_BIT_FETCH_EXISTING_DN_ENTRY)) + { + slapi_pblock_get( pb, SLAPI_ADD_TARGET, &dn ); + slapi_sdn_set_dn_byref(&sdn, dn); + slapi_sdn_get_backend_parent(&sdn,&parentsdn,pb->pb_backend); + /* Check if an entry with the intended DN already exists. */ + done_with_pblock_entry(pb,SLAPI_ADD_EXISTING_DN_ENTRY); /* Could be through this multiple times */ + addr.dn = dn; + addr.uniqueid = NULL; + ldap_result_code= get_copy_of_entry(pb, &addr, &txn, SLAPI_ADD_EXISTING_DN_ENTRY, !is_replicated_operation); + } + /* if we can find the parent by dn or uniqueid, and the operation has requested the parent + then get it */ + if(have_parent_address(&parentsdn, operation->o_params.p.p_add.parentuniqueid) && + slapi_isbitset_int(rc,SLAPI_RTN_BIT_FETCH_PARENT_ENTRY)) + { + done_with_pblock_entry(pb,SLAPI_ADD_PARENT_ENTRY); /* Could be through this multiple times */ + addr.dn = (char*)slapi_sdn_get_ndn (&parentsdn); + addr.uniqueid = operation->o_params.p.p_add.parentuniqueid; + ldap_result_code= get_copy_of_entry(pb, &addr, &txn, SLAPI_ADD_PARENT_ENTRY, !is_replicated_operation); + /* need to set parentsdn or parentuniqueid if either is not set? */ + } + + /* Call the Backend Pre Add plugins */ + slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ldap_result_code); + rc= plugin_call_plugins(pb, SLAPI_PLUGIN_BE_PRE_ADD_FN); + if(rc==-1) + { + /* + * Plugin indicated some kind of failure, + * or that this Operation became a No-Op. + */ + slapi_pblock_get(pb, SLAPI_RESULT_CODE, &ldap_result_code); + goto error_return; + } + /* + * (rc!=-1 && rc!= 0) means that the plugin changed things, so we go around + * the loop once again to get the new present state. + */ + /* JCMREPL - Warning: A Plugin could cause an infinite loop by always returning a result code that requires some action. */ + } + + /* + * Originally (in the U-M LDAP 3.3 code), there was a comment near this + * code about a race condition. The race was that a 2nd entry could be + * added between the time when we check for an already existing entry + * and the cache_add_entry_lock() call below. A race condition no + * longer exists, because now we keep the parent entry locked for + * the duration of the old race condition's window of opportunity. + */ + + /* + * Fetch the parent entry and acquire the cache lock. + */ + if(have_parent_address(&parentsdn, operation->o_params.p.p_add.parentuniqueid)) + { + addr.dn = (char*)slapi_sdn_get_ndn (&parentsdn); + addr.uniqueid = operation->o_params.p.p_add.parentuniqueid; + parententry = find_entry2modify_only(pb,be,&addr,&txn); + if (parententry && parententry->ep_entry) { + if (!operation->o_params.p.p_add.parentuniqueid){ + /* Set the parentuniqueid now */ + operation->o_params.p.p_add.parentuniqueid = slapi_ch_strdup(slapi_entry_get_uniqueid(parententry->ep_entry)); + } + if (slapi_sdn_isempty(&parentsdn)) { + /* Set the parentsdn now */ + slapi_sdn_set_dn_byval(&parentsdn, slapi_entry_get_dn_const(parententry->ep_entry)); + } + } + modify_init(&parent_modify_c,parententry); + } + + /* Check if the entry we have been asked to add already exists */ + { + Slapi_Entry *entry; + slapi_pblock_get( pb, SLAPI_ADD_EXISTING_DN_ENTRY, &entry); + if ( entry != NULL ) + { + /* The entry already exists */ + ldap_result_code= LDAP_ALREADY_EXISTS; + goto error_return; + } + else + { + /* + * did not find the entry - this is good, since we're + * trying to add it, but we have to check whether the + * entry we did match has a referral we should return + * instead. we do this only if managedsait is not on. + */ + if ( !managedsait && !is_tombstone_operation ) + { + int err= 0; + Slapi_DN ancestordn= {0}; + struct backentry *ancestorentry; + ancestorentry= dn2ancestor(pb->pb_backend,&sdn,&ancestordn,&txn,&err); + slapi_sdn_done(&ancestordn); + if ( ancestorentry != NULL ) + { + int sentreferral= check_entry_for_referral(pb, ancestorentry->ep_entry, backentry_get_ndn(ancestorentry), "ldbm_back_add"); + cache_return( &inst->inst_cache, &ancestorentry ); + if(sentreferral) + { + ldap_result_code= -1; /* The result was sent by check_entry_for_referral */ + goto error_return; + } + } + } + } + } + + + if ((operation_is_flag_set(operation,OP_FLAG_ACTION_SCHEMA_CHECK)) && slapi_entry_schema_check(pb, e) != 0) + { + LDAPDebug(LDAP_DEBUG_TRACE, "entry failed schema check\n", 0, 0, 0); + ldap_result_code = LDAP_OBJECT_CLASS_VIOLATION; + slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &ldap_result_message); + goto error_return; + } + + opcsn = operation_get_csn (operation); + if(is_resurect_operation) + { + char *reason = NULL; + /* + * When we resurect a tombstone we must use its UniqueID + * to find the tombstone entry and lock it down in the cache. + */ + addr.dn = NULL; + addr.uniqueid = (char *)slapi_entry_get_uniqueid(e); /* jcm - cast away const */ + tombstoneentry = find_entry2modify( pb, be, &addr, NULL ); + if ( tombstoneentry==NULL ) + { + ldap_result_code= -1; + goto error_return; /* error result sent by find_entry2modify() */ + } + tombstone_in_cache = 1; + + addingentry = backentry_dup( tombstoneentry ); + if ( addingentry==NULL ) + { + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + /* + * To resurect a tombstone we must fix its DN and remove the + * parent UniqueID that we stashed in there. + * + * The entry comes back to life as a Glue entry, so we add the + * magic objectclass. + */ + slapi_pblock_get( pb, SLAPI_ADD_TARGET, &dn ); + slapi_sdn_set_dn_byref(&sdn, dn); + slapi_entry_set_dn(addingentry->ep_entry, slapi_ch_strdup(dn)); /* The DN is passed into the entry. */ + /* LPREPL: the DN is normalized...Somehow who should get a not normalized one */ + addingentry->ep_id = slapi_entry_attr_get_ulong(addingentry->ep_entry,"entryid"); + slapi_entry_attr_delete(addingentry->ep_entry, SLAPI_ATTR_VALUE_PARENT_UNIQUEID); + slapi_entry_delete_string(addingentry->ep_entry, SLAPI_ATTR_OBJECTCLASS, SLAPI_ATTR_VALUE_TOMBSTONE); + /* Now also remove the nscpEntryDN */ + if (slapi_entry_attr_delete(addingentry->ep_entry, SLAPI_ATTR_NSCP_ENTRYDN) != 0){ + LDAPDebug(LDAP_DEBUG_REPL, "Resurrection of %s - Couldn't remove %s\n", dn, SLAPI_ATTR_NSCP_ENTRYDN, 0); + } + + /* And copy the reason from e */ + reason = slapi_entry_attr_get_charptr(e, "nsds5ReplConflict"); + if (reason) { + if (!slapi_entry_attr_hasvalue(addingentry->ep_entry, "nsds5ReplConflict", reason)) { + slapi_entry_add_string(addingentry->ep_entry, "nsds5ReplConflict", reason); + LDAPDebug(LDAP_DEBUG_REPL, "Resurrection of %s - Added Conflict reason %s\n", dn, reason, 0); + } + slapi_ch_free((void **)&reason); + } + /* Clear the Tombstone Flag in the entry */ + slapi_entry_clear_flag(addingentry->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE); + + /* make sure the objectclass + - does not contain any duplicate values + - has CSNs for the new values we added + */ + { + Slapi_Attr *sa = NULL; + Slapi_Value sv; + const struct berval *svbv = NULL; + + /* add the extensibleobject objectclass with csn if not present */ + slapi_entry_attr_find(addingentry->ep_entry, SLAPI_ATTR_OBJECTCLASS, &sa); + slapi_value_init_string(&sv, "extensibleobject"); + svbv = slapi_value_get_berval(&sv); + if (slapi_attr_value_find(sa, svbv)) { /* not found, so add it */ + if (opcsn) { + value_update_csn(&sv, CSN_TYPE_VALUE_UPDATED, opcsn); + } + slapi_attr_add_value(sa, &sv); + } + value_done(&sv); + + /* add the glue objectclass with csn if not present */ + slapi_value_init_string(&sv, "glue"); + svbv = slapi_value_get_berval(&sv); + if (slapi_attr_value_find(sa, svbv)) { /* not found, so add it */ + if (opcsn) { + value_update_csn(&sv, CSN_TYPE_VALUE_UPDATED, opcsn); + } + slapi_attr_add_value(sa, &sv); + } + value_done(&sv); + } + } + else + { + /* + * Try to add the entry to the cache, assign it a new entryid + * and mark it locked. This should only fail if the entry + * already exists. + */ + /* + * next_id will add this id to the list of ids that are pending + * id2entry indexing. + */ + addingentry = backentry_init( e ); + if ( ( addingentry->ep_id = next_id( be ) ) >= MAXID ) { + LDAPDebug( LDAP_DEBUG_ANY, + "add: maximum ID reached, cannot add entry to " + "backend '%s'", be->be_name, 0, 0 ); + ldap_result_code = LDAP_OPERATIONS_ERROR; + goto error_return; + } + addingentry_id_assigned= 1; + + if (!is_fixup_operation) + { + if ( opcsn == NULL && operation->o_csngen_handler ) + { + /* + * Current op is a user request. Opcsn will be assigned + * if the dn is in an updatable replica. + */ + opcsn = entry_assign_operation_csn ( pb, e, parententry ? parententry->ep_entry : NULL ); + } + if ( opcsn != NULL ) + { + entry_set_csn (e, opcsn); + entry_add_dncsn (e, opcsn); + entry_add_rdn_csn (e, opcsn); + entry_set_maxcsn (e, opcsn); + } + } + + if (is_tombstone_operation) + { + /* Directly add the entry as a tombstone */ + /* + * 1) If the entry has an existing DN, change it to be + * "nsuniqueid=<uniqueid>, <old dn>" + * 2) Add the objectclass value "tombstone" and arrange for only + * that value to be indexed. + * 3) If the parent entry was found, set the nsparentuniqueid + * attribute to be the unique id of that parent. + */ + char *untombstoned_dn = slapi_entry_get_dn(e); + if (NULL == untombstoned_dn) + { + untombstoned_dn = ""; + } + slapi_entry_set_dn(addingentry->ep_entry, compute_entry_tombstone_dn(untombstoned_dn, addr.uniqueid)); + /* Work around pb with slapi_entry_add_string (defect 522327) doesn't check duplicate values */ + if (!slapi_entry_attr_hasvalue(addingentry->ep_entry, SLAPI_ATTR_OBJECTCLASS, SLAPI_ATTR_VALUE_TOMBSTONE)) { + slapi_entry_add_string(addingentry->ep_entry, SLAPI_ATTR_OBJECTCLASS, SLAPI_ATTR_VALUE_TOMBSTONE); + slapi_entry_set_flag(addingentry->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE); + } + if (NULL != operation->o_params.p.p_add.parentuniqueid) + { + slapi_entry_add_string(addingentry->ep_entry, SLAPI_ATTR_VALUE_PARENT_UNIQUEID, operation->o_params.p.p_add.parentuniqueid); + } + } + if ( cache_add_tentative( &inst->inst_cache, addingentry, NULL )!= 0 ) + { + LDAPDebug( LDAP_DEBUG_CACHE, "cache_add_tentative concurrency detected\n", 0, 0, 0 ); + ldap_result_code= LDAP_ALREADY_EXISTS; + goto error_return; + } + addingentry_in_cache= 1; + } + + /* + * Get the parent dn and see if the corresponding entry exists. + * If the parent does not exist, only allow the "root" user to + * add the entry. + */ + if ( !slapi_sdn_isempty(&parentsdn) ) + { + /* This is getting the parent */ + if (NULL == parententry) + { + /* Here means that we didn't find the parent */ + int err = 0; + Slapi_DN ancestordn= {0}; + struct backentry *ancestorentry; + + LDAPDebug( LDAP_DEBUG_TRACE, + "parent does not exist, pdn = %s\n", + slapi_sdn_get_dn(&parentsdn), 0, 0 ); + + ancestorentry = dn2ancestor(be, &parentsdn, &ancestordn, &txn, &err ); + cache_return( &inst->inst_cache, &ancestorentry ); + + ldap_result_code= LDAP_NO_SUCH_OBJECT; + ldap_result_matcheddn= slapi_ch_strdup((char *)slapi_sdn_get_dn(&ancestordn)); /* jcm - cast away const. */ + slapi_sdn_done(&ancestordn); + goto error_return; + } + ldap_result_code = plugin_call_acl_plugin (pb, e, NULL, NULL, SLAPI_ACL_ADD, + ACLPLUGIN_ACCESS_DEFAULT, &errbuf ); + if ( ldap_result_code != LDAP_SUCCESS ) + { + LDAPDebug( LDAP_DEBUG_TRACE, "no access to parent\n", 0, 0, 0 ); + ldap_result_message= errbuf; + goto error_return; + } + pid = parententry->ep_id; + } + else + { /* no parent */ + if ( !isroot && !is_replicated_operation) + { + LDAPDebug( LDAP_DEBUG_TRACE, "no parent & not root\n", + 0, 0, 0 ); + ldap_result_code= LDAP_INSUFFICIENT_ACCESS; + goto error_return; + } + parententry = NULL; + pid = 0; + } + + if(is_resurect_operation) + { + /* + * add the entrydn operational attributes + */ + add_update_entrydn_operational_attributes(addingentry); + } + else if (is_tombstone_operation) + { + /* Remove the entrydn operational attributes */ + delete_update_entry_dn_operational_attributes(addingentry); + } + else + { + /* + * add the parentid, entryid and entrydn operational attributes + */ + add_update_entry_operational_attributes(addingentry, pid); + } + + /* + * Before we add the entry, find out if the syntax of the aci + * aci attribute values are correct or not. We don't want to + * the entry if the syntax is incorrect. + */ + if ( plugin_call_acl_verify_syntax (pb, addingentry->ep_entry, &errbuf) != 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, "ACL syntax error\n", 0,0,0); + ldap_result_code= LDAP_INVALID_SYNTAX; + ldap_result_message= errbuf; + goto error_return; + } + + /* Having decided that we're really going to do the operation, let's modify + the in-memory state of the parent to reflect the new child (update + subordinate count specifically */ + if (NULL != parententry) + { + retval = parent_update_on_childchange(&parent_modify_c,1,NULL); /* 1==add */\ + /* The modify context now contains info needed later */ + if (0 != retval) { + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + parent_found = 1; + parententry = NULL; + } + /* + * So, we believe that no code up till here actually added anything + * to persistent store. From now on, we're transacted + */ + + for (retry_count = 0; retry_count < RETRY_TIMES; retry_count++) { + if (retry_count > 0) { + dblayer_txn_abort(li,&txn); + /* We're re-trying */ + LDAPDebug( LDAP_DEBUG_TRACE, "Add Retrying Transaction\n", 0, 0, 0 ); +#ifndef LDBM_NO_BACKOFF_DELAY + { + PRIntervalTime interval; + interval = PR_MillisecondsToInterval(slapi_rand() % 100); + DS_Sleep(interval); + } +#endif + } + retval = dblayer_txn_begin(li,parent_txn,&txn); + if (0 != retval) { + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + retval = id2entry_add( be, addingentry, &txn ); + if (DB_LOCK_DEADLOCK == retval) + { + LDAPDebug( LDAP_DEBUG_ARGS, "add 1 DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (retval != 0) { + LDAPDebug( LDAP_DEBUG_TRACE, "id2entry_add failed, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + if(is_resurect_operation) + { + retval = index_addordel_string(be,SLAPI_ATTR_OBJECTCLASS,SLAPI_ATTR_VALUE_TOMBSTONE,addingentry->ep_id,BE_INDEX_DEL,&txn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "add 2 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "add 1 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + retval = index_addordel_string(be,SLAPI_ATTR_UNIQUEID,slapi_entry_get_uniqueid(addingentry->ep_entry),addingentry->ep_id,BE_INDEX_DEL,&txn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "add 3 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "add 2 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + retval = index_addordel_string(be,SLAPI_ATTR_NSCP_ENTRYDN,slapi_sdn_get_ndn(&sdn),addingentry->ep_id,BE_INDEX_DEL,&txn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "add 4 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "add 3 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + } + if (is_tombstone_operation) + { + retval = index_addordel_entry( be, addingentry, BE_INDEX_ADD | BE_INDEX_TOMBSTONE, &txn ); + } + else + { + retval = index_addordel_entry( be, addingentry, BE_INDEX_ADD, &txn ); + } + if (DB_LOCK_DEADLOCK == retval) + { + LDAPDebug( LDAP_DEBUG_ARGS, "add 5 DEADLOCK\n", 0, 0, 0 ); + /* retry txn */ + continue; + } + if (retval != 0) { + LDAPDebug( LDAP_DEBUG_ANY, "add: attempt to index %lu failed\n", + (u_long)addingentry->ep_id, 0, 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + if (parent_found) { + /* Push out the db modifications from the parent entry */ + retval = modify_update_all(be,pb,&parent_modify_c,&txn); + if (DB_LOCK_DEADLOCK == retval) + { + LDAPDebug( LDAP_DEBUG_ARGS, "add 4 DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "add 1 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + } + /* + * Update the Virtual List View indexes + */ + retval= vlv_update_all_indexes(&txn, be, pb, NULL, addingentry); + if (DB_LOCK_DEADLOCK == retval) + { + LDAPDebug( LDAP_DEBUG_ARGS, "add DEADLOCK vlv_update_index\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "vlv_update_index failed, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + if (retval == 0 ) { + break; + } + + } + if (retry_count == RETRY_TIMES) { + /* Failed */ + LDAPDebug( LDAP_DEBUG_ANY, "Retry count exceeded in add\n", 0, 0, 0 ); + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + /* + * At this point, everything's cool, and the only thing which + * can go wrong is a transaction commit failure. + */ + slapi_pblock_set( pb, SLAPI_ENTRY_PRE_OP, NULL ); + slapi_pblock_set( pb, SLAPI_ENTRY_POST_OP, slapi_entry_dup( addingentry->ep_entry )); + + if(is_resurect_operation) + { + /* + * We can now switch the tombstone entry with the real entry. + */ + if (cache_replace( &inst->inst_cache, tombstoneentry, addingentry ) != 0 ) + { + /* This happens if the dn of addingentry already exists */ + ldap_result_code= LDAP_ALREADY_EXISTS; + cache_unlock_entry( &inst->inst_cache, tombstoneentry ); + goto error_return; + } + /* + * The tombstone was locked down in the cache... we can + * get rid of the entry in the cache now. + */ + cache_unlock_entry( &inst->inst_cache, tombstoneentry ); + cache_return( &inst->inst_cache, &tombstoneentry ); + tombstone_in_cache = 0; /* deleted */ + } + if (parent_found) + { + /* switch the parent entry copy into play */ + modify_switch_entries( &parent_modify_c,be); + } + + retval = dblayer_txn_commit(li,&txn); + if (0 != retval) + { + if (LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto diskfull_return; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + rc= 0; + goto common_return; + +error_return: + if ( addingentry_id_assigned ) + { + next_id_return( be, addingentry->ep_id ); + } + if ( NULL != addingentry ) + { + if ( addingentry_in_cache ) + { + cache_remove(&inst->inst_cache, addingentry); + } + backentry_clear_entry(addingentry); /* e is released in the frontend */ + backentry_free( &addingentry ); /* release the backend wrapper, here */ + } + if(tombstone_in_cache) + { + cache_return(&inst->inst_cache, &tombstoneentry); + } + + if (rc == DB_RUNRECOVERY) { + dblayer_remember_disk_filled(li); + ldbm_nasty("Add",80,rc); + disk_full = 1; + } + + /* It is specifically OK to make this call even when no transaction was in progress */ + dblayer_txn_abort(li,&txn); /* abort crashes in case disk full */ +diskfull_return: + + if (disk_full) + rc= return_on_disk_full(li); + else + rc= SLAPI_FAIL_GENERAL; + +common_return: + + if (addingentry_in_cache) + { + cache_return( &inst->inst_cache, &addingentry ); + } + /* JCMREPL - The bepostop is called even if the operation fails. */ + plugin_call_plugins (pb, SLAPI_PLUGIN_BE_POST_ADD_FN); + + modify_term(&parent_modify_c,be); + done_with_pblock_entry(pb,SLAPI_ADD_EXISTING_DN_ENTRY); + done_with_pblock_entry(pb,SLAPI_ADD_EXISTING_UNIQUEID_ENTRY); + done_with_pblock_entry(pb,SLAPI_ADD_PARENT_ENTRY); + if(dblock_acquired) + { + dblayer_unlock_backend(be); + } + if(ldap_result_code!=-1) + { + slapi_send_ldap_result( pb, ldap_result_code, ldap_result_matcheddn, ldap_result_message, 0, NULL ); + } + slapi_sdn_done(&sdn); + slapi_sdn_done(&parentsdn); + slapi_ch_free( (void**)&ldap_result_matcheddn ); + slapi_ch_free( (void**)&errbuf ); + return rc; +} + +/* + * add the parentid, entryid and entrydn, operational attributes. + * + * Note: This is called from the ldif2ldbm code. + */ +void +add_update_entry_operational_attributes(struct backentry *ep, ID pid) +{ + struct berval bv; + struct berval *bvp[2]; + char buf[40]; /* Enough for an EntryID */ + + bvp[0] = &bv; + bvp[1] = NULL; + + /* parentid */ + /* If the pid is 0, then the entry does not have a parent. It + * may be the case that the entry is a suffix. In any case, + * the parentid attribute should only be added if the entry + * has a parent. */ + if (pid != 0) { + sprintf( buf, "%lu", (u_long)pid ); + bv.bv_val = buf; + bv.bv_len = strlen( buf ); + entry_replace_values( ep->ep_entry, "parentid", bvp ); + } + + /* entryid */ + sprintf( buf, "%lu", (u_long)ep->ep_id ); + bv.bv_val = buf; + bv.bv_len = strlen( buf ); + entry_replace_values( ep->ep_entry, "entryid", bvp ); + + /* entrydn */ + add_update_entrydn_operational_attributes(ep); +} + +/* + * add the entrydn operational attribute. + */ +void +add_update_entrydn_operational_attributes(struct backentry *ep) +{ + struct berval bv; + struct berval *bvp[2]; + + /* entrydn */ + bvp[0] = &bv; + bvp[1] = NULL; + bv.bv_val = (void*)backentry_get_ndn(ep); + bv.bv_len = strlen( bv.bv_val ); + entry_replace_values( ep->ep_entry, "entrydn", bvp ); +} + +/* + * delete the entrydn operational attributes + */ +static void +delete_update_entry_dn_operational_attributes(struct backentry *ep) +{ + slapi_entry_attr_delete( ep->ep_entry, "entrydn"); +} + diff --git a/ldap/servers/slapd/back-ldbm/ldbm_attr.c b/ldap/servers/slapd/back-ldbm/ldbm_attr.c new file mode 100644 index 00000000..246a8d7d --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_attr.c @@ -0,0 +1,635 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* attr.c - backend routines for dealing with attributes */ + +#include "back-ldbm.h" + +extern char **str2charray(); + +struct attrinfo * +attrinfo_new() +{ + struct attrinfo *p= (struct attrinfo *)slapi_ch_calloc(1, sizeof(struct attrinfo)); + p->ai_type= 0; + p->ai_indexmask= 0; + p->ai_plugin= NULL; + p->ai_index_rules= NULL; + p->ai_dblayer= NULL; + p->ai_dblayer_count = 0; + p->ai_idl= NULL; + return p; +} + +void +attrinfo_delete(struct attrinfo **pp) +{ + if(pp!=NULL && *pp!=NULL) + { + idl_release_private(*pp); + slapi_ch_free((void**)&((*pp)->ai_type)); + slapi_ch_free((void**)(*pp)->ai_index_rules); + slapi_ch_free((void**)pp); + *pp= NULL; + } +} + +static int +attrinfo_internal_delete( caddr_t data, caddr_t arg ) +{ + struct attrinfo *n = (struct attrinfo *)data; + attrinfo_delete(&n); + return 0; +} + +void +attrinfo_deletetree(ldbm_instance *inst) +{ + avl_free( inst->inst_attrs, attrinfo_internal_delete ); +} + + +static int +ainfo_type_cmp( + char *type, + struct attrinfo *a +) +{ + return( strcasecmp( type, a->ai_type ) ); +} + +static int +ainfo_cmp( + struct attrinfo *a, + struct attrinfo *b +) +{ + return( strcasecmp( a->ai_type, b->ai_type ) ); +} + +/* + * Called when a duplicate "index" line is encountered. + * + * returns 1 => original from init code, indexmask updated + * 2 => original not from init code, warn the user + * + * Hard coded to return a 1 always... + * + */ + +static int +ainfo_dup( + struct attrinfo *a, + struct attrinfo *b +) +{ + /* merge duplicate indexing information */ + if (b->ai_indexmask == 0 || b->ai_indexmask == INDEX_OFFLINE) { + a->ai_indexmask = INDEX_OFFLINE; /* turns off all indexes */ + charray_free ( a->ai_index_rules ); + a->ai_index_rules = NULL; + } + a->ai_indexmask |= b->ai_indexmask; + if ( b->ai_indexmask & INDEX_RULES ) { + charray_merge( &a->ai_index_rules, b->ai_index_rules, 1 ); + } + + return( 1 ); +} + +void +ainfo_get( + backend *be, + char *type, + struct attrinfo **at +) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + if ( (*at = (struct attrinfo *) avl_find( inst->inst_attrs, type, + ainfo_type_cmp )) == NULL ) { + if ( (*at = (struct attrinfo *) avl_find( inst->inst_attrs, + LDBM_PSEUDO_ATTR_DEFAULT, ainfo_type_cmp )) == NULL ) { + return; + } + } +} + +void +attr_index_config( + backend *be, + char *fname, + int lineno, + int argc, + char **argv, + int init +) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int i, j; + char **attrs; + char **indexes = NULL; + char **index_rules = NULL; + struct attrinfo *a; + int return_value = -1; + + attrs = str2charray( argv[0], "," ); + if ( argc > 1 ) { + indexes = str2charray( argv[1], "," ); + if ( argc > 2 ) { + index_rules = str2charray( argv[2], "," ); + } + } + for ( i = 0; attrs[i] != NULL; i++ ) { + a = attrinfo_new(); + a->ai_type = slapi_attr_basetype( attrs[i], NULL, 0 ); + slapi_attr_type2plugin( a->ai_type, &a->ai_plugin ); + if ( argc == 1 ) { + a->ai_indexmask = (INDEX_PRESENCE | INDEX_EQUALITY | + INDEX_APPROX | INDEX_SUB); + } else { + a->ai_indexmask = 0; + for ( j = 0; indexes[j] != NULL; j++ ) { + if ( strncasecmp( indexes[j], "pres", 4 ) + == 0 ) { + a->ai_indexmask |= INDEX_PRESENCE; + } else if ( strncasecmp( indexes[j], "eq", 2 ) + == 0 ) { + a->ai_indexmask |= INDEX_EQUALITY; + } else if ( strncasecmp( indexes[j], "approx", + 6 ) == 0 ) { + a->ai_indexmask |= INDEX_APPROX; + } else if ( strncasecmp( indexes[j], "sub", 3 ) + == 0 ) { + a->ai_indexmask |= INDEX_SUB; + } else if ( strncasecmp( indexes[j], "none", 4 ) + == 0 ) { + if ( a->ai_indexmask != 0 ) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: line %d: index type \"none\" cannot be combined with other types\n", + fname, lineno, 0); + } + a->ai_indexmask = INDEX_OFFLINE; /* note that the index isn't available */ + } else { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: line %d: unknown index type \"%s\" (ignored)\n", + fname, lineno, indexes[j]); + LDAPDebug(LDAP_DEBUG_ANY, + "valid index types are \"pres\", \"eq\", \"approx\", or \"sub\"\n", + 0, 0, 0); + } + } + + /* compute a->ai_index_rules: */ + j = 0; + if (index_rules != NULL) for (; index_rules[j] != NULL; ++j); + if (j > 0) { /* there are some candidates */ + char** official_rules = (char**) + slapi_ch_malloc ((j + 1) * sizeof (char*)); + size_t k = 0; + for (j = 0; index_rules[j] != NULL; ++j) { + /* Check that index_rules[j] is an official OID */ + char* officialOID = NULL; + IFP mrINDEX = NULL; + Slapi_PBlock* pb = slapi_pblock_new(); + if (!slapi_pblock_set (pb, SLAPI_PLUGIN_MR_OID, index_rules[j]) && + !slapi_pblock_set (pb, SLAPI_PLUGIN_MR_TYPE, a->ai_type) && + !slapi_mr_indexer_create (pb) && + !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX) && + mrINDEX != NULL && + !slapi_pblock_get (pb, SLAPI_PLUGIN_MR_OID, &officialOID) && + officialOID != NULL) { + if (!strcasecmp (index_rules[j], officialOID)) { + official_rules[k++] = slapi_ch_strdup (officialOID); + } else { + char* preamble = slapi_ch_malloc (strlen (fname) + 30); + sprintf (preamble, "%s: line %d", fname, lineno); + LDAPDebug (LDAP_DEBUG_ANY, "%s: use \"%s\" instead of \"%s\" (ignored)\n", + preamble, officialOID, index_rules[j] ); + slapi_ch_free((void**)&preamble); + } + } else { + LDAPDebug (LDAP_DEBUG_ANY, "%s: line %d: " + "unknown index rule \"%s\" (ignored)\n", + fname, lineno, index_rules[j] ); + } + {/* It would improve speed to save the indexer, for future use. + But, for simplicity, we destroy it now: */ + IFP mrDESTROY = NULL; + if (!slapi_pblock_get (pb, SLAPI_PLUGIN_DESTROY_FN, &mrDESTROY) && + mrDESTROY != NULL) { + mrDESTROY (pb); + } + } + slapi_pblock_destroy (pb); + } + official_rules[k] = NULL; + if (k > 0) { + a->ai_index_rules = official_rules; + a->ai_indexmask |= INDEX_RULES; + } else { + slapi_ch_free((void**)&official_rules); + } + } + } +#if 0 /* seems to not matter -- INDEX_FROMINIT is checked nowhere else */ + if ( init ) { + a->ai_indexmask |= INDEX_FROMINIT; + a->ai_indexmask &= ~INDEX_OFFLINE; + } +#endif + + /* initialize the IDL code's private data */ + return_value = idl_init_private(be, a); + if (0 != return_value) { + /* fatal error, exit */ + LDAPDebug(LDAP_DEBUG_ANY,"%s: line %d:Fatal Error: Failed to initialize attribute structure\n", + fname, lineno, 0); + exit( 1 ); + } + + if ( avl_insert( &inst->inst_attrs, a, ainfo_cmp, ainfo_dup ) != 0 ) { + /* duplicate - existing version updated */ + attrinfo_delete(&a); + } + } + charray_free( attrs ); + if ( indexes != NULL ) { + charray_free( indexes ); + } + if ( index_rules != NULL ) { + charray_free( index_rules ); + } +} + +/* + * Function that creates a new attrinfo structure and + * inserts it into the avl tree. This is used by code + * that wants to store attribute-level configuration data + * e.g. attribute encryption, but where the attr_info + * structure doesn't exist because the attribute in question + * is not indexed. + */ +void +attr_create_empty(backend *be,char *type,struct attrinfo **ai) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + struct attrinfo *a = attrinfo_new(); + a->ai_type = slapi_ch_strdup(type); + if ( avl_insert( &inst->inst_attrs, a, ainfo_cmp, ainfo_dup ) != 0 ) { + /* duplicate - existing version updated */ + attrinfo_delete(&a); + ainfo_get(be,type,&a); + } + *ai = a; +} + +/* Code for computed attributes */ +extern char* hassubordinates; +extern char* numsubordinates; + +static int +ldbm_compute_evaluator(computed_attr_context *c,char* type,Slapi_Entry *e,slapi_compute_output_t outputfn) +{ + int rc = 0; + + if ( strcasecmp (type, numsubordinates ) == 0) + { + Slapi_Attr *read_attr = NULL; + /* Check to see whether this attribute is already present in the entry */ + if (0 != slapi_entry_attr_find( e, numsubordinates, &read_attr )) + { + /* If not, we return it as zero */ + Slapi_Attr our_attr; + slapi_attr_init(&our_attr, numsubordinates); + our_attr.a_flags = SLAPI_ATTR_FLAG_OPATTR; + valueset_add_string(&our_attr.a_present_values,"0",CSN_TYPE_UNKNOWN,NULL); + rc = (*outputfn) (c, &our_attr, e); + attr_done(&our_attr); + return (rc); + } + } + if ( strcasecmp (type, hassubordinates ) == 0) + { + Slapi_Attr *read_attr = NULL; + Slapi_Attr our_attr; + slapi_attr_init(&our_attr, hassubordinates); + our_attr.a_flags = SLAPI_ATTR_FLAG_OPATTR; + /* This attribute is always computed */ + /* Check to see whether the subordinate count attribute is already present in the entry */ + rc = slapi_entry_attr_find( e, numsubordinates, &read_attr ); + if ( (0 != rc) || slapi_entry_attr_hasvalue(e,numsubordinates,"0") ) { + /* If not, or present and zero, we return FALSE, otherwise TRUE */ + valueset_add_string(&our_attr.a_present_values,"FALSE",CSN_TYPE_UNKNOWN,NULL); + } else { + valueset_add_string(&our_attr.a_present_values,"TRUE",CSN_TYPE_UNKNOWN,NULL); + } + rc = (*outputfn) (c, &our_attr, e); + attr_done(&our_attr); + return (rc); + } + + return -1; /* I see no ships */ +} + +/* + * string_find(): case sensitive search for the substring str2 within str1. + */ +static +char * string_find ( + const char * str1, + const char * str2 + ) +{ + char *cp = (char *) str1; + char *s1, *s2; + + if ( !*str2 ) + return((char *)str1); + + while (*cp) + { + s1 = cp; + s2 = (char *) str2; + + while ( *s1 && *s2 && !(*s1-*s2) ) + s1++, s2++; + + if (!*s2) + return(cp); + + cp++; + } + + return(NULL); + +} + +/* What are we doing ? + The back-end can't search properly for the hasSubordinates and + numSubordinates attributes. The reason being that they're not + always stored on entries, so filter test fails to do the correct thing. + However, it is possible to rewrite a given search to one + which will work, given that numSubordinates is present when non-zero, + and we maintain a presence index for numSubordinates. + */ +/* Searches we rewrite here : + substrings of the form + (hassubordinates=TRUE) to (&(numsubordinates=*)(numsubordinates>=1)) [indexed] + (hassubordinates=FALSE) to (&(objectclass=*)(!(numsubordinates=*))) [not indexed] + (hassubordinates=*) to (objectclass=*) [not indexed] + (numsubordinates=*) to (objectclass=*) [not indexed] + (numsubordinates=x) to (&(numsubordinates=*)(numsubordinates=x)) [indexed] + (numsubordinates>=x) to (&(numsubordinates=*)(numsubordinates>=x)) [indexed where X > 0] + (numsubordinates<=x) to (&(numsubordinates=*)(numsubordinates<=x)) [indexed] + + anything else involving numsubordinates and hassubordinates we flag as unwilling to perform + +*/ + +/* Before calling this function, you must free all the parts + which will be overwritten, this function dosn't know + how to do that */ +static int replace_filter(Slapi_Filter *f, char *s) +{ + Slapi_Filter *newf = NULL; + Slapi_Filter *temp = NULL; +/* LP: Fix for defect 515161. Crash on AIX + * slapi_str2filter is a nasty function that mangle whatever gets passed in. + * AIX crashes on altering the literal string. + * So we need to allocate the string and then free it. + */ + char *buf = slapi_ch_strdup(s); + + newf = slapi_str2filter(buf); + slapi_ch_free((void **)&buf); + + if (NULL == newf) { + return -1; + } + + /* Now take the parts of newf and put them in f */ + /* An easy way to do this is to preserve the "next" ptr */ + temp = f->f_next; + *f = *newf; + f->f_next = temp; + /* Free the new filter husk */ + slapi_ch_free((void**)&newf); + return 0; +} + +static void find_our_friends(char *s, int *has, int *num) +{ + *has = (0 == strcasecmp(s,"hassubordinates")); + if (!(*has)) { + *num = (0 == strcasecmp(s,"numsubordinates")); + } +} + +/* Free the parts of a filter we're about to overwrite */ +void free_the_filter_bits(Slapi_Filter *f) +{ + /* We need to free: */ + switch ( f->f_choice ) { + case LDAP_FILTER_EQUALITY: + case LDAP_FILTER_GE: + case LDAP_FILTER_LE: + case LDAP_FILTER_APPROX: + ava_done( &f->f_ava ); + break; + + case LDAP_FILTER_PRESENT: + if ( f->f_type != NULL ) { + slapi_ch_free( (void**)&(f->f_type) ); + } + break; + + default: + break; + } +} + +static int grok_and_rewrite_filter(Slapi_Filter *f) +{ + Slapi_Filter *p = NULL; + int has = 0; + int num = 0; + char *rhs = NULL; + struct berval rhs_berval; + + switch ( f->f_choice ) { + case LDAP_FILTER_EQUALITY: + /* Does this involve either of our target attributes ? */ + find_our_friends(f->f_ava.ava_type,&has,&num); + if (has || num) { + rhs = f->f_ava.ava_value.bv_val; + if (has) { + if (0 == strcasecmp(rhs,"TRUE")) { + free_the_filter_bits(f); + replace_filter(f,"(&(numsubordinates=*)(numsubordinates>=1))"); + } else if (0 == strcasecmp(rhs, "FALSE")) { + free_the_filter_bits(f); + replace_filter(f,"(&(objectclass=*)(!(numsubordinates=*)))"); + } else { + return 1; /* Filter we can't rewrite */ + } + } + if (num) { + int rhs_number = 0; + + rhs_number = atoi(rhs); + if (rhs_number > 0) { + + char * theType=f->f_ava.ava_type; + rhs_berval = f->f_ava.ava_value; + replace_filter(f,"(&(numsubordinates=*)(numsubordinates=x))"); + /* Now fixup the resulting filter so that x = rhs */ + slapi_ch_free((void**)&(f->f_and->f_next->f_ava.ava_value.bv_val)); + /*free type also */ + slapi_ch_free((void**)&theType); + + f->f_and->f_next->f_ava.ava_value = rhs_berval; + } else { + if (rhs_number == 0) { + /* This is the same as hassubordinates=FALSE */ + free_the_filter_bits(f); + replace_filter(f,"(&(objectclass=*)(!(numsubordinates=*)))"); + } else { + return 1; + } + } + } + return 0; + } + break; + + case LDAP_FILTER_GE: + find_our_friends(f->f_ava.ava_type,&has,&num); + if (has) { + return 1; /* Makes little sense for this attribute */ + } + if (num) { + int rhs_num = 0; + rhs = f->f_ava.ava_value.bv_val; + /* is the value zero ? */ + rhs_num = atoi(rhs); + if (0 == rhs) { + /* If so, rewrite to same as numsubordinates=* */ + free_the_filter_bits(f); + replace_filter(f,"(objectclass=*)"); + } else { + /* Rewrite to present and GE the rhs */ + char * theType=f->f_ava.ava_type; + rhs_berval = f->f_ava.ava_value; + + replace_filter(f,"(&(numsubordinates=*)(numsubordinates>=x))"); + /* Now fixup the resulting filter so that x = rhs */ + slapi_ch_free((void**)&(f->f_and->f_next->f_ava.ava_value.bv_val)); + /*free type also */ + slapi_ch_free((void**)&theType); + + f->f_and->f_next->f_ava.ava_value = rhs_berval; + } + return 0; + } + break; + + case LDAP_FILTER_LE: + find_our_friends(f->f_ava.ava_type,&has,&num); + if (has) { + return 1; /* Makes little sense for this attribute */ + } + if (num) { + /* One could imagine doing this one, but it's quite hard */ + return 1; + } + break; + + case LDAP_FILTER_APPROX: + find_our_friends(f->f_ava.ava_type,&has,&num); + if (has || num) { + /* Not allowed */ + return 1; + } + break; + + case LDAP_FILTER_SUBSTRINGS: + find_our_friends(f->f_sub_type,&has,&num); + if (has || num) { + /* Not allowed */ + return 1; + } + break; + + case LDAP_FILTER_PRESENT: + find_our_friends(f->f_type,&has,&num); + if (has || num) { + /* we rewrite this search to (objectclass=*) */ + slapi_ch_free((void**)&(f->f_type)); + f->f_type = slapi_ch_strdup("objectclass"); + return 0; + } /* We already weeded out the special search we use use in the console */ + break; + case LDAP_FILTER_AND: + case LDAP_FILTER_OR: + case LDAP_FILTER_NOT: + for ( p = f->f_list; p != NULL; p = p->f_next ) { + grok_and_rewrite_filter( p ); + } + break; + + default: + return -1; /* Bad, might be an extended filter or something */ + } + return -1; +} + +static int +ldbm_compute_rewriter(Slapi_PBlock *pb) +{ + int rc = -1; + char *fstr= NULL; + + /* + * We need to look at the filter and see whether it might contain + * numSubordinates or hasSubordinates. We want to do a quick check + * before we look thoroughly. + */ + slapi_pblock_get( pb, SLAPI_SEARCH_STRFILTER, &fstr ); + + if ( NULL != fstr ) { + char *lc_fstr = (char *)slapi_utf8StrToLower( (unsigned char *)fstr ); + + if (string_find(lc_fstr,"subordinates")) { + Slapi_Filter *f = NULL; + /* Look for special filters we want to leave alone */ + if (0 == strcmp(lc_fstr, "(&(numsubordinates=*)(numsubordinates>=1))" )) { + ; /* Do nothing, this one works OK */ + } else { + /* So let's grok the filter in detail and try to rewrite it */ + slapi_pblock_get( pb, SLAPI_SEARCH_FILTER, &f ); + rc = grok_and_rewrite_filter(f); + if (0 == rc) { + /* he rewrote it ! fixup the string version */ + /* slapi_pblock_set( pb, SLAPI_SEARCH_STRFILTER, newfstr ); */ + } + } + } + + slapi_ch_free_string( &lc_fstr ); + } + return rc; +} + + +int ldbm_compute_init() +{ + int ret = 0; + ret = slapi_compute_add_evaluator(ldbm_compute_evaluator); + if (0 == ret) { + ret = slapi_compute_add_search_rewriter(ldbm_compute_rewriter); + } + return ret; +} + diff --git a/ldap/servers/slapd/back-ldbm/ldbm_attrcrypt.c b/ldap/servers/slapd/back-ldbm/ldbm_attrcrypt.c new file mode 100644 index 00000000..c114fdd6 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_attrcrypt.c @@ -0,0 +1,870 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 1999, 2001-2004 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* This file handles attribute encryption. + */ + + +#include "back-ldbm.h" +#include "attrcrypt.h" +#include "pk11func.h" +#include "keyhi.h" +#include "nss.h" + +/* + * Todo: + * Remember to free the private structures in the attrinfos, so avoid a leak. + */ + +attrcrypt_cipher_entry attrcrypt_cipher_list[] = { {ATTRCRYPT_CIPHER_AES, "AES", CKM_AES_CBC_PAD, CKM_AES_CBC_PAD, CKM_AES_CBC_PAD, 128/8, 16} , + {ATTRCRYPT_CIPHER_DES3 , "3DES" , CKM_DES3_CBC_PAD, CKM_DES3_CBC_PAD, CKM_DES3_CBC_PAD, 112/8, 8}, + {0} }; + +#define KEY_ATTRIBUTE_NAME "nsSymmetricKey" + +/* + * We maintain one of these structures per cipher that we handle + */ + +typedef struct _attrcrypt_cipher_state { + char *cipher_display_name; + PRLock *cipher_lock; + PK11SlotInfo *slot; + PK11SymKey *key; + attrcrypt_cipher_entry *ace; +} attrcrypt_cipher_state; + +struct _attrcrypt_state_private { + attrcrypt_cipher_state *acs_array[1]; +}; + +static int attrcrypt_wrap_key(attrcrypt_cipher_state *acs, PK11SymKey *symmetric_key, SECKEYPublicKey *public_key, SECItem *wrapped_symmetric_key); +static int attrcrypt_unwrap_key(attrcrypt_cipher_state *acs, SECKEYPrivateKey *private_key, SECItem *wrapped_symmetric_key, PK11SymKey **unwrapped_symmetric_key); + +/* + * Copied from front-end because it's private to plugins + */ + +static int +local_valuearray_count( Slapi_Value **va) +{ + int i=0; + if(va!=NULL) + { + while(NULL != va[i]) i++; + } + return(i); +} + +/* + * Helper functions for key management + */ + +static Slapi_Entry * +getConfigEntry( const char *dn, Slapi_Entry **e2 ) { + Slapi_DN sdn; + + slapi_sdn_init_dn_byref( &sdn, dn ); + slapi_search_internal_get_entry( &sdn, NULL, e2, + plugin_get_default_component_id()); + slapi_sdn_done( &sdn ); + return *e2; +} + +/** + * Free an entry + */ +static void +freeConfigEntry( Slapi_Entry ** e ) { + if ( (e != NULL) && (*e != NULL) ) { + slapi_entry_free( *e ); + *e = NULL; + } +} + +static int +attrcrypt_get_ssl_cert_name(char **cert_name) +{ + char *config_entry_dn = "cn=RSA,cn=encryption,cn=config"; + Slapi_Entry *config_entry = NULL; + + *cert_name = NULL; + getConfigEntry(config_entry_dn, &config_entry); + if (NULL == config_entry) { + return -1; + } + *cert_name = slapi_entry_attr_get_charptr( config_entry, "nssslpersonalityssl" ); + freeConfigEntry(&config_entry); + return 0; +} + +/* Retrieve a symmetric key from dse.ldif for a specified cipher */ +static int +attrcrypt_keymgmt_get_key(ldbm_instance *li, attrcrypt_cipher_state *acs, SECKEYPrivateKey *private_key, PK11SymKey **key_from_store) +{ + int ret = 0; + Slapi_Entry *entry = NULL; + char *dn_template = "cn=%s,cn=encrypted attribute keys,cn=%s,cn=ldbm database,cn=plugins,cn=config"; + char *instance_name = li->inst_name; + size_t dn_string_length = 0; + char *dn_string = NULL; + Slapi_Attr *keyattr = NULL; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_keymgmt_get_key\n", 0, 0, 0); + dn_string_length = strlen(dn_template) + strlen(instance_name) + strlen(acs->ace->cipher_display_name); + dn_string = slapi_ch_malloc(dn_string_length); + sprintf(dn_string, dn_template, acs->ace->cipher_display_name, instance_name); + /* Fetch the entry */ + getConfigEntry(dn_string, &entry); + /* Did we find the entry ? */ + if (NULL != entry) { + SECItem key_to_unwrap = {0}; + /* If so then look for the attribute that contains the key */ + slapi_entry_attr_find(entry, KEY_ATTRIBUTE_NAME, &keyattr); + if (keyattr != NULL) { + Slapi_Value *v = NULL; + slapi_valueset_first_value( &keyattr->a_present_values, &v); + key_to_unwrap.len = slapi_value_get_length(v); + key_to_unwrap.data = (void*)slapi_value_get_string(v); + } + /* Unwrap it */ + ret = attrcrypt_unwrap_key(acs, private_key, &key_to_unwrap, key_from_store); + if (entry) { + freeConfigEntry(&entry); + } + } else { + ret = -2; /* Means: we didn't find the entry (which happens if the key has never been generated) */ + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_keymgmt_get_key\n", 0, 0, 0); + return ret; +} + +/* Store a symmetric key for a given cipher in dse.ldif */ +static int +attrcrypt_keymgmt_store_key(ldbm_instance *li, attrcrypt_cipher_state *acs, SECKEYPublicKey *public_key, PK11SymKey *key_to_store) +{ + int ret = 0; + SECItem wrapped_symmetric_key = {0}; + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_keymgmt_store_key\n", 0, 0, 0); + /* Wrap the key and then store it in the right place in dse.ldif */ + ret = attrcrypt_wrap_key(acs, key_to_store, public_key, &wrapped_symmetric_key); + if (!ret) { + /* Make the entry to store */ + Slapi_Entry *e = NULL; + Slapi_PBlock *pb = slapi_pblock_new(); + Slapi_Value *key_value = NULL; + struct berval key_as_berval = {0}; + int rc = 0; + char *entry_template = + "dn: cn=%s,cn=encrypted attribute keys,cn=%s,cn=ldbm database,cn=plugins,cn=config\n" + "objectclass:top\n" + "objectclass:extensibleObject\n" + "cn:%s\n"; + char *instance_name = li->inst_name; + char *entry_string = NULL; + size_t entry_string_length = strlen(entry_template) + strlen(instance_name) + (strlen(acs->ace->cipher_display_name)*2); + entry_string = slapi_ch_malloc(entry_string_length); + sprintf(entry_string, entry_template,acs->ace->cipher_display_name,instance_name,acs->ace->cipher_display_name); + e = slapi_str2entry(entry_string, 0); + /* Add the key as a binary attribute */ + key_as_berval.bv_val = wrapped_symmetric_key.data; + key_as_berval.bv_len = wrapped_symmetric_key.len; + key_value = slapi_value_new_berval(&key_as_berval); + slapi_entry_add_value(e, KEY_ATTRIBUTE_NAME, key_value); + slapi_value_free(&key_value); + /* Store the entry */ + slapi_add_entry_internal_set_pb(pb, e, NULL, li->inst_li->li_identity, 0); + if ((rc = slapi_add_internal_pb(pb)) != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "attrcrypt_keymgmt_store_key: failed to add config key entries to the DSE: %d\n", rc, 0, 0); + } + if (entry_string) { + slapi_ch_free((void**)&entry_string); + } + if (pb) { + slapi_pblock_destroy(pb); + } + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_keymgmt_store_key\n", 0, 0, 0); + return ret; +} + +/* + * Helper functions for key generation and wrapping + */ + +/* Wrap a key with the server's public assymetric key for storage */ +static int +attrcrypt_wrap_key(attrcrypt_cipher_state *acs, PK11SymKey *symmetric_key, SECKEYPublicKey *public_key, SECItem *wrapped_symmetric_key) +{ + int ret = 0; + SECStatus s = 0; + CK_MECHANISM_TYPE wrap_mechanism = CKM_RSA_PKCS; + SECKEYPublicKey *wrapping_key = public_key; + wrapped_symmetric_key->len = slapd_SECKEY_PublicKeyStrength(public_key); + wrapped_symmetric_key->data = slapi_ch_malloc(wrapped_symmetric_key->len); + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_wrap_key\n", 0, 0, 0); + s = slapd_pk11_PubWrapSymKey(wrap_mechanism, wrapping_key, symmetric_key, wrapped_symmetric_key); + if (SECSuccess != s) { + ret = -1; + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_wrap_key: failed to wrap key for cipher %s\n", acs->ace->cipher_display_name, 0, 0); + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_wrap_key\n", 0, 0, 0); + return ret; +} + +/* Unwrap a key previously wrapped with the server's private key */ +static int +attrcrypt_unwrap_key(attrcrypt_cipher_state *acs, SECKEYPrivateKey *private_key, SECItem *wrapped_symmetric_key, PK11SymKey **unwrapped_symmetric_key) +{ + int ret = 0; + CK_MECHANISM_TYPE wrap_mechanism = acs->ace->wrap_mechanism; + SECKEYPrivateKey *unwrapping_key = private_key; + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_unwrap_key\n", 0, 0, 0); + *unwrapped_symmetric_key = slapd_pk11_PubUnwrapSymKey(unwrapping_key, wrapped_symmetric_key, wrap_mechanism, CKA_UNWRAP, 0); + if (NULL == *unwrapped_symmetric_key) { + ret = -1; + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_unwrap_key: failed to unwrap key for cipher %s\n", acs->ace->cipher_display_name, 0, 0); + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_unwrap_key\n", 0, 0, 0); + return ret; +} + +/* Generate a random key for a specified cipher */ +static int +attrcrypt_generate_key(attrcrypt_cipher_state *acs,PK11SymKey **symmetric_key) +{ + int ret = -1; + PK11SymKey *new_symmetric_key = NULL; + *symmetric_key = NULL; + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_generate_key\n", 0, 0, 0); + new_symmetric_key = slapd_pk11_KeyGen(acs->slot, acs->ace->key_gen_mechanism, NULL, acs->ace->key_size, NULL); + if (new_symmetric_key) { + *symmetric_key = new_symmetric_key; + ret = 0; + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_generate_key\n", 0, 0, 0); + return ret; +} + +static int +attrcrypt_fetch_public_key(SECKEYPublicKey **public_key) +{ + int ret = 0; + CERTCertificate *cert = NULL; + SECKEYPublicKey *key = NULL; + PRErrorCode errorCode = 0; + char *default_cert_name = "server-cert"; + char *cert_name = NULL; + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_fetch_public_key\n", 0, 0, 0); + *public_key = NULL; + /* Try to grok the server cert name from the SSL config */ + ret = attrcrypt_get_ssl_cert_name(&cert_name); + if (ret) { + cert_name = default_cert_name; + } + /* We assume that the server core pin stuff is already enabled, via the SSL initialization done in the front-end */ + cert = slapd_pk11_findCertFromNickname(cert_name, NULL); + if (cert == NULL) { + errorCode = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY,"Can't find certificate %s in attrcrypt_fetch_public_key: %d - %s\n", cert_name, errorCode, slapd_pr_strerror(errorCode)); + } + if( cert != NULL ) { + key = slapd_CERT_ExtractPublicKey(cert); + } + if (key == NULL) { + errorCode = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY,"Can't get private key from cert %s in attrcrypt_fetch_public_key: %d - %s\n", cert_name, errorCode, slapd_pr_strerror(errorCode)); + ret = -1; + } + if (cert) { + slapd_pk11_CERT_DestroyCertificate(cert); + } + if (key) { + *public_key = key; + }else { + ret = -1; + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_fetch_public_key\n", 0, 0, 0); + return ret; +} + +static int +attrcrypt_fetch_private_key(SECKEYPrivateKey **private_key) +{ + int ret = 0; + CERTCertificate *cert = NULL; + SECKEYPrivateKey *key = NULL; + PRErrorCode errorCode = 0; + char *default_cert_name = "server-cert"; + char *cert_name = NULL; + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_fetch_private_key\n", 0, 0, 0); + *private_key = NULL; + /* Try to grok the server cert name from the SSL config */ + ret = attrcrypt_get_ssl_cert_name(&cert_name); + if (ret) { + cert_name = default_cert_name; + } + /* We assume that the server core pin stuff is already enabled, via the SSL initialization done in the front-end */ + cert = slapd_pk11_findCertFromNickname(cert_name, NULL); + if (cert == NULL) { + errorCode = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY,"Can't find certificate %s in attrcrypt_fetch_private_key: %d - %s\n", cert_name, errorCode, slapd_pr_strerror(errorCode)); + } + if( cert != NULL ) { + key = slapd_pk11_findKeyByAnyCert(cert, NULL); + } + if (key == NULL) { + errorCode = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY,"Can't get private key from cert %s in attrcrypt_fetch_private_key: %d - %s\n", cert_name, errorCode, slapd_pr_strerror(errorCode)); + ret = -1; + } + if (cert) { + slapd_pk11_CERT_DestroyCertificate(cert); + } + if (key) { + *private_key = key; + } else { + ret = -1; + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_fetch_private_key\n", 0, 0, 0); + return ret; +} + +/* + CKM_AES_CBC_PAD + CKM_DES3_CBC_PAD + */ + +/* Initialize the structure for a single cipher */ +static int +attrcrypt_cipher_init(ldbm_instance *li, attrcrypt_cipher_entry *ace, SECKEYPrivateKey *private_key, SECKEYPublicKey *public_key, attrcrypt_cipher_state *acs) +{ + int ret = 0; + PK11SymKey *symmetric_key = NULL; + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_cipher_init\n", 0, 0, 0); + acs->cipher_lock = PR_NewLock(); + /* Fill in some basic stuff */ + acs->ace = ace; + acs->cipher_display_name = ace->cipher_display_name; + if (NULL == acs->cipher_lock) { + LDAPDebug(LDAP_DEBUG_ANY,"Failed to create cipher lock in attrcrypt_cipher_init\n", 0, 0, 0); + } + acs->slot = slapd_pk11_GetInternalKeySlot(); + if (NULL == acs->slot) { + LDAPDebug(LDAP_DEBUG_ANY,"Failed to create a slot for cipher %s in attrcrypt_cipher_entry\n", acs->cipher_display_name, 0, 0); + goto error; + } + /* Try to get the symmetric key for this cipher */ + ret = attrcrypt_keymgmt_get_key(li,acs,private_key,&symmetric_key); + if (ret) { + if (-2 == ret) { + LDAPDebug(LDAP_DEBUG_ANY,"No symmetric key found for cipher %s in backend %s, attempting to create one...\n", acs->cipher_display_name, li->inst_name, 0); + ret = attrcrypt_generate_key(acs,&symmetric_key); + if (ret) { + LDAPDebug(LDAP_DEBUG_ANY,"Failed to generate key for %s in attrcrypt_cipher_init\n", acs->cipher_display_name, 0, 0); + } + if (symmetric_key) { + ret = attrcrypt_keymgmt_store_key(li,acs,public_key,symmetric_key); + if (ret) { + LDAPDebug(LDAP_DEBUG_ANY,"Failed to store key for cipher %s in attrcrypt_cipher_init\n", acs->cipher_display_name, 0, 0); + } else { + LDAPDebug(LDAP_DEBUG_ANY,"Key for cipher %s successfully generated and stored\n", acs->cipher_display_name, 0, 0); + } + } + + } else { + LDAPDebug(LDAP_DEBUG_ANY,"Failed to retrieve key for cipher %s in attrcrypt_cipher_init\n", acs->cipher_display_name, 0, 0); + } + } + if (symmetric_key) { + /* we loaded the symmetric key, store it in the acs */ + acs->key = symmetric_key; + } +error: + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_cipher_init\n", 0, 0, 0); + return ret; +} + +static void +attrcrypt_acs_list_add(ldbm_instance *li,attrcrypt_cipher_state *acs) +{ + /* Realloc the existing list and add to the end */ + attrcrypt_cipher_state **current = NULL; + size_t list_size = 0; + /* Is the list already there ? */ + if (NULL == li->inst_attrcrypt_state_private) { + /* If not, add it */ + li->inst_attrcrypt_state_private = (attrcrypt_state_private *) slapi_ch_calloc(sizeof(attrcrypt_cipher_state *), 2); /* 2 == The pointer and a NULL terminator */ + } else { + /* Otherwise re-size it */ + for (current = &(li->inst_attrcrypt_state_private->acs_array[0]); *current; current++) { + list_size++; + } + li->inst_attrcrypt_state_private = (attrcrypt_state_private *) slapi_ch_realloc((char*)li->inst_attrcrypt_state_private,sizeof(attrcrypt_cipher_state *) * (list_size + 2)); + li->inst_attrcrypt_state_private->acs_array[list_size + 1] = NULL; + } + li->inst_attrcrypt_state_private->acs_array[list_size] = acs; +} + +int +attrcrypt_init(ldbm_instance *li) +{ + int ret = 0; + attrcrypt_cipher_entry *ace = NULL; + SECKEYPrivateKey *private_key = NULL; + SECKEYPublicKey *public_key = NULL; + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_init\n", 0, 0, 0); + if (slapd_security_library_is_initialized()) { + li->inst_attrcrypt_state_private = NULL; + /* Get the server's private key, which is used to unwrap the stored symmetric keys */ + ret = attrcrypt_fetch_private_key(&private_key); + if (!ret) { + ret = attrcrypt_fetch_public_key(&public_key); + if (!ret) { + for (ace = attrcrypt_cipher_list; ace && ace->cipher_number && !ret; ace++) { + /* Make a state object for this cipher */ + attrcrypt_cipher_state *acs = (attrcrypt_cipher_state *) slapi_ch_calloc(sizeof(attrcrypt_cipher_state),1); + ret = attrcrypt_cipher_init(li, ace, private_key, public_key, acs); + if (ret) { + LDAPDebug(LDAP_DEBUG_ANY,"Failed to initialize cipher %s in attrcrypt_init\n", ace->cipher_display_name, 0, 0); + } else { + /* Since we succeeded, add the acs to the backend instance list */ + attrcrypt_acs_list_add(li,acs); + LDAPDebug(LDAP_DEBUG_TRACE,"Initialized cipher %s in attrcrypt_init\n", ace->cipher_display_name, 0, 0); + } + + } + } + } + } else { + if (li->attrcrypt_configured) { + LDAPDebug(LDAP_DEBUG_ANY,"Warning: encryption is configured in backend %s, but because SSL is not enabled, database encryption is not available and the configuration will be overridden.\n", li->inst_name, 0, 0); + } + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_init : %d\n", ret, 0, 0); + return ret; +} + +/* + * Called by the config code when a new attribute is added, + * to make sure that we already have the runtime state and key + * stored for that cipher. If not, we attmept to make it. + * If this function succeeds, then its ok to go on to use the + * cipher. + */ +int attrcrypt_check_enable_cipher(attrcrypt_cipher_entry *ace) +{ + int ret = 0; + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_check_enable_cipher\n", 0, 0, 0); + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_check_enable_cipher\n", 0, 0, 0); + return ret; +} + +int +attrcrypt_cleanup(attrcrypt_cipher_state *acs) +{ + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_cleanup\n", 0, 0, 0); + if (acs->key) { + slapd_pk11_FreeSymKey(acs->key); + } + if (acs->slot) { + slapd_pk11_FreeSlot(acs->slot); + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_cleanup\n", 0, 0, 0); + return 0; +} + +static attrcrypt_cipher_state * +attrcrypt_get_acs(backend *be, attrcrypt_private *priv) +{ + /* Walk the list of acs objects looking for the one for our cipher */ + int cipher = priv->attrcrypt_cipher; + ldbm_instance *li = (ldbm_instance *) be->be_instance_info; + attrcrypt_state_private* iasp = li->inst_attrcrypt_state_private; + if (iasp) { + attrcrypt_cipher_state **current = &(iasp->acs_array[0]); + while (current) { + if ((*current)->ace->cipher_number == cipher) { + return *current; + } + current++; + } + } + return NULL; +} + +#if defined(DEBUG_ATTRCRYPT) +static void log_bytes(char* format_string, unsigned char *bytes, size_t length) +{ + size_t max_length = 20; + size_t truncated_length = (length > max_length) ? max_length : length; + size_t x = 0; + char *print_buffer = NULL; + char *print_ptr = NULL; + + print_buffer = (char*)slapi_ch_malloc((truncated_length * 3) + 1); + print_ptr = print_buffer; + + for (x = 0; x < truncated_length; x++) { + print_ptr += sprintf(print_ptr, "%02x ", bytes[x]); + } + + LDAPDebug(LDAP_DEBUG_ANY,format_string, print_buffer, length, 0); + + slapi_ch_free((void**)&print_buffer); +} +#endif + +/* Either encipher or decipher an attribute value */ +static int +attrcrypt_crypto_op(attrcrypt_private *priv, backend *be, struct attrinfo *ai, char *in_data, size_t in_size, char **out_data, size_t *out_size, int encrypt) +{ + int ret = 0; + SECStatus secret = 0; + PK11Context* sec_context = NULL; + SECItem iv_item = {0}; + SECItem *security_parameter = NULL; + int output_buffer_length = 0; + int output_buffer_size1 = 0; + int output_buffer_size2 = 0; + unsigned char *output_buffer = NULL; + attrcrypt_cipher_state *acs = NULL; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_crypto_op\n", 0, 0, 0); + acs = attrcrypt_get_acs(be,ai->ai_attrcrypt); + if (NULL == acs) { + /* This happens if SSL/NSS has not been enabled */ + return -1; + } +#if defined(DEBUG_ATTRCRYPT) + if (encrypt) { + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_crypto_op encrypt '%s' (%d)\n", in_data, in_size, 0); + } else { + log_bytes("attrcrypt_crypto_op decrypt '%s' (%d)\n", in_data, in_size); + } +#endif + /* Allocate the output buffer */ + output_buffer_length = in_size + 16; + output_buffer = slapi_ch_malloc(output_buffer_length); + /* Now call NSS to do the cipher op */ + iv_item.data = "aaaaaaaaaaaaaaaa"; /* ptr to an array of IV bytes */ + iv_item.len = acs->ace->iv_length; /* length of the array of IV bytes */ + security_parameter = slapd_pk11_ParamFromIV(acs->ace->cipher_mechanism, &iv_item); + if (NULL == security_parameter) { + int errorCode = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_crypto_op failed to make IV for cipher %s : %d - %s\n", acs->ace->cipher_display_name, errorCode, slapd_pr_strerror(errorCode)); + goto error; + } + sec_context = slapd_pk11_createContextBySymKey(acs->ace->cipher_mechanism, (encrypt ? CKA_ENCRYPT : CKA_DECRYPT), acs->key, security_parameter); + if (NULL == sec_context) { + int errorCode = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_crypto_op failed on cipher %s : %d - %s\n", acs->ace->cipher_display_name, errorCode, slapd_pr_strerror(errorCode)); + goto error; + } + secret = slapd_pk11_cipherOp(sec_context, output_buffer, &output_buffer_size1, output_buffer_length, in_data, in_size); + if (SECSuccess != secret) { + int errorCode = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_crypto_op failed on cipher %s : %d - %s\n", acs->ace->cipher_display_name, errorCode, slapd_pr_strerror(errorCode)); + goto error; + } +#if defined(DEBUG_ATTRCRYPT) + LDAPDebug(LDAP_DEBUG_ANY,"slapd_pk11_cipherOp %d\n", output_buffer_size1, 0, 0); +#endif + secret = slapd_pk11_DigestFinal(sec_context, output_buffer + output_buffer_size1, &output_buffer_size2, output_buffer_length - output_buffer_size1); + if (SECSuccess != secret) { + int errorCode = PR_GetError(); + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_crypto_op digest final failed on cipher %s : %d - %s\n", acs->ace->cipher_display_name, errorCode, slapd_pr_strerror(errorCode)); + goto error; + } else { +#if defined(DEBUG_ATTRCRYPT) + if (encrypt) { + log_bytes("slapd_pk11_DigestFinal '%s' (%d)\n", output_buffer, output_buffer_size1 + output_buffer_size2); + } else { + LDAPDebug(LDAP_DEBUG_ANY,"slapd_pk11_DigestFinal '%s', %d\n", output_buffer, output_buffer_size2, 0); + } +#endif + *out_size = output_buffer_size1 + output_buffer_size2; + *out_data = output_buffer; + } +error: + if (sec_context) { + slapd_pk11_DestroyContext(sec_context, PR_TRUE); + } + if (security_parameter) { + slapd_SECITEM_FreeItem(security_parameter, PR_TRUE); + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_crypto_op\n", 0, 0, 0); + return ret; +} + +static int +attrcrypt_crypto_op_value(attrcrypt_private *priv, backend *be, struct attrinfo *ai, Slapi_Value *invalue, Slapi_Value **outvalue, int encrypt) +{ + int ret = 0; + char *in_data = NULL; + size_t in_size = 0; + char *out_data = NULL; + size_t out_size = 0; + struct berval *bval = NULL; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_crypto_op_value\n", 0, 0, 0); + + bval = (struct berval *) slapi_value_get_berval(invalue); + in_data = bval->bv_val; + in_size = bval->bv_len; + + ret = attrcrypt_crypto_op(priv,be,ai,in_data,in_size,&out_data,&out_size,encrypt); + + if (0 == ret) { + struct berval outbervalue = {0}; + outbervalue.bv_len = out_size; + outbervalue.bv_val = out_data; + /* This call makes a copy of the payload data, so we need to free the original data after making the call */ + *outvalue = slapi_value_new_berval(&outbervalue); + slapi_ch_free((void**)&out_data); + } + + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_crypto_op_value: %d\n", ret, 0, 0); + return ret; +} + +int +attrcrypt_crypto_op_value_replace(attrcrypt_private *priv, backend *be, struct attrinfo *ai, Slapi_Value *inoutvalue, int encrypt) +{ + int ret = 0; + char *in_data = NULL; + size_t in_size = 0; + char *out_data = NULL; + size_t out_size = 0; + struct berval *bval = NULL; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_crypto_op_value_replace\n", 0, 0, 0); + + bval = (struct berval *) slapi_value_get_berval(inoutvalue); + in_data = bval->bv_val; + in_size = bval->bv_len; + + ret = attrcrypt_crypto_op(priv,be,ai,in_data,in_size,&out_data,&out_size,encrypt); + + if (0 == ret) { + struct berval outbervalue = {0}; + outbervalue.bv_len = out_size; + outbervalue.bv_val = out_data; + /* This takes a copy of the payload, so we need to free it now */ + slapi_value_set_berval(inoutvalue,&outbervalue); + slapi_ch_free((void**)&out_data); + } + + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_crypto_op_value_replace: %d\n", ret, 0, 0); + return ret; +} + +static int +attrcrypt_crypto_op_values(attrcrypt_private *priv, backend *be, struct attrinfo *ai, Slapi_Value **invalues, Slapi_Value ***outvalues, int encrypt) +{ + int ret = 0; + int i = 0; + Slapi_Value **encrypted_values = NULL; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_crypto_op_values\n", 0, 0, 0); + encrypted_values = (Slapi_Value **) slapi_ch_calloc(sizeof(Slapi_Value *),local_valuearray_count(invalues) + 1); + for ( i = 0; (invalues[i] != NULL) && (ret == 0); i++ ) { + Slapi_Value *encrypted_value = NULL; + + ret = attrcrypt_crypto_op_value(priv,be,ai,invalues[i],&encrypted_value,encrypt); + if (0 == ret) { + encrypted_values[i] = encrypted_value; + } + } + *outvalues = encrypted_values; + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_crypto_op_values: %d\n", ret, 0, 0); + return ret; +} + +static int +attrcrypt_crypto_op_values_replace(attrcrypt_private *priv, backend *be, struct attrinfo *ai, Slapi_Value **invalues, int encrypt) +{ + int ret = 0; + int i = 0; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_crypto_op_values_replace\n", 0, 0, 0); + for ( i = 0; (invalues[i] != NULL) && (ret == 0); i++ ) { + + ret = attrcrypt_crypto_op_value_replace(priv,be,ai,invalues[i],encrypt); + if (ret) { + break; + } + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_crypto_op_values_replace\n", 0, 0, 0); + return ret; +} + +/* Modifies the entry in-place to decrypt any encrypted attributes */ +int +attrcrypt_decrypt_entry(backend *be, struct backentry *e) +{ + int ret = 0; + int rc = 0; + Slapi_Attr *attr = NULL; + char *type = NULL; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_decrypt_entry\n", 0, 0, 0); + /* Scan through the entry's attributes, looking to see if any are configured for crypto */ + for ( rc = slapi_entry_first_attr( e->ep_entry, &attr ); rc == 0 && attr ; rc = slapi_entry_next_attr( e->ep_entry, attr, &attr )) { + + struct attrinfo *ai = NULL; + Slapi_Value *value = NULL; + int i = 0; + + slapi_attr_get_type( attr, &type ); + ainfo_get(be, type, &ai); + + if (ai && ai->ai_attrcrypt) { + i = slapi_attr_first_value(attr,&value); + while (NULL != value && i != -1) + { + /* Now decrypt the attribute values in place on the original entry */ + ret = attrcrypt_crypto_op_value_replace(ai->ai_attrcrypt,be,ai,value,0); + if (ret) { + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_decrypt_entry: FAILING because decryption operation failed\n", 0, 0, 0); + return ret; + } + i = slapi_attr_next_value(attr,i,&value); + } + /* Now do the same thing with deleted values */ + i = attr_first_deleted_value(attr,&value); + while (NULL != value && i != -1) + { + /* Now decrypt the attribute values in place on the original entry */ + ret = attrcrypt_crypto_op_value_replace(ai->ai_attrcrypt,be,ai,value,0); + if (ret) { + LDAPDebug(LDAP_DEBUG_ANY,"attrcrypt_decrypt_entry: FAILING because decryption operation failed\n", 0, 0, 0); + return ret; + } + i = attr_next_deleted_value(attr,i,&value); + } + } + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_decrypt_entry\n", 0, 0, 0); + return ret; +} + +/* Encrypts attributes on this entry in-place (only changes the attribute data, nothing else) + */ +int +attrcrypt_encrypt_entry_inplace(backend *be, const struct backentry *inout) +{ + int ret = 0; + int rc = 0; + char *type = NULL; + Slapi_Attr *attr = NULL; + Slapi_Value **svals = NULL; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_encrypt_entry_inplace\n", 0, 0, 0); + /* Scan the entry's attributes looking for any that are configured for encryption */ + for ( rc = slapi_entry_first_attr( inout->ep_entry, &attr ); rc == 0; + rc = slapi_entry_next_attr( inout->ep_entry, attr, &attr ) ) { + + struct attrinfo *ai = NULL; + + slapi_attr_get_type( attr, &type ); + + ainfo_get(be, type, &ai); + + if (ai && ai->ai_attrcrypt) { + svals = attr_get_present_values(attr); + if (svals) { + /* Now encrypt the attribute values in place on the new entry */ + ret = attrcrypt_crypto_op_values_replace(ai->ai_attrcrypt,be,ai,svals,1); + } + } + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_encrypt_entry_inplace\n", 0, 0, 0); + return ret; +} + +/* Makes a copy of the entry that has all necessary attributes encrypted + * as a performance optimization, if there are no attributes configured + * for encryption in the entry, then no copy is returned. + */ +int +attrcrypt_encrypt_entry(backend *be, const struct backentry *in, struct backentry **out) +{ + int ret = 0; + int rc = 0; + struct backentry *new_entry = NULL; + char *type = NULL; + Slapi_Attr *attr = NULL; + Slapi_Value **svals = NULL; + Slapi_Value **new_vals = NULL; + + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_encrypt_entry\n", 0, 0, 0); + *out = NULL; + /* Scan the entry's attributes looking for any that are configured for encryption */ + for ( rc = slapi_entry_first_attr( in->ep_entry, &attr ); rc == 0; + rc = slapi_entry_next_attr( in->ep_entry, attr, &attr ) ) { + + struct attrinfo *ai = NULL; + + slapi_attr_get_type( attr, &type ); + + ainfo_get(be, type, &ai); + + if (ai && ai->ai_attrcrypt) { + svals = attr_get_present_values(attr); + if (svals) { + /* If we find one, did we make the new entry yet ? */ + if (NULL == new_entry) { + /* If not then make it now as a copy of the old entry */ + new_entry = backentry_dup((struct backentry *)in); + } + /* Now encrypt the attribute values in place on the new entry */ + ret = attrcrypt_crypto_op_values(ai->ai_attrcrypt,be,ai,svals,&new_vals,1); + if (ret) { + LDAPDebug(LDAP_DEBUG_ANY,"Error: attrcrypt_crypto_op_values failed in attrcrypt_encrypt_entry\n", 0, 0, 0); + break; + } + /* DBDB does this call free the old value memory ? */ + slapi_entry_attr_replace_sv(new_entry->ep_entry, type, new_vals); + } + } + } + *out = new_entry; + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_encrypt_entry\n", 0, 0, 0); + return ret; +} + +/* + * Encrypt an index key. There is never any need to decrypt index keys since + * we only ever look them up using plain text. + */ +int +attrcrypt_encrypt_index_key(backend *be, struct attrinfo *ai, const struct berval *in, struct berval **out) +{ + int ret = 0; + char *in_data = in->bv_val; + size_t in_size = in->bv_len; + char *out_data = NULL; + size_t out_size = 0; + struct berval *out_berval = NULL; + + if (ai->ai_attrcrypt) { + LDAPDebug(LDAP_DEBUG_TRACE,"-> attrcrypt_encrypt_index_key\n", 0, 0, 0); + ret = attrcrypt_crypto_op(ai->ai_attrcrypt,be,ai, in_data,in_size,&out_data,&out_size, 1); + if (0 == ret) { + out_berval = (struct berval *)ber_alloc(); + if (NULL == out_berval) { + return ENOMEM; + } + out_berval->bv_len = out_size; + /* Because we're making a new berval, we copy the payload pointer in */ + /* It's now the responsibility of our caller to free that data */ + out_berval->bv_val = out_data; + *out = out_berval; + } + LDAPDebug(LDAP_DEBUG_TRACE,"<- attrcrypt_encrypt_index_key\n", 0, 0, 0); + } + return ret; +} + diff --git a/ldap/servers/slapd/back-ldbm/ldbm_attrcrypt_config.c b/ldap/servers/slapd/back-ldbm/ldbm_attrcrypt_config.c new file mode 100644 index 00000000..7ec93554 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_attrcrypt_config.c @@ -0,0 +1,298 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2004 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* This file handles configuration information that is specific + * to ldbm instance attribute encryption configuration. + */ + +/* DBDB I left in the Sun copyright statement because some of the code + * in this file is derived from an older file : ldbm_index_config.c + */ + +#include "back-ldbm.h" +#include "attrcrypt.h" + +/* Forward declarations for the callbacks */ +int ldbm_instance_attrcrypt_config_add_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); +int ldbm_instance_attrcrypt_config_delete_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); + +/* + +Config entries look like this: + +dn: cn=<attributeName>, cn=encrypted attributes, cn=databaseName, cn=ldbm database, cn=plugins, cn=config +objectclass: top +objectclass: nsAttributeEncryption +cn: <attributeName> +nsEncryptionAlgorithm: <cipherName> + +*/ + +static int +ldbm_attrcrypt_parse_cipher(char* cipher_display_name) +{ + attrcrypt_cipher_entry *ce = attrcrypt_cipher_list; + while (ce->cipher_number) { + if (0 == strcmp(ce->cipher_display_name,cipher_display_name)) { + return ce->cipher_number; + } + ce++; + } + return 0; +} + +static int +ldbm_attrcrypt_parse_entry(ldbm_instance *inst, Slapi_Entry *e, + char **attribute_name, + int *cipher) +{ + Slapi_Attr *attr; + const struct berval *attrValue; + Slapi_Value *sval; + + *cipher = 0; + *attribute_name = NULL; + + /* Get the name of the attribute to index which will be the value + * of the cn attribute. */ + if (slapi_entry_attr_find(e, "cn", &attr) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "Warning: malformed attribute encryption entry %s\n", + slapi_entry_get_dn(e), 0, 0); + return LDAP_OPERATIONS_ERROR; + } + + slapi_attr_first_value(attr, &sval); + attrValue = slapi_value_get_berval(sval); + *attribute_name = slapi_ch_strdup(attrValue->bv_val); + + /* Get the list of index types from the entry. */ + if (0 == slapi_entry_attr_find(e, "nsEncryptionAlgorithm", &attr)) { + slapi_attr_first_value(attr, &sval); + if (sval) { + attrValue = slapi_value_get_berval(sval); + *cipher = ldbm_attrcrypt_parse_cipher(attrValue->bv_val); + if (0 == *cipher) + LDAPDebug(LDAP_DEBUG_ANY, "Warning: attempt to configure unrecognized cipher %s in encrypted attribute config entry %s\n", + attrValue->bv_val, slapi_entry_get_dn(e), 0); + } + } + return LDAP_SUCCESS; +} + +static void +ldbm_instance_attrcrypt_enable(struct attrinfo *ai, int cipher) +{ + attrcrypt_private *priv = NULL; + if (NULL == ai->ai_attrcrypt) { + /* No existing private structure, allocate one */ + ai->ai_attrcrypt = (attrcrypt_private*) slapi_ch_calloc(1, sizeof(attrcrypt_private)); + } + priv = ai->ai_attrcrypt; + priv->attrcrypt_cipher = cipher; +} + +static void +ldbm_instance_attrcrypt_disable(struct attrinfo *ai) +{ + if (NULL != ai->ai_attrcrypt) { + /* Don't free the structure here, because other threads might be + * concurrently referencing it. + */ + ai->ai_attrcrypt = 0; + } +} + +/* + * Config DSE callback for attribute encryption entry add. + */ +int +ldbm_instance_attrcrypt_config_add_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* eAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + char *attribute_name = NULL; + int cipher = 0; + int ret = 0; + + returntext[0] = '\0'; + + /* For add, we parse the entry, then check the attribute exists, + * then check that indexing config does not preclude us encrypting it, + * and finally we set the private structure in the attrinfo for the attribute. + */ + + *returncode = ldbm_attrcrypt_parse_entry(inst, e, &attribute_name , &cipher); + + if (*returncode == LDAP_SUCCESS) { + + struct attrinfo *ai = NULL; + + /* If the cipher was invalid, return unwilling to perform */ + if (0 == cipher) { + returntext = "invalid cipher"; + *returncode = LDAP_UNWILLING_TO_PERFORM; + ret = SLAPI_DSE_CALLBACK_ERROR; + } else { + + ainfo_get(inst->inst_be, attribute_name, &ai); + /* If we couldn't find a non-default attrinfo, then that means + * that no indexing or encryption has yet been defined for this attribute + * therefore , create a new attrinfo structure now. + */ + if ((ai == NULL) || (0 == strcmp(LDBM_PSEUDO_ATTR_DEFAULT, ai->ai_type) )) { + /* If this attribute doesn't exist in the schema, then we DO NOT fail + * (this is because entensible objects and disabled schema checking allow + * non-schema attributes to exist. + */ + /* Make a new attrinfo object */ + attr_create_empty(inst->inst_be,attribute_name,&ai); + } + if (ai) { + ldbm_instance_attrcrypt_enable(ai, cipher); + /* Remember that we have some encryption enabled, so we can be intelligent about warning when SSL is not enabled */ + inst->attrcrypt_configured = 1; + } else { + LDAPDebug(LDAP_DEBUG_ANY, "Warning: attempt to encryption on a non-existent attribute: %s\n", + attribute_name, 0, 0); + returntext = "attribute does not exist"; + *returncode = LDAP_UNWILLING_TO_PERFORM; + ret = SLAPI_DSE_CALLBACK_ERROR; + } + ret = SLAPI_DSE_CALLBACK_OK; + } + + } else { + ret = SLAPI_DSE_CALLBACK_ERROR; + } + if (attribute_name) { + slapi_ch_free(&attribute_name); + } + return ret; +} + +/* + * Temp callback that gets called for each attribute encryption entry when a new + * instance is starting up. + */ +int +ldbm_attrcrypt_init_entry_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + return ldbm_instance_attrcrypt_config_add_callback(pb,e,entryAfter,returncode,returntext,arg); +} + +/* + * Config DSE callback for attribute encryption deletes. + */ +int +ldbm_instance_attrcrypt_config_delete_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + char *attribute_name = NULL; + int cipher = 0; + int ret = SLAPI_DSE_CALLBACK_ERROR; + + returntext[0] = '\0'; + + /* For add, we parse the entry, then check the attribute exists, + * then check that indexing config does not preclude us encrypting it, + * and finally we set the private structure in the attrinfo for the attribute. + */ + + *returncode = ldbm_attrcrypt_parse_entry(inst, e, &attribute_name , &cipher); + + if (*returncode == LDAP_SUCCESS) { + + struct attrinfo *ai = NULL; + + ainfo_get(inst->inst_be, attribute_name, &ai); + if (ai == NULL && (0 == strcmp(LDBM_PSEUDO_ATTR_DEFAULT, ai->ai_type)) ) { + LDAPDebug(LDAP_DEBUG_ANY, "Warning: attempt to delete encryption for non-existant attribute: %s\n", + attribute_name, 0, 0); + } else { + ldbm_instance_attrcrypt_disable(ai); + ret = SLAPI_DSE_CALLBACK_OK; + } + } + if (attribute_name) { + slapi_ch_free((void **)&attribute_name); + } + return ret; +} + +/* + * Config DSE callback for index entry changes. + * + * this function is huge! + */ +int +ldbm_instance_attrcrypt_config_modify_callback(Slapi_PBlock *pb, Slapi_Entry *e, + Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + Slapi_Attr *attr; + Slapi_Value *sval; + const struct berval *attrValue; + struct attrinfo *ainfo = NULL; + LDAPMod **mods; + int i = 0; + int j = 0; + + returntext[0] = '\0'; + *returncode = LDAP_SUCCESS; + slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods); + + slapi_entry_attr_find(e, "cn", &attr); + slapi_attr_first_value(attr, &sval); + attrValue = slapi_value_get_berval(sval); + ainfo_get(inst->inst_be, attrValue->bv_val, &ainfo); + if (NULL == ainfo) { + return SLAPI_DSE_CALLBACK_ERROR; + } + + for (i = 0; mods[i] != NULL; i++) { + + char *config_attr = (char *)mods[i]->mod_type; + + /* There are basically three cases in the modify: + * 1. The attribute was added + * 2. The attribute was deleted + * 3. The attribute was modified (deleted and added). + * Now, of these three, only #3 is legal. + * This is because the attribute is mandatory and single-valued in the schema. + * We handle this as follows: an add will always replace what's there (if anything). + * a delete will remove what's there as long as it matches what's being deleted. + * this is to avoid ordering problems with the adds and deletes. + */ + + if (strcasecmp(config_attr, "nsEncryptionAlgorithm") == 0) { + + if ((mods[i]->mod_op & ~LDAP_MOD_BVALUES) == LDAP_MOD_ADD) { + + for (j = 0; mods[i]->mod_bvalues[j] != NULL; j++) { + int cipher = ldbm_attrcrypt_parse_cipher(mods[i]->mod_bvalues[j]->bv_val); + if (0 == cipher) { + /* Tried to configure an invalid cipher */ + } + ldbm_instance_attrcrypt_enable(ainfo,cipher); + } + continue; + } + if ((mods[i]->mod_op & ~LDAP_MOD_BVALUES) == LDAP_MOD_DELETE) { + if ((mods[i]->mod_bvalues == NULL) || + (mods[i]->mod_bvalues[0] == NULL)) { + /* Not legal */ + return SLAPI_DSE_CALLBACK_ERROR; + } else { + for (j = 0; mods[i]->mod_bvalues[j] != NULL; j++) { + /* Code before here should ensure that we only ever delete something that was already here */ + ldbm_instance_attrcrypt_disable(ainfo); + } + } + continue; + } + } + } + return SLAPI_DSE_CALLBACK_OK; +} + diff --git a/ldap/servers/slapd/back-ldbm/ldbm_bind.c b/ldap/servers/slapd/back-ldbm/ldbm_bind.c new file mode 100644 index 00000000..de70c601 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_bind.c @@ -0,0 +1,245 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* bind.c - ldbm backend bind and unbind routines */ + +#include "back-ldbm.h" + +#if defined( XP_WIN32 ) + +typedef enum LDAPWAEnum { + LDAPWA_NoDomainAttr = -3, + LDAPWA_InvalidCredentials = -2, + LDAPWA_Failure = -1, + LDAPWA_Success= 0 +} LDAPWAStatus; + +int +GetDomainUsername( + char *pszNTuserdomainid, + char *pszNTDomain, + char *pszNTUsername +) +{ + char *pszAttr, *pDomain, *pUsername; + + if( !pszNTuserdomainid ) + return( 1 ); + + // Split the specially constructed attribute. + pszAttr = slapi_ch_strdup( pszNTuserdomainid ); + + pDomain = pszAttr; + + pUsername = strchr( pszAttr, ':' ); + if( pUsername == NULL ) + return( 1 ); + + // Set the end of the NT Domain name, + // and the start of the NT username. + *pUsername = (char)NULL; + pUsername++; + + strcpy( pszNTDomain, pDomain); + strcpy( pszNTUsername, pUsername); + + slapi_ch_free( (void**)&pszAttr ); + + return( 0 ); +} + +/* Attempt Windows NT Authentication, using the password from the client app, + with the NT Domain and NT username, both stored in the entry. + If successful, the ldap_bind() is completed successsfully. */ + +LDAPWAStatus +WindowsAuthentication( + struct backentry *e, + struct berval *cred +) +{ + Slapi_Attr *a; + Slapi_Value *sval = NULL; + int iStatus; + char szNTDomain[MAX_PATH], szNTUsername[MAX_PATH]; + HANDLE hToken = NULL; + BOOL bLogonStatus = FALSE; + int i= -1; + + /* Get the NT Domain and username - if the entry has such an attribute */ + if( !e || !e->ep_entry || + slapi_entry_attr_find( e->ep_entry, "ntuserdomainid", &a ) != 0) + { + return( LDAPWA_NoDomainAttr ); + } + + i= slapi_attr_first_value( a, &sval ); + if(sval==NULL) + { + return( LDAPWA_NoDomainAttr ); + } + + while(i != -1) + { + const struct berval *val = slapi_value_get_berval(sval); + char * colon = NULL; + + if (!val->bv_val || (strlen(val->bv_val) > (MAX_PATH<<1))) { + LDAPDebug( LDAP_DEBUG_TRACE, + "WindowsAuthentication => validation FAILED for \"%s\" on NT Domain : " + "ntuserdomainid attr value too long\n", + val->bv_val, 0, 0); + i= slapi_attr_next_value(a, i, &sval); + continue; + } + colon = strchr( val->bv_val, ':' ); + if (!colon || ((colon - val->bv_val)/sizeof(char) > MAX_PATH)) { + if (!colon) { + LDAPDebug( LDAP_DEBUG_TRACE, + "WindowsAuthentication => validation FAILED for \"%s\" on NT Domain : " + "a colon is missing in ntuserdomainid attr value\n", + val->bv_val, 0, 0); + } + else { + LDAPDebug( LDAP_DEBUG_TRACE, + "WindowsAuthentication => validation FAILED for \"%s\" on NT Domain : " + "domain in ntuserdomainid attr value too long\n", + val->bv_val, 0, 0); + } + i= slapi_attr_next_value(a, i, &sval); + continue; + } + + if(( iStatus = GetDomainUsername( val->bv_val, + szNTDomain, + szNTUsername )) != 0) + { + i= slapi_attr_next_value(a, i, &sval); + continue; + } + +#if !defined( LOGON32_LOGON_NETWORK ) +/* This is specified in the WIn32 LogonUser() documentation, but not defined + in the Visual C++ 4.2 include file winbase.h. A search of the lastest version + of this file at www.microsoft.com finds that LOGON32_LOGON_NETWORK == 3. + */ +#define LOGON32_LOGON_NETWORK 3 +#endif + /* Now do the Logon attempt */ + bLogonStatus = LogonUser( szNTUsername, // string that specifies the user name + szNTDomain, // string that specifies the domain or server + cred->bv_val, // string that specifies the password + LOGON32_LOGON_NETWORK, // the type of logon operation, + LOGON32_PROVIDER_DEFAULT, // specifies the logon provider + &hToken ); // pointer to variable to receive token handle + if( bLogonStatus && hToken ) + CloseHandle( hToken ); + + if( bLogonStatus ) + { + // Successful validation + LDAPDebug( LDAP_DEBUG_TRACE, + "WindowsAuthentication => validated \"%s\" on NT Domain \"%s\"\n", + szNTUsername, szNTDomain, 0 ); + return( LDAPWA_Success ); + } + else + { + LDAPDebug( LDAP_DEBUG_TRACE, + "WindowsAuthentication => validation FAILED for \"%s\" on NT Domain \"%s\", reason %d\n", + szNTUsername, szNTDomain, GetLastError() ); + return( LDAPWA_InvalidCredentials ); + } + i= slapi_attr_next_value(a, i, &sval); + } + + + return( LDAPWA_Failure ); + +} +#endif + +int +ldbm_back_bind( Slapi_PBlock *pb ) +{ + backend *be; + ldbm_instance *inst; + int method; + struct berval *cred; + struct ldbminfo *li; + struct backentry *e; + Slapi_Attr *attr; + Slapi_Value **bvals; + entry_address *addr; + + /* get parameters */ + slapi_pblock_get( pb, SLAPI_BACKEND, &be ); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_TARGET_ADDRESS, &addr ); + slapi_pblock_get( pb, SLAPI_BIND_METHOD, &method ); + slapi_pblock_get( pb, SLAPI_BIND_CREDENTIALS, &cred ); + + inst = (ldbm_instance *) be->be_instance_info; + + /* always allow noauth simple binds (front end will send the result) */ + if ( method == LDAP_AUTH_SIMPLE && cred->bv_len == 0 ) { + return( SLAPI_BIND_ANONYMOUS ); + } + + /* + * find the target entry. find_entry() takes care of referrals + * and sending errors if the entry does not exist. + */ + if (( e = find_entry( pb, be, addr, NULL /* no txn */ )) == NULL ) { + return( SLAPI_BIND_FAIL ); + } + + switch ( method ) { + case LDAP_AUTH_SIMPLE: + { + Slapi_Value cv; + if ( slapi_entry_attr_find( e->ep_entry, "userpassword", &attr ) != 0 ) { +#if defined( XP_WIN32 ) + if( WindowsAuthentication( e, cred ) == LDAPWA_Success ) { + break; + } +#endif + slapi_send_ldap_result( pb, LDAP_INAPPROPRIATE_AUTH, NULL, + NULL, 0, NULL ); + cache_return( &inst->inst_cache, &e ); + return( SLAPI_BIND_FAIL ); + } + bvals= attr_get_present_values(attr); + slapi_value_init_berval(&cv,cred); + if ( slapi_pw_find_sv( bvals, &cv ) != 0 ) { +#if defined( XP_WIN32 ) + /* One last try - attempt Windows authentication, + if the user has a Windows account. */ + if( WindowsAuthentication( e, cred ) == LDAPWA_Success ) { + break; + } +#endif + slapi_send_ldap_result( pb, LDAP_INVALID_CREDENTIALS, NULL, + NULL, 0, NULL ); + cache_return( &inst->inst_cache, &e ); + value_done(&cv); + return( SLAPI_BIND_FAIL ); + } + value_done(&cv); + } + break; + + default: + slapi_send_ldap_result( pb, LDAP_STRONG_AUTH_NOT_SUPPORTED, NULL, + "auth method not supported", 0, NULL ); + cache_return( &inst->inst_cache, &e ); + return( SLAPI_BIND_FAIL ); + } + + cache_return( &inst->inst_cache, &e ); + + /* success: front end will send result */ + return( SLAPI_BIND_SUCCESS ); +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_compare.c b/ldap/servers/slapd/back-ldbm/ldbm_compare.c new file mode 100644 index 00000000..a63f6009 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_compare.c @@ -0,0 +1,77 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* compare.c - ldbm backend compare routine */ + +#include "back-ldbm.h" + +int +ldbm_back_compare( Slapi_PBlock *pb ) +{ + backend *be; + ldbm_instance *inst; + struct ldbminfo *li; + struct backentry *e; + int err; + char *type; + struct berval *bval; + entry_address *addr; + Slapi_Value compare_value; + int result; + int ret = 0; + Slapi_DN *namespace_dn; + + + slapi_pblock_get( pb, SLAPI_BACKEND, &be ); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_TARGET_ADDRESS, &addr); + slapi_pblock_get( pb, SLAPI_COMPARE_TYPE, &type ); + slapi_pblock_get( pb, SLAPI_COMPARE_VALUE, &bval ); + + inst = (ldbm_instance *) be->be_instance_info; + /* get the namespace dn */ + namespace_dn = (Slapi_DN*)slapi_be_getsuffix(be, 0); + + if ( (e = find_entry( pb, be, addr, NULL )) == NULL ) { + return( -1 ); /* error result sent by find_entry() */ + } + + err = slapi_access_allowed (pb, e->ep_entry, type, bval, SLAPI_ACL_COMPARE); + if ( err != LDAP_SUCCESS ) { + slapi_send_ldap_result( pb, err, NULL, NULL, 0, NULL ); + ret = 1; + } else { + + slapi_value_init_berval(&compare_value,bval); + + err = slapi_vattr_namespace_value_compare(e->ep_entry,namespace_dn,type,&compare_value,&result,0); + + if (0 != err) { + /* Was the attribute not found ? */ + if (SLAPI_VIRTUALATTRS_NOT_FOUND == err) { + slapi_send_ldap_result( pb, LDAP_NO_SUCH_ATTRIBUTE, NULL, NULL,0, NULL ); + ret = 1; + } else { + /* Some other problem, call it an operations error */ + slapi_send_ldap_result( pb, LDAP_OPERATIONS_ERROR, NULL, NULL,0, NULL ); + ret = -1; + } + } else { + /* Interpret the result */ + if (result) { + /* Compare true */ + slapi_send_ldap_result( pb, LDAP_COMPARE_TRUE, NULL, NULL, 0, NULL ); + } else { + /* Compare false */ + slapi_send_ldap_result( pb, LDAP_COMPARE_FALSE, NULL, NULL, 0, NULL ); + } + ret = 0; + } + value_done(&compare_value); + } + + cache_return( &inst->inst_cache, &e ); + return( ret ); +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.c b/ldap/servers/slapd/back-ldbm/ldbm_config.c new file mode 100644 index 00000000..59e197f3 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_config.c @@ -0,0 +1,1730 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* ldbm_config.c - Handles configuration information that is global to all ldbm instances. */ + +#include "back-ldbm.h" +#include "dblayer.h" + +/* Forward declarations */ +static int parse_ldbm_config_entry(struct ldbminfo *li, Slapi_Entry *e, config_info *config_array); + +/* Forward callback declarations */ +int ldbm_config_search_entry_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_config_modify_entry_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); + +static char *ldbm_skeleton_entries[] = +{ + "dn:cn=config, cn=%s, cn=plugins, cn=config\n" + "objectclass:top\n" + "objectclass:extensibleObject\n" + "cn:config\n", + + "dn:cn=monitor, cn=%s, cn=plugins, cn=config\n" + "objectclass:top\n" + "objectclass:extensibleObject\n" + "cn:monitor\n", + + "dn:cn=database, cn=monitor, cn=%s, cn=plugins, cn=config\n" + "objectclass:top\n" + "objectclass:extensibleObject\n" + "cn:database\n", + + "" +}; + +/* Used to add an array of entries, like the one above and + * ldbm_instance_skeleton_entries in ldbm_instance_config.c, to the dse. + * Returns 0 on success. + */ +int ldbm_config_add_dse_entries(struct ldbminfo *li, char **entries, char *string1, char *string2, char *string3, int flags) +{ + int x; + Slapi_Entry *e; + Slapi_PBlock *util_pb = NULL; + int rc; + char entry_string[512]; + int dont_write_file = 0; + + if (flags & LDBM_INSTANCE_CONFIG_DONT_WRITE) { + dont_write_file = 1; + } + + for(x = 0; strlen(entries[x]) > 0; x++) { + util_pb = slapi_pblock_new(); + sprintf(entry_string, entries[x], string1, string2, string3); + e = slapi_str2entry(entry_string, 0); + slapi_add_entry_internal_set_pb(util_pb, e, NULL, li->li_identity, 0); + slapi_pblock_set(util_pb, SLAPI_DSE_DONT_WRITE_WHEN_ADDING, + &dont_write_file); + if ((rc = slapi_add_internal_pb(util_pb)) != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "Unable to add config entries to the DSE: %d\n", rc, 0, 0); + } + slapi_pblock_destroy(util_pb); + } + + return 0; +} + +/* used to add a single entry, special case of above */ +int ldbm_config_add_dse_entry(struct ldbminfo *li, char *entry, int flags) +{ + char *entries[] = { "%s", "" }; + + return ldbm_config_add_dse_entries(li, entries, entry, NULL, NULL, flags); +} + +/* Finds an entry in a config_info array with the given name. Returns + * the entry on success and NULL when not found. + */ +config_info *get_config_info(config_info *config_array, char *attr_name) +{ + int x; + + for(x = 0; config_array[x].config_name != NULL; x++) { + if (!strcasecmp(config_array[x].config_name, attr_name)) { + return &(config_array[x]); + } + } + return NULL; +} + +/*------------------------------------------------------------------------ + * Get and set functions for ldbm and dblayer variables + *----------------------------------------------------------------------*/ +static void *ldbm_config_lookthroughlimit_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) (li->li_lookthroughlimit); +} + +static int ldbm_config_lookthroughlimit_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + /* Do whatever we can to make sure the data is ok. */ + + if (apply) { + li->li_lookthroughlimit = val; + } + + return retval; +} + +static void *ldbm_config_mode_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) (li->li_mode); +} + +static int ldbm_config_mode_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + /* Do whatever we can to make sure the data is ok. */ + + if (apply) { + li->li_mode = val; + } + + return retval; +} + +static void *ldbm_config_allidsthreshold_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) (li->li_allidsthreshold); +} + +static int ldbm_config_allidsthreshold_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + /* Do whatever we can to make sure the data is ok. */ + + /* Catch attempts to configure a stupidly low allidsthreshold */ + if (val < 100) { + val = 100; + } + + if (apply) { + li->li_allidsthreshold = val; + } + + return retval; +} + +static void *ldbm_config_directory_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + /* Remember get functions of type string need to return + * alloced memory. */ + return (void *) slapi_ch_strdup(li->li_new_directory); +} + +static int ldbm_config_directory_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + char *val = (char *) value; + char tmpbuf[BUFSIZ]; + + errorbuf[0] = '\0'; + + if (!apply) { + /* we should really do some error checking here. */ + return retval; + } + + if (CONFIG_PHASE_RUNNING == phase) { + slapi_ch_free((void **) &(li->li_new_directory)); + li->li_new_directory = slapi_ch_strdup(val); + LDAPDebug(LDAP_DEBUG_ANY, "New db directory location will not take affect until the server is restarted\n", 0, 0, 0); + } else { + if (!strcmp(val, "get default")) { + /* Generate the default db directory name. The default db directory + * should be the instance directory with a '/db' thrown on the end. + * We need to read cn=config to get the instance dir. */ + /* We use this funky "get default" string for the caller to + * tell us that it has no idea what the db directory should + * be. This code figures it out be reading cn=config. */ + + Slapi_PBlock *search_pb; + Slapi_Entry **entries = NULL; + Slapi_Attr *attr = NULL; + Slapi_Value *v = NULL; + const char *s = NULL; + int res; + + search_pb = slapi_pblock_new(); + slapi_search_internal_set_pb(search_pb, "cn=config", LDAP_SCOPE_BASE, + "objectclass=*", NULL, 0, NULL, NULL, li->li_identity, 0); + slapi_search_internal_pb(search_pb); + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_RESULT, &res); + + if (res != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "ERROR: ldbm plugin unable to read cn=config\n", + 0, 0, 0); + goto done; + } + + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries); + if (NULL == entries) { + LDAPDebug(LDAP_DEBUG_ANY, "ERROR: ldbm plugin unable to read cn=config\n", + 0, 0, 0); + res = LDAP_OPERATIONS_ERROR; + goto done; + } + + res = slapi_entry_attr_find(entries[0], "nsslapd-instancedir", &attr); + if (res != 0 || attr == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR: ldbm plugin unable to read attribute nsslapd-instancedir from cn=config\n", + 0, 0, 0); + res = LDAP_OPERATIONS_ERROR; + goto done; + } + + if ( slapi_attr_first_value(attr,&v) != 0 + || ( NULL == v ) + || ( NULL == ( s = slapi_value_get_string( v )))) { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR: ldbm plugin unable to read attribute nsslapd-instancedir from cn=config\n", + 0, 0, 0); + res = LDAP_OPERATIONS_ERROR; + goto done; + } + +done: + slapi_pblock_destroy(search_pb); + if (res != LDAP_SUCCESS) { + return res; + } + sprintf(tmpbuf, "%s/db", s ); + val = tmpbuf; + } + slapi_ch_free((void **) &(li->li_new_directory)); + slapi_ch_free((void **) &(li->li_directory)); + li->li_new_directory = slapi_ch_strdup(val); + li->li_directory = slapi_ch_strdup(val); + } + + return retval; +} + +static void *ldbm_config_dbcachesize_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) (li->li_new_dbcachesize); +} + +static int ldbm_config_dbcachesize_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + size_t val = (size_t) value; + + if (apply) { + /* Stop the user configuring a stupidly small cache */ + /* min: 8KB (page size) * def thrd cnts (threadnumber==20). */ +#define DBDEFMINSIZ 500000 + if (val < DBDEFMINSIZ) { + LDAPDebug( LDAP_DEBUG_ANY,"WARNING: cache too small, increasing to %dK bytes\n", DBDEFMINSIZ/1000, 0, 0); + val = DBDEFMINSIZ; + } + + if (CONFIG_PHASE_RUNNING == phase) { + li->li_new_dbcachesize = val; + LDAPDebug(LDAP_DEBUG_ANY, "New db cache size will not take affect until the server is restarted\n", 0, 0, 0); + } else { + li->li_new_dbcachesize = val; + li->li_dbcachesize = val; + } + + } + + return retval; +} + +static void *ldbm_config_maxpassbeforemerge_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) (li->li_maxpassbeforemerge); +} + +static int ldbm_config_maxpassbeforemerge_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + if (val < 0) { + LDAPDebug( LDAP_DEBUG_ANY,"WARNING: maxpassbeforemerge will not take negative value\n", 0, 0, 0); + val = 100; + } + + li->li_maxpassbeforemerge = val; + } + + return retval; +} + + +static void *ldbm_config_dbncache_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) (li->li_new_dbncache); +} + +static int ldbm_config_dbncache_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + if (val < 0) { + LDAPDebug( LDAP_DEBUG_ANY,"WARNING: ncache will not take negative value\n", 0, 0, 0); + val = 0; + } + + if (CONFIG_PHASE_RUNNING == phase) { + li->li_new_dbncache = val; + LDAPDebug(LDAP_DEBUG_ANY, "New db ncache will not take affect until the server is restarted\n", 0, 0, 0); + } else { + li->li_new_dbncache = val; + li->li_dbncache = val; + } + + } + + return retval; +} + +static void *ldbm_config_db_logdirectory_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + /* Remember get functions of type string need to return + * alloced memory. */ + /* if dblayer_log_directory is set to a string different from "" + * then it has been set, return this variable + * otherwise it is set to default, use the instance home directory + */ + if (strlen(li->li_dblayer_private->dblayer_log_directory) > 0) + return (void *) slapi_ch_strdup(li->li_dblayer_private->dblayer_log_directory); + else + return (void *) slapi_ch_strdup(li->li_new_directory); + +} + +static int ldbm_config_db_logdirectory_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + char *val = (char *) value; + + if (apply) { + slapi_ch_free((void **) &(li->li_dblayer_private->dblayer_log_directory)); + li->li_dblayer_private->dblayer_log_directory = slapi_ch_strdup(val); + } + + return retval; +} + +static void *ldbm_config_db_durable_transactions_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_durable_transactions; +} + +static int ldbm_config_db_durable_transactions_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_durable_transactions = val; + } + + return retval; +} + +static void *ldbm_config_db_lockdown_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_lockdown; +} + +static int ldbm_config_db_lockdown_set( + void *arg, + void *value, + char *errorbuf, + int phase, + int apply +) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_lockdown = val; + } + + return retval; +} + +static void *ldbm_config_db_circular_logging_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_circular_logging; +} + +static int ldbm_config_db_circular_logging_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_circular_logging = val; + } + + return retval; +} + +static void *ldbm_config_db_transaction_logging_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_enable_transactions; +} + +static int ldbm_config_db_transaction_logging_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_enable_transactions = val; + } + + return retval; +} + +static void *ldbm_config_db_logbuf_size_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_logbuf_size; +} + +static int ldbm_config_db_logbuf_size_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + size_t val = (size_t) value; + + if (apply) { + li->li_dblayer_private->dblayer_logbuf_size = val; + } + + return retval; +} + +static void *ldbm_config_db_checkpoint_interval_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_checkpoint_interval; +} + +static int ldbm_config_db_checkpoint_interval_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_checkpoint_interval = val; + } + + return retval; +} + +static void *ldbm_config_db_page_size_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_page_size; +} + +static int ldbm_config_db_page_size_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + size_t val = (size_t) value; + + if (apply) { + li->li_dblayer_private->dblayer_page_size = val; + } + + return retval; +} + +static void *ldbm_config_db_index_page_size_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_index_page_size; +} + +static int ldbm_config_db_index_page_size_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + size_t val = (size_t) value; + + if (apply) { + li->li_dblayer_private->dblayer_index_page_size = val; + } + + return retval; +} + +static void *ldbm_config_db_idl_divisor_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_idl_divisor; +} + +static int ldbm_config_db_idl_divisor_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_idl_divisor = val; + } + + return retval; +} + +static void *ldbm_config_db_logfile_size_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_logfile_size; +} + +static int ldbm_config_db_logfile_size_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + size_t val = (size_t) value; + + if (apply) { + li->li_dblayer_private->dblayer_logfile_size = val; + } + + return retval; +} + +static void *ldbm_config_db_spin_count_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_spin_count; +} + +static int ldbm_config_db_spin_count_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_spin_count = val; + } + + return retval; +} + +static void *ldbm_config_db_trickle_percentage_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_trickle_percentage; +} + +static int ldbm_config_db_trickle_percentage_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (val < 0 || val > 100) { + sprintf(errorbuf, "Error: Invalid value for %s (%d). Must be between 0 and 100\n", CONFIG_DB_TRICKLE_PERCENTAGE, val); + LDAPDebug(LDAP_DEBUG_ANY, "%s", errorbuf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + } + + if (apply) { + li->li_dblayer_private->dblayer_trickle_percentage = val; + } + + return retval; +} + +static void *ldbm_config_db_verbose_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_verbose; +} + +static int ldbm_config_db_verbose_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_verbose = val; + } + + return retval; +} + +static void *ldbm_config_db_debug_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_debug; +} + +static int ldbm_config_db_debug_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_debug = val; + } + + return retval; +} + +static void *ldbm_config_db_named_regions_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_named_regions; +} + +static int ldbm_config_db_named_regions_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_named_regions = val; + } + + return retval; +} + +static void *ldbm_config_db_private_mem_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_private_mem; +} + +static int ldbm_config_db_private_mem_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_private_mem = val; + } + + return retval; +} + +static void *ldbm_config_db_private_import_mem_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_private_import_mem; +} + +static int ldbm_config_db_private_import_mem_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_private_import_mem = val; + } + + return retval; +} + +static void *ldbm_config_db_shm_key_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_shm_key; +} + +static int ldbm_config_db_shm_key_set( + void *arg, + void *value, + char *errorbuf, + int phase, + int apply +) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_shm_key = val; + } + + return retval; +} + +static void *ldbm_config_db_lock_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_lock_config; +} + + +static int ldbm_config_db_lock_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + size_t val = (size_t) value; + + if (apply) { + if (CONFIG_PHASE_RUNNING == phase) { + li->li_dblayer_private->dblayer_lock_config = val; + LDAPDebug(LDAP_DEBUG_ANY, "New db cache size will not take affect until the server is restarted\n", 0, 0, 0); + } else { + li->li_dblayer_private->dblayer_lock_config = val; + } + + } + + return retval; +} +static void *ldbm_config_db_cache_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_cache_config; +} + +static int ldbm_config_db_cache_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_cache_config = val; + } + + return retval; +} + +static void *ldbm_config_db_debug_checkpointing_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->db_debug_checkpointing; +} + +static int ldbm_config_db_debug_checkpointing_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->db_debug_checkpointing = val; + } + + return retval; +} + +static void *ldbm_config_db_home_directory_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + /* Remember get functions of type string need to return + * alloced memory. */ + return (void *) slapi_ch_strdup(li->li_dblayer_private->dblayer_dbhome_directory); +} + +static int ldbm_config_db_home_directory_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + char *val = (char *) value; + + if (apply) { + slapi_ch_free((void **) &(li->li_dblayer_private->dblayer_dbhome_directory)); + li->li_dblayer_private->dblayer_dbhome_directory = slapi_ch_strdup(val); + } + + return retval; +} + +static void *ldbm_config_import_cache_autosize_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + return (void *)(li->li_import_cache_autosize); +} + +static int ldbm_config_import_cache_autosize_set(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + if (apply) + li->li_import_cache_autosize = (int)value; + return LDAP_SUCCESS; +} + +static void *ldbm_config_cache_autosize_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + return (void *)(li->li_cache_autosize); +} + +static int ldbm_config_cache_autosize_set(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + if (apply) + li->li_cache_autosize = (int)value; + return LDAP_SUCCESS; +} + +static void *ldbm_config_cache_autosize_split_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + return (void *)(li->li_cache_autosize_split); +} + +static int ldbm_config_cache_autosize_split_set(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + if (apply) + li->li_cache_autosize_split = (int)value; + return LDAP_SUCCESS; +} + +static void *ldbm_config_import_cachesize_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + return (void *)(li->li_import_cachesize); +} + +static int ldbm_config_import_cachesize_set(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + if (apply) + li->li_import_cachesize = (size_t)value; + return LDAP_SUCCESS; +} + +static void *ldbm_config_index_buffer_size_get(void *arg) +{ + return (void *)import_get_index_buffer_size(); +} + +static int ldbm_config_index_buffer_size_set(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + if (apply) + import_configure_index_buffer_size((size_t)value); + return LDAP_SUCCESS; +} + +static void *ldbm_config_idl_get_idl_new(void *arg) +{ + if (idl_get_idl_new()) + return slapi_ch_strdup("new"); + else + return slapi_ch_strdup("old"); +} + +static int ldbm_config_idl_set_tune(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + if (!strcasecmp("new", value)) + idl_set_tune(4096); + else + idl_set_tune(0); + return LDAP_SUCCESS; +} + +static void *ldbm_config_serial_lock_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_fat_lock; +} + +static int ldbm_config_serial_lock_set(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + if (apply) { + li->li_fat_lock = (int) value; + } + + return LDAP_SUCCESS; +} + +static void *ldbm_config_legacy_errcode_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_legacy_errcode; +} + +static int ldbm_config_legacy_errcode_set(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + if (apply) { + li->li_legacy_errcode = (int) value; + } + + return LDAP_SUCCESS; +} + +static int +ldbm_config_set_bypass_filter_test(void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + if (apply) { + char *myvalue = (char *)value; + + if (0 == strcasecmp(myvalue, "on")) { + li->li_filter_bypass = 1; + li->li_filter_bypass_check = 0; + } else if (0 == strcasecmp(myvalue, "verify")) { + li->li_filter_bypass = 1; + li->li_filter_bypass_check = 1; + } else { + li->li_filter_bypass = 0; + li->li_filter_bypass_check = 0; + } + } + return LDAP_SUCCESS; +} + +static void *ldbm_config_get_bypass_filter_test(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + char *retstr = NULL; + + if (li->li_filter_bypass) { + if (li->li_filter_bypass_check) { + /* meaningful only if is bypass filter test called */ + retstr = slapi_ch_strdup("verify"); + } else { + retstr = slapi_ch_strdup("on"); + } + } else { + retstr = slapi_ch_strdup("off"); + } + return (void *)retstr; +} + +static int ldbm_config_set_use_vlv_index(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int val = (int) value; + + if (apply) { + int setval = 0; + if (val) { + li->li_use_vlv = 1; + } else { + li->li_use_vlv = 0; + } + } + return LDAP_SUCCESS; +} + +static void *ldbm_config_get_use_vlv_index(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_use_vlv; +} + +static int +ldbm_config_exclude_from_export_set( void *arg, void *value, char *errorbuf, + int phase, int apply) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + + if ( apply ) { + if ( NULL != li->li_attrs_to_exclude_from_export ) { + charray_free( li->li_attrs_to_exclude_from_export ); + li->li_attrs_to_exclude_from_export = NULL; + } + + if ( NULL != value ) { + char *dupvalue = slapi_ch_strdup( value ); + li->li_attrs_to_exclude_from_export = str2charray( dupvalue, " " ); + slapi_ch_free((void**)&dupvalue); + } + } + + return LDAP_SUCCESS; +} + +static void * +ldbm_config_exclude_from_export_get( void *arg ) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + char *p, *retstr = NULL; + size_t len = 0; + + if ( NULL != li->li_attrs_to_exclude_from_export && + NULL != li->li_attrs_to_exclude_from_export[0] ) { + int i; + + for ( i = 0; li->li_attrs_to_exclude_from_export[i] != NULL; ++i ) { + len += strlen( li->li_attrs_to_exclude_from_export[i] ) + 1; + } + p = retstr = slapi_ch_malloc( len ); + for ( i = 0; li->li_attrs_to_exclude_from_export[i] != NULL; ++i ) { + if ( i > 0 ) { + *p++ = ' '; + } + strcpy( p, li->li_attrs_to_exclude_from_export[i] ); + p += strlen( p ); + } + *p = '\0'; + } else { + retstr = slapi_ch_strdup( "" ); + } + + return (void *)retstr; +} + +static void *ldbm_config_db_tx_max_get(void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + + return (void *) li->li_dblayer_private->dblayer_tx_max; +} + +static int ldbm_config_db_tx_max_set( + void *arg, + void *value, + char *errorbuf, + int phase, + int apply +) +{ + struct ldbminfo *li = (struct ldbminfo *) arg; + int retval = LDAP_SUCCESS; + int val = (int) value; + + if (apply) { + li->li_dblayer_private->dblayer_tx_max = val; + } + + return retval; +} + + +/*------------------------------------------------------------------------ + * Configuration array for ldbm and dblayer variables + *----------------------------------------------------------------------*/ +static config_info ldbm_config[] = { + {CONFIG_LOOKTHROUGHLIMIT, CONFIG_TYPE_INT, "5000", &ldbm_config_lookthroughlimit_get, &ldbm_config_lookthroughlimit_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_MODE, CONFIG_TYPE_INT_OCTAL, "0600", &ldbm_config_mode_get, &ldbm_config_mode_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_IDLISTSCANLIMIT, CONFIG_TYPE_INT, "4000", &ldbm_config_allidsthreshold_get, &ldbm_config_allidsthreshold_set, CONFIG_FLAG_ALWAYS_SHOW}, + {CONFIG_DIRECTORY, CONFIG_TYPE_STRING, "", &ldbm_config_directory_get, &ldbm_config_directory_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DBCACHESIZE, CONFIG_TYPE_SIZE_T, "10000000", &ldbm_config_dbcachesize_get, &ldbm_config_dbcachesize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DBNCACHE, CONFIG_TYPE_INT, "0", &ldbm_config_dbncache_get, &ldbm_config_dbncache_set, CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_MAXPASSBEFOREMERGE, CONFIG_TYPE_INT, "100", &ldbm_config_maxpassbeforemerge_get, &ldbm_config_maxpassbeforemerge_set, 0}, + + /* dblayer config attributes */ + {CONFIG_DB_LOGDIRECTORY, CONFIG_TYPE_STRING, "", &ldbm_config_db_logdirectory_get, &ldbm_config_db_logdirectory_set, CONFIG_FLAG_ALWAYS_SHOW}, + {CONFIG_DB_DURABLE_TRANSACTIONS, CONFIG_TYPE_ONOFF, "on", &ldbm_config_db_durable_transactions_get, &ldbm_config_db_durable_transactions_set, CONFIG_FLAG_ALWAYS_SHOW}, + {CONFIG_DB_CIRCULAR_LOGGING, CONFIG_TYPE_ONOFF, "on", &ldbm_config_db_circular_logging_get, &ldbm_config_db_circular_logging_set, 0}, + {CONFIG_DB_TRANSACTION_LOGGING, CONFIG_TYPE_ONOFF, "on", &ldbm_config_db_transaction_logging_get, &ldbm_config_db_transaction_logging_set, CONFIG_FLAG_ALWAYS_SHOW}, + {CONFIG_DB_CHECKPOINT_INTERVAL, CONFIG_TYPE_INT, "60", &ldbm_config_db_checkpoint_interval_get, &ldbm_config_db_checkpoint_interval_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DB_TRANSACTION_BATCH, CONFIG_TYPE_INT, "0", &dblayer_get_batch_transactions, &dblayer_set_batch_transactions, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DB_LOGBUF_SIZE, CONFIG_TYPE_SIZE_T, "0", &ldbm_config_db_logbuf_size_get, &ldbm_config_db_logbuf_size_set, CONFIG_FLAG_ALWAYS_SHOW}, + {CONFIG_DB_PAGE_SIZE, CONFIG_TYPE_SIZE_T, "0", &ldbm_config_db_page_size_get, &ldbm_config_db_page_size_set, 0}, + {CONFIG_DB_INDEX_PAGE_SIZE, CONFIG_TYPE_SIZE_T, "0", &ldbm_config_db_index_page_size_get, &ldbm_config_db_index_page_size_set, 0}, + {CONFIG_DB_IDL_DIVISOR, CONFIG_TYPE_INT, "0", &ldbm_config_db_idl_divisor_get, &ldbm_config_db_idl_divisor_set, 0}, + {CONFIG_DB_LOGFILE_SIZE, CONFIG_TYPE_SIZE_T, "0", &ldbm_config_db_logfile_size_get, &ldbm_config_db_logfile_size_set, 0}, + {CONFIG_DB_TRICKLE_PERCENTAGE, CONFIG_TYPE_INT, "5", &ldbm_config_db_trickle_percentage_get, &ldbm_config_db_trickle_percentage_set, 0}, + {CONFIG_DB_SPIN_COUNT, CONFIG_TYPE_INT, "0", &ldbm_config_db_spin_count_get, &ldbm_config_db_spin_count_set, 0}, + {CONFIG_DB_VERBOSE, CONFIG_TYPE_ONOFF, "off", &ldbm_config_db_verbose_get, &ldbm_config_db_verbose_set, 0}, + {CONFIG_DB_DEBUG, CONFIG_TYPE_ONOFF, "on", &ldbm_config_db_debug_get, &ldbm_config_db_debug_set, 0}, + {CONFIG_DB_NAMED_REGIONS, CONFIG_TYPE_ONOFF, "off", &ldbm_config_db_named_regions_get, &ldbm_config_db_named_regions_set, 0}, + {CONFIG_DB_LOCK, CONFIG_TYPE_INT, "10000", &ldbm_config_db_lock_get, &ldbm_config_db_lock_set, 0}, + {CONFIG_DB_PRIVATE_MEM, CONFIG_TYPE_ONOFF, "off", &ldbm_config_db_private_mem_get, &ldbm_config_db_private_mem_set, 0}, + {CONFIG_DB_PRIVATE_IMPORT_MEM, CONFIG_TYPE_ONOFF, "on", &ldbm_config_db_private_import_mem_get, &ldbm_config_db_private_import_mem_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DB_SHM_KEY, CONFIG_TYPE_LONG, "389389", &ldbm_config_db_shm_key_get, &ldbm_config_db_shm_key_set, 0}, + {CONFIG_DB_CACHE, CONFIG_TYPE_INT, "0", &ldbm_config_db_cache_get, &ldbm_config_db_cache_set, 0}, + {CONFIG_DB_DEBUG_CHECKPOINTING, CONFIG_TYPE_ONOFF, "off", &ldbm_config_db_debug_checkpointing_get, &ldbm_config_db_debug_checkpointing_set, 0}, + {CONFIG_DB_HOME_DIRECTORY, CONFIG_TYPE_STRING, "", &ldbm_config_db_home_directory_get, &ldbm_config_db_home_directory_set, 0}, + {CONFIG_IMPORT_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "-1", &ldbm_config_import_cache_autosize_get, &ldbm_config_import_cache_autosize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_CACHE_AUTOSIZE, CONFIG_TYPE_INT, "0", &ldbm_config_cache_autosize_get, &ldbm_config_cache_autosize_set, 0}, + {CONFIG_CACHE_AUTOSIZE_SPLIT, CONFIG_TYPE_INT, "50", &ldbm_config_cache_autosize_split_get, &ldbm_config_cache_autosize_split_set, 0}, + {CONFIG_IMPORT_CACHESIZE, CONFIG_TYPE_SIZE_T, "20000000", &ldbm_config_import_cachesize_get, &ldbm_config_import_cachesize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, +#if defined(USE_NEW_IDL) + {CONFIG_IDL_SWITCH, CONFIG_TYPE_STRING, "new", &ldbm_config_idl_get_idl_new, &ldbm_config_idl_set_tune, CONFIG_FLAG_ALWAYS_SHOW}, +#else + {CONFIG_IDL_SWITCH, CONFIG_TYPE_STRING, "old", &ldbm_config_idl_get_idl_new, &ldbm_config_idl_set_tune, CONFIG_FLAG_ALWAYS_SHOW}, +#endif + {CONFIG_BYPASS_FILTER_TEST, CONFIG_TYPE_STRING, "on", &ldbm_config_get_bypass_filter_test, &ldbm_config_set_bypass_filter_test, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_USE_VLV_INDEX, CONFIG_TYPE_ONOFF, "on", &ldbm_config_get_use_vlv_index, &ldbm_config_set_use_vlv_index, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_DB_LOCKDOWN, CONFIG_TYPE_ONOFF, "off", &ldbm_config_db_lockdown_get, &ldbm_config_db_lockdown_set, 0}, + {CONFIG_INDEX_BUFFER_SIZE, CONFIG_TYPE_INT, "0", &ldbm_config_index_buffer_size_get, &ldbm_config_index_buffer_size_set, 0}, + {CONFIG_EXCLUDE_FROM_EXPORT, CONFIG_TYPE_STRING, + CONFIG_EXCLUDE_FROM_EXPORT_DEFAULT_VALUE, + &ldbm_config_exclude_from_export_get, + &ldbm_config_exclude_from_export_set, CONFIG_FLAG_ALWAYS_SHOW}, + {CONFIG_DB_TX_MAX, CONFIG_TYPE_INT, "200", &ldbm_config_db_tx_max_get, &ldbm_config_db_tx_max_set, 0}, + {CONFIG_SERIAL_LOCK, CONFIG_TYPE_ONOFF, "on", &ldbm_config_serial_lock_get, &ldbm_config_serial_lock_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_USE_LEGACY_ERRORCODE, CONFIG_TYPE_ONOFF, "off", &ldbm_config_legacy_errcode_get, &ldbm_config_legacy_errcode_set, 0}, + {NULL, 0, NULL, NULL, NULL, 0} +}; + +void ldbm_config_setup_default(struct ldbminfo *li) +{ + config_info *config; + char err_buf[BUFSIZ]; + + for (config = ldbm_config; config->config_name != NULL; config++) { + ldbm_config_set((void *)li, config->config_name, ldbm_config, NULL /* use default */, err_buf, CONFIG_PHASE_INITIALIZATION, 1 /* apply */); + } +} + +void +ldbm_config_read_instance_entries(struct ldbminfo *li, const char *backend_type) +{ + Slapi_PBlock *tmp_pb; + char basedn[BUFSIZ]; + Slapi_Entry **entries = NULL; + + /* Construct the base dn of the subtree that holds the instance entries. */ + sprintf(basedn, "cn=%s, cn=plugins, cn=config", backend_type); + + /* Do a search of the subtree containing the instance entries */ + tmp_pb = slapi_pblock_new(); + slapi_search_internal_set_pb(tmp_pb, basedn, LDAP_SCOPE_SUBTREE, "(objectclass=nsBackendInstance)", NULL, 0, NULL, NULL, li->li_identity, 0); + slapi_search_internal_pb (tmp_pb); + slapi_pblock_get(tmp_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries); + if (entries!=NULL) { + int i; + for (i=0; entries[i]!=NULL; i++) { + ldbm_instance_add_instance_entry_callback(NULL, entries[i], NULL, NULL, NULL, li); + } + } + + slapi_free_search_results_internal(tmp_pb); + slapi_pblock_destroy(tmp_pb); +} + +/* Reads in any config information held in the dse for the ldbm plugin. + * Creates dse entries used to configure the ldbm plugin and dblayer + * if they don't already exist. Registers dse callback functions to + * maintain those dse entries. Returns 0 on success. + */ +int ldbm_config_load_dse_info(struct ldbminfo *li) +{ + Slapi_PBlock *search_pb; + Slapi_Entry **entries = NULL; + int res; + char dn[BUFSIZ]; + + /* We try to read the entry + * cn=config, cn=ldbm database, cn=plugins, cn=config. If the entry is + * there, then we process the config information it stores. + */ + sprintf(dn, "cn=config, cn=%s, cn=plugins, cn=config", + li->li_plugin->plg_name); + search_pb = slapi_pblock_new(); + slapi_search_internal_set_pb(search_pb, dn, LDAP_SCOPE_BASE, + "objectclass=*", NULL, 0, NULL, NULL, li->li_identity, 0); + slapi_search_internal_pb (search_pb); + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_RESULT, &res); + + if (LDAP_NO_SUCH_OBJECT == res) { + /* Add skeleten dse entries for the ldbm plugin */ + ldbm_config_add_dse_entries(li, ldbm_skeleton_entries, + li->li_plugin->plg_name, NULL, NULL, 0); + } else if (res != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "Error accessing the ldbm config DSE\n", + 0, 0, 0); + return 1; + } else { + /* Need to parse the configuration information for the ldbm + * plugin that is held in the DSE. */ + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, + &entries); + if (NULL == entries || entries[0] == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "Error accessing the ldbm config DSE\n", + 0, 0, 0); + return 1; + } + parse_ldbm_config_entry(li, entries[0], ldbm_config); + } + + if (search_pb) { + slapi_free_search_results_internal(search_pb); + slapi_pblock_destroy(search_pb); + } + + /* Find all the instance entries and create a Slapi_Backend and an + * ldbm_instance for each */ + ldbm_config_read_instance_entries(li, li->li_plugin->plg_name); + + /* setup the dse callback functions for the ldbm backend config entry */ + sprintf(dn, "cn=config, cn=%s, cn=plugins, cn=config", + li->li_plugin->plg_name); + slapi_config_register_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_config_search_entry_callback, + (void *) li); + slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_config_modify_entry_callback, + (void *) li); + slapi_config_register_callback(DSE_OPERATION_WRITE, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_config_search_entry_callback, + (void *) li); + + /* setup the dse callback functions for the ldbm backend monitor entry */ + sprintf(dn, "cn=monitor, cn=%s, cn=plugins, cn=config", + li->li_plugin->plg_name); + slapi_config_register_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_back_monitor_search, + (void *)li); + + /* And the ldbm backend database monitor entry */ + sprintf(dn, "cn=database, cn=monitor, cn=%s, cn=plugins, cn=config", + li->li_plugin->plg_name); + slapi_config_register_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_back_dbmonitor_search, + (void *)li); + + /* setup the dse callback functions for the ldbm backend instance + * entries */ + sprintf(dn, "cn=%s, cn=plugins, cn=config", li->li_plugin->plg_name); + slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsBackendInstance)", + ldbm_instance_add_instance_entry_callback, (void *) li); + slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_POSTOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsBackendInstance)", + ldbm_instance_postadd_instance_entry_callback, (void *) li); + slapi_config_register_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsBackendInstance)", + ldbm_instance_delete_instance_entry_callback, (void *) li); + slapi_config_register_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_POSTOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsBackendInstance)", + ldbm_instance_post_delete_instance_entry_callback, (void *) li); + + return 0; +} + + +/* Utility function used in creating config entries. Using the + * config_info, this function gets info and formats in the correct + * way. + */ +void ldbm_config_get(void *arg, config_info *config, char *buf) +{ + char *tmp_string; + + if (config == NULL) { + buf[0] = '\0'; + } + + switch(config->config_type) { + case CONFIG_TYPE_INT: + sprintf(buf, "%d", (int) config->config_get_fn(arg)); + break; + case CONFIG_TYPE_INT_OCTAL: + sprintf(buf, "%o", (int) config->config_get_fn(arg)); + break; + case CONFIG_TYPE_LONG: + sprintf(buf, "%ld", (long) config->config_get_fn(arg)); + break; + case CONFIG_TYPE_SIZE_T: + sprintf(buf, "%lu", (size_t) config->config_get_fn(arg)); + break; + case CONFIG_TYPE_STRING: + /* Remember the get function for strings returns memory + * that must be freed. */ + tmp_string = (char *) config->config_get_fn(arg); + sprintf(buf, "%s", (char *) tmp_string); + slapi_ch_free((void **)&tmp_string); + break; + case CONFIG_TYPE_ONOFF: + if ((int) config->config_get_fn(arg)) { + sprintf(buf, "on"); + } else { + sprintf(buf, "off"); + } + break; + } +} + +/* + * Returns: + * SLAPI_DSE_CALLBACK_ERROR on failure + * SLAPI_DSE_CALLBACK_OK on success + */ +int ldbm_config_search_entry_callback(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + char buf[BUFSIZ]; + struct berval *vals[2]; + struct berval val; + struct ldbminfo *li= (struct ldbminfo *) arg; + config_info *config; + + vals[0] = &val; + vals[1] = NULL; + + returntext[0] = '\0'; + + PR_Lock(li->li_config_mutex); + + for(config = ldbm_config; config->config_name != NULL; config++) { + /* Go through the ldbm_config table and fill in the entry. */ + + if (!(config->config_flags & (CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_PREVIOUSLY_SET))) { + /* This config option shouldn't be shown */ + continue; + } + + ldbm_config_get((void *) li, config, buf); + + val.bv_val = buf; + val.bv_len = strlen(buf); + slapi_entry_attr_replace(e, config->config_name, vals); + } + + PR_Unlock(li->li_config_mutex); + + *returncode = LDAP_SUCCESS; + return SLAPI_DSE_CALLBACK_OK; +} + + +int ldbm_config_ignored_attr(char *attr_name) +{ + /* These are the names of attributes that are in the + * config entries but are not config attributes. */ + if (!strcasecmp("objectclass", attr_name) || + !strcasecmp("cn", attr_name) || + !strcasecmp("creatorsname", attr_name) || + !strcasecmp("modifiersname", attr_name) || + !strcasecmp("createtimestamp", attr_name) || + !strcasecmp("numsubordinates", attr_name) || + !strcasecmp("modifytimestamp", attr_name)) { + return 1; + } else { + return 0; + } +} + +/* Returns LDAP_SUCCESS on success */ +int ldbm_config_set(void *arg, char *attr_name, config_info *config_array, struct berval *bval, char *err_buf, int phase, int apply_mod) +{ + config_info *config; + int use_default; + int int_val; + long long_val; + size_t sz_val; + PRInt64 llval; + int maxint = (int)(((unsigned int)~0)>>1); + int minint = ~maxint; + PRInt64 llmaxint; + PRInt64 llminint; + int err = 0; + char *str_val; + int retval = 0; + + LL_I2L(llmaxint, maxint); + LL_I2L(llminint, minint); + + config = get_config_info(config_array, attr_name); + if (NULL == config) { + LDAPDebug(LDAP_DEBUG_CONFIG, "Unknown config attribute %s\n", attr_name, 0, 0); + sprintf(err_buf, "Unknown config attribute %s\n", attr_name); + return LDAP_SUCCESS; /* Ignore unknown attributes */ + } + + /* Some config attrs can't be changed while the server is running. */ + if (phase == CONFIG_PHASE_RUNNING && + !(config->config_flags & CONFIG_FLAG_ALLOW_RUNNING_CHANGE)) { + sprintf(err_buf, "%s can't be modified while the server is running.\n", attr_name); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + } + + /* If the config phase is initialization or if bval is NULL, we will use + * the default value for the attribute. */ + if (CONFIG_PHASE_INITIALIZATION == phase || NULL == bval) { + use_default = 1; + } else { + use_default = 0; + + /* Since we are setting the value for the config attribute, we + * need to turn on the CONFIG_FLAG_PREVIOUSLY_SET flag to make + * sure this attribute is shown. */ + config->config_flags |= CONFIG_FLAG_PREVIOUSLY_SET; + } + + switch(config->config_type) { + case CONFIG_TYPE_INT: + if (use_default) { + str_val = config->config_default_value; + } else { + str_val = bval->bv_val; + } + /* get the value as a 64 bit value */ + llval = db_atoi(str_val, &err); + /* check for parsing error (e.g. not a number) */ + if (err) { + sprintf(err_buf, "Error: value %s for attr %s is not a number\n", + str_val, attr_name); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + /* check for overflow */ + } else if (LL_CMP(llval, >, llmaxint)) { + sprintf(err_buf, "Error: value %s for attr %s is greater than the maximum %d\n", + str_val, attr_name, maxint); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + /* check for underflow */ + } else if (LL_CMP(llval, <, llminint)) { + sprintf(err_buf, "Error: value %s for attr %s is less than the minimum %d\n", + str_val, attr_name, minint); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + } + /* convert 64 bit value to 32 bit value */ + LL_L2I(int_val, llval); + retval = config->config_set_fn(arg, (void *) int_val, err_buf, phase, apply_mod); + break; + case CONFIG_TYPE_INT_OCTAL: + if (use_default) { + int_val = (int) strtol(config->config_default_value, NULL, 8); + } else { + int_val = (int) strtol((char *)bval->bv_val, NULL, 8); + } + retval = config->config_set_fn(arg, (void *) int_val, err_buf, phase, apply_mod); + break; + case CONFIG_TYPE_LONG: + if (use_default) { + str_val = config->config_default_value; + } else { + str_val = bval->bv_val; + } + /* get the value as a 64 bit value */ + llval = db_atoi(str_val, &err); + /* check for parsing error (e.g. not a number) */ + if (err) { + sprintf(err_buf, "Error: value %s for attr %s is not a number\n", + str_val, attr_name); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + /* check for overflow */ + } else if (LL_CMP(llval, >, llmaxint)) { + sprintf(err_buf, "Error: value %s for attr %s is greater than the maximum %d\n", + str_val, attr_name, maxint); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + /* check for underflow */ + } else if (LL_CMP(llval, <, llminint)) { + sprintf(err_buf, "Error: value %s for attr %s is less than the minimum %d\n", + str_val, attr_name, minint); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + } + /* convert 64 bit value to 32 bit value */ + LL_L2I(long_val, llval); + retval = config->config_set_fn(arg, (void *) long_val, err_buf, phase, apply_mod); + break; + case CONFIG_TYPE_SIZE_T: + if (use_default) { + str_val = config->config_default_value; + } else { + str_val = bval->bv_val; + } + + /* get the value as a size_t value */ + sz_val = db_strtoul(str_val, &err); + + /* check for parsing error (e.g. not a number) */ + if (err == EINVAL) { + sprintf(err_buf, "Error: value %s for attr %s is not a number\n", + str_val, attr_name); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + /* check for overflow */ + } else if (err == ERANGE) { + sprintf(err_buf, "Error: value %s for attr %s is outside the range of representable values\n", + str_val, attr_name); + LDAPDebug(LDAP_DEBUG_ANY, "%s", err_buf, 0, 0); + return LDAP_UNWILLING_TO_PERFORM; + } + retval = config->config_set_fn(arg, (void *) sz_val, err_buf, phase, apply_mod); + break; + case CONFIG_TYPE_STRING: + if (use_default) { + retval = config->config_set_fn(arg, config->config_default_value, err_buf, phase, apply_mod); + } else { + retval = config->config_set_fn(arg, bval->bv_val, err_buf, phase, apply_mod); + } + break; + case CONFIG_TYPE_ONOFF: + if (use_default) { + int_val = !strcasecmp(config->config_default_value, "on"); + } else { + int_val = !strcasecmp((char *) bval->bv_val, "on"); + } + retval = config->config_set_fn(arg, (void *) int_val, err_buf, phase, apply_mod); + break; + } + + return retval; +} + + +static int parse_ldbm_config_entry(struct ldbminfo *li, Slapi_Entry *e, config_info *config_array) +{ + Slapi_Attr *attr = NULL; + + for (slapi_entry_first_attr(e, &attr); attr; slapi_entry_next_attr(e, attr, &attr)) { + char *attr_name = NULL; + Slapi_Value *sval = NULL; + struct berval *bval; + char err_buf[BUFSIZ]; + + slapi_attr_get_type(attr, &attr_name); + + /* There are some attributes that we don't care about, like objectclass. */ + if (ldbm_config_ignored_attr(attr_name)) { + continue; + } + + slapi_attr_first_value(attr, &sval); + bval = (struct berval *) slapi_value_get_berval(sval); + + if (ldbm_config_set(li, attr_name, config_array, bval, err_buf, CONFIG_PHASE_STARTUP, 1 /* apply */) != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "Error with config attribute %s : %s\n", attr_name, err_buf, 0); + return 1; + } + } + return 0; +} + +/* + * Returns: + * SLAPI_DSE_CALLBACK_ERROR on failure + * SLAPI_DSE_CALLBACK_OK on success + */ +int ldbm_config_modify_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg) +{ + int i; + char *attr_name; + LDAPMod **mods; + int rc = LDAP_SUCCESS; + int apply_mod = 0; + struct ldbminfo *li= (struct ldbminfo *) arg; + + /* This lock is probably way too conservative, but we don't expect much + * contention for it. */ + PR_Lock(li->li_config_mutex); + + slapi_pblock_get( pb, SLAPI_MODIFY_MODS, &mods ); + + returntext[0] = '\0'; + + /* + * First pass: set apply mods to 0 so only input validation will be done; + * 2nd pass: set apply mods to 1 to apply changes to internal storage + */ + for ( apply_mod = 0; apply_mod <= 1 && LDAP_SUCCESS == rc; apply_mod++ ) { + for (i = 0; mods[i] && LDAP_SUCCESS == rc; i++) { + attr_name = mods[i]->mod_type; + + /* There are some attributes that we don't care about, like modifiersname. */ + if (ldbm_config_ignored_attr(attr_name)) { + continue; + } + + if ((mods[i]->mod_op & LDAP_MOD_DELETE) || + ((mods[i]->mod_op & ~LDAP_MOD_BVALUES) == LDAP_MOD_ADD)) { + rc= LDAP_UNWILLING_TO_PERFORM; + sprintf(returntext, "%s attributes is not allowed", + (mods[i]->mod_op & LDAP_MOD_DELETE) ? "Deleting" : "Adding"); + } else if (mods[i]->mod_op & LDAP_MOD_REPLACE) { + /* This assumes there is only one bval for this mod. */ + rc = ldbm_config_set((void *) li, attr_name, ldbm_config, + ( mods[i]->mod_bvalues == NULL ) ? NULL + : mods[i]->mod_bvalues[0], returntext, + ((li->li_flags&LI_FORCE_MOD_CONFIG)? + CONFIG_PHASE_INTERNAL:CONFIG_PHASE_RUNNING), + apply_mod); + } + } + } + + PR_Unlock(li->li_config_mutex); + + *returncode= rc; + if(LDAP_SUCCESS == rc) { + return SLAPI_DSE_CALLBACK_OK; + } + else { + return SLAPI_DSE_CALLBACK_ERROR; + } +} + + +/* This function is used to set config attributes. It can be used as a + * shortcut to doing an internal modify operation on the config DSE. + */ +void ldbm_config_internal_set(struct ldbminfo *li, char *attrname, char *value) +{ + char err_buf[BUFSIZ]; + struct berval bval; + + bval.bv_val = value; + bval.bv_len = strlen(value); + + if (ldbm_config_set((void *) li, attrname, ldbm_config, &bval, + err_buf, CONFIG_PHASE_INTERNAL, 1 /* apply */) != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, + "Internal Error: Error setting instance config attr %s to %s: %s\n", + attrname, value, err_buf); + exit(1); + } +} + +/* + * replace_ldbm_config_value: + * - update an ldbm database config value + */ +void replace_ldbm_config_value(char *conftype, char *val, struct ldbminfo *li) +{ + Slapi_PBlock pb; + Slapi_Mods smods; + + pblock_init(&pb); + slapi_mods_init(&smods, 1); + slapi_mods_add(&smods, LDAP_MOD_REPLACE, conftype, strlen(val), val); + slapi_modify_internal_set_pb(&pb, + "cn=config,cn=ldbm database,cn=plugins,cn=config", + slapi_mods_get_ldapmods_byref(&smods), + NULL, NULL, li->li_identity, 0); + slapi_modify_internal_pb(&pb); + pblock_done(&pb); +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_config.h b/ldap/servers/slapd/back-ldbm/ldbm_config.h new file mode 100644 index 00000000..a26a73ed --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_config.h @@ -0,0 +1,133 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +#ifndef _LDBM_CONFIG_H_ +#define _LDBM_CONFIG_H_ + +struct config_info; +typedef struct config_info config_info; + +typedef int config_set_fn_t(void *arg, void *value, char *errorbuf, int phase, int apply); +typedef void *config_get_fn_t(void *arg); + /* The value for these is passed around as a + * void *, the actual value should be gotten + * by casting the void * as shown below. */ +#define CONFIG_TYPE_ONOFF 1 /* val = (int) value */ +#define CONFIG_TYPE_STRING 2 /* val = (char *) value - The get functions + * for this type must return alloced memory + * that should be freed by the caller. */ +#define CONFIG_TYPE_INT 3 /* val = (int) value */ +#define CONFIG_TYPE_LONG 4 /* val = (long) value */ +#define CONFIG_TYPE_INT_OCTAL 5 /* Same as CONFIG_TYPE_INT, but shown in + * octal */ +#define CONFIG_TYPE_SIZE_T 6 /* val = (size_t) value */ + +/* How changes to some config attributes are handled depends on what + * "phase" the server is in. Initialization, reading the config + * information at startup, or actually running. */ +#define CONFIG_PHASE_INITIALIZATION 1 +#define CONFIG_PHASE_STARTUP 2 +#define CONFIG_PHASE_RUNNING 3 +#define CONFIG_PHASE_INTERNAL 4 + +#define CONFIG_FLAG_PREVIOUSLY_SET 1 +#define CONFIG_FLAG_ALWAYS_SHOW 2 +#define CONFIG_FLAG_ALLOW_RUNNING_CHANGE 4 + +struct config_info { + char *config_name; + int config_type; + char *config_default_value; + config_get_fn_t *config_get_fn; + config_set_fn_t *config_set_fn; + int config_flags; +}; + +#define CONFIG_INSTANCE "nsslapd-instance" +#define CONFIG_LOOKTHROUGHLIMIT "nsslapd-lookthroughlimit" +#define CONFIG_IDLISTSCANLIMIT "nsslapd-idlistscanlimit" +#define CONFIG_DIRECTORY "nsslapd-directory" +#define CONFIG_MODE "nsslapd-mode" +#define CONFIG_DBCACHESIZE "nsslapd-dbcachesize" +#define CONFIG_DBNCACHE "nsslapd-dbncache" +#define CONFIG_MAXPASSBEFOREMERGE "nsslapd-maxpassbeforemerge" +#define CONFIG_IMPORT_CACHE_AUTOSIZE "nsslapd-import-cache-autosize" +#define CONFIG_CACHE_AUTOSIZE "nsslapd-cache-autosize" +#define CONFIG_CACHE_AUTOSIZE_SPLIT "nsslapd-cache-autosize-split" +#define CONFIG_IMPORT_CACHESIZE "nsslapd-import-cachesize" +#define CONFIG_INDEX_BUFFER_SIZE "nsslapd-index-buffer-size" +#define CONFIG_EXCLUDE_FROM_EXPORT "nsslapd-exclude-from-export" +#define CONFIG_EXCLUDE_FROM_EXPORT_DEFAULT_VALUE \ + "entrydn entryid dncomp parentid numSubordinates" + +/* dblayer config options - These are hidden from the user + * and can't be updated on the fly. */ +#define CONFIG_DB_LOGDIRECTORY "nsslapd-db-logdirectory" +#define CONFIG_DB_DURABLE_TRANSACTIONS "nsslapd-db-durable-transaction" +#define CONFIG_DB_CIRCULAR_LOGGING "nsslapd-db-circular-logging" +#define CONFIG_DB_TRANSACTION_LOGGING "nsslapd-db-transaction-logging" +#define CONFIG_DB_CHECKPOINT_INTERVAL "nsslapd-db-checkpoint-interval" +#define CONFIG_DB_TRANSACTION_BATCH "nsslapd-db-transaction-batch-val" +#define CONFIG_DB_LOGBUF_SIZE "nsslapd-db-logbuf-size" +#define CONFIG_DB_PAGE_SIZE "nsslapd-db-page-size" +#define CONFIG_DB_INDEX_PAGE_SIZE "nsslapd-db-index-page-size" /* With the new + idl design, the large 8Kbyte pages we use are not + optimal. The page pool churns very quickly as we add new IDs under a + sustained add load. Smaller pages stop this happening so much and + consequently make us spend less time flushing dirty pages on checkpoints. + But 8K is still a good page size for id2entry. So we now allow different + page sizes for the primary and secondary indices. */ +#define CONFIG_DB_IDL_DIVISOR "nsslapd-db-idl-divisor" +#define CONFIG_DB_LOGFILE_SIZE "nsslapd-db-logfile-size" +#define CONFIG_DB_TRICKLE_PERCENTAGE "nsslapd-db-trickle-percentage" +#define CONFIG_DB_SPIN_COUNT "nsslapd-db-spin-count" +#define CONFIG_DB_VERBOSE "nsslapd-db-verbose" +#define CONFIG_DB_DEBUG "nsslapd-db-debug" +#define CONFIG_DB_LOCK "nsslapd-db-locks" +#define CONFIG_DB_NAMED_REGIONS "nsslapd-db-named-regions" +#define CONFIG_DB_PRIVATE_MEM "nsslapd-db-private-mem" +#define CONFIG_DB_PRIVATE_IMPORT_MEM "nsslapd-db-private-import-mem" +#define CONFIG_DB_SHM_KEY "nsslapd-db-shm-key" +#define CONFIG_DB_CACHE "nsslapd-db-cache" +#define CONFIG_DB_DEBUG_CHECKPOINTING "nsslapd-db-debug-checkpointing" +#define CONFIG_DB_HOME_DIRECTORY "nsslapd-db-home-directory" +#define CONFIG_DB_LOCKDOWN "nsslapd-db-lockdown" +#define CONFIG_DB_TX_MAX "nsslapd-db-tx-max" + +#define CONFIG_IDL_SWITCH "nsslapd-idl-switch" +#define CONFIG_BYPASS_FILTER_TEST "nsslapd-search-bypass-filter-test" +#define CONFIG_USE_VLV_INDEX "nsslapd-search-use-vlv-index" +#define CONFIG_SERIAL_LOCK "nsslapd-serial-lock" + +/* instance config options */ +#define CONFIG_INSTANCE_CACHESIZE "nsslapd-cachesize" +#define CONFIG_INSTANCE_CACHEMEMSIZE "nsslapd-cachememsize" +#define CONFIG_INSTANCE_SUFFIX "nsslapd-suffix" +#define CONFIG_INSTANCE_READONLY "nsslapd-readonly" +#define CONFIG_INSTANCE_DIR "nsslapd-directory" + +#define CONFIG_INSTANCE_REQUIRE_INDEX "nsslapd-require-index" + +#define CONFIG_USE_LEGACY_ERRORCODE "nsslapd-do-not-use-vlv-error" + +#define LDBM_INSTANCE_CONFIG_DONT_WRITE 1 + +/* Some fuctions in ldbm_config.c used by ldbm_instance_config.c */ +int ldbm_config_add_dse_entries(struct ldbminfo *li, char **entries, char *string1, char *string2, char *string3, int flags); +int ldbm_config_add_dse_entry(struct ldbminfo *li, char *entry, int flags); +void ldbm_config_get(void *arg, config_info *config, char *buf); +int ldbm_config_set(void *arg, char *attr_name, config_info *config_array, struct berval *bval, char *err_buf, int phase, int apply_mod); +int ldbm_config_ignored_attr(char *attr_name); + +/* Functions in ldbm_instance_config.c used in ldbm_config.c */ +int ldbm_instance_config_load_dse_info(ldbm_instance *inst); +int ldbm_instance_config_add_index_entry(ldbm_instance *inst, int argc, + char **argv, int flags); +int +ldbm_instance_index_config_enable_index(ldbm_instance *inst, Slapi_Entry* e); +int ldbm_instance_create_default_user_indexes(ldbm_instance *inst); + + +#endif /* _LDBM_CONFIG_H_ */ diff --git a/ldap/servers/slapd/back-ldbm/ldbm_delete.c b/ldap/servers/slapd/back-ldbm/ldbm_delete.c new file mode 100644 index 00000000..55d8f162 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_delete.c @@ -0,0 +1,633 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +/* delete.c - ldbm backend delete routine */ + +#include "back-ldbm.h" + +int +ldbm_back_delete( Slapi_PBlock *pb ) +{ + backend *be; + ldbm_instance *inst; + struct ldbminfo *li = NULL; + struct backentry *e = NULL; + struct backentry *tombstone = NULL; + char *dn = NULL; + back_txn txn; + back_txnid parent_txn; + int retval = -1; + char *msg; + char *errbuf = NULL; + int retry_count = 0; + int disk_full = 0; + int parent_found = 0; + modify_context parent_modify_c = {0}; + int rc; + int ldap_result_code= LDAP_SUCCESS; + char *ldap_result_message= NULL; + Slapi_DN sdn; + char *e_uniqueid = NULL; + Slapi_DN *nscpEntrySDN = NULL; + int dblock_acquired= 0; + Slapi_Operation *operation; + CSN *opcsn = NULL; + int is_fixup_operation = 0; + int is_replicated_operation= 0; + int is_tombstone_entry = 0; /* True if the current entry is alreday a tombstone */ + int delete_tombstone_entry = 0; /* We must remove the given tombstone entry from the DB */ + int create_tombstone_entry = 0; /* We perform a "regular" LDAP delete but since we use */ + /* replication, we must create a new tombstone entry */ + int tombstone_in_cache = 0; + entry_address *addr; + int addordel_flags = 0; /* passed to index_addordel */ + + slapi_pblock_get( pb, SLAPI_BACKEND, &be); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_DELETE_TARGET, &dn ); + slapi_pblock_get( pb, SLAPI_TARGET_ADDRESS, &addr); + slapi_pblock_get( pb, SLAPI_PARENT_TXN, (void**)&parent_txn ); + slapi_pblock_get( pb, SLAPI_OPERATION, &operation ); + slapi_pblock_get( pb, SLAPI_IS_REPLICATED_OPERATION, &is_replicated_operation ); + + if (pb->pb_conn) + { + slapi_log_error (SLAPI_LOG_TRACE, "ldbm_back_delete", "enter conn=%d op=%d\n", pb->pb_conn->c_connid, operation->o_opid); + } + + is_fixup_operation = operation_is_flag_set(operation,OP_FLAG_REPL_FIXUP); + delete_tombstone_entry = operation_is_flag_set(operation, OP_FLAG_TOMBSTONE_ENTRY); + + inst = (ldbm_instance *) be->be_instance_info; + + slapi_sdn_init_dn_byref(&sdn,dn); + + dblayer_txn_init(li,&txn); + + /* The dblock serializes writes to the database, + * which reduces deadlocking in the db code, + * which means that we run faster. + * + * But, this lock is re-enterant for the fixup + * operations that the URP code in the Replication + * plugin generates. + */ + if(SERIALLOCK(li) && !operation_is_flag_set(operation,OP_FLAG_REPL_FIXUP)) + { + dblayer_lock_backend(be); + dblock_acquired= 1; + } + + /* + * We are about to pass the last abandon test, so from now on we are + * committed to finish this operation. Set status to "will complete" + * before we make our last abandon check to avoid race conditions in + * the code that processes abandon operations. + */ + if (operation) { + operation->o_status = SLAPI_OP_STATUS_WILL_COMPLETE; + } + if ( slapi_op_abandoned( pb ) ) { + goto error_return; + } + + /* Don't call pre-op for Tombstone entries */ + if (!delete_tombstone_entry) + { + /* + * Some present state information is passed through the PBlock to the + * backend pre-op plugin. To ensure a consistent snapshot of this state + * we wrap the reading of the entry with the dblock. + */ + ldap_result_code= get_copy_of_entry(pb, addr, &txn, SLAPI_DELETE_EXISTING_ENTRY, !is_replicated_operation); + slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ldap_result_code); + if(plugin_call_plugins(pb, SLAPI_PLUGIN_BE_PRE_DELETE_FN)==-1) + { + /* + * Plugin indicated some kind of failure, + * or that this Operation became a No-Op. + */ + slapi_pblock_get(pb, SLAPI_RESULT_CODE, &ldap_result_code); + goto error_return; + } + } + + + /* find and lock the entry we are about to modify */ + if ( (e = find_entry2modify( pb, be, addr, NULL )) == NULL ) + { + ldap_result_code= -1; + goto error_return; /* error result sent by find_entry2modify() */ + } + + if ( slapi_entry_has_children( e->ep_entry ) ) + { + ldap_result_code= LDAP_NOT_ALLOWED_ON_NONLEAF; + goto error_return; + } + + /* + * Sanity check to avoid to delete a non-tombstone or to tombstone again + * a tombstone entry. This should not happen (see bug 561003). + */ + is_tombstone_entry = slapi_entry_flag_is_set(e->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE); + if (delete_tombstone_entry) { + PR_ASSERT(is_tombstone_entry); + if (!is_tombstone_entry) { + slapi_log_error(SLAPI_LOG_FATAL, "ldbm_back_delete", + "Attempt to delete a non-tombstone entry %s\n", dn); + delete_tombstone_entry = 0; + } + } else { + PR_ASSERT(!is_tombstone_entry); + if (is_tombstone_entry) { + slapi_log_error(SLAPI_LOG_FATAL, "ldbm_back_delete", + "Attempt to Tombstone again a tombstone entry %s\n", dn); + delete_tombstone_entry = 1; + } + } + + /* + * If a CSN is set, we need to tombstone the entry, + * rather than deleting it outright. + */ + opcsn = operation_get_csn (operation); + if (!delete_tombstone_entry) + { + if (opcsn == NULL && !is_fixup_operation && operation->o_csngen_handler) + { + /* + * Current op is a user request. Opcsn will be assigned + * by entry_assign_operation_csn() if the dn is in an + * updatable replica. + */ + opcsn = entry_assign_operation_csn ( pb, e->ep_entry, NULL ); + } + if (opcsn != NULL) + { + if (!is_fixup_operation) + { + entry_set_maxcsn (e->ep_entry, opcsn); + } + /* + * We are dealing with replication and if we haven't been called to + * remove a tombstone, then it's because we want to create a new one. + */ + if ( slapi_operation_get_replica_attr (pb, operation, "nsds5ReplicaTombstonePurgeInterval", &create_tombstone_entry) == 0) + { + create_tombstone_entry = (create_tombstone_entry < 0) ? 0 : 1; + } + } + } + +#if DEBUG + slapi_log_error(SLAPI_LOG_REPL, "ldbm_back_delete", + "entry: %s - flags: delete %d is_tombstone_entry %d create %d \n", + dn, delete_tombstone_entry, is_tombstone_entry, create_tombstone_entry); +#endif + + /* Save away a copy of the entry, before modifications */ + slapi_pblock_set( pb, SLAPI_ENTRY_PRE_OP, slapi_entry_dup( e->ep_entry )); + + /* JCMACL - Shouldn't the access check be before the has children check... + * otherwise we're revealing the fact that an entry exists and has children */ + ldap_result_code = plugin_call_acl_plugin (pb, e->ep_entry, NULL, NULL, SLAPI_ACL_DELETE, + ACLPLUGIN_ACCESS_DEFAULT, &errbuf ); + if ( ldap_result_code != LDAP_SUCCESS ) + { + ldap_result_message= errbuf; + goto error_return; + } + + /* + * Get the entry's parent. We do this here because index_read + * seems to deadlock the database when dblayer_txn_begin is + * called. + */ + if (!delete_tombstone_entry) + { + Slapi_DN parentsdn; + + slapi_sdn_init(&parentsdn); + slapi_sdn_get_backend_parent(&sdn,&parentsdn,pb->pb_backend); + if ( !slapi_sdn_isempty(&parentsdn) ) + { + struct backentry *parent = NULL; + entry_address parent_addr; + + parent_addr.dn = (char*)slapi_sdn_get_ndn (&parentsdn); + parent_addr.uniqueid = NULL; + parent = find_entry2modify_only(pb,be,&parent_addr,&txn); + if (NULL != parent) { + int isglue; + size_t haschildren = 0; + + /* Unfortunately findentry doesn't tell us whether it just didn't find the entry, or if + there was an error, so we have to assume that the parent wasn't found */ + parent_found = 1; + + /* Modify the parent in memory */ + modify_init(&parent_modify_c,parent); + retval = parent_update_on_childchange(&parent_modify_c,2,&haschildren); /* 2==delete */\ + /* The modify context now contains info needed later */ + if (0 != retval) { + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + /* + * Replication urp_post_delete will delete the parent entry + * if it is a glue entry without any more children. + * Those urp condition checkings are done here to + * save unnecessary entry dup. + */ + isglue = slapi_entry_attr_hasvalue (parent_modify_c.new_entry->ep_entry, + SLAPI_ATTR_OBJECTCLASS, "glue"); + if ( opcsn && parent_modify_c.new_entry && !haschildren && isglue) + { + slapi_pblock_set ( pb, SLAPI_DELETE_GLUE_PARENT_ENTRY, + slapi_entry_dup (parent_modify_c.new_entry->ep_entry) ); + } + } + } + slapi_sdn_done(&parentsdn); + } + + if(create_tombstone_entry) + { + /* + * The entry is not removed from the disk when we tombstone an + * entry. We change the DN, add objectclass=tombstone, and record + * the UniqueID of the parent entry. + */ + const char *childuniqueid= slapi_entry_get_uniqueid(e->ep_entry); + const char *parentuniqueid= NULL; + char *tombstone_dn = compute_entry_tombstone_dn(slapi_entry_get_dn(e->ep_entry), + childuniqueid); + Slapi_Value *tomb_value; + + nscpEntrySDN = slapi_entry_get_sdn(e->ep_entry); + + /* Copy the entry unique_id for URP conflict checking */ + e_uniqueid = slapi_ch_strdup(childuniqueid); + + if(parent_modify_c.old_entry!=NULL) + { + /* The suffix entry has no parent */ + parentuniqueid= slapi_entry_get_uniqueid(parent_modify_c.old_entry->ep_entry); + } + tombstone = backentry_dup( e ); + slapi_entry_set_dn(tombstone->ep_entry,tombstone_dn); /* Consumes DN */ + /* Set tombstone flag on ep_entry */ + slapi_entry_set_flag(tombstone->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE); + + if(parentuniqueid!=NULL) + { + /* The suffix entry has no parent */ + slapi_entry_add_string(tombstone->ep_entry, SLAPI_ATTR_VALUE_PARENT_UNIQUEID, parentuniqueid); + } + if(nscpEntrySDN!=NULL) + { + slapi_entry_add_string(tombstone->ep_entry, SLAPI_ATTR_NSCP_ENTRYDN, slapi_sdn_get_ndn(nscpEntrySDN)); + } + tomb_value = slapi_value_new_string(SLAPI_ATTR_VALUE_TOMBSTONE); + value_update_csn(tomb_value, CSN_TYPE_VALUE_UPDATED, + operation_get_csn(operation)); + slapi_entry_add_value(tombstone->ep_entry, SLAPI_ATTR_OBJECTCLASS, tomb_value); + slapi_value_free(&tomb_value); + + /* XXXggood above used to be: slapi_entry_add_string(tombstone->ep_entry, SLAPI_ATTR_OBJECTCLASS, SLAPI_ATTR_VALUE_TOMBSTONE); */ + /* JCMREPL - Add a description of what's going on? */ + } + + /* + * So, we believe that no code up till here actually added anything + * to the persistent store. From now on, we're transacted + */ + + for (retry_count = 0; retry_count < RETRY_TIMES; retry_count++) { + if (retry_count > 0) { + dblayer_txn_abort(li,&txn); + /* We're re-trying */ + LDAPDebug( LDAP_DEBUG_TRACE, "Delete Retrying Transaction\n", 0, 0, 0 ); +#ifndef LDBM_NO_BACKOFF_DELAY + { + PRIntervalTime interval; + interval = PR_MillisecondsToInterval(slapi_rand() % 100); + DS_Sleep(interval); + } +#endif + } + retval = dblayer_txn_begin(li,parent_txn,&txn); + if (0 != retval) { + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + if(create_tombstone_entry) + { + /* + * The entry is not removed from the disk when we tombstone an + * entry. We change the DN, add objectclass=tombstone, and record + * the UniqueID of the parent entry. + */ + retval = id2entry_add( be, tombstone, &txn ); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 1 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Abort and re-try */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_ANY, "id2entry_add failed, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + tombstone_in_cache = 1; + } + else + { + /* delete the entry from disk */ + retval = id2entry_delete( be, e, &txn ); + if (DB_LOCK_DEADLOCK == retval) + { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 2 DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (retval != 0 ) { + if (retval == DB_RUNRECOVERY || + LDBM_OS_ERR_IS_DISKFULL(retval)) { + disk_full = 1; + } + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + } + /* delete from attribute indexes */ + addordel_flags = BE_INDEX_DEL|BE_INDEX_PRESENCE; + if (delete_tombstone_entry) + { + addordel_flags |= BE_INDEX_TOMBSTONE; /* tell index code we are deleting a tombstone */ + } + retval = index_addordel_entry( be, e, addordel_flags, &txn ); + if (DB_LOCK_DEADLOCK == retval) + { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 1 DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (retval != 0) { + LDAPDebug( LDAP_DEBUG_TRACE, "index_del_entry failed\n", 0, 0, 0 ); + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + if(create_tombstone_entry) + { + /* + * The tombstone entry is removed from all attribute indexes + * above, but we want it to remain in the nsUniqueID and nscpEntryDN indexes + * and for objectclass=tombstone. + */ + retval = index_addordel_string(be,SLAPI_ATTR_OBJECTCLASS,SLAPI_ATTR_VALUE_TOMBSTONE,tombstone->ep_id,BE_INDEX_ADD,&txn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 4 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "delete 1 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + retval = index_addordel_string(be,SLAPI_ATTR_UNIQUEID,slapi_entry_get_uniqueid(tombstone->ep_entry),tombstone->ep_id,BE_INDEX_ADD,&txn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 5 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "delete 2 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + retval = index_addordel_string(be,SLAPI_ATTR_NSCP_ENTRYDN, slapi_sdn_get_ndn(nscpEntrySDN),tombstone->ep_id,BE_INDEX_ADD,&txn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 6 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "delete 3 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + } else if (delete_tombstone_entry) + { + /* + * We need to remove the Tombstone entry from the remaining indexes: + * objectclass=nsTombstone, nsUniqueID, nscpEntryDN + */ + char *nscpedn = NULL; + + retval = index_addordel_string(be,SLAPI_ATTR_OBJECTCLASS,SLAPI_ATTR_VALUE_TOMBSTONE,e->ep_id,BE_INDEX_DEL,&txn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 4 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "delete 1 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + retval = index_addordel_string(be,SLAPI_ATTR_UNIQUEID,slapi_entry_get_uniqueid(e->ep_entry),e->ep_id,BE_INDEX_DEL,&txn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 5 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "delete 2 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + nscpedn = slapi_entry_attr_get_charptr(e->ep_entry, SLAPI_ATTR_NSCP_ENTRYDN); + if (nscpedn) { + retval = index_addordel_string(be,SLAPI_ATTR_NSCP_ENTRYDN, nscpedn, e->ep_id,BE_INDEX_DEL,&txn); + slapi_ch_free((void **)&nscpedn); + if (DB_LOCK_DEADLOCK == retval) { + LDAPDebug( LDAP_DEBUG_ARGS, "delete 6 DB_LOCK_DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "delete 3 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + } + } + + if (parent_found) { + /* Push out the db modifications from the parent entry */ + retval = modify_update_all(be,pb,&parent_modify_c,&txn); + if (DB_LOCK_DEADLOCK == retval) + { + LDAPDebug( LDAP_DEBUG_ARGS, "del 4 DEADLOCK\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_TRACE, "delete 3 BAD, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + } + /* + * first check if searchentry needs to be removed + * Remove the entry from the Virtual List View indexes. + * + */ + if(!delete_tombstone_entry && + !vlv_delete_search_entry(pb,e->ep_entry,inst)) { + retval = vlv_update_all_indexes(&txn, be, pb, e, NULL); + } + + if (DB_LOCK_DEADLOCK == retval) + { + LDAPDebug( LDAP_DEBUG_ARGS, "delete DEADLOCK vlv_update_index\n", 0, 0, 0 ); + /* Retry txn */ + continue; + } + if (retval != 0 ) { + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + if (retval == 0 ) { + break; + } + } + if (retry_count == RETRY_TIMES) { + /* Failed */ + LDAPDebug( LDAP_DEBUG_ANY, "Retry count exceeded in delete\n", 0, 0, 0 ); + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + retval = dblayer_txn_commit(li,&txn); + if (0 != retval) + { + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + /* delete from cache and clean up */ + cache_remove(&inst->inst_cache, e); + cache_unlock_entry( &inst->inst_cache, e ); + cache_return( &inst->inst_cache, &e ); + if (parent_found) + { + /* Replace the old parent entry with the newly modified one */ + modify_switch_entries( &parent_modify_c,be); + } + + + rc= 0; + goto common_return; + +error_return: + if (e!=NULL) { + cache_unlock_entry( &inst->inst_cache, e ); + cache_return( &inst->inst_cache, &e ); + } + if (tombstone_in_cache) + { + cache_remove( &inst->inst_cache, tombstone ); + } + else + { + backentry_free( &tombstone ); + } + + if (retval == DB_RUNRECOVERY) { + dblayer_remember_disk_filled(li); + ldbm_nasty("Delete",79,retval); + disk_full = 1; + } + + if (disk_full) { + rc= return_on_disk_full(li); + goto diskfull_return; + } + else + rc= SLAPI_FAIL_GENERAL; + + /* It is specifically OK to make this call even when no transaction was in progress */ + dblayer_txn_abort(li,&txn); /* abort crashes in case disk full */ + +common_return: + if (tombstone_in_cache) + { + cache_return( &inst->inst_cache, &tombstone ); + } + + /* + * The bepostop is called even if the operation fails, + * but not if the operation is purging tombstones. + */ + if (!delete_tombstone_entry) { + plugin_call_plugins (pb, SLAPI_PLUGIN_BE_POST_DELETE_FN); + } + +diskfull_return: + if(ldap_result_code!=-1) + { + slapi_send_ldap_result( pb, ldap_result_code, NULL, ldap_result_message, 0, NULL ); + } + modify_term(&parent_modify_c,be); + if(dblock_acquired) + { + dblayer_unlock_backend(be); + } + if (rc == 0 && opcsn && !is_fixup_operation && !delete_tombstone_entry) + { + /* URP Naming Collision + * When an entry is deleted by a replicated delete operation + * we must check for entries that have had a naming collision + * with this entry. Now that this name has been given up, one + * of those entries can take over the name. + */ + slapi_pblock_set(pb, SLAPI_URP_NAMING_COLLISION_DN, slapi_ch_strdup (dn)); + } + done_with_pblock_entry(pb, SLAPI_DELETE_EXISTING_ENTRY); + slapi_ch_free((void**)&errbuf); + slapi_sdn_done(&sdn); + slapi_ch_free_string(&e_uniqueid); + if (pb->pb_conn) + { + slapi_log_error (SLAPI_LOG_TRACE, "ldbm_back_delete", "leave conn=%d op=%d\n", pb->pb_conn->c_connid, operation->o_opid); + } + return rc; +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_index_config.c b/ldap/servers/slapd/back-ldbm/ldbm_index_config.c new file mode 100644 index 00000000..bd604cdb --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_index_config.c @@ -0,0 +1,698 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* This file handles configuration information that is specific + * to ldbm instance indexes. + */ + +#include "back-ldbm.h" +#include "dblayer.h" + +/* Forward declarations for the callbacks */ +int ldbm_instance_index_config_add_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); +int ldbm_instance_index_config_delete_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); + + + + + +/* attrinfo2ConfIndexes: converts attrinfo into "pres,eq,sub,approx" + * as seen in index entries within dse.ldif + */ +static char *attrinfo2ConfIndexes (struct attrinfo *pai) +{ + char buffer[128]; + + buffer[0] = '\0'; + if (!(IS_INDEXED( pai->ai_indexmask ))) { /* skip if no index */ + strcat (buffer, "none"); + } + + if (pai->ai_indexmask & INDEX_PRESENCE) { + if (strlen (buffer)) { + strcat (buffer, ","); + } + strcat (buffer, "pres"); + } + if (pai->ai_indexmask & INDEX_EQUALITY) { + if (strlen (buffer)) { + strcat (buffer, ","); + } + strcat (buffer, "eq"); + } + if (pai->ai_indexmask & INDEX_APPROX) { + if (strlen(buffer)) { + strcat (buffer, ","); + } + strcat (buffer, "approx"); + } + if (pai->ai_indexmask & INDEX_SUB) { + if (strlen (buffer)) { + strcat (buffer, ","); + } + strcat (buffer, "sub"); + } + + return (slapi_ch_strdup (buffer) ); +} + + +/* attrinfo2ConfMatchingRules: converts attrinfo into matching rule oids, as + * seen in index entries within dse.ldif + */ +static char *attrinfo2ConfMatchingRules (struct attrinfo *pai) +{ + int i; + char buffer[1024]; + + buffer[0] = '\0'; + + if (pai->ai_index_rules) { + strcat (buffer, "\t"); + for (i = 0; pai->ai_index_rules[i]; i++) { + strcat (buffer, pai->ai_index_rules[i]); + if (pai->ai_index_rules[i+1]) { + strcat (buffer, ","); + } + } + } + return (slapi_ch_strdup (buffer) ); +} + + +/* used by the two callbacks below, to parse an index entry into something + * awkward that we can pass to attr_index_config(). + */ +#define MAX_TMPBUF 256 +#define ZCAT_SAFE(_buf, _x1, _x2) do { \ + if (strlen(_buf) + strlen(_x1) + strlen(_x2) + 2 < MAX_TMPBUF) { \ + strcat(_buf, _x1); \ + strcat(_buf, _x2); \ + } \ +} while (0) +static int ldbm_index_parse_entry(ldbm_instance *inst, Slapi_Entry *e, + const char *trace_string, + char **index_name) +{ + char *arglist[] = { NULL, NULL, NULL, NULL }; + int argc = 0, i; + Slapi_Attr *attr; + const struct berval *attrValue; + Slapi_Value *sval; + char tmpBuf[MAX_TMPBUF]; + + /* Get the name of the attribute to index which will be the value + * of the cn attribute. */ + if (slapi_entry_attr_find(e, "cn", &attr) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "Warning: malformed index entry %s\n", + slapi_entry_get_dn(e), 0, 0); + return LDAP_OPERATIONS_ERROR; + } + + slapi_attr_first_value(attr, &sval); + attrValue = slapi_value_get_berval(sval); + arglist[argc++] = slapi_ch_strdup(attrValue->bv_val); + if (index_name != NULL) { + *index_name = slapi_ch_strdup(attrValue->bv_val); + } + + /* Get the list of index types from the entry. */ + if (0 == slapi_entry_attr_find(e, "nsIndexType", &attr)) { + for (i = slapi_attr_first_value(attr, &sval); i != -1; + i = slapi_attr_next_value(attr, i, &sval)) { + attrValue = slapi_value_get_berval(sval); + if (0 == i) { + tmpBuf[0] = 0; + ZCAT_SAFE(tmpBuf, "", attrValue->bv_val); + } else { + ZCAT_SAFE(tmpBuf, ",", attrValue->bv_val); + } + } + arglist[argc++] = slapi_ch_strdup(tmpBuf); + } + + /* Get the list of matching rules from the entry. */ + if (0 == slapi_entry_attr_find(e, "nsMatchingRule", &attr)) { + for (i = slapi_attr_first_value(attr, &sval); i != -1; + i = slapi_attr_next_value(attr, i, &sval)) { + attrValue = slapi_value_get_berval(sval); + if (0 == i) { + tmpBuf[0] = 0; + ZCAT_SAFE(tmpBuf, "", attrValue->bv_val); + } else { + ZCAT_SAFE(tmpBuf, ",", attrValue->bv_val); + } + } + arglist[argc++] = slapi_ch_strdup(tmpBuf); + } + + arglist[argc] = NULL; + attr_index_config(inst->inst_be, (char *)trace_string, 0, argc, arglist, 0); + for (i = 0; i < argc; i++) { + slapi_ch_free((void **)&arglist[i]); + } + return LDAP_SUCCESS; +} + + +/* + * Temp callback that gets called for each index entry when a new + * instance is starting up. + */ +int +ldbm_index_init_entry_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + + returntext[0] = '\0'; + *returncode = ldbm_index_parse_entry(inst, e, "from ldbm instance init", + NULL); + if (*returncode == LDAP_SUCCESS) { + return SLAPI_DSE_CALLBACK_OK; + } else { + sprintf(returntext, "Problem initializing index entry %s\n", + slapi_entry_get_dn(e)); + return SLAPI_DSE_CALLBACK_ERROR; + } +} + +/* + * Config DSE callback for index additions. + */ +int +ldbm_instance_index_config_add_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* eAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + char *index_name; + + returntext[0] = '\0'; + *returncode = ldbm_index_parse_entry(inst, e, "from DSE add", &index_name); + if (*returncode == LDAP_SUCCESS) { + struct attrinfo *ai = NULL; + + /* if the index is a "system" index, we assume it's being added by + * by the server, and it's okay for the index to go online immediately. + * if not, we set the index "offline" so it won't actually be used + * until someone runs db2index on it. + */ + if (! ldbm_attribute_always_indexed(index_name)) { + ainfo_get(inst->inst_be, index_name, &ai); + PR_ASSERT(ai != NULL); + ai->ai_indexmask |= INDEX_OFFLINE; + } + slapi_ch_free((void **)&index_name); + return SLAPI_DSE_CALLBACK_OK; + } else { + return SLAPI_DSE_CALLBACK_ERROR; + } +} + +/* + * Config DSE callback for index deletes. + */ +int +ldbm_instance_index_config_delete_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + char *arglist[4]; + Slapi_Attr *attr; + Slapi_Value *sval; + const struct berval *attrValue; + int argc = 0; + int rc = SLAPI_DSE_CALLBACK_OK; + struct attrinfo *ainfo = NULL; + + returntext[0] = '\0'; + *returncode = LDAP_SUCCESS; + + slapi_entry_attr_find(e, "cn", &attr); + slapi_attr_first_value(attr, &sval); + attrValue = slapi_value_get_berval(sval); + + arglist[argc++] = slapi_ch_strdup(attrValue->bv_val); + arglist[argc++] = slapi_ch_strdup("none"); + arglist[argc] = NULL; + attr_index_config(inst->inst_be, "From DSE delete", 0, argc, arglist, 0); + slapi_ch_free((void **)&arglist[0]); + slapi_ch_free((void **)&arglist[1]); + + ainfo_get(inst->inst_be, attrValue->bv_val, &ainfo); + + if (NULL == ainfo) { + *returncode = LDAP_UNAVAILABLE; + rc = SLAPI_DSE_CALLBACK_ERROR; + } else { + if (dblayer_erase_index_file(inst->inst_be, ainfo, 0 /* do chkpt */)) { + *returncode = LDAP_UNWILLING_TO_PERFORM; + rc = SLAPI_DSE_CALLBACK_ERROR; + } + } + + return rc; +} + +/* + * Config DSE callback for index entry changes. + * + * this function is huge! + */ +int +ldbm_instance_index_config_modify_callback(Slapi_PBlock *pb, Slapi_Entry *e, + Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + Slapi_Attr *attr; + Slapi_Value *sval; + const struct berval *attrValue; + struct attrinfo *ainfo = NULL; + LDAPMod **mods; + char *arglist[4]; + char *config_attr; + char *origIndexTypes, *origMatchingRules; + char **origIndexTypesArray = NULL; + char **origMatchingRulesArray = NULL; + char **addIndexTypesArray = NULL; + char **addMatchingRulesArray = NULL; + char **deleteIndexTypesArray = NULL; + char **deleteMatchingRulesArray = NULL; + int i, j; + int dodeletes = 0; + char tmpBuf[MAX_TMPBUF]; + + returntext[0] = '\0'; + *returncode = LDAP_SUCCESS; + slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods); + + slapi_entry_attr_find(e, "cn", &attr); + slapi_attr_first_value(attr, &sval); + attrValue = slapi_value_get_berval(sval); + ainfo_get(inst->inst_be, attrValue->bv_val, &ainfo); + if (NULL == ainfo) { + return SLAPI_DSE_CALLBACK_ERROR; + } + + origIndexTypes = attrinfo2ConfIndexes(ainfo); + origMatchingRules = attrinfo2ConfMatchingRules(ainfo); + origIndexTypesArray = str2charray(origIndexTypes, ","); + origMatchingRulesArray = str2charray(origMatchingRules, ","); + + for (i = 0; mods[i] != NULL; i++) { + config_attr = (char *)mods[i]->mod_type; + + if (strcasecmp(config_attr, "nsIndexType") == 0) { + if ((mods[i]->mod_op & ~LDAP_MOD_BVALUES) == LDAP_MOD_ADD) { + for (j = 0; mods[i]->mod_bvalues[j] != NULL; j++) { + charray_add(&addIndexTypesArray, + slapi_ch_strdup(mods[i]->mod_bvalues[j]->bv_val)); + } + continue; + } + if ((mods[i]->mod_op & ~LDAP_MOD_BVALUES) == LDAP_MOD_DELETE) { + if ((mods[i]->mod_bvalues == NULL) || + (mods[i]->mod_bvalues[0] == NULL)) { + if (deleteIndexTypesArray) { + charray_free(deleteIndexTypesArray); + } + deleteIndexTypesArray = charray_dup(origIndexTypesArray); + } else { + for (j = 0; mods[i]->mod_bvalues[j] != NULL; j++) { + charray_add(&deleteIndexTypesArray, + slapi_ch_strdup(mods[i]->mod_bvalues[j]->bv_val)); + } + } + continue; + } + } + if (strcasecmp(config_attr, "nsMatchingRule") == 0) { + if ((mods[i]->mod_op & ~LDAP_MOD_BVALUES) == LDAP_MOD_ADD) { + for (j = 0; mods[i]->mod_bvalues[j] != NULL; j++) { + charray_add(&addMatchingRulesArray, + slapi_ch_strdup(mods[i]->mod_bvalues[j]->bv_val)); + } + continue; + } + if ((mods[i]->mod_op & ~LDAP_MOD_BVALUES) == LDAP_MOD_DELETE) { + if ((mods[i]->mod_bvalues == NULL) || + (mods[i]->mod_bvalues[0] == NULL)) { + if (deleteMatchingRulesArray) { + charray_free(deleteMatchingRulesArray); + } + deleteMatchingRulesArray = charray_dup(origMatchingRulesArray); + } else { + for (j = 0; mods[i]->mod_bvalues[j] != NULL; j++) { + charray_add(&deleteMatchingRulesArray, + slapi_ch_strdup(mods[i]->mod_bvalues[j]->bv_val)); + } + } + continue; + } + } + } + + /* create the new set of index types */ + if (deleteIndexTypesArray) { + for (i = 0; origIndexTypesArray[i] != NULL; i++) { + if (charray_inlist(deleteIndexTypesArray, + origIndexTypesArray[i])) { + slapi_ch_free((void **)&(origIndexTypesArray[i])); + dodeletes = 1; + if (origIndexTypesArray[i+1] != NULL) { + for (j = i+1; origIndexTypesArray[j] != NULL; j++) { + origIndexTypesArray[j-1] = origIndexTypesArray[j]; + } + origIndexTypesArray[j-1] = NULL; + i--; + } + } + } + } + + if (addIndexTypesArray) { + for (i = 0; addIndexTypesArray[i] != NULL; i++) { + if (!charray_inlist(origIndexTypesArray, addIndexTypesArray[i])) { + charray_add(&origIndexTypesArray, + slapi_ch_strdup(addIndexTypesArray[i])); + } + } + } + + if (deleteMatchingRulesArray) { + for (i = 0; origMatchingRulesArray[i] != NULL; i++) { + if (charray_inlist(deleteMatchingRulesArray, + origMatchingRulesArray[i])) { + slapi_ch_free((void **)&(origMatchingRulesArray[i])); + dodeletes = 1; + if (origMatchingRulesArray[i+1] != NULL) { + for (j = i+1; origMatchingRulesArray[j] != NULL; j++) { + origMatchingRulesArray[j-1] = origMatchingRulesArray[j]; + } + origMatchingRulesArray[j-1] = NULL; + i--; + } + } + } + } + + if (addMatchingRulesArray) { + for (i = 0; addMatchingRulesArray[i] != NULL; i++) { + if (!charray_inlist(origMatchingRulesArray, + addMatchingRulesArray[i])) { + charray_add(&origMatchingRulesArray, + slapi_ch_strdup(addMatchingRulesArray[i])); + } + } + } + + if (dodeletes) { + i = 0; + arglist[i++] = slapi_ch_strdup(attrValue->bv_val); + arglist[i++] = slapi_ch_strdup("none"); + arglist[i] = NULL; + attr_index_config(inst->inst_be, "from DSE modify", 0, i, arglist, 0); + + /* Free args */ + slapi_ch_free((void **)&arglist[0]); + slapi_ch_free((void **)&arglist[1]); + } + + i = 0; + arglist[i++] = slapi_ch_strdup(attrValue->bv_val); + if (origIndexTypesArray && origIndexTypesArray[0]) { + tmpBuf[0] = 0; + ZCAT_SAFE(tmpBuf, "", origIndexTypesArray[0]); + for (j = 1; origIndexTypesArray[j] != NULL; j++) { + ZCAT_SAFE(tmpBuf, ",", origIndexTypesArray[j]); + } + arglist[i++] = slapi_ch_strdup(tmpBuf); + } else { + arglist[i++] = slapi_ch_strdup("none"); + } + + if (origMatchingRulesArray && origMatchingRulesArray[0]) { + tmpBuf[0] = 0; + ZCAT_SAFE(tmpBuf, "", origMatchingRulesArray[0]); + for (j = 1; origMatchingRulesArray[j] != NULL; j++) { + ZCAT_SAFE(tmpBuf, ",", origMatchingRulesArray[j]); + } + arglist[i++] = slapi_ch_strdup(tmpBuf); + } + + arglist[i] = NULL; + attr_index_config(inst->inst_be, "from DSE modify", 0, i, arglist, 0); + + /* Free args */ + for (i=0; arglist[i]; i++) { + slapi_ch_free((void **)&arglist[i]); + } + + if(origIndexTypesArray) { + charray_free(origIndexTypesArray); + } + if(origMatchingRulesArray) { + charray_free(origMatchingRulesArray); + } + if(addIndexTypesArray) { + charray_free(addIndexTypesArray); + } + if(deleteIndexTypesArray) { + charray_free(deleteIndexTypesArray); + } + if(addMatchingRulesArray) { + charray_free(addMatchingRulesArray); + } + if(deleteMatchingRulesArray) { + charray_free(deleteMatchingRulesArray); + } + if (origIndexTypes) { + slapi_ch_free ((void **)&origIndexTypes); + } + if (origMatchingRules) { + slapi_ch_free ((void **)&origMatchingRules); + } + + return SLAPI_DSE_CALLBACK_OK; +} + +/* add index entries to the per-instance DSE (used only from instance.c) */ +int ldbm_instance_config_add_index_entry( + ldbm_instance *inst, + int argc, + char **argv, + int flags +) +{ + char **attrs = NULL; + char **indexes = NULL; + char **matchingRules = NULL; + char eBuf[BUFSIZ]; + int i = 0; + int j = 0; + char *basetype = NULL; + char tmpAttrsStr[256]; + char tmpIndexesStr[256]; + char tmpMatchingRulesStr[1024]; + struct ldbminfo *li = inst->inst_li; + + if ((argc < 2) || (NULL == argv) || (NULL == argv[0]) || + (NULL == argv[1])) { + return(-1); + } + + strcpy(tmpAttrsStr,argv[0]); + attrs = str2charray( tmpAttrsStr, "," ); + strcpy(tmpIndexesStr,argv[1]); + indexes = str2charray( tmpIndexesStr, ","); + + if(argc > 2) { + strcpy(tmpMatchingRulesStr,argv[2]); + matchingRules = str2charray( tmpMatchingRulesStr, ","); + } + + for(i=0; attrs[i] !=NULL; i++) + { + if('\0' == attrs[i][0]) continue; + basetype = slapi_attr_basetype(attrs[i], NULL, 0); + sprintf(eBuf, + "dn: cn=%s, cn=index, cn=%s, cn=%s, cn=plugins, cn=config\n" + "objectclass:top\n" + "objectclass:nsIndex\n" + "cn:%s\n" + "nsSystemIndex:%s\n", + basetype, inst->inst_name, li->li_plugin->plg_name, + basetype, + (ldbm_attribute_always_indexed(basetype)?"true":"false")); + for(j=0; indexes[j] != NULL; j++) + { + strcat(eBuf, "nsIndexType:"); + strcat(eBuf,indexes[j]); + strcat(eBuf,"\n"); + } + if((argc>2)&&(argv[2])) + { + for(j=0; matchingRules[j] != NULL; j++) + { + strcat(eBuf,"nsMatchingRule:"); + strcat(eBuf,matchingRules[j]); + strcat(eBuf,"\n"); + } + } + + ldbm_config_add_dse_entry(li, eBuf, flags); + + slapi_ch_free((void**)&basetype); + } + + if(NULL != attrs) { + charray_free(attrs); + } + if(NULL != indexes) { + charray_free(indexes); + } + if(NULL != matchingRules) { + charray_free(matchingRules); + } + return (0); +} + +int +ldbm_instance_index_config_enable_index(ldbm_instance *inst, Slapi_Entry* e) +{ + char *index_name; + int rc; + + rc=ldbm_index_parse_entry(inst, e, "from DSE add", &index_name); + if (rc == LDAP_SUCCESS) { + struct attrinfo *ai = NULL; + + /* Assume the caller knows if it is OK to go online immediatly */ + + ainfo_get(inst->inst_be, index_name, &ai); + PR_ASSERT(ai != NULL); + ai->ai_indexmask &= ~INDEX_OFFLINE; + slapi_ch_free((void **)&index_name); + } + return rc; +} + + +/* +** create the default user-defined indexes +*/ + +int ldbm_instance_create_default_user_indexes(ldbm_instance *inst) +{ + + /* + ** Search for user-defined default indexes and add them + ** to the backend instance beeing created. + */ + + Slapi_PBlock *aPb; + Slapi_Entry **entries = NULL; + Slapi_Attr *attr; + Slapi_Value *sval = NULL; + const struct berval *attrValue; + char *argv[ 8 ]; + char basedn[BUFSIZ]; + char tmpBuf[MAX_TMPBUF]; + char tmpBuf2[MAX_TMPBUF]; + int argc; + + struct ldbminfo *li; + + /* write the dse file only on the final index */ + int flags = LDBM_INSTANCE_CONFIG_DONT_WRITE; + + if (NULL == inst) { + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: can't initialize default user indexes (invalid instance).\n", 0,0,0); + return -1; + } + + li = inst->inst_li; + strcpy(tmpBuf,""); + + /* Construct the base dn of the subtree that holds the default user indexes. */ + sprintf(basedn, "cn=default indexes, cn=config, cn=%s, cn=plugins, cn=config", + li->li_plugin->plg_name); + + /* Do a search of the subtree containing the index entries */ + aPb = slapi_pblock_new(); + slapi_search_internal_set_pb(aPb, basedn, LDAP_SCOPE_SUBTREE, + "(objectclass=nsIndex)", NULL, 0 , NULL, NULL, li->li_identity, 0); + slapi_search_internal_pb (aPb); + slapi_pblock_get(aPb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries); + if (entries!=NULL) { + int i,j; + for (i=0; entries[i]!=NULL; i++) { + + /* Get the name of the attribute to index which will be the value + * of the cn attribute. */ + + if (slapi_entry_attr_find(entries[i], "cn", &attr) != 0) { + LDAPDebug(LDAP_DEBUG_ANY,"Warning: malformed index entry %s. Index ignored.\n", + slapi_entry_get_dn(entries[i]), 0, 0); + continue; + } + slapi_attr_first_value(attr, &sval); + attrValue = slapi_value_get_berval(sval); + argv[0] = attrValue->bv_val; + argc=1; + + /* Get the list of index types from the entry. */ + + if (0 == slapi_entry_attr_find(entries[i], "nsIndexType", &attr)) { + for (j = slapi_attr_first_value(attr, &sval); j != -1; + j = slapi_attr_next_value(attr, j, &sval)) { + attrValue = slapi_value_get_berval(sval); + if (0 == j) { + tmpBuf[0] = 0; + ZCAT_SAFE(tmpBuf, "", attrValue->bv_val); + } else { + ZCAT_SAFE(tmpBuf, ",", attrValue->bv_val); + } + } + argv[argc]=tmpBuf; + argc++; + } + + /* Get the list of matching rules from the entry. */ + + if (0 == slapi_entry_attr_find(entries[i], "nsMatchingRule", &attr)) { + for (j = slapi_attr_first_value(attr, &sval); j != -1; + j = slapi_attr_next_value(attr, j, &sval)) { + attrValue = slapi_value_get_berval(sval); + if (0 == j) { + tmpBuf2[0] = 0; + ZCAT_SAFE(tmpBuf2, "", attrValue->bv_val); + } else { + ZCAT_SAFE(tmpBuf2, ",", attrValue->bv_val); + } + } + argv[argc]=tmpBuf2; + argc++; + } + + argv[argc]=NULL; + + /* Create the index entry in the backend */ + + if (entries[i+1] == NULL) { + /* write the dse file only on the final index */ + flags = 0; + } + + ldbm_instance_config_add_index_entry(inst, argc, argv, flags); + + /* put the index online */ + + ldbm_instance_index_config_enable_index(inst, entries[i]); + } + } + + slapi_free_search_results_internal(aPb); + slapi_pblock_destroy(aPb); + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c b/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c new file mode 100644 index 00000000..a5819d6f --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c @@ -0,0 +1,997 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* This file handles configuration information that is specific + * to ldbm instances. + */ + +#include "back-ldbm.h" +#include "dblayer.h" + +/* Forward declarations for the callbacks */ +int ldbm_instance_search_config_entry_callback(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_instance_modify_config_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); + +static char *ldbm_instance_attrcrypt_filter = "(objectclass=nsAttributeEncryption)"; + +/* dse entries add for a new ldbm instance */ +static char *ldbm_instance_skeleton_entries[] = +{ + "dn:cn=monitor, cn=%s, cn=%s, cn=plugins, cn=config\n" + "objectclass:top\n" + "objectclass:extensibleObject\n" + "cn:monitor\n", + + "dn:cn=index, cn=%s, cn=%s, cn=plugins, cn=config\n" + "objectclass:top\n" + "objectclass:extensibleObject\n" + "cn:index\n", + + "dn:cn=encrypted attributes, cn=%s, cn=%s, cn=plugins, cn=config\n" + "objectclass:top\n" + "objectclass:extensibleObject\n" + "cn:encrypted attributes\n", + + "dn:cn=encrypted attribute keys, cn=%s, cn=%s, cn=plugins, cn=config\n" + "objectclass:top\n" + "objectclass:extensibleObject\n" + "cn:encrypted attribute keys\n", + + "" +}; + + +/*------------------------------------------------------------------------ + * Get and set functions for ldbm instance variables + *----------------------------------------------------------------------*/ +static void * +ldbm_instance_config_cachesize_get(void *arg) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + + return (void *) cache_get_max_entries(&(inst->inst_cache)); +} + +static int +ldbm_instance_config_cachesize_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + int retval = LDAP_SUCCESS; + long val = (long) value; + + /* Do whatever we can to make sure the data is ok. */ + + if (apply) { + cache_set_max_entries(&(inst->inst_cache), val); + } + + return retval; +} + +static void * +ldbm_instance_config_cachememsize_get(void *arg) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + + return (void *) cache_get_max_size(&(inst->inst_cache)); +} + +static int +ldbm_instance_config_cachememsize_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + ldbm_instance *inst = (ldbm_instance *) arg; + int retval = LDAP_SUCCESS; + size_t val = (size_t) value; + + /* Do whatever we can to make sure the data is ok. */ + + if (apply) { + cache_set_max_size(&(inst->inst_cache), val); + } + + return retval; +} + +static void * +ldbm_instance_config_readonly_get(void *arg) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + + return (void *)inst->inst_be->be_readonly; +} + +static void * +ldbm_instance_config_instance_dir_get(void *arg) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + + if (inst->inst_dir_name == NULL) + return slapi_ch_strdup(""); + else if (inst->inst_parent_dir_name) + { + int len = strlen(inst->inst_parent_dir_name) + + strlen(inst->inst_dir_name) + 2; + char *full_inst_dir = (char *)slapi_ch_malloc(len); + sprintf(full_inst_dir, "%s%c%s", + inst->inst_parent_dir_name, get_sep(inst->inst_parent_dir_name), + inst->inst_dir_name); + return full_inst_dir; + } + else + return slapi_ch_strdup(inst->inst_dir_name); +} + +static void * +ldbm_instance_config_require_index_get(void *arg) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + + return (void *)inst->require_index; +} + +static int +ldbm_instance_config_instance_dir_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + + if (!apply) { + return LDAP_SUCCESS; + } + + if ((value == NULL) || (strlen(value) == 0)) + { + inst->inst_dir_name = NULL; + inst->inst_parent_dir_name = NULL; + } + else + { + char *dir = (char *)value; + if (is_fullpath(dir)) + { + char sep = get_sep(dir); + char *p = strrchr(dir, sep); + if (NULL == p) /* never happens, tho */ + { + inst->inst_parent_dir_name = NULL; + inst->inst_dir_name = slapi_ch_strdup(dir); + } + else + { + *p = '\0'; + inst->inst_parent_dir_name = slapi_ch_strdup(dir); + inst->inst_dir_name = slapi_ch_strdup(p+1); + *p = sep; + } + } + else + { + inst->inst_parent_dir_name = NULL; + inst->inst_dir_name = slapi_ch_strdup(dir); + } + } + return LDAP_SUCCESS; +} + +static int +ldbm_instance_config_readonly_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + + if (!apply) { + return LDAP_SUCCESS; + } + + if (CONFIG_PHASE_RUNNING == phase) { + /* if the instance is busy, we'll save the user's readonly settings + * but won't change them until the instance is un-busy again. + */ + if (! (inst->inst_flags & INST_FLAG_BUSY)) { + slapi_mtn_be_set_readonly(inst->inst_be, (int)value); + } + if ((int)value) { + inst->inst_flags |= INST_FLAG_READONLY; + } else { + inst->inst_flags &= ~INST_FLAG_READONLY; + } + } else { + slapi_be_set_readonly(inst->inst_be, (int)value); + } + + return LDAP_SUCCESS; +} + +static int +ldbm_instance_config_require_index_set(void *arg, void *value, char *errorbuf, int phase, int apply) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + + if (!apply) { + return LDAP_SUCCESS; + } + + inst->require_index = (int)value; + + return LDAP_SUCCESS; +} + + +/*------------------------------------------------------------------------ + * ldbm instance configuration array + *----------------------------------------------------------------------*/ +static config_info ldbm_instance_config[] = { + {CONFIG_INSTANCE_CACHESIZE, CONFIG_TYPE_LONG, "-1", &ldbm_instance_config_cachesize_get, &ldbm_instance_config_cachesize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_INSTANCE_CACHEMEMSIZE, CONFIG_TYPE_SIZE_T, "10485760", &ldbm_instance_config_cachememsize_get, &ldbm_instance_config_cachememsize_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_INSTANCE_READONLY, CONFIG_TYPE_ONOFF, "off", &ldbm_instance_config_readonly_get, &ldbm_instance_config_readonly_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_INSTANCE_REQUIRE_INDEX, CONFIG_TYPE_ONOFF, "off", &ldbm_instance_config_require_index_get, &ldbm_instance_config_require_index_set, CONFIG_FLAG_ALWAYS_SHOW|CONFIG_FLAG_ALLOW_RUNNING_CHANGE}, + {CONFIG_INSTANCE_DIR, CONFIG_TYPE_STRING, NULL, &ldbm_instance_config_instance_dir_get, &ldbm_instance_config_instance_dir_set, CONFIG_FLAG_ALWAYS_SHOW}, + {NULL, 0, NULL, NULL, NULL, 0} +}; + +void +ldbm_instance_config_setup_default(ldbm_instance *inst) +{ + config_info *config; + char err_buf[BUFSIZ]; + + for (config = ldbm_instance_config; config->config_name != NULL; config++) { + ldbm_config_set((void *)inst, config->config_name, ldbm_instance_config, NULL /* use default */, err_buf, CONFIG_PHASE_INITIALIZATION, 1 /* apply */); + } +} + +static int +parse_ldbm_instance_entry(Slapi_Entry *e, char **instance_name) +{ + Slapi_Attr *attr = NULL; + + for (slapi_entry_first_attr(e, &attr); attr; + slapi_entry_next_attr(e, attr, &attr)) { + char *attr_name = NULL; + + slapi_attr_get_type(attr, &attr_name); + if (strcasecmp(attr_name, "cn") == 0) { + Slapi_Value *sval = NULL; + struct berval *bval; + slapi_attr_first_value(attr, &sval); + bval = (struct berval *) slapi_value_get_berval(sval); + *instance_name = slapi_ch_strdup((char *)bval->bv_val); + } + } + return 0; +} + +/* When a new instance is started, we need to read the dse to + * find out what indexes should be maintained. This function + * does that. Returns 0 on success. */ +static int +read_instance_index_entries(ldbm_instance *inst) +{ + Slapi_PBlock *tmp_pb; + int scope = LDAP_SCOPE_SUBTREE; + char basedn[BUFSIZ]; + const char *searchfilter = "(objectclass=nsIndex)"; + + /* Construct the base dn of the subtree that holds the index entries + * for this instance. */ + sprintf(basedn, "cn=index, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, inst->inst_li->li_plugin->plg_name); + + /* Set up a tmp callback that will handle the init for each index entry */ + slapi_config_register_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, + basedn, scope, searchfilter, ldbm_index_init_entry_callback, + (void *) inst); + + /* Do a search of the subtree containing the index entries */ + tmp_pb = slapi_pblock_new(); + slapi_search_internal_set_pb(tmp_pb, basedn, LDAP_SCOPE_SUBTREE, + searchfilter, NULL, 0, NULL, NULL, inst->inst_li->li_identity, 0); + slapi_search_internal_pb (tmp_pb); + + /* Remove the tmp callback */ + slapi_config_remove_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, + basedn, scope, searchfilter, ldbm_index_init_entry_callback); + slapi_free_search_results_internal(tmp_pb); + slapi_pblock_destroy(tmp_pb); + + return 0; +} + +/* When a new instance is started, we need to read the dse to + * find out what attributes should be encrypted. This function + * does that. Returns 0 on success. */ +static int +read_instance_attrcrypt_entries(ldbm_instance *inst) +{ + Slapi_PBlock *tmp_pb; + int scope = LDAP_SCOPE_SUBTREE; + char basedn[BUFSIZ]; + const char *searchfilter = ldbm_instance_attrcrypt_filter; + + /* Construct the base dn of the subtree that holds the index entries + * for this instance. */ + sprintf(basedn, "cn=encrypted attributes, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, inst->inst_li->li_plugin->plg_name); + + /* Set up a tmp callback that will handle the init for each index entry */ + slapi_config_register_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, + basedn, scope, searchfilter, ldbm_attrcrypt_init_entry_callback, + (void *) inst); + + /* Do a search of the subtree containing the index entries */ + tmp_pb = slapi_pblock_new(); + slapi_search_internal_set_pb(tmp_pb, basedn, LDAP_SCOPE_SUBTREE, + searchfilter, NULL, 0, NULL, NULL, inst->inst_li->li_identity, 0); + slapi_search_internal_pb (tmp_pb); + + /* Remove the tmp callback */ + slapi_config_remove_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, + basedn, scope, searchfilter, ldbm_attrcrypt_init_entry_callback); + slapi_free_search_results_internal(tmp_pb); + slapi_pblock_destroy(tmp_pb); + + return 0; +} + +/* Handles the parsing of the config entry for an ldbm instance. Returns 0 + * on success. */ +static int +parse_ldbm_instance_config_entry(ldbm_instance *inst, Slapi_Entry *e, config_info *config_array) +{ + Slapi_Attr *attr = NULL; + + for (slapi_entry_first_attr(e, &attr); attr; + slapi_entry_next_attr(e, attr, &attr)) { + char *attr_name = NULL; + Slapi_Value *sval = NULL; + struct berval *bval; + char err_buf[BUFSIZ]; + + slapi_attr_get_type(attr, &attr_name); + + /* There are some attributes that we don't care about, + * like objectclass. */ + if (ldbm_config_ignored_attr(attr_name)) { + continue; + } + + /* We have to handle suffix attributes a little differently */ + if (strcasecmp(attr_name, CONFIG_INSTANCE_SUFFIX) == 0) { + Slapi_DN suffix; + + slapi_attr_first_value(attr, &sval); + bval = (struct berval *) slapi_value_get_berval(sval); + slapi_sdn_init_dn_byref(&suffix, bval->bv_val); + if (!slapi_be_issuffix(inst->inst_be, &suffix)) { + be_addsuffix(inst->inst_be, &suffix); + } + slapi_sdn_done(&suffix); + continue; + } + + /* We are assuming that each of these attributes are to have + * only one value. If they have more than one value, like + * the nsslapd-suffix attribute, then they need to be + * handled differently. */ + slapi_attr_first_value(attr, &sval); + bval = (struct berval *) slapi_value_get_berval(sval); + + if (ldbm_config_set((void *) inst, attr_name, config_array, bval, + err_buf, CONFIG_PHASE_STARTUP, 1 /* apply */) != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "Error with config attribute %s : %s\n", + attr_name, err_buf, 0); + return 1; + } + } + + /* Read the index entries */ + read_instance_index_entries(inst); + /* Read the attribute encryption entries */ + read_instance_attrcrypt_entries(inst); + + return 0; +} + +/* general-purpose callback to deny an operation */ +static int ldbm_instance_deny_config(Slapi_PBlock *pb, Slapi_Entry *e, + Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + *returncode = LDAP_UNWILLING_TO_PERFORM; + return SLAPI_DSE_CALLBACK_ERROR; +} + +/* Reads in any config information held in the dse for the given + * entry. Creates dse entries used to configure the given instance + * if they don't already exist. Registers dse callback functions to + * maintain those dse entries. Returns 0 on success. */ +int +ldbm_instance_config_load_dse_info(ldbm_instance *inst) +{ + struct ldbminfo *li = inst->inst_li; + Slapi_PBlock *search_pb; + Slapi_Entry **entries = NULL; + int res; + char dn[BUFSIZ]; + + /* We try to read the entry + * cn=instance_name, cn=ldbm database, cn=plugins, cn=config. If the + * entry is there, then we process the config information it stores. + */ + sprintf(dn, "cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + search_pb = slapi_pblock_new(); + slapi_search_internal_set_pb(search_pb, dn, LDAP_SCOPE_BASE, + "objectclass=*", NULL, 0, NULL, NULL, + li->li_identity, 0); + slapi_search_internal_pb (search_pb); + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_RESULT, &res); + + if (res != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "Error accessing the config DSE\n", 0, 0, 0); + return 1; + } else { + /* Need to parse the configuration information for the ldbm + * plugin that is held in the DSE. */ + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, + &entries); + if ((!entries) || (!entries[0])) { + LDAPDebug(LDAP_DEBUG_ANY, "Error accessing the config DSE\n", + 0, 0, 0); + return 1; + } + parse_ldbm_instance_config_entry(inst, entries[0], + ldbm_instance_config); + } + + if (search_pb) + { + slapi_free_search_results_internal(search_pb); + slapi_pblock_destroy(search_pb); + } + + /* now check for cn=monitor -- if not present, add default child entries */ + search_pb = slapi_pblock_new(); + sprintf(dn, "cn=monitor, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + slapi_search_internal_set_pb(search_pb, dn, LDAP_SCOPE_BASE, + "objectclass=*", NULL, 0, NULL, NULL, + li->li_identity, 0); + slapi_search_internal_pb(search_pb); + slapi_pblock_get(search_pb, SLAPI_PLUGIN_INTOP_RESULT, &res); + + if (res == LDAP_NO_SUCH_OBJECT) { + /* Add skeleton dse entries for this instance */ + ldbm_config_add_dse_entries(li, ldbm_instance_skeleton_entries, + inst->inst_name, li->li_plugin->plg_name, + inst->inst_name, 0); + } + + if (search_pb) { + slapi_free_search_results_internal(search_pb); + slapi_pblock_destroy(search_pb); + } + + /* setup the dse callback functions for the ldbm instance config entry */ + sprintf(dn, "cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + slapi_config_register_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", + ldbm_instance_search_config_entry_callback, (void *) inst); + slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", + ldbm_instance_modify_config_entry_callback, (void *) inst); + slapi_config_register_callback(DSE_OPERATION_WRITE, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", + ldbm_instance_search_config_entry_callback, (void *) inst); + slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", + ldbm_instance_deny_config, (void *)inst); + /* delete is handled by a callback set in ldbm_config.c */ + + /* don't forget the monitor! */ + sprintf(dn, "cn=monitor, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + /* make callback on search; deny add/modify/delete */ + slapi_config_register_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_back_monitor_instance_search, + (void *)inst); + slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=*)", ldbm_instance_deny_config, + (void *)inst); + slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_instance_deny_config, + (void *)inst); + /* delete is okay */ + + /* Callbacks to handle indexes */ + sprintf(dn, "cn=index, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsIndex)", + ldbm_instance_index_config_add_callback, (void *) inst); + slapi_config_register_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsIndex)", + ldbm_instance_index_config_delete_callback, (void *) inst); + slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsIndex)", + ldbm_instance_index_config_modify_callback, (void *) inst); + + /* Callbacks to handle attribute encryption */ + sprintf(dn, "cn=encrypted attributes, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + slapi_config_register_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, ldbm_instance_attrcrypt_filter, + ldbm_instance_attrcrypt_config_add_callback, (void *) inst); + slapi_config_register_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, ldbm_instance_attrcrypt_filter, + ldbm_instance_attrcrypt_config_delete_callback, (void *) inst); + slapi_config_register_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, ldbm_instance_attrcrypt_filter, + ldbm_instance_attrcrypt_config_modify_callback, (void *) inst); + + return 0; +} + +/* + * Config. DSE callback for instance entry searches. + */ +int +ldbm_instance_search_config_entry_callback(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + char buf[BUFSIZ]; + struct berval *vals[2]; + struct berval val; + ldbm_instance *inst = (ldbm_instance *) arg; + config_info *config; + int x; + const Slapi_DN *suffix; + + vals[0] = &val; + vals[1] = NULL; + + returntext[0] = '\0'; + + /* show the suffixes */ + attrlist_delete(&e->e_attrs, CONFIG_INSTANCE_SUFFIX); + x = 0; + do { + suffix = slapi_be_getsuffix(inst->inst_be, x); + if (suffix != NULL) { + val.bv_val = (char *) slapi_sdn_get_dn(suffix); + val.bv_len = strlen (val.bv_val); + attrlist_merge( &e->e_attrs, CONFIG_INSTANCE_SUFFIX, vals ); + } + x++; + } while(suffix!=NULL); + + PR_Lock(inst->inst_config_mutex); + + for(config = ldbm_instance_config; config->config_name != NULL; config++) { + /* Go through the ldbm_config table and fill in the entry. */ + + if (!(config->config_flags & (CONFIG_FLAG_ALWAYS_SHOW | CONFIG_FLAG_PREVIOUSLY_SET))) { + /* This config option shouldn't be shown */ + continue; + } + + ldbm_config_get((void *) inst, config, buf); + + val.bv_val = buf; + val.bv_len = strlen(buf); + slapi_entry_attr_replace(e, config->config_name, vals); + } + + PR_Unlock(inst->inst_config_mutex); + + *returncode = LDAP_SUCCESS; + return SLAPI_DSE_CALLBACK_OK; +} + +/* This function is used by the instance modify callback to add a new + * suffix. It return LDAP_SUCCESS on success. + */ +int +add_suffix(ldbm_instance *inst, struct berval **bvals, int apply_mod, char *returntext) +{ + Slapi_DN suffix; + int x; + + returntext[0] = '\0'; + for (x = 0; bvals[x]; x++) { + slapi_sdn_init_dn_byref(&suffix, bvals[x]->bv_val); + if (!slapi_be_issuffix(inst->inst_be, &suffix) && apply_mod) { + be_addsuffix(inst->inst_be, &suffix); + } + slapi_sdn_done(&suffix); + } + + return LDAP_SUCCESS; +} + +/* + * Config. DSE callback for instance entry modifies. + */ +int +ldbm_instance_modify_config_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg) +{ + int i; + char *attr_name; + LDAPMod **mods; + int rc = LDAP_SUCCESS; + int apply_mod = 0; + ldbm_instance *inst = (ldbm_instance *) arg; + + /* This lock is probably way too conservative, but we don't expect much + * contention for it. */ + PR_Lock(inst->inst_config_mutex); + + slapi_pblock_get( pb, SLAPI_MODIFY_MODS, &mods ); + + returntext[0] = '\0'; + + /* + * First pass: set apply mods to 0 so only input validation will be done; + * 2nd pass: set apply mods to 1 to apply changes to internal storage + */ + for ( apply_mod = 0; apply_mod <= 1 && LDAP_SUCCESS == rc; apply_mod++ ) { + for (i = 0; mods[i] && LDAP_SUCCESS == rc; i++) { + attr_name = mods[i]->mod_type; + + if (strcasecmp(attr_name, CONFIG_INSTANCE_SUFFIX) == 0) { + /* naughty naughty, we don't allow this */ + rc = LDAP_UNWILLING_TO_PERFORM; + if (returntext) { + sprintf(returntext, + "Can't change the root suffix of a backend"); + } + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm: modify attempted to change the root suffix " + "of a backend (which is not allowed)\n", + 0, 0, 0); + continue; + } + + /* There are some attributes that we don't care about, like + * modifiersname. */ + if (ldbm_config_ignored_attr(attr_name)) { + continue; + } + + if ((mods[i]->mod_op & LDAP_MOD_DELETE) || + (mods[i]->mod_op & LDAP_MOD_ADD)) { + rc= LDAP_UNWILLING_TO_PERFORM; + sprintf(returntext, "%s attributes is not allowed", + (mods[i]->mod_op & LDAP_MOD_DELETE) ? + "Deleting" : "Adding"); + } else if (mods[i]->mod_op & LDAP_MOD_REPLACE) { + /* This assumes there is only one bval for this mod. */ + rc = ldbm_config_set((void *) inst, attr_name, + ldbm_instance_config, mods[i]->mod_bvalues[0], returntext, + CONFIG_PHASE_RUNNING, apply_mod); + } + } + } + + PR_Unlock(inst->inst_config_mutex); + + *returncode = rc; + if (LDAP_SUCCESS == rc) { + return SLAPI_DSE_CALLBACK_OK; + } else { + return SLAPI_DSE_CALLBACK_ERROR; + } +} + +/* This function is used to set instance config attributes. It can be used as a + * shortcut to doing an internal modify operation on the config DSE. + */ +void +ldbm_instance_config_internal_set(ldbm_instance *inst, char *attrname, char *value) +{ + char err_buf[BUFSIZ]; + struct berval bval; + + bval.bv_val = value; + bval.bv_len = strlen(value); + + if (ldbm_config_set((void *) inst, attrname, ldbm_instance_config, &bval, + err_buf, CONFIG_PHASE_INTERNAL, 1 /* apply */) != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, + "Internal Error: Error setting instance config attr %s to %s: %s\n", + attrname, value, err_buf); + exit(1); + } +} + + + +static int ldbm_instance_generate(struct ldbminfo *li, char *instance_name, + Slapi_Backend **ret_be) +{ + Slapi_Backend *new_be = NULL; + int rc = 0; + + /* Create a new instance, process config info for it, + * and then call slapi_be_new and create a new backend here + */ + new_be = slapi_be_new(LDBM_DATABASE_TYPE_NAME /* type */, instance_name, + 0 /* public */, 1 /* do log changes */); + new_be->be_database = li->li_plugin; + ldbm_instance_create(new_be, instance_name); + + ldbm_instance_config_load_dse_info(new_be->be_instance_info); + rc = ldbm_instance_create_default_indexes(new_be); + + if (ret_be != NULL) { + *ret_be = new_be; + } + + return rc; +} + +int +ldbm_instance_postadd_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + backend *be = NULL; + struct ldbm_instance *inst; + char *instance_name; + struct ldbminfo *li = (struct ldbminfo *)arg; + int rval = 0; + + parse_ldbm_instance_entry(entryBefore, &instance_name); + ldbm_instance_generate(li, instance_name, &be); + + inst = ldbm_instance_find_by_name(li, instance_name); + + /* Add default indexes */ + ldbm_instance_create_default_user_indexes(inst); + + /* Initialize and register callbacks for VLV indexes */ + vlv_init(inst); + + /* this is an ACTUAL ADD being done while the server is running! + * start up the appropriate backend... + */ + rval = ldbm_instance_start(be); + if (0 != rval) + { + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm_instance_postadd_instance_entry_callback: " + "ldbm_instnace_start (%s) failed (%d)\n", + instance_name, rval, 0); + } + + slapi_ch_free((void **)&instance_name); + + /* instance must be fully ready before we call this */ + slapi_mtn_be_started(be); + + return SLAPI_DSE_CALLBACK_OK; +} + +int +ldbm_instance_add_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + char *instance_name; + struct ldbm_instance *inst= NULL; + struct ldbminfo *li= (struct ldbminfo *) arg; + int rc = 0; + + parse_ldbm_instance_entry(entryBefore, &instance_name); + + /* Make sure we don't create two instances with the same name. */ + inst = ldbm_instance_find_by_name(li, instance_name); + if (inst != NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "WARNING: ldbm instance %s already exists\n", + instance_name, 0, 0); + if (returntext != NULL) + sprintf(returntext, "An ldbm instance with the name %s already exists\n", + instance_name); + if (returncode != NULL) + *returncode = LDAP_UNWILLING_TO_PERFORM; + slapi_ch_free((void **)&instance_name); + return SLAPI_DSE_CALLBACK_ERROR; + } + + if (pb == NULL) { + /* called during startup -- do the rest now */ + rc = ldbm_instance_generate(li, instance_name, NULL); + } + /* if called during a normal ADD operation, the postadd callback + * will do the rest. + */ + + slapi_ch_free((void **)&instance_name); + return (rc == 0) ? SLAPI_DSE_CALLBACK_OK : SLAPI_DSE_CALLBACK_ERROR; +} + + + + +/* unregister the DSE callbacks on a backend -- this needs to be done when + * deleting a backend, so that adding the same backend later won't cause + * these expired callbacks to be called. + */ +static void ldbm_instance_unregister_callbacks(ldbm_instance *inst) +{ + struct ldbminfo *li = inst->inst_li; + char dn[BUFSIZ]; + + /* tear down callbacks for the instance config entry */ + sprintf(dn, "cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + slapi_config_remove_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", + ldbm_instance_search_config_entry_callback); + slapi_config_remove_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", + ldbm_instance_modify_config_entry_callback); + slapi_config_remove_callback(DSE_OPERATION_WRITE, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", + ldbm_instance_search_config_entry_callback); + slapi_config_remove_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", + ldbm_instance_deny_config); + + /* now the cn=monitor entry */ + sprintf(dn, "cn=monitor, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + slapi_config_remove_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_back_monitor_instance_search); + slapi_config_remove_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=*)", ldbm_instance_deny_config); + slapi_config_remove_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_BASE, "(objectclass=*)", ldbm_instance_deny_config); + + /* now the cn=index entries */ + sprintf(dn, "cn=index, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + slapi_config_remove_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsIndex)", + ldbm_instance_index_config_add_callback); + slapi_config_remove_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsIndex)", + ldbm_instance_index_config_delete_callback); + slapi_config_remove_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, "(objectclass=nsIndex)", + ldbm_instance_index_config_modify_callback); + + /* now the cn=encrypted attributes entries */ + sprintf(dn, "cn=encrypted attributes, cn=%s, cn=%s, cn=plugins, cn=config", + inst->inst_name, li->li_plugin->plg_name); + slapi_config_remove_callback(SLAPI_OPERATION_ADD, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, ldbm_instance_attrcrypt_filter, + ldbm_instance_attrcrypt_config_add_callback); + slapi_config_remove_callback(SLAPI_OPERATION_DELETE, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, ldbm_instance_attrcrypt_filter, + ldbm_instance_attrcrypt_config_delete_callback); + slapi_config_remove_callback(SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, dn, + LDAP_SCOPE_SUBTREE, ldbm_instance_attrcrypt_filter, + ldbm_instance_attrcrypt_config_modify_callback); + + vlv_remove_callbacks(inst); +} + + +int +ldbm_instance_post_delete_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + char *instance_name; + struct ldbminfo *li = (struct ldbminfo *)arg; + struct ldbm_instance *inst = NULL; + + parse_ldbm_instance_entry(entryBefore, &instance_name); + inst = ldbm_instance_find_by_name(li, instance_name); + + if (inst == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: instance '%s' does not exist! (2)\n", + instance_name, 0, 0); + if (returntext) { + sprintf(returntext, "No ldbm instance exists with the name '%s' (2)\n", + instance_name); + } + if (returncode) { + *returncode = LDAP_UNWILLING_TO_PERFORM; + } + slapi_ch_free((void **)&instance_name); + return SLAPI_DSE_CALLBACK_ERROR; + } + + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: removing '%s'.\n", instance_name, 0, 0); + + { + struct ldbminfo *li = (struct ldbminfo *) inst->inst_be->be_database->plg_private; + dblayer_private *priv = (dblayer_private*) li->li_dblayer_private; + struct dblayer_private_env *pEnv = priv->dblayer_env; + if(pEnv) { + PRDir *dirhandle = NULL; + char dbName[MAXPATHLEN*2]; + char *dbNamep = NULL; + char *p; + int dbbasenamelen, dbnamelen; + int rc; + if (inst->inst_dir_name == NULL){ + dblayer_get_instance_data_dir(inst->inst_be); + } + dirhandle = PR_OpenDir(inst->inst_dir_name); + /* the db dir instance may have been removed already */ + if (dirhandle){ + dbNamep = dblayer_get_full_inst_dir(li, inst, + dbName, MAXPATHLEN*2); + dbbasenamelen = strlen(dbNamep); + dbnamelen = dbbasenamelen + 14; /* "/id2entry.db#" + '\0' */ + if (dbnamelen > MAXPATHLEN*2) + { + dbNamep = (char *)slapi_ch_realloc(dbNamep, dbnamelen); + } + p = dbNamep + dbbasenamelen; + sprintf(p, "%c%s%s", get_sep(dbNamep), + "id2entry", LDBM_FILENAME_SUFFIX); + rc = dblayer_db_remove(pEnv, dbName, 0); + PR_ASSERT(rc == 0); + if (dbNamep != dbName) + slapi_ch_free_string(&dbNamep); + PR_CloseDir(dirhandle); + } /* non-null dirhandle */ + } /* non-null pEnv */ + } + + ldbm_instance_unregister_callbacks(inst); + slapi_be_free(&inst->inst_be); + ldbm_instance_destroy(inst); + slapi_ch_free((void **)&instance_name); + + return SLAPI_DSE_CALLBACK_OK; +} + +int +ldbm_instance_delete_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + char *instance_name; + struct ldbminfo *li = (struct ldbminfo *)arg; + struct ldbm_instance *inst = NULL; + + parse_ldbm_instance_entry(entryBefore, &instance_name); + inst = ldbm_instance_find_by_name(li, instance_name); + if (inst == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: instance '%s' does not exist!\n", + instance_name, 0, 0); + if (returntext) { + sprintf(returntext, "No ldbm instance exists with the name '%s'\n", + instance_name); + } + if (returncode) { + *returncode = LDAP_UNWILLING_TO_PERFORM; + } + slapi_ch_free((void **)&instance_name); + return SLAPI_DSE_CALLBACK_ERROR; + } + + /* check if some online task is happening */ + if (instance_set_busy(inst) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: '%s' is in the middle of a task. " + "Cancel the task or wait for it to finish, " + "then try again.\n", instance_name, 0, 0); + if (returntext) { + sprintf(returntext, "ldbm instance '%s' is in the middle of a " + "task. Cancel the task or wait for it to finish, " + "then try again.\n", instance_name); + } + if (returncode) { + *returncode = LDAP_UNWILLING_TO_PERFORM; + } + slapi_ch_free((void **)&instance_name); + return SLAPI_DSE_CALLBACK_ERROR; + } + + /* okay, we're gonna delete this database instance. take it offline. */ + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: Bringing %s offline...\n", + instance_name, 0, 0); + slapi_mtn_be_stopping(inst->inst_be); + dblayer_instance_close(inst->inst_be); + cache_destroy_please(&inst->inst_cache); + slapi_ch_free((void **)&instance_name); + + return SLAPI_DSE_CALLBACK_OK; +} + diff --git a/ldap/servers/slapd/back-ldbm/ldbm_modify.c b/ldap/servers/slapd/back-ldbm/ldbm_modify.c new file mode 100644 index 00000000..1df2e6fd --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_modify.c @@ -0,0 +1,567 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* modify.c - ldbm backend modify routine */ + +#include "back-ldbm.h" + +extern char *numsubordinates; +extern char *hassubordinates; + +static void remove_illegal_mods(LDAPMod **mods); +static int mods_have_effect (Slapi_Entry *entry, Slapi_Mods *smods); + +/* Modify context structure constructor, sans allocation */ +void modify_init(modify_context *mc,struct backentry *old_entry) +{ + /* Store the old entry */ + PR_ASSERT(NULL == mc->old_entry); + PR_ASSERT(NULL == mc->new_entry); + + mc->old_entry = old_entry; + mc->new_entry_in_cache = 0; +} + +int modify_apply_mods(modify_context *mc, Slapi_Mods *smods) +{ + int ret = 0; + /* Make a copy of the entry */ + PR_ASSERT(mc->old_entry != NULL); + PR_ASSERT(mc->new_entry == NULL); + mc->new_entry = backentry_dup(mc->old_entry); + PR_ASSERT(smods!=NULL); + if ( mods_have_effect (mc->new_entry->ep_entry, smods) ) { + ret = entry_apply_mods( mc->new_entry->ep_entry, slapi_mods_get_ldapmods_byref(smods)); + } + mc->smods= smods; + return ret; +} + +/* Modify context structure destructor */ +int modify_term(modify_context *mc,struct backend *be) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + + slapi_mods_free(&mc->smods); + /* Unlock and return entries */ + if (NULL != mc->old_entry) { + cache_unlock_entry(&inst->inst_cache, mc->old_entry); + cache_return( &(inst->inst_cache), &(mc->old_entry) ); + mc->old_entry= NULL; + } + if (mc->new_entry_in_cache) { + cache_return( &(inst->inst_cache), &(mc->new_entry) ); + } else { + backentry_free(&(mc->new_entry)); + } + mc->new_entry= NULL; + return 0; +} + +/* Modify context structure member to switch entries in the cache */ +int modify_switch_entries(modify_context *mc,backend *be) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int ret = 0; + if (mc->old_entry!=NULL && mc->new_entry!=NULL) { + ret = cache_replace(&(inst->inst_cache), mc->old_entry, mc->new_entry); + if (ret == 0) mc->new_entry_in_cache = 1; + } + return ret; +} + +/* This routine does that part of a modify operation which involves + updating the on-disk data: updates idices, id2entry. + Copes properly with DB_LOCK_DEADLOCK. The caller must be able to cope with + DB_LOCK_DEADLOCK returned. + The caller is presumed to proceed as follows: + Find the entry you want to modify; + Lock it for modify; + Make a copy of it; (call backentry_dup() ) + Apply modifications to the copy in memory (call entry_apply_mods() ) + begin transaction; + Do any other mods to on-disk data you want + Call this routine; + Commit transaction; + You pass it environment data: struct ldbminfo, pb (not sure why, but the vlv code seems to need it) + the copy of the entry before modfication, the entry after modification; + an LDAPMods array containing the modifications performed +*/ +int modify_update_all(backend *be, Slapi_PBlock *pb, + modify_context *mc, + back_txn *txn) +{ + static char *function_name = "modify_update_all"; + int retval = 0; + + /* + * Update the ID to Entry index. + * Note that id2entry_add replaces the entry, so the Entry ID stays the same. + */ + retval = id2entry_add( be, mc->new_entry, txn ); + if ( 0 != retval ) { + if (DB_LOCK_DEADLOCK != retval) + { + ldbm_nasty(function_name,66,retval); + } + goto error; + } + retval = index_add_mods( be, (const LDAPMod **)slapi_mods_get_ldapmods_byref(mc->smods), mc->old_entry, mc->new_entry, txn ); + if ( 0 != retval ) { + if (DB_LOCK_DEADLOCK != retval) + { + ldbm_nasty(function_name,65,retval); + } + goto error; + } + /* + * Remove the old entry from the Virtual List View indexes. + * Add the new entry to the Virtual List View indexes. + * Because the VLV code calls slapi_filter_test(), which requires a pb (why?), + * we allow the caller sans pb to get everything except vlv indexing. + */ + if (NULL != pb) { + retval= vlv_update_all_indexes(txn, be, pb, mc->old_entry, mc->new_entry); + if ( 0 != retval ) { + if (DB_LOCK_DEADLOCK != retval) + { + ldbm_nasty(function_name,64,retval); + } + goto error; + } + } +error: + return retval; +} + +int +ldbm_back_modify( Slapi_PBlock *pb ) +{ + backend *be; + ldbm_instance *inst; + struct ldbminfo *li; + struct backentry *e, *ec = NULL; + Slapi_Entry *postentry = NULL; + LDAPMod **mods; + back_txn txn; + back_txnid parent_txn; + int retval = -1; + char *msg; + char *errbuf = NULL; + int retry_count = 0; + int disk_full = 0; + int ldap_result_code= LDAP_SUCCESS; + char *ldap_result_message= NULL; + int rc = 0; + Slapi_Operation *operation; + int dblock_acquired= 0; + entry_address *addr; + int change_entry = 0; + int ec_in_cache = 0; + int is_fixup_operation= 0; + CSN *opcsn = NULL; + int repl_op; + + slapi_pblock_get( pb, SLAPI_BACKEND, &be); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_TARGET_ADDRESS, &addr ); + slapi_pblock_get( pb, SLAPI_MODIFY_MODS, &mods ); + slapi_pblock_get( pb, SLAPI_PARENT_TXN, (void**)&parent_txn ); + slapi_pblock_get( pb, SLAPI_OPERATION, &operation ); + slapi_pblock_get (pb, SLAPI_IS_REPLICATED_OPERATION, &repl_op); + is_fixup_operation = operation_is_flag_set(operation,OP_FLAG_REPL_FIXUP); + inst = (ldbm_instance *) be->be_instance_info; + + dblayer_txn_init(li,&txn); + + /* The dblock serializes writes to the database, + * which reduces deadlocking in the db code, + * which means that we run faster. + * + * But, this lock is re-enterant for the fixup + * operations that the URP code in the Replication + * plugin generates. + */ + if(SERIALLOCK(li) && !operation_is_flag_set(operation,OP_FLAG_REPL_FIXUP)) + { + dblayer_lock_backend(be); + dblock_acquired= 1; + } + + /* find and lock the entry we are about to modify */ + if ( (e = find_entry2modify( pb, be, addr, NULL )) == NULL ) { + ldap_result_code= -1; + goto error_return; /* error result sent by find_entry2modify() */ + } + + if ( !is_fixup_operation ) + { + opcsn = operation_get_csn (operation); + if (NULL == opcsn && operation->o_csngen_handler) + { + /* + * Current op is a user request. Opcsn will be assigned + * if the dn is in an updatable replica. + */ + opcsn = entry_assign_operation_csn ( pb, e->ep_entry, NULL ); + } + if (opcsn) + { + entry_set_maxcsn (e->ep_entry, opcsn); + } + } + + /* Save away a copy of the entry, before modifications */ + slapi_pblock_set( pb, SLAPI_ENTRY_PRE_OP, slapi_entry_dup( e->ep_entry )); + + if ( (ldap_result_code = plugin_call_acl_mods_access( pb, e->ep_entry, mods, &errbuf)) != LDAP_SUCCESS ) { + ldap_result_message= errbuf; + goto error_return; + } + + /* create a copy of the entry and apply the changes to it */ + if ( (ec = backentry_dup( e )) == NULL ) { + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + remove_illegal_mods(mods); + + /* ec is the entry that our bepreop should get to mess with */ + slapi_pblock_set( pb, SLAPI_MODIFY_EXISTING_ENTRY, ec->ep_entry ); + slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ldap_result_code); + plugin_call_plugins(pb, SLAPI_PLUGIN_BE_PRE_MODIFY_FN); + slapi_pblock_get(pb, SLAPI_RESULT_CODE, &ldap_result_code); + /* The Plugin may have messed about with some of the PBlock parameters... ie. mods */ + slapi_pblock_get( pb, SLAPI_MODIFY_MODS, &mods ); + + { + Slapi_Mods smods; + CSN *csn = operation_get_csn(operation); + slapi_mods_init_byref(&smods,mods); + if ( (change_entry = mods_have_effect (ec->ep_entry, &smods)) ) { + ldap_result_code = entry_apply_mods_wsi(ec->ep_entry, &smods, csn, operation_is_flag_set(operation,OP_FLAG_REPLICATED)); + /* + * XXXmcs: it would be nice to get back an error message from + * the above call so we could pass it along to the client, e.g., + * "duplicate value for attribute givenName." + */ + } else { + /* If the entry was not actually changed, we still need to + * set the SLAPI_ENTRY_POST_OP field in the pblock (post-op + * plugins expect that field to be present for all modify + * operations that return LDAP_SUCCESS). + */ + postentry = slapi_entry_dup( e->ep_entry ); + slapi_pblock_set ( pb, SLAPI_ENTRY_POST_OP, postentry ); + postentry = NULL; /* avoid removal/free in error_return code */ + } + slapi_mods_done(&smods); + if ( !change_entry || ldap_result_code != 0 ) { + /* change_entry == 0 is not an error, but we need to free lock etc */ + goto error_return; + } + } + + /* + * If we are not handling a replicated operation, AND if the + * objectClass attribute type was modified in any way, expand + * the objectClass values to reflect the inheritance hierarchy. + * [blackflag 624152]: repl_op covers both regular and legacy replication + */ + if(!repl_op) + { + int i; + + for ( i = 0; mods[i] != NULL; ++i ) { + if ( 0 == strcasecmp( SLAPI_ATTR_OBJECTCLASS, mods[i]->mod_type )) { + slapi_schema_expand_objectclasses( ec->ep_entry ); + break; + } + } + } + + /* + * We are about to pass the last abandon test, so from now on we are + * committed to finish this operation. Set status to "will complete" + * before we make our last abandon check to avoid race conditions in + * the code that processes abandon operations. + */ + if (operation) { + operation->o_status = SLAPI_OP_STATUS_WILL_COMPLETE; + } + if ( slapi_op_abandoned( pb ) ) { + goto error_return; + } + + /* check that the entry still obeys the schema */ + if ( (operation_is_flag_set(operation,OP_FLAG_ACTION_SCHEMA_CHECK)) && + slapi_entry_schema_check( pb, ec->ep_entry ) != 0 ) { + ldap_result_code= LDAP_OBJECT_CLASS_VIOLATION; + slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &ldap_result_message); + goto error_return; + } + + /* + * make sure the entry contains all values in the RDN. + * if not, the modification must have removed them. + */ + if ( ! slapi_entry_rdn_values_present( ec->ep_entry ) ) { + ldap_result_code= LDAP_NOT_ALLOWED_ON_RDN; + goto error_return; + } + + for (retry_count = 0; retry_count < RETRY_TIMES; retry_count++) { + + if (retry_count > 0) { + dblayer_txn_abort(li,&txn); + LDAPDebug( LDAP_DEBUG_TRACE, "Modify Retrying Transaction\n", 0, 0, 0 ); +#ifndef LDBM_NO_BACKOFF_DELAY + { + PRIntervalTime interval; + interval = PR_MillisecondsToInterval(slapi_rand() % 100); + DS_Sleep(interval); + } +#endif + } + + /* Nothing above here modifies persistent store, everything after here is subject to the transaction */ + retval = dblayer_txn_begin(li,parent_txn,&txn); + + if (0 != retval) { + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + /* + * Update the ID to Entry index. + * Note that id2entry_add replaces the entry, so the Entry ID stays the same. + */ + retval = id2entry_add( be, ec, &txn ); + if (DB_LOCK_DEADLOCK == retval) + { + /* Abort and re-try */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_ANY, "id2entry_add failed, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + ec_in_cache = 1; + retval = index_add_mods( be, (const LDAPMod**)mods, e, ec, &txn ); + if (DB_LOCK_DEADLOCK == retval) + { + /* Abort and re-try */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_ANY, "index_add_mods failed, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + /* + * Remove the old entry from the Virtual List View indexes. + * Add the new entry to the Virtual List View indexes. + */ + retval= vlv_update_all_indexes(&txn, be, pb, e, ec); + if (DB_LOCK_DEADLOCK == retval) + { + /* Abort and re-try */ + continue; + } + if (0 != retval) { + LDAPDebug( LDAP_DEBUG_ANY, "vlv_update_index failed, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + if (0 == retval) { + break; + } + } + if (retry_count == RETRY_TIMES) { + LDAPDebug( LDAP_DEBUG_ANY, "Retry count exceeded in modify\n", 0, 0, 0 ); + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + if (cache_replace( &inst->inst_cache, e, ec ) != 0 ) { + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + postentry = slapi_entry_dup( ec->ep_entry ); + slapi_pblock_set( pb, SLAPI_ENTRY_POST_OP, postentry ); + + /* invalidate virtual cache */ + ec->ep_entry->e_virtual_watermark = 0; + + /* we must return both e (which has been deleted) and new entry ec */ + cache_unlock_entry( &inst->inst_cache, e ); + cache_return( &inst->inst_cache, &e ); + /* + * LP Fix of crash when the commit will fail: + * If the commit fail, the common error path will + * try to unlock the entry again and crash (PR_ASSERT + * in debug mode. + * By just setting e to NULL, we avoid this. It's OK since + * we don't use e after that in the normal case. + */ + e = NULL; + + retval = dblayer_txn_commit(li,&txn); + if (0 != retval) { + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + rc= 0; + goto common_return; + +error_return: + if (ec_in_cache) + { + cache_remove( &inst->inst_cache, ec ); + } + else + { + backentry_free(&ec); + } + if ( postentry != NULL ) + { + slapi_entry_free( postentry ); + postentry = NULL; + slapi_pblock_set( pb, SLAPI_ENTRY_POST_OP, NULL ); + } + + if (e!=NULL) { + cache_unlock_entry( &inst->inst_cache, e); + cache_return( &inst->inst_cache, &e); + } + + if (retval == DB_RUNRECOVERY) { + dblayer_remember_disk_filled(li); + ldbm_nasty("Modify",81,retval); + disk_full = 1; + } + + if (disk_full) + rc= return_on_disk_full(li); + else if (ldap_result_code != LDAP_SUCCESS) { + /* It is specifically OK to make this call even when no transaction was in progress */ + dblayer_txn_abort(li,&txn); /* abort crashes in case disk full */ + rc= SLAPI_FAIL_GENERAL; + } + + +common_return: + + if (ec_in_cache) + { + cache_return( &inst->inst_cache, &ec ); + } + /* JCMREPL - The bepostop is called even if the operation fails. */ + if (!disk_full) + plugin_call_plugins (pb, SLAPI_PLUGIN_BE_POST_MODIFY_FN); + + if(dblock_acquired) + { + dblayer_unlock_backend(be); + } + if(ldap_result_code!=-1) + { + slapi_send_ldap_result( pb, ldap_result_code, NULL, ldap_result_message, 0, NULL ); + } + + slapi_ch_free( (void**)&errbuf); + return rc; +} + +/* Function removes mods which are not allowed over-the-wire */ +static void +remove_illegal_mods(LDAPMod **mods) +{ + int i, j; + LDAPMod *tmp; + + /* remove any attempts by the user to modify these attrs */ + for ( i = 0; mods[i] != NULL; i++ ) { + if ( strcasecmp( mods[i]->mod_type, numsubordinates ) == 0 + || strcasecmp( mods[i]->mod_type, hassubordinates ) == 0 ) + { + tmp = mods[i]; + for ( j = i; mods[j] != NULL; j++ ) { + mods[j] = mods[j + 1]; + } + slapi_ch_free( (void**)&(tmp->mod_type) ); + if ( tmp->mod_bvalues != NULL ) { + ber_bvecfree( tmp->mod_bvalues ); + } + slapi_ch_free( (void**)&tmp ); + i--; + } + } +} + +/* A mod has no effect if it is trying to replace a non-existing + * attribute with null value + */ +static int +mods_have_effect (Slapi_Entry *entry, Slapi_Mods *smods) +{ + LDAPMod *mod; + Slapi_Attr *attr; + int have_effect = 1; + int j; + + /* Mods have effect if there is at least a non-replace mod or + * a non-null-value mod. + */ + for ( j = 0; j < smods->num_mods - 1; j++ ) { + if ( (mod = smods->mods[j]) != NULL ) { + if ( (mod->mod_op & LDAP_MOD_REPLACE) == 0 || + mod->mod_vals.modv_bvals && + strcasecmp (mod->mod_type, "modifiersname") && + strcasecmp (mod->mod_type, "modifytime") ) { + goto done; + } + } + } + + if ( entry && entry->e_sdn.dn ) { + for ( j = 0; j < smods->num_mods - 1; j++ ) { + if ( (mod = smods->mods[j]) != NULL && + strcasecmp (mod->mod_type, "modifiersname") && + strcasecmp (mod->mod_type, "modifytime") ) { + for ( attr = entry->e_attrs; attr; attr = attr->a_next ) { + /* Mods have effect if at least a null-value-mod is + * to actually remove an existing attribute + */ + if ( strcasecmp ( mod->mod_type, attr->a_type ) == 0 ) { + goto done; + } + } + have_effect = 0; + } + } + + } + +done: + + /* Return true would let the flow continue along the old path before + * this function was added + */ + return have_effect; +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c new file mode 100644 index 00000000..8658886a --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c @@ -0,0 +1,1403 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* modrdn.c - ldbm backend modrdn routine */ + +#include "back-ldbm.h" + +static const char *moddn_get_newdn(Slapi_PBlock *pb, Slapi_DN *dn_olddn, Slapi_DN *dn_newrdn, Slapi_DN *dn_newsuperiordn); +static void moddn_unlock_and_return_entries(backend *be,struct backentry **targetentry, struct backentry **existingentry); +static int moddn_newrdn_mods(Slapi_PBlock *pb, const char *olddn, struct backentry *ec, Slapi_Mods *smods, Slapi_Mods *smods_wsi, int is_repl_op); +static IDList *moddn_get_children(back_txn *ptxn, Slapi_PBlock *pb, backend *be, struct backentry *parententry, Slapi_DN *parentdn, struct backentry ***child_entries, struct backentry ***child_entry_copies); +static int moddn_rename_children(back_txn *ptxn, Slapi_PBlock *pb, backend *be, IDList *children, Slapi_DN *dn_parentdn, Slapi_DN *dn_newsuperiordn, struct backentry *child_entries[], struct backentry *child_entry_copies[]); +static int modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *e, struct backentry *ec, Slapi_Mods *smods1, Slapi_Mods *smods2, Slapi_Mods *smods3); +static void mods_remove_nsuniqueid(Slapi_Mods *smods); + +int +ldbm_back_modrdn( Slapi_PBlock *pb ) +{ + backend *be; + ldbm_instance *inst; + struct ldbminfo *li; + struct backentry *e= NULL; + struct backentry *ec= NULL; + int ec_in_cache= 0; + back_txn txn; + back_txnid parent_txn; + int retval = -1; + char *msg; + Slapi_Entry *postentry = NULL; + char *errbuf = NULL; + int disk_full = 0; + int retry_count = 0; + int ldap_result_code= LDAP_SUCCESS; + char *ldap_result_message= NULL; + char *ldap_result_matcheddn= NULL; + struct backentry *parententry= NULL; + struct backentry *newparententry= NULL; + struct backentry *existingentry= NULL; + modify_context parent_modify_context = {0}; + modify_context newparent_modify_context = {0}; + IDList *children= NULL; + struct backentry **child_entries= NULL; + struct backentry **child_entry_copies= NULL; + Slapi_DN dn_olddn; + Slapi_DN dn_newdn; + Slapi_DN dn_newrdn; + Slapi_DN dn_newsuperiordn; + Slapi_DN dn_parentdn; + int rc; + int isroot; + LDAPMod **mods; + Slapi_Mods smods_operation_wsi = {0}; + Slapi_Mods smods_generated = {0}; + Slapi_Mods smods_generated_wsi = {0}; + Slapi_Operation *operation; + int dblock_acquired= 0; + int is_replicated_operation= 0; + int is_fixup_operation = 0; + entry_address new_addr; + entry_address *old_addr; + entry_address oldparent_addr; + entry_address *newsuperior_addr; + char *dn; + char ebuf[BUFSIZ]; + CSN *opcsn = NULL; + + slapi_sdn_init(&dn_newdn); + slapi_sdn_init(&dn_parentdn); + + slapi_pblock_get( pb, SLAPI_MODRDN_TARGET, &dn ); + slapi_pblock_get( pb, SLAPI_BACKEND, &be); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_PARENT_TXN, (void**)&parent_txn ); + slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot ); + slapi_pblock_get( pb, SLAPI_OPERATION, &operation ); + slapi_pblock_get( pb, SLAPI_IS_REPLICATED_OPERATION, &is_replicated_operation ); + is_fixup_operation = operation_is_flag_set(operation,OP_FLAG_REPL_FIXUP); + + if (pb->pb_conn) + { + slapi_log_error (SLAPI_LOG_TRACE, "ldbm_back_modrdn", "enter conn=%d op=%d\n", pb->pb_conn->c_connid, operation->o_opid); + } + + inst = (ldbm_instance *) be->be_instance_info; + { + char *newrdn, *newsuperiordn; + slapi_pblock_get( pb, SLAPI_MODRDN_NEWRDN, &newrdn ); + slapi_pblock_get( pb, SLAPI_MODRDN_NEWSUPERIOR, &newsuperiordn ); + slapi_sdn_init_dn_byref(&dn_olddn,dn); + slapi_sdn_init_dn_byref(&dn_newrdn,newrdn); + slapi_sdn_init_dn_byref(&dn_newsuperiordn,newsuperiordn); + slapi_sdn_get_parent(&dn_olddn,&dn_parentdn); + } + + /* if old and new superior are equals, newsuperior should not be set + * Here we have to reset newsuperiordn in order to save processing and + * avoid later deadlock when trying to fetch twice the same entry + */ + if (slapi_sdn_compare(&dn_newsuperiordn, &dn_parentdn) == 0) + { + slapi_sdn_done(&dn_newsuperiordn); + slapi_sdn_init_dn_byref(&dn_newsuperiordn,NULL); + } + + /* Replicated Operations are allowed to change the superior */ + if ( !is_replicated_operation && !slapi_sdn_isempty(&dn_newsuperiordn)) + { + slapi_send_ldap_result( pb, LDAP_UNWILLING_TO_PERFORM, NULL, + "server does not support moving of entries", 0, NULL ); + return( -1 ); + } + + dblayer_txn_init(li,&txn); + + /* The dblock serializes writes to the database, + * which reduces deadlocking in the db code, + * which means that we run faster. + * + * But, this lock is re-enterant for the fixup + * operations that the URP code in the Replication + * plugin generates. + * + * Also some URP post-op operations are called after + * the backend has committed the change and released + * the dblock. Acquire the dblock again for them + * if OP_FLAG_ACTION_INVOKE_FOR_REPLOP is set. + */ + if(SERIALLOCK(li) && (!operation_is_flag_set(operation,OP_FLAG_REPL_FIXUP) || operation_is_flag_set(operation,OP_FLAG_ACTION_INVOKE_FOR_REPLOP))) + { + dblayer_lock_backend(be); + dblock_acquired= 1; + } + + /* Work out what the new name of the entry will be */ + { + const char *newdn= moddn_get_newdn(pb,&dn_olddn,&dn_newrdn,&dn_newsuperiordn); + slapi_sdn_set_dn_passin(&dn_newdn,newdn); + } + + rc= 0; + rc= slapi_setbit_int(rc,SLAPI_RTN_BIT_FETCH_EXISTING_DN_ENTRY); + rc= slapi_setbit_int(rc,SLAPI_RTN_BIT_FETCH_PARENT_ENTRY); + rc= slapi_setbit_int(rc,SLAPI_RTN_BIT_FETCH_NEWPARENT_ENTRY); + rc= slapi_setbit_int(rc,SLAPI_RTN_BIT_FETCH_TARGET_ENTRY); + while(rc!=0) + { + /* JCM - copying entries can be expensive... should optimize */ + /* + * Some present state information is passed through the PBlock to the + * backend pre-op plugin. To ensure a consistent snapshot of this state + * we wrap the reading of the entry with the dblock. + */ + if(slapi_isbitset_int(rc,SLAPI_RTN_BIT_FETCH_EXISTING_DN_ENTRY)) + { + const char *newdn = NULL; + char * newrdn = NULL; + + /* see if an entry with the new name already exists */ + done_with_pblock_entry(pb,SLAPI_MODRDN_EXISTING_ENTRY); /* Could be through this multiple times */ + slapi_sdn_done(&dn_newrdn); + slapi_pblock_get(pb, SLAPI_MODRDN_NEWRDN, &newrdn); + slapi_sdn_init_dn_byref(&dn_newrdn,newrdn); + newdn= moddn_get_newdn(pb,&dn_olddn,&dn_newrdn,&dn_newsuperiordn); + slapi_sdn_set_dn_passin(&dn_newdn,newdn); + new_addr.dn = (char*)slapi_sdn_get_ndn (&dn_newdn); + new_addr.uniqueid = NULL; + ldap_result_code= get_copy_of_entry(pb, &new_addr, &txn, SLAPI_MODRDN_EXISTING_ENTRY, 0); + } + if(slapi_isbitset_int(rc,SLAPI_RTN_BIT_FETCH_PARENT_ENTRY)) + { + /* find and lock the old parent entry */ + done_with_pblock_entry(pb,SLAPI_MODRDN_PARENT_ENTRY); /* Could be through this multiple times */ + oldparent_addr.dn = (char*)slapi_sdn_get_ndn (&dn_parentdn); + oldparent_addr.uniqueid = NULL; + ldap_result_code= get_copy_of_entry(pb, &oldparent_addr, &txn, SLAPI_MODRDN_PARENT_ENTRY, !is_replicated_operation); + } + if(slapi_sdn_get_ndn(&dn_newsuperiordn)!=NULL && slapi_isbitset_int(rc,SLAPI_RTN_BIT_FETCH_NEWPARENT_ENTRY)) + { + /* JCM - Could check that this really is a new superior, and not the same old one. Compare parentdn & newsuperior */ + /* find and lock the new parent entry */ + done_with_pblock_entry(pb,SLAPI_MODRDN_NEWPARENT_ENTRY); /* Could be through this multiple times */ + /* JCMREPL - If this is a replicated operation then should fetch new superior with uniqueid */ + slapi_pblock_get (pb, SLAPI_MODRDN_NEWSUPERIOR_ADDRESS, &newsuperior_addr); + ldap_result_code= get_copy_of_entry(pb, newsuperior_addr, &txn, SLAPI_MODRDN_NEWPARENT_ENTRY, !is_replicated_operation); + } + if(slapi_isbitset_int(rc,SLAPI_RTN_BIT_FETCH_TARGET_ENTRY)) + { + /* find and lock the entry we are about to modify */ + done_with_pblock_entry(pb,SLAPI_MODRDN_TARGET_ENTRY); /* Could be through this multiple times */ + slapi_pblock_get (pb, SLAPI_TARGET_ADDRESS, &old_addr); + ldap_result_code= get_copy_of_entry(pb, old_addr, &txn, SLAPI_MODRDN_TARGET_ENTRY, !is_replicated_operation); + if(ldap_result_code==LDAP_OPERATIONS_ERROR) + { + /* JCM - Usually the call to find_entry2modify would generate the result code. */ + /* JCM !!! */ + goto error_return; + } + } + /* Call the Backend Pre ModRDN plugins */ + slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ldap_result_code); + rc= plugin_call_plugins(pb, SLAPI_PLUGIN_BE_PRE_MODRDN_FN); + if(rc==-1) + { + /* + * Plugin indicated some kind of failure, + * or that this Operation became a No-Op. + */ + slapi_pblock_get(pb, SLAPI_RESULT_CODE, &ldap_result_code); + goto error_return; + } + /* + * (rc!=-1) means that the plugin changed things, so we go around + * the loop once again to get the new present state. + */ + /* JCMREPL - Warning: A Plugin could cause an infinite loop by always returning a result code that requires some action. */ + } + + /* find and lock the entry we are about to modify */ + /* JCMREPL - Argh, what happens about the stinking referrals? */ + slapi_pblock_get (pb, SLAPI_TARGET_ADDRESS, &old_addr); + e = find_entry2modify( pb, be, old_addr, NULL ); + if ( e == NULL ) + { + ldap_result_code= -1; + goto error_return; /* error result sent by find_entry2modify() */ + } + + /* Check that an entry with the same DN doesn't already exist. */ + { + Slapi_Entry *entry; + slapi_pblock_get( pb, SLAPI_MODRDN_EXISTING_ENTRY, &entry); + if(entry!=NULL) + { + ldap_result_code= LDAP_ALREADY_EXISTS; + goto error_return; + } + } + + /* Fetch and lock the parent of the entry that is moving */ + oldparent_addr.dn = (char*)slapi_sdn_get_ndn (&dn_parentdn); + oldparent_addr.uniqueid = NULL; + parententry = find_entry2modify_only( pb, be, &oldparent_addr, NULL ); + modify_init(&parent_modify_context,parententry); + + /* Fetch and lock the new parent of the entry that is moving */ + if(slapi_sdn_get_ndn(&dn_newsuperiordn)!=NULL) + { + slapi_pblock_get (pb, SLAPI_MODRDN_NEWSUPERIOR_ADDRESS, &newsuperior_addr); + newparententry = find_entry2modify_only( pb, be, newsuperior_addr, NULL); + modify_init(&newparent_modify_context,newparententry); + } + + opcsn = operation_get_csn (operation); + if (!is_fixup_operation) + { + if ( opcsn == NULL && operation->o_csngen_handler) + { + /* + * Current op is a user request. Opcsn will be assigned + * if the dn is in an updatable replica. + */ + opcsn = entry_assign_operation_csn ( pb, e->ep_entry, parententry ? parententry->ep_entry : NULL ); + } + if ( opcsn != NULL ) + { + entry_set_maxcsn (e->ep_entry, opcsn); + } + } + + /* + * Now that we have the old entry, we reset the old DN and recompute + * the new DN. Why? Because earlier when we computed the new DN, we did + * not have the old entry, so we used the DN that was presented as the + * target DN in the ModRDN operation itself, and we would prefer to + * preserve the case and spacing that are in the actual entry's DN + * instead. Otherwise, a ModRDN operation will potentially change an + * entry's entire DN (at least with respect to case and spacing). + */ + slapi_sdn_copy( slapi_entry_get_sdn_const( e->ep_entry ), &dn_olddn ); + if (newparententry != NULL) { + /* don't forget we also want to preserve case of new superior */ + slapi_sdn_copy(slapi_entry_get_sdn_const(newparententry->ep_entry), &dn_newsuperiordn); + } + slapi_sdn_set_dn_passin(&dn_newdn, + moddn_get_newdn(pb, &dn_olddn, &dn_newrdn, &dn_newsuperiordn)); + + /* Check that we're allowed to add an entry below the new superior */ + if ( newparententry == NULL ) + { + /* There may not be a new parent because we don't intend there to be one. */ + if(slapi_sdn_get_ndn(&dn_newsuperiordn)!=NULL) + { + /* If the new entry is to be a suffix, and we're root, then it's OK that the new parent doesn't exist */ + if(!(slapi_dn_isbesuffix(pb,slapi_sdn_get_ndn(&dn_newdn)) && isroot)) + { + /* Here means that we didn't find the parent */ + int err = 0; + Slapi_DN ancestordn = {0}; + struct backentry *ancestorentry; + ancestorentry= dn2ancestor(be,&dn_newdn,&ancestordn,&txn,&err); + cache_return( &inst->inst_cache, &ancestorentry ); + ldap_result_matcheddn= slapi_ch_strdup((char *) slapi_sdn_get_dn(&ancestordn)); + ldap_result_code= LDAP_NO_SUCH_OBJECT; + LDAPDebug( LDAP_DEBUG_TRACE, "New superior does not exist matched %s, newsuperior = %s\n", + ldap_result_matcheddn == NULL ? "NULL" : ldap_result_matcheddn, slapi_sdn_get_ndn(&dn_newsuperiordn), 0 ); + slapi_sdn_done(&ancestordn); + goto error_return; + } + } + } + else + { + ldap_result_code= plugin_call_acl_plugin (pb, newparententry->ep_entry, NULL, NULL, SLAPI_ACL_ADD, ACLPLUGIN_ACCESS_DEFAULT, &errbuf ); + if ( ldap_result_code != LDAP_SUCCESS ) + { + ldap_result_message= errbuf; + LDAPDebug( LDAP_DEBUG_TRACE, "No access to new superior.\n", 0, 0, 0 ); + goto error_return; + } + } + + /* Check that the target entry has a parent */ + if ( parententry == NULL ) + { + /* If the entry a suffix, and we're root, then it's OK that the parent doesn't exist */ + if(!(slapi_dn_isbesuffix(pb,slapi_sdn_get_ndn(&dn_olddn)) && isroot)) + { + /* Here means that we didn't find the parent */ + ldap_result_matcheddn = slapi_ch_strdup((char *) slapi_entry_get_dn(parententry->ep_entry)); + ldap_result_code= LDAP_NO_SUCH_OBJECT; + LDAPDebug( LDAP_DEBUG_TRACE, "Parent does not exist matched %s, parentdn = %s\n", + ldap_result_matcheddn == NULL ? "NULL" : ldap_result_matcheddn, slapi_sdn_get_ndn(&dn_parentdn), 0 ); + goto error_return; + } + } + + /* Replicated Operations are allowed to rename entries with children */ + if ( !is_replicated_operation && slapi_entry_has_children( e->ep_entry )) + { + ldap_result_code = LDAP_NOT_ALLOWED_ON_NONLEAF; + goto error_return; + } + + + /* + * JCM - All the child entries must be locked in the cache, so the size of + * subtree that can be renamed is limited by the cache size. + */ + + /* Save away a copy of the entry, before modifications */ + slapi_pblock_set( pb, SLAPI_ENTRY_PRE_OP, slapi_entry_dup( e->ep_entry )); + + /* create a copy of the entry and apply the changes to it */ + if ( (ec = backentry_dup( e )) == NULL ) + { + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + /* JCMACL - Should be performed before the child check. */ + /* JCMACL - Why is the check performed against the copy, rather than the existing entry? */ + /*ldap_result_code = plugin_call_acl_plugin (pb, ec->ep_entry, NULL , NULL , SLAPI_ACL_WRITE, ACLPLUGIN_ACCESS_DEFAULT, &errbuf );*/ + ldap_result_code = plugin_call_acl_plugin (pb, ec->ep_entry, + NULL /*attr*/, NULL /*value*/, SLAPI_ACL_WRITE, + ACLPLUGIN_ACCESS_MODRDN, &errbuf ); + if ( ldap_result_code != LDAP_SUCCESS ) + { + goto error_return; + } + + slapi_entry_set_sdn( ec->ep_entry, &dn_newdn ); + + /* create it in the cache - prevents others from creating it */ + if ( cache_add_tentative( &inst->inst_cache, ec, NULL ) != 0 ) { + /* somebody must've created it between dn2entry() and here */ + /* JCMREPL - Hmm... we can't permit this to happen...? */ + ldap_result_code= LDAP_ALREADY_EXISTS; + goto error_return; + } + ec_in_cache= 1; + + /* Build the list of modifications required to the existing entry */ + { + slapi_mods_init(&smods_generated,4); + slapi_mods_init(&smods_generated_wsi,4); + ldap_result_code = moddn_newrdn_mods(pb, slapi_sdn_get_ndn(&dn_olddn), ec, &smods_generated, &smods_generated_wsi, + is_replicated_operation); + if (ldap_result_code != LDAP_SUCCESS) { + if (ldap_result_code == LDAP_UNWILLING_TO_PERFORM) + ldap_result_message = "Modification of old rdn attribute type not allowed."; + goto error_return; + } + /* + * Remove the old entrydn index entry, and add the new one. + */ + slapi_mods_add( &smods_generated, LDAP_MOD_DELETE, "entrydn", strlen(backentry_get_ndn(e)), backentry_get_ndn(e)); + slapi_mods_add( &smods_generated, LDAP_MOD_REPLACE, "entrydn", strlen(backentry_get_ndn(ec)), backentry_get_ndn(ec)); + + /* + * Update parentid if we have a new superior. + */ + if(slapi_sdn_get_dn(&dn_newsuperiordn)!=NULL) { + char buf[40]; /* Enough for an ID */ + + if (parententry != NULL) { + sprintf( buf, "%lu", (u_long)parententry->ep_id ); + slapi_mods_add_string(&smods_generated, LDAP_MOD_DELETE, "parentid", buf); + } + if (newparententry != NULL) { + sprintf( buf, "%lu", (u_long)newparententry->ep_id ); + slapi_mods_add_string(&smods_generated, LDAP_MOD_REPLACE, "parentid", buf); + } + } + } + + slapi_pblock_get( pb, SLAPI_MODIFY_MODS, &mods ); + slapi_mods_init_byref(&smods_operation_wsi,mods); + + /* + * We are about to pass the last abandon test, so from now on we are + * committed to finish this operation. Set status to "will complete" + * before we make our last abandon check to avoid race conditions in + * the code that processes abandon operations. + */ + if (operation) { + operation->o_status = SLAPI_OP_STATUS_WILL_COMPLETE; + } + if ( slapi_op_abandoned( pb ) ) { + goto error_return; + } + + /* + * First, we apply the generated mods that do not involve any state information. + */ + if ( entry_apply_mods( ec->ep_entry, slapi_mods_get_ldapmods_byref(&smods_generated) ) != 0 ) + { + ldap_result_code= LDAP_OPERATIONS_ERROR; + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm_modrdn: entry_apply_mods failed for entry %s\n", + escape_string(slapi_entry_get_dn_const(ec->ep_entry), ebuf), 0, 0); + goto error_return; + } + + /* + * Now we apply the generated mods that do involve state information. + */ + if (slapi_mods_get_num_mods(&smods_generated_wsi)>0) + { + if (entry_apply_mods_wsi(ec->ep_entry, &smods_generated_wsi, operation_get_csn(operation), is_replicated_operation)!=0) + { + ldap_result_code= LDAP_OPERATIONS_ERROR; + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm_modrdn: entry_apply_mods_wsi failed for entry %s\n", + escape_string(slapi_entry_get_dn_const(ec->ep_entry), ebuf), 0, 0); + goto error_return; + } + } + + /* + * Now we apply the operation mods that do involve state information. + * (Operational attributes). + * The following block looks redundent to the one above. But it may + * be necessary - check the comment for version 1.3.16.22.2.76 of + * this file and compare that version with its previous one. + */ + if (slapi_mods_get_num_mods(&smods_operation_wsi)>0) + { + if (entry_apply_mods_wsi(ec->ep_entry, &smods_operation_wsi, operation_get_csn(operation), is_replicated_operation)!=0) + { + ldap_result_code= LDAP_OPERATIONS_ERROR; + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm_modrdn: entry_apply_mods_wsi (operational attributes) failed for entry %s\n", + escape_string(slapi_entry_get_dn_const(ec->ep_entry), ebuf), 0, 0); + goto error_return; + } + } + /* check that the entry still obeys the schema */ + if ( slapi_entry_schema_check( pb, ec->ep_entry ) != 0 ) { + ldap_result_code = LDAP_OBJECT_CLASS_VIOLATION; + slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &ldap_result_message); + goto error_return; + } + + /* + * Update the DN CSN of the entry. + */ + entry_add_dncsn(ec->ep_entry,operation_get_csn(operation)); + entry_add_rdn_csn(ec->ep_entry,operation_get_csn(operation)); + + /* + * If the entry has a new superior then the subordinate count + * of the parents must be updated. + */ + if(slapi_sdn_get_dn(&dn_newsuperiordn)!=NULL) + { + /* + * Update the subordinate count of the parents to reflect the moved child. + */ + if ( parententry!=NULL ) + { + retval = parent_update_on_childchange(&parent_modify_context,2,NULL); /* 2==delete */ + /* The parent modify context now contains info needed later */ + if (0 != retval) + { + goto error_return; + } + } + if ( newparententry!=NULL ) + { + retval = parent_update_on_childchange(&newparent_modify_context,1,NULL); /* 1==add */ + /* The newparent modify context now contains info needed later */ + if (0 != retval) + { + goto error_return; + } + } + } + + /* + * If the entry has children then we're going to have to rename them all. + */ + if (slapi_entry_has_children( e->ep_entry )) + { + /* JCM - This is where the subtree lock will appear */ + children= moddn_get_children(&txn, pb, be, e, &dn_olddn, &child_entries, &child_entry_copies); + /* JCM - Shouldn't we perform an access control check on all the children. */ + /* JCMREPL - But, the replication client has total rights over its subtree, so no access check needed. */ + /* JCM - A subtree move could break ACIs, static groups, and dynamic groups. */ + } + + /* + * So, we believe that no code up till here actually added anything + * to persistent store. From now on, we're transacted + */ + for (retry_count = 0; retry_count < RETRY_TIMES; retry_count++) + { + if (retry_count > 0) + { + dblayer_txn_abort(li,&txn); + /* We're re-trying */ + LDAPDebug( LDAP_DEBUG_TRACE, "Modrdn Retrying Transaction\n", 0, 0, 0 ); + } + retval = dblayer_txn_begin(li,parent_txn,&txn); + if (0 != retval) { + ldap_result_code= LDAP_OPERATIONS_ERROR; + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + goto error_return; + } + + /* + * Update the indexes for the entry. + */ + retval = modrdn_rename_entry_update_indexes(&txn, pb, li, e, ec, &smods_generated, &smods_generated_wsi, &smods_operation_wsi); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + continue; + } + if (retval != 0 ) + { + LDAPDebug( LDAP_DEBUG_TRACE, "modrdn_rename_entry_update_indexes failed, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + } + + /* + * add new name to index + */ + { + char **rdns; + int i; + if ( (rdns = ldap_explode_rdn( slapi_sdn_get_dn(&dn_newrdn), 0 )) != NULL ) + { + for ( i = 0; rdns[i] != NULL; i++ ) + { + char *type; + Slapi_Value *svp[2]; + Slapi_Value sv; + memset(&sv,0,sizeof(Slapi_Value)); + if ( slapi_rdn2typeval( rdns[i], &type, &sv.bv ) != 0 ) + { + char ebuf[ BUFSIZ ]; + LDAPDebug( LDAP_DEBUG_ANY, "modrdn: rdn2typeval (%s) failed\n", + escape_string( rdns[i], ebuf ), 0, 0 ); + goto error_return; + } + svp[0] = &sv; + svp[1] = NULL; + retval = index_addordel_values_sv( be, type, svp, NULL, ec->ep_id, BE_INDEX_ADD, &txn ); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + continue; + } + if (retval != 0 ) + { + LDAPDebug( LDAP_DEBUG_ANY, "modrdn: could not add new value to index, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + } + } + ldap_value_free( rdns ); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + goto error_return; + } + } + } + if (slapi_sdn_get_dn(&dn_newsuperiordn)!=NULL) + { + /* Push out the db modifications from the parent entry */ + retval = modify_update_all(be, pb, &parent_modify_context, &txn); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + continue; + } + if (0 != retval) + { + LDAPDebug( LDAP_DEBUG_TRACE, "moddn: could not update parent, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + } + /* Push out the db modifications from the new parent entry */ + if(retval==0) + { + retval = modify_update_all(be, pb, &newparent_modify_context, &txn); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + continue; + } + if (0 != retval) + { + LDAPDebug( LDAP_DEBUG_TRACE, "moddn: could not update parent, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + } + } + } + + /* + * Update ancestorid index. + */ + if (slapi_sdn_get_dn(&dn_newsuperiordn)!=NULL) { + retval = ldbm_ancestorid_move_subtree(be, &dn_olddn, &dn_newdn, e->ep_id, children, &txn); + if (retval != 0) { + if (retval == DB_LOCK_DEADLOCK) continue; + if (retval == DB_RUNRECOVERY || LDBM_OS_ERR_IS_DISKFULL(retval)) + disk_full = 1; + goto error_return; + } + } + + /* + * If the entry has children, then rename them all. + */ + if (children!=NULL) + { + retval= moddn_rename_children( &txn, pb, be, children, &dn_olddn, &dn_newdn, child_entries, child_entry_copies); + } + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + continue; + } + if (retval != 0) + { + if (retval == DB_RUNRECOVERY || LDBM_OS_ERR_IS_DISKFULL(retval)) + disk_full = 1; + goto error_return; + } + + break; /* retval==0, Done, Terminate the loop */ + } + if (retry_count == RETRY_TIMES) + { + /* Failed */ + LDAPDebug( LDAP_DEBUG_ANY, "Retry count exceeded in modrdn\n", 0, 0, 0 ); + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + postentry = slapi_entry_dup( ec->ep_entry ); + + if(parententry!=NULL) + { + modify_switch_entries( &parent_modify_context,be); + } + if(newparententry!=NULL) + { + modify_switch_entries( &newparent_modify_context,be); + } + + retval = dblayer_txn_commit(li,&txn); + if (0 != retval) + { + if (LDBM_OS_ERR_IS_DISKFULL(retval)) disk_full = 1; + ldap_result_code= LDAP_OPERATIONS_ERROR; + goto error_return; + } + + if(children!=NULL) + { + int i=0; + for (; child_entries[i]!=NULL; i++) { + cache_unlock_entry( &inst->inst_cache, child_entries[i]) ; + cache_return( &inst->inst_cache, &(child_entries[i]) ); + cache_return( &inst->inst_cache, &(child_entry_copies[i]) ); + } + } + + retval= 0; +#if 0 /* this new entry in the cache can be used for future; don't remove it */ + /* remove from cache so that memory can be freed by cache_return */ + if (ec_in_cache) { + cache_remove(&inst->inst_cache, ec); + } +#endif + goto common_return; + +error_return: + /* result already sent above - just free stuff */ + if ( NULL != postentry ) + { + slapi_entry_free( postentry ); + postentry= NULL; + } + if( ec!=NULL ) { + if (ec_in_cache) { + cache_remove(&inst->inst_cache, ec); + } else { + backentry_free( &ec ); + } + } + if(children!=NULL) + { + int i=0; + for(;child_entries[i]!=NULL;i++) { + cache_unlock_entry(&inst->inst_cache, child_entries[i]); + cache_return(&inst->inst_cache, &(child_entries[i])); + if (child_entry_copies[i] != NULL) { + cache_remove(&inst->inst_cache, child_entry_copies[i]); + cache_return( &inst->inst_cache, &(child_entry_copies[i]) ); + } + } + } + + + if (retval == DB_RUNRECOVERY) { + dblayer_remember_disk_filled(li); + ldbm_nasty("ModifyDN",82,retval); + disk_full = 1; + } + + if (disk_full) + { + retval = return_on_disk_full(li); + } + else + { + /* It is specifically OK to make this call even when no transaction was in progress */ + dblayer_txn_abort(li,&txn); /* abort crashes in case disk full */ + retval= SLAPI_FAIL_GENERAL; + } + +common_return: + + /* Free up the resource we don't need any more */ + if(ec_in_cache) { + cache_return( &inst->inst_cache, &ec ); + } + + /* + * The bepostop is called even if the operation fails. + */ + plugin_call_plugins (pb, SLAPI_PLUGIN_BE_POST_MODRDN_FN); + + if (ldap_result_code!=-1) + { + slapi_send_ldap_result( pb, ldap_result_code, ldap_result_matcheddn, + ldap_result_message, 0,NULL ); + } + slapi_mods_done(&smods_operation_wsi); + slapi_mods_done(&smods_generated); + slapi_mods_done(&smods_generated_wsi); + moddn_unlock_and_return_entries(be,&e,&existingentry); + slapi_ch_free((void**)&child_entries); + slapi_ch_free((void**)&child_entry_copies); + slapi_ch_free((void**)&ldap_result_matcheddn); + idl_free(children); + slapi_sdn_done(&dn_olddn); + slapi_sdn_done(&dn_newdn); + slapi_sdn_done(&dn_newrdn); + slapi_sdn_done(&dn_newsuperiordn); + slapi_sdn_done(&dn_parentdn); + modify_term(&parent_modify_context,be); + modify_term(&newparent_modify_context,be); + done_with_pblock_entry(pb,SLAPI_MODRDN_EXISTING_ENTRY); + done_with_pblock_entry(pb,SLAPI_MODRDN_PARENT_ENTRY); + done_with_pblock_entry(pb,SLAPI_MODRDN_NEWPARENT_ENTRY); + done_with_pblock_entry(pb,SLAPI_MODRDN_TARGET_ENTRY); + if(dblock_acquired) + { + dblayer_unlock_backend(be); + } + slapi_ch_free((void**)&errbuf); + if (retval == 0 && opcsn != NULL && !is_fixup_operation) + { + slapi_pblock_set(pb, SLAPI_URP_NAMING_COLLISION_DN, slapi_ch_strdup (dn)); + } + slapi_pblock_set( pb, SLAPI_ENTRY_POST_OP, postentry ); + if (pb->pb_conn) + { + slapi_log_error (SLAPI_LOG_TRACE, "ldbm_back_modrdn", "leave conn=%d op=%d\n", pb->pb_conn->c_connid, operation->o_opid); + } + return retval; +} + +/* + * Work out what the new DN of the entry will be. + */ +static const char * +moddn_get_newdn(Slapi_PBlock *pb, Slapi_DN *dn_olddn, Slapi_DN *dn_newrdn, Slapi_DN *dn_newsuperiordn) +{ + char *newdn; + const char *newrdn= slapi_sdn_get_dn(dn_newrdn); + const char *newsuperiordn= slapi_sdn_get_dn(dn_newsuperiordn); + + if( newsuperiordn!=NULL) + { + /* construct the new dn */ + if(slapi_dn_isroot(newsuperiordn)) + { + newdn= slapi_ch_strdup(newrdn); + } + else + { + newdn= slapi_dn_plus_rdn(newsuperiordn, newrdn); /* JCM - Use Slapi_RDN */ + } + } + else + { + /* construct the new dn */ + char *pdn; + const char *dn= slapi_sdn_get_dn(dn_olddn); + pdn = slapi_dn_beparent( pb, dn ); + if ( pdn != NULL ) + { + newdn= slapi_dn_plus_rdn(pdn, newrdn); /* JCM - Use Slapi_RDN */ + } + else + { + newdn= slapi_ch_strdup(newrdn); + } + slapi_ch_free( (void**)&pdn ); + } + return newdn; +} + +/* + * Return the entries to the cache. + */ +static void +moddn_unlock_and_return_entries( + backend *be, + struct backentry **targetentry, + struct backentry **existingentry) + { + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + + /* Something bad happened so we should give back all the entries */ + if ( *targetentry!=NULL ) { + cache_unlock_entry(&inst->inst_cache, *targetentry); + cache_return( &inst->inst_cache, targetentry ); + *targetentry= NULL; + } + if ( *existingentry!=NULL ) { + cache_return( &inst->inst_cache, existingentry ); + *existingentry= NULL; + } + } + + +/* + * JCM - There was a problem with multi-valued RDNs where + * JCM - there was an intersection of the two sets RDN Components + * JCM - and the deleteoldrdn flag was set. A value was deleted + * JCM - but not re-added because the value is found to already + * JCM - exist. + * + * This function returns 1 if it is necessary to add an RDN value + * to the entry. This is necessary if either: + * 1 the attribute or the value is not present in the entry, or + * 2 the attribute is present, deleteoldrdn is set, and the RDN value + * is in the deleted list. + * + * For example, suppose you rename cn=a to cn=a+sn=b. The cn=a value + * is removed from the entry and then readded. + */ + +static int +moddn_rdn_add_needed ( + struct backentry *ec, + char *type, + struct berval *bvp, + int deleteoldrdn, + Slapi_Mods *smods_wsi +) +{ + Slapi_Attr *attr; + LDAPMod *mod; + + if (slapi_entry_attr_find(ec->ep_entry, type, &attr) != 0 || + slapi_attr_value_find( attr, bvp ) != 0 ) + { + return 1; + } + + if (deleteoldrdn == 0) return 0; + + /* in a multi-valued RDN, the RDN value might have been already + * put on the smods_wsi list to be deleted, yet might still be + * in the target RDN. + */ + + for (mod = slapi_mods_get_first_mod(smods_wsi); + mod != NULL; + mod = slapi_mods_get_next_mod(smods_wsi)) { + if (((mod->mod_op & ~LDAP_MOD_BVALUES) == LDAP_MOD_DELETE) && + (strcasecmp(mod->mod_type, type) == 0) && + (mod->mod_bvalues != NULL) && + (slapi_attr_value_cmp(attr, *mod->mod_bvalues, bvp) == 0)) { + return 1; + } + } + + return 0; +} + +/* + * Build the list of modifications to apply to the Existing Entry + * With State Information: + * - delete old rdn values from the entry if deleteoldrdn is set + * - add new rdn values to the entry + * Without State Information + * - No changes + */ +static int +moddn_newrdn_mods(Slapi_PBlock *pb, const char *olddn, struct backentry *ec, Slapi_Mods *smods, Slapi_Mods *smods_wsi, int is_repl_op) +{ + char ebuf[BUFSIZ]; + char **rdns = NULL; + char **dns = NULL; + int deleteoldrdn; + char *type = NULL; + char *dn = NULL; + char *newrdn = NULL; + int i; + struct berval *bvps[2]; + struct berval bv; + + bvps[0] = &bv; + bvps[1] = NULL; + + /* slapi_pblock_get( pb, SLAPI_MODRDN_TARGET, &dn ); */ + slapi_pblock_get( pb, SLAPI_MODRDN_NEWRDN, &newrdn ); + slapi_pblock_get( pb, SLAPI_MODRDN_DELOLDRDN, &deleteoldrdn ); + + + /* + * This loop removes the old RDN of the existing entry. + */ + if (deleteoldrdn) { + int baddn = 0; /* set to true if could not parse dn */ + int badrdn = 0; /* set to true if could not parse rdn */ + dn = slapi_ch_strdup(olddn); + dns = ldap_explode_dn( dn, 0 ); + if ( dns != NULL ) + { + rdns = ldap_explode_rdn( dns[0], 0 ); + if ( rdns != NULL ) + { + for ( i = 0; rdns[i] != NULL; i++ ) + { + /* delete from entry attributes */ + if ( deleteoldrdn && slapi_rdn2typeval( rdns[i], &type, &bv ) == 0 ) + { + /* check if user is allowed to modify the specified attribute */ + /* + * It would be better to do this check in the front end + * end inside op_shared_rename(), but unfortunately we + * don't have access to the target entry there. + */ + if (!op_shared_is_allowed_attr (type, is_repl_op)) + { + ldap_value_free( rdns ); + ldap_value_free( dns ); + slapi_ch_free_string(&dn); + return LDAP_UNWILLING_TO_PERFORM; + } + if (strcasecmp (type, SLAPI_ATTR_UNIQUEID) != 0) + slapi_mods_add_modbvps( smods_wsi, LDAP_MOD_DELETE, type, bvps ); + } + } + ldap_value_free( rdns ); + } + else + { + badrdn = 1; + } + ldap_value_free( dns ); + } + else + { + baddn = 1; + } + slapi_ch_free_string(&dn); + + if ( baddn || badrdn ) + { + LDAPDebug( LDAP_DEBUG_TRACE, "moddn_newrdn_mods failed: olddn=%s baddn=%d badrdn=%d\n", + escape_string(olddn, ebuf), baddn, badrdn); + return LDAP_OPERATIONS_ERROR; + } + } + /* + * add new RDN values to the entry (non-normalized) + */ + rdns = ldap_explode_rdn( newrdn, 0 ); + if ( rdns != NULL ) + { + for ( i = 0; rdns[i] != NULL; i++ ) + { + if ( slapi_rdn2typeval( rdns[i], &type, &bv ) != 0) { + continue; + } + + /* add to entry if it's not already there or if was + * already deleted + */ + if (moddn_rdn_add_needed(ec, type, &bv, + deleteoldrdn, + smods_wsi) == 1) { + slapi_mods_add_modbvps( smods_wsi, LDAP_MOD_ADD, type, bvps ); + } + } + ldap_value_free( rdns ); + } + else + { + LDAPDebug( LDAP_DEBUG_TRACE, "moddn_newrdn_mods failed: could not parse new rdn %s\n", + escape_string(newrdn, ebuf), 0, 0); + return LDAP_OPERATIONS_ERROR; + } + + return LDAP_SUCCESS; +} + +static void +mods_remove_nsuniqueid(Slapi_Mods *smods) +{ + int i; + + LDAPMod **mods = slapi_mods_get_ldapmods_byref(smods); + for ( i = 0; mods[i] != NULL; i++ ) { + if (!strcasecmp(mods[i]->mod_type, SLAPI_ATTR_UNIQUEID)) { + mods[i]->mod_op = LDAP_MOD_IGNORE; + } + } +} + + +/* + * Update the indexes to reflect the DN change made. + * e is the entry before, ec the entry after. + * mods contains the list of attribute change made. + */ +static int +modrdn_rename_entry_update_indexes(back_txn *ptxn, Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *e, struct backentry *ec, Slapi_Mods *smods1, Slapi_Mods *smods2, Slapi_Mods *smods3) +{ + backend *be; + ldbm_instance *inst; + int retval= 0; + char *msg; + + slapi_pblock_get( pb, SLAPI_BACKEND, &be); + inst = (ldbm_instance *) be->be_instance_info; + + /* + * Update the ID to Entry index. + * Note that id2entry_add replaces the entry, so the Entry ID stays the same. + */ + retval = id2entry_add( be, ec, ptxn ); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + goto error_return; + } + if (retval != 0) + { + LDAPDebug( LDAP_DEBUG_ANY, "id2entry_add failed, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + goto error_return; + } + if(smods1!=NULL && slapi_mods_get_num_mods(smods1)>0) + { + /* + * update the indexes: lastmod, rdn, etc. + */ + retval = index_add_mods( be, (const LDAPMod **)slapi_mods_get_ldapmods_byref(smods1), e, ec, ptxn ); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + goto error_return; + } + if (retval != 0) + { + LDAPDebug( LDAP_DEBUG_TRACE, "index_add_mods 1 failed, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + goto error_return; + } + } + if(smods2!=NULL && slapi_mods_get_num_mods(smods2)>0) + { + /* + * smods2 contains the state generated mods. One of them might be the removal of a "nsuniqueid" rdn component + * previously gnerated through a conflict resolution. We need to make sure we don't remove the index for "nsuniqueid" + * so let's get it out from the mods before calling index_add_mods... + */ + mods_remove_nsuniqueid(smods2); + /* + * update the indexes: lastmod, rdn, etc. + */ + retval = index_add_mods( be, (const LDAPMod **)slapi_mods_get_ldapmods_byref(smods2), e, ec, ptxn ); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + goto error_return; + } + if (retval != 0) + { + LDAPDebug( LDAP_DEBUG_TRACE, "index_add_mods 2 failed, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + goto error_return; + } + } + if(smods3!=NULL && slapi_mods_get_num_mods(smods3)>0) + { + /* + * update the indexes: lastmod, rdn, etc. + */ + retval = index_add_mods( be, (const LDAPMod **)slapi_mods_get_ldapmods_byref(smods3), e, ec, ptxn ); + if (DB_LOCK_DEADLOCK == retval) + { + /* Retry txn */ + goto error_return; + } + if (retval != 0) + { + LDAPDebug( LDAP_DEBUG_TRACE, "index_add_mods 3 failed, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + goto error_return; + } + } + /* + * Remove the old entry from the Virtual List View indexes. + * Add the new entry to the Virtual List View indexes. + */ + retval= vlv_update_all_indexes(ptxn, be, pb, e, ec); + if (DB_LOCK_DEADLOCK == retval) + { + /* Abort and re-try */ + goto error_return; + } + if (retval != 0) + { + LDAPDebug( LDAP_DEBUG_TRACE, "vlv_update_all_indexes failed, err=%d %s\n", retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + goto error_return; + } + if (cache_replace( &inst->inst_cache, e, ec ) != 0 ) { + retval= -1; + goto error_return; + } +error_return: + return retval; +} + + +/* + */ +static int +moddn_rename_child_entry( + back_txn *ptxn, + Slapi_PBlock *pb, + struct ldbminfo *li, + struct backentry *e, + struct backentry *ec, + int parentdncomps, + char **newsuperiordns, + int newsuperiordncomps, + CSN *opcsn) +{ + /* + * Construct the new DN for the entry by taking the old DN + * excluding the old parent entry DN, and adding the new + * superior entry DN. + * + * ldap_explode_dn is probably a bit slow, but it knows about + * DN escaping which is pretty complicated, and we wouldn't + * want to reimplement that here. + * + * JCM - This was written before Slapi_RDN... so this could be made much neater. + */ + int retval; + char *olddn; + char *newdn; + char **olddns; + int olddncomps= 0; + int need= 1; /* For the '\0' */ + int i; + + olddn = slapi_entry_get_dn(ec->ep_entry); + olddns = ldap_explode_dn( olddn, 0 ); + for(;olddns[olddncomps]!=NULL;olddncomps++); + for(i=0;i<olddncomps-parentdncomps;i++) + { + need+= strlen(olddns[i]) + 2; /* For the ", " */ + } + for(i=0;i<newsuperiordncomps;i++) + { + need+= strlen(newsuperiordns[i]) + 2; /* For the ", " */ + } + need--; /* We don't have a comma on the end of the last component */ + newdn= slapi_ch_malloc(need); + newdn[0]= '\0'; + for(i=0;i<olddncomps-parentdncomps;i++) + { + strcat(newdn,olddns[i]); + strcat(newdn,", "); + } + for(i=0;i<newsuperiordncomps;i++) + { + strcat(newdn,newsuperiordns[i]); + if(i<newsuperiordncomps-1) + { + /* We don't have a comma on the end of the last component */ + strcat(newdn,", "); + } + } + ldap_value_free( olddns ); + slapi_entry_set_dn( ec->ep_entry, newdn ); + add_update_entrydn_operational_attributes (ec); + + /* + * Update the DN CSN of the entry. + */ + { + entry_add_dncsn(e->ep_entry, opcsn); + entry_add_rdn_csn(e->ep_entry, opcsn); + entry_set_maxcsn(e->ep_entry, opcsn); + } + { + Slapi_Mods smods; + slapi_mods_init(&smods, 2); + slapi_mods_add( &smods, LDAP_MOD_DELETE, "entrydn", strlen( backentry_get_ndn(e) ), backentry_get_ndn(e) ); + slapi_mods_add( &smods, LDAP_MOD_REPLACE, "entrydn", strlen( backentry_get_ndn(ec) ), backentry_get_ndn(ec) ); + /* + * Update all the indexes. + */ + retval= modrdn_rename_entry_update_indexes(ptxn, pb, li, e, ec, &smods, NULL, NULL); /* JCMREPL - Should the children get updated modifiersname and lastmodifiedtime? */ + slapi_mods_done(&smods); + } + return retval; +} + +/* + * Rename all the children of an entry who's name has changed. + */ +static int +moddn_rename_children( + back_txn *ptxn, + Slapi_PBlock *pb, + backend *be, + IDList *children, + Slapi_DN *dn_parentdn, + Slapi_DN *dn_newsuperiordn, + struct backentry *child_entries[], + struct backentry *child_entry_copies[]) +{ + /* Iterate over the children list renaming every child */ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + Slapi_Operation *operation; + CSN *opcsn; + int retval= 0, i; + char **newsuperiordns; + int newsuperiordncomps= 0; + int parentdncomps= 0; + + /* + * Break down the parent entry dn into its components. + */ + { + char **parentdns; + parentdns = ldap_explode_dn( slapi_sdn_get_dn(dn_parentdn), 0 ); + for(;parentdns[parentdncomps]!=NULL;parentdncomps++); + ldap_value_free( parentdns ); + } + + /* + * Break down the new superior entry dn into its components. + */ + newsuperiordns = ldap_explode_dn( slapi_sdn_get_dn(dn_newsuperiordn), 0 ); + for(;newsuperiordns[newsuperiordncomps]!=NULL;newsuperiordncomps++); + + /* + * Iterate over the child entries renaming them. + */ + slapi_pblock_get( pb, SLAPI_OPERATION, &operation ); + opcsn = operation_get_csn (operation); + for (i = 0; retval == 0 && child_entries[i] != NULL; i++) { + retval= moddn_rename_child_entry(ptxn, pb, li, child_entries[i], child_entry_copies[i], parentdncomps, newsuperiordns, newsuperiordncomps, opcsn ); + } + if (retval != 0) { + while (child_entries[i] != NULL) { + backentry_free(&(child_entry_copies[i])); + i++; + } + } + ldap_value_free( newsuperiordns ); + return retval; +} + + +/* + * Get an IDList of all the children of an entry. + */ +static IDList * +moddn_get_children(back_txn *ptxn, Slapi_PBlock *pb, backend *be, struct backentry *parententry, Slapi_DN *dn_parentdn, struct backentry ***child_entries, struct backentry ***child_entry_copies) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int err= 0; + IDList *candidates; + IDList *result_idl = NULL; + char filterstr[20]; + Slapi_Filter *filter; + NIDS nids; + int entrynumber= 0; + ID id; + idl_iterator sr_current; /* the current position in the search results */ + struct backentry *e= NULL; + + /* Fetch a candidate list of all the entries below the entry being moved */ + strcpy( filterstr, "objectclass=*" ); + filter = slapi_str2filter( filterstr ); + candidates= subtree_candidates(pb, be, slapi_sdn_get_ndn(dn_parentdn), parententry, filter, 1 /* ManageDSAIT */, NULL /* allids_before_scopingp */, &err); + slapi_filter_free(filter,1); + + if (candidates!=NULL) + { + sr_current = idl_iterator_init(candidates); + result_idl= idl_alloc(candidates->b_nids); + do + { + id = idl_iterator_dereference_increment(&sr_current, candidates); + if ( id!=NOID ) + { + int err= 0; + e = id2entry( be, id, NULL, &err ); + if (e!=NULL) + { + /* The subtree search will have included the parent entry in the result set */ + if (e!=parententry) + { + /* Check that the candidate entry is really below the base. */ + if(slapi_dn_issuffix( backentry_get_ndn(e), slapi_sdn_get_ndn(dn_parentdn))) + { + idl_append(result_idl,id); + } + } + cache_return(&inst->inst_cache, &e); + } + } + } while (id!=NOID); + idl_free(candidates); + } + + nids = result_idl ? result_idl->b_nids : 0; + + *child_entries= (struct backentry**)slapi_ch_calloc(sizeof(struct backentry*),nids+1); + *child_entry_copies= (struct backentry**)slapi_ch_calloc(sizeof(struct backentry*),nids+1); + + sr_current = idl_iterator_init(result_idl); + do { + id = idl_iterator_dereference_increment(&sr_current, result_idl); + if ( id!=NOID ) { + e= cache_find_id( &inst->inst_cache, id ); + if ( e != NULL ) { + cache_lock_entry(&inst->inst_cache, e); + (*child_entries)[entrynumber]= e; + (*child_entry_copies)[entrynumber]= backentry_dup(e); + entrynumber++; + } + } + } while (id!=NOID); + + return result_idl; +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_search.c b/ldap/servers/slapd/back-ldbm/ldbm_search.c new file mode 100644 index 00000000..ee33011d --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_search.c @@ -0,0 +1,1345 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* search.c - ldbm backend search function */ +/* view with ts=4 */ + +#include "back-ldbm.h" +#include "vlv_srch.h" + +/* prototypes */ +static int build_candidate_list( Slapi_PBlock *pb, backend *be, + struct backentry *e, const char * base, int scope, + int *lookup_returned_allidsp, IDList** candidates); +static IDList *base_candidates( Slapi_PBlock *pb, struct backentry *e ); +static IDList *onelevel_candidates( Slapi_PBlock *pb, backend *be, const char *base, struct backentry *e, Slapi_Filter *filter, int managedsait, int *lookup_returned_allidsp, int *err ); +static back_search_result_set* new_search_result_set(IDList* idl,int vlv, int lookthroughlimit); +static void delete_search_result_set( back_search_result_set **sr ); +static int can_skip_filter_test( Slapi_PBlock *pb, struct slapi_filter *f, + int scope, IDList *idl ); + +/* This is for performance testing, allows us to disable ACL checking altogether */ +#if defined(DISABLE_ACL_CHECK) +#define ACL_CHECK_FLAG 0 +#else +#define ACL_CHECK_FLAG 1 +#endif + +#define ISLEGACY(be) (be?(be->be_instance_info?(((ldbm_instance *)be->be_instance_info)->inst_li?(((ldbm_instance *)be->be_instance_info)->inst_li->li_legacy_errcode):0):0):0) + +static int +compute_lookthrough_limit( Slapi_PBlock *pb, struct ldbminfo *li ) +{ + Slapi_Connection *conn = NULL; + int limit; + + slapi_pblock_get( pb, SLAPI_CONNECTION, &conn); + + if ( slapi_reslimit_get_integer_limit( conn, + li->li_reslimit_lookthrough_handle, &limit ) + != SLAPI_RESLIMIT_STATUS_SUCCESS ) { + /* + * no limit associated with binder/connection or some other error + * occurred. use the default. + */ + int isroot = 0; + + slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot ); + if (isroot) { + limit = -1; + } else { + PR_Lock(li->li_config_mutex); + limit = li->li_lookthroughlimit; + PR_Unlock(li->li_config_mutex); + } + } + + return( limit ); +} + +/* don't free the berval, just clean it */ +static void +berval_done(struct berval *val) +{ + slapi_ch_free_string(&val->bv_val); +} + +/* + * We call this function as we exit ldbm_back_search + */ +int ldbm_back_search_cleanup(Slapi_PBlock *pb, struct ldbminfo *li, sort_spec_thing *sort_control, int ldap_result, char* ldap_result_description, int function_result, Slapi_DN *sdn, struct vlv_request *vlv_request_control) +{ + if(sort_control!=NULL) + { + sort_spec_free(sort_control); + } + if(ldap_result>=LDAP_SUCCESS) + { + slapi_send_ldap_result( pb, ldap_result, NULL, ldap_result_description, 0, NULL ); + } + { + /* hack hack --- code to free the result set if we don't need it */ + /* We get it and check to see if the structure was ever used */ + back_search_result_set *sr = NULL; + slapi_pblock_get( pb, SLAPI_SEARCH_RESULT_SET, &sr ); + if ( (NULL != sr) && (function_result != 0) ) { + delete_search_result_set(&sr); + } + } + slapi_sdn_done(sdn); + if (vlv_request_control) + { + berval_done(&vlv_request_control->value); + } + return function_result; +} + +/* + * Return values from ldbm_back_search are: + * + * 0: Success. A result set is in the pblock. No results have been + * sent to the client. + * 1: Success. The result has already been sent to the client. + * -1: An error occurred, and results have been sent to the client. + * -2: Disk Full. Abandon ship! + */ +int +ldbm_back_search( Slapi_PBlock *pb ) +{ + /* Search stuff */ + backend *be; + ldbm_instance *inst; + struct ldbminfo *li; + struct backentry *e; + IDList *candidates= NULL; + char *base; + Slapi_DN basesdn; + int scope; + LDAPControl **controls = NULL; + Slapi_Operation *operation; + entry_address *addr; + + /* SORT control stuff */ + int sort = 0; + int vlv = 0; + struct berval *sort_spec = NULL; + int is_sorting_critical = 0; + int is_sorting_critical_orig = 0; + sort_spec_thing *sort_control = NULL; + + /* VLV control stuff */ + int virtual_list_view = 0; + struct berval *vlv_spec = NULL; + int is_vlv_critical = 0; + struct vlv_request vlv_request_control; + back_search_result_set *sr = NULL; + + /* Fix for bugid #394184, SD, 20 Jul 00 */ + int tmp_err = -1; /* must be lower than LDAP_SUCCESS */ + char * tmp_desc = NULL; + /* end Fix for defect #394184 */ + + int lookup_returned_allids = 0; + int backend_count = 1; + static int print_once = 1; + + slapi_pblock_get( pb, SLAPI_BACKEND, &be ); + slapi_pblock_get( pb, SLAPI_OPERATION, &operation); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_SEARCH_TARGET, &base ); + slapi_pblock_get( pb, SLAPI_TARGET_ADDRESS, &addr); + slapi_pblock_get( pb, SLAPI_SEARCH_SCOPE, &scope ); + slapi_pblock_get( pb, SLAPI_REQCONTROLS, &controls ); + slapi_pblock_get( pb, SLAPI_BACKEND_COUNT, &backend_count ); + + inst = (ldbm_instance *) be->be_instance_info; + + slapi_sdn_init_dn_ndn_byref(&basesdn,base); /* normalized by front end*/ + /* Initialize the result set structure here because we need to use it during search processing */ + /* Beware that if we exit this routine sideways, we might leak this structure */ + sr = new_search_result_set( NULL, 0, + compute_lookthrough_limit( pb, li )); + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_SET, sr ); + + /* clear this out so we can free it later */ + memset(&vlv_request_control, 0, sizeof(vlv_request_control)); + if ( NULL != controls ) + { + /* Are we being asked to sort the results ? */ + sort = slapi_control_present( controls, LDAP_CONTROL_SORTREQUEST, &sort_spec, &is_sorting_critical_orig ); + if(sort) + { + int r= parse_sort_spec(sort_spec, &sort_control); + if(r!=0) + { + /* Badly formed SORT control */ + return ldbm_back_search_cleanup(pb, li, sort_control, LDAP_PROTOCOL_ERROR, "Sort Control", SLAPI_FAIL_GENERAL, &basesdn, NULL); + } + } + is_sorting_critical = is_sorting_critical_orig; + + /* Are we to provide a virtual view of the list? */ + if ((vlv = slapi_control_present( controls, LDAP_CONTROL_VLVREQUEST, &vlv_spec, &is_vlv_critical))) + { + if(sort) + { + int r = vlv_parse_request_control( be, vlv_spec, &vlv_request_control ); + if(r!=LDAP_SUCCESS) + { + /* Badly formed VLV control */ + return ldbm_back_search_cleanup(pb, li, sort_control, r, "VLV Control", SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + } + { + /* Access Control Check to see if the client is allowed to use the VLV Control. */ + Slapi_Entry *feature; + char dn[128]; + char *dummyAttr = "dummy#attr"; + char *dummyAttrs[2] = { NULL, NULL }; + + dummyAttrs[0] = dummyAttr; + + sprintf(dn,"dn: oid=%s,cn=features,cn=config",LDAP_CONTROL_VLVREQUEST); + feature= slapi_str2entry(dn,0); + r= plugin_call_acl_plugin (pb, feature, dummyAttrs, NULL, SLAPI_ACL_READ, ACLPLUGIN_ACCESS_DEFAULT, NULL); + slapi_entry_free(feature); + if(r!=LDAP_SUCCESS) + { + /* Client isn't allowed to do this. */ + return ldbm_back_search_cleanup(pb, li, sort_control, r, "VLV Control", SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + } + } + /* + * Sorting must always be critical for VLV; Force it be so. + */ + is_sorting_critical= 1; + virtual_list_view= 1; + } + else + { + /* Can't have a VLV control without a SORT control */ + return ldbm_back_search_cleanup(pb, li, sort_control, LDAP_SORT_CONTROL_MISSING, "VLV Control", SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + } + } + } + if ((virtual_list_view || sort) && backend_count > 0) + { + char *ctrlstr = NULL; + struct vlv_response vlv_response = {0}; + if (virtual_list_view) + { + if (sort) + { + ctrlstr = "The VLV and sort controls cannot be processed"; + } + else + { + ctrlstr = "The VLV control cannot be processed"; + } + } + else + { + if (sort) + { + ctrlstr = "The sort control cannot be processed"; + } + } + + PR_ASSERT(NULL != ctrlstr); + + if (print_once) + { + LDAPDebug(LDAP_DEBUG_ANY, + "ERROR: %s " + "when more than one backend is involved. " + "VLV indexes that will never be used should be removed.\n", + ctrlstr, 0, 0); + print_once = 0; + } + + /* 402380: mapping tree must refuse VLV and SORT control + * when several backends are impacted by a search */ + if (0 != is_vlv_critical) + { + vlv_response.result = LDAP_UNWILLING_TO_PERFORM; + vlv_make_response_control(pb, &vlv_response); + if (sort) + { + make_sort_response_control(pb, LDAP_UNWILLING_TO_PERFORM, NULL); + } + if (ISLEGACY(be)) + { + return ldbm_back_search_cleanup(pb, li, sort_control, + LDAP_UNWILLING_TO_PERFORM, ctrlstr, + SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + } + else + { + return ldbm_back_search_cleanup(pb, li, sort_control, + LDAP_VIRTUAL_LIST_VIEW_ERROR, ctrlstr, + SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + } + } + else + { + if (0 != is_sorting_critical_orig) + { + if (virtual_list_view) + { + vlv_response.result = LDAP_UNWILLING_TO_PERFORM; + vlv_make_response_control(pb, &vlv_response); + } + make_sort_response_control(pb, LDAP_UNWILLING_TO_PERFORM, NULL); + return ldbm_back_search_cleanup(pb, li, sort_control, + LDAP_UNAVAILABLE_CRITICAL_EXTENSION, ctrlstr, + SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + } + else /* vlv and sorting are not critical, so ignore the control */ + { + if (virtual_list_view) + { + vlv_response.result = LDAP_UNWILLING_TO_PERFORM; + vlv_make_response_control(pb, &vlv_response); + } + if (sort) + { + make_sort_response_control(pb, LDAP_UNWILLING_TO_PERFORM, NULL); + } + sort = 0; + virtual_list_view = 0; + } + } + } + + /* + * Get the base object for the search. + * The entry "" will never be contained in the database, + * so treat it as a special case. + */ + if ( *base == '\0' ) + { + e = NULL; + } + else + { + if ( ( e = find_entry( pb, be, addr, NULL )) == NULL ) + { + /* error or referral sent by find_entry */ + return ldbm_back_search_cleanup(pb, li, sort_control, -1, NULL, 1, &basesdn, &vlv_request_control); + } + } + + /* + * If this is a persistent search then the client is only + * interested in entries that change, so we skip building + * a candidate list. + */ + if (operation_is_flag_set( operation, OP_FLAG_PS_CHANGESONLY )) + { + candidates = NULL; + } + else + { + time_t time_up= 0; + int lookthrough_limit = 0; + struct vlv_response vlv_response_control; + int abandoned= 0; + int vlv_rc; + /* + * Build a list of IDs for this entry and scope + */ + if ((NULL != controls) && (sort)) { + switch (vlv_search_build_candidate_list(pb, &basesdn, &vlv_rc, sort_control, (vlv ? &vlv_request_control : NULL), &candidates, &vlv_response_control)) { + case VLV_ACCESS_DENIED: + return ldbm_back_search_cleanup(pb, li, sort_control, vlv_rc, "VLV Control", SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + + case VLV_BLD_LIST_FAILED: + return ldbm_back_search_cleanup(pb, li, sort_control, vlv_response_control.result, NULL, SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + + case LDAP_SUCCESS: + /* Log to the access log the particulars of this sort request */ + /* Log message looks like this: SORT <key list useful for input to ldapsearch> <#candidates> | <unsortable> */ + sort_log_access(pb,sort_control,NULL); + /* Since a pre-computed index was found for the VLV Search then + * the candidate list now contains exactly what should be returned. + * There's no need to sort or trim the candidate list. + * + * However, the client will be expecting a Sort Response control + */ + if (LDAP_SUCCESS != make_sort_response_control( pb, 0, NULL ) ) + { + return ldbm_back_search_cleanup(pb, li, sort_control, LDAP_OPERATIONS_ERROR, "Sort Response Control", SLAPI_FAIL_GENERAL, &basesdn, &vlv_request_control); + } + } + } + if(candidates==NULL) + { + int rc = build_candidate_list(pb, be, e, base, scope, + &lookup_returned_allids, &candidates); + if (rc) + { + /* Error result sent by build_candidate_list */ + return ldbm_back_search_cleanup(pb, li, sort_control, -1, NULL, rc, &basesdn, &vlv_request_control); + } + /* + * If we're sorting then we must check what administrative + * limits should be imposed. Work out at what time to give + * up, and how many entries we should sift through. + */ + if (sort && (NULL != candidates)) + { + time_t optime = 0; + time_t tlimit = 0; + + slapi_pblock_get( pb, SLAPI_SEARCH_TIMELIMIT, &tlimit ); + slapi_pblock_get( pb, SLAPI_OPINITIATED_TIME, &optime ); + /* + * (tlimit==-1) means no time limit + */ + time_up = ( tlimit==-1 ? -1 : optime + tlimit); + + lookthrough_limit = compute_lookthrough_limit( pb, li ); + } + + /* + * If we're presenting a virtual list view, then apply the + * search filter before sorting. + */ + if (virtual_list_view && (NULL != candidates)) + { + int r= 0; + IDList *idl= NULL; + Slapi_Filter *filter= NULL; + slapi_pblock_get( pb, SLAPI_SEARCH_FILTER, &filter ); + r= vlv_filter_candidates(be, pb, candidates, &basesdn, scope, filter, &idl, lookthrough_limit, time_up); + if(r==0) + { + idl_free(candidates); + candidates= idl; + } + else + { + return ldbm_back_search_cleanup(pb, li, sort_control, r, NULL, -1, &basesdn, &vlv_request_control); + } + } + /* + * Client wants the server to sort the results. + */ + if (sort && (NULL != candidates)) + { + /* Before we haste off to sort the candidates, we need to + * prepare some information for the purpose of imposing the + * administrative limits. + * We figure out the time when the time limit will be up. + * We can't use the size limit because we might be sorting + * a candidate list larger than the result set. + * But, we can use the lookthrough limit---we count each + * time we access an entry as one look and act accordingly. + */ + + char *sort_error_type = NULL; + int sort_return_value = 0; + + /* Log to the access log the particulars of this sort request */ + /* Log message looks like this: SORT <key list useful for input to ldapsearch> <#candidates> | <unsortable> */ + sort_log_access(pb,sort_control,candidates); + sort_return_value = sort_candidates( be, lookthrough_limit, time_up, pb, candidates, sort_control, &sort_error_type ); + /* Fix for bugid # 394184, SD, 20 Jul 00 */ + /* replace the hard coded return value by the appropriate LDAP error code */ + switch (sort_return_value) { + case LDAP_SUCCESS: /* Everything OK */ + vlv_response_control.result= LDAP_SUCCESS; + break; + case LDAP_PROTOCOL_ERROR: /* A protocol error */ + return ldbm_back_search_cleanup(pb, li, sort_control, LDAP_PROTOCOL_ERROR, "Sort Control", -1, &basesdn, &vlv_request_control); + case LDAP_UNWILLING_TO_PERFORM: /* Too hard */ + case LDAP_OPERATIONS_ERROR: /* Operation error */ + case LDAP_TIMELIMIT_EXCEEDED: /* Timeout */ + vlv_response_control.result= LDAP_TIMELIMIT_EXCEEDED; + break; + case LDAP_ADMINLIMIT_EXCEEDED: /* Admin limit exceeded */ + vlv_response_control.result= LDAP_ADMINLIMIT_EXCEEDED; + break; + case LDAP_OTHER: /* Abandoned */ + abandoned= 1; /* So that we don't return a result code */ + is_sorting_critical= 1; /* In order to have the results discarded */ + break; + default: /* Should never get here */ + break; + } + /* End fix for bug # 394184 */ + /* + * If the sort control was marked as critical, and there was an error in sorting, + * don't return any entries, and return unavailableCriticalExtension in the + * searchResultDone message. + */ + /* Fix for bugid #394184, SD, 05 Jul 00 */ + /* we were not actually returning unavailableCriticalExtension; + now fixed (hopefully !) */ + if (is_sorting_critical && (0 != sort_return_value)) + { + idl_free(candidates); + candidates = idl_alloc(0); + tmp_err = LDAP_UNAVAILABLE_CRITICAL_EXTENSION; + tmp_desc = "Sort Response Control"; + } + /* end Fix for bugid #394184 */ + /* Generate the control returned to the client to indicate sort result */ + if (LDAP_SUCCESS != make_sort_response_control( pb, sort_return_value, sort_error_type ) ) + { + return ldbm_back_search_cleanup(pb, li, sort_control, (abandoned?-1:LDAP_PROTOCOL_ERROR), "Sort Response Control", -1, &basesdn, &vlv_request_control); + } + } + /* + * If we're presenting a virtual list view, then the candidate list + * must be trimmed down to just the range of entries requested. + */ + if (virtual_list_view) + { + if (NULL != candidates && candidates->b_nids>0) + { + IDList *idl= NULL; + vlv_response_control.result= vlv_trim_candidates(be, candidates, sort_control, &vlv_request_control, &idl, &vlv_response_control); + if(vlv_response_control.result==0) + { + idl_free(candidates); + candidates= idl; + } + else + { + return ldbm_back_search_cleanup(pb, li, sort_control, vlv_response_control.result, NULL, -1, &basesdn, &vlv_request_control); + } + } + else + { + vlv_response_control.targetPosition= 0; + vlv_response_control.contentCount= 0; + vlv_response_control.result= LDAP_SUCCESS; + } + } + } + if (virtual_list_view) + { + if(LDAP_SUCCESS != vlv_make_response_control( pb, &vlv_response_control )) + { + return ldbm_back_search_cleanup(pb, li, sort_control, (abandoned?-1:LDAP_PROTOCOL_ERROR), "VLV Response Control", -1, &basesdn, &vlv_request_control); + } + /* Log the VLV operation */ + vlv_print_access_log(pb,&vlv_request_control,&vlv_response_control); + } + } + + cache_return( &inst->inst_cache, &e ); + + /* + * if the candidate list is an allids list, arrange for access log + * to record that fact. + */ + if ( NULL != candidates && ALLIDS( candidates )) { + unsigned int opnote = SLAPI_OP_NOTE_UNINDEXED; + int ri = 0; + + /* + * Return error if nsslapd-require-index is set and + * this is not an internal operation. + * We hope the plugins know what they are doing! + */ + if (!operation_is_flag_set(operation, OP_FLAG_INTERNAL)) { + + PR_Lock(inst->inst_config_mutex); + ri = inst->require_index; + PR_Unlock(inst->inst_config_mutex); + + if (ri) { + idl_free(candidates); + candidates = idl_alloc(0); + tmp_err = LDAP_UNWILLING_TO_PERFORM; + tmp_desc = "Search is not indexed"; + } + } + + slapi_pblock_set( pb, SLAPI_OPERATION_NOTES, &opnote ); + } + + sr->sr_candidates = candidates; + sr->sr_virtuallistview = virtual_list_view; + + /* check to see if we can skip the filter test */ + if ( li->li_filter_bypass && NULL != candidates && !virtual_list_view + && !lookup_returned_allids ) { + Slapi_Filter *filter= NULL; + + slapi_pblock_get( pb, SLAPI_SEARCH_FILTER, &filter ); + if ( can_skip_filter_test( pb, filter, scope, candidates)) { + sr->sr_flags |= SR_FLAG_CAN_SKIP_FILTER_TEST; + } + } + + /* Fix for bugid #394184, SD, 05 Jul 00 */ + /* tmp_err == -1: no error */ + return ldbm_back_search_cleanup(pb, li, sort_control, tmp_err, tmp_desc, (tmp_err == -1 ? 0 : -1), &basesdn, &vlv_request_control); + /* end Fix for bugid #394184 */ +} + +/* + * Build a candidate list for this backentry and scope. + * Could be a BASE, ONELEVEL, or SUBTREE search. + * + * Returns: + * 0 - success + * <0 - fail + * + */ +static int +build_candidate_list( Slapi_PBlock *pb, backend *be, struct backentry *e, + const char * base, int scope, int *lookup_returned_allidsp, + IDList** candidates) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + int managedsait= 0; + Slapi_Filter *filter= NULL; + int err= 0; + int r= 0; + + slapi_pblock_get( pb, SLAPI_SEARCH_FILTER, &filter ); + slapi_pblock_get( pb, SLAPI_MANAGEDSAIT, &managedsait ); + + switch ( scope ) { + case LDAP_SCOPE_BASE: + *candidates = base_candidates( pb, e ); + break; + + case LDAP_SCOPE_ONELEVEL: + *candidates = onelevel_candidates( pb, be, base, e, filter, managedsait, + lookup_returned_allidsp, &err ); + break; + + case LDAP_SCOPE_SUBTREE: + *candidates = subtree_candidates(pb, be, base, e, filter, managedsait, + lookup_returned_allidsp, &err); + break; + + default: + slapi_send_ldap_result( pb, LDAP_PROTOCOL_ERROR, NULL, "Bad scope", 0, NULL ); + r = SLAPI_FAIL_GENERAL; + } + if ( 0 != err && DB_NOTFOUND != err ) { + LDAPDebug( LDAP_DEBUG_ANY, "database error %d\n", err, 0, 0 ); + slapi_send_ldap_result( pb, LDAP_OPERATIONS_ERROR, NULL, NULL, + 0, NULL ); + if (LDBM_OS_ERR_IS_DISKFULL(err)) r = return_on_disk_full(li); + else r = SLAPI_FAIL_GENERAL; + } + + /* + * If requested, set a flag to indicate whether the indexed + * lookup returned an ALLIDs block. Note that this is taken care of + * above already for subtree searches. + */ + if ( NULL != lookup_returned_allidsp ) { + if ( 0 == err ) { + if ( !(*lookup_returned_allidsp) && LDAP_SCOPE_SUBTREE != scope ) { + *lookup_returned_allidsp = + ( NULL != *candidates && ALLIDS( *candidates )); + } + } else { + *lookup_returned_allidsp = 0; + } + } + + LDAPDebug(LDAP_DEBUG_TRACE, "candidate list has %lu ids\n", + *candidates ? (*candidates)->b_nids : 0L, 0, 0); + + return r; +} + +/* + * Build a candidate list for a BASE scope search. + */ +static IDList * +base_candidates(Slapi_PBlock *pb, struct backentry *e) +{ + IDList *idl= idl_alloc( 1 ); + idl_append( idl, NULL == e ? 0 : e->ep_id ); + return( idl ); +} + +/* + * Modify the filter to include entries of the referral objectclass + * + * make (|(originalfilter)(objectclass=referral)) + * + * "focref, forr" are temporary filters which the caller must free + * non-recursively when done with the returned filter. + */ +static Slapi_Filter* +create_referral_filter(Slapi_Filter* filter, Slapi_Filter** focref, Slapi_Filter** forr) +{ + char *buf = slapi_ch_strdup( "objectclass=referral" ); + + *focref = slapi_str2filter( buf ); + *forr = slapi_filter_join( LDAP_FILTER_OR, filter, *focref ); + + slapi_ch_free((void **)&buf); + return *forr; +} + +/* + * Modify the filter to be a one level search. + * + * (&(parentid=idofbase)(|(originalfilter)(objectclass=referral))) + * + * "fid2kids, focref, fand, forr" are temporary filters which the + * caller must free'd non-recursively when done with the returned filter. + * + * This function is exported for the VLV code to use. + */ +Slapi_Filter* +create_onelevel_filter(Slapi_Filter* filter, const struct backentry *baseEntry, int managedsait, Slapi_Filter** fid2kids, Slapi_Filter** focref, Slapi_Filter** fand, Slapi_Filter** forr) +{ + Slapi_Filter *ftop= filter; + char buf[40]; + + if ( !managedsait ) + { + ftop= create_referral_filter(filter, focref, forr); + } + + sprintf( buf, "parentid=%lu", (u_long)(baseEntry != NULL ? baseEntry->ep_id : 0) ); + *fid2kids = slapi_str2filter( buf ); + *fand = slapi_filter_join( LDAP_FILTER_AND, ftop, *fid2kids ); + + return *fand; +} + +/* + * Build a candidate list for a ONELEVEL scope search. + */ +static IDList * +onelevel_candidates( + Slapi_PBlock *pb, + backend *be, + const char *base, + struct backentry *e, + Slapi_Filter *filter, + int managedsait, + int *lookup_returned_allidsp, + int *err +) +{ + Slapi_Filter *fid2kids= NULL; + Slapi_Filter *focref= NULL; + Slapi_Filter *fand= NULL; + Slapi_Filter *forr= NULL; + Slapi_Filter *ftop= NULL; + IDList *candidates; + + /* + * modify the filter to be something like this: + * + * (&(parentid=idofbase)(|(originalfilter)(objectclass=referral))) + */ + + ftop= create_onelevel_filter(filter, e, managedsait, &fid2kids, &focref, &fand, &forr); + + /* from here, it's just like subtree_candidates */ + candidates = filter_candidates( pb, be, base, ftop, NULL, 0, err ); + + *lookup_returned_allidsp = slapi_be_is_flag_set(be, SLAPI_BE_FLAG_DONT_BYPASS_FILTERTEST); + + /* free up just the filter stuff we allocated above */ + slapi_filter_free( fid2kids, 0 ); + slapi_filter_free( fand, 0 ); + slapi_filter_free( forr, 0 ); + slapi_filter_free( focref, 0 ); + + return( candidates ); +} + + +#define GRABSIZE2 50 +#define BUF_ALLOC_CAT( cpyfunc, s ) { \ + int len = 2 * strlen( s ); \ + while ( bmax - bcur < len + 1 ) { \ + bmax += GRABSIZE2; \ + buf = slapi_ch_realloc( buf, bmax ); \ + } \ + cpyfunc( buf + bcur, s ); \ + bcur += strlen( buf + bcur ); \ +} + +/* + * We need to modify the filter to be something like this: + * + * (|(originalfilter)(objectclass=referral)) + * + * the "objectclass=referral" part is used to select referrals to return. + * it is only included if the managedsait service control is not set. + * + * This function is exported for the VLV code to use. + */ +Slapi_Filter* +create_subtree_filter(Slapi_Filter* filter, int managedsait, Slapi_Filter** focref, Slapi_Filter** forr) +{ + Slapi_Filter *ftop= filter; + + if ( !managedsait ) + { + ftop= create_referral_filter(filter, focref, forr); + } + + return ftop; +} + + +static int +nscpentrydn_check_filter(Slapi_Filter *f) +{ + if (!f || (f->f_choice != LDAP_FILTER_AND)) + return 0; /* Not nscpEntryDN filter */ + + if ( 0 == strcasecmp ( f->f_and->f_avtype, SLAPI_ATTR_NSCP_ENTRYDN)) { + return 1; /* Contains a nscpEntryDN filter */ + } else if ( 0 == strcasecmp ( f->f_and->f_next->f_avtype, SLAPI_ATTR_NSCP_ENTRYDN)) { + return 1; + } + return 0; /* Not nscpEntryDN filter */ +} + + +/* + * Build a candidate list for a SUBTREE scope search. + */ +IDList * +subtree_candidates( + Slapi_PBlock *pb, + backend *be, + const char *base, + const struct backentry *e, + Slapi_Filter *filter, + int managedsait, + int *allids_before_scopingp, + int *err +) +{ + Slapi_Filter *focref= NULL; + Slapi_Filter *forr= NULL; + Slapi_Filter *ftop= NULL; + IDList *candidates; + PRBool has_tombstone_filter; + int isroot = 0; + + /* make (|(originalfilter)(objectclass=referral)) */ + ftop= create_subtree_filter(filter, managedsait, &focref, &forr); + + /* Fetch a candidate list for the original filter */ + candidates = filter_candidates( pb, be, base, ftop, NULL, 0, err ); + slapi_filter_free( forr, 0 ); + slapi_filter_free( focref, 0 ); + + /* set 'allids before scoping' flag */ + if ( NULL != allids_before_scopingp ) { + *allids_before_scopingp = ( NULL != candidates && ALLIDS( candidates )); + } + + has_tombstone_filter = (filter->f_flags & SLAPI_FILTER_TOMBSTONE); + slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot ); + + /* + * Apply the DN components if the candidate list is greater than + * our threshold, and if the filter is not "(objectclass=nstombstone)", + * since tombstone entries are not indexed in the ancestorid index. + */ + if(candidates!=NULL && ( idl_length(candidates)>FILTER_TEST_THRESHOLD) && !has_tombstone_filter) + { + IDList *tmp = candidates, *descendants = NULL; + + *err = ldbm_ancestorid_read(be, NULL, e->ep_id, &descendants); + idl_insert(&descendants, e->ep_id); + candidates = idl_intersection(be, candidates, descendants); + idl_free(tmp); + idl_free(descendants); + } + /* + * If the search is initiated by the Directory Manager, + * and the filter includes objectclass=nsTombstone, + * then we union the candidate list with all the tombstone + * entries in this backend instance. + */ + if (has_tombstone_filter && isroot && !nscpentrydn_check_filter(filter)) + { + IDList *idl; + IDList *tmp= candidates; + struct slapi_filter f = {0}; + f.f_choice = LDAP_FILTER_EQUALITY; + f.f_avtype = "objectclass"; + f.f_avvalue.bv_val = SLAPI_ATTR_VALUE_TOMBSTONE; + f.f_avvalue.bv_len = strlen(SLAPI_ATTR_VALUE_TOMBSTONE); + f.f_next= NULL; + idl = filter_candidates( pb, be, NULL, &f, NULL, 0, err ); + + /* + * If that gave allids then try (nscpentrydn=*) instead. + * The nscpentrydn equality index contains all the tombstones + * and can be used to resolve a presence filter without + * hitting allids. + */ + if (idl && ALLIDS(idl)) { + idl_free(idl); + f.f_choice = LDAP_FILTER_PRESENT; + f.f_avtype = SLAPI_ATTR_NSCP_ENTRYDN; + idl = filter_candidates( pb, be, NULL, &f, NULL, 0, err ); + } + + candidates = idl_union( be, idl, tmp ); + idl_free( idl ); + idl_free( tmp ); + } + + return( candidates ); +} + +static int grok_filter(struct slapi_filter *f); +#if 0 +/* Helper for grok_filter() */ +static int +grok_filter_list(struct slapi_filter *flist) +{ + struct slapi_filter *f; + + /* Scan the clauses of the AND filter, if any of them fails the grok, then we fail */ + for ( f = flist; f != NULL; f = f->f_next ) { + if ( !grok_filter(f) ) { + return( 0 ); + } + } + return( 1 ); +} +#endif + +/* Helper function for can_skip_filter_test() */ +static int grok_filter(struct slapi_filter *f) +{ + switch ( f->f_choice ) { + case LDAP_FILTER_EQUALITY: + return 1; /* If there's an ID list and an equality filter, we can skip the filter test */ + case LDAP_FILTER_SUBSTRINGS: + return 0; + + case LDAP_FILTER_GE: + return 1; + + case LDAP_FILTER_LE: + return 1; + + case LDAP_FILTER_PRESENT: + return 1; /* If there's an ID list, and a presence filter, we can skip the filter test */ + + case LDAP_FILTER_APPROX: + return 0; + + case LDAP_FILTER_EXTENDED: + return 0; + + case LDAP_FILTER_AND: + return 0; /* Unless we check to see whether the presence and equality branches + of the search filter were all indexed, we get things wrong here, + so let's punt for now */ + /* return grok_filter_list(f->f_and); AND clauses are potentially OK */ + + case LDAP_FILTER_OR: + return 0; + + case LDAP_FILTER_NOT: + return 0; + + default: + return 0; + } +} + +/* Routine which says whether or not the indices produced a "correct" answer */ +static int +can_skip_filter_test( + Slapi_PBlock *pb, + struct slapi_filter *f, + int scope, + IDList *idl +) +{ + /* Is the ID list ALLIDS ? */ + if ( ALLIDS(idl)) { + /* If so, then can't optimize */ + return 0; + } + + /* Is this a base scope search? */ + if ( scope == LDAP_SCOPE_BASE ) { + /* + * If so, then we can't optimize. Why not? Because we only consult + * the entrydn index in producing our 1 candidate, and that means + * we have not used the filter to produce the candidate list. + */ + return 0; + } + + /* Grok the filter and tell me if it has only equality components in it */ + return grok_filter(f); +} + + + +/* + * Return the next entry in the result set. The entry is returned + * in the pblock. + * Returns 0 normally. If -1 is returned, it means that some + * exceptional condition, e.g. timelimit exceeded has occurred, + * and this routine has sent a result to the client. If zero + * is returned and no entry is available in the PBlock, then + * we've iterated through all the entries. + */ +int +ldbm_back_next_search_entry( Slapi_PBlock *pb ) +{ + return ldbm_back_next_search_entry_ext( pb, 0 ); +} + +int +ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension ) +{ + backend *be; + ldbm_instance *inst; + struct ldbminfo *li; + int scope; + int managedsait; + Slapi_Attr *attr; + Slapi_Filter *filter; + char *base; + back_search_result_set *sr; + ID id; + struct backentry *e; + int nentries; + time_t curtime, stoptime, optime; + int tlimit, llimit, slimit, isroot; + struct berval **urls = NULL; + int err; + Slapi_DN basesdn; + char *target_uniqueid; + + slapi_pblock_get( pb, SLAPI_BACKEND, &be ); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_SEARCH_SCOPE, &scope ); + slapi_pblock_get( pb, SLAPI_MANAGEDSAIT, &managedsait ); + slapi_pblock_get( pb, SLAPI_SEARCH_FILTER, &filter ); + slapi_pblock_get( pb, SLAPI_SEARCH_TARGET, &base ); + slapi_pblock_get( pb, SLAPI_NENTRIES, &nentries ); + slapi_pblock_get( pb, SLAPI_SEARCH_SIZELIMIT, &slimit ); + slapi_pblock_get( pb, SLAPI_SEARCH_TIMELIMIT, &tlimit ); + slapi_pblock_get( pb, SLAPI_OPINITIATED_TIME, &optime ); + slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot ); + slapi_pblock_get( pb, SLAPI_SEARCH_REFERRALS, &urls ); + slapi_pblock_get( pb, SLAPI_SEARCH_RESULT_SET, &sr ); + slapi_pblock_get( pb, SLAPI_TARGET_UNIQUEID, &target_uniqueid ); + + inst = (ldbm_instance *) be->be_instance_info; + + slapi_sdn_init_dn_ndn_byref(&basesdn,base); /* normalized by front end */ + /* Return to the cache the entry we handed out last time */ + /* If we are using the extension, the front end will tell + * us when to do this so we don't do it now */ + if ( !use_extension ) + { + cache_return( &inst->inst_cache, &(sr->sr_entry) ); + } + + if(sr->sr_vlventry != NULL && !use_extension ) + { + /* This empty entry was handed out last time because the ACL check failed on a VLV Search. */ + /* The empty entry has a pointer to the cache entry dn... make sure we don't free the dn */ + /* which belongs to the cache entry. */ + slapi_entry_free( sr->sr_vlventry ); + sr->sr_vlventry = NULL; + } + + stoptime = optime + tlimit; + llimit = sr->sr_lookthroughlimit; + + /* Find the next candidate entry and return it. */ + while ( 1 ) + { + + /* check for abandon */ + if ( slapi_op_abandoned( pb )) + { + delete_search_result_set( &sr ); + if ( use_extension ) { + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, NULL ); + } + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY, NULL ); + slapi_sdn_done(&basesdn); + return -1; + } + + /* check time limit */ + curtime = current_time(); + if ( tlimit != -1 && curtime > stoptime ) + { + slapi_send_ldap_result( pb, LDAP_TIMELIMIT_EXCEEDED, NULL, NULL, nentries, urls ); + delete_search_result_set( &sr ); + if ( use_extension ) { + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, NULL ); + } + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY, NULL ); + slapi_sdn_done(&basesdn); + return -1; + } + + /* check lookthrough limit */ + if ( llimit != -1 && sr->sr_lookthroughcount >= llimit ) + { + slapi_send_ldap_result( pb, LDAP_ADMINLIMIT_EXCEEDED, NULL, NULL, nentries, urls ); + delete_search_result_set( &sr ); + if ( use_extension ) { + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, NULL ); + } + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY, NULL ); + slapi_sdn_done(&basesdn); + return -1; + } + + /* get the entry */ + id = idl_iterator_dereference_increment(&(sr->sr_current), sr->sr_candidates); + if ( id == NOID ) + { + /* No more entries */ + delete_search_result_set( &sr ); + if ( use_extension ) { + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, NULL ); + } + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY, NULL ); + slapi_sdn_done(&basesdn); + return 0; + } + + ++sr->sr_lookthroughcount; /* checked above */ + + /* get the entry */ + if ( (e = id2entry( be, id, NULL, &err )) == NULL ) + { + if ( err != 0 && err != DB_NOTFOUND ) + { + LDAPDebug( LDAP_DEBUG_ANY, "next_search_entry db err %d\n", err, 0, 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(err)) + { + /* disk full in the middle of returning search results + * is gonna be traumatic. unavoidable. + */ + slapi_send_ldap_result(pb, LDAP_OPERATIONS_ERROR, NULL, NULL, 0, NULL); + slapi_sdn_done(&basesdn); + return return_on_disk_full(li); + } + } + LDAPDebug( LDAP_DEBUG_ARGS, "candidate %lu not found\n", (u_long)id, 0, 0 ); + if ( err == DB_NOTFOUND ) + { + /* Since we didn't really look at this entry, we should + * decrement the lookthrough counter (it was just incremented). + * If we didn't do this, it would be possible to go over the + * lookthrough limit when there are fewer entries in the database + * than the lookthrough limit. This could happen on an ALLIDS + * search after adding a bunch of entries and then deleting + * them. */ + --sr->sr_lookthroughcount; + } + continue; + } + e->ep_vlventry = NULL; + sr->sr_entry = e; + + /* + * If it's a referral, return it without checking the + * filter explicitly here since it's only a candidate anyway. Do + * check the scope though. + */ + if ( !managedsait && slapi_entry_attr_find( e->ep_entry, "ref", &attr ) == 0) + { + Slapi_Value **refs= attr_get_present_values(attr); + if ( refs == NULL || refs[0] == NULL ) + { + char ebuf[ BUFSIZ ]; + LDAPDebug( LDAP_DEBUG_ANY, "null ref in (%s)\n", escape_string( backentry_get_ndn(e), ebuf ), 0, 0 ); + } + else if ( slapi_sdn_scope_test( backentry_get_sdn(e), &basesdn, scope )) + { + if ( use_extension ) { + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, e ); + } + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY, e->ep_entry ); + slapi_sdn_done(&basesdn); + return 0; + } + } + else + { + /* + * As per slapi_filter_test: + * 0 filter matched + * -1 filter did not match + * >0 an ldap error code + */ + int filter_test = -1; + + if((slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_LDAPSUBENTRY) + && !filter_flag_is_set(filter,SLAPI_FILTER_LDAPSUBENTRY)) || + (slapi_entry_flag_is_set(e->ep_entry,SLAPI_ENTRY_FLAG_TOMBSTONE) + && (!isroot || !filter_flag_is_set(filter, SLAPI_FILTER_TOMBSTONE)))) + { + /* If the entry is an LDAP subentry and filter don't filter subentries OR + * the entry is a TombStone and filter don't filter Tombstone + * don't return the entry + */ + /* ugaston - we don't want to mistake this filter failure with the one below due to ACL, + * because whereas the former should be read as 'no entry must be returned', the latter + * might still lead to return an empty entry. */ + filter_test=-1; + } + else + { + /* it's a regular entry, check if it matches the filter, and passes the ACL check */ + if ( 0 != ( sr->sr_flags & SR_FLAG_CAN_SKIP_FILTER_TEST )) { + /* Since we do access control checking in the filter test (?Why?) we need to check access now */ + LDAPDebug( LDAP_DEBUG_FILTER, "Bypassing filter test\n", 0, 0, 0 ); + if ( ACL_CHECK_FLAG ) { + filter_test = slapi_vattr_filter_test_ext( pb, e->ep_entry, filter, ACL_CHECK_FLAG, 1 /* Only perform access checking, thank you */); + } else { + filter_test = 0; + } + if (li->li_filter_bypass_check) { + int ft_rc; + + LDAPDebug( LDAP_DEBUG_FILTER, "Checking bypass\n", 0, 0, 0 ); + ft_rc = slapi_vattr_filter_test( pb, e->ep_entry, filter, + ACL_CHECK_FLAG ); + if (filter_test != ft_rc) { + /* Oops ! This means that we thought we could bypass the filter test, but noooo... */ + char ebuf[ BUFSIZ ]; + LDAPDebug( LDAP_DEBUG_ANY, "Filter bypass ERROR on entry %s\n", escape_string( backentry_get_ndn(e), ebuf ), 0, 0 ); + filter_test = ft_rc; /* Fix the error */ + } + } + } else { + /* Old-style case---we need to do a filter test */ + filter_test = slapi_vattr_filter_test( pb, e->ep_entry, filter, ACL_CHECK_FLAG); + } + } + if ( (filter_test == 0) || (sr->sr_virtuallistview && (filter_test != -1)) ) + /* ugaston - if filter failed due to subentries or tombstones (filter_test=-1), + * just forget about it, since we don't want to return anything at all. */ + { + if ( slapi_uniqueIDCompareString(target_uniqueid, e->ep_entry->e_uniqueid) || + slapi_sdn_scope_test( backentry_get_sdn(e), &basesdn, scope )) + { + /* check size limit */ + if ( slimit >= 0 ) + { + if ( --slimit < 0 ) { + cache_return( &inst->inst_cache, &e ); + delete_search_result_set( &sr ); + slapi_send_ldap_result( pb, LDAP_SIZELIMIT_EXCEEDED, NULL, NULL, nentries, urls ); + slapi_sdn_done(&basesdn); + return -1; + } + slapi_pblock_set( pb, SLAPI_SEARCH_SIZELIMIT, &slimit ); + } + if ( (filter_test != 0) && sr->sr_virtuallistview) + { + /* Slapi Filter Test failed. + * Must be that the ACL check failed. + * Send back an empty entry. + */ + sr->sr_vlventry = slapi_entry_alloc(); + slapi_entry_init(sr->sr_vlventry,slapi_ch_strdup(slapi_entry_get_dn_const(e->ep_entry)),NULL); + e->ep_vlventry = sr->sr_vlventry; + if ( use_extension ) { + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, e ); + } + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY, sr->sr_vlventry ); + } else { + if ( use_extension ) { + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY_EXT, e ); + } + slapi_pblock_set( pb, SLAPI_SEARCH_RESULT_ENTRY, e->ep_entry ); + } + slapi_sdn_done(&basesdn); + return 0; + } + else + { + cache_return ( &inst->inst_cache, &(sr->sr_entry) ); + } + } + else + { + /* Failed the filter test, and this isn't a VLV Search */ + cache_return( &inst->inst_cache, &(sr->sr_entry) ); + } + } + } + /*NOTREACHED*/ + slapi_sdn_done(&basesdn); +} + + +static back_search_result_set* +new_search_result_set(IDList *idl, int vlv, int lookthroughlimit) +{ + back_search_result_set *p= (back_search_result_set *)slapi_ch_malloc( sizeof( back_search_result_set )); + p->sr_candidates = idl; + p->sr_current = idl_iterator_init(idl); + p->sr_entry = NULL; + p->sr_lookthroughcount = 0; + p->sr_lookthroughlimit = lookthroughlimit; + p->sr_virtuallistview= vlv; + p->sr_vlventry = NULL; + p->sr_flags = 0; + return p; +} + +static void +delete_search_result_set( back_search_result_set **sr ) +{ + if ( NULL == sr || NULL == *sr) + { + return; + } + if ( NULL != (*sr)->sr_candidates ) + { + idl_free( (*sr)->sr_candidates ); + } + slapi_ch_free( (void**)sr ); +} + + +int +ldbm_back_entry_release( Slapi_PBlock *pb, void *backend_info_ptr ) { + backend *be; + ldbm_instance *inst; + + if ( backend_info_ptr == NULL ) + return 1; + + slapi_pblock_get( pb, SLAPI_BACKEND, &be ); + inst = (ldbm_instance *) be->be_instance_info; + + cache_return( &inst->inst_cache, (struct backentry **)&backend_info_ptr ); + + if( ((struct backentry *) backend_info_ptr)->ep_vlventry != NULL ) + { + /* This entry was created during a vlv search whose acl check failed. It needs to be + * freed here */ + slapi_entry_free( ((struct backentry *) backend_info_ptr)->ep_vlventry ); + ((struct backentry *) backend_info_ptr)->ep_vlventry = NULL; + } + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/ldbm_unbind.c b/ldap/servers/slapd/back-ldbm/ldbm_unbind.c new file mode 100644 index 00000000..9d3e80fa --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldbm_unbind.c @@ -0,0 +1,14 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* unbind.c - handle an ldap unbind operation */ + +#include "back-ldbm.h" + +int +ldbm_back_unbind( Slapi_PBlock *pb ) +{ + return( 0 ); +} diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c new file mode 100644 index 00000000..1930cb51 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c @@ -0,0 +1,2440 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2004 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* ldif2ldbm.c + * + * common functions for import (old and new) and export + * the export code (db2ldif) + * code for db2index (is this still in use?) + */ + +#include "back-ldbm.h" +#include "vlv_srch.h" +#include "dblayer.h" +#include "import.h" + +static char *sourcefile = "ldif2ldbm.c"; + + +static int db2index_add_indexed_attr(backend *be, char *attrString); + +static int ldbm_exclude_attr_from_export( struct ldbminfo *li, + const char *attr, int dump_uniqueid ); + + +/********** common routines for classic/deluxe import code **********/ + +static size_t import_config_index_buffer_size = DEFAULT_IMPORT_INDEX_BUFFER_SIZE; + +void import_configure_index_buffer_size(size_t size) +{ + import_config_index_buffer_size = size; +} + +size_t import_get_index_buffer_size() { + return import_config_index_buffer_size; +} + +static PRIntn import_subcount_hash_compare_keys(const void *v1, const void *v2) +{ + return( ((ID)v1 == (ID)v2 ) ? 1 : 0); +} + +static PRIntn import_subcount_hash_compare_values(const void *v1, const void *v2) +{ + return( ((size_t)v1 == (size_t)v2 ) ? 1 : 0); +} + +static PLHashNumber import_subcount_hash_fn(const void *id) +{ + return (PLHashNumber) id; +} + +void import_subcount_stuff_init(import_subcount_stuff *stuff) +{ + stuff->hashtable = PL_NewHashTable(IMPORT_SUBCOUNT_HASHTABLE_SIZE, + import_subcount_hash_fn, import_subcount_hash_compare_keys, + import_subcount_hash_compare_values, NULL, NULL); +} + +void import_subcount_stuff_term(import_subcount_stuff *stuff) +{ + if ( stuff != NULL && stuff->hashtable != NULL ) { + PL_HashTableDestroy(stuff->hashtable); + } +} + +/* fetch include/exclude DNs from the pblock and normalize them -- + * returns true if there are any include/exclude DNs + * [used by both ldif2db and db2ldif] + */ +int ldbm_back_fetch_incl_excl(Slapi_PBlock *pb, char ***include, + char ***exclude) +{ + char **pb_incl, **pb_excl; + char subtreeDn[BUFSIZ]; + char *normSubtreeDn; + int i; + + slapi_pblock_get(pb, SLAPI_LDIF2DB_INCLUDE, &pb_incl); + slapi_pblock_get(pb, SLAPI_LDIF2DB_EXCLUDE, &pb_excl); + *include = *exclude = NULL; + + /* normalize */ + if (pb_excl) { + for (i = 0; pb_excl[i]; i++) { + strcpy(subtreeDn, pb_excl[i]); + normSubtreeDn = slapi_dn_normalize_case(subtreeDn); + charray_add(exclude, slapi_ch_strdup(normSubtreeDn)); + } + } + if (pb_incl) { + for (i = 0; pb_incl[i]; i++) { + strcpy(subtreeDn, pb_incl[i]); + normSubtreeDn = slapi_dn_normalize_case(subtreeDn); + charray_add(include, slapi_ch_strdup(normSubtreeDn)); + } + } + return (pb_incl || pb_excl); +} + +void ldbm_back_free_incl_excl(char **include, char **exclude) +{ + if (include) { + charray_free(include); + } + if (exclude) { + charray_free(exclude); + } +} + +/* check if a DN is in the include list but NOT the exclude list + * [used by both ldif2db and db2ldif] + */ +int ldbm_back_ok_to_dump(const char *dn, char **include, char **exclude) +{ + int i = 0; + + if (!(include || exclude)) + return(1); + + if (exclude) { + i = 0; + while (exclude[i]) { + if (slapi_dn_issuffix(dn,exclude[i])) + return(0); + i++; + } + } + + if (include) { + i = 0; + while (include[i]) { + if (slapi_dn_issuffix(dn,include[i])) + return(1); + i++; + } + /* not in include... bye. */ + return(0); + } + + return(1); +} + + +/* + * add_op_attrs - add the parentid, entryid, dncomp, + * and entrydn operational attributes to an entry. + * Also---new improved washes whiter than white version + * now removes any bogus operational attributes you're not + * allowed to specify yourself on entries. + * Currenty the list of these is: numSubordinates, hasSubordinates + */ +int add_op_attrs(Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *ep, + int *status) +{ + backend *be; + const char *pdn; + ID pid = 0; + + slapi_pblock_get(pb, SLAPI_BACKEND, &be); + + /* + * add the parentid and entryid operational attributes + */ + + if (NULL != status) { + *status = IMPORT_ADD_OP_ATTRS_OK; + } + + /* parentid */ + if ( (pdn = slapi_dn_parent( backentry_get_ndn(ep))) != NULL ) { + struct berval bv; + IDList *idl; + int err = 0; + + /* + * read the entrydn index to get the id of the parent + * If this entry's parent is not present in the index, + * we'll get a DB_NOTFOUND error here. + * In olden times, we just ignored this, but now... + * we see this as meaning that the entry is either a + * suffix entry, or its erroneous. So, we signal this to the + * caller via the status parameter. + */ + bv.bv_val = (char *)pdn; + bv.bv_len = strlen(pdn); + if ( (idl = index_read( be, "entrydn", indextype_EQUALITY, &bv, NULL, + &err )) != NULL ) { + pid = idl_firstid( idl ); + idl_free( idl ); + } else if ( 0 != err ) { + if (DB_NOTFOUND != err ) { + LDAPDebug( LDAP_DEBUG_ANY, "database error %d\n", err, 0, 0 ); + slapi_ch_free( (void**)&pdn ); + return( -1 ); + } else { + if (NULL != status) { + *status = IMPORT_ADD_OP_ATTRS_NO_PARENT; + } + } + } + slapi_ch_free( (void**)&pdn ); + } else { + if (NULL != status) { + *status = IMPORT_ADD_OP_ATTRS_NO_PARENT; + } + } + + /* Get rid of attributes you're not allowed to specify yourself */ + slapi_entry_delete_values( ep->ep_entry, hassubordinates, NULL ); + slapi_entry_delete_values( ep->ep_entry, numsubordinates, NULL ); + + /* Add the entryid, parentid and entrydn operational attributes */ + /* Note: This function is provided by the Add code */ + add_update_entry_operational_attributes(ep, pid); + + return( 0 ); +} + +/********** functions for maintaining the subordinate count **********/ + +/* Update subordinate count in a hint list, given the parent's ID */ +int import_subcount_mother_init(import_subcount_stuff *mothers, ID parent_id, + size_t count) +{ + PR_ASSERT(NULL == PL_HashTableLookup(mothers->hashtable,(void*)parent_id)); + PL_HashTableAdd(mothers->hashtable,(void*)parent_id,(void*)count); + return 0; +} + +/* Look for a subordinate count in a hint list, given the parent's ID */ +static int import_subcount_mothers_lookup(import_subcount_stuff *mothers, + ID parent_id, size_t *count) +{ + size_t stored_count = 0; + + *count = 0; + /* Lookup hash table for ID */ + stored_count = (size_t)PL_HashTableLookup(mothers->hashtable, + (void*)parent_id); + /* If present, return the count found */ + if (0 != stored_count) { + *count = stored_count; + return 0; + } + return -1; +} + +/* Update subordinate count in a hint list, given the parent's ID */ +int import_subcount_mother_count(import_subcount_stuff *mothers, ID parent_id) +{ + size_t stored_count = 0; + + /* Lookup the hash table for the target ID */ + stored_count = (size_t)PL_HashTableLookup(mothers->hashtable, + (void*)parent_id); + PR_ASSERT(0 != stored_count); + /* Increment the count */ + stored_count++; + PL_HashTableAdd(mothers->hashtable, (void*)parent_id, (void*)stored_count); + return 0; +} + +static int import_update_entry_subcount(backend *be, ID parentid, + size_t sub_count) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int ret = 0; + modify_context mc = {0}; + char value_buffer[20]; /* enough digits for 2^64 children */ + struct backentry *e = NULL; + int isreplace = 0; + + /* Get hold of the parent */ + e = id2entry(be,parentid,NULL,&ret); + if ( (NULL == e) || (0 != ret)) { + ldbm_nasty(sourcefile,5,ret); + return (0 == ret) ? -1 : ret; + } + /* Lock it (not really required since we're single-threaded here, but + * let's do it so we can reuse the modify routines) */ + cache_lock_entry( &inst->inst_cache, e ); + modify_init(&mc,e); + sprintf(value_buffer,"%lu",sub_count); + /* attr numsubordinates could already exist in the entry, + let's check whether it's already there or not */ + isreplace = (attrlist_find(e->ep_entry->e_attrs, numsubordinates) != NULL); + { + int op = isreplace ? LDAP_MOD_REPLACE : LDAP_MOD_ADD; + Slapi_Mods *smods= slapi_mods_new(); + + slapi_mods_add(smods, op | LDAP_MOD_BVALUES, numsubordinates, + strlen(value_buffer), value_buffer); + ret = modify_apply_mods(&mc,smods); /* smods passed in */ + } + if (0 == ret || LDAP_TYPE_OR_VALUE_EXISTS == ret) { + /* This will correctly index subordinatecount: */ + ret = modify_update_all(be,NULL,&mc,NULL); + if (0 == ret) { + modify_switch_entries( &mc,be); + } + } + modify_term(&mc,be); + return ret; +} + +struct _import_subcount_trawl_info { + struct _import_subcount_trawl_info *next; + ID id; + size_t sub_count; +}; +typedef struct _import_subcount_trawl_info import_subcount_trawl_info; + +static void import_subcount_trawl_add(import_subcount_trawl_info **list, ID id) +{ + import_subcount_trawl_info *new_info = CALLOC(import_subcount_trawl_info); + + new_info->next = *list; + new_info->id = id; + *list = new_info; +} + +static int import_subcount_trawl(backend *be, import_subcount_trawl_info *trawl_list) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + ID id = 1; + int ret = 0; + import_subcount_trawl_info *current = NULL; + char value_buffer[20]; /* enough digits for 2^64 children */ + + /* OK, we do */ + /* We open id2entry and iterate through it */ + /* Foreach entry, we check to see if its parentID matches any of the + * values in the trawl list . If so, we bump the sub count for that + * parent in the list. + */ + while (1) { + struct backentry *e = NULL; + + /* Get the next entry */ + e = id2entry(be,id,NULL,&ret); + if ( (NULL == e) || (0 != ret)) { + if (DB_NOTFOUND == ret) { + break; + } else { + ldbm_nasty(sourcefile,8,ret); + return ret; + } + } + for (current = trawl_list; current != NULL; current = current->next) { + sprintf(value_buffer,"%lu",(u_long)current->id); + if (slapi_entry_attr_hasvalue(e->ep_entry,"parentid",value_buffer)) { + /* If this entry's parent ID matches one we're trawling for, + * bump its count */ + current->sub_count++; + } + } + /* Free the entry */ + cache_remove(&inst->inst_cache, e); + cache_return(&inst->inst_cache, &e); + id++; + } + /* Now update the parent entries from the list */ + for (current = trawl_list; current != NULL; current = current->next) { + /* Update the parent entry with the correctly counted subcount */ + ret = import_update_entry_subcount(be,current->id,current->sub_count); + if (0 != ret) { + ldbm_nasty(sourcefile,10,ret); + break; + } + } + return ret; +} + +/* + * Function: update_subordinatecounts + * + * Returns: Nothing + * + */ +int update_subordinatecounts(backend *be, import_subcount_stuff *mothers, + DB_TXN *txn) +{ + int ret = 0; + DB *db = NULL; + DBC *dbc = NULL; + struct attrinfo *ai = NULL; + DBT key = {0}; + DBT data = {0}; + import_subcount_trawl_info *trawl_list = NULL; + + /* Open the parentid index */ + ainfo_get( be, "parentid", &ai ); + + /* Open the parentid index file */ + if ( (ret = dblayer_get_index_file( be, ai, &db, DBOPEN_CREATE )) != 0 ) { + ldbm_nasty(sourcefile,67,ret); + return(ret); + } + + /* Get a cursor so we can walk through the parentid */ + ret = db->cursor(db,txn,&dbc,0); + if (ret != 0 ) { + ldbm_nasty(sourcefile,68,ret); + dblayer_release_index_file( be, ai, db ); + return ret; + } + + /* Walk along the index */ + while (1) { + size_t sub_count = 0; + int found_count = 1; + ID parentid = 0; + + /* Foreach key which is an equality key : */ + data.flags = DB_DBT_MALLOC; + key.flags = DB_DBT_MALLOC; + ret = dbc->c_get(dbc,&key,&data,DB_NEXT_NODUP); + if (NULL != data.data) { + free(data.data); + data.data = NULL; + } + if (0 != ret) { + if (ret != DB_NOTFOUND) { + ldbm_nasty(sourcefile,62,ret); + } + if (NULL != key.data) { + free(key.data); + key.data = NULL; + } + break; + } + if (*(char*)key.data == EQ_PREFIX) { + char *idptr = NULL; + + /* construct the parent's ID from the key */ + /* Look for the ID in the hint list supplied by the caller */ + /* If its there, we know the answer already */ + idptr = (((char *) key.data) + 1); + parentid = (ID) atol(idptr); + PR_ASSERT(0 != parentid); + ret = import_subcount_mothers_lookup(mothers,parentid,&sub_count); + if (0 != ret) { + IDList *idl = NULL; + + /* If it's not, we need to compute it ourselves: */ + /* Load the IDL matching the key */ + key.flags = DB_DBT_REALLOC; + ret = NEW_IDL_NO_ALLID; + idl = idl_fetch(be,db,&key,NULL,NULL,&ret); + if ( (NULL == idl) || (0 != ret)) { + ldbm_nasty(sourcefile,4,ret); + dblayer_release_index_file( be, ai, db ); + return (0 == ret) ? -1 : ret; + } + /* The number of IDs in the IDL tells us the number of + * subordinates for the entry */ + /* Except, the number might be above the allidsthreshold, + * in which case */ + if (ALLIDS(idl)) { + /* We add this ID to the list for which to trawl */ + import_subcount_trawl_add(&trawl_list,parentid); + found_count = 0; + } else { + /* We get the count from the IDL */ + sub_count = idl->b_nids; + } + idl_free(idl); + } + /* Did we get the count ? */ + if (found_count) { + PR_ASSERT(0 != sub_count); + /* If so, update the parent now */ + import_update_entry_subcount(be,parentid,sub_count); + } + } + if (NULL != key.data) { + free(key.data); + key.data = NULL; + } + } + + ret = dbc->c_close(dbc); + if (0 != ret) { + ldbm_nasty(sourcefile,6,ret); + } + dblayer_release_index_file( be, ai, db ); + + /* Now see if we need to go trawling through id2entry for the info + * we need */ + if (NULL != trawl_list) { + ret = import_subcount_trawl(be,trawl_list); + if (0 != ret) { + ldbm_nasty(sourcefile,7,ret); + } + } + return(ret); +} + + +/********** ldif2db entry point **********/ + +/* + Some notes about this stuff: + + The front-end does call our init routine before calling us here. + So, we get the regular chance to parse the config file etc. + However, it does _NOT_ call our start routine, so we need to + do whatever work that did and which we need for this work , here. + Furthermore, the front-end simply exits after calling us, so we need + to do any cleanup work here also. + */ + +/* + * ldbm_back_ldif2ldbm - backend routine to convert an ldif file to + * a database. + */ +int ldbm_back_ldif2ldbm( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + ldbm_instance *inst = NULL; + char *instance_name; + int ret, task_flags; + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name ); + + /* BEGIN complex dependencies of various initializations. */ + /* hopefully this will go away once import is not run standalone... */ + + slapi_pblock_get(pb, SLAPI_TASK_FLAGS, &task_flags); + if (task_flags & TASK_RUNNING_FROM_COMMANDLINE) { + li->li_flags |= TASK_RUNNING_FROM_COMMANDLINE; + ldbm_config_load_dse_info(li); + autosize_import_cache(li); + } + + /* Find the instance that the ldif2db will be done on. */ + inst = ldbm_instance_find_by_name(li, instance_name); + if (NULL == inst) { + LDAPDebug(LDAP_DEBUG_ANY, "Unknown ldbm instance %s\n", instance_name, + 0, 0); + return -1; + } + + /* check if an import/restore is already ongoing... */ + if (instance_set_busy(inst) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + inst->inst_name, 0, 0); + return -1; + } + + /***** prepare & init libdb and dblayer *****/ + + if (! (task_flags & TASK_RUNNING_FROM_COMMANDLINE)) { + /* shutdown this instance of the db */ + LDAPDebug(LDAP_DEBUG_ANY, "Bringing %s offline...\n", + instance_name, 0, 0); + slapi_mtn_be_disable(inst->inst_be); + + cache_clear(&inst->inst_cache); + dblayer_instance_close(inst->inst_be); + dblayer_delete_indices(inst); + } else { + /* from the command line, libdb needs to be started up */ + ldbm_config_internal_set(li, CONFIG_DB_TRANSACTION_LOGGING, "off"); + + if (0 != (ret = dblayer_start(li, DBLAYER_IMPORT_MODE)) ) { + if (LDBM_OS_ERR_IS_DISKFULL(ret)) { + LDAPDebug(LDAP_DEBUG_ANY, "ERROR: Failed to init database. " + "There is either insufficient disk space or " + "insufficient memory available to initialize the " + "database.\n", 0, 0, 0); + LDAPDebug(LDAP_DEBUG_ANY,"Please check that\n" + "1) disks are not full,\n" + "2) no file exceeds the file size limit,\n" + "3) the configured dbcachesize is not too large for the available memory on this machine.\n", + 0, 0, 0); + } else { + LDAPDebug(LDAP_DEBUG_ANY, "ERROR: Failed to init database " + "(error %d: %s)\n", ret, dblayer_strerror(ret), 0); + } + goto fail; + } + } + + /* Delete old database files */ + dblayer_delete_instance_dir(inst->inst_be); + /* it's okay to fail -- the directory might have already been deleted */ + + /* dblayer_instance_start will init the id2entry index. */ + /* it also (finally) fills in inst_dir_name */ + ret = dblayer_instance_start(inst->inst_be, DBLAYER_IMPORT_MODE); + if (ret != 0) { + goto fail; + } + + vlv_init(inst); + + /***** done init libdb and dblayer *****/ + + /* always use "new" import code now */ + slapi_pblock_set(pb, SLAPI_BACKEND, inst->inst_be); + return ldbm_back_ldif2ldbm_deluxe(pb); + +fail: + /* DON'T enable the backend -- leave it offline */ + instance_set_not_busy(inst); + return ret; +} + + +/********** db2ldif, db2index **********/ + + +/* fetch an IDL for the series of subtree specs */ +/* (used for db2ldif) */ +static IDList *ldbm_fetch_subtrees(backend *be, char **include, int *err) +{ + int i; + ID id; + IDList *idltotal = NULL, *idltmp; + back_txn *txn = NULL; + struct berval bv; + + /* for each subtree spec... */ + for (i = 0; include[i]; i++) { + IDList *idl = NULL; + + /* + * First map the suffix to its entry ID. + * Note that the suffix is already normalized. + */ + bv.bv_val = include[i]; + bv.bv_len = strlen(include[i]); + idl = index_read(be, "entrydn", indextype_EQUALITY, &bv, txn, err); + if (idl == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "warning: entrydn not indexed on '%s'\n", + include[i], 0, 0); + continue; + } + id = idl_firstid(idl); + idl_free(idl); + idl = NULL; + + /* + * Now get all the descendants of that suffix. + */ + *err = ldbm_ancestorid_read(be, txn, id, &idl); + if (idl == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "warning: ancestorid not indexed on %lu\n", + id, 0, 0); + continue; + } + + /* Insert the suffix itself */ + idl_insert(&idl, id); + + /* Merge the idlists */ + if (! idltotal) { + idltotal = idl; + } else if (idl) { + idltmp = idl_union(be, idltotal, idl); + idl_free(idltotal); + idl_free(idl); + idltotal = idltmp; + } + } + + return idltotal; +} + +#define FD_STDOUT 1 + + +/* + * ldbm_back_ldbm2ldif - backend routine to convert database to an + * ldif file. + * (reunified at last) + */ +int +ldbm_back_ldbm2ldif( Slapi_PBlock *pb ) +{ + backend *be; + struct ldbminfo *li = NULL; + DB *db = NULL; + DBC *dbc = NULL; + struct backentry *ep; + DBT key = {0}; + DBT data = {0}; + char *type, *fname = NULL; + int len, printkey, rc, ok_index; + int return_value = 0; + int nowrap = 0; + int nobase64 = 0; + NIDS idindex = 0; + ID temp_id; + char **exclude_suffix = NULL; + char **include_suffix = NULL; + int decrypt = 0; + int dump_replica = 0; + int dump_uniqueid = 1; + int fd; + IDList *idl = NULL; /* optimization for -s include lists */ + int cnt = 0, lastcnt = 0; + int options = 0; + int keepgoing = 1; + int isfirst = 1; + int appendmode = 0; + int appendmode_1 = 0; + int noversion = 0; + ID lastid; + int task_flags; + Slapi_Task *task; + int run_from_cmdline = 0; + char *instance_name; + ldbm_instance *inst; + int str2entry_options= 0; + int retry; + int we_start_the_backends = 0; + int server_running; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> ldbm_back_ldbm2ldif\n", 0, 0, 0 ); + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_TASK_FLAGS, &task_flags ); + slapi_pblock_get( pb, SLAPI_DB2LDIF_DECRYPT, &decrypt ); + slapi_pblock_get( pb, SLAPI_DB2LDIF_SERVER_RUNNING, &server_running ); + run_from_cmdline = (task_flags & TASK_RUNNING_FROM_COMMANDLINE); + + dump_replica = pb->pb_ldif_dump_replica; + if (run_from_cmdline) { + li->li_flags |= TASK_RUNNING_FROM_COMMANDLINE; + if (!dump_replica) { + we_start_the_backends = 1; + } + } + + if (we_start_the_backends) { + /* No ldbm be's exist until we process the config information. */ + + /* + * Note that we should only call this once. If we're + * dumping several backends then it gets called multiple + * times and we get warnings in the error log like this: + * WARNING: ldbm instance NetscapeRoot already exists + */ + ldbm_config_load_dse_info(li); + } + + if (run_from_cmdline && li->li_dblayer_private->dblayer_private_mem + && server_running) + { + LDAPDebug(LDAP_DEBUG_ANY, + "Cannot export the database while the server is running and " + "nsslapd-db-private-mem option is used, " + "please use ldif2db.pl\n", 0, 0, 0); + return_value = -1; + goto bye; + } + + if (run_from_cmdline) { + + /* Now that we have processed the config information, we look for + * the be that should do the db2ldif. */ + slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name); + inst = ldbm_instance_find_by_name(li, instance_name); + if (NULL == inst) { + LDAPDebug(LDAP_DEBUG_ANY, "Unknown ldbm instance %s\n", + instance_name, 0, 0); + return_value = -1; + goto bye; + } + /* [605974] command db2ldif should not be able to run when on-line + * import is running */ + if (dblayer_in_import(inst)) { + LDAPDebug(LDAP_DEBUG_ANY, "instance %s is busy\n", + instance_name, 0, 0); + return_value = -1; + goto bye; + } + + /* store the be in the pb */ + be = inst->inst_be; + slapi_pblock_set(pb, SLAPI_BACKEND, be); + } else { + slapi_pblock_get(pb, SLAPI_BACKEND, &be); + inst = (ldbm_instance *)be->be_instance_info; + + if (NULL == inst) { + LDAPDebug(LDAP_DEBUG_ANY, "Unknown ldbm instance\n", 0, 0, 0); + return_value = -1; + goto bye; + } + + /* check if an import/restore is already ongoing... */ + if (instance_set_busy(inst) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: '%s' is already in the middle" + " of another task and cannot be disturbed.\n", + inst->inst_name, 0, 0); + return_value = -1; + goto bye; + } + } + + slapi_pblock_get( pb, SLAPI_BACKEND_TASK, &task ); + + ldbm_back_fetch_incl_excl(pb, &include_suffix, &exclude_suffix); + + str2entry_options= (dump_replica?0:SLAPI_STR2ENTRY_TOMBSTONE_CHECK); + + slapi_pblock_get( pb, SLAPI_DB2LDIF_FILE, &fname ); + slapi_pblock_get( pb, SLAPI_DB2LDIF_PRINTKEY, &printkey ); + slapi_pblock_get( pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &dump_uniqueid ); + + /* tsk, overloading printkey. shame on me. */ + ok_index = !(printkey & EXPORT_ID2ENTRY_ONLY); + printkey &= ~EXPORT_ID2ENTRY_ONLY; + + nobase64 = (printkey & EXPORT_MINIMAL_ENCODING); + printkey &= ~EXPORT_MINIMAL_ENCODING; + nowrap = (printkey & EXPORT_NOWRAP); + printkey &= ~EXPORT_NOWRAP; + appendmode = (printkey & EXPORT_APPENDMODE); + printkey &= ~EXPORT_APPENDMODE; + appendmode_1 = (printkey & EXPORT_APPENDMODE_1); + printkey &= ~EXPORT_APPENDMODE_1; + noversion = (printkey & EXPORT_NOVERSION); + printkey &= ~EXPORT_NOVERSION; + + /* decide whether to dump uniqueid */ + if (dump_uniqueid) + options |= SLAPI_DUMP_UNIQUEID; + if (nowrap) + options |= SLAPI_DUMP_NOWRAP; + if (nobase64) + options |= SLAPI_DUMP_MINIMAL_ENCODING; + if (dump_replica) + options |= SLAPI_DUMP_STATEINFO; + + if (fname == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "db2ldif: no LDIF filename supplied\n", + 0, 0, 0); + return_value = -1; + goto bye; + } + + if (strcmp(fname, "-")) { /* not '-' */ + if (appendmode) { + if (appendmode_1) { + fd = dblayer_open_huge_file(fname, O_WRONLY|O_CREAT|O_TRUNC, + SLAPD_DEFAULT_FILE_MODE); + } else { + fd = dblayer_open_huge_file(fname, O_WRONLY|O_CREAT|O_APPEND, + SLAPD_DEFAULT_FILE_MODE); + } + } else { + /* open it */ + fd = dblayer_open_huge_file(fname, O_WRONLY|O_CREAT|O_TRUNC, + SLAPD_DEFAULT_FILE_MODE); + } + if (fd < 0) { + LDAPDebug(LDAP_DEBUG_ANY, "db2ldif: can't open %s: %d (%s)\n", + fname, errno, dblayer_strerror(errno)); + return_value = -1; + goto bye; + } + } else { /* '-' */ + fd = FD_STDOUT; + } + + if ( we_start_the_backends ) { + if (0 != dblayer_start(li,DBLAYER_EXPORT_MODE)) { + LDAPDebug( LDAP_DEBUG_ANY, "db2ldif: Failed to init database\n", + 0, 0, 0 ); + return_value = -1; + goto bye; + } + /* dblayer_instance_start will init the id2entry index. */ + if (0 != dblayer_instance_start(be, DBLAYER_EXPORT_MODE)) { + LDAPDebug(LDAP_DEBUG_ANY, "db2ldif: Failed to init instance\n", + 0, 0, 0); + return_value = -1; + goto bye; + } + } + + /* idl manipulation requires nextid to be init'd now */ + if (include_suffix && ok_index) + get_ids_from_disk(be); + + if ((( dblayer_get_id2entry( be, &db )) != 0) || (db == NULL)) { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open/create id2entry\n", + 0, 0, 0 ); + ldbm_back_free_incl_excl(include_suffix, exclude_suffix); + return_value = -1; + goto bye; + } + + /* if an include_suffix was given (and we're pretty sure the + * entrydn and ancestorid indexes are valid), we try to + * assemble an id-list of candidates instead of plowing thru + * the whole database. this is a big performance improvement + * when exporting config info (which is usually on the order + * of 100 entries) from a database that may be on the order of + * GIGS in size. + */ + { + /* Here, we assume that the table is ordered in EID-order, + * which it is ! + */ + /* get a cursor to we can walk over the table */ + return_value = db->cursor(db,NULL,&dbc,0); + if (0 != return_value ) { + LDAPDebug( LDAP_DEBUG_ANY, + "Failed to get cursor for db2ldif\n", + 0, 0, 0 ); + ldbm_back_free_incl_excl(include_suffix, exclude_suffix); + return_value = -1; + goto bye; + } + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + return_value = dbc->c_get(dbc,&key,&data,DB_LAST); + if (0 != return_value) { + keepgoing = 0; + } else { + lastid = id_stored_to_internal((char *)key.data); + free( key.data ); + free( data.data ); + isfirst = 1; + } + } + if (include_suffix && ok_index && !dump_replica) { + int err; + + idl = ldbm_fetch_subtrees(be, include_suffix, &err); + if (! idl) { + /* most likely, indexes are bad. */ + LDAPDebug(LDAP_DEBUG_ANY, + "Failed to fetch subtree lists (error %d) %s\n", + err, dblayer_strerror(err), 0); + LDAPDebug(LDAP_DEBUG_ANY, + "Possibly the entrydn or ancestorid index is corrupted or " + "does not exist.\n", 0, 0, 0); + LDAPDebug(LDAP_DEBUG_ANY, + "Attempting direct unindexed export instead.\n", + 0, 0, 0); + ok_index = 0; + idl = NULL; + } else if (ALLIDS(idl)) { + /* allids list is no help at all -- revert to trawling + * the whole list. */ + ok_index = 0; + idl_free(idl); + idl = NULL; + } + idindex = 0; + } + + /* When user has specifically asked not to print the version + * or when this is not the first backend that is append into + * this file : don't print the version + */ + if ((!noversion) && ((!appendmode) || (appendmode_1))) { + char vstr[64]; + int myversion = 1; /* XXX: ldif version; + * needs to be modified when version + * control begins. + */ + + sprintf(vstr, "version: %d\n\n", myversion); + write(fd, vstr, strlen(vstr)); + } + + while ( keepgoing ) { + Slapi_Attr *this_attr, *next_attr; + + /* + * All database operations in a transactional environment, + * including non-transactional reads can receive a return of + * DB_LOCK_DEADLOCK. Which operation gets aborted depends + * on the deadlock detection policy, but can include + * non-transactional reads (in which case the single + * operation should just be retried). + */ + + if (idl) { + /* exporting from an ID list */ + if (idindex >= idl->b_nids) + break; + id_internal_to_stored(idl->b_ids[idindex], (char *)&temp_id); + key.data = (char *)&temp_id; + key.size = sizeof(temp_id); + data.flags = DB_DBT_MALLOC; + + for (retry = 0; retry < RETRY_TIMES; retry++) { + return_value = db->get(db, NULL, &key, &data, 0); + if (return_value != DB_LOCK_DEADLOCK) break; + } + if (return_value) { + LDAPDebug(LDAP_DEBUG_ANY, "db2ldif: failed to read " + "entry %lu, err %d\n", (u_long)idl->b_ids[idindex], + return_value, 0); + return_value = -1; + break; + } + /* back to internal format: */ + temp_id = idl->b_ids[idindex]; + idindex++; + } else { + /* follow the cursor */ + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + if (isfirst) { + for (retry = 0; retry < RETRY_TIMES; retry++) { + return_value = dbc->c_get(dbc,&key,&data,DB_FIRST); + if (return_value != DB_LOCK_DEADLOCK) break; + } + isfirst = 0; + } else { + for (retry = 0; retry < RETRY_TIMES; retry++) { + return_value = dbc->c_get(dbc,&key,&data,DB_NEXT); + if (return_value != DB_LOCK_DEADLOCK) break; + } + } + + if (0 != return_value) + break; + + /* back to internal format */ + temp_id = id_stored_to_internal((char *)key.data); + free(key.data); + } + + /* call post-entry plugin */ + plugin_call_entryfetch_plugins( (char **) &data.dptr, &data.dsize ); + + ep = backentry_alloc(); + ep->ep_entry = slapi_str2entry( data.data, str2entry_options ); + free(data.data); + + if ( (ep->ep_entry) != NULL ) { + ep->ep_id = temp_id; + cnt++; + } else { + LDAPDebug( LDAP_DEBUG_ANY, + "skipping badly formatted entry with id %lu\n", + (u_long)temp_id, 0, 0 ); + backentry_free( &ep ); + continue; + } + if (!ldbm_back_ok_to_dump(backentry_get_ndn(ep), include_suffix, + exclude_suffix)) { + backentry_free( &ep ); + continue; + } + if(!dump_replica && slapi_entry_flag_is_set(ep->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE)) + { + /* We only dump the tombstones if the user needs to create a replica from the ldif */ + backentry_free( &ep ); + continue; + } + + + /* do not output attributes that are in the "exclude" list */ + /* Also, decrypt any encrypted attributes, if we're asked to */ + rc = slapi_entry_first_attr( ep->ep_entry, &this_attr ); + while (0 == rc) { + rc = slapi_entry_next_attr( ep->ep_entry, + this_attr, &next_attr ); + slapi_attr_get_type( this_attr, &type ); + if ( ldbm_exclude_attr_from_export( li, type, dump_uniqueid )) { + slapi_entry_delete_values( ep->ep_entry, type, NULL ); + } + this_attr = next_attr; + } + if (decrypt) { + /* Decrypt in place */ + rc = attrcrypt_decrypt_entry(be, ep); + if (rc) { + LDAPDebug(LDAP_DEBUG_ANY,"Failed to decrypt entry%s\n", ep->ep_entry->e_sdn , 0, 0); + } + } + + data.data = slapi_entry2str_with_options( ep->ep_entry, &len, options ); + data.size = len + 1; + + if ( printkey & EXPORT_PRINTKEY ) { + char idstr[32]; + + sprintf(idstr, "# entry-id: %lu\n", (u_long)ep->ep_id); + write(fd, idstr, strlen(idstr)); + } + write(fd, data.data, len); + write(fd, "\n", 1); + if (cnt % 1000 == 0) { + int percent; + + if (idl) { + percent = (idindex*100 / idl->b_nids); + } else { + percent = (ep->ep_id*100 / lastid); + } + if (task != NULL) { + slapi_task_log_status(task, + "%s: Processed %d entries (%d%%).", + inst->inst_name, cnt, percent); + slapi_task_log_notice(task, + "%s: Processed %d entries (%d%%).", + inst->inst_name, cnt, percent); + } + LDAPDebug(LDAP_DEBUG_ANY, + "export %s: Processed %d entries (%d%%).\n", + inst->inst_name, cnt, percent); + lastcnt = cnt; + } + + backentry_free( &ep ); + free( data.data ); + } + /* DB_NOTFOUND -> successful end */ + if (return_value == DB_NOTFOUND) + return_value = 0; + + /* done cycling thru entries to write */ + if (lastcnt != cnt) { + if (task) { + slapi_task_log_status(task, + "%s: Processed %d entries (100%%).", + inst->inst_name, cnt); + slapi_task_log_notice(task, + "%s: Processed %d entries (100%%).", + inst->inst_name, cnt); + } + LDAPDebug(LDAP_DEBUG_ANY, + "export %s: Processed %d entries (100%%).\n", + inst->inst_name, cnt, 0); + } + + if (idl) { + idl_free(idl); + } + if (dbc) { + dbc->c_close(dbc); + } + + dblayer_release_id2entry( be, db ); + ldbm_back_free_incl_excl(include_suffix, exclude_suffix); + + if (fd != FD_STDOUT) { + close(fd); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= ldbm_back_ldbm2ldif\n", 0, 0, 0 ); + + if (we_start_the_backends && 0 != dblayer_flush(li)) { + LDAPDebug( LDAP_DEBUG_ANY, "db2ldif: Failed to flush database\n", + 0, 0, 0 ); + } + + if (we_start_the_backends) { + if (0 != dblayer_close(li,DBLAYER_EXPORT_MODE)) { + LDAPDebug( LDAP_DEBUG_ANY, + "db2ldif: Failed to close database\n", + 0, 0, 0 ); + } + } else if (run_from_cmdline && dump_replica) { + /* + * It should not be necessary to close the dblayer here. + * However it masks complex thread timing issues that + * prevent a correct shutdown of the plugins. Closing the + * dblayer here means we cannot dump multiple replicas + * using -r, but the server doesn't allow that either. + */ + + /* + * Use DBLAYER_NORMAL_MODE to match the value that was provided + * to dblayer_start() and ensure creation of the guardian file. + */ + if (0 != dblayer_close(li,DBLAYER_NORMAL_MODE)) { + LDAPDebug( LDAP_DEBUG_ANY, + "db2ldif: Failed to close database\n", + 0, 0, 0 ); + } + } + + if (!run_from_cmdline) { + instance_set_not_busy(inst); + } + +bye: + if (inst != NULL) { + PR_Lock(inst->inst_config_mutex); + inst->inst_flags &= ~INST_FLAG_BUSY; + PR_Unlock(inst->inst_config_mutex); + } + + return( return_value ); +} + + +static void ldbm2index_bad_vlv(Slapi_Task *task, ldbm_instance *inst, + char *index) +{ + char *text = vlv_getindexnames(inst->inst_be); + + if (task) { + slapi_task_log_status(task, "%s: Unknown VLV index '%s'", + inst->inst_name, index); + slapi_task_log_notice(task, "%s: Unknown VLV index '%s'", + inst->inst_name, index); + slapi_task_log_notice(task, "%s: Known VLV indexes are: %s", + inst->inst_name, text); + } + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm2index: Unknown VLV Index named '%s'\n", index, 0, 0); + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm2index: Known VLV Indexes are: %s\n", text, 0, 0); + slapi_ch_free((void**)&text); +} + +/* + * ldbm_back_ldbm2index - backend routine to create a new index from an + * existing database + */ +int +ldbm_back_ldbm2index(Slapi_PBlock *pb) +{ + char *instance_name; + struct ldbminfo *li; + int task_flags, run_from_cmdline; + ldbm_instance *inst; + backend *be; + DB *db = NULL; + DBC *dbc = NULL; + char **indexAttrs = NULL; + struct vlvIndex **pvlv= NULL; + DBT key = {0}; + DBT data = {0}; + IDList *idl = NULL; /* optimization for vlv index creation */ + int numvlv = 0; + int return_value = -1; + ID temp_id; + int i, j; + ID lastid; + struct backentry *ep; + char *type; + NIDS idindex = 0; + int count = 0; + Slapi_Attr *attr; + Slapi_Task *task; + int ret = 0; + int isfirst = 1; + int index_aid = 0; /* index ancestorid */ + + LDAPDebug( LDAP_DEBUG_TRACE, "=> ldbm_back_ldbm2index\n", 0, 0, 0 ); + + slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name); + slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &li); + slapi_pblock_get(pb, SLAPI_TASK_FLAGS, &task_flags); + run_from_cmdline = (task_flags & TASK_RUNNING_FROM_COMMANDLINE); + slapi_pblock_get(pb, SLAPI_BACKEND_TASK, &task); + + if (run_from_cmdline) { + /* No ldbm backend exists until we process the config info. */ + li->li_flags |= TASK_RUNNING_FROM_COMMANDLINE; + ldbm_config_load_dse_info(li); + } + + inst = ldbm_instance_find_by_name(li, instance_name); + if (NULL == inst) { + if (task) { + slapi_task_log_notice(task, "Unknown ldbm instance %s", + instance_name, 0, 0); + } + LDAPDebug(LDAP_DEBUG_ANY, "Unknown ldbm instance %s\n", + instance_name, 0, 0); + return -1; + } + be = inst->inst_be; + slapi_pblock_set(pb, SLAPI_BACKEND, be); + + /* would love to be able to turn off transactions here, but i don't + * think it's in the cards... + */ + if (run_from_cmdline) { + /* Turn off transactions */ + ldbm_config_internal_set(li, CONFIG_DB_TRANSACTION_LOGGING, "off"); + + if (0 != dblayer_start(li,DBLAYER_INDEX_MODE)) { + LDAPDebug( LDAP_DEBUG_ANY, + "ldbm2index: Failed to init database\n", 0, 0, 0 ); + return( -1 ); + } + + /* dblayer_instance_start will init the id2entry index. */ + if (0 != dblayer_instance_start(be, DBLAYER_INDEX_MODE)) { + LDAPDebug(LDAP_DEBUG_ANY, "db2ldif: Failed to init instance\n", + 0, 0, 0); + return -1; + } + + /* Initialise the Virtual List View code */ + vlv_init(inst); + } + + /* make sure no other tasks are going, and set the backend readonly */ + if (instance_set_busy_and_readonly(inst) != 0) { + LDAPDebug(LDAP_DEBUG_ANY, "ldbm: '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + inst->inst_name, 0, 0); + return -1; + } + + if ((( dblayer_get_id2entry( be, &db )) != 0 ) || (db == NULL)) { + LDAPDebug( LDAP_DEBUG_ANY, "Could not open/create id2entry\n", + 0, 0, 0 ); + instance_set_not_busy(inst); + return( -1 ); + } + + /* get a cursor to we can walk over the table */ + return_value = db->cursor(db, NULL, &dbc, 0); + if (0 != return_value ) { + LDAPDebug( LDAP_DEBUG_ANY, + "Failed to get cursor for ldbm2index\n", 0, 0, 0 ); + dblayer_release_id2entry(be, db); + instance_set_not_busy(inst); + return( -1 ); + } + + /* ask for the last id so we can give cute percentages */ + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + return_value = dbc->c_get(dbc, &key, &data, DB_LAST); + if (return_value == DB_NOTFOUND) { + lastid = 0; + isfirst = 0; /* neither a first nor a last */ + } else if (return_value == 0) { + lastid = id_stored_to_internal((char *)key.data); + free(key.data); + free(data.data); + isfirst = 1; + } else { + LDAPDebug(LDAP_DEBUG_ANY, + "Failed to seek within id2entry (BAD %d)\n", + return_value, 0 ,0); + dbc->c_close(dbc); + dblayer_release_id2entry(be, db); + instance_set_not_busy(inst); + return( -1 ); + } + + /* Work out which indexes we should build */ + /* explanation: for archaic reasons, the list of indexes is passed to + * ldif2index as a string list, where each string either starts with a + * 't' (normal index) or a 'T' (vlv index). + * example: "tcn" (normal index cn) + */ + { + char **attrs = NULL; + struct vlvIndex *p = NULL; + struct attrinfo *ai = NULL; + + slapi_pblock_get(pb, SLAPI_DB2INDEX_ATTRS, &attrs); + for (i = 0; attrs[i] != NULL; i++) { + switch(attrs[i][0]) { + case 't': /* attribute type to index */ + db2index_add_indexed_attr(be, attrs[i]); + ainfo_get(be, attrs[i]+1, &ai); + /* the ai was added above, if it didn't already exist */ + PR_ASSERT(ai != NULL); + if (strcasecmp(attrs[i]+1, "ancestorid") == 0) { + if (task) { + slapi_task_log_notice(task, "%s: Indexing ancestorid", + inst->inst_name); + } + LDAPDebug(LDAP_DEBUG_ANY, "%s: Indexing ancestorid\n", + inst->inst_name, 0, 0); + index_aid = 1; + } else { + charray_add(&indexAttrs, attrs[i]+1); + ai->ai_indexmask |= INDEX_OFFLINE; + if (task) { + slapi_task_log_notice(task, "%s: Indexing attribute: %s", + inst->inst_name, attrs[i]+1); + } + LDAPDebug(LDAP_DEBUG_ANY, "%s: Indexing attribute: %s\n", + inst->inst_name, attrs[i]+1, 0); + } + dblayer_erase_index_file(be, ai, i/* chkpt; 1st time only */); + break; + case 'T': /* VLV Search to index */ + p = vlv_find_searchname((attrs[i])+1, be); + if (p == NULL) { + ldbm2index_bad_vlv(task, inst, attrs[i]+1); + ret = -1; + goto out; + } else { + vlvIndex_go_offline(p, be); + if (pvlv == NULL) { + pvlv = (struct vlvIndex **)slapi_ch_calloc(1, + sizeof(struct vlvIndex *)); + } else { + pvlv = (struct vlvIndex **)slapi_ch_realloc((char*)pvlv, + (numvlv+1)*sizeof(struct vlvIndex *)); + } + pvlv[numvlv] = p; + numvlv++; + /* Get rid of the index if it already exists */ + PR_Delete(vlvIndex_filename(p)); + if (task) { + slapi_task_log_notice(task, "%s: Indexing VLV: %s", + inst->inst_name, attrs[i]+1); + } + LDAPDebug(LDAP_DEBUG_ANY, "%s: Indexing VLV: %s\n", + inst->inst_name, attrs[i]+1, 0); + } + break; + } + } + } + + /* if we're only doing vlv indexes, we can accomplish this with an + * idl composed from the ancestorid list, instead of traversing the + * entire database. + */ + if (!indexAttrs && !index_aid && pvlv) { + int i, err; + char **suffix_list = NULL; + + /* create suffix list */ + for (i = 0; i < numvlv; i++) { + char *s = slapi_ch_strdup(slapi_sdn_get_dn(vlvIndex_getBase(pvlv[i]))); + + s = slapi_dn_normalize_case(s); + charray_add(&suffix_list, s); + } + idl = ldbm_fetch_subtrees(be, suffix_list, &err); + charray_free(suffix_list); + if (! idl) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: WARNING: Failed to fetch subtree lists: (%d) %s\n", + inst->inst_name, err, dblayer_strerror(err)); + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Possibly the entrydn or ancestorid index is " + "corrupted or does not exist.\n", inst->inst_name, 0, 0); + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Attempting brute-force method instead.\n", + inst->inst_name, 0, 0); + if (task) { + slapi_task_log_notice(task, + "%s: WARNING: Failed to fetch subtree lists (err %d) -- " + "attempting brute-force method instead.", + inst->inst_name, err); + } + } else if (ALLIDS(idl)) { + /* that's no help. */ + idl_free(idl); + idl = NULL; + } + } + + if (idl) { + /* don't need that cursor, we have a shopping list. */ + dbc->c_close(dbc); + idindex = 0; + } + + /* Bug 603120: slapd dumps core while indexing and deleting the db at the + * same time. Now added the lock for the indexing code too. + */ + vlv_acquire_lock(be); + while (1) { + if (idl) { + if (idindex >= idl->b_nids) + break; + id_internal_to_stored(idl->b_ids[idindex], (char *)&temp_id); + key.data = (char *)&temp_id; + key.size = sizeof(temp_id); + data.flags = DB_DBT_MALLOC; + + return_value = db->get(db, NULL, &key, &data, 0); + if (return_value) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed " + "to read database, errno=%d (%s)\n", + inst->inst_name, return_value, + dblayer_strerror(return_value)); + if (task) { + slapi_task_log_notice(task, + "%s: Failed to read database, err %d (%s)", + inst->inst_name, return_value, + dblayer_strerror(return_value)); + } + break; + } + /* back to internal format: */ + temp_id = idl->b_ids[idindex]; + idindex++; + } else { + key.flags = DB_DBT_MALLOC; + data.flags = DB_DBT_MALLOC; + if (isfirst) { + return_value = dbc->c_get(dbc, &key, &data, DB_FIRST); + isfirst = 0; + } else{ + return_value = dbc->c_get(dbc, &key, &data, DB_NEXT); + } + + if (0 != return_value) { + if (DB_NOTFOUND == return_value) { + break; + } else { + LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed to read database, " + "errno=%d (%s)\n", inst->inst_name, return_value, + dblayer_strerror(return_value)); + if (task) { + slapi_task_log_notice(task, + "%s: Failed to read database, err %d (%s)", + inst->inst_name, return_value, + dblayer_strerror(return_value)); + } + break; + } + } + temp_id = id_stored_to_internal((char *)key.data); + free(key.data); + } + + /* call post-entry plugin */ + plugin_call_entryfetch_plugins( (char **) &data.dptr, &data.dsize ); + + ep = backentry_alloc(); + ep->ep_entry = slapi_str2entry( data.data, 0 ); + free(data.data); + + if ( ep->ep_entry != NULL ) { + ep->ep_id = temp_id; + } else { + if (task) { + slapi_task_log_notice(task, + "%s: WARNING: skipping badly formatted entry (id %lu)", + inst->inst_name, (u_long)temp_id); + } + LDAPDebug(LDAP_DEBUG_ANY, + "%s: WARNING: skipping badly formatted entry (id %lu)\n", + inst->inst_name, (u_long)temp_id, 0); + backentry_free( &ep ); + continue; + } + + if ( add_op_attrs( pb, li, ep, NULL ) != 0 ) { + if (task) { + slapi_task_log_notice(task, + "%s: ERROR: Could not add op attrs to entry (id %lu)", + inst->inst_name, (u_long)ep->ep_id); + } + LDAPDebug(LDAP_DEBUG_ANY, + "%s: ERROR: Could not add op attrs to entry (id %lu)\n", + inst->inst_name, (u_long)ep->ep_id, 0); + backentry_free( &ep ); + ret = -1; + goto out; + } + + /* + * Update the attribute indexes + */ + if (indexAttrs != NULL) { + for (i = slapi_entry_first_attr(ep->ep_entry, &attr); i == 0; + i = slapi_entry_next_attr(ep->ep_entry, attr, &attr)) { + Slapi_Value **svals; + int rc = 0; + + slapi_attr_get_type( attr, &type ); + for ( j = 0; indexAttrs[j] != NULL; j++ ) { + if (slapi_attr_type_cmp(indexAttrs[j], type, + SLAPI_TYPE_CMP_SUBTYPE) == 0 ) { + back_txn txn; + svals = attr_get_present_values(attr); + + if (run_from_cmdline) + { + txn.back_txn_txn = NULL; + } + else + { + rc = dblayer_txn_begin(li, NULL, &txn); + if (0 != rc) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: ERROR: failed to begin txn for update " + "index '%s'\n", + inst->inst_name, indexAttrs[j], 0); + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Error %d: %s\n", inst->inst_name, rc, + dblayer_strerror(rc)); + if (task) { + slapi_task_log_notice(task, + "%s: ERROR: failed to begin txn for " + "update index '%s' (err %d: %s)", + inst->inst_name, indexAttrs[j], rc, + dblayer_strerror(rc)); + } + ret = -2; + goto out; + } + } + rc = index_addordel_values_sv( + be, indexAttrs[j], svals, + NULL, ep->ep_id, BE_INDEX_ADD, &txn); + if (rc != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: ERROR: failed to update index '%s'\n", + inst->inst_name, indexAttrs[j], 0); + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Error %d: %s\n", inst->inst_name, rc, + dblayer_strerror(rc)); + if (task) { + slapi_task_log_notice(task, + "%s: ERROR: failed to update index '%s' " + "(err %d: %s)", inst->inst_name, + indexAttrs[j], rc, dblayer_strerror(rc)); + } + if (!run_from_cmdline) + dblayer_txn_abort(li, &txn); + ret = -2; + goto out; + } + if (!run_from_cmdline) + { + rc = dblayer_txn_commit(li, &txn); + if (0 != rc) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: ERROR: failed to commit txn for " + "update index '%s'\n", + inst->inst_name, indexAttrs[j], 0); + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Error %d: %s\n", inst->inst_name, rc, + dblayer_strerror(rc)); + if (task) { + slapi_task_log_notice(task, + "%s: ERROR: failed to commit txn for " + "update index '%s' " + "(err %d: %s)", inst->inst_name, + indexAttrs[j], rc, dblayer_strerror(rc)); + } + ret = -2; + goto out; + } + } + } + } + } + } + + /* + * Update the Virtual List View indexes + */ + for ( j = 0; j<numvlv; j++ ) { + back_txn txn; + int rc = 0; + if (run_from_cmdline) + { + txn.back_txn_txn = NULL; + } + else + if (!run_from_cmdline) + { + rc = dblayer_txn_begin(li, NULL, &txn); + if (0 != rc) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: ERROR: failed to begin txn for update index '%s'\n", + inst->inst_name, indexAttrs[j], 0); + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Error %d: %s\n", inst->inst_name, rc, + dblayer_strerror(rc)); + if (task) { + slapi_task_log_notice(task, + "%s: ERROR: failed to begin txn for update index '%s' " + "(err %d: %s)", inst->inst_name, + indexAttrs[j], rc, dblayer_strerror(rc)); + } + ret = -2; + goto out; + } + } + vlv_update_index(pvlv[j], &txn, li, pb, NULL, ep); + if (!run_from_cmdline) + { + rc = dblayer_txn_commit(li, &txn); + if (0 != rc) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: ERROR: failed to commit txn for update index '%s'\n", + inst->inst_name, indexAttrs[j], 0); + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Error %d: %s\n", inst->inst_name, rc, + dblayer_strerror(rc)); + if (task) { + slapi_task_log_notice(task, + "%s: ERROR: failed to commit txn for update index '%s' " + "(err %d: %s)", inst->inst_name, + indexAttrs[j], rc, dblayer_strerror(rc)); + } + ret = -2; + goto out; + } + } + } + + /* + * Update the ancestorid index + */ + if (index_aid) { + int rc; + + rc = ldbm_ancestorid_index_entry(be, ep, BE_INDEX_ADD, NULL); + if (rc != 0) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: ERROR: failed to update index 'ancestorid'\n", + inst->inst_name, 0, 0); + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Error %d: %s\n", inst->inst_name, rc, + dblayer_strerror(rc)); + if (task) { + slapi_task_log_notice(task, + "%s: ERROR: failed to update index 'ancestorid' " + "(err %d: %s)", inst->inst_name, + rc, dblayer_strerror(rc)); + } + ret = -2; + goto out; + } + } + + count++; + if ((count % 1000) == 0) { + int percent; + + if (idl) { + percent = (idindex*100 / (idl->b_nids ? idl->b_nids : 1)); + } else { + percent = (ep->ep_id*100 / (lastid ? lastid : 1)); + } + if (task) { + task->task_progress = (idl ? idindex : ep->ep_id); + task->task_work = (idl ? idl->b_nids : lastid); + slapi_task_status_changed(task); + slapi_task_log_status(task, "%s: Indexed %d entries (%d%%).", + inst->inst_name, count, percent); + slapi_task_log_notice(task, "%s: Indexed %d entries (%d%%).", + inst->inst_name, count, percent); + } + LDAPDebug(LDAP_DEBUG_ANY, "%s: Indexed %d entries (%d%%).\n", + inst->inst_name, count, percent); + } + + backentry_free( &ep ); + } + vlv_release_lock(be); + + /* if we got here, we finished successfully */ + + /* activate all the indexes we added */ + for (i = 0; indexAttrs && indexAttrs[i]; i++) { + struct attrinfo *ai = NULL; + + ainfo_get(be, indexAttrs[i], &ai); + PR_ASSERT(ai != NULL); + ai->ai_indexmask &= ~INDEX_OFFLINE; + } + for (i = 0; i < numvlv; i++) { + vlvIndex_go_online(pvlv[i], be); + } + + if (task) { + slapi_task_log_status(task, "%s: Finished indexing.", + inst->inst_name); + slapi_task_log_notice(task, "%s: Finished indexing.", + inst->inst_name); + } + LDAPDebug(LDAP_DEBUG_ANY, "%s: Finished indexing.\n", + inst->inst_name, 0, 0); + +out: + if (idl) { + idl_free(idl); + } else { + dbc->c_close(dbc); + } + dblayer_release_id2entry( be, db ); + + instance_set_not_busy(inst); + + LDAPDebug( LDAP_DEBUG_TRACE, "<= ldbm_back_ldbm2index\n", 0, 0, 0 ); + + if (run_from_cmdline) { + if (0 != dblayer_flush(li)) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Failed to flush database\n", inst->inst_name, 0, 0); + } + dblayer_instance_close(be); + if (0 != dblayer_close(li,DBLAYER_INDEX_MODE)) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Failed to close database\n", inst->inst_name, 0, 0); + } + } + + if (indexAttrs) { + slapi_ch_free((void **)&indexAttrs); + } + + return (ret); +} + +/* + * The db2index mode of slapd accepts commandline specification of + * an attribute to be indexed and the types of indexes to be created. + * The format is: + * (ns-)slapd db2index -tattributeName[:indextypes[:matchingrules]] + * where indextypes and matchingrules(OIDs) are comma separated lists + * e.g., + * -tuid:eq,pres + * -tuid:sub:2.1.15.17.blah + */ +static int +db2index_add_indexed_attr(backend *be, char *attrString) +{ + char *iptr = NULL; + char *mptr = NULL; + char *nsslapd_index_value[4]; + int argc = 0; + int i; + + if (NULL == (iptr = strchr(attrString, ':'))) { + return(0); + } + iptr[0] = '\0'; + iptr++; + + nsslapd_index_value[argc++] = slapi_ch_strdup(attrString+1); + + if (NULL != (mptr = strchr(iptr, ':'))) { + mptr[0] = '\0'; + mptr++; + } + nsslapd_index_value[argc++] = slapi_ch_strdup(iptr); + if (NULL != mptr) { + nsslapd_index_value[argc++] = slapi_ch_strdup(mptr); + } + nsslapd_index_value[argc] = NULL; + attr_index_config(be, "from db2index()", 0, argc, nsslapd_index_value, 0); + + for ( i=0; i<argc; i++ ) { + slapi_ch_free((void **)&nsslapd_index_value[i]); + } + return(0); +} + + +/* + * Determine if the given normalized 'attr' is to be excluded from LDIF + * exports. + * + * Returns a non-zero value if: + * 1) The 'attr' is in the configured list of attribute types that + * are to be excluded. + * OR 2) dump_uniqueid is non-zero and 'attr' is the unique ID attribute. + * + * Return 0 if the attribute is not to be excluded. + */ +static int +ldbm_exclude_attr_from_export( struct ldbminfo *li , const char *attr, + int dump_uniqueid ) + +{ + int i, rc = 0; + + if ( !dump_uniqueid && 0 == strcasecmp( SLAPI_ATTR_UNIQUEID, attr )) { + rc = 1; /* exclude */ + + } else if ( NULL != li && NULL != li->li_attrs_to_exclude_from_export ) { + for ( i = 0; li->li_attrs_to_exclude_from_export[i] != NULL; ++i ) { + if ( 0 == strcasecmp( li->li_attrs_to_exclude_from_export[i], + attr )) { + rc = 1; /* exclude */ + break; + } + } + } + + return( rc ); +} + +#if defined(UPGRADEDB) +/* + * ldbm_back_upgradedb - + * + * functions to convert idl from the old format to the new one + * (604921) Support a database uprev process any time post-install + */ + +void upgradedb_core(Slapi_PBlock *pb, ldbm_instance *inst); +int upgradedb_copy_logfiles(struct ldbminfo *li, char *destination_dir, int restore, int *cnt); +int upgradedb_delete_indices_4cmd(ldbm_instance *inst); +void normalize_dir(char *dir); + +/* + * ldbm_back_upgradedb - + * check the DB version and if it's old idl'ed index, + * then reindex using new idl. + * + * standalone only -- not allowed to run while DS is up. + */ +int ldbm_back_upgradedb(Slapi_PBlock *pb) +{ + struct ldbminfo *li; + Object *inst_obj = NULL; + ldbm_instance *inst = NULL; + int run_from_cmdline = 0; + int task_flags = 0; + int server_running = 0; + int rval = 0; + int backup_rval = 0; + char *dest_dir = NULL; + char *orig_dest_dir = NULL; + char *home_dir = NULL; + int up_flags; + int i; + Slapi_Task *task; + char inst_dir[MAXPATHLEN]; + char *inst_dirp = NULL; + + slapi_pblock_get(pb, SLAPI_SEQ_TYPE, &up_flags); + slapi_log_error(SLAPI_LOG_TRACE, "upgrade DB", "Reindexing all...\n"); + slapi_pblock_get(pb, SLAPI_TASK_FLAGS, &task_flags); + slapi_pblock_get(pb, SLAPI_BACKEND_TASK, &task); + slapi_pblock_get(pb, SLAPI_DB2LDIF_SERVER_RUNNING, &server_running); + + run_from_cmdline = (task_flags & TASK_RUNNING_FROM_COMMANDLINE); + slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &li); + if (run_from_cmdline) + { + if (!(up_flags & SLAPI_UPGRADEDB_SKIPINIT)) + { + ldbm_config_load_dse_info(li); + } + autosize_import_cache(li); + } + else + { + Object *inst_obj, *inst_obj2; + ldbm_instance *inst = NULL; + + /* server is up -- mark all backends busy */ + slapi_log_error(SLAPI_LOG_TRACE, "upgrade DB", + "server is up -- marking all LDBM backends busy\n"); + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) + { + inst = (ldbm_instance *)object_get_data(inst_obj); + /* check if an import/restore is already ongoing... */ + /* BUSY flag is cleared at the end of import_main (join thread); + it should not cleared in this thread [610347] */ + if (instance_set_busy(inst) != 0) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "ldbm: '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + inst->inst_name); + if (task) + { + slapi_task_log_notice(task, + "Backend '%s' is already in the middle of " + "another task and cannot be disturbed.\n", + inst->inst_name); + } + + /* painfully, we have to clear the BUSY flags on the + * backends we'd already marked... + */ + for (inst_obj2 = objset_first_obj(li->li_instance_set); + inst_obj2 && (inst_obj2 != inst_obj); + inst_obj2 = objset_next_obj(li->li_instance_set, inst_obj2)) + { + inst = (ldbm_instance *)object_get_data(inst_obj2); + instance_set_not_busy(inst); + } + object_release(inst_obj2); + object_release(inst_obj); + return -1; + } + } + } + + inst_obj = objset_first_obj(li->li_instance_set); + if (inst_obj) + { + inst = (ldbm_instance *)object_get_data(inst_obj); + if (!(up_flags & SLAPI_UPGRADEDB_FORCE)) + { /* upgrade idl to new */ + li->li_flags |= LI_FORCE_MOD_CONFIG; + /* set new idl */ + ldbm_config_internal_set(li, CONFIG_IDL_SWITCH, "new"); + /* First check the dbversion */ + rval = check_db_inst_version(inst); + if (!(DBVERSION_NEED_IDL_OLD2NEW & rval)) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Index version is up-to-date\n"); + return 0; + } + } + } + else + { + slapi_log_error(SLAPI_LOG_FATAL, + "upgrade DB", "No instance to be upgraded\n"); + return -1; + } + + /* we are going to go forward */ + /* + * First, backup index files and checkpoint log files + * since the server is not up and running, we can just copy them. + */ + slapi_pblock_get( pb, SLAPI_SEQ_VAL, &dest_dir ); + if (NULL == dest_dir) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Backup directory is not specified.\n"); + return -1; + } + + { + int cnt = 0; + PRFileInfo info; + + orig_dest_dir = dest_dir; + normalize_dir(dest_dir); + /* clean up the backup dir first, then create it */ + rval = PR_GetFileInfo(dest_dir, &info); + if (PR_SUCCESS == rval) + { + if (PR_FILE_DIRECTORY == info.type) /* directory exists */ + { + time_t tm = time(0); /* long */ + + char *tmpname = (char *)slapi_ch_malloc(strlen(dest_dir) + 32); + sprintf(tmpname, "%s/%d", dest_dir, tm); + dest_dir = tmpname; + } + else /* not a directory */ + PR_Delete(dest_dir); + } + + if (mkdir_p(dest_dir, 0700) < 0) + goto fail0; + + while (1) + { + inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN); + backup_rval = dblayer_copy_directory(li, NULL /* task */, + inst_dirp, dest_dir, 0/*backup*/, + &cnt, 0, 1); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + if (backup_rval < 0) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Warning: Failed to backup index files (instance %s).\n", + inst_dirp); + goto fail1; + } + + /* delete index files to be reindexed */ + if (run_from_cmdline) + { + if (0 != upgradedb_delete_indices_4cmd(inst)) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Can't clean up indices in %s\n", inst->inst_dir_name); + goto fail1; + } + } + else + { + if (0 != dblayer_delete_indices(inst)) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Can't clean up indices in %s\n", inst->inst_dir_name); + goto fail1; + } + } + + inst_obj = objset_next_obj(li->li_instance_set, inst_obj); + if (NULL == inst_obj) + break; + inst = (ldbm_instance *)object_get_data(inst_obj); + } + + /* copy checkpoint logs */ + backup_rval += upgradedb_copy_logfiles(li, dest_dir, 0, &cnt); + } + + if (run_from_cmdline) + ldbm_config_internal_set(li, CONFIG_DB_TRANSACTION_LOGGING, "off"); + + inst_obj = objset_first_obj(li->li_instance_set); + for (i = 0; NULL != inst_obj; i++) + { + if (run_from_cmdline) + { + /* need to call dblayer_start for each instance, + since dblayer_close is called in upgradedb_core => + ldbm_back_ldif2ldbm_deluxe */ + if (0 != dblayer_start(li, DBLAYER_IMPORT_MODE)) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "upgradedb: Failed to init database\n"); + goto fail1; + } + } + + inst = (ldbm_instance *)object_get_data(inst_obj); + slapi_pblock_set(pb, SLAPI_BACKEND, inst->inst_be); + slapi_pblock_set(pb, SLAPI_BACKEND_INSTANCE_NAME, inst->inst_name); + upgradedb_core(pb, inst); + inst_obj = objset_next_obj(li->li_instance_set, inst_obj); + } + + /* upgrade idl to new; otherwise no need to modify idl-switch */ + if (!(up_flags & SLAPI_UPGRADEDB_FORCE)) + { + replace_ldbm_config_value(CONFIG_IDL_SWITCH, "new", li); + } + + home_dir = dblayer_get_home_dir(li, NULL); + + /* write db version files */ + dbversion_write(li, home_dir, NULL); + + inst_obj = objset_first_obj(li->li_instance_set); + while (NULL != inst_obj) + { + char *inst_dirp = NULL; + inst_dirp = dblayer_get_full_inst_dir(li, inst, inst_dir, MAXPATHLEN); + inst = (ldbm_instance *)object_get_data(inst_obj); + dbversion_write(li, inst_dirp, NULL); + inst_obj = objset_next_obj(li->li_instance_set, inst_obj); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + } + + /* close the database down again */ + if (run_from_cmdline) + { + if (0 != dblayer_flush(li)) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Failed to flush database\n"); + } + if (0 != dblayer_close(li,DBLAYER_IMPORT_MODE)) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Failed to close database\n"); + goto fail1; + } + } + + /* delete backup */ + if (NULL != dest_dir) + ldbm_delete_dirs(dest_dir); + + if (dest_dir != orig_dest_dir) + slapi_ch_free_string(&dest_dir); + + return 0; + +fail1: + if (0 != dblayer_flush(li)) + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Failed to flush database\n"); + + /* Ugly! (we started dblayer with DBLAYER_IMPORT_MODE) + * We just want not to generate a guardian file... + */ + if (0 != dblayer_close(li,DBLAYER_ARCHIVE_MODE)) + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Failed to close database\n"); + + /* restore from the backup, if possible */ + if (NULL != dest_dir) + { + if (0 == backup_rval) /* only when the backup succeeded... */ + { + int cnt = 0; + + inst_obj = objset_first_obj(li->li_instance_set); + while (NULL != inst_obj) + { + inst = (ldbm_instance *)object_get_data(inst_obj); + + inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN); + backup_rval = dblayer_copy_directory(li, NULL /* task */, + inst->inst_dir_name, + dest_dir, 1/*restore*/, + &cnt, 0, 1); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + if (backup_rval < 0) + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Failed to restore index files (instance %s).\n", + inst->inst_name); + + inst_obj = objset_next_obj(li->li_instance_set, inst_obj); + } + + backup_rval = upgradedb_copy_logfiles(li, dest_dir, 1, &cnt); + if (backup_rval < 0) + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "Failed to restore log files.\n"); + } + + /* anyway clean up the backup dir */ + ldbm_delete_dirs(dest_dir); + } + +fail0: + if (dest_dir != orig_dest_dir) + slapi_ch_free_string(&dest_dir); + + return rval; +} + +void normalize_dir(char *dir) +{ + int l = strlen(dir); + if ('/' == dir[l-1] || '\\' == dir[l-1]) + { + dir[l-1] = '\0'; + } +} + +#define LOG "log." +#define LOGLEN 4 +int upgradedb_copy_logfiles(struct ldbminfo *li, char *destination_dir, + int restore, int *cnt) +{ + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + char *src; + char *dest; + int srclen; + int destlen; + int rval = 0; + int len0 = 0; + int len1 = 0; + char *from = NULL; + char *to = NULL; + + *cnt = 0; + if (restore) + { + src = destination_dir; + dest = li->li_directory; + } + else + { + src = li->li_directory; + dest = destination_dir; + } + srclen = strlen(src); + destlen = strlen(dest); + + /* Open the instance dir so we can look what's in it. */ + dirhandle = PR_OpenDir(src); + if (NULL == dirhandle) + return -1; + + while (NULL != (direntry = + PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) + { + if (NULL == direntry->name) + break; + + if (0 == strncmp(direntry->name, LOG, 4)) + { + int filelen = strlen(direntry->name); + char *p, *endp; + int fromlen, tolen; + int notalog = 0; + + endp = (char *)direntry->name + filelen; + for (p = (char *)direntry->name + LOGLEN; p < endp; p++) + { + if (!isdigit(*p)) + { + notalog = 1; + break; + } + } + if (notalog) + continue; /* go to next file */ + + fromlen = srclen + filelen + 2; + if (len0 < fromlen) + { + slapi_ch_free_string(&from); + from = slapi_ch_calloc(1, fromlen); + len0 = fromlen; + } + sprintf(from, "%s/%s", src, direntry->name); + tolen = destlen + filelen + 2; + if (len1 < tolen) + { + slapi_ch_free_string(&to); + to = slapi_ch_calloc(1, tolen); + len1 = tolen; + } + sprintf(to, "%s/%s", dest, direntry->name); + if (NULL == from || NULL == to) + break; + rval = dblayer_copyfile(from, to, 1, DEFAULT_MODE); + if (rval < 0) + break; + cnt++; + } + } + slapi_ch_free_string(&from); + slapi_ch_free_string(&to); + PR_CloseDir(dirhandle); + + return rval; +} + +int upgradedb_delete_indices_4cmd(ldbm_instance *inst) +{ + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + int rval = 0; + char fullpath[MAXPATHLEN]; + char *fullpathp = fullpath; + char inst_dir[MAXPATHLEN]; + char *inst_dirp = dblayer_get_full_inst_dir(inst->inst_li, inst, + inst_dir, MAXPATHLEN); + + slapi_log_error(SLAPI_LOG_TRACE, "upgrade DB", + "upgradedb_delete_indices_4cmd: %s\n"); + dirhandle = PR_OpenDir(inst_dirp); + if (!dirhandle) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "upgradedb_delete_indices_4cmd: PR_OpenDir failed\n"); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return -1; + } + + while (NULL != (direntry = + PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) + { + PRFileInfo info; + int len; + + if (! direntry->name) + break; + + if (0 == strcmp(direntry->name, ID2ENTRY LDBM_FILENAME_SUFFIX)) + continue; + + len = strlen(inst_dirp) + strlen(direntry->name) + 2; + if (len > MAXPATHLEN) + { + fullpathp = (char *)slapi_ch_malloc(len); + } + sprintf(fullpathp, "%s/%s", inst_dirp, direntry->name); + rval = PR_GetFileInfo(fullpathp, &info); + if (PR_SUCCESS == rval && PR_FILE_DIRECTORY != info.type) + { + PR_Delete(fullpathp); + slapi_log_error(SLAPI_LOG_TRACE, "upgrade DB", + "upgradedb_delete_indices_4cmd: %s deleted\n", fullpath); + } + if (fullpathp != fullpath) + slapi_ch_free_string(&fullpathp); + } + PR_CloseDir(dirhandle); + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return rval; +} + +/* + * upgradedb_core + */ +void upgradedb_core(Slapi_PBlock *pb, ldbm_instance *inst) +{ + backend *be = NULL; + int task_flags = 0; + int run_from_cmdline = 0; + + slapi_pblock_get(pb, SLAPI_TASK_FLAGS, &task_flags); + run_from_cmdline = (task_flags & TASK_RUNNING_FROM_COMMANDLINE); + + be = inst->inst_be; + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "%s: Start upgradedb.\n", inst->inst_name); + + if (!run_from_cmdline) + { + /* shutdown this instance of the db */ + slapi_log_error(SLAPI_LOG_TRACE, "upgrade DB", + "Bringing %s offline...\n", inst->inst_name); + slapi_mtn_be_disable(inst->inst_be); + + cache_clear(&inst->inst_cache); + dblayer_instance_close(be); + } + + /* dblayer_instance_start will init the id2entry index. */ + if (0 != dblayer_instance_start(be, DBLAYER_IMPORT_MODE)) + { + slapi_log_error(SLAPI_LOG_FATAL, "upgrade DB", + "upgradedb: Failed to init instance %s\n", inst->inst_name); + return; + } + + if (run_from_cmdline) + vlv_init(inst); /* Initialise the Virtual List View code */ + + ldbm_back_ldif2ldbm_deluxe(pb); +} + +#endif /* UPGRADEDB */ diff --git a/ldap/servers/slapd/back-ldbm/libback-ldbm.def b/ldap/servers/slapd/back-ldbm/libback-ldbm.def new file mode 100644 index 00000000..0967d9c5 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/libback-ldbm.def @@ -0,0 +1,13 @@ +; BEGIN COPYRIGHT BLOCK +; Copyright 2001 Sun Microsystems, Inc. +; Portions copyright 1999, 2001-2003 Netscape Communications Corporation. +; All rights reserved. +; END COPYRIGHT BLOCK +; +DESCRIPTION 'Directory Server 2.0 DB Backend Plugin' +EXPORTS + ldbm_back_init @2 + plugin_init_debug_level @3 +; ldbm_back_changelog_init @4 + + diff --git a/ldap/servers/slapd/back-ldbm/matchrule.c b/ldap/servers/slapd/back-ldbm/matchrule.c new file mode 100644 index 00000000..0d7197ab --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/matchrule.c @@ -0,0 +1,126 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* matchrule.c */ + + +#include "back-ldbm.h" + +/* NPCTE fix for bug # 394184, SD, 20 Jul 00 */ +/* replace the hard coded return value by the appropriate LDAP error code */ +/* + * Returns: 0 -- OK now is: LDAP_SUCCESS (fix for bug #394184) + * -1 -- protocol error now is: LDAP_PROTOCOL_ERROR + * -3 -- operation error now is: LDAP_OPERATIONS_ERROR + */ +int +create_matchrule_indexer(Slapi_PBlock **pb,char* matchrule,char* type) +{ + IFP mrINDEX = NULL; + int return_value = LDAP_SUCCESS; + unsigned int sort_indicator = SLAPI_PLUGIN_MR_USAGE_SORT; + + if(pb==NULL) + { + return LDAP_OPERATIONS_ERROR; + } + + if(*pb==NULL) + { + *pb = slapi_pblock_new(); + } + if(*pb==NULL) + { + /* Memory allocation faliure */ + /* Operations error to the calling routine */ + return LDAP_OPERATIONS_ERROR; + } + + /* If these fail, it's an operations error */ + return_value |= slapi_pblock_set (*pb, SLAPI_PLUGIN_MR_OID, matchrule); + return_value |= slapi_pblock_set (*pb, SLAPI_PLUGIN_MR_TYPE, type); + return_value |= slapi_pblock_set (*pb, SLAPI_PLUGIN_MR_USAGE, (void*)&sort_indicator); + if (0 != return_value) + { + return LDAP_OPERATIONS_ERROR; + } + + /* If this fails, could be operations error, or that OID is not supported */ + return_value = slapi_mr_indexer_create (*pb); + if (0 != return_value) + { + return LDAP_PROTOCOL_ERROR; + } + + /* If these fail, ops error */ + return_value = slapi_pblock_get (*pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX); + + if ( (0 != return_value) || (mrINDEX == NULL) ) + { + return LDAP_OPERATIONS_ERROR; + } + else + { + return LDAP_SUCCESS; + } +} +/* End NPCTE fix for bug # 394184 */ + +int +destroy_matchrule_indexer(Slapi_PBlock *pb) +{ + IFP mrDESTROY = NULL; + if (!slapi_pblock_get (pb, SLAPI_PLUGIN_DESTROY_FN, &mrDESTROY)) + { + if (mrDESTROY != NULL) + { + mrDESTROY (pb); + } + } + return 0; +} + + +/* + * This routine returns pointer to memory which is owned by the plugin, so don't + * free it. Gets freed by the next call to this routine, or when the indexer + * is destroyed + */ +int +matchrule_values_to_keys(Slapi_PBlock *pb,struct berval **input_values,struct berval ***output_values) +{ + IFP mrINDEX = NULL; + + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX); + slapi_pblock_set (pb, SLAPI_PLUGIN_MR_VALUES, input_values); + mrINDEX (pb); + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_KEYS, output_values); + return 0; +} + +/* + * This routine returns pointer to memory which is owned by the plugin, so don't + * free it. Gets freed by the next call to this routine, or when the indexer + * is destroyed + */ +int +matchrule_values_to_keys_sv(Slapi_PBlock *pb,Slapi_Value **input_values,Slapi_Value ***output_values) +{ + IFP mrINDEX = NULL; + struct berval **bvi, **bvo; + + valuearray_get_bervalarray(input_values, &bvi); + + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_INDEX_FN, &mrINDEX); + slapi_pblock_set (pb, SLAPI_PLUGIN_MR_VALUES, bvi); + mrINDEX (pb); + slapi_pblock_get (pb, SLAPI_PLUGIN_MR_KEYS, &bvo); + + slapi_pblock_set (pb, SLAPI_PLUGIN_MR_VALUES, NULL); + ber_bvecfree(bvi); + + valuearray_init_bervalarray(bvo, output_values); + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/misc.c b/ldap/servers/slapd/back-ldbm/misc.c new file mode 100644 index 00000000..b2a8d6de --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/misc.c @@ -0,0 +1,356 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* misc.c - backend misc routines */ + +#include "back-ldbm.h" + +/* Takes a return code supposed to be errno or from lidb + which we don't expect to see and prints a handy log message */ +void ldbm_nasty(const char* str, int c, int err) +{ + char *msg = NULL; + char buffer[200]; + if (err == DB_LOCK_DEADLOCK) { + sprintf(buffer,"%s WARNING %d",str,c); + LDAPDebug(LDAP_DEBUG_TRACE,"%s, err=%d %s\n", + buffer,err,(msg = dblayer_strerror( err )) ? msg : ""); + } else if (err == DB_RUNRECOVERY) { + LDAPDebug(LDAP_DEBUG_ANY,"FATAL ERROR at %s (%d); server stopping as database recovery needed.\n", str,c,0); + exit(1); + } else { + sprintf(buffer,"%s BAD %d",str,c); + LDAPDebug(LDAP_DEBUG_ANY,"%s, err=%d %s\n", + buffer,err,(msg = dblayer_strerror( err )) ? msg : ""); + } +} + +/* Put a message in the access log, complete with connection ID and operation ID */ +void ldbm_log_access_message(Slapi_PBlock *pblock,char *string) +{ + int ret = 0; + int connection_id = 0; + int operation_id = 0; + Operation *operation = NULL; /* DBDB this is sneaky---opid should be covered by the API directly */ + + ret = slapi_pblock_get(pblock,SLAPI_OPERATION,&operation); + if (0 != ret) { + return; + } + ret = slapi_pblock_get(pblock,SLAPI_CONN_ID,&connection_id); + if (0 != ret) { + return; + } + operation_id = operation->o_opid; + slapi_log_access( LDAP_DEBUG_STATS, "conn=%d op=%d %s\n",connection_id, operation_id,string); +} + +int return_on_disk_full(struct ldbminfo *li) +{ + dblayer_remember_disk_filled(li); + return SLAPI_FAIL_DISKFULL; +} + + +/* System Indexes */ + +static const char *systemIndexes[] = { + "entrydn", + "parentid", + "objectclass", + "aci", + "numsubordinates", + SLAPI_ATTR_UNIQUEID, + SLAPI_ATTR_NSCP_ENTRYDN, + ATTR_NSDS5_REPLCONFLICT, + NULL +}; + +int +ldbm_attribute_always_indexed(const char *attrtype) +{ + int r= 0; + if(NULL != attrtype) + { + int i=0; + while (!r && systemIndexes[i] != NULL) + { + if(!strcasecmp(attrtype,systemIndexes[i])) + { + r= 1; + } + i++; + } + } + return(r); +} + + + +/* + * Given an entry dn and a uniqueid, compute the + * DN of the entry's tombstone. Returns a pointer + * to an allocated block of memory. + */ +char * +compute_entry_tombstone_dn(const char *entrydn, const char *uniqueid) +{ + const char *tombstone_dn_pattern = "%s=%s, %s"; + char *tombstone_dn; + + PR_ASSERT(NULL != entrydn); + PR_ASSERT(NULL != uniqueid); + + tombstone_dn = slapi_ch_malloc(strlen(SLAPI_ATTR_UNIQUEID) + + strlen(tombstone_dn_pattern) + + strlen(uniqueid) + + strlen(entrydn) + 1); + sprintf(tombstone_dn, tombstone_dn_pattern, + SLAPI_ATTR_UNIQUEID, + uniqueid, + entrydn); + return tombstone_dn; +} + + +/* mark a backend instance "busy" + * returns 0 on success, -1 if the instance is ALREADY busy + */ +int instance_set_busy(ldbm_instance *inst) +{ + PR_Lock(inst->inst_config_mutex); + if (inst->inst_flags & INST_FLAG_BUSY) { + PR_Unlock(inst->inst_config_mutex); + return -1; + } + + inst->inst_flags |= INST_FLAG_BUSY; + PR_Unlock(inst->inst_config_mutex); + return 0; +} + +int instance_set_busy_and_readonly(ldbm_instance *inst) +{ + PR_Lock(inst->inst_config_mutex); + if (inst->inst_flags & INST_FLAG_BUSY) { + PR_Unlock(inst->inst_config_mutex); + return -1; + } + + inst->inst_flags |= INST_FLAG_BUSY; + + /* save old readonly state */ + if (slapi_be_get_readonly(inst->inst_be)) { + inst->inst_flags |= INST_FLAG_READONLY; + } else { + inst->inst_flags &= ~INST_FLAG_READONLY; + } + slapi_mtn_be_set_readonly(inst->inst_be, 1); + + PR_Unlock(inst->inst_config_mutex); + return 0; +} + +/* mark a backend instance to be not "busy" anymore */ +void instance_set_not_busy(ldbm_instance *inst) +{ + int readonly; + + PR_Lock(inst->inst_config_mutex); + inst->inst_flags &= ~INST_FLAG_BUSY; + /* set backend readonly flag to match instance flags again + * (sometimes the instance changes the readonly status when it's busy) + */ + readonly = (inst->inst_flags & INST_FLAG_READONLY ? 1 : 0); + slapi_mtn_be_set_readonly(inst->inst_be, readonly); + PR_Unlock(inst->inst_config_mutex); +} + +void +allinstance_set_not_busy(struct ldbminfo *li) +{ + ldbm_instance *inst; + Object *inst_obj; + + /* server is up -- mark all backends busy */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + instance_set_not_busy(inst); + } + if (inst_obj) + object_release(inst_obj); +} + +void +allinstance_set_busy(struct ldbminfo *li) +{ + ldbm_instance *inst; + Object *inst_obj; + + /* server is up -- mark all backends busy */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + instance_set_busy(inst); + } + if (inst_obj) + object_release(inst_obj); +} + +int +is_anyinstance_busy(struct ldbminfo *li) +{ + ldbm_instance *inst; + Object *inst_obj; + int rval = 0; + + /* server is up -- mark all backends busy */ + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + PR_Lock(inst->inst_config_mutex); + rval = inst->inst_flags & INST_FLAG_BUSY; + PR_Unlock(inst->inst_config_mutex); + if (0 != rval) { + break; + } + } + if (inst_obj) + object_release(inst_obj); + return rval; +} + +/* + * delete the given file/directory and its sub files/directories + */ +int +ldbm_delete_dirs(char *path) +{ + PRDir *dirhandle = NULL; + PRDirEntry *direntry = NULL; + char fullpath[MAXPATHLEN]; + int rval = 0; + PRFileInfo info; + + dirhandle = PR_OpenDir(path); + if (! dirhandle) + { + PR_Delete(path); + return 0; + } + + while (NULL != (direntry = + PR_ReadDir(dirhandle, PR_SKIP_DOT | PR_SKIP_DOT_DOT))) + { + if (! direntry->name) + break; + + sprintf(fullpath, "%s/%s", path, direntry->name); + rval = PR_GetFileInfo(fullpath, &info); + if (PR_SUCCESS == rval) + { + if (PR_FILE_DIRECTORY == info.type) + rval += ldbm_delete_dirs(fullpath); + } + if (PR_FILE_DIRECTORY != info.type) + PR_Delete(fullpath); + } + PR_CloseDir(dirhandle); + /* remove the directory itself too */ + rval += PR_RmDir(path); + return rval; +} + +char +get_sep(char *path) +{ + if (NULL == path) + return '/'; /* default */ + if (NULL != strchr(path, '/')) + return '/'; + if (NULL != strchr(path, '\\')) + return '\\'; + return '/'; /* default */ +} + +/* mkdir -p */ +int +mkdir_p(char *dir, unsigned int mode) +{ + PRFileInfo info; + int rval; + char sep = get_sep(dir); + + rval = PR_GetFileInfo(dir, &info); + if (PR_SUCCESS == rval) + { + if (PR_FILE_DIRECTORY != info.type) /* not a directory */ + { + PR_Delete(dir); + if (PR_SUCCESS != PR_MkDir(dir, mode)) + { + LDAPDebug(LDAP_DEBUG_ANY, "mkdir_p %s: error %d (%s)\n", + dir, PR_GetError(),slapd_pr_strerror(PR_GetError())); + return -1; + } + } + return 0; + } + else + { + /* does not exist */ + char *p, *e; + char c[2] = {0, 0}; + int len = strlen(dir); + rval = 0; + + e = dir + len - 1; + if (*e == sep) + { + c[1] = *e; + *e = '\0'; + } + + c[0] = '/'; + p = strrchr(dir, sep); + if (NULL != p) + { + *p = '\0'; + rval = mkdir_p(dir, mode); + *p = c[0]; + } + if (c[1]) + *e = c[1]; + if (0 != rval) + return rval; + if (PR_SUCCESS != PR_MkDir(dir, mode)) + { + LDAPDebug(LDAP_DEBUG_ANY, "mkdir_p %s: error %d (%s)\n", + dir, PR_GetError(),slapd_pr_strerror(PR_GetError())); + return -1; + } + return 0; + } +} + +int +is_fullpath(char *path) +{ + int len; + if (NULL == path || '\0' == *path) + return 0; + + if ('/' == *path || '\\' == *path) + return 1; + + len = strlen(path); + if (len > 2) + { + if (':' == path[1] && ('/' == path[2] || '\\' == path[2])) /* Windows */ + return 1; + } + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/monitor.c b/ldap/servers/slapd/back-ldbm/monitor.c new file mode 100644 index 00000000..1c5a2960 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/monitor.c @@ -0,0 +1,274 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* monitor.c - ldbm backend monitor function */ + +#include "back-ldbm.h" +#include "dblayer.h" /* XXXmcs: not sure this is good to do... */ +#include <sys/stat.h> + + +#define MSET(_attr) do { \ + val.bv_val = buf; \ + val.bv_len = strlen(buf); \ + attrlist_replace(&e->e_attrs, (_attr), vals); \ +} while (0) + +#define MSETF(_attr, _x) do { \ + char tmp_atype[37]; \ + sprintf(tmp_atype, _attr, _x); \ + MSET(tmp_atype); \ +} while (0) + + +/* DSE callback to monitor stats for a particular instance */ +int ldbm_back_monitor_instance_search(Slapi_PBlock *pb, Slapi_Entry *e, + Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + struct ldbminfo *li = NULL; + struct berval val; + struct berval *vals[2]; + char buf[BUFSIZ]; + u_long hits, tries; + long nentries,maxentries; + size_t size,maxsize; +/* NPCTE fix for bugid 544365, esc 0. <P.R> <04-Jul-2001> */ + struct stat astat; +/* end of NPCTE fix for bugid 544365 */ + DB_MPOOL_FSTAT **mpfstat = NULL; + int i,j; + + /* Get the LDBM Info structure for the ldbm backend */ + if (inst->inst_be->be_database == NULL) { + *returncode= LDAP_OPERATIONS_ERROR; + return SLAPI_DSE_CALLBACK_ERROR; + } + + li = (struct ldbminfo *)inst->inst_be->be_database->plg_private; + if (li == NULL) { + *returncode= LDAP_OPERATIONS_ERROR; + return SLAPI_DSE_CALLBACK_ERROR; + } + + if (inst->inst_be->be_state != BE_STATE_STARTED) + { + *returncode = LDAP_SUCCESS; + return SLAPI_DSE_CALLBACK_OK; + } + + vals[0] = &val; + vals[1] = NULL; + + /* database name */ + sprintf(buf, "%s", li->li_plugin->plg_name); + MSET("database"); + + /* read-only status */ + sprintf( buf, "%d", inst->inst_be->be_readonly ); + MSET("readOnly"); + + /* fetch cache statistics */ + cache_get_stats(&(inst->inst_cache), &hits, &tries, + &nentries, &maxentries, &size, &maxsize); + sprintf(buf, "%lu", hits); + MSET("entryCacheHits"); + sprintf(buf, "%lu", tries); + MSET("entryCacheTries"); + sprintf(buf, "%lu", (unsigned long)(100.0*(double)hits / (double)(tries > 0 ? tries : 1))); + MSET("entryCacheHitRatio"); + sprintf(buf, "%lu", size); + MSET("currentEntryCacheSize"); + sprintf(buf, "%lu", maxsize); + MSET("maxEntryCacheSize"); + sprintf(buf, "%ld", nentries); + MSET("currentEntryCacheCount"); + sprintf(buf, "%ld", maxentries); + MSET("maxEntryCacheCount"); + +#ifdef DEBUG + { + /* debugging for hash statistics */ + char *x; + cache_debug_hash(&(inst->inst_cache), &x); + val.bv_val = x; + val.bv_len = strlen(x); + attrlist_replace(&e->e_attrs, "entrycache-hashtables", vals); + slapi_ch_free((void **)&x); + } +#endif + + if (dblayer_memp_stat(li, NULL, &mpfstat) != 0) { + *returncode = LDAP_OPERATIONS_ERROR; + return SLAPI_DSE_CALLBACK_ERROR; + } + + for (i = 0;(mpfstat[i] && (mpfstat[i]->file_name != NULL)); i++) { +#ifdef _WIN32 + int fpos = 0; +#endif + char *absolute_pathname = NULL; + size_t absolute_pathname_size = 0; + + /* only print out stats on files used by this instance */ + if (strlen(mpfstat[i]->file_name) < strlen(inst->inst_dir_name)) + continue; + if (strncmp(mpfstat[i]->file_name, inst->inst_dir_name, + strlen(inst->inst_dir_name)) != 0) + continue; + + /* Since the filenames are now relative, we need to construct an absolute version + * for the purpose of stat() etc below... + */ + if (absolute_pathname) { + slapi_ch_free(&absolute_pathname); + } + absolute_pathname_size = strlen(inst->inst_parent_dir_name) + strlen(mpfstat[i]->file_name) + 2; + absolute_pathname = slapi_ch_malloc(absolute_pathname_size); + sprintf(absolute_pathname, "%s%c%s" , inst->inst_parent_dir_name, get_sep(inst->inst_parent_dir_name), mpfstat[i]->file_name ); + +/* NPCTE fix for bugid 544365, esc 0. <P.R> <04-Jul-2001> */ + /* Hide statistic of deleted files (mainly indexes) */ + if (stat(absolute_pathname,&astat)) + continue; + /* If the file has been re-created after been deleted + * We should show only statistics for the last instance + * Since SleepyCat returns the statistic of the last open file first, + * we should only display the first statistic record for a given file + */ + for (j=0;j<i;j++) + if (!strcmp(mpfstat[i]->file_name,mpfstat[j]->file_name)) + break; + if (j<i) + continue; +/* end of NPCTE fix for bugid 544365 */ + + /* Get each file's stats */ + sprintf(buf, "%s", mpfstat[i]->file_name); +#ifdef _WIN32 + /* + * For NT, switch the last + * backslash to a foward + * slash. - RJP + */ + for (fpos = strlen(buf); fpos >= 0; fpos--) { + if (buf[fpos] == '\\') { + buf[fpos] = '/'; + break; + } + } +#endif + MSETF("dbFilename-%d", i); + + sprintf(buf, "%u", mpfstat[i]->st_cache_hit); + MSETF("dbFileCacheHit-%d", i); + sprintf(buf, "%u", mpfstat[i]->st_cache_miss); + MSETF("dbFileCacheMiss-%d", i); + sprintf(buf, "%u", mpfstat[i]->st_page_in); + MSETF("dbFilePageIn-%d", i); + sprintf(buf, "%u", mpfstat[i]->st_page_out); + MSETF("dbFilePageOut-%d", i); + + if (absolute_pathname) { + slapi_ch_free(&absolute_pathname); + } + + } + +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR + DB_VERSION_PATCH <= 3204 + /* In DB 3.2.4 and earlier, we need to free each element */ + for (i = 0; mpfstat[i]; i++) + free(mpfstat[i]); +#endif + free(mpfstat); + + *returncode = LDAP_SUCCESS; + return SLAPI_DSE_CALLBACK_OK; + +} + + +/* monitor global ldbm stats */ +int ldbm_back_monitor_search(Slapi_PBlock *pb, Slapi_Entry *e, + Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + struct ldbminfo *li = (struct ldbminfo *)arg; + struct berval val; + struct berval *vals[2]; + char buf[BUFSIZ]; + DB_MPOOL_STAT *mpstat = NULL; + DB_MPOOL_FSTAT **mpfstat = NULL; + u_int32_t cache_tries; + + vals[0] = &val; + vals[1] = NULL; + + /* database name */ + sprintf(buf, "%s", li->li_plugin->plg_name); + MSET("database"); + + /* we have to ask for file stats in order to get correct global stats */ + if (dblayer_memp_stat(li, &mpstat, &mpfstat) != 0) { + *returncode = LDAP_OPERATIONS_ERROR; + return SLAPI_DSE_CALLBACK_ERROR; + } + + /* cache hits*/ + sprintf(buf, "%u", mpstat->st_cache_hit); + MSET("dbCacheHits"); + + /* cache tries*/ + cache_tries = (mpstat->st_cache_miss + mpstat->st_cache_hit); + sprintf(buf, "%u", cache_tries); + MSET("dbCacheTries"); + + /* cache hit ratio*/ + sprintf(buf, "%lu", (unsigned long)(100.0 * (double)mpstat->st_cache_hit / (double)(cache_tries > 0 ? cache_tries : 1) )); + MSET("dbCacheHitRatio"); + + sprintf(buf, "%u", mpstat->st_page_in); + MSET("dbCachePageIn"); + sprintf(buf, "%u", mpstat->st_page_out); + MSET("dbCachePageOut"); + sprintf(buf, "%u", mpstat->st_ro_evict); + MSET("dbCacheROEvict"); + sprintf(buf, "%u", mpstat->st_rw_evict); + MSET("dbCacheRWEvict"); + + free(mpstat); + + if (mpfstat) { +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR + DB_VERSION_PATCH <= 3204 + /* In DB 3.2.4 and earlier, we need to free each element */ + int i; + for (i = 0; mpfstat[i]; i++) + free(mpfstat[i]); +#endif + free(mpfstat); + } + + *returncode = LDAP_SUCCESS; + return SLAPI_DSE_CALLBACK_OK; +} + + +/* monitor global ldbm database stats */ +int +ldbm_back_dbmonitor_search(Slapi_PBlock *pb, Slapi_Entry *e, + Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg) +{ + dblayer_private *dbpriv = NULL; + struct ldbminfo *li = NULL; + + PR_ASSERT(NULL != arg); + li = (struct ldbminfo*)arg; + dbpriv = (dblayer_private*)li->li_dblayer_private; + PR_ASSERT(NULL != dbpriv); + + perfctrs_as_entry( e, dbpriv->perf_private, dbpriv->dblayer_env->dblayer_DB_ENV); + + *returncode = LDAP_SUCCESS; + return SLAPI_DSE_CALLBACK_OK; +} diff --git a/ldap/servers/slapd/back-ldbm/nextid.c b/ldap/servers/slapd/back-ldbm/nextid.c new file mode 100644 index 00000000..12773768 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/nextid.c @@ -0,0 +1,204 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* id.c - keep track of the next id to be given out */ + +#include "back-ldbm.h" + +ID +next_id(backend *be) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + ID id; + + /*Lock*/ + PR_Lock( inst->inst_nextid_mutex ); + + /*Test if nextid hasn't been initialized. */ + if (inst->inst_nextid < 1) { + LDAPDebug( LDAP_DEBUG_ANY, + "ldbm backend instance: nextid not initialized... exiting.\n", 0,0,0); + exit(1); + } + + /*Increment the in-memory nextid*/ + inst->inst_nextid++; + + id = inst->inst_nextid - 1; + + /*unlock*/ + PR_Unlock( inst->inst_nextid_mutex ); + + /* if ID is above the threshold, the database may need rebuilding soon */ + if (id >= ID_WARNING_THRESHOLD) { + if ( id >= MAXID ) { + LDAPDebug( LDAP_DEBUG_ANY, + "ldbm backend instance: FATAL ERROR: backend '%s' has no" + "IDs left. DATABASE MUST BE REBUILT.\n", be->be_name, 0, + 0); + id = MAXID; + } else { + LDAPDebug( LDAP_DEBUG_ANY, + "ldbm backend instance: WARNING: backend '%s' may run out " + "of IDs. Please, rebuild database.\n", be->be_name, 0, 0); + } + } + return( id ); +} + +void +next_id_return( backend *be, ID id ) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + + /*Lock*/ + PR_Lock( inst->inst_nextid_mutex ); + + /*Test if nextid hasn't been initialized. */ + if (inst->inst_nextid < 1) { + LDAPDebug( LDAP_DEBUG_ANY, + "ldbm backend instance: nextid not initialized... exiting\n", 0,0,0); + exit(1); + } + + if ( id != inst->inst_nextid - 1 ) { + PR_Unlock( inst->inst_nextid_mutex ); + return; + } + + /*decrement the in-memory version*/ + inst->inst_nextid--; + + /*unlock this bad boy*/ + PR_Unlock( inst->inst_nextid_mutex ); +} + +ID +next_id_get( backend *be ) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + ID id; + + /*lock*/ + PR_Lock( inst->inst_nextid_mutex ); + + /*Test if nextid hasn't been initialized.*/ + if (inst->inst_nextid < 1) { + LDAPDebug( LDAP_DEBUG_ANY, + "ldbm backend instance: nextid not initialized... exiting\n", 0,0,0); + exit(1); + } + + id = inst->inst_nextid; + PR_Unlock( inst->inst_nextid_mutex ); + + return( id ); +} + +/* + * Function: get_ids_from_disk + * + * Returns: squat + * + * Description: Opend the id2entry file and obtains the largest + * ID in use, and sets li->li_nextid. If no IDs + * could be read from id2entry, li->li_nextid + * is set to 1. + */ +void +get_ids_from_disk(backend *be) +{ + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + DB *id2entrydb; /*the id2entry database*/ + int return_value = -1; + + /*For the nextid, we go directly to the id2entry database, + and grab the max ID*/ + + /*Get a copy of the id2entry database*/ + if ( (return_value = dblayer_get_id2entry( be, &id2entrydb )) != 0 ) { + id2entrydb = NULL; + } + + /* lock the nextid mutex*/ + PR_Lock( inst->inst_nextid_mutex ); + + /* + * If there is no id2entry database, then we can assume that there + * are no entries, and that nextid should be 1 + */ + if (id2entrydb == NULL) { + inst->inst_nextid = 1; + + /* unlock */ + PR_Unlock( inst->inst_nextid_mutex ); + return; + + } else { + + /*Get the last key*/ + DBC *dbc = NULL; + DBT key = {0}; /*For the nextid*/ + DBT Value = {0}; + Value.flags = DB_DBT_MALLOC; + key.flags = DB_DBT_MALLOC; + return_value = id2entrydb->cursor(id2entrydb,NULL,&dbc,0); + if (0 == return_value) { + return_value = dbc->c_get(dbc,&key,&Value,DB_LAST); + if (0 == return_value) { + inst->inst_nextid = id_stored_to_internal(key.dptr) + 1; + } + if (NULL != key.data) { + free(key.data); + } + if (NULL != Value.data) { + free(Value.data); + } + dbc->c_close(dbc); + } + if ( (key.dptr == NULL) || (0 != return_value) ) { + inst->inst_nextid = 1; + + /*close the cache*/ + dblayer_release_id2entry( be, id2entrydb ); + + /* unlock */ + PR_Unlock( inst->inst_nextid_mutex ); + return; + } + + } + + /*close the cache*/ + dblayer_release_id2entry( be, id2entrydb ); + + /* unlock */ + PR_Unlock( inst->inst_nextid_mutex ); +} + + +/* routines to turn an internal machine-representation ID into the one we store (big-endian) */ + +void id_internal_to_stored(ID i,char *b) +{ + if ( sizeof(ID) > 4 ) { + memset (b+4, 0, sizeof(ID)-4); + } + + b[0] = (char)(i >> 24); + b[1] = (char)(i >> 16); + b[2] = (char)(i >> 8); + b[3] = (char)i; +} + +ID id_stored_to_internal(char* b) +{ + ID i; + i = (ID)b[3] & 0x000000ff; + i |= (((ID)b[2]) << 8) & 0x0000ff00; + i |= (((ID)b[1]) << 16) & 0x00ff0000; + i |= ((ID)b[0]) << 24; + return i; +} diff --git a/ldap/servers/slapd/back-ldbm/parents.c b/ldap/servers/slapd/back-ldbm/parents.c new file mode 100644 index 00000000..80fec7ac --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/parents.c @@ -0,0 +1,105 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* parents.c - where the adults live */ + +#include "back-ldbm.h" + +char *numsubordinates = "numsubordinates"; +char *hassubordinates = "hassubordinates"; + +/* Routine where any in-memory modification of a parent entry happens on some state-change in + one of its children. vaid op values are: 1 == child entry newly added, 2 == child entry about to be + deleted; 3 == child modified, in which case childmods points to the modifications. The child entry + passed is in the state which reflects the mods having been appied. op ==3 HAS NOT BEEN IMPLEMENTED YET + The routine is allowed to modify the parent entry, and to return a set of LDAPMods reflecting + the changes it made. The LDAPMods array must be freed by the called by calling ldap_free_mods(p,1) + + */ +int parent_update_on_childchange(modify_context *mc,int op, size_t *new_sub_count ) +{ + int ret = 0; + int mod_op = 0; + Slapi_Attr *read_attr = NULL; + size_t current_sub_count = 0; + int already_present = 0; + + if (new_sub_count) + *new_sub_count = 0; + + /* Check nobody is trying to use op == 3, it's not implemented yet */ + PR_ASSERT( (op == 1) || (op == 2)); + + /* We want to invent a mods set to be passed to modify_apply_mods() */ + + /* For now, we're only interested in subordinatecount. + We first examine the present value for the attribute. + If it isn't present and we're adding, we assign value 1 to the attribute and add it. + If it is present, we increment or decrement depending upon whether we're adding or deleting. + If the value after decrementing is zero, we remove it. + */ + + /* Get the present value of the subcount attr, or 0 if not present */ + ret = slapi_entry_attr_find(mc->old_entry->ep_entry,numsubordinates,&read_attr); + if (0 == ret) { + /* decode the value */ + Slapi_Value *sval; + slapi_attr_first_value( read_attr, &sval ); + if (sval!=NULL) { + const struct berval *bval = slapi_value_get_berval(sval); + if(NULL != bval) { + already_present = 1; + current_sub_count = atol(bval->bv_val); + } + } + } + /* are we adding ? */ + if ( (1 == op) && !already_present) { + /* If so, and the parent entry does not already have a subcount attribute, we need to add it */ + mod_op = LDAP_MOD_ADD; + } else { + if (2 == op) { + if (!already_present) { + /* This means that something is wrong---deleting a child but no subcount present on parent */ + LDAPDebug( LDAP_DEBUG_ANY, "numsubordinates assertion failure\n", 0, 0, 0 ); + return -1; + } else { + if (current_sub_count == 1) { + mod_op = LDAP_MOD_DELETE; + } else { + mod_op = LDAP_MOD_REPLACE; + } + } + } else { + mod_op = LDAP_MOD_REPLACE; + } + } + + /* Mow compute the new value */ + if (1 == op) { + current_sub_count++; + } else { + current_sub_count--; + } + + { + Slapi_Mods *smods= slapi_mods_new(); + if (mod_op == LDAP_MOD_DELETE) + { + slapi_mods_add(smods, mod_op | LDAP_MOD_BVALUES, numsubordinates, 0, NULL); + } + else + { + char value_buffer[20]; /* enough digits for 2^64 children */ + sprintf(value_buffer,"%lu", current_sub_count); + slapi_mods_add(smods, mod_op | LDAP_MOD_BVALUES, numsubordinates, strlen(value_buffer), value_buffer); + } + ret = modify_apply_mods(mc,smods); /* smods passed in */ + } + + if (new_sub_count) + *new_sub_count = current_sub_count; + return ret; +} diff --git a/ldap/servers/slapd/back-ldbm/perfctrs.c b/ldap/servers/slapd/back-ldbm/perfctrs.c new file mode 100644 index 00000000..0457e170 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/perfctrs.c @@ -0,0 +1,444 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* Database performance counters stuff */ +#include "back-ldbm.h" + +#include "perfctrs.h" + +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 4000 +#define TXN_STAT(env, statp, flags, malloc) \ + (env)->txn_stat((env), (statp), (flags)) +#define MEMP_STAT(env, gsp, fsp, flags, malloc) \ + (env)->memp_stat((env), (gsp), (fsp), (flags)) +#define LOG_STAT(env, spp, flags, malloc) (env)->log_stat((env), (spp), (flags)) +#define LOCK_STAT(env, statp, flags, malloc) \ + (env)->lock_stat((env), (statp), (flags)) + +#else /* older than db 4.0 */ +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 3300 +#define TXN_STAT(env, statp, flags, malloc) txn_stat((env), (statp)) +#define MEMP_STAT(env, gsp, fsp, flags, malloc) memp_stat((env), (gsp), (fsp)) +#define LOG_STAT(env, spp, flags, malloc) log_stat((env), (spp)) +#define LOCK_STAT(env, statp, flags, malloc) lock_stat((env), (statp)) + +#else /* older than db 3.3 */ +#define TXN_STAT(env, statp, flags, malloc) txn_stat((env), (statp), (malloc)) +#define MEMP_STAT(env, gsp, fsp, flags, malloc) + memp_stat((env), (gsp), (fsp), (malloc)) +#define LOG_STAT(env, spp, flags, malloc) log_stat((env), (spp), (malloc)) +#define LOCK_STAT(env, statp, flags, malloc) lock_stat((env), (statp), (malloc)) +#endif +#endif + +static void perfctrs_update(perfctrs_private *priv, DB_ENV *db_env); +static void perfctr_add_to_entry( Slapi_Entry *e, char *type, + PRUint32 countervalue ); + +/* + * Win32 specific code (to support the Windows NT/2000 Performance Monitor). + */ +#if defined(_WIN32) +static +char * string_concatenate(char *a, char* b) +{ + size_t string_length = 0; + char *string = NULL; + + string_length = strlen(a) + strlen(b) + 1; + string = malloc(string_length); + if (NULL == string) { + return string; + } + sprintf(string,"%s%s",a,b); + return string; +} + +static void init_shared_memory(perfctrs_private *priv) +{ + performance_counters *perf = (performance_counters*)priv->memory; + if (NULL != perf) { + memset(perf,sizeof(performance_counters),0); + } +} + +static int open_event(char *name, perfctrs_private *priv) +{ + HANDLE hEvent = INVALID_HANDLE_VALUE; + + hEvent = OpenEvent(EVENT_ALL_ACCESS,FALSE,name); + if (NULL == hEvent) { + hEvent = CreateEvent(NULL,FALSE,FALSE,name); + if (NULL == hEvent) { + LDAPDebug(LDAP_DEBUG_ANY,"BAD EV 1, err=%d\n",GetLastError(),0,0); + return -1; + } + } + priv->hEvent = hEvent; + return 0; +} + +static int open_shared_memory(char *name, perfctrs_private *priv) +{ + HANDLE hMapping = INVALID_HANDLE_VALUE; + void *pMemory = NULL; + /* We fear a bug in NT where it fails to attach to an existing region on calling CreateFileMapping, so let's call OpenFileMapping first */ + hMapping = OpenFileMapping(FILE_MAP_ALL_ACCESS,FALSE,name); + if (NULL == hMapping) { + hMapping = CreateFileMapping((HANDLE)0xFFFFFFFF,NULL,PAGE_READWRITE,0,sizeof(performance_counters),name); + if (NULL == hMapping) { + LDAPDebug(LDAP_DEBUG_ANY,"BAD MAP 1, err=%d\n",GetLastError(),0,0); + return -1; + } + } + /* If we got to here, we have the mapping object open */ + pMemory = MapViewOfFile(hMapping,FILE_MAP_ALL_ACCESS,0,0,0); + if (NULL == pMemory) { + LDAPDebug(LDAP_DEBUG_ANY,"BAD MAP 2, err=%d\n",GetLastError(),0,0); + return -1; + } + priv->memory = pMemory; + priv->hMemory = hMapping; + return 0; +} +#endif + +/* Init perf ctrs */ +void perfctrs_init(struct ldbminfo *li, perfctrs_private **ret_priv) +{ + perfctrs_private *priv = NULL; + +#if defined(_WIN32) + /* XXX What's my instance name ? */ + + /* + * We have a single DB environment for all backend databases. + * Therefore the instance name can be the server instance name. + * To match the db perf ctr DLL the instance name should be the + * name of a key defined in the registry under: + * HKEY_LOCAL_MACHINE\SOFTWARE\Netscape\Directory\5 + * i.e. slapd-servername + */ + + char *string = NULL; + char *instance_name = li->li_plugin->plg_name; /* XXX does not identify server instance */ +#endif + + *ret_priv = NULL; + +#if defined(_WIN32) + /* + * On Windows, the performance counters reside in shared memory. + */ + if (NULL == instance_name) { + return; + } + /* Invent the name for the shared memory region */ + string = string_concatenate(instance_name,PERFCTRS_REGION_SUFFIX); + if (NULL == string) { + return; + } +#endif + + /* + * We need the perfctrs_private area on all platforms. + */ + priv = calloc(1,sizeof(perfctrs_private)); + if (NULL == priv) { + return; + } + +#if defined(_WIN32) + /* Try to open the shared memory region */ + open_shared_memory(string,priv); + free(string); + /* Invent the name for the update mutex */ + string = string_concatenate(instance_name,PERFCTRS_MUTEX_SUFFIX); + if (NULL == string) { + return; + } + open_event(string,priv); + free(string); + init_shared_memory(priv); + +#else + /* + * On other platforms, the performance counters reside in regular memory. + */ + if ( NULL == ( priv->memory = calloc( 1, sizeof( performance_counters )))) { + return; + } +#endif + + *ret_priv = priv; +} + +/* Terminate perf ctrs */ +void perfctrs_terminate(perfctrs_private **priv) +{ +#if defined(_WIN32) + if (NULL != (*priv)->memory) { + UnmapViewOfFile((*priv)->memory); + } + if (NULL != (*priv)->hMemory) { + CloseHandle((*priv)->hMemory); + } + if (NULL != (*priv)->hEvent) { + CloseHandle((*priv)->hEvent); + } +#else + if (NULL != (*priv)->memory) { + free((*priv)->memory); + } +#endif + + free( (*priv) ); + (*priv) = NULL; +} + +/* Wait while checking for perfctr update requests */ +void perfctrs_wait(size_t milliseconds,perfctrs_private *priv,DB_ENV *db_env) +{ +#if defined(_WIN32) + if (NULL != priv) { + DWORD ret = 0; + if (NULL != priv->hEvent) { + /* Sleep waiting on the perfctrs update event */ + ret = WaitForSingleObject(priv->hEvent,milliseconds); + /* If we didn't time out, update the perfctrs */ + if (ret == WAIT_OBJECT_0) { + perfctrs_update(priv,db_env); + } + } else { + Sleep(milliseconds); + } + } +#else + /* Just sleep */ + PRIntervalTime interval; /*NSPR timeout stuffy*/ + interval = PR_MillisecondsToInterval(milliseconds); + DS_Sleep(interval); +#endif +} + +/* Update perfctrs */ +static +void perfctrs_update(perfctrs_private *priv, DB_ENV *db_env) +{ + int ret = 0; + performance_counters *perf; + if (NULL == priv) { + return; + } + if (NULL == db_env) { + return; + } + perf = (performance_counters*)priv->memory; + if (NULL == perf) { + return; + } + /* Call libdb to get the various stats */ + if (NULL != db_env->lg_handle) + { + DB_LOG_STAT *logstat = NULL; + ret = LOG_STAT(db_env,&logstat,0,malloc); + if (0 == ret) { + perf->log_region_wait_rate = logstat->st_region_wait; + perf->log_write_rate = 1024*1024*logstat->st_w_mbytes + logstat->st_w_bytes; + perf->log_bytes_since_checkpoint = 1024*1024*logstat->st_wc_mbytes + logstat->st_wc_bytes; + } + free(logstat); + } + if (NULL != db_env->tx_handle) + { + DB_TXN_STAT *txnstat = NULL; + ret = TXN_STAT(db_env, &txnstat, 0, malloc); + if (0 == ret) { + perf->active_txns = txnstat->st_nactive; + perf->commit_rate = txnstat->st_ncommits; + perf->abort_rate = txnstat->st_naborts; + perf->txn_region_wait_rate = txnstat->st_region_wait; + } + if (txnstat) + free(txnstat); + } + if (NULL != db_env->lk_handle) + { + DB_LOCK_STAT *lockstat = NULL; + ret = LOCK_STAT(db_env,&lockstat,0,malloc); + if (0 == ret) { + perf->lock_region_wait_rate = lockstat->st_region_wait; + perf->deadlock_rate = lockstat->st_ndeadlocks; + perf->configured_locks = lockstat->st_maxlocks; + perf->current_locks = lockstat->st_nlocks; + perf->max_locks = lockstat->st_maxnlocks; + perf->lockers = lockstat->st_nlockers; + perf->lock_conflicts = lockstat->st_nconflicts; + perf->lock_request_rate = lockstat->st_nrequests; + perf->current_lock_objects = lockstat->st_nobjects; + perf->max_lock_objects = lockstat->st_maxnobjects; + } + free(lockstat); + } + if (NULL != db_env->mp_handle) + { + DB_MPOOL_STAT *mpstat = NULL; + ret = MEMP_STAT(db_env,&mpstat,NULL,0,malloc); + if (0 == ret) { +#define ONEG 1073741824 + perf->cache_size_bytes = mpstat->st_gbytes * ONEG + mpstat->st_bytes; + perf->page_access_rate = mpstat->st_cache_hit + mpstat->st_cache_miss; + perf->cache_hit = mpstat->st_cache_hit; + perf->cache_try = mpstat->st_cache_hit + mpstat->st_cache_miss; + perf->page_create_rate = mpstat->st_page_create; + perf->page_read_rate = mpstat->st_page_in; + perf->page_write_rate = mpstat->st_page_out; + perf->page_ro_evict_rate = mpstat->st_ro_evict; + perf->page_rw_evict_rate = mpstat->st_rw_evict; + perf->hash_buckets = mpstat->st_hash_buckets; + perf->hash_search_rate = mpstat->st_hash_searches; + perf->longest_chain_length = mpstat->st_hash_longest; + perf->hash_elements_examine_rate = mpstat->st_hash_examined; + perf->pages_in_use = mpstat->st_page_dirty + mpstat->st_page_clean; + perf->dirty_pages = mpstat->st_page_dirty; + perf->clean_pages = mpstat->st_page_clean; + perf->page_trickle_rate = mpstat->st_page_trickle; + perf->cache_region_wait_rate = mpstat->st_region_wait; + free(mpstat); + } + } + /* Place the stats in the shared memory region */ + /* Bump the sequence number */ + perf->sequence_number++; +} + + + +/* + * Define a map (array of structures) which is used to retrieve performance + * counters from the performance_counters structure and map them to an + * LDAP attribute type. + */ + +#define SLAPI_LDBM_PERFCTR_AT_PREFIX "nsslapd-db-" +typedef struct slapi_ldbm_perfctr_at_map { + char *pam_type; /* name of LDAP attribute type */ + size_t pam_offset; /* offset into performance_counters struct */ +} SlapiLDBMPerfctrATMap; + +static SlapiLDBMPerfctrATMap perfctr_at_map[] = { + { SLAPI_LDBM_PERFCTR_AT_PREFIX "abort-rate", + offsetof( performance_counters, abort_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "active-txns", + offsetof( performance_counters, active_txns ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "cache-hit", + offsetof( performance_counters, cache_hit ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "cache-try", + offsetof( performance_counters, cache_try ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "cache-region-wait-rate", + offsetof( performance_counters, cache_region_wait_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "cache-size-bytes", + offsetof( performance_counters, cache_size_bytes ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "clean-pages", + offsetof( performance_counters, clean_pages ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "commit-rate", + offsetof( performance_counters, commit_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "deadlock-rate", + offsetof( performance_counters, deadlock_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "dirty-pages", + offsetof( performance_counters, dirty_pages ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "hash-buckets", + offsetof( performance_counters, hash_buckets ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "hash-elements-examine-rate", + offsetof( performance_counters, hash_elements_examine_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "hash-search-rate", + offsetof( performance_counters, hash_search_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "lock-conflicts", + offsetof( performance_counters, lock_conflicts ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "lock-region-wait-rate", + offsetof( performance_counters, lock_region_wait_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "lock-request-rate", + offsetof( performance_counters, lock_request_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "lockers", + offsetof( performance_counters, lockers ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "configured-locks", + offsetof( performance_counters, configured_locks ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "current-locks", + offsetof( performance_counters, current_locks ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "max-locks", + offsetof( performance_counters, max_locks ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "current-lock-objects", + offsetof( performance_counters, current_lock_objects ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "max-lock-objects", + offsetof( performance_counters, max_lock_objects ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "log-bytes-since-checkpoint", + offsetof( performance_counters, log_bytes_since_checkpoint ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "log-region-wait-rate", + offsetof( performance_counters, log_region_wait_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "log-write-rate", + offsetof( performance_counters, log_write_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "longest-chain-length", + offsetof( performance_counters, longest_chain_length ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "objects-locked", + offsetof( performance_counters, page_access_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "page-create-rate", + offsetof( performance_counters, page_create_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "page-read-rate", + offsetof( performance_counters, page_read_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "page-ro-evict-rate", + offsetof( performance_counters, page_ro_evict_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "page-rw-evict-rate", + offsetof( performance_counters, page_rw_evict_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "page-trickle-rate", + offsetof( performance_counters, page_trickle_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "page-write-rate", + offsetof( performance_counters, page_write_rate ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "pages-in-use", + offsetof( performance_counters, pages_in_use ) }, + { SLAPI_LDBM_PERFCTR_AT_PREFIX "txn-region-wait-rate", + offsetof( performance_counters, txn_region_wait_rate ) }, +}; +#define SLAPI_LDBM_PERFCTR_AT_MAP_COUNT \ + (sizeof(perfctr_at_map) / sizeof(SlapiLDBMPerfctrATMap)) + + +/* + * Set attributes and values in entry `e' based on performance counter + * information (from `priv'). + */ +void +perfctrs_as_entry( Slapi_Entry *e, perfctrs_private *priv, DB_ENV *db_env ) +{ + performance_counters *perf; + int i; + + if (priv == NULL) return; + + perf = (performance_counters*)priv->memory; + + /* + * First, update the values so they are current. + */ + perfctrs_update( priv, db_env ); + + /* + * Then convert all the counters to attribute values. + */ + for ( i = 0; i < SLAPI_LDBM_PERFCTR_AT_MAP_COUNT; ++i ) { + perfctr_add_to_entry( e, perfctr_at_map[i].pam_type, + *((PRUint32 *)((char *)perf + perfctr_at_map[i].pam_offset))); + } +} + + +static void +perfctr_add_to_entry( Slapi_Entry *e, char *type, PRUint32 countervalue ) +{ + /* + * XXXmcs: the following line assumes that long's are 32 bits or larger, + * which we assume in other places too I am sure. + */ + slapi_entry_attr_set_ulong( e, type, (unsigned long)countervalue ); +} diff --git a/ldap/servers/slapd/back-ldbm/perfctrs.h b/ldap/servers/slapd/back-ldbm/perfctrs.h new file mode 100644 index 00000000..8aed8d55 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/perfctrs.h @@ -0,0 +1,51 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* Structure definition for performance data */ +/* This stuff goes in shared memory, so make sure the packing is consistent */ + +struct _performance_counters { + PRUint32 sequence_number; + PRUint32 lock_region_wait_rate; + PRUint32 deadlock_rate; + PRUint32 configured_locks; + PRUint32 current_locks; + PRUint32 max_locks; + PRUint32 lockers; + PRUint32 current_lock_objects; + PRUint32 max_lock_objects; + PRUint32 lock_conflicts; + PRUint32 lock_request_rate; + PRUint32 log_region_wait_rate; + PRUint32 log_write_rate; + PRUint32 log_bytes_since_checkpoint; + PRUint32 cache_size_bytes; + PRUint32 page_access_rate; + PRUint32 cache_hit; + PRUint32 cache_try; + PRUint32 page_create_rate; + PRUint32 page_read_rate; + PRUint32 page_write_rate; + PRUint32 page_ro_evict_rate; + PRUint32 page_rw_evict_rate; + PRUint32 hash_buckets; + PRUint32 hash_search_rate; + PRUint32 longest_chain_length; + PRUint32 hash_elements_examine_rate; + PRUint32 pages_in_use; + PRUint32 dirty_pages; + PRUint32 clean_pages; + PRUint32 page_trickle_rate; + PRUint32 cache_region_wait_rate; + PRUint32 active_txns; + PRUint32 commit_rate; + PRUint32 abort_rate; + PRUint32 txn_region_wait_rate; +}; +typedef struct _performance_counters performance_counters; + +#define PERFCTRS_REGION_SUFFIX "-sm" +#define PERFCTRS_MUTEX_SUFFIX "-mx" + diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h new file mode 100644 index 00000000..4e8ed4df --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h @@ -0,0 +1,582 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2004 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +#ifndef _PROTO_BACK_LDBM +#define _PROTO_BACK_LDBM + +/* + * attr.c + */ +struct attrinfo * attrinfo_new(); +void attrinfo_delete(struct attrinfo **pp); +void ainfo_get( backend *be, char *type, struct attrinfo **at ); +void attr_masks( backend *be, char *type, int *indexmask, + int *syntaxmask ); +void attr_masks_ex( backend *be, char *type, int *indexmask, + int *syntaxmask, struct attrinfo **at ); +void attr_index_config( backend *be, char *fname, int lineno, + int argc, char **argv, int init ); +int ldbm_compute_init(); +void attrinfo_deletetree(ldbm_instance *inst); +void attr_create_empty(backend *be,char *type,struct attrinfo **ai); + +/* + * cache.c + */ +int cache_init(struct cache *cache, size_t maxsize, long maxentries); +void cache_clear(struct cache *cache); +void cache_destroy_please(struct cache *cache); +void cache_set_max_size(struct cache *cache, size_t bytes); +void cache_set_max_entries(struct cache *cache, long entries); +size_t cache_get_max_size(struct cache *cache); +long cache_get_max_entries(struct cache *cache); +void cache_get_stats(struct cache *cache, u_long *hits, u_long *tries, + long *entries,long *maxentries, + size_t *size, size_t *maxsize); +void cache_debug_hash(struct cache *cache, char **out); +int cache_remove(struct cache *cache, struct backentry *e); +void cache_return(struct cache *cache, struct backentry **bep); +struct backentry *cache_find_dn(struct cache *cache, const char *dn, unsigned long ndnlen); +struct backentry *cache_find_id(struct cache *cache, ID id); +struct backentry *cache_find_uuid(struct cache *cache, const char *uuid); +int cache_add(struct cache *cache, struct backentry *e, + struct backentry **alt); +int cache_add_tentative(struct cache *cache, struct backentry *e, + struct backentry **alt); +int cache_lock_entry(struct cache *cache, struct backentry *e); +void cache_unlock_entry(struct cache *cache, struct backentry *e); +int cache_replace(struct cache *cache, struct backentry *olde, + struct backentry *newe); + +Hashtable *new_hash(u_long size, u_long offset, HashFn hfn, + HashTestFn tfn); +int add_hash(Hashtable *ht, void *key, size_t keylen, void *entry, + void **alt); +int find_hash(Hashtable *ht, const void *key, size_t keylen, void **entry); +int remove_hash(Hashtable *ht, const void *key, size_t keylen); + +/* + * dblayer.c + */ +int dblayer_init(struct ldbminfo *li); +int dblayer_terminate(struct ldbminfo *li); +int dblayer_start(struct ldbminfo *li, int dbmode); +int dblayer_flush(struct ldbminfo *li ); +int dblayer_close(struct ldbminfo *li, int dbmode ); +void dblayer_pre_close(struct ldbminfo *li); +int dblayer_post_close(struct ldbminfo *li, int dbmode ); +int dblayer_instance_close(backend *be); +int dblayer_get_index_file(backend *be,struct attrinfo *a, DB** ppDB, int create); +int dblayer_release_index_file(backend *be,struct attrinfo *a, DB* pDB); +int dblayer_erase_index_file(backend *be, struct attrinfo *a, int no_force_chkpt); +int dblayer_erase_index_file_nolock(backend *be, struct attrinfo *a, int no_force_chkpt); +int dblayer_get_id2entry(backend *be, DB **ppDB); +int dblayer_release_id2entry(backend *be, DB *pDB); +int dblayer_get_aux_id2entry(backend *be, DB **ppDB, DB_ENV **ppEnv); +int dblayer_release_aux_id2entry(backend *be, DB *pDB, DB_ENV *pEnv); +int dblayer_txn_init(struct ldbminfo *li, back_txn *txn); +int dblayer_txn_begin(struct ldbminfo *li,back_txnid parent_txn, back_txn *txn); +int dblayer_txn_commit(struct ldbminfo *li, back_txn *txn); +int dblayer_txn_abort(struct ldbminfo *li, back_txn *txn); +int dblayer_read_txn_abort(struct ldbminfo *li, back_txn *txn); +int dblayer_read_txn_begin(struct ldbminfo *li,back_txnid parent_txn, back_txn *txn); +int dblayer_read_txn_commit(struct ldbminfo *li, back_txn *txn); +size_t dblayer_get_optimal_block_size(struct ldbminfo *li); +void dblayer_unlock_backend(backend *be); +void dblayer_lock_backend(backend *be); +int dblayer_plugin_begin(Slapi_PBlock *pb); +int dblayer_plugin_commit(Slapi_PBlock *pb); +int dblayer_plugin_abort(Slapi_PBlock *pb); +int dblayer_memp_stat(struct ldbminfo *li, DB_MPOOL_STAT **gsp,DB_MPOOL_FSTAT ***fsp); +int dblayer_memp_stat_instance(ldbm_instance *inst, DB_MPOOL_STAT **gsp, DB_MPOOL_FSTAT ***fsp); +int dblayer_backup(struct ldbminfo *li, char *destination_directory, + Slapi_Task *task); +int dblayer_restore(struct ldbminfo *li, char* source_directory, Slapi_Task *task); +int dblayer_copy_directory(struct ldbminfo *li, Slapi_Task *task, + char *instance_dir, char *destination_dir, + int restore, int *cnt, int instance_dir_flag, + int indexonly); +int dblayer_copyfile(char* source, char * destination, int overwrite, int mode); +int dblayer_delete_instance_dir(backend *be); +int dblayer_delete_database(struct ldbminfo *li); +int dblayer_database_size(struct ldbminfo *li, unsigned int *size); +int dblayer_terminate(struct ldbminfo *li); +int dblayer_close_indexes(backend *be); +int dblayer_open_file(backend *be, char* indexname, int create, int index_flags, DB **ppDB); +int dblayer_close_file(DB *db); +void dblayer_sys_pages(size_t *pagesize, size_t *pages, size_t *procpages, size_t *availpages); +int dblayer_is_cachesize_sane(size_t *cachesize); +void dblayer_remember_disk_filled(struct ldbminfo *li); +int dblayer_open_huge_file(const char *path, int oflag, int mode); +int dblayer_instance_start(backend *be, int normal_mode); +int dblayer_make_new_instance_data_dir(backend *be); +int dblayer_get_instance_data_dir(backend *be); +char *dblayer_strerror(int error); +PRInt64 db_atol(char *str, int *err); +PRInt64 db_atoi(char *str, int *err); +unsigned long db_strtoul(const char *str, int *err); +int dblayer_set_batch_transactions(void *arg, void *value, char *errorbuf, int phase, int apply); +void *dblayer_get_batch_transactions(void *arg); +int dblayer_in_import(ldbm_instance *inst); + +int dblayer_update_db_ext(ldbm_instance *inst, char *oldext, char *newext); +void dblayer_set_recovery_required(struct ldbminfo *li); + +char *dblayer_get_home_dir(struct ldbminfo *li, int *dbhome); +char *dblayer_get_full_inst_dir(struct ldbminfo *li, ldbm_instance *inst, + char *buf, int buflen); +void autosize_import_cache(struct ldbminfo *li); + + +/* + * dn2entry.c + */ +struct backentry *dn2entry(Slapi_Backend *be, const Slapi_DN *sdn, back_txn *txn, int *err); +struct backentry *dn2entry_or_ancestor(Slapi_Backend *be, const Slapi_DN *sdn, Slapi_DN *ancestor, back_txn *txn, int *err); +struct backentry *dn2ancestor(Slapi_Backend *be,const Slapi_DN *sdn,Slapi_DN *ancestordn,back_txn *txn,int *err); +int get_copy_of_entry(Slapi_PBlock *pb, const entry_address *addr, back_txn *txn, int plock_parameter, int must_exist); +void done_with_pblock_entry(Slapi_PBlock *pb, int plock_parameter); + +/* + * uniqueid2entry.c + */ +struct backentry * uniqueid2entry(backend *be, const char *uniqueid, + back_txn *txn, int *err); + +/* + * filterindex.c + */ +IDList * filter_candidates( Slapi_PBlock *pb, backend *be, const char *base, Slapi_Filter *f, Slapi_Filter *nextf, int range, int *err ); + +/* + * findentry.c + */ +struct backentry * find_entry2modify( Slapi_PBlock *pb, Slapi_Backend *be, const entry_address *addr, back_txn *txn ); +struct backentry * find_entry( Slapi_PBlock *pb, Slapi_Backend *be, const entry_address *addr, back_txn *txn ); +struct backentry * find_entry2modify_only( Slapi_PBlock *pb, Slapi_Backend *be, const entry_address *addr, back_txn *txn); +struct backentry * find_entry_only( Slapi_PBlock *pb, Slapi_Backend *be, const entry_address *addr, back_txn *txn); +int check_entry_for_referral(Slapi_PBlock *pb, Slapi_Entry *entry, char *matched, const char *callingfn); + +/* + * haschildren.c + */ +int has_children( struct ldbminfo *li, struct backentry *p, back_txn *txn, int *err ); + +/* + * id2entry.c + */ +int id2entry_add( backend *be, struct backentry *e, back_txn *txn ); +int id2entry_add_ext( backend *be, struct backentry *e, back_txn *txn, int encrypt ); +int id2entry_delete( backend *be, struct backentry *e, back_txn *txn ); +struct backentry * id2entry( backend *be, ID id, back_txn *txn, int *err ); + +/* + * idl.c + */ +IDList * idl_alloc( NIDS nids ); +void idl_free( IDList *idl ); +NIDS idl_length(IDList *idl); +int idl_is_allids(IDList *idl); +int idl_append( IDList *idl, ID id); +void idl_insert(IDList **idl, ID id); +IDList * idl_allids( backend *be ); +IDList * idl_fetch( backend *be, DB* db, DBT *key, DB_TXN *txn, struct attrinfo *a, int *err ); +int idl_insert_key( backend *be, DB* db, DBT *key, ID id, DB_TXN *txn, struct attrinfo *a,int *disposition ); +int idl_delete_key( backend *be, DB *db, DBT *key, ID id, DB_TXN *txn, struct attrinfo *a ); +IDList * idl_intersection( backend *be, IDList *a, IDList *b ); +IDList * idl_union( backend *be, IDList *a, IDList *b ); +int idl_notin( backend *be, IDList *a, IDList *b , IDList **new_result); +ID idl_firstid( IDList *idl ); +ID idl_nextid( IDList *idl, ID id ); +int idl_init_private(backend *be, struct attrinfo *a); +int idl_release_private(struct attrinfo *a); + +idl_iterator idl_iterator_init(const IDList *idl); +idl_iterator idl_iterator_increment(idl_iterator *i); +idl_iterator idl_iterator_decrement(idl_iterator *i); +ID idl_iterator_dereference(idl_iterator i, const IDList *idl); +ID idl_iterator_dereference_increment(idl_iterator *i, const IDList *idl); +size_t idl_sizeof(IDList *idl); +int idl_store_block(backend *be,DB *db,DBT *key,IDList *idl,DB_TXN *txn,struct attrinfo *a); +void idl_set_tune(int val); +int idl_get_tune(); +size_t idl_get_allidslimit(struct attrinfo *a); +int idl_get_idl_new(); +int idl_new_compare_dups( +#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 3200 + DB *db, +#endif + const DBT *a, + const DBT *b +); + +/* + * index.c + */ +int index_addordel_entry( backend *be, struct backentry *e, int flags, back_txn *txn ); +int index_add_mods( backend *be, const LDAPMod**mods, struct backentry *olde, struct backentry *newe, back_txn *txn ); +int index_addordel_string(backend *be, const char *type, const char *s, ID id, int flags, back_txn *txn); +int index_addordel_values_sv( backend *be, const char *type, Slapi_Value **vals, Slapi_Value **evals, ID id, int flags, back_txn *txn ); +int index_addordel_values_ext_sv( backend *be, const char *type, Slapi_Value **vals, Slapi_Value **evals, ID id, int flags, back_txn *txn,int *idl_disposition, void *buffer_handle ); +int id_array_init(Id_Array *new_guy, int size); + +IDList* index_read( backend *be, char *type, const char* indextype, const struct berval* val, back_txn *txn, int *err ); +IDList* index_read_ext( backend *be, char *type, const char* indextype, const struct berval* val, back_txn *txn, int *err, int *unindexed ); +IDList* index_range_read( Slapi_PBlock *pb, backend *be, char *type, const char* indextype, int ftype, struct berval* val, struct berval* nextval, int range, back_txn *txn, int *err ); +const char *encode( const struct berval* data, char buf[BUFSIZ] ); + +extern const char* indextype_PRESENCE; +extern const char* indextype_EQUALITY; +extern const char* indextype_APPROX; +extern const char* indextype_SUB; + +int index_buffer_init(size_t size,int flags,void **h); +int index_buffer_flush(void *h,backend *be, DB_TXN *txn,struct attrinfo *a); +int index_buffer_terminate(void *h); + +/* + * instance.c + */ +int ldbm_instance_create(backend *be, char *name); +int ldbm_instance_create_default_indexes(backend *be); +int ldbm_instance_start(backend *be); +int ldbm_instance_stop(backend *be); +int ldbm_instance_startall(struct ldbminfo *li); +int ldbm_instance_stopall(struct ldbminfo *li); +ldbm_instance *ldbm_instance_find_by_name(struct ldbminfo *li, char *name); +int ldbm_instance_destroy(ldbm_instance *inst); + +/* + * ldif2ldbm.c + */ +int import_subcount_mother_init(import_subcount_stuff *mothers,ID parent_id, size_t count); +int import_subcount_mother_count(import_subcount_stuff *mothers,ID parent_id); +void import_subcount_stuff_init(import_subcount_stuff *stuff); +void import_subcount_stuff_term(import_subcount_stuff *stuff); +int update_subordinatecounts(backend *be,import_subcount_stuff *mothers, DB_TXN *txn); +void import_configure_index_buffer_size(size_t size); +size_t import_get_index_buffer_size(); +int ldbm_back_fetch_incl_excl(Slapi_PBlock *pb, char ***include, + char ***exclude); +void ldbm_back_free_incl_excl(char **include, char **exclude); +int ldbm_back_ok_to_dump(const char *dn, char **include, char **exclude); +int ldbm_back_wire_import(Slapi_PBlock *pb); +void *factory_constructor(void *object, void *parent); +void factory_destructor(void *extension, void *object, void *parent); + +/* + * modify.c + */ +int modify_update_all(backend *be, Slapi_PBlock *pb,modify_context *mc,back_txn *txn); +void modify_init(modify_context *mc,struct backentry *old_entry); +int modify_apply_mods(modify_context *mc, Slapi_Mods *smods); +int modify_term(modify_context *mc,backend *be); +int modify_switch_entries(modify_context *mc,backend *be); + +/* + * add.c + */ +void add_update_entry_operational_attributes(struct backentry *ep, ID pid); +void add_update_entrydn_operational_attributes(struct backentry *ep); + +/* + * misc.c + */ +void ldbm_nasty(const char* str, int c, int err); +void ldbm_log_access_message(Slapi_PBlock *pblock,char *string); +int return_on_disk_full(struct ldbminfo *li); +int ldbm_attribute_always_indexed(const char *attrtype); +void ldbm_destroy_instance_name(struct ldbminfo *li); +char *compute_entry_tombstone_dn(const char *entrydn, const char *uniqueid); +int instance_set_busy(ldbm_instance *inst); +int instance_set_busy_and_readonly(ldbm_instance *inst); +void instance_set_not_busy(ldbm_instance *inst); +void allinstance_set_busy(struct ldbminfo *li); +void allinstance_set_not_busy(struct ldbminfo *li); +int is_anyinstance_busy(struct ldbminfo *li); +int ldbm_delete_dirs(char *path); +int mkdir_p(char *dir, unsigned int mode); +int is_fullpath(char *path); +char get_sep(char *path); + +/* + * nextid.c + */ +ID next_id( backend *be ); +void next_id_return( backend *be, ID id ); +ID next_id_get( backend *be ); +void id_internal_to_stored(ID,char*); +ID id_stored_to_internal(char*); +#if 0 +int write_dbversion( ldbm_instance *inst ); +#endif +void get_ids_from_disk(backend *be); +void get_both_ids( struct ldbminfo *li, ID *nextid, ID *nextid2index ); + +/* + * backentry.c + */ +struct backentry *backentry_init( Slapi_Entry *e ); +struct backentry *backentry_alloc(); +void backentry_free( struct backentry **bep ); +struct backentry *backentry_dup( struct backentry * ); +void backentry_clear_entry( struct backentry * ); +char *backentry_get_ndn(const struct backentry *e); +const Slapi_DN *backentry_get_sdn(const struct backentry *e); + +/* + * parents.c + */ +int parent_update_on_childchange(modify_context *mc,int op, size_t *numofchildren); + +/* + * perfctrs.c + */ +void perfctrs_wait(size_t milliseconds,perfctrs_private *priv,DB_ENV *db_env); +void perfctrs_init(struct ldbminfo *li,perfctrs_private **priv); +void perfctrs_terminate(perfctrs_private **priv); +void perfctrs_as_entry( Slapi_Entry *e, perfctrs_private *priv, DB_ENV *db_env ); + +/* + * rmdb.c + */ +int ldbm_back_rmdb( Slapi_PBlock *pb ); + +/* + * sort.c + */ + +/* + * Definitions for sort spec object + */ +struct sort_spec_thing +{ + char *type; + char *matchrule; /* Matching rule string */ + int order; /* 0 == ascending, 1 == decending */ + struct sort_spec_thing *next; /* Link to the next one */ + Slapi_PBlock *mr_pb; /* For matchrule indexing */ + value_compare_fn_type compare_fn; /* For non-matchrule indexing */ +}; +typedef struct sort_spec_thing sort_spec_thing; +typedef struct sort_spec_thing sort_spec; + +void sort_spec_free(sort_spec *s); +int sort_candidates(backend *be, int lookthrough_limit, time_t time_up, Slapi_PBlock *pb, IDList *candidates, sort_spec_thing *sort_spec, char **sort_error_type) ; +int make_sort_response_control ( Slapi_PBlock *pb, int code, char *error_type); +int parse_sort_spec(struct berval *sort_spec_ber, sort_spec **ps); +struct berval* attr_value_lowest(struct berval **values, value_compare_fn_type compare_fn); +int sort_attr_compare(struct berval ** value_a, struct berval ** value_b, value_compare_fn_type compare_fn); +void sort_log_access(Slapi_PBlock *pb,sort_spec_thing *s,IDList *candidates); + +/* + * dbsize.c + */ +int ldbm_db_size( Slapi_PBlock *pb ); + +/* + * external functions + */ +int ldbm_back_bind( Slapi_PBlock *pb ); +int ldbm_back_unbind( Slapi_PBlock *pb ); +int ldbm_back_search( Slapi_PBlock *pb ); +int ldbm_back_compare( Slapi_PBlock *pb ); +int ldbm_back_modify( Slapi_PBlock *pb ); +int ldbm_back_modrdn( Slapi_PBlock *pb ); +int ldbm_back_add( Slapi_PBlock *pb ); +int ldbm_back_delete( Slapi_PBlock *pb ); +int ldbm_back_abandon( Slapi_PBlock *pb ); +int ldbm_back_config( Slapi_PBlock *pb ); +int ldbm_back_close( Slapi_PBlock *pb ); +int ldbm_back_cleanup( Slapi_PBlock *pb ); +void ldbm_back_instance_set_destructor(void **arg); +int ldbm_back_flush( Slapi_PBlock *pb ); +int ldbm_back_start( Slapi_PBlock *pb ); +int ldbm_back_seq( Slapi_PBlock *pb ); +int ldbm_back_ldif2ldbm( Slapi_PBlock *pb ); +int ldbm_back_ldbm2ldif( Slapi_PBlock *pb ); +int ldbm_back_ldbm2ldifalt( Slapi_PBlock *pb ); +int ldbm_back_ldbm2index( Slapi_PBlock *pb ); +int ldbm_back_archive2ldbm( Slapi_PBlock *pb ); +int ldbm_back_ldbm2archive( Slapi_PBlock *pb ); +#if defined(UPGRADEDB) +int ldbm_back_upgradedb( Slapi_PBlock *pb ); +#endif +int ldbm_back_next_search_entry( Slapi_PBlock *pb ); +int ldbm_back_next_search_entry_ext( Slapi_PBlock *pb, int use_extension ); +int ldbm_back_db_test( Slapi_PBlock *pb ); +int ldbm_back_entry_release( Slapi_PBlock *pb, void *backend_info_ptr ); +int ldbm_back_init( Slapi_PBlock *pb ); + +/* + * monitor.c + */ + +int ldbm_back_monitor_search(Slapi_PBlock *pb, Slapi_Entry* e, + Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_back_monitor_instance_search(Slapi_PBlock *pb, Slapi_Entry *e, + Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_back_dbmonitor_search(Slapi_PBlock *pb, Slapi_Entry *e, + Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); + +/* + * vlv.c + */ +struct vlv_request +{ + unsigned long beforeCount; + unsigned long afterCount; + unsigned long tag; + unsigned long index; + unsigned long contentCount; + struct berval value; +}; + +struct vlv_response +{ + unsigned long targetPosition; + unsigned long contentCount; + unsigned long result; +}; + +int vlv_init(ldbm_instance *inst); +int vlv_remove_callbacks(ldbm_instance *inst); +const Slapi_Entry **vlv_get_search_entries(); +struct vlvIndex* vlv_find_searchname(const char * name, backend *be); +struct vlvIndex* vlv_find_indexname(const char * name, backend *be); +char *vlv_getindexnames(); +int vlv_search_build_candidate_list(Slapi_PBlock *pb, const Slapi_DN *base, int *rc, const sort_spec* sort_control, + const struct vlv_request *vlv_request_control, IDList** candidates, struct vlv_response *vlv_response_control); +int vlv_update_index(struct vlvIndex* p, back_txn *txn, struct ldbminfo *li, Slapi_PBlock *pb, struct backentry* oldEntry, struct backentry* newEntry); +int vlv_update_all_indexes(back_txn *txn, backend *be, Slapi_PBlock *pb, struct backentry* oldEntry, struct backentry* newEntry); +int vlv_filter_candidates(backend *be, Slapi_PBlock *pb, const IDList *candidates, const Slapi_DN *base, int scope, Slapi_Filter *filter, IDList** filteredCandidates,int lookthrough_limit, time_t time_up); +int vlv_trim_candidates(backend *be, const IDList *candidates, const sort_spec* sort_control, const struct vlv_request *vlv_request_control, IDList** filteredCandidates,struct vlv_response *pResponse); +int vlv_parse_request_control(backend *be, struct berval *vlv_spec_ber, struct vlv_request* vlvp); +int vlv_make_response_control(Slapi_PBlock *pb, const struct vlv_response* vlvp); +void vlv_getindices(IFP callback_fn,void *param, backend *be); +void vlv_print_access_log(Slapi_PBlock *pb,struct vlv_request* vlvi, struct vlv_response *vlvo); +void vlv_grok_new_import_entry(const struct backentry *e, backend *be); +IDList *vlv_find_index_by_filter(struct backend *be, const char *base, + Slapi_Filter *f); +int vlv_delete_search_entry(Slapi_PBlock *pb, Slapi_Entry* e, ldbm_instance *inst); +void vlv_acquire_lock(backend *be); +void vlv_release_lock(backend *be); +int vlv_isvlv(char *filename); + +/* + * Indexfile.c + */ +int indexfile_delete_all_keys(backend *be,char* type,back_txn *txn); +int indexfile_primary_modifyall(backend *be, LDAPMod **mods_to_perform,char **indexes_to_update,back_txn *txn); + +/* + * bedse.c + */ +#if 0 +int bedse_init(); +int bedse_search(Slapi_PBlock *pb); +struct dse_callback *bedse_register_callback(int operation, const Slapi_DN *base, int scope, const char *filter, int (*fn)(Slapi_PBlock *,Slapi_Entry *,Slapi_Entry *,int*,char*,void *), void *fn_arg); +void bedse_remove_callback(int operation, const Slapi_DN *base, int scope, const char *filter, int (*fn)(Slapi_PBlock *,Slapi_Entry *,Slapi_Entry *,int*,char*,void *)); +int bedse_add_index_entry(int argc, char **argv); +#endif + +/* + * search.c + */ +Slapi_Filter* create_onelevel_filter(Slapi_Filter* filter, const struct backentry *e, int managedsait, Slapi_Filter** fid2kids, Slapi_Filter** focref, Slapi_Filter** fand, Slapi_Filter** forr); +Slapi_Filter* create_subtree_filter(Slapi_Filter* filter, int managedsait, Slapi_Filter** focref, Slapi_Filter** forr); +IDList* subtree_candidates(Slapi_PBlock *pb, backend *be, const char *base, const struct backentry *e, Slapi_Filter *filter, int managedsait, int *allids_before_scopingp, int *err); +void search_set_tune(struct ldbminfo *li,int val); +int search_get_tune(struct ldbminfo *li); + +/* + * matchrule.c + */ +int create_matchrule_indexer(Slapi_PBlock **pb,char* matchrule,char* type); +int destroy_matchrule_indexer(Slapi_PBlock *pb); +int matchrule_values_to_keys(Slapi_PBlock *pb,struct berval **input_values,struct berval ***output_values); +int matchrule_values_to_keys_sv(Slapi_PBlock *pb,Slapi_Value **input_values, Slapi_Value ***output_values); + +/* + * upgrade.c + */ +int check_db_version(struct ldbminfo *li, int *action); +int check_db_inst_version(ldbm_instance *inst); +#if defined(UPGRADEDB) +int adjust_idl_switch(char *ldbmversion, struct ldbminfo *li); +#endif +int ldbm_upgrade(ldbm_instance *inst, int action); +int lookup_dbversion(char *dbversion, int flag); + + +/* + * init.c + */ +int ldbm_attribute_always_indexed(const char *attrtype); + +/* + * dbversion.c + */ +int dbversion_write(struct ldbminfo *li, const char *dir, const char *dversion); +int dbversion_read(struct ldbminfo *li, const char *directory, + char *ldbmversion, char *dataversion); +int dbversion_exists(struct ldbminfo *li, const char *directory); + +/* + * config_ldbm.c + */ +int ldbm_config_load_dse_info(struct ldbminfo *li); +void ldbm_config_setup_default(struct ldbminfo *li); +void ldbm_config_internal_set(struct ldbminfo *li, char *attrname, char *value); +void ldbm_instance_config_internal_set(ldbm_instance *inst, char *attrname, char *value); +void ldbm_instance_config_setup_default(ldbm_instance *inst); +int ldbm_instance_postadd_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_instance_add_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_instance_delete_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_instance_post_delete_instance_entry_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); +/* Index config functions */ +int ldbm_index_init_entry_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_instance_index_config_add_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); +int ldbm_instance_index_config_delete_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); +int ldbm_instance_index_config_modify_callback(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); +/* Attribute Encryption config functions */ +int ldbm_attrcrypt_init_entry_callback(Slapi_PBlock *pb, Slapi_Entry* e, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg); +int ldbm_instance_attrcrypt_config_add_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); +int ldbm_instance_attrcrypt_config_delete_callback(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, int *returncode, char *returntext, void *arg); +int ldbm_instance_attrcrypt_config_modify_callback(Slapi_PBlock *pb, Slapi_Entry *e, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg); + +void replace_ldbm_config_value(char *conftype, char *val, struct ldbminfo *li); + +/* + * ancestorid.c + */ +int ldbm_ancestorid_create_index(backend *be); +int ldbm_ancestorid_index_entry(backend *be, struct backentry *e, int flags, back_txn *txn); +int ldbm_ancestorid_read(backend *be, back_txn *txn, ID id, IDList **idl); +int ldbm_ancestorid_move_subtree( + backend *be, + const Slapi_DN *olddn, + const Slapi_DN *newdn, + ID id, + IDList *subtree_idl, + back_txn *txn +); + +#endif + +/* + * import-threads.c + */ +int dse_conf_backup(struct ldbminfo *li, char *destination_directory); +int dse_conf_verify(struct ldbminfo *li, char *src_dir); + +/* + * ldbm_attrcrypt.c + */ +int attrcrypt_decrypt_entry(backend *be, struct backentry *e); +int attrcrypt_encrypt_entry_inplace(backend *be, const struct backentry *inout); +int attrcrypt_encrypt_entry(backend *be, const struct backentry *in, struct backentry **out); +int attrcrypt_encrypt_index_key(backend *be, struct attrinfo *ai, const struct berval *in, struct berval **out); +int attrcrypt_init(ldbm_instance *li); diff --git a/ldap/servers/slapd/back-ldbm/rmdb.c b/ldap/servers/slapd/back-ldbm/rmdb.c new file mode 100644 index 00000000..d4b760bd --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/rmdb.c @@ -0,0 +1,54 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* + * rmdb.c - ldbm backend routine which deletes an entire database. + * This routine is not exposed in the public SLAPI interface. It + * is called by the replication subsystem when then changelog must + * be erased. + */ + +#include "back-ldbm.h" + +int +ldbm_back_rmdb( Slapi_PBlock *pb ) +{ + struct ldbminfo *li = NULL; + /* char *directory = NULL;*/ + int return_value = -1; + Slapi_Backend *be; + + slapi_pblock_get( pb, SLAPI_BACKEND, &be ); + + if (be->be_state != BE_STATE_STOPPED) + { + LDAPDebug( LDAP_DEBUG_TRACE, + "ldbm_back_cleanup: warning - backend is in a wrong state - %d\n", + be->be_state, 0, 0 ); + return 0; + } + + PR_Lock (be->be_state_lock); + + if (be->be_state != BE_STATE_STOPPED) + { + LDAPDebug( LDAP_DEBUG_TRACE, + "ldbm_back_cleanup: warning - backend is in a wrong state - %d\n", + be->be_state, 0, 0 ); + PR_Unlock (be->be_state_lock); + return 0; + } + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); +/* slapi_pblock_get( pb, SLAPI_SEQ_VAL, &directory );*/ + return_value = dblayer_delete_database( li ); + + if (return_value == 0) + be->be_state = BE_STATE_DELETED; + + PR_Unlock (be->be_state_lock); + + return return_value; +} diff --git a/ldap/servers/slapd/back-ldbm/seq.c b/ldap/servers/slapd/back-ldbm/seq.c new file mode 100644 index 00000000..6a61fd2e --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/seq.c @@ -0,0 +1,262 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* seq.c - ldbm backend sequential access function */ + +#include "back-ldbm.h" + +#define SEQ_LITTLE_BUFFER_SIZE 100 + +/* + * Access the database sequentially. + * There are 4 ways to call this routine. In each case, the equality index + * for "attrname" is consulted: + * 1) If the SLAPI_SEQ_TYPE parameter is SLAPI_SEQ_FIRST, then this routine + * will find the smallest key greater than or equal to the SLAPI_SEQ_VAL + * parameter, and return all entries that key's IDList. If SLAPI_SEQ_VAL + * is NULL, then the smallest key is retrieved and the associaated + * entries are returned. + * 2) If the SLAPI_SEQ_TYPE parameter is SLAPI_SEQ_NEXT, then this routine + * will find the smallest key strictly greater than the SLAPI_SEQ_VAL + * parameter, and return all entries that key's IDList. + * 3) If the SLAPI_SEQ_TYPE parameter is SLAPI_SEQ_PREV, then this routine + * will find the greatest key strictly less than the SLAPI_SEQ_VAL + * parameter, and return all entries that key's IDList. + * 4) If the SLAPI_SEQ_TYPE parameter is SLAPI_SEQ_LAST, then this routine + * will find the largest equality key in the index and return all entries + * which match that key. The SLAPI_SEQ_VAL parameter is ignored. + */ +int +ldbm_back_seq( Slapi_PBlock *pb ) +{ + backend *be; + ldbm_instance *inst; + struct ldbminfo *li; + IDList *idl = NULL; + int err = LDAP_SUCCESS; + DB *db; + DBC *dbc = NULL; + int type; + char *attrname, *val; + int isroot; + struct attrinfo *ai = NULL; + int return_value = -1; + int nentries = 0; + int retry_count=0; + + /* Decode arguments */ + slapi_pblock_get( pb, SLAPI_BACKEND, &be); + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + slapi_pblock_get( pb, SLAPI_SEQ_TYPE, &type ); + slapi_pblock_get( pb, SLAPI_SEQ_ATTRNAME, &attrname ); + slapi_pblock_get( pb, SLAPI_SEQ_VAL, &val ); + slapi_pblock_get( pb, SLAPI_REQUESTOR_ISROOT, &isroot ); + + inst = (ldbm_instance *) be->be_instance_info; + + /* Validate arguments */ + if ( type != SLAPI_SEQ_FIRST && + type != SLAPI_SEQ_LAST && + type != SLAPI_SEQ_NEXT && + type != SLAPI_SEQ_PREV ) + { + slapi_send_ldap_result( pb, LDAP_PROTOCOL_ERROR, NULL, + "Bad seq access type", 0, NULL ); + return( -1 ); + } + + /* get a database */ + + ainfo_get( be, attrname, &ai ); + LDAPDebug( LDAP_DEBUG_ARGS, + " seq: indextype: %s indexmask: 0x%x seek type: %d\n", + ai->ai_type, ai->ai_indexmask, type ); + if ( ! (INDEX_EQUALITY & ai->ai_indexmask) ) { + LDAPDebug( LDAP_DEBUG_TRACE, + "seq: caller specified un-indexed attribute %s\n", + attrname ? attrname : "", 0, 0 ); + slapi_send_ldap_result( pb, LDAP_UNWILLING_TO_PERFORM, NULL, + "Unindexed seq access type", 0, NULL ); + return -1; + } + + if ( (return_value = dblayer_get_index_file( be, ai, &db, DBOPEN_CREATE )) != 0 ) { + LDAPDebug( LDAP_DEBUG_ANY, + "<= ldbm_back_seq NULL (could not open index file for attribute %s)\n", + attrname, 0, 0 ); + slapi_send_ldap_result( pb, LDAP_OPERATIONS_ERROR, NULL, NULL, 0, NULL ); + return -1; + } + + /* First, get a database cursor */ + + return_value = db->cursor(db,NULL,&dbc,0); + + if (0 == return_value) + { + DBT data = {0}; + DBT key = {0}; + char little_buffer[SEQ_LITTLE_BUFFER_SIZE]; + char *big_buffer = NULL; + char keystring = EQ_PREFIX; + + /* Set data */ + data.flags = DB_DBT_MALLOC; + + /* Set up key */ + key.flags = DB_DBT_MALLOC; + if (NULL == val) + { + /* this means, goto the first equality key */ + /* seek to key >= "=" */ + key.data = &keystring; + key.size = 1; + } + else + { + size_t key_length = strlen(val) + 2; + if (key_length <= SEQ_LITTLE_BUFFER_SIZE) { + key.data = &little_buffer; + } else { + big_buffer = slapi_ch_malloc(key_length); + if (NULL == big_buffer) { + /* memory allocation failure */ + dblayer_release_index_file( be, ai, db ); + return -1; + } + key.data = big_buffer; + } + key.size = sprintf(key.data,"%c%s",EQ_PREFIX,val); + } + + /* decide which type of operation we're being asked to do and do the db bit */ + /* The c_get call always mallocs memory for data.data */ + /* The c_get call mallocs memory for key.data, except for DB_SET */ + /* after this, we leave data containing the retrieved IDL, or NULL if we didn't get it */ + + switch (type) { + case SLAPI_SEQ_FIRST: + /* if (NULL == val) goto the first equality key ( seek to key >= "=" ) */ + /* else goto the first equality key >= val ( seek to key >= "=val" )*/ + return_value = dbc->c_get(dbc,&key,&data,DB_SET_RANGE); + break; + case SLAPI_SEQ_NEXT: + /* seek to the indicated =value, then seek to the next entry, */ + return_value = dbc->c_get(dbc,&key,&data,DB_SET); + if (0 == return_value) + { + free(data.data); + return_value = dbc->c_get(dbc,&key,&data,DB_NEXT); + } + else + { + /* DB_SET doesn't allocate key data. Make sure we don't try to free it... */ + key.data= NULL; + } + break; + case SLAPI_SEQ_PREV: + /* seek to the indicated =value, then seek to the previous entry, */ + return_value = dbc->c_get(dbc,&key,&data,DB_SET); + if (0 == return_value ) + { + free(data.data); + return_value = dbc->c_get(dbc,&key,&data,DB_PREV); + } + else + { + /* DB_SET doesn't allocate key data. Make sure we don't try to free it... */ + key.data= NULL; + } + break; + case SLAPI_SEQ_LAST: + /* seek to the first possible key after all the equality keys (">"), then seek back one */ + { + keystring = EQ_PREFIX + 1; + key.data = &keystring; + key.size = 1; + return_value = dbc->c_get(dbc,&key,&data,DB_SET_RANGE); + if (0 == return_value || DB_NOTFOUND == return_value) + { + free(data.data); + return_value = dbc->c_get(dbc,&key,&data,DB_PREV); + } + } + break; + default: + PR_ASSERT(0); + } + + dbc->c_close(dbc); + + if (0 == return_value && key.data!=NULL) + { + + /* Now check that the key we eventually settled on was an equality key ! */ + if (*((char*)key.data) == EQ_PREFIX) + { + /* Retrieve the idlist for this key */ + key.flags = 0; + for (retry_count = 0; retry_count < IDL_FETCH_RETRY_COUNT; retry_count++) { + err = NEW_IDL_DEFAULT; + idl = idl_fetch( be, db, &key, NULL, ai, &err ); + if(err == DB_LOCK_DEADLOCK) { + ldbm_nasty("ldbm_back_seq deadlock retry", 1600, err); + continue; + } else { + break; + } + } + } + } + if(retry_count == IDL_FETCH_RETRY_COUNT) { + ldbm_nasty("ldbm_back_seq retry count exceeded",1645,err); + } else if ( err != 0 && err != DB_NOTFOUND ) { + ldbm_nasty("ldbm_back_seq database error", 1650, err); + } + free( data.data ); + if ( key.data != little_buffer && key.data != &keystring ) { + free( key.data ); + } + free( big_buffer ); + } + + /* null idlist means there were no matching keys */ + if ( idl != NULL ) + { + /* + * Step through the IDlist. For each ID, get the entry + * and send it. + */ + ID id; + struct backentry *e; + for ( id = idl_firstid( idl ); id != NOID; + id = idl_nextid( idl, id )) + { + if (( e = id2entry( be, id, NULL, &err )) == NULL ) + { + if ( err != LDAP_SUCCESS ) + { + LDAPDebug( LDAP_DEBUG_ANY, "seq id2entry err %d\n", err, 0, 0 ); + } + LDAPDebug( LDAP_DEBUG_ARGS, + "ldbm_back_seq: candidate %lu not found\n", + (u_long)id, 0, 0 ); + continue; + } + if ( slapi_send_ldap_search_entry( pb, e->ep_entry, NULL, NULL, 0 ) == 0 ) + { + nentries++; + } + cache_return( &inst->inst_cache, &e ); + } + idl_free( idl ); + } + + dblayer_release_index_file( be, ai, db ); + + slapi_send_ldap_result( pb, LDAP_SUCCESS == err ? LDAP_SUCCESS : LDAP_OPERATIONS_ERROR, NULL, NULL, nentries, NULL ); + + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/sort.c b/ldap/servers/slapd/back-ldbm/sort.c new file mode 100644 index 00000000..4a25e068 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/sort.c @@ -0,0 +1,1031 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* Code to implement result sorting */ + +#include "back-ldbm.h" + +#define CHECK_INTERVAL 10 /* The frequency whith which we'll check the admin limits */ + +/* Structure to carry the things we need down the call stack */ +struct baggage_carrier { + backend *be; /* For id2entry */ + Slapi_PBlock *pb; /* For slapi_op_abandoned */ + time_t stoptime; /* For timelimit policing */ + int lookthrough_limit; + int check_counter; /* Used to avoid checking every 100ns */ +}; +typedef struct baggage_carrier baggage_carrier; + +static int slapd_qsort (baggage_carrier *bc,IDList *list,sort_spec *s); +static int print_out_sort_spec(char* buffer,sort_spec *s,int *size); + +static void sort_spec_thing_free(sort_spec_thing *s) +{ + if (NULL != s->type) { + slapi_ch_free((void **)&s->type); + } + if (NULL != s->matchrule) { + slapi_ch_free( (void**)&s->matchrule); + } + if (NULL != s->mr_pb) { + destroy_matchrule_indexer(s->mr_pb); + slapi_pblock_destroy (s->mr_pb); + } + slapi_ch_free( (void**)&s); +} + +static sort_spec_thing *sort_spec_thing_allocate() +{ + return (sort_spec_thing *) slapi_ch_calloc(1,sizeof (sort_spec_thing)); +} + +void sort_spec_free(sort_spec *s) +{ + /* Walk down the list freeing */ + sort_spec_thing *t = (sort_spec_thing*)s; + sort_spec_thing *p = NULL; + do { + p = t->next; + sort_spec_thing_free(t); + t = p; + } while (p); +} + +static sort_spec_thing * sort_spec_thing_new(char *type, char* matchrule, int reverse) +{ + sort_spec_thing *s = sort_spec_thing_allocate(); + if (NULL == s) { + return s; + } + s->type = type; + s->matchrule = matchrule; + s->order = reverse; + return s; +} + +void sort_log_access(Slapi_PBlock *pb,sort_spec_thing *s,IDList *candidates) +{ +#define SORT_LOG_BSZ 64 +#define SORT_LOG_PAD 22 /* space for the number of candidates */ + char stack_buffer[SORT_LOG_BSZ + SORT_LOG_PAD]; + char *buffer = NULL; + int ret = 0; + int size = SORT_LOG_BSZ + SORT_LOG_PAD; + char *prefix = "SORT "; + int prefix_size = strlen(prefix); + + buffer = stack_buffer; + size -= sprintf(buffer,"%s",prefix); + ret = print_out_sort_spec(buffer+prefix_size,s,&size); + if (0 != ret) { + /* It wouldn't fit in the buffer */ + buffer = slapi_ch_malloc(prefix_size + size + SORT_LOG_PAD); + sprintf(buffer,"%s",prefix); + ret = print_out_sort_spec(buffer+prefix_size,s,&size); + } + if (candidates) { + if (ALLIDS(candidates)) { + sprintf(buffer+size+prefix_size,"(*)"); + } else { + sprintf(buffer+size+prefix_size,"(%lu)",(u_long)candidates->b_nids); + } + } + /* Now output it */ + ldbm_log_access_message(pb,buffer); + if (buffer != stack_buffer) { + slapi_ch_free( (void**)&buffer); + } +} + +/* Fix for bug # 394184, SD, 20 Jul 00 */ +/* replace the hard coded return value by the appropriate LDAP error code */ +/* also removed an useless if (0 == return_value) {} statement */ +/* Given a candidate list and a list of sort order specifications, sort this, or cop out */ +/* Returns: 0 -- sorted OK now is: LDAP_SUCCESS (fix for bug #394184) + * -1 -- protocol error now is: LDAP_PROTOCOL_ERROR + * -2 -- too hard to sort these now is: LDAP_UNWILLING_TO_PERFORM + * -3 -- operation error now is: LDAP_OPERATIONS_ERROR + * -4 -- timeout now is: LDAP_TIMELIMIT_EXCEEDED + * -5 -- admin limit exceeded now is: LDAP_ADMINLIMIT_EXCEEDED + * -6 -- abandoned now is: LDAP_OTHER + */ +/* + * So here's the plan: + * Plan A: We do a regular quicksort on the entries. + * Plan B: Through some hint given us from on high, we + * determine that the entries are _already_ + * sorted as requested, thus we do nothing ! + * Plan C: We determine that sorting these suckers is + * far too hard for us to even try, so we refuse. + */ +int sort_candidates(backend *be,int lookthrough_limit,time_t time_up, Slapi_PBlock *pb, + IDList *candidates, sort_spec_thing *s, char **sort_error_type) +{ + int return_value = LDAP_SUCCESS; + baggage_carrier bc = {0}; + sort_spec_thing *this_s = NULL; + + /* We refuse to sort a non-existent IDlist */ + if (NULL == candidates) { + return LDAP_UNWILLING_TO_PERFORM; + } + /* we refuse to sort a candidate list which is vast */ + if (ALLIDS(candidates)) { + LDAPDebug( LDAP_DEBUG_TRACE, "Asked to sort ALLIDS candidate list, refusing\n",0, 0, 0 ); + return LDAP_UNWILLING_TO_PERFORM; + } + + /* Iterate over the sort types */ + for (this_s = s; this_s; this_s=this_s->next) { + if (NULL == this_s->matchrule) { + void *pi; + int return_value = 0; + return_value = slapi_attr_type2plugin( this_s->type, &pi ); + if (0 == return_value) { + return_value = plugin_call_syntax_get_compare_fn( pi, &(this_s->compare_fn) ); + } + if (return_value != 0 ) { + LDAPDebug( LDAP_DEBUG_TRACE, "Attempting to sort a non-ordered attribute (%s)\n",this_s->type, 0, 0 ); + /* DBDB we should set the error type here */ + return_value = LDAP_UNWILLING_TO_PERFORM; + *sort_error_type = this_s->type; + return return_value; + } + } else { + /* Need to---find the matching rule plugin, + * tell it it needs to do ordering for this OID + * see whether it agrees---if not signal error to client + * Then later use it for generating ordering keys. + * finally, free it up + */ + return_value = create_matchrule_indexer(&this_s->mr_pb,this_s->matchrule,this_s->type); + if (LDAP_SUCCESS != return_value) { + *sort_error_type = this_s->type; + return return_value; + } + this_s->compare_fn = slapi_berval_cmp; + } + } + + bc.be = be; + bc.pb = pb; + bc.stoptime = time_up; + bc.lookthrough_limit = lookthrough_limit; + bc.check_counter = 1; + + return_value = slapd_qsort(&bc,candidates,s); + LDAPDebug( LDAP_DEBUG_TRACE, "<= Sorting done\n",0, 0, 0 ); + + return return_value; +} +/* End fix for bug # 394184 */ + +/* Fix for bug # 394184, SD, 20 Jul 00 */ +/* fix and cleanup (switch(code) {} removed) */ +/* arg 'code' has now the correct sortResult value */ +int +make_sort_response_control ( Slapi_PBlock *pb, int code, char *error_type) { + + LDAPControl new_ctrl = {0}; + BerElement *ber= NULL; + struct berval *bvp = NULL; + int rc = -1; + int control_code = code; + + /* + SortResult ::= SEQUENCE { + sortResult ENUMERATED { + success (0), -- results are sorted + operationsError (1), -- server internal failure + timeLimitExceeded (3), -- timelimit reached before + -- sorting was completed + strongAuthRequired (8), -- refused to return sorted + -- results via insecure + -- protocol + adminLimitExceeded (11), -- too many matching entries + -- for the server to sort + noSuchAttribute (16), -- unrecognized attribute + -- type in sort key + inappropriateMatching (18), -- unrecognized or inappro- + -- priate matching rule in + -- sort key + insufficientAccessRights (50), -- refused to return sorted + -- results to this client + busy (51), -- too busy to process + unwillingToPerform (53), -- unable to sort + other (80) + }, + attributeType [0] AttributeType OPTIONAL } + + */ + + if ( ( ber = ber_alloc()) == NULL ) { + return -1; + } + + if (( rc = ber_printf( ber, "{e", control_code )) != -1 ) { + if ( rc != -1 && NULL != error_type ) { + rc = ber_printf( ber, "s", error_type ); + } + if ( rc != -1 ) { + rc = ber_printf( ber, "}" ); + } + } + if ( rc != -1 ) { + rc = ber_flatten( ber, &bvp ); + } + + ber_free( ber, 1 ); + + if ( rc == -1 ) { + return rc; + } + + new_ctrl.ldctl_oid = LDAP_CONTROL_SORTRESPONSE; + new_ctrl.ldctl_value = *bvp; + new_ctrl.ldctl_iscritical = 1; + + if ( slapi_pblock_set( pb, SLAPI_ADD_RESCONTROL, &new_ctrl ) != 0 ) { + ber_bvfree(bvp); + return( -1 ); + } + + ber_bvfree(bvp); + return( LDAP_SUCCESS ); +} +/* End fix for bug #394184 */ + +static int term_tag(unsigned long tag) +{ + return ( (LBER_END_OF_SEQORSET == tag) || (LBER_ERROR == tag) ); +} + +/* hacky function to convert a sort spec to a string + you specify a buffer and a size. If the thing won't fit, it returns + non-zero, and the size needed. Pass NULL buffer to just get the size */ +static int print_out_sort_spec(char* buffer,sort_spec *s,int *size) +{ + /* Walk down the list printing */ + sort_spec_thing *t = (sort_spec_thing*)s; + sort_spec_thing *p = NULL; + int buffer_size = 0; + int input_size = 0; + + if (NULL != size) { + input_size = *size; + } + do { + p = t->next; + + buffer_size += strlen(t->type); + if (t->order) { + buffer_size += 1; /* For the '-' */ + } + if (NULL != t->matchrule) { + /* space for matchrule + semicolon */ + buffer_size += strlen(t->matchrule) + 1; + } + buffer_size += 1; /* for the space */ + if ( (NULL != buffer) && (buffer_size <= input_size) ) { + /* write into the buffer */ + buffer += sprintf(buffer,"%s%s%s%s ", + t->order ? "-" : "", + t->type, + ( NULL == t->matchrule ) ? "" : ";", + ( NULL == t->matchrule ) ? "" : t->matchrule); + } + + t = p; + } while (p); + if (NULL != size) { + *size = buffer_size; + } + if (buffer_size <= input_size) { + return 0; + } else { + return 1; + } +} + +int parse_sort_spec(struct berval *sort_spec_ber, sort_spec **ps) +{ + /* So here we call ber_scanf to get the sort spec */ + /* This control looks like this : + SortKeyList ::= SEQUENCE OF SEQUENCE { + attributeType AttributeType, + orderingRule [0] MatchingRuleId OPTIONAL, + reverseOrder [1] BOOLEAN DEFAULT FALSE } + */ + BerElement *ber = NULL; + sort_spec_thing *listhead = NULL; + unsigned long tag = 0; + unsigned long len = 0; + char *last = NULL; + sort_spec_thing *listpointer = NULL; + char *type = NULL; + char *matchrule = NULL; + int rc = LDAP_SUCCESS; + + ber = ber_init(sort_spec_ber); + if(ber==NULL) + { + return -1; + } + + /* Work our way along the BER, one sort spec at a time */ + for ( tag = ber_first_element( ber, &len, &last ); !term_tag(tag); tag = ber_next_element( ber, &len, last )) { + /* we're now pointing at the beginning of a sequence of type, matching rule and reverse indicator */ + + char *inner_last = NULL; + char *rtype = NULL; + int reverse = 0; + unsigned long next_tag = 0; + sort_spec_thing *s = NULL; + unsigned long return_value; + + next_tag = ber_first_element( ber, &len, &inner_last ); + + /* The type is not optional */ + + return_value = ber_scanf(ber,"a",&rtype); + if (LBER_ERROR == return_value) { + rc = LDAP_PROTOCOL_ERROR; + goto err; + } + /* normalize */ + type = slapi_attr_syntax_normalize(rtype); + free(rtype); + + /* Now look for the next tag. */ + + next_tag = ber_next_element(ber,&len, inner_last); + + /* Are we done ? */ + if ( !term_tag(next_tag) ) { + /* Is it the matching rule ? */ + if (LDAP_TAG_SK_MATCHRULE == next_tag) { + /* If so, get it */ + ber_scanf(ber,"a",&matchrule); + /* That can be followed by a reverse indicator */ + next_tag = ber_next_element(ber,&len, inner_last); + if (LDAP_TAG_SK_REVERSE == next_tag) { + /* Get the reverse sort indicator here */ + ber_scanf(ber,"b",&reverse); + /* The protocol police say--"You must have other than your default value" */ + if (0 == reverse) { + /* Protocol error */ + rc = LDAP_PROTOCOL_ERROR; + goto err; + } + } else { + /* Perhaps we're done now ? */ + if (LBER_END_OF_SEQORSET != next_tag) { + /* Protocol error---we got a matching rule, but followed by something other + * than reverse or end of sequence. + */ + rc = LDAP_PROTOCOL_ERROR; + goto err; + } + } + } else { + /* Is it the reverse indicator ? */ + if (LDAP_TAG_SK_REVERSE == next_tag) { + /* If so, get it */ + ber_scanf(ber,"b",&reverse); + } else { + /* Protocol error---tag which isn't either of the legal ones came first */ + rc = LDAP_PROTOCOL_ERROR; + goto err; + } + } + } + + s = sort_spec_thing_new(type,matchrule,reverse); + if (NULL == s) { + /* Memory allocation failed */ + rc = LDAP_OPERATIONS_ERROR; + goto err; + } + type = matchrule = NULL; + if (NULL != listpointer) { + listpointer->next = s; + } + listpointer = s; + if (NULL == listhead) { + listhead = s; + } + + } + + if (NULL == listhead) { /* LP - defect #559792 - don't return null listhead */ + *ps = NULL; + rc = LDAP_PROTOCOL_ERROR; + goto err; + } + + /* the ber encoding is no longer needed */ + ber_free(ber,1); + + *ps = (sort_spec *)listhead; + + + return LDAP_SUCCESS; + + err: + if (listhead) sort_spec_free((sort_spec*) listhead); + slapi_ch_free((void**)&type); + slapi_ch_free((void**)&matchrule); + ber_free(ber,1); + + return rc; +} + +#if 0 +static int attr_value_compare(struct berval *value_a, struct berval *value_b) +{ + /* return value_cmp(value_a,value_b,syntax,3); */ + return strcasecmp(value_a->bv_val, value_b->bv_val); +} +#endif + +struct berval* attr_value_lowest(struct berval **values, value_compare_fn_type compare_fn) +{ + /* We iterate through the values, storing our last best guess as to the lowest */ + struct berval *lowest_so_far = values[0]; + struct berval *this_one = NULL; + + for (this_one = *values; this_one; this_one = *values++) { + if (compare_fn(lowest_so_far,this_one) > 0) { + lowest_so_far = this_one; + } + } + return lowest_so_far; +} + +int sort_attr_compare(struct berval ** value_a, struct berval ** value_b, value_compare_fn_type compare_fn) +{ + /* So, the thing we need to do here is to look out for multi-valued + * attributes. When we get one of those, we need to look through all the + * values to find the lowest one (per X.511 edict). We then use that one to + * compare against the other. We should really put some logic in here to + * prevent us partying on an attribute with thousands of values for a long time. + */ + struct berval *compare_value_a = NULL; + struct berval *compare_value_b = NULL; + + compare_value_a = attr_value_lowest(value_a, compare_fn); + compare_value_b = attr_value_lowest(value_b, compare_fn); + + return compare_fn(compare_value_a,compare_value_b); + +} + + +#if 0 +/* USE THE _SV VERSION NOW */ + +/* Comparison routine, called by qsort. + * The job here is to return the correct value + * for the operation a < b + * Returns: + * <0 when a < b + * 0 when a == b + * >0 when a > b + */ +static int compare_entries(ID *id_a, ID *id_b, sort_spec *s,baggage_carrier *bc, int *error) +{ + /* We get passed the IDs, but need to fetch the entries in order to + * perform the comparison . + */ + struct backentry *a = NULL; + struct backentry *b = NULL; + int result = 0; + sort_spec_thing *this_one = NULL; + int return_value = -1; + backend *be = bc->be; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int err; + + *error = 1; + a = id2entry(be,*id_a,NULL,&err); + if (NULL == a) { + if (0 != err ) { + LDAPDebug(LDAP_DEBUG_ANY,"compare_entries db err %d\n",err,0,0); + } + /* Were up a creek without paddle here */ + /* Best to log error and set some flag */ + return 0; + } + b = id2entry(be,*id_b,NULL,&err); + if (NULL == b) { + if (0 != err ) { + LDAPDebug(LDAP_DEBUG_ANY,"compare_entries db err %d\n",err,0,0); + } + return 0; + } + /* OK, now we have the entries, so we work our way down the attribute list comparing as we go */ + for (this_one = (sort_spec_thing*)s; this_one ; this_one = this_one->next) { + + char *type = this_one->type; + int order = this_one->order; + Slapi_Attr *attr_a = NULL; + Slapi_Attr *attr_b = NULL; + struct berval **value_a = NULL; + struct berval **value_b = NULL; + + /* Get the two attribute values from the entries */ + return_value = slapi_entry_attr_find(a->ep_entry,type,&attr_a); + return_value = slapi_entry_attr_find(b->ep_entry,type,&attr_b); + /* What do we do if one or more of the entries lacks this attribute ? */ + /* if one lacks the attribute */ + if (NULL == attr_a) { + /* then if the other does too, they're equal */ + if (NULL == attr_b) { + result = 0; + continue; + } else + { + /* If one has the attribute, and the other + * doesn't, the missing attribute is the + * LARGER one. (bug #108154) -robey + */ + result = 1; + break; + } + } + if (NULL == attr_b) { + result = -1; + break; + } + /* Somewhere in here, we need to go sideways for match rule case + * we need to call the match rule plugin to get the attribute values + * converted into ordering keys. Then we proceed as usual to use those, + * but ensuring that we don't leak memory anywhere. This works as follows: + * the code assumes that the attrs are references into the entry, so + * doesn't try to free them. We need to note at the right place that + * we're on the matchrule path, and accordingly free the keys---this turns out + * to be when we free the indexer */ + if (NULL == s->matchrule) { + /* Non-match rule case */ + /* xxxPINAKI + needs modification + + value_a = attr_a->a_vals; + value_b = attr_b->a_vals; + */ + } else { + /* Match rule case */ + struct berval **actual_value_b = NULL; + struct berval **temp_value = NULL; + + /* xxxPINAKI + needs modification + struct berval **actual_value_a = NULL; + + actual_value_a = attr_a->a_vals; + actual_value_b = attr_b->a_vals; + matchrule_values_to_keys(s->mr_pb,actual_value_a,&temp_value); + */ + /* Now copy it, so the second call doesn't crap on it */ + value_a = slapi_ch_bvecdup(temp_value); /* Really, we'd prefer to not call the chXXX variant...*/ + matchrule_values_to_keys(s->mr_pb,actual_value_b,&value_b); + } + /* Compare them */ + if (!order) { + result = sort_attr_compare(value_a, value_b, s->compare_fn); + } else { + /* If reverse, invert the sense of the comparison */ + result = sort_attr_compare(value_b, value_a, s->compare_fn); + } + /* Time to free up the attribute allocated above */ + if (NULL != s->matchrule) { + ber_bvecfree(value_a); + } + /* Are they equal ? */ + if (0 != result) { + /* If not, we're done */ + break; + } + /* If so, proceed to the next attribute for comparison */ + } + cache_return(&inst->inst_cache,&a); + cache_return(&inst->inst_cache,&b); + *error = 0; + return result; +} +#endif + +/* Comparison routine, called by qsort. + * The job here is to return the correct value + * for the operation a < b + * Returns: + * <0 when a < b + * 0 when a == b + * >0 when a > b + */ +static int compare_entries_sv(ID *id_a, ID *id_b, sort_spec *s,baggage_carrier *bc, int *error) +{ + /* We get passed the IDs, but need to fetch the entries in order to + * perform the comparison . + */ + struct backentry *a = NULL; + struct backentry *b = NULL; + int result = 0; + sort_spec_thing *this_one = NULL; + int return_value = -1; + backend *be = bc->be; + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; + int err; + + *error = 1; + a = id2entry(be,*id_a,NULL,&err); + if (NULL == a) { + if (0 != err ) { + LDAPDebug(LDAP_DEBUG_ANY,"compare_entries db err %d\n",err,0,0); + } + /* Were up a creek without paddle here */ + /* Best to log error and set some flag */ + return 0; + } + b = id2entry(be,*id_b,NULL,&err); + if (NULL == b) { + if (0 != err ) { + LDAPDebug(LDAP_DEBUG_ANY,"compare_entries db err %d\n",err,0,0); + } + return 0; + } + /* OK, now we have the entries, so we work our way down the attribute list comparing as we go */ + for (this_one = (sort_spec_thing*)s; this_one ; this_one = this_one->next) { + + char *type = this_one->type; + int order = this_one->order; + Slapi_Attr *attr_a = NULL; + Slapi_Attr *attr_b = NULL; + struct berval **value_a = NULL; + struct berval **value_b = NULL; + + /* Get the two attribute values from the entries */ + return_value = slapi_entry_attr_find(a->ep_entry,type,&attr_a); + return_value = slapi_entry_attr_find(b->ep_entry,type,&attr_b); + /* What do we do if one or more of the entries lacks this attribute ? */ + /* if one lacks the attribute */ + if (NULL == attr_a) { + /* then if the other does too, they're equal */ + if (NULL == attr_b) { + result = 0; + continue; + } else + { + /* If one has the attribute, and the other + * doesn't, the missing attribute is the + * LARGER one. (bug #108154) -robey + */ + result = 1; + break; + } + } + if (NULL == attr_b) { + result = -1; + break; + } + /* Somewhere in here, we need to go sideways for match rule case + * we need to call the match rule plugin to get the attribute values + * converted into ordering keys. Then we proceed as usual to use those, + * but ensuring that we don't leak memory anywhere. This works as follows: + * the code assumes that the attrs are references into the entry, so + * doesn't try to free them. We need to note at the right place that + * we're on the matchrule path, and accordingly free the keys---this turns out + * to be when we free the indexer */ + if (NULL == s->matchrule) { + /* Non-match rule case */ + valuearray_get_bervalarray(valueset_get_valuearray(&attr_a->a_present_values),&value_a); + valuearray_get_bervalarray(valueset_get_valuearray(&attr_b->a_present_values),&value_b); + } else { + /* Match rule case */ + struct berval **actual_value_a = NULL; + struct berval **actual_value_b = NULL; + struct berval **temp_value = NULL; + + valuearray_get_bervalarray(valueset_get_valuearray(&attr_a->a_present_values),&actual_value_a); + valuearray_get_bervalarray(valueset_get_valuearray(&attr_b->a_present_values),&actual_value_b); + matchrule_values_to_keys(s->mr_pb,actual_value_a,&temp_value); + /* Now copy it, so the second call doesn't crap on it */ + value_a = slapi_ch_bvecdup(temp_value); /* Really, we'd prefer to not call the chXXX variant...*/ + matchrule_values_to_keys(s->mr_pb,actual_value_b,&value_b); + if (actual_value_a) ber_bvecfree(actual_value_a); + if (actual_value_b) ber_bvecfree(actual_value_b); + } + /* Compare them */ + if (!order) { + result = sort_attr_compare(value_a, value_b, s->compare_fn); + } else { + /* If reverse, invert the sense of the comparison */ + result = sort_attr_compare(value_b, value_a, s->compare_fn); + } + /* Time to free up the attributes allocated above */ + if (NULL != s->matchrule) { + ber_bvecfree(value_a); + } else { + ber_bvecfree(value_a); + ber_bvecfree(value_b); + } + /* Are they equal ? */ + if (0 != result) { + /* If not, we're done */ + break; + } + /* If so, proceed to the next attribute for comparison */ + } + cache_return(&inst->inst_cache,&a); + cache_return(&inst->inst_cache,&b); + *error = 0; + return result; +} + +/* Fix for bug # 394184, SD, 20 Jul 00 */ +/* replace the hard coded return value by the appropriate LDAP error code */ +/* + * Returns: + * 0: Everything OK now is: LDAP_SUCCESS (fix for bug #394184) + * -1: A protocol error now is: LDAP_PROTOCOL_ERROR + * -2: Too hard now is: LDAP_UNWILLING_TO_PERFORM + * -3: Operation error now is: LDAP_OPERATIONS_ERROR + * -4: Timeout now is: LDAP_TIMELIMIT_EXCEEDED + * -5: Admin limit exceeded now is: LDAP_ADMINLIMIT_EXCEEDED + * -6: Abandoned now is: LDAP_OTHER + */ +static int sort_nazi(baggage_carrier *bc) +{ + time_t curtime = 0; + /* check for abandon */ + if ( slapi_op_abandoned( bc->pb)) { + return LDAP_OTHER; + } + + /* Check to see if our journey is really necessary */ + + if (0 == ((bc->check_counter)++ % CHECK_INTERVAL) ) { + + /* check time limit */ + curtime = current_time(); + if ( bc->stoptime != -1 && curtime > bc->stoptime ) { + return LDAP_TIMELIMIT_EXCEEDED; + } + + /* Fix for bugid #394184, SD, 05 Jul 00 */ + /* not sure this is the appropriate place to do this; + since the entries are swaped in slapd_qsort, some of them are most + probably counted more than once */ + /* hence commenting out the following test and moving it into slapd_qsort */ + /* check lookthrough limit */ + /* if ( bc->lookthrough_limit != -1 && (bc->lookthrough_limit -= CHECK_INTERVAL) < 0 ) { + return LDAP_ADMINLIMIT_EXCEEDED; + } */ + /* end for bugid #394184 */ + + } + return LDAP_SUCCESS; +} +/* End fix for bug # 394184 */ + +/* prototypes for local routines */ +static void shortsort(baggage_carrier *bc,ID *lo, ID *hi,sort_spec *s ); +static void swap (ID *a,ID *b); + +/* this parameter defines the cutoff between using quick sort and + insertion sort for arrays; arrays with lengths shorter or equal to the + below value use insertion sort */ + +#define CUTOFF 8 /* testing shows that this is good value */ + + +/* Fix for bug # 394184, SD, 20 Jul 00 */ +/* replace the hard coded return value by the appropriate LDAP error code */ +/* Our qsort needs to police the client timeout and lookthrough limit ? + * It knows how to compare entries, so we don't bother with all the void * stuff. + */ +/* + * Returns: + * 0: Everything OK now is: LDAP_SUCCESS (fix for bug #394184) + * -1: A protocol error now is: LDAP_PROTOCOL_ERROR + * -2: Too hard now is: LDAP_UNWILLING_TO_PERFORM + * -3: Operation error now is: LDAP_OPERATIONS_ERROR + * -4: Timeout now is: LDAP_TIMELIMIT_EXCEEDED + * -5: Admin limit exceeded now is: LDAP_ADMINLIMIT_EXCEEDED + * -6: Abandoned now is: LDAP_OTHER + */ +static int slapd_qsort(baggage_carrier *bc,IDList *list, sort_spec *s) +{ + ID *lo, *hi; /* ends of sub-array currently sorting */ + ID *mid; /* points to middle of subarray */ + ID *loguy, *higuy; /* traveling pointers for partition step */ + NIDS size; /* size of the sub-array */ + ID *lostk[30], *histk[30]; + int stkptr; /* stack for saving sub-array to be processed */ + NIDS num = list->b_nids; + int return_value = LDAP_SUCCESS; + int error = 0; + + /* Note: the number of stack entries required is no more than + 1 + log2(size), so 30 is sufficient for any array */ + if (num < 2 ) + return LDAP_SUCCESS; /* nothing to do */ + + stkptr = 0; /* initialize stack */ + + lo = &(list->b_ids[0]); + hi = &(list->b_ids[num-1]); /* initialize limits */ + + /* Fix for bugid #394184, SD, 20 Jul 00 */ + if ( bc->lookthrough_limit != -1 && ( bc->lookthrough_limit <= (int) list->b_nids) ) { + return LDAP_ADMINLIMIT_EXCEEDED; + } + /* end Fix for bugid #394184 */ + + /* this entry point is for pseudo-recursion calling: setting + lo and hi and jumping to here is like recursion, but stkptr is + prserved, locals aren't, so we preserve stuff on the stack */ +recurse: + + size = (hi - lo) + 1; /* number of el's to sort */ + + /* below a certain size, it is faster to use a O(n^2) sorting method */ + if (size <= CUTOFF) { + shortsort(bc,lo, hi, s ); + } + else { + /* First we pick a partititioning element. The efficiency of the + algorithm demands that we find one that is approximately the + median of the values, but also that we select one fast. Using + the first one produces bad performace if the array is already + sorted, so we use the middle one, which would require a very + wierdly arranged array for worst case performance. Testing shows + that a median-of-three algorithm does not, in general, increase + performance. */ + + mid = lo + (size / 2); /* find middle element */ + swap(mid, lo); /* swap it to beginning of array */ + + /* We now wish to partition the array into three pieces, one + consisiting of elements <= partition element, one of elements + equal to the parition element, and one of element >= to it. This + is done below; comments indicate conditions established at every + step. */ + + loguy = lo; + higuy = hi + 1; + + /* Note that higuy decreases and loguy increases on every iteration, + so loop must terminate. */ + for (;;) { + /* lo <= loguy < hi, lo < higuy <= hi + 1, + A[i] <= A[lo] for lo <= i <= loguy, + A[i] >= A[lo] for higuy <= i <= hi */ + + do { + loguy ++; + } while (loguy <= hi && compare_entries_sv(loguy, lo, s, bc, &error) <= 0); + + /* lo < loguy <= hi+1, A[i] <= A[lo] for lo <= i < loguy, + either loguy > hi or A[loguy] > A[lo] */ + + do { + higuy --; + } while (higuy > lo && compare_entries_sv(higuy, lo, s, bc, &error) >= 0); + + /* lo-1 <= higuy <= hi, A[i] >= A[lo] for higuy < i <= hi, + either higuy <= lo or A[higuy] < A[lo] */ + + if (higuy < loguy) + break; + + /* if loguy > hi or higuy <= lo, then we would have exited, so + A[loguy] > A[lo], A[higuy] < A[lo], + loguy < hi, highy > lo */ + + swap(loguy, higuy); + + /* Check admin and time limits here on the sort */ + if ( LDAP_SUCCESS != (return_value = sort_nazi(bc)) ) + { + return return_value; + } + + /* A[loguy] < A[lo], A[higuy] > A[lo]; so condition at top + of loop is re-established */ + } + + /* A[i] >= A[lo] for higuy < i <= hi, + A[i] <= A[lo] for lo <= i < loguy, + higuy < loguy, lo <= higuy <= hi + implying: + A[i] >= A[lo] for loguy <= i <= hi, + A[i] <= A[lo] for lo <= i <= higuy, + A[i] = A[lo] for higuy < i < loguy */ + + swap(lo, higuy); /* put partition element in place */ + + /* OK, now we have the following: + A[i] >= A[higuy] for loguy <= i <= hi, + A[i] <= A[higuy] for lo <= i < higuy + A[i] = A[lo] for higuy <= i < loguy */ + + /* We've finished the partition, now we want to sort the subarrays + [lo, higuy-1] and [loguy, hi]. + We do the smaller one first to minimize stack usage. + We only sort arrays of length 2 or more.*/ + + if ( higuy - 1 - lo >= hi - loguy ) { + if (lo + 1 < higuy) { + lostk[stkptr] = lo; + histk[stkptr] = higuy - 1; + ++stkptr; + } /* save big recursion for later */ + + if (loguy < hi) { + lo = loguy; + goto recurse; /* do small recursion */ + } + } + else { + if (loguy < hi) { + lostk[stkptr] = loguy; + histk[stkptr] = hi; + ++stkptr; /* save big recursion for later */ + } + + if (lo + 1 < higuy) { + hi = higuy - 1; + goto recurse; /* do small recursion */ + } + } + } + + /* We have sorted the array, except for any pending sorts on the stack. + Check if there are any, and do them. */ + + --stkptr; + if (stkptr >= 0) { + lo = lostk[stkptr]; + hi = histk[stkptr]; + goto recurse; /* pop subarray from stack */ + } + else + return LDAP_SUCCESS; /* all subarrays done */ +} +/* End fix for bug # 394184 */ + + +static void shortsort ( + baggage_carrier *bc, + ID *lo, + ID *hi, + sort_spec *s + ) +{ + ID *p, *max; + int error = 0; + + /* Note: in assertions below, i and j are alway inside original bound of + array to sort. */ + + while (hi > lo) { + /* A[i] <= A[j] for i <= j, j > hi */ + max = lo; + for (p = lo+1; p <= hi; p++) { + /* A[i] <= A[max] for lo <= i < p */ + if (compare_entries_sv(p,max,s,bc,&error) > 0) { + max = p; + } + /* A[i] <= A[max] for lo <= i <= p */ + } + + /* A[i] <= A[max] for lo <= i <= hi */ + + swap(max, hi); + + /* A[i] <= A[hi] for i <= hi, so A[i] <= A[j] for i <= j, j >= hi */ + + hi--; + + /* A[i] <= A[j] for i <= j, j > hi, loop top condition established */ + } + /* A[i] <= A[j] for i <= j, j > lo, which implies A[i] <= A[j] for i < j, + so array is sorted */ +} + +static void swap (ID *a,ID *b) +{ + ID tmp; + + if ( a != b ) { + tmp = *a; + *a = *b; + *b = tmp; + } +} + + diff --git a/ldap/servers/slapd/back-ldbm/start.c b/ldap/servers/slapd/back-ldbm/start.c new file mode 100644 index 00000000..f87b7112 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/start.c @@ -0,0 +1,177 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* + * start.c + */ + +#include "back-ldbm.h" + +/* + * Start the LDBM plugin, and all its instances. + */ +int +ldbm_back_start( Slapi_PBlock *pb ) +{ + struct ldbminfo *li; + static int initialized = 0; + char *home_dir; + int action; + int retval; + + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm backend starting\n", 0, 0, 0 ); + + slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li ); + + /* parse the config file here */ + ldbm_config_load_dse_info(li); + + /* register with the binder-based resource limit subsystem so that */ + /* lookthroughlimit can be supported on a per-connection basis. */ + if ( slapi_reslimit_register( SLAPI_RESLIMIT_TYPE_INT, + LDBM_LOOKTHROUGHLIMIT_AT, &li->li_reslimit_lookthrough_handle ) + != SLAPI_RESLIMIT_STATUS_SUCCESS ) { + LDAPDebug( LDAP_DEBUG_ANY, "start: Resource limit registration failed\n", + 0, 0, 0 ); + return SLAPI_FAIL_GENERAL; + } + + /* If the db directory hasn't been set yet, we need to set it to + * the default. */ + if ('\0' == li->li_directory[0]) { + /* "get default" is a special string that tells the config + * routines to figure out the default db directory by + * reading cn=config. */ + ldbm_config_internal_set(li, CONFIG_DIRECTORY, "get default"); + } + + /* sanity check the autosizing values, + no value or sum of values larger than 100. + */ + if ( (li->li_cache_autosize > 100) || + (li->li_cache_autosize_split > 100) || + (li->li_import_cache_autosize > 100) || + ((li->li_cache_autosize > 0) && (li->li_import_cache_autosize > 0) && + (li->li_cache_autosize + li->li_import_cache_autosize > 100)) ) + { + LDAPDebug( LDAP_DEBUG_ANY, "cache autosizing: bad settings, " + "value or sum of values can not larger than 100.\n", 0, 0, 0 ); + } else + /* if cache autosize was selected, select the cache sizes now */ + if ((li->li_cache_autosize > 0) || (li->li_import_cache_autosize > 0)) { + size_t pagesize, pages, procpages, availpages; + + dblayer_sys_pages(&pagesize, &pages, &procpages, &availpages); + if (pagesize) { + char s[32]; /* big enough to hold %ld */ + unsigned long cache_size_to_configure = 0; + int zone_pages, db_pages, entry_pages, import_pages; + Object *inst_obj; + ldbm_instance *inst; + /* autosizing dbCache and entryCache */ + if (li->li_cache_autosize) { + zone_pages = (li->li_cache_autosize * pages) / 100; + /* now split it according to user prefs */ + db_pages = (li->li_cache_autosize_split * zone_pages) / 100; + /* fudge an extra instance into our calculations... */ + entry_pages = (zone_pages - db_pages) / + (objset_size(li->li_instance_set) + 1); + LDAPDebug(LDAP_DEBUG_ANY, "cache autosizing. found %dk physical memory\n", + pages*(pagesize/1024), 0, 0); + LDAPDebug(LDAP_DEBUG_ANY, "cache autosizing: db cache: %dk, " + "each entry cache (%d total): %dk\n", + db_pages*(pagesize/1024), objset_size(li->li_instance_set), + entry_pages*(pagesize/1024)); + + /* libdb allocates 1.25x the amount we tell it to, but only for values < 500Meg */ + if (cache_size_to_configure < (500 * MEGABYTE)) { + cache_size_to_configure = (unsigned long)((db_pages * pagesize) / 1.25); + } else { + cache_size_to_configure = (unsigned long)(db_pages * pagesize); + } + sprintf(s, "%lu", cache_size_to_configure); + ldbm_config_internal_set(li, CONFIG_DBCACHESIZE, s); + li->li_cache_autosize_ec = (unsigned long)entry_pages * pagesize; + + for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; + inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { + inst = (ldbm_instance *)object_get_data(inst_obj); + cache_set_max_entries(&(inst->inst_cache), -1); + cache_set_max_size(&(inst->inst_cache), li->li_cache_autosize_ec); + } + } + /* autosizing importCache */ + if (li->li_import_cache_autosize) { + /* For some reason, -1 means 50 ... */ + if (li->li_import_cache_autosize == -1) { + li->li_import_cache_autosize = 50; + } + import_pages = (li->li_import_cache_autosize * pages) / 100; + LDAPDebug(LDAP_DEBUG_ANY, "cache autosizing: import cache: %dk \n", + import_pages*(pagesize/1024), NULL, NULL); + + sprintf(s, "%lu", (unsigned long)(import_pages * pagesize)); + ldbm_config_internal_set(li, CONFIG_IMPORT_CACHESIZE, s); + } + } + } + + retval = check_db_version(li, &action); + if (0 != retval) + { + LDAPDebug( LDAP_DEBUG_ANY, "start: db version is not supported\n", + 0, 0, 0); + return SLAPI_FAIL_GENERAL; + } + + if (action & DBVERSION_UPGRADE_3_4) + { + retval = dblayer_start(li,DBLAYER_CLEAN_RECOVER_MODE); + } + else + { + retval = dblayer_start(li,DBLAYER_NORMAL_MODE); + } + if (0 != retval) { + char *msg; + LDAPDebug( LDAP_DEBUG_ANY, "start: Failed to init database, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) return return_on_disk_full(li); + else return SLAPI_FAIL_GENERAL; + } + + /* Walk down the instance list, starting all the instances. */ + retval = ldbm_instance_startall(li); + if (0 != retval) { + char *msg; + LDAPDebug( LDAP_DEBUG_ANY, "start: Failed to start databases, err=%d %s\n", + retval, (msg = dblayer_strerror( retval )) ? msg : "", 0 ); + if (LDBM_OS_ERR_IS_DISKFULL(retval)) return return_on_disk_full(li); + else return SLAPI_FAIL_GENERAL; + } + + /* write DBVERSION file if one does not exist */ + home_dir = dblayer_get_home_dir(li, NULL); + if (!dbversion_exists(li, home_dir)) + { + dbversion_write (li, home_dir, NULL); + } + + + /* this function is called every time new db is initialized */ + /* currently it is called the 2nd time when changelog db is */ + /* dynamically created. Code below should only be called once */ + if (!initialized) + { + ldbm_compute_init(); + + initialized = 1; + } + + LDAPDebug( LDAP_DEBUG_TRACE, "ldbm backend done starting\n", 0, 0, 0 ); + + return( 0 ); + +} diff --git a/ldap/servers/slapd/back-ldbm/tools/index_dump/Makefile b/ldap/servers/slapd/back-ldbm/tools/index_dump/Makefile new file mode 100644 index 00000000..7b41ae90 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/tools/index_dump/Makefile @@ -0,0 +1,40 @@ +# +# BEGIN COPYRIGHT BLOCK +# Copyright 2001 Sun Microsystems, Inc. +# Portions copyright 1999, 2001-2003 Netscape Communications Corporation. +# All rights reserved. +# END COPYRIGHT BLOCK +# +# +# GNU Makefile for Directory Server libback-ldbm +# + +LDAP_SRC = ../../../../.. +MCOM_ROOT = ../../../../../../.. + +OBJDEST = $(OBJDIR)/lib/libback-ldbm +LIBDIR = $(LDAP_LIBDIR) + +include $(MCOM_ROOT)/netsite/nsdefs.mk +include $(MCOM_ROOT)/netsite/nsconfig.mk +include $(LDAP_SRC)/nsldap.mk +include $(MCOM_ROOT)/netsite/ns_usedb.mk + +INCLUDES += -I$(LDAP_SRC)/servers/slapd + +INDEX_DUMP_OBJS= index_dump.o + +OBJS = $(addprefix $(OBJDEST)/, $(INDEX_DUMP_OBJS)) + +all: $(OBJDEST) $(LIBDIR) $(SLIBBACK_LDBM) $(LIBBACK_LDBM) + +veryclean: clean + +clean: + +$(OBJDEST): + $(MKDIR) $(OBJDEST) + +$(BINDIR): + $(MKDIR) $(LIBDIR) + diff --git a/ldap/servers/slapd/back-ldbm/tools/index_dump/index_dump.c b/ldap/servers/slapd/back-ldbm/tools/index_dump/index_dump.c new file mode 100644 index 00000000..66998e49 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/tools/index_dump/index_dump.c @@ -0,0 +1,206 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +void configure __P((char *)); +DB_ENV *db_init __P((char *)); +void pheader __P((DB *, int)); +void usage __P((void)); + +const char + *progname = "db_dump"; /* Program name. */ + +int +main(argc, argv) + int argc; + char *argv[]; +{ + extern char *optarg; + extern int optind; + DB *dbp; + DBC *dbcp; + DBT key, data; + DB_ENV *dbenv; + int ch, checkprint, dflag; + char *home; + + home = NULL; + checkprint = dflag = 0; + while ((ch = getopt(argc, argv, "df:h:p")) != EOF) + switch (ch) { + case 'd': + dflag = 1; + break; + case 'f': + if (freopen(optarg, "w", stdout) == NULL) + err(1, "%s", optarg); + break; + case 'h': + home = optarg; + break; + case 'p': + checkprint = 1; + break; + case '?': + default: + usage(); + } + argc -= optind; + argv += optind; + + if (argc != 1) + usage(); + + if (dflag) { + if (home != NULL) + errx(1, + "the -d and -h options may not both be specified"); + if (checkprint) + errx(1, + "the -d and -p options may not both be specified"); + } + /* Initialize the environment. */ + dbenv = dflag ? NULL : db_init(home); + + /* Open the DB file. */ + if ((errno = + db_open(argv[0], DB_UNKNOWN, DB_RDONLY, 0, dbenv, NULL, &dbp)) != 0) + err(1, "%s", argv[0]); + + /* DB dump. */ + if (dflag) { + (void)__db_dump(dbp, NULL, 1); + if ((errno = dbp->close(dbp, 0)) != 0) + err(1, "close"); + exit (0); + } + + /* Get a cursor and step through the database. */ + if ((errno = dbp->cursor(dbp, NULL, &dbcp)) != 0) { + (void)dbp->close(dbp, 0); + err(1, "cursor"); + } + + /* Print out the header. */ + pheader(dbp, checkprint); + + /* Print out the key/data pairs. */ + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + while ((errno = dbcp->c_get(dbcp, &key, &data, DB_NEXT)) == 0) { + if (dbp->type != DB_RECNO && + (errno = __db_prdbt(&key, checkprint, stdout)) != 0) + break; + if ((errno = __db_prdbt(&data, checkprint, stdout)) != 0) + break; + } + + if (errno != DB_NOTFOUND) + err(1, "cursor get"); + + if ((errno = dbp->close(dbp, 0)) != 0) + err(1, "close"); + return (0); +} + +/* + * db_init -- + * Initialize the environment. + */ +DB_ENV * +db_init(home) + char *home; +{ + DB_ENV *dbenv; + + if ((dbenv = (DB_ENV *)calloc(sizeof(DB_ENV), 1)) == NULL) { + errno = ENOMEM; + err(1, NULL); + } + dbenv->db_errfile = stderr; + dbenv->db_errpfx = progname; + + if ((errno = + db_appinit(home, NULL, dbenv, DB_CREATE | DB_USE_ENVIRON)) != 0) + err(1, "db_appinit"); + return (dbenv); +} + +/* + * pheader -- + * Write out the header information. + */ +void +pheader(dbp, pflag) + DB *dbp; + int pflag; +{ + DB_BTREE_STAT *btsp; + HTAB *hashp; + HASHHDR *hdr; + db_pgno_t pgno; + + printf("format=%s\n", pflag ? "print" : "bytevalue"); + switch (dbp->type) { + case DB_BTREE: + printf("type=btree\n"); + if ((errno = dbp->stat(dbp, &btsp, NULL, 0)) != 0) + err(1, "dbp->stat"); + if (F_ISSET(dbp, DB_BT_RECNUM)) + printf("recnum=1\n"); + if (btsp->bt_maxkey != 0) + printf("bt_maxkey=%lu\n", (u_long)btsp->bt_maxkey); + if (btsp->bt_minkey != 0) + printf("bt_minkey=%lu\n", (u_long)btsp->bt_minkey); + break; + case DB_HASH: + printf("type=hash\n"); + hashp = dbp->internal; + pgno = PGNO_METADATA; + if (memp_fget(dbp->mpf, &pgno, 0, &hdr) == 0) { + if (hdr->ffactor != 0) + printf("h_ffactor=%lu\n", (u_long)hdr->ffactor); + if (hdr->nelem != 0) + printf("h_nelem=%lu\n", (u_long)hdr->nelem); + (void)memp_fput(dbp->mpf, hdr, 0); + } + break; + case DB_RECNO: + printf("type=recno\n"); + if (F_ISSET(dbp, DB_RE_RENUMBER)) + printf("renumber=1\n"); + if (F_ISSET(dbp, DB_RE_FIXEDLEN)) + printf("re_len=%lu\n", (u_long)btsp->bt_re_len); + if (F_ISSET(dbp, DB_RE_PAD)) + printf("re_pad=%#x\n", btsp->bt_re_pad); + break; + case DB_UNKNOWN: + abort(); + /* NOTREACHED */ + } + + if (F_ISSET(dbp, DB_AM_DUP)) + printf("duplicates=1\n"); + + if (dbp->dbenv->db_lorder != 0) + printf("db_lorder=%lu\n", (u_long)dbp->dbenv->db_lorder); + + if (!F_ISSET(dbp, DB_AM_PGDEF)) + printf("db_pagesize=%lu\n", (u_long)dbp->pgsize); + + printf("HEADER=END\n"); +} + +/* + * usage -- + * Display the usage message. + */ +void +usage() +{ + (void)fprintf(stderr, + "usage: db_dump [-dp] [-f file] [-h home] db_file\n"); + exit(1); +} diff --git a/ldap/servers/slapd/back-ldbm/uniqueid2entry.c b/ldap/servers/slapd/back-ldbm/uniqueid2entry.c new file mode 100644 index 00000000..31f53898 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/uniqueid2entry.c @@ -0,0 +1,74 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* uniqueid2entry.c - given a dn return an entry */ + +#include "back-ldbm.h" + +/* + * uniqueid2entry - look up uniqueid in the cache/indexes and return the + * corresponding entry. + */ + +struct backentry * +uniqueid2entry( + backend *be, + const char *uniqueid, + back_txn *txn, + int *err +) +{ +#ifdef UUIDCACHE_ON + ldbm_instance *inst = (ldbm_instance *) be->be_instance_info; +#endif + struct berval idv; + IDList *idl = NULL; + struct backentry *e = NULL; + + LDAPDebug( LDAP_DEBUG_TRACE, "=> uniqueid2entry \"%s\"\n", uniqueid, + 0, 0 ); +#ifdef UUIDCACHE_ON + e = cache_find_uuid(&inst->inst_cache, uniqueid); +#endif + if (e == NULL) { + /* convert dn to entry id */ + *err = 0; + idv.bv_val = (void*)uniqueid; + idv.bv_len = strlen( idv.bv_val ); + + if ( (idl = index_read( be, SLAPI_ATTR_UNIQUEID, indextype_EQUALITY, &idv, txn, + err )) == NULL ) { + if ( *err != 0 && *err != DB_NOTFOUND ) { + goto ext; + } + } else { + /* convert entry id to entry */ + if ( (e = id2entry( be, idl_firstid( idl ), txn, err )) + != NULL ) { + goto ext; + } else { + if ( *err != 0 && *err != DB_NOTFOUND ) { + goto ext; + } + /* + * this is pretty bad anyway. the dn was in the + * SLAPI_ATTR_UNIQUEID index, but we could not + * read the entry from the id2entry index. + * what should we do? + */ + } + } + } else { + goto ext; + } + +ext: + if (NULL != idl) { + slapi_ch_free( (void**)&idl); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= uniqueid2entry %p\n", e, 0, 0 ); + return( e ); +} diff --git a/ldap/servers/slapd/back-ldbm/upgrade.c b/ldap/servers/slapd/back-ldbm/upgrade.c new file mode 100644 index 00000000..1c0bfb9d --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/upgrade.c @@ -0,0 +1,352 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ + +/* upgrade.c --- upgrade from a previous version of the database */ + +#include "back-ldbm.h" + +#if 0 +static char* filename = "upgrade.c"; +#endif +/* + * ldbm_compat_versions holds DBVERSION strings for all versions of the + * database with which we are (upwards) compatible. If check_db_version + * encounters a database with a version that is not listed in this array, + * we display a warning message. + */ + +db_upgrade_info ldbm_version_suss[] = { +#if defined(USE_NEW_IDL) + {LDBM_VERSION,DBVERSION_NEW_IDL,DBVERSION_NO_UPGRADE}, + {LDBM_VERSION_OLD,DBVERSION_OLD_IDL,DBVERSION_NO_UPGRADE}, +#else + /* default: old idl (DS6.2) */ + {LDBM_VERSION_NEW,DBVERSION_NEW_IDL,DBVERSION_NO_UPGRADE}, + {LDBM_VERSION,DBVERSION_OLD_IDL,DBVERSION_NO_UPGRADE}, +#endif + {LDBM_VERSION_61,DBVERSION_NEW_IDL,DBVERSION_UPGRADE_3_4}, + {LDBM_VERSION_60,DBVERSION_OLD_IDL,DBVERSION_UPGRADE_3_4}, + {NULL,0,0} +}; + + +/* clear the following flag to suppress "database files do not exist" warning +int ldbm_warn_if_no_db = 0; +*/ +/* global LDBM version in the db home */ + +int +lookup_dbversion(char *dbversion, int flag) +{ + int i, matched = 0; + int rval = 0; + + for ( i = 0; ldbm_version_suss[i].old_version_string != NULL; ++i ) + { + if ( strcmp( dbversion, ldbm_version_suss[i].old_version_string ) == 0 ) + { + matched = 1; + break; + } + } + if ( matched ) + { + if ( flag & DBVERSION_TYPE ) + { + rval |= ldbm_version_suss[i].type; + } + if ( flag & DBVERSION_ACTION ) + { + rval |= ldbm_version_suss[i].action; + } + } + return rval; +} + +/* + * this function reads the db/DBVERSION file and check + * 1) if the db version is supported, and + * 2) if the db version requires some migration operation + * + * return: 0: supported + * DBVERSION_NOT_SUPPORTED: not supported + * + * action: 0: nothing is needed + * DBVERSION_UPGRADE_3_4: db3->db4 uprev is needed + */ +int +check_db_version( struct ldbminfo *li, int *action ) +{ + int value = 0; + char ldbmversion[BUFSIZ]; + char dataversion[BUFSIZ]; + + *action = 0; + dbversion_read(li, li->li_directory,ldbmversion,dataversion); + if (0 == strlen(ldbmversion)) + return 0; + + value = lookup_dbversion( ldbmversion, DBVERSION_TYPE | DBVERSION_ACTION); + if ( !value ) + { + LDAPDebug( LDAP_DEBUG_ANY, + "ERROR: Database version mismatch (expecting " + "'%s' but found '%s' in directory %s)\n", + LDBM_VERSION, ldbmversion, li->li_directory ); + /* + * A non-zero return here will cause slapd to exit during startup. + */ + return DBVERSION_NOT_SUPPORTED; + } + if ( value & DBVERSION_UPGRADE_3_4 ) + { + dblayer_set_recovery_required(li); + *action = DBVERSION_UPGRADE_3_4; + } + return 0; +} + +/* + * this function reads the db/<inst>/DBVERSION file and check + * 1) if the db version is supported, and + * 2) if the db version matches the idl configuration + * (nsslapd-idl-switch: new|old) + * note that old idl will disappear from the next major update (6.5? 7.0?) + * + * return: 0: supported and the version matched + * DBVERSION_NEED_IDL_OLD2NEW: old->new uprev is needed + * (used in convindices) + * DBVERSION_NEED_IDL_NEW2OLD: old db is found, for the new idl config + * DBVERSION_NOT_SUPPORTED: not supported + * + * DBVERSION_UPGRADE_3_4: db3->db4 uprev is needed + */ +int +check_db_inst_version( ldbm_instance *inst ) +{ + int value = 0; + char ldbmversion[BUFSIZ]; + char dataversion[BUFSIZ]; + int rval = 0; + char inst_dir[MAXPATHLEN*2]; + char *inst_dirp = NULL; + + inst_dirp = + dblayer_get_full_inst_dir(inst->inst_li, inst, inst_dir, MAXPATHLEN*2); + + dbversion_read(inst->inst_li, inst_dirp,ldbmversion,dataversion); + if (0 == strlen(ldbmversion)) + return rval; + + value = lookup_dbversion( ldbmversion, DBVERSION_TYPE | DBVERSION_ACTION); + if ( !value ) + { + LDAPDebug( LDAP_DEBUG_ANY, + "ERROR: Database version mismatch (expecting " + "'%s' but found '%s' in directory %s)\n", + LDBM_VERSION, ldbmversion, inst->inst_dir_name ); + /* + * A non-zero return here will cause slapd to exit during startup. + */ + return DBVERSION_NOT_SUPPORTED; + } + + /* recognize the difference between an old/new database regarding idl + * (406922) */ + if (idl_get_idl_new() && !(value & DBVERSION_NEW_IDL) ) + { + rval |= DBVERSION_NEED_IDL_OLD2NEW; + } + else if (!idl_get_idl_new() && !(value & DBVERSION_OLD_IDL) ) + { + rval |= DBVERSION_NEED_IDL_NEW2OLD; + } + if ( value & DBVERSION_UPGRADE_3_4 ) + { + rval |= DBVERSION_UPGRADE_3_4; + } + if (inst_dirp != inst_dir) + slapi_ch_free_string(&inst_dirp); + return rval; +} + +#if defined(UPGRADEDB) +/* + * adjust_idl_switch + * if the current nsslapd-idl-switch is different from ldbmversion, + * update the value of nsslapd-idl-switch (in LDBM_CONFIG_ENTRY) + */ +int +adjust_idl_switch(char *ldbmversion, struct ldbminfo *li) +{ + int rval = 0; + + li->li_flags |= LI_FORCE_MOD_CONFIG; +#if defined(USE_NEW_IDL) + if ((0 == strcmp(ldbmversion, LDBM_VERSION)) || + (0 == strcmp(ldbmversion, LDBM_VERSION_61))) /* db: new idl */ +#else + if ((0 == strcmp(ldbmversion, LDBM_VERSION_NEW)) || + (0 == strcmp(ldbmversion, LDBM_VERSION_61))) /* db: new idl */ +#endif + { + if (!idl_get_idl_new()) /* config: old idl */ + { + replace_ldbm_config_value(CONFIG_IDL_SWITCH, "new", li); + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: Dbversion %s does not meet nsslapd-idl-switch: \"old\"; " + "nsslapd-idl-switch is updated to \"new\"\n", + + ldbmversion, 0, 0); + } + } +#if defined(USE_NEW_IDL) + else if ((0 == strcmp(ldbmversion, LDBM_VERSION_OLD)) || + (0 == strcmp(ldbmversion, LDBM_VERSION_60))) /* db: old */ +#else + else if ((0 == strcmp(ldbmversion, LDBM_VERSION)) || /* ds6.2: old */ + (0 == strcmp(ldbmversion, LDBM_VERSION_60))) /* db: old */ +#endif + { + if (idl_get_idl_new()) /* config: new */ + { + replace_ldbm_config_value(CONFIG_IDL_SWITCH, "old", li); + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: Dbversion %s does not meet nsslapd-idl-switch: \"new\"; " + "nsslapd-idl-switch is updated to \"old\"\n", + ldbmversion, 0, 0); + } + } + else + { + LDAPDebug(LDAP_DEBUG_ANY, + "Warning: Dbversion %s is not supported\n", + ldbmversion, 0, 0); + rval = 1; + } + + /* ldbminfo is a common resource; should clean up when the job is done */ + li->li_flags &= ~LI_FORCE_MOD_CONFIG; + return rval; +} +#endif + +/* Do the work to upgrade a database if needed */ +/* When we're called, the database files have been opened, and any +recovery needed has been performed. */ +int ldbm_upgrade(ldbm_instance *inst, int action) +{ + int rval = 0; + + if (0 == action) + { + return rval; + } + + if (action & DBVERSION_UPGRADE_3_4) /* upgrade from db3 to db4 */ + { + /* basically, db4 supports db3. + * so, what we need to do is rename XXX.db3 to XXX.db4 */ + + int rval = dblayer_update_db_ext(inst, LDBM_SUFFIX_OLD, LDBM_SUFFIX); + if (0 == rval) + { +#if defined(USE_NEW_IDL) + if (idl_get_idl_new()) + { + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm_upgrade: Upgrading instance %s to %s%s is successfully done.\n", + inst->inst_name, LDBM_VERSION_BASE, PRODUCTTEXT); + } + else + { + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm_upgrade: Upgrading instance %s to %s%s is successfully done.\n", + inst->inst_name, LDBM_VERSION_OLD, 0); + } +#else + LDAPDebug(LDAP_DEBUG_ANY, + "ldbm_upgrade: Upgrading instance %s to %s%s is successfully done.\n", + inst->inst_name, LDBM_VERSION_BASE, PRODUCTTEXT); +#endif + } + else + { + /* recovery effort ... */ + dblayer_update_db_ext(inst, LDBM_SUFFIX, LDBM_SUFFIX_OLD); + return rval; + } + } + + return 0; /* Means that the database is new */ +} + +/* Here's the upgrade process : + Delete all the keys from the parentid index + Scan the id2entry file: + Remove any hassubordinates attribute present + Update the parentid index, maintaining a hash of high-count parents + Scan the newly created parentid index updating the subordinatecount attributes. + + Most of the functionality is implemented in the import code. + */ +#if 0 +static int upgrade_db_3x_40(backend *be) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + int ret = 0; + back_txn txn; + + static char* indexes_modified[] = {"parentid", "numsubordinates", NULL}; + + LDAPDebug( LDAP_DEBUG_ANY, "WARNING: Detected a database older than this server, upgrading data...\n",0,0,0); + + dblayer_txn_init(li,&txn); + ret = dblayer_txn_begin(li,NULL,&txn); + if (0 != ret) { + ldbm_nasty(filename,69,ret); + goto error; + } + ret = indexfile_delete_all_keys(be,"parentid",&txn); + if (0 != ret) { + ldbm_nasty(filename,70,ret); + goto error; + } + + { + Slapi_Mods smods; + slapi_mods_init(&smods,1); + /* Mods are to remove the hassubordinates attribute */ + slapi_mods_add(&smods, LDAP_MOD_DELETE, "hassubordinates", 0, NULL); + /* This function takes care of generating the subordinatecount attribute and indexing it */ + ret = indexfile_primary_modifyall(be,slapi_mods_get_ldapmods_byref(&smods),indexes_modified,&txn); + slapi_mods_done(&smods); + } + + if (0 != ret) { + ldbm_nasty(filename,61,ret); + } + +error: + if (0 != ret ) { + dblayer_txn_abort(li,&txn); + } else { + ret = dblayer_txn_commit(li,&txn); + if (0 != ret) { + ldbm_nasty(filename,60,ret); + } else { + /* Now update DBVERSION file */ + } + } + if (0 == ret) { + LDAPDebug( LDAP_DEBUG_ANY, "...upgrade complete.\n",0,0,0); + } else { + LDAPDebug( LDAP_DEBUG_ANY, "ERROR: Attempt to upgrade the older database FAILED.\n",0,0,0); + } + return ret; +} + +#endif diff --git a/ldap/servers/slapd/back-ldbm/vlv.c b/ldap/servers/slapd/back-ldbm/vlv.c new file mode 100644 index 00000000..8397ef0e --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/vlv.c @@ -0,0 +1,1947 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* vlv.c */ + + +/* + * References to on-line documentation here. + * + * http://BLUES/users/dboreham/publish/Design_Documentation/RFCs/draft-ietf-asid-ldapv3-virtuallistview-01.html + * http://warp.mcom.com/server/directory-server/clientsdk/hammerhead/design/virtuallistview.html + * ftp://ftp.ietf.org/internet-drafts/draft-ietf-ldapext-ldapv3-vlv-00.txt + * http://rocknroll/users/merrells/publish/vlvimplementation.html + */ + + +#include "back-ldbm.h" +#include "vlv_srch.h" +#include "vlv_key.h" + +static PRUint32 vlv_trim_candidates_byindex(PRUint32 length, const struct vlv_request *vlv_request_control); +static PRUint32 vlv_trim_candidates_byvalue(backend *be, const IDList *candidates, const sort_spec* sort_control, const struct vlv_request *vlv_request_control); +static int vlv_build_candidate_list( backend *be, struct vlvIndex* p, const struct vlv_request *vlv_request_control, IDList** candidates, struct vlv_response *vlv_response_control); + +/* New mutex for vlv locking +PRRWLock * vlvSearchList_lock=NULL; +static struct vlvSearch *vlvSearchList= NULL; +*/ + +#define ISLEGACY(be) (be?(be->be_instance_info?(((ldbm_instance *)be->be_instance_info)->inst_li?(((ldbm_instance *)be->be_instance_info)->inst_li->li_legacy_errcode):0):0):0) + +/* Callback to add a new VLV Search specification. Added write lock.*/ + +int vlv_AddSearchEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + ldbm_instance *inst = (ldbm_instance *)arg; + struct vlvSearch* newVlvSearch= vlvSearch_new(); + backend *be = inst->inst_be; + + vlvSearch_init(newVlvSearch, pb, entryBefore, inst); + PR_RWLock_Wlock(be->vlvSearchList_lock); + vlvSearch_addtolist(newVlvSearch, (struct vlvSearch **)&be->vlvSearchList); + PR_RWLock_Unlock(be->vlvSearchList_lock); + return SLAPI_DSE_CALLBACK_OK; +} + +/* Callback to add a new VLV Index specification. Added write lock.*/ + +int vlv_AddIndexEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + struct vlvSearch *parent; + backend *be= ((ldbm_instance*)arg)->inst_be; + Slapi_DN parentdn; + + slapi_sdn_init(&parentdn); + slapi_sdn_get_parent(slapi_entry_get_sdn(entryBefore),&parentdn); + { + PR_RWLock_Wlock(be->vlvSearchList_lock); + parent= vlvSearch_finddn((struct vlvSearch *)be->vlvSearchList, &parentdn); + if(parent!=NULL) + { + struct vlvIndex* newVlvIndex= vlvIndex_new(); + newVlvIndex->vlv_be=be; + vlvIndex_init(newVlvIndex, be, parent, entryBefore); + vlvSearch_addIndex(parent, newVlvIndex); + } + PR_RWLock_Unlock(be->vlvSearchList_lock); + } + slapi_sdn_done(&parentdn); + return SLAPI_DSE_CALLBACK_OK; +} + +/* Callback to delete a VLV Index specification. Added write lock.*/ + +int vlv_DeleteSearchEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + struct vlvSearch* p=NULL; + backend *be= ((ldbm_instance*)arg)->inst_be; + + PR_RWLock_Wlock(be->vlvSearchList_lock); + p = vlvSearch_finddn((struct vlvSearch *)be->vlvSearchList, slapi_entry_get_sdn(entryBefore)); + if(p!=NULL) + { + LDAPDebug( LDAP_DEBUG_ANY, "Deleted Virtual List View Search (%s).\n", p->vlv_name, 0, 0); + vlvSearch_removefromlist((struct vlvSearch **)&be->vlvSearchList,p->vlv_dn); + vlvSearch_delete(&p); + } + PR_RWLock_Unlock(be->vlvSearchList_lock); + return SLAPI_DSE_CALLBACK_OK; +} + + +/* Stub Callback to delete a VLV Index specification.*/ + +int vlv_DeleteIndexEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + LDAPDebug( LDAP_DEBUG_ANY, "Deleted Virtual List View Index.\n", 0, 0, 0); + return SLAPI_DSE_CALLBACK_OK; +} + + +/* Callback to modify a VLV Search specification. Added read lock.*/ + +int vlv_ModifySearchEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + struct vlvSearch* p=NULL; + backend *be= ((ldbm_instance*)arg)->inst_be; + + PR_RWLock_Rlock(be->vlvSearchList_lock); + p= vlvSearch_finddn((struct vlvSearch *)be->vlvSearchList, slapi_entry_get_sdn(entryBefore)); + if(p!=NULL) + { + LDAPDebug( LDAP_DEBUG_ANY, "Modified Virtual List View Search (%s), which will be enabled when the database is rebuilt.\n", p->vlv_name, 0, 0); + } + PR_RWLock_Unlock(be->vlvSearchList_lock); + return SLAPI_DSE_CALLBACK_DO_NOT_APPLY; +} + + +/* Stub callback to modify a VLV Index specification. */ + +int vlv_ModifyIndexEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + LDAPDebug( LDAP_DEBUG_ANY, "Modified Virtual List View Index.\n", 0, 0, 0); + return SLAPI_DSE_CALLBACK_DO_NOT_APPLY; +} + + +/* Callback to rename a VLV Search specification. Added read lock.*/ + +int vlv_ModifyRDNSearchEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + struct vlvSearch* p=NULL; + backend *be= ((ldbm_instance*)arg)->inst_be; + + PR_RWLock_Rlock(be->vlvSearchList_lock); + p= vlvSearch_finddn((struct vlvSearch *)be->vlvSearchList, slapi_entry_get_sdn(entryBefore)); + if(p!=NULL) + { + LDAPDebug( LDAP_DEBUG_ANY, "Modified Virtual List View Search (%s), which will be enabled when the database is rebuilt.\n", p->vlv_name, 0, 0); + } + PR_RWLock_Unlock(be->vlvSearchList_lock); + return SLAPI_DSE_CALLBACK_DO_NOT_APPLY; +} + + +/* Stub callback to modify a VLV Index specification. */ + +int vlv_ModifyRDNIndexEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + LDAPDebug( LDAP_DEBUG_ANY, "Modified Virtual List View Index.\n", 0, 0, 0); + return SLAPI_DSE_CALLBACK_DO_NOT_APPLY; +} + +/* Something may have just read a VLV Entry. */ + +int vlv_SearchIndexEntry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + char *name= slapi_entry_attr_get_charptr(entryBefore,type_vlvName); + backend *be= ((ldbm_instance*)arg)->inst_be; + if (name!=NULL) + { + struct vlvIndex* p= vlv_find_searchname(name, be); /* lock list */ + slapi_ch_free((void **) &name); + if(p!=NULL) + { + if(vlvIndex_enabled(p)) + { + slapi_entry_attr_set_charptr(entryBefore, type_vlvEnabled, "1"); + } + else + { + slapi_entry_attr_set_charptr(entryBefore, type_vlvEnabled, "0"); + } + slapi_entry_attr_set_ulong(entryBefore, type_vlvUses, p->vlv_uses); + } + } + return SLAPI_DSE_CALLBACK_OK; +} + +/* Handle results of a search for objectclass "vlvIndex". Called by vlv_init at inittime -- no need to lock*/ + +static int +vlv_init_index_entry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + struct vlvIndex* newVlvIndex; + struct vlvSearch* pSearch; + Slapi_Backend *be= ((ldbm_instance*)arg)->inst_be; + char ebuf[BUFSIZ]; + + if(be!=NULL) + { + Slapi_DN parentdn; + + slapi_sdn_init(&parentdn); + newVlvIndex= vlvIndex_new(); + slapi_sdn_get_parent(slapi_entry_get_sdn(entryBefore),&parentdn); + pSearch= vlvSearch_finddn((struct vlvSearch *)be->vlvSearchList, &parentdn); + if (pSearch == NULL) { + LDAPDebug( LDAP_DEBUG_ANY, "Parent doesn't exist for entry %s.\n", + escape_string(slapi_entry_get_dn(entryBefore), ebuf), 0, 0); + } + else { + vlvIndex_init(newVlvIndex, be, pSearch, entryBefore); + vlvSearch_addIndex(pSearch, newVlvIndex); + } + slapi_sdn_done(&parentdn); + } + return SLAPI_DSE_CALLBACK_OK; +} + +/* Handle results of a search for objectclass "vlvSearch". Called by vlv_init at inittime -- no need to lock*/ + +static int +vlv_init_search_entry(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* entryAfter, int *returncode, char *returntext, void *arg) +{ + struct vlvSearch* newVlvSearch= vlvSearch_new(); + ldbm_instance *inst = (ldbm_instance*)arg; + backend *be= inst->inst_be; + + vlvSearch_init(newVlvSearch, pb, entryBefore, inst); + vlvSearch_addtolist(newVlvSearch, (struct vlvSearch **)&be->vlvSearchList); + return SLAPI_DSE_CALLBACK_OK; +} + +/* Look at a new entry, and the set of VLV searches, and see whether +there are any which have deferred initialization and which can now +be initialized given the new entry. Added write lock. */ + + +void vlv_grok_new_import_entry(const struct backentry *e, backend *be) +{ + struct vlvSearch* p = NULL; + static int seen_them_all = 0; + int any_not_done = 0; + + + PR_RWLock_Wlock(be->vlvSearchList_lock); + if (seen_them_all) { + PR_RWLock_Unlock(be->vlvSearchList_lock); + return; + } + p=(struct vlvSearch *)be->vlvSearchList; + + /* Walk the list of searches */ + for(;p!=NULL;p= p->vlv_next) + /* is this one not initialized ? */ + if (0 == p->vlv_initialized) { + any_not_done = 1; + /* Is its base the entry we have here ? */ + if (0 == slapi_sdn_compare(backentry_get_sdn(e),p->vlv_base) ) { + /* Then initialize it */ + vlvSearch_reinit(p,e); + } + } + if (!any_not_done) { + seen_them_all = 1; + } + PR_RWLock_Unlock(be->vlvSearchList_lock); +} + +/* + * Search for the VLV entries which describe the pre-computed indexes we + * support. Register administartion DSE callback functions. + * This is exported to the backend initialisation routine. + * 'inst' may be NULL for non-slapd initialization... + */ +int +vlv_init(ldbm_instance *inst) +{ + /* The FE DSE *must* be initialised before we get here */ + int return_value= LDAP_SUCCESS; + int scope= LDAP_SCOPE_SUBTREE; + char *basedn, buf[512]; + const char *searchfilter = "(objectclass=vlvsearch)"; + const char *indexfilter = "(objectclass=vlvindex)"; + backend *be= inst->inst_be; + + /* Initialize lock first time through */ + if(be->vlvSearchList_lock == NULL) { + char *rwlockname = (char *)slapi_ch_malloc(sizeof("vlvSearchList") + + strlen(inst->inst_name) + 2); + sprintf(rwlockname, "vlvSearchList_%s", inst->inst_name); + be->vlvSearchList_lock = PR_NewRWLock(PR_RWLOCK_RANK_NONE, rwlockname); + slapi_ch_free((void**)&rwlockname); + } + if (NULL != (struct vlvSearch *)be->vlvSearchList) + { + struct vlvSearch *t = NULL; + struct vlvSearch *nt = NULL; + PR_RWLock_Wlock(be->vlvSearchList_lock); + for (t = (struct vlvSearch *)be->vlvSearchList; NULL != t; ) + { + nt = t->vlv_next; + vlvSearch_delete(&t); + t = nt; + } + be->vlvSearchList = NULL; + PR_RWLock_Unlock(be->vlvSearchList_lock); + } + if (inst == NULL) { + basedn = NULL; + } else { + sprintf(buf, "cn=%s,cn=%s,cn=plugins,cn=config", + inst->inst_name, inst->inst_li->li_plugin->plg_name); + basedn = buf; + } + + /* Find the VLV Search Entries */ + { + Slapi_PBlock *tmp_pb; + slapi_config_register_callback(SLAPI_OPERATION_SEARCH,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_init_search_entry,(void *)inst); + tmp_pb= slapi_search_internal(basedn, scope, searchfilter, NULL, NULL, 0); + slapi_config_remove_callback(SLAPI_OPERATION_SEARCH,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_init_search_entry); + slapi_free_search_results_internal(tmp_pb); + slapi_pblock_destroy(tmp_pb); + } + + /* Find the VLV Index Entries */ + { + Slapi_PBlock *tmp_pb; + slapi_config_register_callback(SLAPI_OPERATION_SEARCH,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_init_index_entry,(void*)inst); + tmp_pb= slapi_search_internal(basedn, scope, indexfilter, NULL, NULL, 0); + slapi_config_remove_callback(SLAPI_OPERATION_SEARCH,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_init_index_entry); + slapi_free_search_results_internal(tmp_pb); + slapi_pblock_destroy(tmp_pb); + } + + /* Only need to register these callbacks for SLAPD mode... */ + if(basedn!=NULL) + { + slapi_config_register_callback(SLAPI_OPERATION_SEARCH,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_SearchIndexEntry,(void*)inst); + slapi_config_register_callback(SLAPI_OPERATION_ADD,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_AddSearchEntry,(void*)inst); + slapi_config_register_callback(SLAPI_OPERATION_ADD,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_AddIndexEntry,(void*)inst); + slapi_config_register_callback(SLAPI_OPERATION_MODIFY,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_ModifySearchEntry,(void*)inst); + slapi_config_register_callback(SLAPI_OPERATION_MODIFY,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_ModifyIndexEntry,(void*)inst); + slapi_config_register_callback(SLAPI_OPERATION_DELETE,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_DeleteSearchEntry,(void*)inst); + slapi_config_register_callback(SLAPI_OPERATION_DELETE,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_DeleteIndexEntry,(void*)inst); + slapi_config_register_callback(SLAPI_OPERATION_MODRDN,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_ModifyRDNSearchEntry,(void*)inst); + slapi_config_register_callback(SLAPI_OPERATION_MODRDN,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_ModifyRDNIndexEntry,(void*)inst); + } + + return return_value; +} + +/* Removes callbacks from above when instance is removed. */ + +int +vlv_remove_callbacks(ldbm_instance *inst) { + + int return_value= LDAP_SUCCESS; + int scope= LDAP_SCOPE_SUBTREE; + char *basedn, buf[512]; + const char *searchfilter = "(objectclass=vlvsearch)"; + const char *indexfilter = "(objectclass=vlvindex)"; + + if (inst == NULL) { + basedn = NULL; + } else { + sprintf(buf, "cn=%s,cn=%s,cn=plugins,cn=config", + inst->inst_name, inst->inst_li->li_plugin->plg_name); + basedn = buf; + } + if(basedn!=NULL) + { + slapi_config_remove_callback(SLAPI_OPERATION_SEARCH,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_SearchIndexEntry); + slapi_config_remove_callback(SLAPI_OPERATION_ADD,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_AddSearchEntry); + slapi_config_remove_callback(SLAPI_OPERATION_ADD,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_AddIndexEntry); + slapi_config_remove_callback(SLAPI_OPERATION_MODIFY,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_ModifySearchEntry); + slapi_config_remove_callback(SLAPI_OPERATION_MODIFY,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_ModifyIndexEntry); + slapi_config_remove_callback(SLAPI_OPERATION_DELETE,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_DeleteSearchEntry); + slapi_config_remove_callback(SLAPI_OPERATION_DELETE,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_DeleteIndexEntry); + slapi_config_remove_callback(SLAPI_OPERATION_MODRDN,DSE_FLAG_PREOP,basedn,scope,searchfilter,vlv_ModifyRDNSearchEntry); + slapi_config_remove_callback(SLAPI_OPERATION_MODRDN,DSE_FLAG_PREOP,basedn,scope,indexfilter,vlv_ModifyRDNIndexEntry); + } + return return_value; +} + +/* Find an enabled index which matches this description. */ + +static struct vlvIndex* +vlv_find_search(backend *be, const Slapi_DN *base, int scope, const char *filter, const sort_spec* sort_control) +{ + return vlvSearch_findenabled(be,(struct vlvSearch *)be->vlvSearchList,base,scope,filter,sort_control); +} + + +/* Find a search which matches this name. Added read lock. */ + +struct vlvIndex* +vlv_find_searchname(const char * name, backend *be) +{ + struct vlvIndex *p=NULL; + + PR_RWLock_Rlock(be->vlvSearchList_lock); + p=vlvSearch_findname((struct vlvSearch *)be->vlvSearchList,name); + PR_RWLock_Unlock(be->vlvSearchList_lock); + return p; +} + +/* Find a search which matches this indexname. Added to read lock */ + +struct vlvIndex* +vlv_find_indexname(const char * name, backend *be) +{ + + struct vlvIndex *p=NULL; + + PR_RWLock_Rlock(be->vlvSearchList_lock); + p=vlvSearch_findindexname((struct vlvSearch *)be->vlvSearchList,name); + PR_RWLock_Unlock(be->vlvSearchList_lock); + return p; +} + + +/* Get a list of known VLV Indexes. Added read lock */ + +char * +vlv_getindexnames(backend *be) +{ + char *n=NULL; + + PR_RWLock_Rlock(be->vlvSearchList_lock); + n=vlvSearch_getnames((struct vlvSearch *)be->vlvSearchList); + PR_RWLock_Unlock(be->vlvSearchList_lock); + return n; +} + +/* Return the list of VLV indices to the import code. Added read lock */ + +void +vlv_getindices(IFP callback_fn,void *param, backend *be) +{ + /* Traverse the list, calling the import code's callback function */ + struct vlvSearch* ps = NULL; + + PR_RWLock_Rlock(be->vlvSearchList_lock); + ps = (struct vlvSearch *)be->vlvSearchList; + for(;ps!=NULL;ps= ps->vlv_next) + { + struct vlvIndex* pi= ps->vlv_index; + for(;pi!=NULL;pi= pi->vlv_next) + { + callback_fn(pi->vlv_attrinfo,param); + } + } + PR_RWLock_Unlock(be->vlvSearchList_lock); +} + +/* + * Create a key for the entry in the vlv index. + * + * The key is a composite of a value from each sorted attribute. + * + * If a sorted attribute has many values, then the key is built + * with the attribute value with the lowest value. + * + * The primary sorted attribute value is followed by a 0x00 to + * ensure that short attribute values appear before longer ones. + * + * Many entries may have the same attribute values, which would + * generate the same composite key, so we append the EntryID + * to ensure the uniqueness of the key. + * + * Always creates a key. Never returns NULL. + */ +static struct vlv_key * +vlv_create_key(struct vlvIndex* p, struct backentry* e) +{ + struct berval val, *lowest_value = NULL; + unsigned char char_min = 0x00; + unsigned char char_max = 0xFF; + struct vlv_key *key= vlv_key_new(); + if(p->vlv_sortkey!=NULL) + { + /* Foreach sorted attribute... */ + int sortattr= 0; + while(p->vlv_sortkey[sortattr]!=NULL) + { + Slapi_Attr* attr= attrlist_find(e->ep_entry->e_attrs, p->vlv_sortkey[sortattr]->sk_attrtype); + { + /* + * If there's a matching rule associated with the sorted + * attribute then use the indexer to mangle the attr values. + * This ensures that the international characters will + * collate in the correct order. + */ + + /* xxxPINAKI */ + /* need to free some stuff! */ + Slapi_Value **cvalue = NULL; + struct berval **value = NULL; + int free_value= 0; + if (attr != NULL && !valueset_isempty(&attr->a_present_values)) + { + /* Sorted attribute found. */ + int totalattrs; + if (p->vlv_sortkey[sortattr]->sk_matchruleoid==NULL) + { + /* No matching rule. Syntax Plugin mangles value. */ + Slapi_Value **va= valueset_get_valuearray(&attr->a_present_values); + slapi_call_syntax_values2keys_sv( p->vlv_syntax_plugin[sortattr], va, &cvalue, LDAP_FILTER_EQUALITY ); + valuearray_get_bervalarray(cvalue,&value); + + /* XXXSD need to free some more stuff */ + { + int numval; + for (numval=0; cvalue&&cvalue[numval];numval++) { + slapi_value_free(&cvalue[numval]); + } + if (cvalue) + slapi_ch_free((void **)&cvalue); + } + + free_value= 1; + } + else + { + /* Matching rule. Do the magic mangling. Plugin owns the memory. */ + if(p->vlv_mrpb[sortattr]!=NULL) + { + /* xxxPINAKI */ + struct berval **bval=NULL; + Slapi_Value **va= valueset_get_valuearray(&attr->a_present_values); + valuearray_get_bervalarray(va,&bval); + matchrule_values_to_keys(p->vlv_mrpb[sortattr],bval,&value); + } + } + for(totalattrs=0;value[totalattrs]!=NULL;totalattrs++) {}; /* Total Number of Attributes */ + if(totalattrs==1) + { + lowest_value= value[0]; + } + else + { + lowest_value = attr_value_lowest(value, slapi_berval_cmp); + } + } /* end of if (attr != NULL && ...) */ + if(p->vlv_sortkey[sortattr]->sk_reverseorder) + { + /* + * This attribute is reverse sorted, so we must + * invert the attribute value so that the keys + * will be in the correct order. + */ + unsigned int i; + char *attributeValue = NULL; + /* Bug 605477 : Don't malloc 0 bytes */ + if (attr != NULL && lowest_value->bv_len != 0) { + attributeValue = (char*)slapi_ch_malloc(lowest_value->bv_len); + for(i=0;i<lowest_value->bv_len;i++) + { + attributeValue[i]= UCHAR_MAX - ((char*)lowest_value->bv_val)[i]; + } + val.bv_len= lowest_value->bv_len; + val.bv_val= (void*)attributeValue; + } else { + /* Reverse Sort: We use an attribute value of 0x00 when + * there is no attribute value or attrbute is absent + */ + val.bv_val= (void*)&char_min; + val.bv_len= 1; + } + vlv_key_addattr(key,&val); + slapi_ch_free((void**)&attributeValue); + } + else + { + /* + * This attribute is forward sorted, so add the + * attribute value to the end of all the keys. + */ + + /* If the forward-sorted attribute is absent or has no + * value, we need to use the value of 0xFF. + */ + if (attr != NULL && lowest_value->bv_len > 0) { + vlv_key_addattr(key,lowest_value); + } else { + val.bv_val = (void*)&char_max; + val.bv_len = 1; + vlv_key_addattr(key,&val); + } + } + if(sortattr==0) + { + /* + * If this is the first attribute (the typedown attribute) + * then it should be followed by a zero. This is to ensure + * that shorter attribute values appear before longer ones. + */ + char zero = 0; + val.bv_len= 1; + val.bv_val= (void*)&zero; + vlv_key_addattr(key,&val); + } + if(free_value) + { + ber_bvecfree(value); + } + } + sortattr++; + } + } + { + /* Append the EntryID to the key to ensure uniqueness */ + val.bv_len= sizeof(e->ep_id); + val.bv_val= (void*)&e->ep_id; + vlv_key_addattr(key,&val); + } + return key; +} + +/* + * Insert or Delete the entry to or from the index + */ + +static int +do_vlv_update_index(back_txn *txn, struct ldbminfo *li, Slapi_PBlock *pb, struct vlvIndex* pIndex, struct backentry* entry, int insert) +{ + backend *be; + int rc= 0; + DB *db = NULL; + DB_TXN *db_txn = NULL; + struct vlv_key *key = NULL; + + slapi_pblock_get(pb, SLAPI_BACKEND, &be); + + rc = dblayer_get_index_file(be, pIndex->vlv_attrinfo, &db, DBOPEN_CREATE); + if (rc != 0) { + if(rc != DB_LOCK_DEADLOCK) + LDAPDebug(LDAP_DEBUG_ANY, "VLV: can't get index file '%s' (err %d)\n", + pIndex->vlv_attrinfo->ai_type, rc, 0); + return rc; + } + + key = vlv_create_key(pIndex,entry); + if (NULL != txn) { + db_txn = txn->back_txn_txn; + } else { + /* Very bad idea to do this outside of a transaction */ + } + + if (insert) { + DBT data = {0}; + data.size = sizeof(entry->ep_id); + data.data = &entry->ep_id; + rc = db->put(db, db_txn, &key->key, &data, 0); + if (rc == 0) { + LDAPDebug(LDAP_DEBUG_TRACE, + "vlv_update_index: %s Insert %s ID=%lu\n", + pIndex->vlv_name, key->key.data, (u_long)entry->ep_id); + vlvIndex_increment_indexlength(pIndex, db, txn); + } else if (rc == DB_RUNRECOVERY) { + ldbm_nasty(pIndex->vlv_name,77,rc); + } else if(rc != DB_LOCK_DEADLOCK) { + /* jcm: This error is valid if the key already exists. + * Identical multi valued attr values could do this. */ + LDAPDebug(LDAP_DEBUG_TRACE, + "vlv_update_index: %s Insert %s ID=%lu FAILED\n", + pIndex->vlv_name, key->key.data, (u_long)entry->ep_id); + } + } else { + LDAPDebug(LDAP_DEBUG_TRACE, + "vlv_update_index: %s Delete %s\n", + pIndex->vlv_name, key->key.data, 0); + rc = db->del(db, db_txn, &key->key, 0); + if (rc == 0) { + vlvIndex_decrement_indexlength(pIndex, db, txn); + } else if (rc == DB_RUNRECOVERY) { + ldbm_nasty(pIndex->vlv_name,78,rc); + } else if (rc != DB_LOCK_DEADLOCK) { + LDAPDebug(LDAP_DEBUG_TRACE, + "vlv_update_index: %s Delete %s FAILED\n", + pIndex->vlv_name, key->key.data, 0); + } + } + + vlv_key_delete(&key); + dblayer_release_index_file(be, pIndex->vlv_attrinfo, db); + return rc; +} + +/* + * Given an entry modification check if a VLV index needs to be updated. + */ + +int +vlv_update_index(struct vlvIndex* p, back_txn *txn, struct ldbminfo *li, Slapi_PBlock *pb, struct backentry* oldEntry, struct backentry* newEntry) +{ + int return_value=0; + /* Check if the old entry is in this VLV index */ + if(oldEntry!=NULL) + { + if(slapi_sdn_scope_test(backentry_get_sdn(oldEntry),vlvIndex_getBase(p),vlvIndex_getScope(p))) + { + if(slapi_filter_test( pb, oldEntry->ep_entry, vlvIndex_getFilter(p), 0 /* No ACL Check */) == 0 ) + { + /* Remove the entry from the index */ + return_value=do_vlv_update_index(txn, li, pb, p, oldEntry, 0 /* Delete Key */); + } + } + } + /* Check if the new entry should be in the VLV index */ + if(newEntry!=NULL) + { + if(slapi_sdn_scope_test(backentry_get_sdn(newEntry),vlvIndex_getBase(p),vlvIndex_getScope(p))) + { + if(slapi_filter_test( pb, newEntry->ep_entry, vlvIndex_getFilter(p), 0 /* No ACL Check */) == 0 ) + { + /* Add the entry to the index */ + return_value=do_vlv_update_index(txn, li, pb, p, newEntry, 1 /* Insert Key */); + } + } + } + return return_value; +} + +/* + * Given an entry modification check if a VLV index needs to be updated. + * + * This is called for every modifying operation, so it must be very efficient. + * + * We need to know if we're adding, deleting, or modifying + * because we could be leaving and/or joining an index + * + * ADD: oldEntry==NULL && newEntry!=NULL + * DEL: oldEntry!=NULL && newEntry==NULL + * MOD: oldEntry!=NULL && newEntry!=NULL + * + * JCM: If only non-sorted attributes are changed, then the indexes don't need updating. + * JCM: Detecting this fact, given multi-valued atribibutes, might be tricky... + * Added write lock +*/ + +int +vlv_update_all_indexes(back_txn *txn, backend *be, Slapi_PBlock *pb, struct backentry* oldEntry, struct backentry* newEntry) +{ + int return_value= LDAP_SUCCESS; + struct vlvSearch* ps=NULL; + struct ldbminfo *li = ((ldbm_instance *)be->be_instance_info)->inst_li; + + PR_RWLock_Wlock(be->vlvSearchList_lock); + ps = (struct vlvSearch *)be->vlvSearchList; + for(;ps!=NULL;ps= ps->vlv_next) + { + struct vlvIndex* pi= ps->vlv_index; + for (return_value = LDAP_SUCCESS; return_value == LDAP_SUCCESS && pi!=NULL; pi=pi->vlv_next) + return_value=vlv_update_index(pi, txn, li, pb, oldEntry, newEntry); + } + PR_RWLock_Unlock(be->vlvSearchList_lock); + return return_value; +} + +/* + * Determine the range of record numbers to return. + * Prevent an underrun, or overrun. + */ + /* jcm: Should we make sure that start < stop */ + +static void +determine_result_range(const struct vlv_request *vlv_request_control, PRUint32 index, PRUint32 length, PRUint32* pstart, PRUint32 *pstop) +{ + if (vlv_request_control == NULL) + { + *pstart= 0; + if (0 == length) /* 609377: index size could be 0 */ + { + *pstop= 0; + } + else + { + *pstop= length - 1; + } + } + else + { + /* Make sure we don't run off the start */ + if(index < vlv_request_control->beforeCount) + { + *pstart= 0; + } + else + { + *pstart= index - vlv_request_control->beforeCount; + } + /* Make sure we don't run off the end */ + if(ULONG_MAX - index > vlv_request_control->afterCount) + { + *pstop= index + vlv_request_control->afterCount; + } + else + { + *pstop= ULONG_MAX; + } + /* Client tried to index off the end */ + if (0 == length) /* 609377: index size could be 0 */ + { + *pstop= 0; + } + else if(*pstop > length - 1) + { + *pstop= length - 1; + } + } + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_determine_result_range: Result Range %lu-%lu\n", *pstart, *pstop, 0 ); +} + +/* + * This is a utility function to pass the client + * supplied attribute value through the appropriate + * matching rule indexer. + * + * It allocates a berval vector which the caller + * must free. + */ + +static struct berval ** +vlv_create_matching_rule_value( Slapi_PBlock* pb, struct berval *original_value) +{ + struct berval **value= NULL; + if(pb!=NULL) + { + struct berval **outvalue = NULL; + struct berval *invalue[2]; + invalue[0]= original_value; /* jcm: cast away const */ + invalue[1]= NULL; + /* The plugin owns the memory it returns in outvalue */ + matchrule_values_to_keys(pb,invalue,&outvalue); + if(outvalue!=NULL) + { + value= slapi_ch_bvecdup(outvalue); + } + } + if(value==NULL) + { + struct berval *outvalue[2]; + outvalue[0]= original_value; /* jcm: cast away const */ + outvalue[1]= NULL; + value= slapi_ch_bvecdup(outvalue); + } + return value; +} + + +/* + * Find the record number in a VLV index for a given attribute value. + * The returned index is counted from zero. + */ + +static PRUint32 +vlv_build_candidate_list_byvalue( struct vlvIndex* p, DBC *dbc, PRUint32 length, const struct vlv_request *vlv_request_control) +{ + PRUint32 si= 0; /* The Selected Index */ + int err= 0; + DBT key= {0}; + DBT data= {0}; + /* + * If the primary sorted attribute has an associated + * matching rule, then we must mangle the typedown + * value. + */ + struct berval **typedown_value= NULL; + struct berval *invalue[2]; + invalue[0]= (struct berval *)&vlv_request_control->value; /* jcm: cast away const */ + invalue[1]= NULL; + if (p->vlv_sortkey[0]->sk_matchruleoid==NULL) + { + slapi_call_syntax_values2keys(p->vlv_syntax_plugin[0],invalue,&typedown_value,LDAP_FILTER_EQUALITY); /* JCM SLOW FUNCTION */ + } + else + { + typedown_value= vlv_create_matching_rule_value(p->vlv_mrpb[0],(struct berval *)&vlv_request_control->value); /* jcm: cast away const */ + } + if(p->vlv_sortkey[0]->sk_reverseorder) + { + /* + * The primary attribute is reverse sorted, so we must + * invert the typedown value in order to match the key. + */ + unsigned int i; + for(i=0;i<(*typedown_value)->bv_len;i++) + { + ((char*)(*typedown_value)->bv_val)[i]= UCHAR_MAX - ((char*)(*typedown_value)->bv_val)[i]; + } + } + + key.flags= DB_DBT_MALLOC; + key.size= typedown_value[0]->bv_len; + key.data= typedown_value[0]->bv_val; + data.flags= DB_DBT_MALLOC; + err= dbc->c_get(dbc,&key,&data,DB_SET_RANGE); + if(err==0) + { + free(data.data); + err= dbc->c_get(dbc,&key,&data,DB_GET_RECNO); + if(err==0) + { + si= *((db_recno_t*)data.data); + /* Records are numbered from one. */ + si--; + free(data.data); + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_build_candidate_list_byvalue: Found. Index=%lu\n",si,0,0); + } + else + { + /* Couldn't get the record number for the record we found. */ + } + } + else + { + /* Couldn't find an entry which matches the value, + * so return the last entry + * (609377) when the index file is empty, there is no "last entry". + */ + if (0 == length) + { + si = 0; + } + else + { + si = length - 1; + } + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_build_candidate_list_byvalue: Not Found. Index=%lu\n",si,0,0); + } + ber_bvecfree((struct berval**)typedown_value); + return si; +} + +static int +vlv_idl_sort_cmp(const void *x, const void *y) +{ + return *(ID *)x - *(ID *)y; +} + +/* build a candidate list (IDL) from a VLV index, given the starting index + * and the ending index (as an inclusive list). + * returns 0 on success, or an LDAP error code. + */ +int vlv_build_idl(PRUint32 start, PRUint32 stop, DB *db, DBC *dbc, + IDList **candidates, int dosort) +{ + IDList *idl = NULL; + int err; + PRUint32 recno; + DBT key = {0}; + DBT data = {0}; + ID id; + + idl = idl_alloc(stop-start+1); + if (!idl) { + /* out of memory :( */ + return LDAP_OPERATIONS_ERROR; + } + recno = start+1; + key.size = sizeof(recno); + key.data = &recno; + key.flags = DB_DBT_MALLOC; + data.ulen = sizeof(ID); + data.data = &id; + data.flags = DB_DBT_USERMEM; /* don't alloc */ + err = dbc->c_get(dbc, &key, &data, DB_SET_RECNO); + while ((err == 0) && (recno <= stop+1)) { + if (key.data != &recno) + free(key.data); + idl_append(idl, *(ID *)data.data); + if (++recno <= stop+1) { + err = dbc->c_get(dbc, &key, &data, DB_NEXT); + } + } + if (err != 0) { + /* some db error...? */ + LDAPDebug(LDAP_DEBUG_ANY, "vlv_build_idl: can't follow db cursor " + "(err %d)\n", err, 0, 0); + if (err == ENOMEM) + LDAPDebug(LDAP_DEBUG_ANY, " nomem: wants %d key, %d data\n", + key.size, data.size, 0); + return LDAP_OPERATIONS_ERROR; + } + + /* success! */ + if (idl) { + if (candidates) + { + if (dosort) + { + qsort((void *)&idl->b_ids[0], idl->b_nids, + (size_t)sizeof(ID), vlv_idl_sort_cmp); + } + *candidates = idl; + } + else + idl_free(idl); /* ??? */ + } + return LDAP_SUCCESS; +} + + +/* This function does vlv_access, searching and building list all while holding read lock + + 1. vlv_find_search fails, set: + unsigned int opnote = SLAPI_OP_NOTE_UNINDEXED; + slapi_pblock_set( pb, SLAPI_OPERATION_NOTES, &opnote ); + return FIND_SEARCH FAILED + + 2. vlvIndex_accessallowed fails + return VLV_LDBM_ACCESS_DENIED + + 3. vlv_build_candidate_list fails: + return VLV_BLD_LIST_FAILED + + 4. return LDAP_SUCCESS +*/ + +int +vlv_search_build_candidate_list(Slapi_PBlock *pb, const Slapi_DN *base, int *vlv_rc, const sort_spec* sort_control, + const struct vlv_request *vlv_request_control, + IDList** candidates, struct vlv_response *vlv_response_control) { + struct vlvIndex* pi = NULL; + backend *be; + int scope, rc=LDAP_SUCCESS; + char *fstr; + + slapi_pblock_get( pb, SLAPI_BACKEND, &be ); + slapi_pblock_get( pb, SLAPI_SEARCH_SCOPE, &scope ); + slapi_pblock_get( pb, SLAPI_SEARCH_STRFILTER, &fstr ); + PR_RWLock_Rlock(be->vlvSearchList_lock); + if((pi=vlv_find_search(be, base, scope, fstr, sort_control)) == NULL) { + unsigned int opnote = SLAPI_OP_NOTE_UNINDEXED; + slapi_pblock_set( pb, SLAPI_OPERATION_NOTES, &opnote ); + rc = VLV_FIND_SEARCH_FAILED; + } else if((*vlv_rc=vlvIndex_accessallowed(pi, pb)) != LDAP_SUCCESS) { + rc = VLV_ACCESS_DENIED; + } else if ((*vlv_rc=vlv_build_candidate_list(be,pi,vlv_request_control,candidates,vlv_response_control)) != LDAP_SUCCESS) { + rc = VLV_BLD_LIST_FAILED; + vlv_response_control->result=*vlv_rc; + } + PR_RWLock_Unlock(be->vlvSearchList_lock); + return rc; +} + +/* + * Given the SORT and VLV controls return a candidate list from the + * pre-computed index file. + * + * Returns: + * success (0), + * operationsError (1), + * unwillingToPerform (53), + * timeLimitExceeded (3), + * adminLimitExceeded (11), + * indexRangeError (61), + * other (80) + */ + + +static int +vlv_build_candidate_list( backend *be, struct vlvIndex* p, const struct vlv_request *vlv_request_control, IDList** candidates, struct vlv_response *vlv_response_control) +{ + int return_value = LDAP_SUCCESS; + DB *db = NULL; + DBC *dbc = NULL; + int rc, err; + PRUint32 si = 0; /* The Selected Index */ + PRUint32 length; + int do_trim= 1; + + LDAPDebug(LDAP_DEBUG_TRACE, + "=> vlv_build_candidate_list: %s %s Using VLV Index %s\n", + slapi_sdn_get_dn(vlvIndex_getBase(p)), p->vlv_search->vlv_filter, + vlvIndex_getName(p)); + if (!vlvIndex_online(p)) { + return -1; + } + rc = dblayer_get_index_file(be, p->vlv_attrinfo, &db, 0); + if (rc != 0) { + /* shouldn't happen */ + LDAPDebug(LDAP_DEBUG_ANY, "VLV: can't get index file '%s' (err %d)\n", + p->vlv_attrinfo->ai_type, rc, 0); + return -1; + } + + err = db->cursor(db, 0 /* txn */, &dbc, 0); + if (err != 0) { + /* shouldn't happen */ + LDAPDebug(LDAP_DEBUG_ANY, "VLV: couldn't get cursor (err %d)\n", + rc, 0, 0); + return -1; + } + + length = vlvIndex_get_indexlength(p, db, 0 /* txn */); + + /* Increment the usage counter */ + vlvIndex_incrementUsage(p); + + if (vlv_request_control) + { + switch(vlv_request_control->tag) { + case 0: /* byIndex */ + si = vlv_trim_candidates_byindex(length, vlv_request_control); + break; + case 1: /* byValue */ + si = vlv_build_candidate_list_byvalue(p, dbc, length, + vlv_request_control); + if (si==length) { + do_trim = 0; + *candidates = idl_alloc(0); + } + break; + default: + /* Some wierd tag value. Shouldn't ever happen */ + if (ISLEGACY(be)) { + return_value = LDAP_OPERATIONS_ERROR; + } else { + return_value = LDAP_VIRTUAL_LIST_VIEW_ERROR; + } + break; + } + + /* Tell the client what the real content count is. + * Client counts from 1. */ + vlv_response_control->targetPosition = si + 1; + vlv_response_control->contentCount = length; + vlv_response_control->result = return_value; + } + + if ((return_value == LDAP_SUCCESS) && do_trim) { + /* Work out the range of records to return */ + PRUint32 start, stop; + determine_result_range(vlv_request_control, si, length, &start, &stop); + + /* fetch the idl */ + return_value = vlv_build_idl(start, stop, db, dbc, candidates, 0); + } + dbc->c_close(dbc); + + dblayer_release_index_file( be, p->vlv_attrinfo, db ); + return return_value; +} + +/* + * Given a candidate list and a filter specification, filter the candidate list + * + * Returns: + * success (0), + * operationsError (1), + * unwillingToPerform (53), + * timeLimitExceeded (3), + * adminLimitExceeded (11), + * indexRangeError (61), + * other (80) + */ +int +vlv_filter_candidates(backend *be, Slapi_PBlock *pb, const IDList *candidates, const Slapi_DN *base, int scope, Slapi_Filter *filter, IDList** filteredCandidates, int lookthrough_limit, time_t time_up) +{ + IDList* resultIdl= NULL; + int return_value = LDAP_SUCCESS; + + /* Refuse to filter a non-existent IDlist */ + if (NULL == candidates) + { + return LDAP_UNWILLING_TO_PERFORM; + } + + LDAPDebug( LDAP_DEBUG_TRACE, "=> vlv_filter_candidates: Filtering %lu Candidates\n",(u_long)candidates->b_nids, 0, 0 ); + + if (0 == return_value && candidates->b_nids>0) + { + /* jcm: Could be an idlist function. create_filtered_idlist */ + /* Iterate over the ID List applying the filter */ + int lookedat= 0; + int done= 0; + int counter= 0; + ID id = NOID; + idl_iterator current = idl_iterator_init(candidates); + resultIdl= idl_alloc(candidates->b_nids); + do + { + id = idl_iterator_dereference_increment(¤t, candidates); + if ( id != NOID ) + { + int err= 0; + struct backentry *e= NULL; + e = id2entry( be, id, NULL, &err ); + if ( e == NULL ) + { + /* + * The ALLIDS ID List contains IDs for which there is no entry. + * This is because the entries have been deleted. An error in + * this case is ok. + */ + if(!(ALLIDS(candidates) && err==DB_NOTFOUND)) + { + LDAPDebug( LDAP_DEBUG_ANY, "vlv_filter_candidates: Candidate %lu not found err=%d\n", (u_long)id, err, 0 ); + } + } + else + { + lookedat++; + if(slapi_sdn_scope_test(backentry_get_sdn(e),base,scope)) + { + if ( slapi_filter_test( pb, e->ep_entry, filter, 0 /* No ACL Check */) == 0 ) + { + /* The entry passed the filter test, add the id to the list */ + LDAPDebug( LDAP_DEBUG_TRACE, "vlv_filter_candidates: Candidate %lu Passed Filter\n", (u_long)id, 0, 0 ); + idl_append(resultIdl,id); + } + } + cache_return(&(((ldbm_instance *) be->be_instance_info)->inst_cache), &e); + } + } + + done= slapi_op_abandoned(pb); + + /* Check to see if our journey is really necessary */ + if ( counter++ % 10 == 0 ) + { + /* check time limit */ + time_t curtime = current_time(); + if ( time_up != -1 && curtime > time_up ) + { + return_value= LDAP_TIMELIMIT_EXCEEDED; + done= 1; + } + /* check lookthrough limit */ + if ( lookthrough_limit != -1 && lookedat>lookthrough_limit ) + { + return_value= LDAP_ADMINLIMIT_EXCEEDED; + done= 1; + } + } + } while (!done && id!=NOID); + } + if(filteredCandidates!=NULL) + *filteredCandidates= resultIdl; + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_filter_candidates: Filtering done\n",0, 0, 0 ); + + return return_value; +} + +/* + * Given a candidate list and a virtual list view specification, trim the candidate list + * + * Returns: + * success (0), + * operationsError (1), + * unwillingToPerform (53), + * timeLimitExceeded (3), + * adminLimitExceeded (11), + * indexRangeError (61), + * other (80) + */ +int +vlv_trim_candidates(backend *be, const IDList *candidates, const sort_spec* sort_control, const struct vlv_request *vlv_request_control, IDList** trimmedCandidates,struct vlv_response *vlv_response_control) +{ + IDList* resultIdl= NULL; + int return_value= LDAP_SUCCESS; + PRUint32 si= 0; /* The Selected Index */ + int do_trim= 1; + + /* Refuse to trim a non-existent IDlist */ + if (NULL == candidates || candidates->b_nids==0) + { + return LDAP_UNWILLING_TO_PERFORM; + } + + switch(vlv_request_control->tag) + { + case 0: /* byIndex */ + si= vlv_trim_candidates_byindex(candidates->b_nids, vlv_request_control); + break; + case 1: /* byValue */ + si= vlv_trim_candidates_byvalue(be, candidates, sort_control, vlv_request_control); + /* Don't bother sending results if the attribute value wasn't found */ + if(si==candidates->b_nids) + { + do_trim= 0; + resultIdl= idl_alloc(0); + } + break; + default: + /* Some wierd tag value. Shouldn't ever happen */ + if (ISLEGACY(be)) { + return_value = LDAP_OPERATIONS_ERROR; + } else { + return_value = LDAP_VIRTUAL_LIST_VIEW_ERROR; + } + break; + } + + /* Tell the client what the real content count is. Clients count from 1 */ + vlv_response_control->targetPosition= si + 1; + vlv_response_control->contentCount= candidates->b_nids; + + if(return_value==LDAP_SUCCESS && do_trim) + { + /* Work out the range of records to return */ + PRUint32 start, stop; + determine_result_range(vlv_request_control,si,candidates->b_nids,&start,&stop); + /* Build a new list containing the (start..stop) range */ + /* JCM: Should really be a function in idlist.c to copy a range */ + resultIdl= idl_alloc(stop-start+1); + { + PRUint32 cursor= 0; + for(cursor=start;cursor<=stop;cursor++) + { + LDAPDebug( LDAP_DEBUG_TRACE, "vlv_trim_candidates: Include ID %lu\n",(u_long)candidates->b_ids[cursor], 0, 0 ); + idl_append(resultIdl,candidates->b_ids[cursor]); + } + } + } + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_trim_candidates: Trimmed list contains %lu entries.\n",(u_long)resultIdl->b_nids, 0, 0 ); + if(trimmedCandidates!=NULL) + *trimmedCandidates= resultIdl; + return return_value; +} + +/* + * Work out the Selected Index given the length of the candidate list + * and the request control from the client. + * + * If the client sends Index==0 we behave as if I=1 + * If the client sends Index==Size==1 we behave as if I=1, S=0 + */ +static PRUint32 +vlv_trim_candidates_byindex(PRUint32 length, const struct vlv_request *vlv_request_control) +{ + PRUint32 si= 0; /* The Selected Index */ + LDAPDebug( LDAP_DEBUG_TRACE, "=> vlv_trim_candidates_byindex: length=%lu index=%lu size=%lu\n",length, vlv_request_control->index, vlv_request_control->contentCount ); + if(vlv_request_control->index==0) + { + /* Always select the first entry in the list */ + si= 0; + } + else + { + if(vlv_request_control->contentCount==0) + { + /* The client has no idea what the content count might be. */ + /* Can't scale the index, so use as is */ + si= vlv_request_control->index; + if (0 == length) /* 609377: index size could be 0 */ + { + if (si > 0) + { + si = length; + } + } + else if(si > length - 1) + { + si= length - 1; + } + } + else + { + if(vlv_request_control->index>=vlv_request_control->contentCount) + { + /* Always select the last entry in the list */ + if (0 == length) /* 609377: index size could be 0 */ + { + si = 0; + } + else + { + si= length-1; + } + } + else + { + /* The three components of this expression are (PRUint32) and may well have a value up to ULONG_MAX */ + /* SelectedIndex = ActualContentCount * ( ClientIndex / ClientContentCount ) */ + si= ((PRUint32)((double)length * (double)(vlv_request_control->index / (double)vlv_request_control->contentCount ))); + } + } + } + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_trim_candidates_byindex: Selected Index %lu\n",si, 0, 0 ); + return si; +} + +/* + * Iterate over the Candidate ID List looking for an entry >= the provided attribute value. + */ +static PRUint32 +vlv_trim_candidates_byvalue(backend *be, const IDList *candidates, const sort_spec* sort_control, const struct vlv_request *vlv_request_control) +{ + PRUint32 si= 0; /* The Selected Index */ + PRUint32 low= 0; + PRUint32 high= candidates->b_nids-1; + PRUint32 current= 0; + ID id = NOID; + int found= 0; + struct berval **typedown_value; + + /* For non-matchrule indexing */ + value_compare_fn_type compare_fn= NULL; + + /* + * If the primary sorted attribute has an associated + * matching rule, then we must mangle the typedown + * value. + */ + if (sort_control->matchrule==NULL) + { + void *pi= NULL; + if(slapi_attr_type2plugin(sort_control->type, &pi)==0) + { + struct berval *invalue[2]; + invalue[0]= (struct berval *)&vlv_request_control->value; /* jcm: cast away const */ + invalue[1]= NULL; + slapi_call_syntax_values2keys(pi,invalue,&typedown_value,LDAP_FILTER_EQUALITY); /* JCM SLOW FUNCTION */ + plugin_call_syntax_get_compare_fn( pi, &compare_fn ); + if (compare_fn == NULL) { + LDAPDebug(LDAP_DEBUG_ANY, "vlv_trim_candidates_byvalue: " + "attempt to compare an unordered attribute", + 0, 0, 0); + compare_fn = slapi_berval_cmp; + } + } + } + else + { + typedown_value= vlv_create_matching_rule_value(sort_control->mr_pb,(struct berval *)&vlv_request_control->value); + compare_fn= slapi_berval_cmp; + } + /* + * Perform a binary search over the candidate list + */ + do { + int err= 0; + struct backentry *e= NULL; + if(!sort_control->order) + { + current = (low + high)/2; + } + else + { + current = (1 + low + high)/2; + } + id= candidates->b_ids[current]; + e = id2entry( be, id, NULL, &err ); + if ( e == NULL ) + { + LDAPDebug( LDAP_DEBUG_ANY, "vlv_trim_candidates_byvalue: Candidate ID %lu not found err=%d\n", (u_long)id, err, 0 ); + } + else + { + /* Check if vlv_request_control->value is greater than or equal to the primary key. */ + int match; + Slapi_Attr *attr; + if ( (NULL != compare_fn) && (slapi_entry_attr_find( e->ep_entry, sort_control->type, &attr ) == 0) ) + { + /* + * If there's a matching rule associated with the primary + * attribute then use the indexer to mangle the attr values. + */ + Slapi_Value **csn_value = valueset_get_valuearray(&attr->a_present_values); + struct berval **entry_value = /* xxxPINAKI needs modification attr->a_vals */NULL; + if(sort_control->mr_pb!=NULL) + { + struct berval **tmp_entry_value = NULL; + + valuearray_get_bervalarray(csn_value,&tmp_entry_value); + /* Matching rule. Do the magic mangling. Plugin owns the memory. */ + matchrule_values_to_keys(sort_control->mr_pb,/* xxxPINAKI needs modification attr->a_vals */tmp_entry_value,&entry_value); + } + else + { + valuearray_get_bervalarray(csn_value,&entry_value); + } + if(!sort_control->order) + { + match= sort_attr_compare(entry_value, (struct berval**)typedown_value, compare_fn); + } + else + { + match= sort_attr_compare((struct berval**)typedown_value, entry_value, compare_fn); + } + } + else + { + /* + * This attribute doesn't exist on this entry. + */ + if(sort_control->order) + { + match= 1; + } + else + { + match= 0; + } + } + if(!sort_control->order) + { + if (match>=0) + { + high= current; + } + else + { + low= current+1; + } + } + else + { + if (match>=0) + { + high= current-1; + } + else + { + low= current; + } + } + if (low>=high) + { + found= 1; + si= high; + if(si==candidates->b_nids && !match) + { + /* Couldn't find an entry which matches the value, so return contentCount */ + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_trim_candidates_byvalue: Not Found. Index %lu\n",si, 0, 0 ); + si= candidates->b_nids; + } + else + { + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_trim_candidates_byvalue: Found. Index %lu\n",si, 0, 0 ); + } + } + } + } while (!found); + ber_bvecfree((struct berval**)typedown_value); + return si; +} + +/* + * Encode the VLV RESPONSE control. + * + * Create a virtual list view response control, + * and add it to the PBlock to be returned to the client. + * + * Returns: + * success ( 0 ) + * operationsError (1), + */ +int +vlv_make_response_control (Slapi_PBlock *pb, const struct vlv_response* vlvp) +{ + BerElement *ber= NULL; + struct berval *bvp = NULL; + int rc = -1; + + /* + VirtualListViewResponse ::= SEQUENCE { + targetPosition INTEGER (0 .. maxInt), + contentCount INTEGER (0 .. maxInt), + virtualListViewResult ENUMERATED { + success (0), + operationsError (1), + unwillingToPerform (53), + insufficientAccessRights (50), + busy (51), + timeLimitExceeded (3), + adminLimitExceeded (11), + sortControlMissing (60), + indexRangeError (61), + other (80) } } + */ + + if ( ( ber = ber_alloc()) == NULL ) + { + return rc; + } + + rc = ber_printf( ber, "{iie}", vlvp->targetPosition, vlvp->contentCount, vlvp->result ); + if ( rc != -1 ) + { + rc = ber_flatten( ber, &bvp ); + } + + ber_free( ber, 1 ); + + if ( rc != -1 ) + { + LDAPControl new_ctrl = {0}; + new_ctrl.ldctl_oid = LDAP_CONTROL_VLVRESPONSE; + new_ctrl.ldctl_value = *bvp; + new_ctrl.ldctl_iscritical = 1; + rc= slapi_pblock_set( pb, SLAPI_ADD_RESCONTROL, &new_ctrl ); + ber_bvfree(bvp); + } + + LDAPDebug( LDAP_DEBUG_TRACE, "<= vlv_make_response_control: Index=%lu Size=%lu Result=%lu\n", vlvp->targetPosition, vlvp->contentCount, vlvp->result ); + + return (rc==-1?LDAP_OPERATIONS_ERROR:LDAP_SUCCESS); +} + +/* + * Generate a logging string for the vlv request and response + */ +void vlv_print_access_log(Slapi_PBlock *pb,struct vlv_request* vlvi, struct vlv_response *vlvo) +{ +#define VLV_LOG_BS (21*6 + 4 + 5) /* space for 20-digit values for all parameters + 'VLV ' + status */ + char stack_buffer[VLV_LOG_BS]; + char *buffer = stack_buffer; + char *p; + + if (vlvi->value.bv_len > 20) { + buffer = slapi_ch_malloc(VLV_LOG_BS + vlvi->value.bv_len); + } + p = buffer; + p+= sprintf(p,"VLV "); + if (0 == vlvi->tag) { + /* By Index case */ + p+= sprintf(p,"%ld:%ld:%ld:%ld", + vlvi->beforeCount , + vlvi->afterCount , + vlvi->index , + vlvi->contentCount + ); + } else { + /* By value case */ +#define VLV_LOG_SS 32 + char stack_string[VLV_LOG_SS]; + char *string = stack_string; + + if (vlvi->value.bv_len >= VLV_LOG_SS) { + string = slapi_ch_malloc(vlvi->value.bv_len+1); + } + strncpy(string,vlvi->value.bv_val,vlvi->value.bv_len); + string[vlvi->value.bv_len] = '\0'; + p += sprintf(p,"%ld:%ld:%s", + vlvi->beforeCount , + vlvi->afterCount , + string + ); + if (string != stack_string) { + slapi_ch_free( (void**)&string); + } + } + /* Now the response info */ + p += sprintf(p," %ld:%ld (%ld)", + vlvo->targetPosition , + vlvo->contentCount, + vlvo->result + ); + + + ldbm_log_access_message(pb,buffer); + + if (buffer != stack_buffer) { + slapi_ch_free( (void**)&buffer); + } +} + +/* + * Decode the VLV REQUEST control. + * + * If the client sends Index==0 we behave as if I=1 + * + * Returns: + * success (0), + * operationsError (1), + * + */ +int +vlv_parse_request_control( backend *be, struct berval *vlv_spec_ber,struct vlv_request* vlvp) +{ + /* This control looks like this : + + VirtualListViewRequest ::= SEQUENCE { + beforeCount INTEGER (0 .. maxInt), + afterCount INTEGER (0 .. maxInt), + CHOICE { + byIndex [0] SEQUENCE { + index INTEGER (0 .. maxInt), + contentCount INTEGER (0 .. maxInt) } + greaterThanOrEqual [1] assertionValue } + */ + BerElement *ber = NULL; + int return_value = LDAP_SUCCESS; + PRUint32 rc= 0; + long long_beforeCount; + long long_afterCount; + long long_index; + long long_contentCount; + + vlvp->value.bv_len = 0; + vlvp->value.bv_val = NULL; + + ber = ber_init(vlv_spec_ber); + rc = ber_scanf(ber,"{ii",&long_beforeCount,&long_afterCount); + vlvp->beforeCount = long_beforeCount; + vlvp->afterCount = long_afterCount; + if (LBER_ERROR == rc) + { + return_value= LDAP_OPERATIONS_ERROR; + } + else + { + LDAPDebug( LDAP_DEBUG_TRACE, "vlv_parse_request_control: Before=%lu After=%lu\n", vlvp->beforeCount, vlvp->afterCount, 0 ); + rc = ber_scanf(ber,"t",&vlvp->tag); + switch(vlvp->tag) + { + case LDAP_TAG_VLV_BY_INDEX: + /* byIndex */ + vlvp->tag= 0; + rc = ber_scanf(ber,"{ii}}",&long_index,&long_contentCount); + vlvp->index = long_index; + vlvp->contentCount = long_contentCount; + if (LBER_ERROR == rc) + { + if (ISLEGACY(be)) { + return_value = LDAP_OPERATIONS_ERROR; + } else { + return_value = LDAP_VIRTUAL_LIST_VIEW_ERROR; + } + } + else + { + /* Client Counts from 1. */ + if(vlvp->index!=0) + { + vlvp->index--; + } + LDAPDebug( LDAP_DEBUG_TRACE, "vlv_parse_request_control: Index=%lu Content=%lu\n", vlvp->index, vlvp->contentCount, 0 ); + } + break; + case LDAP_TAG_VLV_BY_VALUE: + /* byValue */ + vlvp->tag= 1; + rc = ber_scanf(ber,"o}",&vlvp->value); + if (LBER_ERROR == rc) + { + if (ISLEGACY(be)) { + return_value = LDAP_OPERATIONS_ERROR; + } else { + return_value = LDAP_VIRTUAL_LIST_VIEW_ERROR; + } + } + { + /* jcm: isn't there a utility fn to do this? */ + char *p= slapi_ch_malloc(vlvp->value.bv_len+1); + strncpy(p,vlvp->value.bv_val,vlvp->value.bv_len); + p[vlvp->value.bv_len]= '\0'; + LDAPDebug( LDAP_DEBUG_TRACE, "vlv_parse_request_control: Value=%s\n", p, 0, 0 ); + slapi_ch_free( (void**)&p); + } + break; + default: + if (ISLEGACY(be)) { + return_value = LDAP_OPERATIONS_ERROR; + } else { + return_value = LDAP_VIRTUAL_LIST_VIEW_ERROR; + } + } + } + + /* the ber encoding is no longer needed */ + ber_free(ber,1); + + return return_value; +} + +/* given a slapi_filter, check if there's a vlv index that matches that + * filter. if so, return the IDL for that index (else return NULL). + * -- a vlv index will match ONLY if that vlv index is subtree-scope and + * has the same search base and search filter. + * added read lock */ + +IDList *vlv_find_index_by_filter(struct backend *be, const char *base, + Slapi_Filter *f) +{ + struct vlvSearch *t = NULL; + struct vlvIndex *vi; + Slapi_DN base_sdn; + PRUint32 length; + int err; + DB *db = NULL; + DBC *dbc = NULL; + IDList *idl; + Slapi_Filter *vlv_f; + + PR_RWLock_Rlock(be->vlvSearchList_lock); + slapi_sdn_init_dn_byref(&base_sdn, base); + for (t = (struct vlvSearch *)be->vlvSearchList; t; t = t->vlv_next) { + /* all vlv "filters" start with (|(xxx)(objectclass=referral)). + * we only care about the (xxx) part. + */ + vlv_f = t->vlv_slapifilter->f_or; + if ((t->vlv_scope == LDAP_SCOPE_SUBTREE) && + (slapi_sdn_compare(t->vlv_base, &base_sdn) == 0) && + (slapi_filter_compare(vlv_f, f) == 0)) { + /* found match! */ + slapi_sdn_done(&base_sdn); + + /* is there an index that's ready? */ + vi = t->vlv_index; + while (!vlvIndex_online(vi) && vi) { + vi = vi->vlv_next; + } + if (!vi) { + /* no match */ + LDAPDebug(LDAP_DEBUG_TRACE, "vlv: no index online for %s\n", + t->vlv_filter, 0, 0); + PR_RWLock_Unlock(be->vlvSearchList_lock); + return NULL; + } + + if (dblayer_get_index_file(be, vi->vlv_attrinfo, &db, 0) == 0) { + err = db->cursor(db, 0 /* txn */, &dbc, 0); + if (err == 0) { + length = vlvIndex_get_indexlength(vi, db, 0 /* txn */); + if (length == 0) /* 609377: index size could be 0 */ + { + LDAPDebug(LDAP_DEBUG_TRACE, "vlv: index %s is empty\n", + t->vlv_filter, 0, 0); + idl = NULL; + } + else + { + err = vlv_build_idl(0, length-1, db, dbc, &idl, 1 /* dosort */); + } + dbc->c_close(dbc); + } + dblayer_release_index_file(be, vi->vlv_attrinfo, db); + if (err == 0) { + PR_RWLock_Unlock(be->vlvSearchList_lock); + return idl; + } else { + LDAPDebug(LDAP_DEBUG_ANY, "vlv find index: err %d\n", + err, 0, 0); + PR_RWLock_Unlock(be->vlvSearchList_lock); + return NULL; + } + } + } + } + PR_RWLock_Unlock(be->vlvSearchList_lock); + /* no match */ + slapi_sdn_done(&base_sdn); + return NULL; +} + + + +/* replace c with c2 in string -- probably exists somewhere but I can't find it slapi maybe? */ + +static void replace_char(char *name, char c, char c2) +{ + int x; + + for (x = 0; name[x] != '\0'; x++) { + if (c == name[x]) { + name[x] = c2; + } + } +} + +/* similar to what the console GUI does */ + +char *create_vlv_search_tag(const char* dn) { + char *tmp2=strdup(dn); + + replace_char(tmp2,',',' '); + replace_char(tmp2,'"','-'); + replace_char(tmp2,'+','_'); + return tmp2; +} + +/* Builds strings from Slapi_DN similar console GUI. Uses those dns to + delete vlvsearch's if they match. New write lock. + */ + +#define LDBM_PLUGIN_ROOT ", cn=ldbm database, cn=plugins, cn=config" +#define TAG "cn=by MCC " + +int vlv_delete_search_entry(Slapi_PBlock *pb, Slapi_Entry* e, ldbm_instance *inst) +{ + int rc=0; + Slapi_PBlock *tmppb; + Slapi_DN *newdn; + struct vlvSearch* p=NULL; + char *buf, *buf2, *tag1, *tag2; + const char *dn= slapi_sdn_get_dn(&e->e_sdn); + backend *be= inst->inst_be; + + tag1=create_vlv_search_tag(dn); + buf=slapi_ch_malloc(strlen("cn=MCC ")+strlen(tag1)+strlen(", cn=")+strlen(inst->inst_name)+strlen(LDBM_PLUGIN_ROOT) + 1); + sprintf(buf,"%s%s%s%s%s","cn=MCC ",tag1,", cn=",inst->inst_name,LDBM_PLUGIN_ROOT); + newdn=slapi_sdn_new_dn_byval(buf); + PR_RWLock_Wlock(be->vlvSearchList_lock); + p = vlvSearch_finddn((struct vlvSearch *)be->vlvSearchList, newdn); + if(p!=NULL) + { + LDAPDebug( LDAP_DEBUG_ANY, "Deleted Virtual List View Search (%s).\n", p->vlv_name, 0, 0); + tag2=create_vlv_search_tag(dn); + buf2=slapi_ch_malloc(strlen(TAG)+strlen(tag2)+strlen(buf)+2); + sprintf(buf2,"%s%s,%s",TAG,tag2,buf); + vlvSearch_removefromlist((struct vlvSearch **)&be->vlvSearchList,p->vlv_dn); + /* This line release lock to prevent recursive deadlock caused by slapi_internal_delete calling vlvDeleteSearchEntry */ + PR_RWLock_Unlock(be->vlvSearchList_lock); + vlvSearch_delete(&p); + tmppb = slapi_pblock_new(); + slapi_delete_internal_set_pb(tmppb, buf2, NULL, NULL, + (void *)plugin_get_default_component_id(), 0); + slapi_delete_internal_pb(tmppb); + slapi_pblock_get (tmppb, SLAPI_PLUGIN_INTOP_RESULT, &rc); + if(rc != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "vlv_delete_search_entry:can't delete dse entry '%s'\n", buf2, 0, 0); + } + pblock_done(tmppb); + pblock_init(tmppb); + slapi_delete_internal_set_pb(tmppb, buf, NULL, NULL, + (void *)plugin_get_default_component_id(), 0); + slapi_delete_internal_pb(tmppb); + slapi_pblock_get (tmppb, SLAPI_PLUGIN_INTOP_RESULT, &rc); + if(rc != LDAP_SUCCESS) { + LDAPDebug(LDAP_DEBUG_ANY, "vlv_delete_search_entry:can't delete dse entry '%s'\n", buf, 0, 0); + } + slapi_pblock_destroy(tmppb); + slapi_ch_free((void **)&tag2); + slapi_ch_free((void **)&buf2); + } else { + PR_RWLock_Unlock(be->vlvSearchList_lock); + } + slapi_ch_free((void **)&tag1); + slapi_ch_free((void **)&buf); + slapi_sdn_free(&newdn); + return rc; +} + +void +vlv_acquire_lock(backend *be) +{ + LDAPDebug(LDAP_DEBUG_TRACE, "vlv_acquire_lock => trying to acquire the lock\n", 0, 0, 0); + PR_RWLock_Wlock(be->vlvSearchList_lock); +} + +void +vlv_release_lock(backend *be) +{ + LDAPDebug(LDAP_DEBUG_TRACE, "vlv_release_lock => trying to release the lock\n", 0, 0, 0); + PR_RWLock_Unlock(be->vlvSearchList_lock); +} diff --git a/ldap/servers/slapd/back-ldbm/vlv_key.c b/ldap/servers/slapd/back-ldbm/vlv_key.c new file mode 100644 index 00000000..d80aaa34 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/vlv_key.c @@ -0,0 +1,69 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* vlv_key.c */ + + +#include "back-ldbm.h" +#include "vlv_key.h" + +/* + * These functions manipulate keys for the virtual list view indexes. + * A key consists of a string of attribute values concatinated together, + * plus an entry DN to ensure uniqueness. + */ + +struct vlv_key * +vlv_key_new() +{ + struct vlv_key *p= (struct vlv_key*)slapi_ch_malloc(sizeof(struct vlv_key)); + p->keymem= 64; + memset(&p->key,0,sizeof(DBT)); + p->key.data= slapi_ch_malloc(p->keymem); + p->key.size= 0; + return p; +} + +void +vlv_key_delete(struct vlv_key **p) +{ + slapi_ch_free(&((*p)->key.data)); + slapi_ch_free((void **)p); +} + +#if 0 +static void +vlv_key_copy(const struct vlv_key *p1,struct vlv_key *p2) +{ + p2->keymem= p1->keymem; + p2->key.data= slapi_ch_realloc(p2->key.data,p2->keymem); + strcpy(p2->key.data, p1->key.data); + p2->key.size= p1->key.size; +} +#endif + +/* + * Add an attribute value to the end of a composite key. + */ +void +vlv_key_addattr(struct vlv_key *p,struct berval *val) +{ + /* If there isn't room then allocate some more memory */ + unsigned int need = p->key.size + val->bv_len; + if(need > p->keymem) + { + p->keymem*= 2; + if(need > p->keymem) + { + p->keymem= need; + } + p->key.data= slapi_ch_realloc(p->key.data,p->keymem); + } + memcpy(((char*)p->key.data)+p->key.size, val->bv_val, val->bv_len); + p->key.size+= val->bv_len; +} + + + diff --git a/ldap/servers/slapd/back-ldbm/vlv_key.h b/ldap/servers/slapd/back-ldbm/vlv_key.h new file mode 100644 index 00000000..d7436629 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/vlv_key.h @@ -0,0 +1,22 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* vlv_key.h */ + + +#if !defined(__VLV_KEY_H) +#define __VLV_KEY_H + +struct vlv_key +{ + PRUint32 keymem; + DBT key; +}; + +struct vlv_key *vlv_key_new(); +void vlv_key_delete(struct vlv_key **p); +void vlv_key_addattr(struct vlv_key *p,struct berval *val); + +#endif diff --git a/ldap/servers/slapd/back-ldbm/vlv_srch.c b/ldap/servers/slapd/back-ldbm/vlv_srch.c new file mode 100644 index 00000000..0f72c418 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/vlv_srch.c @@ -0,0 +1,901 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* vlv_srch.c */ + + +#include "back-ldbm.h" +#include "vlv_srch.h" + +/* Attributes for vlvSearch */ +char* const type_vlvName = "cn"; +char* const type_vlvBase = "vlvBase"; +char* const type_vlvScope = "vlvScope"; +char* const type_vlvFilter = "vlvFilter"; + +/* Attributes for vlvIndex */ +char* const type_vlvSort = "vlvSort"; +char* const type_vlvFilename = "vlvFilename"; +char* const type_vlvEnabled = "vlvEnabled"; +char* const type_vlvUses = "vlvUses"; + +static const char *file_prefix= "vlv#"; /* '#' used to avoid collision with real attributes */ +static const char *file_suffix= LDBM_FILENAME_SUFFIX; + +static int vlvIndex_createfilename(struct vlvIndex* pIndex, char **ppc); + +static int vlvIndex_equal(const struct vlvIndex* p1, const sort_spec* sort_control); +static void vlvIndex_checkforindex(struct vlvIndex* p, backend *be); + +/* + * Create a new vlvSearch object + */ +struct vlvSearch* +vlvSearch_new() +{ + struct vlvSearch* p = (struct vlvSearch*)slapi_ch_calloc(1,sizeof(struct vlvSearch)); + if(p!=NULL) + { + p->vlv_e= NULL; + p->vlv_dn= NULL; + p->vlv_name= NULL; + p->vlv_base= NULL; + p->vlv_scope= LDAP_SCOPE_BASE; + p->vlv_filter= NULL; + p->vlv_slapifilter= NULL; + p->vlv_index= NULL; + p->vlv_next= NULL; + } + return p; +} + +/* + * Trim spaces off the end of the string + */ +static void +trimspaces(char *s) +{ + PRUint32 i= strlen(s) - 1; + while(i > 0 && isascii(s[i]) && isspace(s[i])) + { + s[i]= '\0'; + i--; + } +} + +/* + * Re-Initialise a vlvSearch object + */ +void +vlvSearch_reinit(struct vlvSearch* p, const struct backentry *base) +{ + if (p->vlv_initialized) { + return; /* no work to do */ + } + if (LDAP_SCOPE_ONELEVEL != p->vlv_scope) { + /* Only kind we re-init is onelevel searches */ + return; + } + /* Now down to work */ + if (NULL != p->vlv_slapifilter) { + slapi_filter_free(p->vlv_slapifilter,1); + } + p->vlv_slapifilter= slapi_str2filter( p->vlv_filter ); + filter_normalize(p->vlv_slapifilter); + /* make (&(parentid=idofbase)(|(originalfilter)(objectclass=referral))) */ + { + Slapi_Filter *fid2kids= NULL; + Slapi_Filter *focref= NULL; + Slapi_Filter *fand= NULL; + Slapi_Filter *forr= NULL; + p->vlv_slapifilter= create_onelevel_filter(p->vlv_slapifilter, base, 0 /* managedsait */, &fid2kids, &focref, &fand, &forr); + } +} + +/* + * Initialise a vlvSearch object + */ +void +vlvSearch_init(struct vlvSearch* p, Slapi_PBlock *pb, const Slapi_Entry *e, ldbm_instance *inst) +{ + /* VLV specification */ + /* Need to copy the entry here because this one is in the cache, + * not forever ! */ + p->vlv_e= slapi_entry_dup( e ); + p->vlv_dn= slapi_sdn_dup(slapi_entry_get_sdn_const(e)); + p->vlv_name= slapi_entry_attr_get_charptr(e,type_vlvName); + p->vlv_base= slapi_sdn_new_dn_passin(slapi_entry_attr_get_charptr(e,type_vlvBase)); + p->vlv_scope= slapi_entry_attr_get_int(e,type_vlvScope); + p->vlv_filter= slapi_entry_attr_get_charptr(e,type_vlvFilter); + p->vlv_initialized = 1; + + /* JCM: Should perform some validation and report errors to the error log */ + /* JCM: Add brackets around the filter if none are there... */ + trimspaces(p->vlv_name); + trimspaces(p->vlv_filter); + + if(strlen(p->vlv_filter)>0) + { + /* Convert the textual filter, into a Slapi_Filter structure */ + p->vlv_slapifilter= slapi_str2filter( p->vlv_filter ); + filter_normalize(p->vlv_slapifilter); + } + + /* JCM: Really should convert the slapifilter into a string and use that. */ + + /* Convert the filter based on the scope of the search */ + switch(p->vlv_scope) + { + case LDAP_SCOPE_BASE: + /* Don't need to alter the filter */ + break; + case LDAP_SCOPE_ONELEVEL: + { + /* + * Get the base object for the search. + * The entry "" will never be contained in the database, + * so treat it as a special case. + */ + struct backentry *e= NULL; + if ( !slapi_sdn_isempty(p->vlv_base)) { + Slapi_Backend *oldbe = NULL; + entry_address addr; + + /* switch context to the target backend */ + slapi_pblock_get(pb, SLAPI_BACKEND, &oldbe); + slapi_pblock_set(pb, SLAPI_BACKEND, inst->inst_be); + slapi_pblock_set(pb, SLAPI_PLUGIN, inst->inst_be->be_database); + + addr.dn = (char*)slapi_sdn_get_ndn (p->vlv_base); + addr.uniqueid = NULL; + e = find_entry( pb, inst->inst_be, &addr, NULL ); + /* Check to see if the entry is absent. If it is, mark this search + * as not initialized */ + if (NULL == e) { + p->vlv_initialized = 0; + /* We crash on anyhow, and rely on the fact that the filter + * we create is bogus to prevent chaos */ + } + + /* switch context back to the DSE backend */ + slapi_pblock_set(pb, SLAPI_BACKEND, oldbe); + slapi_pblock_set(pb, SLAPI_PLUGIN, oldbe->be_database); + } + + /* make (&(parentid=idofbase)(|(originalfilter)(objectclass=referral))) */ + { + Slapi_Filter *fid2kids= NULL; + Slapi_Filter *focref= NULL; + Slapi_Filter *fand= NULL; + Slapi_Filter *forr= NULL; + p->vlv_slapifilter= create_onelevel_filter(p->vlv_slapifilter, e, 0 /* managedsait */, &fid2kids, &focref, &fand, &forr); + /* jcm: fid2kids, focref, fand, and forr get freed when we free p->vlv_slapifilter */ + cache_return(&inst->inst_cache,&e); + } + } + break; + case LDAP_SCOPE_SUBTREE: + { + /* make (|(originalfilter)(objectclass=referral))) */ + /* No need for scope-filter since we apply a scope test before the filter test */ + Slapi_Filter *focref= NULL; + Slapi_Filter *forr= NULL; + p->vlv_slapifilter= create_subtree_filter(p->vlv_slapifilter, 0 /* managedsait */, &focref, &forr); + /* jcm: focref and forr get freed when we free p->vlv_slapifilter */ + } + break; + } +} + +/* + * Destroy an existing vlvSearch object + */ +void +vlvSearch_delete(struct vlvSearch** ppvs) +{ + if(ppvs!=NULL && *ppvs!=NULL) + { + struct vlvIndex *pi, *ni; + slapi_sdn_free(&((*ppvs)->vlv_dn)); + slapi_ch_free((void**)&((*ppvs)->vlv_name)); + slapi_sdn_free(&((*ppvs)->vlv_base)); + slapi_ch_free((void**)&((*ppvs)->vlv_filter)); + slapi_filter_free((*ppvs)->vlv_slapifilter,1); + for(pi= (*ppvs)->vlv_index;pi!=NULL;) + { + ni= pi->vlv_next; + if(pi->vlv_be != NULL) { + vlvIndex_go_offline(pi,pi->vlv_be); + } + vlvIndex_delete(&pi); + pi= ni; + } + slapi_ch_free((void**)ppvs); + *ppvs= NULL; + } +} + +/* + * Add a search to a list. + * + * We add it to the end of the list because there could + * be other threads traversing the list at this time. + */ +void +vlvSearch_addtolist(struct vlvSearch* p, struct vlvSearch** pplist) +{ + if(pplist!=NULL && p!=NULL) + { + p->vlv_next= NULL; + if(*pplist==NULL) + { + *pplist= p; + } + else + { + struct vlvSearch* last= *pplist; + for(;last->vlv_next!=NULL;last=last->vlv_next); + last->vlv_next= p; + } + } +} + + +/* + * Compare two VLV Searches to see if they're the same, based on their VLV Search specification. + */ +static struct vlvIndex * +vlvSearch_equal(const struct vlvSearch* p1, const Slapi_DN *base, int scope, const char *filter, const sort_spec* sort_control) +{ + struct vlvIndex *pi= NULL; + int r= (slapi_sdn_compare(p1->vlv_base,base)==0); + if(r) r= (p1->vlv_scope==scope); + if(r) r= (strcasecmp(p1->vlv_filter,filter)==0); + if(r) + { + pi= p1->vlv_index; + r= 0; + for(;!r && pi!=NULL;) + { + r= vlvIndex_equal(pi, sort_control); + if(!r) + { + pi= pi->vlv_next; + } + } + } + return pi; +} + +/* + * Find an enabled VLV Search in a list which matches the + * description provided in "base, scope, filter, sort_control" + */ +struct vlvIndex* +vlvSearch_findenabled(backend *be,struct vlvSearch* plist, const Slapi_DN *base, int scope, const char *filter, const sort_spec* sort_control) +{ + struct vlvSearch *t= plist; + struct vlvIndex *pi= NULL; + for(; (t!=NULL) && (pi == NULL); t= t->vlv_next) + { + pi= vlvSearch_equal(t,base,scope,filter,sort_control); + if(pi!=NULL) + { + if(!vlvIndex_enabled(pi)) + { + /* + * A VLV Spec which matched the search criteria was found. + * But it hasn't been enabled yet. Check to see if the + * index is there. But, only check once every 60 seconds. + */ + time_t curtime = current_time(); + if(curtime>pi->vlv_lastchecked+60) + { + vlvIndex_checkforindex(pi, be); + pi->vlv_lastchecked= current_time(); + } + } + if(!vlvIndex_enabled(pi)) + { + pi= NULL; + } + } + } + return pi; +} + +/* + * Find a VLV Search in a list which matches the name + */ +struct vlvIndex* +vlvSearch_findname(const struct vlvSearch* plist, const char *name) +{ + const struct vlvSearch* t= plist; + for(; t!=NULL ; t= t->vlv_next) + { + struct vlvIndex *pi= t->vlv_index; + for(;pi!=NULL;pi= pi->vlv_next) + { + if(strcasecmp(pi->vlv_name,name)==0) + { + return pi; + } + } + } + return NULL; +} + +/* + * Find a VLV Search in a list which matches the index name + */ +struct vlvIndex* +vlvSearch_findindexname(const struct vlvSearch* plist, const char *name) +{ + const struct vlvSearch* t= plist; + for(; t!=NULL ; t= t->vlv_next) + { + struct vlvIndex *pi= t->vlv_index; + for(;pi!=NULL;pi= pi->vlv_next) + { + if(strcasecmp(pi->vlv_attrinfo->ai_type,name)==0) + { + return pi; + } + } + } + return NULL; +} + +/* + * Get a list of VLV Index names. + * The returned pointer must be freed with slapi_ch_free + */ +char * +vlvSearch_getnames(const struct vlvSearch* plist) +{ + /* Work out how long the string will be */ + char *text; + int length= 5; /* enough to hold 'none' */ + const struct vlvSearch* t= plist; + for(; t!=NULL ; t= t->vlv_next) + { + struct vlvIndex *pi= t->vlv_index; + for(;pi!=NULL;pi= pi->vlv_next) + { + length+= strlen(pi->vlv_name) + 4; + } + } + /* Build a comma delimited list of Index names */ + text= slapi_ch_malloc(length); + if(length==5) + { + strcpy(text,"none"); + } + else + { + text[0]= '\0'; + t= plist; + for(; t!=NULL ; t= t->vlv_next) + { + struct vlvIndex *pi= t->vlv_index; + for(;pi!=NULL;pi= pi->vlv_next) + { + sprintf(text + strlen(text),"'%s', ",pi->vlv_name); + } + } + } + return text; +} + +/* + * Find a VLV Search in a list, based on the DN. + */ +struct vlvSearch* +vlvSearch_finddn(const struct vlvSearch* plist, const Slapi_DN *dn) +{ + const struct vlvSearch* curr= plist; + for(; curr!=NULL && slapi_sdn_compare(curr->vlv_dn,dn)!=0; curr= curr->vlv_next); + return (struct vlvSearch*)curr; +} + +/* + * Remove a VLV Search from a list, based on the DN. + */ +void +vlvSearch_removefromlist(struct vlvSearch** pplist, const Slapi_DN *dn) +{ + int done= 0; + struct vlvSearch* prev= NULL; + struct vlvSearch* curr= *pplist; + while(curr!=NULL && !done) + { + if(slapi_sdn_compare(curr->vlv_dn,dn)==0) + { + if(curr==*pplist) + { + *pplist= curr->vlv_next; + } + else + { + prev->vlv_next= curr->vlv_next; + } + done= 1; + } + else + { + prev= curr; + curr= curr->vlv_next; + } + } +} + +/* + * Access Control Check to see if the client is allowed to use this VLV Search. + */ +int +vlvSearch_accessallowed(struct vlvSearch *p, Slapi_PBlock *pb) +{ + char *attrs[2] = { NULL, NULL}; + + attrs[0] = type_vlvName; + return (plugin_call_acl_plugin ( pb, (Slapi_Entry*)p->vlv_e, attrs, NULL, + SLAPI_ACL_READ, ACLPLUGIN_ACCESS_READ_ON_VLV, NULL ) ); +} + +const Slapi_DN *vlvSearch_getBase(struct vlvSearch* p) +{ + return p->vlv_base; +} + +int vlvSearch_getScope(struct vlvSearch* p) +{ + return p->vlv_scope; +} + +Slapi_Filter *vlvSearch_getFilter(struct vlvSearch* p) +{ + return p->vlv_slapifilter; +} + +int vlvSearch_isVlvSearchEntry(Slapi_Entry *e) +{ + return slapi_entry_attr_hasvalue(e, "objectclass", "vlvsearch"); +} + +void vlvSearch_addIndex(struct vlvSearch *pSearch, struct vlvIndex *pIndex) +{ + pIndex->vlv_next= NULL; + if(pSearch->vlv_index==NULL) + { + pSearch->vlv_index= pIndex; + } + else + { + struct vlvIndex* last= pSearch->vlv_index; + for(;last->vlv_next!=NULL;last=last->vlv_next); + last->vlv_next= pIndex; + } +} + +/* ============================================================================================== */ + +/* + * Create a new vlvIndex object + */ +struct vlvIndex* +vlvIndex_new() +{ + struct vlvIndex* p = (struct vlvIndex*)slapi_ch_calloc(1,sizeof(struct vlvIndex)); + if(p!=NULL) + { + p->vlv_sortspec= NULL; + p->vlv_attrinfo= attrinfo_new(); + p->vlv_sortkey= NULL; + p->vlv_filename= NULL; + p->vlv_mrpb= NULL; + p->vlv_syntax_plugin= NULL; + p->vlv_indexlength_lock= PR_NewLock(); + p->vlv_indexlength_cached= 0; + p->vlv_indexlength= 0; + p->vlv_online = 1; + p->vlv_enabled = 0; + p->vlv_lastchecked= 0; + p->vlv_uses= 0; + p->vlv_search= NULL; + p->vlv_next= NULL; + } + return p; +} + +/* + * Destroy an existing vlvIndex object + */ +void +vlvIndex_delete(struct vlvIndex** ppvs) +{ + if(ppvs!=NULL && *ppvs!=NULL) + { + slapi_ch_free((void**)&((*ppvs)->vlv_sortspec)); + { + int n; + for(n=0;(*ppvs)->vlv_sortkey[n]!=NULL;n++) + { + if((*ppvs)->vlv_mrpb[n] != NULL) { + destroy_matchrule_indexer((*ppvs)->vlv_mrpb[n]); + slapi_pblock_destroy((*ppvs)->vlv_mrpb[n]); + } + } + } + ldap_free_sort_keylist((*ppvs)->vlv_sortkey); + attrinfo_delete(&((*ppvs)->vlv_attrinfo)); + slapi_ch_free((void**)&((*ppvs)->vlv_mrpb)); + slapi_ch_free((void**)&((*ppvs)->vlv_syntax_plugin)); + PR_DestroyLock((*ppvs)->vlv_indexlength_lock); + slapi_ch_free((void**)ppvs); + *ppvs= NULL; + } +} + +/* + * Initialise a vlvSearch object + */ +void +vlvIndex_init(struct vlvIndex* p, backend *be, struct vlvSearch* pSearch, const Slapi_Entry *e) +{ + struct ldbminfo *li = (struct ldbminfo *) be->be_database->plg_private; + char *filename= NULL; + + if (NULL == p) + return; + + /* JCM: Should perform some validation and report errors to the error log */ + /* JCM: Add brackets around the filter if none are there... */ + p->vlv_sortspec= slapi_entry_attr_get_charptr(e,type_vlvSort); + trimspaces(p->vlv_sortspec); + + p->vlv_name= slapi_entry_attr_get_charptr(e,type_vlvName); + trimspaces(p->vlv_name); + + p->vlv_search= pSearch; + + /* Convert the textual sort specification into a keylist structure */ + ldap_create_sort_keylist(&(p->vlv_sortkey),p->vlv_sortspec); + { + /* + * For each sort attribute find the appropriate syntax plugin, + * and if it has a matching rule, create a matching rule indexer object. + */ + int n; + for(n=0;p->vlv_sortkey[n]!=NULL;n++); + p->vlv_mrpb= (Slapi_PBlock**)slapi_ch_calloc(n+1,sizeof(Slapi_PBlock*)); + p->vlv_syntax_plugin= (void **)(Slapi_PBlock**)slapi_ch_calloc(n+1,sizeof(Slapi_PBlock*)); + for(n=0;p->vlv_sortkey[n]!=NULL;n++) + { + slapi_attr_type2plugin( p->vlv_sortkey[n]->sk_attrtype, &p->vlv_syntax_plugin[n] ); + if(p->vlv_sortkey[n]->sk_matchruleoid!=NULL) + { + create_matchrule_indexer(&p->vlv_mrpb[n],p->vlv_sortkey[n]->sk_matchruleoid,p->vlv_sortkey[n]->sk_attrtype); + } + + } + + } + + /* Create an index filename for the search */ + if(vlvIndex_createfilename(p,&filename)) + { + p->vlv_filename= slapi_ch_malloc(strlen(file_prefix) + strlen(filename) + strlen(file_suffix) + 1); + sprintf(p->vlv_filename,"%s%s%s",file_prefix,filename,file_suffix); + + /* Create an attrinfo structure */ + p->vlv_attrinfo->ai_type= slapi_ch_malloc(strlen(file_prefix) + strlen(filename) + 1); + sprintf(p->vlv_attrinfo->ai_type,"%s%s",file_prefix,filename); + p->vlv_attrinfo->ai_indexmask= INDEX_VLV; + + /* Check if the index file actually exists */ + if(li!=NULL) + { + vlvIndex_checkforindex(p, be); + } + p->vlv_lastchecked= current_time(); + } + slapi_ch_free((void**)&filename); +} + +/* + * Determine how many {key,data} pairs there are in the VLV Index. + * We only work out the length of the index once, then we cache + * it and maintain it. + */ +PRUint32 +vlvIndex_get_indexlength(struct vlvIndex* p, DB *db, back_txn *txn) +{ + if (NULL == p) + return 0; + + if(!p->vlv_indexlength_cached) + { + DBC *dbc = NULL; + DB_TXN *db_txn = NULL; + int err= 0; + if (NULL != txn) + { + db_txn = txn->back_txn_txn; + } + err = db->cursor(db, db_txn, &dbc, 0); + if(err==0) + { + DBT key= {0}; + DBT data= {0}; + key.flags= DB_DBT_MALLOC; + data.flags= DB_DBT_MALLOC; + err= dbc->c_get(dbc,&key,&data,DB_LAST); + if(err==0) + { + free(key.data); key.data= NULL; + free(data.data); data.data= NULL; + err= dbc->c_get(dbc,&key,&data,DB_GET_RECNO); + if(err==0) + { + PR_Lock(p->vlv_indexlength_lock); + p->vlv_indexlength_cached= 1; + p->vlv_indexlength= *((db_recno_t*)data.data); + PR_Unlock(p->vlv_indexlength_lock); + free(data.data); + } + } + dbc->c_close(dbc); + } + else + { + /* couldn't get cursor??? */ + } + } + return p->vlv_indexlength; +} + +/* + * Increment the index length count. + * We keep track of the index length for efficiency. + */ +void +vlvIndex_increment_indexlength(struct vlvIndex* p, DB *db, back_txn *txn) +{ + if (NULL == p) + return; + + if(p->vlv_indexlength_cached) + { + PR_Lock(p->vlv_indexlength_lock); + p->vlv_indexlength++; + PR_Unlock(p->vlv_indexlength_lock); + } + else + { + p->vlv_indexlength= vlvIndex_get_indexlength(p, db, txn); + } +} + +/* + * Decrement the index length count. + * We keep track of the index length for efficiency. + */ +void +vlvIndex_decrement_indexlength(struct vlvIndex* p, DB *db, back_txn *txn) +{ + if (NULL == p) + return; + + if(p->vlv_indexlength_cached) + { + /* jcm: Check for underflow? */ + PR_Lock(p->vlv_indexlength_lock); + p->vlv_indexlength--; + PR_Unlock(p->vlv_indexlength_lock); + } + else + { + p->vlv_indexlength= vlvIndex_get_indexlength(p, db, txn); + } +} + +/* + * Increment the usage counter + */ +void +vlvIndex_incrementUsage(struct vlvIndex* p) +{ + if (NULL == p) + return; + p->vlv_uses++; +} + +/* + * Get the filename of the index. + */ +const char * +vlvIndex_filename(const struct vlvIndex* p) +{ + if (NULL == p) + return NULL; + return p->vlv_filename; +} + +/* + * Check if the index is available. + */ +int vlvIndex_enabled(const struct vlvIndex* p) +{ + if (NULL == p) + return 0; + return p->vlv_enabled; +} + +int vlvIndex_online(const struct vlvIndex *p) +{ + if (NULL == p) + return 0; + return p->vlv_online; +} + +void vlvIndex_go_offline(struct vlvIndex *p, backend *be) +{ + if (NULL == p) + return; + p->vlv_online = 0; + p->vlv_enabled = 0; + p->vlv_indexlength = 0; + p->vlv_attrinfo->ai_indexmask |= INDEX_OFFLINE; + dblayer_erase_index_file_nolock(be, p->vlv_attrinfo, 1 /* chkpt if not busy */); +} + +void vlvIndex_go_online(struct vlvIndex *p, backend *be) +{ + if (NULL == p) + return; + p->vlv_attrinfo->ai_indexmask &= ~INDEX_OFFLINE; + p->vlv_online = 1; + vlvIndex_checkforindex(p, be); +} + + +/* + * Access Control Check to see if the client is allowed to use this VLV Index. + */ +int +vlvIndex_accessallowed(struct vlvIndex *p, Slapi_PBlock *pb) +{ + if (NULL == p) + return 0; + return vlvSearch_accessallowed(p->vlv_search, pb); +} + +const Slapi_DN *vlvIndex_getBase(struct vlvIndex* p) +{ + if (NULL == p) + return NULL; + return vlvSearch_getBase(p->vlv_search); +} + +int vlvIndex_getScope(struct vlvIndex* p) +{ + if (NULL == p) + return 0; + return vlvSearch_getScope(p->vlv_search); +} + +Slapi_Filter *vlvIndex_getFilter(struct vlvIndex* p) +{ + if (NULL == p) + return NULL; + return vlvSearch_getFilter(p->vlv_search); +} + +const char *vlvIndex_getName(struct vlvIndex* p) +{ + if (NULL == p) + return NULL; + return p->vlv_name; +} + +/* + * JCM: Could also match reverse sense of index and use in reverse. + */ +static int +vlvIndex_equal(const struct vlvIndex* p1, const sort_spec* sort_control) +{ + int r= 1; + const sort_spec *t1= sort_control; + LDAPsortkey *t2= p1->vlv_sortkey[0]; + int n= 1; + for(;t1!=NULL && t2!=NULL && r;t1= t1->next,t2=p1->vlv_sortkey[n],n++) + { + r= (t1->order && t2->sk_reverseorder) || (!t1->order && !t2->sk_reverseorder); + if(r) r= (strcasecmp(t1->type, t2->sk_attrtype)==0); + if(r) + { + if(t1->matchrule==NULL && t2->sk_matchruleoid==NULL) + { + r= 1; + } + else if(t1->matchrule!=NULL && t2->sk_matchruleoid!=NULL) + { + r= (strcasecmp(t1->matchrule, t2->sk_matchruleoid)==0); + } + else + { + r= 0; + } + } + } + if(r) r= (t1==NULL && t2==NULL); + return r; +} + +/* + * Check if the index file actually exists, + * and set vlv_enabled appropriately + */ +static void +vlvIndex_checkforindex(struct vlvIndex* p, backend *be) +{ + DB *db = NULL; + + /* if the vlv index is offline (being generated), don't even look */ + if (! p->vlv_online) + return; + + if (dblayer_get_index_file(be, p->vlv_attrinfo, &db, 0) == 0) { + p->vlv_enabled = 1; + dblayer_release_index_file( be, p->vlv_attrinfo, db ); + } else { + p->vlv_enabled = 0; + } +} + +int vlvIndex_isVlvIndexEntry(Slapi_Entry *e) +{ + return slapi_entry_attr_hasvalue(e, "objectclass", "vlvindex"); +} + +/* + * Create the filename for the index. + * Extract all the alphanumeric characters from the descriptive name. + * Convert to all lower case. + */ +static int +vlvIndex_createfilename(struct vlvIndex* pIndex, char **ppc) +{ + int filenameValid= 1; + unsigned int i; + char *p, *filename; + filename= slapi_ch_malloc(strlen(pIndex->vlv_name) + 1); + p= filename; + for(i=0;i<strlen(pIndex->vlv_name);i++) + { + if(isalnum(pIndex->vlv_name[i])) + { + *p= TOLOWER( pIndex->vlv_name[i] ); + p++; + } + } + *p= '\0'; + if(strlen(filename)==0) + { + LDAPDebug( LDAP_DEBUG_ANY, "Couldn't generate valid filename from Virtual List View Index Name (%s). Need some alphabetical characters.\n", pIndex->vlv_name, 0, 0); + filenameValid= 0; + } + /* JCM: Check if this file clashes with another VLV Index filename */ + *ppc= filename; + return filenameValid; +} + +int +vlv_isvlv(char *filename) +{ + if (0 == strncmp(filename, file_prefix, 4)) + return 1; + return 0; +} diff --git a/ldap/servers/slapd/back-ldbm/vlv_srch.h b/ldap/servers/slapd/back-ldbm/vlv_srch.h new file mode 100644 index 00000000..c892f6b4 --- /dev/null +++ b/ldap/servers/slapd/back-ldbm/vlv_srch.h @@ -0,0 +1,134 @@ +/** BEGIN COPYRIGHT BLOCK + * Copyright 2001 Sun Microsystems, Inc. + * Portions copyright 1999, 2001-2003 Netscape Communications Corporation. + * All rights reserved. + * END COPYRIGHT BLOCK **/ +/* vlv_srch.h */ + + +#if !defined(__VLV_SRCH_H) +#define __VLV_SRCH_H + +extern char* const type_vlvName; +extern char* const type_vlvBase; +extern char* const type_vlvScope; +extern char* const type_vlvFilter; +extern char* const type_vlvSort; +extern char* const type_vlvFilename; +extern char* const type_vlvEnabled; +extern char* const type_vlvUses; + +/* + * This structure is the internal representation of a VLV Search. + */ +struct vlvSearch +{ + /* The VLV Search Specification Entry */ + const Slapi_Entry *vlv_e; + + /* Extracted from the VLV Search Specification entry */ + Slapi_DN *vlv_dn; + char *vlv_name; + Slapi_DN *vlv_base; + int vlv_scope; + char *vlv_filter; + int vlv_initialized; + + /* Derived from the VLV Entry */ + Slapi_Filter *vlv_slapifilter; + + /* List of Indexes for this Search */ + struct vlvIndex* vlv_index; + + /* The next VLV Search in the list */ + struct vlvSearch* vlv_next; +}; + +struct vlvIndex +{ + char *vlv_name; + char *vlv_sortspec; + + /* Derived from the VLV Entry */ + LDAPsortkey **vlv_sortkey; + + /* The Index filename */ + char *vlv_filename; + + /* Attribute Structure maps filename onto index */ + struct attrinfo *vlv_attrinfo; + + /* Syntax Plugin. One for each LDAPsortkey */ + void **vlv_syntax_plugin; + + /* Matching Rule PBlock. One for each LDAPsortkey */ + Slapi_PBlock **vlv_mrpb; + + /* Keep track of the index length */ + PRLock *vlv_indexlength_lock; + int vlv_indexlength_cached; + db_recno_t vlv_indexlength; + + int vlv_enabled; /* index file is there & ready */ + int vlv_online; /* turned off when generating index */ + + /* The last time we checked to see if the index file was available */ + time_t vlv_lastchecked; + + /* The number of uses this search has received since start up */ + PRUint32 vlv_uses; + + struct backend* vlv_be; /* need backend to remove the index when done */ + + /* The parent Search Specification for this Index */ + struct vlvSearch* vlv_search; + + /* The next VLV Index in the list */ + struct vlvIndex* vlv_next; +}; + +struct vlvSearch* vlvSearch_new(); +void vlvSearch_init(struct vlvSearch*, Slapi_PBlock *pb, const Slapi_Entry *e, ldbm_instance *inst); +void vlvSearch_reinit(struct vlvSearch* p, const struct backentry *base); +void vlvSearch_delete(struct vlvSearch** ppvs); +void vlvSearch_addtolist(struct vlvSearch* p, struct vlvSearch** pplist); +struct vlvSearch* vlvSearch_find(const struct vlvSearch* plist, const char *base, int scope, const char *filter, const char *sortspec); +struct vlvIndex* vlvSearch_findenabled(backend *be,struct vlvSearch* plist, const Slapi_DN *base, int scope, const char *filter, const sort_spec* sort_control); +struct vlvSearch* vlvSearch_finddn(const struct vlvSearch* plist, const Slapi_DN *dn); +struct vlvIndex* vlvSearch_findname(const struct vlvSearch* plist, const char *name); +struct vlvIndex* vlvSearch_findindexname(const struct vlvSearch* plist, const char *name); +char *vlvSearch_getnames(const struct vlvSearch* plist); +void vlvSearch_removefromlist(struct vlvSearch** pplist, const Slapi_DN *dn); +int vlvSearch_accessallowed(struct vlvSearch *p, Slapi_PBlock *pb); +const Slapi_DN *vlvSearch_getBase(struct vlvSearch* p); +int vlvSearch_getScope(struct vlvSearch* p); +Slapi_Filter *vlvSearch_getFilter(struct vlvSearch* p); +int vlvSearch_isVlvSearchEntry(Slapi_Entry *e); +void vlvSearch_addIndex(struct vlvSearch *pSearch, struct vlvIndex *pIndex); + + +struct vlvIndex* vlvIndex_new(); +void vlvIndex_init(struct vlvIndex* p, backend *be, struct vlvSearch* pSearch, const Slapi_Entry *e); +void vlvIndex_delete(struct vlvIndex** ppvs); +PRUint32 vlvIndex_get_indexlength(struct vlvIndex* p, DB *db, back_txn *txn); +void vlvIndex_increment_indexlength(struct vlvIndex* p, DB *db, back_txn *txn); +void vlvIndex_decrement_indexlength(struct vlvIndex* p, DB *db, back_txn *txn); +void vlvIndex_incrementUsage(struct vlvIndex* p); +const char *vlvIndex_filename(const struct vlvIndex* p); +int vlvIndex_enabled(const struct vlvIndex* p); +int vlvIndex_online(const struct vlvIndex *p); +void vlvIndex_go_offline(struct vlvIndex *p, backend *be); +void vlvIndex_go_online(struct vlvIndex *p, backend *be); +int vlvIndex_accessallowed(struct vlvIndex *p, Slapi_PBlock *pb); +const Slapi_DN *vlvIndex_getBase(struct vlvIndex* p); +int vlvIndex_getScope(struct vlvIndex* p); +Slapi_Filter *vlvIndex_getFilter(struct vlvIndex* p); +const char *vlvIndex_getName(struct vlvIndex* p); +int vlvIndex_isVlvIndexEntry(Slapi_Entry *e); + +#define VLV_ACCESS_DENIED -1 +#define VLV_BLD_LIST_FAILED -2 +#define VLV_FIND_SEARCH_FAILED -3 + + +#endif |