Bug 531642 - EntryUSN: RFE: a configuration option to make entryusn "global"

https://bugzilla.redhat.com/show_bug.cgi?id=531642
Resolves: 531642
Fix description:
1. Introduced a config parameter nsslapd-entryusn-global: on|off to
   enable | disable the global mode.  By default, off.
   In the global mode, search on root dse returns "lastusn: <num>"
   without the backend subtype (e.g., "lastusn;userroot: <num>")
2. Added slapi_get_next_suffix_ext to mapping_tree.c, which visits
   children as well as siblings in the mapping tree.
   (Note: slapi_get_next_suffix does just siblings.)
3. import (ldif2db) adds "entryusn: 0" to every entry unless the
   entry already contains the entryusn attribute.
4. ldbm_back_delete, ldbm_back_modify, ldbm_back_modrdn: set
   ldap_result_code to pblock so that bepost plugin could see if
   the operation was successful or not.
See also http://directory.fedoraproject.org/wiki/Entry_USN#Global_mode

10 files changed, 122 insertions, 27 deletions

diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h
index f0e290eb..f4a6f434 100644
--- a/ldap/servers/slapd/back-ldbm/back-ldbm.h
+++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h
@@ -635,6 +635,7 @@ struct ldbminfo {
     int li_flags;
     int li_fat_lock;         /* 608146 -- make this configurable, first */
     int li_legacy_errcode;   /* 615428 -- in case legacy err code is expected */
+    Slapi_Counter *li_global_usn_counter; /* global USN counter */
 };
 
 /* li_flags could store these bits defined in ../slapi-plugin.h
@@ -840,4 +841,7 @@ typedef struct _back_search_result_set
  */ 
 #define LDBM_ERROR_FOUND_DUPDN 9999
 
+/* Initial entryusn value */
+#define INITIALUSN (PRUint64)(-1)
+
 #endif /* _back_ldbm_h_ */
diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c
index 3a65590b..a17ba5ce 100644
--- a/ldap/servers/slapd/back-ldbm/dblayer.c
+++ b/ldap/servers/slapd/back-ldbm/dblayer.c
@@ -657,6 +657,10 @@ int dblayer_terminate(struct ldbminfo *li)
     slapi_ch_free((void**)&priv);
     li->li_dblayer_private = NULL;
 
+    if (config_get_entryusn_global()) {
+        slapi_counter_destroy(&li->li_global_usn_counter);
+    }
+
     return 0;
 }
 
diff --git a/ldap/servers/slapd/back-ldbm/import-threads.c b/ldap/servers/slapd/back-ldbm/import-threads.c
index 8ea64774..7107eb41 100644
--- a/ldap/servers/slapd/back-ldbm/import-threads.c
+++ b/ldap/servers/slapd/back-ldbm/import-threads.c
@@ -666,11 +666,30 @@ import_producer(void *param)
             pw_encodevals( (Slapi_Value **)va ); /* jcm - cast away const */
         }
 
-         if (job->flags & FLAG_ABORT) { 
-             backentry_free(&ep);
-             goto error;
-         }
+        if (job->flags & FLAG_ABORT) { 
+            backentry_free(&ep);
+            goto error;
+        }
 
+        /* 
+         * Check if entryusn plugin is enabled.  
+         * If yes, add "entryusn: 0" to the entry 
+         * if it does not have the attr type .
+         */
+        if (plugin_enabled("USN", (void *)plugin_get_default_component_id())) {
+            if (slapi_entry_attr_find(ep->ep_entry, SLAPI_ATTR_ENTRYUSN,
+                                                      &attr)) { /* not found */
+                /* add entryusn: 0 to the entry */
+                Slapi_Value *usn_value = NULL;
+                struct berval usn_berval = {0};
+                usn_berval.bv_val = slapi_ch_smprintf("0");
+                usn_berval.bv_len = strlen(usn_berval.bv_val);
+                usn_value = slapi_value_new_berval(&usn_berval);
+                slapi_entry_add_value(ep->ep_entry,
+                                      SLAPI_ATTR_ENTRYUSN, usn_value);
+                slapi_value_free(&usn_value);
+            }
+        }
 
         /* Now we have this new entry, all decoded
          * Next thing we need to do is:
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_delete.c b/ldap/servers/slapd/back-ldbm/ldbm_delete.c
index 36ded0c9..d2c0401c 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_delete.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_delete.c
@@ -892,6 +892,8 @@ common_return:
 		backentry_free( &tombstone );
 	}
 	
+	/* result code could be used in the bepost plugin functions. */
+	slapi_pblock_get(pb, SLAPI_RESULT_CODE, &ldap_result_code);
 	/*
 	 * The bepostop is called even if the operation fails,
 	 * but not if the operation is purging tombstones.
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c b/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c
index 462a501f..882f10a2 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_instance_config.c
@@ -855,9 +855,21 @@ static int ldbm_instance_generate(struct ldbminfo *li, char *instance_name,
     rc = ldbm_instance_create_default_indexes(new_be);
 
     /* if USN plugin is enabled, set slapi_counter */
-    if (plugin_enabled("USN", li->li_identity)) {
-        /* slapi_counter_new sets the initial value to 0 */
-        new_be->be_usn_counter = slapi_counter_new();
+    if (plugin_enabled("USN", li->li_identity) && ldbm_back_isinitialized()) {
+        /* 
+         * ldbm_back is already initialized. 
+         * I.e., a new instance is being added.
+         * If not initialized, ldbm_usn_init is called later and
+         * be usn counter is initialized there.
+         */
+        if (config_get_entryusn_global()) {
+            /* global usn counter is already created. 
+             * set it to be_usn_counter. */
+            new_be->be_usn_counter = li->li_global_usn_counter;
+        } else {
+            new_be->be_usn_counter = slapi_counter_new();
+            slapi_counter_set_value(new_be->be_usn_counter, INITIALUSN);
+        }
     }
 
     if (ret_be != NULL) {
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_modify.c b/ldap/servers/slapd/back-ldbm/ldbm_modify.c
index 30a99b18..529f55e0 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_modify.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_modify.c
@@ -332,7 +332,6 @@ ldbm_back_modify( Slapi_PBlock *pb )
 		}
 		if ( !change_entry || ldap_result_code != 0 ) {
 			/* change_entry == 0 is not an error, but we need to free lock etc */
-			slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ldap_result_code);
 			goto error_return;
 		}
 	}
@@ -550,7 +549,11 @@ common_return:
 	{
 		CACHE_RETURN( &inst->inst_cache, &ec );
 	}
-	/* JCMREPL - The bepostop is called even if the operation fails. */
+
+	/* result code could be used in the bepost plugin functions. */
+	slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ldap_result_code);
+
+	/* The bepostop is called even if the operation fails. */
 	if (!disk_full)
 		plugin_call_plugins (pb, SLAPI_PLUGIN_BE_POST_MODIFY_FN);
 
@@ -560,10 +563,10 @@ common_return:
 	}
 	if(ldap_result_code!=-1)
 	{
-    	slapi_send_ldap_result( pb, ldap_result_code, NULL, ldap_result_message, 0, NULL );
+		slapi_send_ldap_result( pb, ldap_result_code, NULL, ldap_result_message, 0, NULL );
 	}
 	
-    slapi_ch_free( (void**)&errbuf);
+	slapi_ch_free_string(&errbuf);
 	return rc;
 }
 
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c
index dbe53e5b..4848c83e 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_modrdn.c
@@ -995,6 +995,9 @@ common_return:
     }
 
     moddn_unlock_and_return_entries(be,&e,&existingentry);
+
+    /* result code could be used in the bepost plugin functions. */
+    slapi_pblock_get(pb, SLAPI_RESULT_CODE, &ldap_result_code);
     /*
      * The bepostop is called even if the operation fails.
      */
diff --git a/ldap/servers/slapd/back-ldbm/ldbm_usn.c b/ldap/servers/slapd/back-ldbm/ldbm_usn.c
index 4bcd95ec..2bb64bc2 100644
--- a/ldap/servers/slapd/back-ldbm/ldbm_usn.c
+++ b/ldap/servers/slapd/back-ldbm/ldbm_usn.c
@@ -67,6 +67,9 @@ ldbm_usn_init(struct ldbminfo  *li)
     int rc = 0;
     Slapi_Backend *be = NULL;
     PRUint64 last_usn = 0;
+    PRUint64 global_last_usn = INITIALUSN;
+    int isglobal = config_get_entryusn_global();
+    int isfirst = 1;
 
     /* if USN is not enabled, return immediately */
     if (!plugin_enabled("USN", li->li_identity)) {
@@ -75,15 +78,34 @@ ldbm_usn_init(struct ldbminfo  *li)
 
     /* Search each namingContext in turn */
     for ( sdn = slapi_get_first_suffix( &node, 0 ); sdn != NULL;
-          sdn = slapi_get_next_suffix( &node, 0 )) {
+          sdn = slapi_get_next_suffix_ext( &node, 0 )) {
         be = slapi_mapping_tree_find_backend_for_sdn(sdn);
-        slapi_log_error(SLAPI_LOG_TRACE, "ldbm_usn_init",
-                        "backend: %s\n", be->be_name);
+        slapi_log_error(SLAPI_LOG_BACKLDBM, "ldbm_usn_init",
+                    "backend: %s \n", be->be_name, isglobal?"(global mode)":"");
         rc = usn_get_last_usn(be, &last_usn);
         if (0 == rc) { /* only when the last usn is available */
-            be->be_usn_counter = slapi_counter_new();
-            slapi_counter_set_value(be->be_usn_counter, last_usn);
-            slapi_counter_increment(be->be_usn_counter); /* stores next usn */
+            if (isglobal) {
+                if (isfirst) {
+                    li->li_global_usn_counter = slapi_counter_new();
+                    isfirst = 0;
+                }
+                /* share one counter */
+                be->be_usn_counter = li->li_global_usn_counter;
+                /* Initialize global_last_usn;
+                 * Set the largest last_usn among backends */
+                if ((global_last_usn == INITIALUSN) ||
+                   ((last_usn != INITIALUSN) && (global_last_usn < last_usn))) {
+                    global_last_usn = last_usn;
+                }
+                slapi_counter_set_value(be->be_usn_counter, global_last_usn);
+                /* stores next usn */
+                slapi_counter_increment(be->be_usn_counter);
+            } else {
+                be->be_usn_counter = slapi_counter_new();
+                slapi_counter_set_value(be->be_usn_counter, last_usn);
+                /* stores next usn */
+                slapi_counter_increment(be->be_usn_counter);
+            }
         }
     }
 bail:
@@ -110,7 +132,7 @@ usn_get_last_usn(Slapi_Backend *be, PRUint64 *last_usn)
     memset(&key, 0, sizeof(key));
     memset(&value, 0, sizeof(key));
 
-    *last_usn = -1; /* to start from 0 */
+    *last_usn = INITIALUSN; /* to start from 0 */
 
     /* Open the entryusn index */
     ainfo_get(be, SLAPI_ATTR_ENTRYUSN, &ai);
@@ -179,14 +201,32 @@ int
 ldbm_set_last_usn(Slapi_Backend *be)
 {
     PRUint64 last_usn = 0;
-    int rc = usn_get_last_usn(be, &last_usn);
-
-    if (0 == rc) { /* only when the last usn is available */
-        /* destroy old counter, if any */
-        slapi_counter_destroy(&(be->be_usn_counter));
-        be->be_usn_counter = slapi_counter_new();
-        slapi_counter_set_value(be->be_usn_counter, last_usn);
-        slapi_counter_increment(be->be_usn_counter); /* stores next usn */
+    PRUint64 current_usn = 0;
+    int isglobal = config_get_entryusn_global();
+    int rc = -1;
+
+    if (NULL == be) {
+        slapi_log_error(SLAPI_LOG_FATAL, "ldbm_set_last_usn",
+                        "Empty backend\n");
+        return rc;
+    }
+
+    if (isglobal) {
+       struct ldbminfo *li = (struct ldbminfo *)be->be_database->plg_private;
+       /* destroy old counter, if any */
+       slapi_counter_destroy(&(li->li_global_usn_counter));
+       ldbm_usn_init(li);
+    } else {
+        slapi_log_error(SLAPI_LOG_BACKLDBM, "ldbm_set_last_usn",
+                        "backend: %s\n", be->be_name);
+        rc = usn_get_last_usn(be, &last_usn);
+        if (0 == rc) { /* only when the last usn is available */
+            /* destroy old counter, if any */
+            slapi_counter_destroy(&(be->be_usn_counter));
+            be->be_usn_counter = slapi_counter_new();
+            slapi_counter_set_value(be->be_usn_counter, last_usn);
+            slapi_counter_increment(be->be_usn_counter); /* stores next usn */
+        }
     }
 
     return rc;
diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h
index 90ac5700..615d9373 100644
--- a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h
+++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h
@@ -479,6 +479,7 @@ int ldbm_back_entry_release( Slapi_PBlock *pb, void *backend_info_ptr );
 void ldbm_back_search_results_release( void **search_results );
 int ldbm_back_init( Slapi_PBlock *pb ); 
 void ldbm_back_prev_search_results( Slapi_PBlock *pb );
+int ldbm_back_isinitialized();
 
 /*
  * monitor.c
diff --git a/ldap/servers/slapd/back-ldbm/start.c b/ldap/servers/slapd/back-ldbm/start.c
index 8ea44aa1..6c19c768 100644
--- a/ldap/servers/slapd/back-ldbm/start.c
+++ b/ldap/servers/slapd/back-ldbm/start.c
@@ -46,6 +46,14 @@
 
 #include "back-ldbm.h"
 
+static int initialized = 0;
+
+int
+ldbm_back_isinitialized()
+{
+    return initialized;
+}
+
 /*
  * Start the LDBM plugin, and all its instances.
  */
@@ -53,7 +61,6 @@ int
 ldbm_back_start( Slapi_PBlock *pb )
 {
   struct ldbminfo  *li;
-  static int initialized = 0;
   char *home_dir;
   int action;
   int retval;