Add code to reindex data during cloning without replication

When setting up a clone, indexes are added before the
replication agreements are set up and the consumer is initialized.
Thus, as data is replicated and added to the clone db, the
data is indexed.

When cloning is done with the replication agreements already set
up and the data replicated, the existing data is not indexed and
cannot be accessed in searches.  The data needs to be reindexed.

Related to ticket 1414

5 files changed, 78 insertions, 30 deletions

diff --git a/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java b/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
index c8ab38ce7..a417be4a3 100644
--- a/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
+++ b/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
@@ -88,6 +88,7 @@ import netscape.security.x509.X500Name;
 import netscape.security.x509.X509CertImpl;
 import netscape.security.x509.X509Key;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.velocity.context.Context;
 import org.mozilla.jss.CryptoManager;
 import org.mozilla.jss.CryptoManager.NicknameConflictException;
@@ -1346,6 +1347,7 @@ public class ConfigurationUtils {
         boolean remove = cs.getBoolean("preop.database.removeData", false);
         boolean createNewDB = cs.getBoolean("preop.database.createNewDB", true);
         boolean setupReplication = cs.getBoolean("preop.database.setupReplication", true);
+        boolean reindexData = cs.getBoolean("preop.database.reindexData", false);
 
         IConfigStore dbCfg = cs.getSubStore("internaldb");
         ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
@@ -1419,6 +1421,9 @@ public class ConfigurationUtils {
                     // On the other hand, if we are not setting up replication, then we
                     // are assuming that replication is already taken care of, and schema
                     // has already been replicated.  No need to add.
+
+                    // Also, data will be replicated from master to clone
+                    // so clone does not need the data
                     boolean replicateSchema = cs.getBoolean("preop.internaldb.replicateSchema", true);
                     if (!replicateSchema || !setupReplication) {
                         importLDIFS("preop.internaldb.schema.ldif", conn);
@@ -1427,9 +1432,15 @@ public class ConfigurationUtils {
 
                     // add the index before replication, add VLV indexes afterwards
                     importLDIFS("preop.internaldb.index_ldif", conn);
+
+                    if (!setupReplication && reindexData) {
+                        // data has already been replicated but not yet indexed -
+                        // re-index here
+                        populateIndexes(conn);
+                    }
                 } else {
-                    // data will be replicated from the master to the clone
-                    // so clone does not need the data
+                    // this is the normal non-clone case
+                    // import schema, database, initial data and indexes
                     importLDIFS("preop.internaldb.schema.ldif", conn);
                     importLDIFS("preop.internaldb.ldif", conn);
                     importLDIFS("preop.internaldb.data_ldif", conn);
@@ -1444,6 +1455,51 @@ public class ConfigurationUtils {
         }
     }
 
+    private static void populateIndexes(LDAPConnection conn) throws EPropertyNotFound, IOException, EBaseException {
+        CMS.debug("populateIndexes(): start");
+        IConfigStore cs = CMS.getConfigStore();
+
+        importLDIFS("preop.internaldb.index_task_ldif", conn, false);
+
+        /* For populating indexes, we need to check if the task has completed.
+           Presence of nsTaskExitCode means task is complete
+         */
+        String wait_dn = cs.getString("preop.internaldb.index_wait_dn", "");
+        if (!StringUtils.isEmpty(wait_dn)) {
+            wait_for_task(conn, wait_dn);
+        }
+    }
+
+    private static void wait_for_task(LDAPConnection conn, String wait_dn) {
+        LDAPEntry task = null;
+        boolean taskComplete = false;
+        CMS.debug("Checking wait_dn " + wait_dn);
+        do {
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException e) {
+                // restore the interrupted status
+                Thread.currentThread().interrupt();
+            }
+
+            try {
+                task = conn.read(wait_dn, (String[]) null);
+                if (task != null) {
+                    LDAPAttribute attr = task.getAttribute("nsTaskExitCode");
+                    if (attr != null) {
+                        taskComplete = true;
+                        String val = (String) attr.getStringValues().nextElement();
+                        if (val.compareTo("0") != 0) {
+                            CMS.debug("Error in populating indexes: nsTaskExitCode=" + val);
+                        }
+                    }
+                }
+            } catch (Exception le) {
+                CMS.debug("Still checking wait_dn '" + wait_dn + "' (" + le.toString() + ")");
+            }
+        } while (!taskComplete);
+    }
+
     private static void createBaseEntry(String baseDN, LDAPConnection conn) throws EBaseException {
         try {
             CMS.debug("Creating base DN: " + baseDN);
@@ -1624,7 +1680,11 @@ public class ConfigurationUtils {
         }
     }
 
-    public static void importLDIFS(String param, LDAPConnection conn) throws IOException, EPropertyNotFound,
+    public static void importLDIFS(String param, LDAPConnection conn) throws EPropertyNotFound, IOException, EBaseException {
+        importLDIFS(param, conn, true);
+    }
+
+    public static void importLDIFS(String param, LDAPConnection conn, boolean suppressErrors) throws IOException, EPropertyNotFound,
             EBaseException {
         IConfigStore cs = CMS.getConfigStore();
 
@@ -1706,6 +1766,9 @@ public class ConfigurationUtils {
                 for (String error : errors) {
                     CMS.debug(error);
                 }
+                if (!suppressErrors) {
+                    throw new EBaseException("LDAP Errors in importing " + filename);
+                }
             }
         }
     }
@@ -1836,33 +1899,7 @@ public class ConfigurationUtils {
              */
             String wait_dn = cs.getString("preop.internaldb.wait_dn", "");
             if (!wait_dn.equals("")) {
-                LDAPEntry task = null;
-                boolean taskComplete = false;
-                CMS.debug("Checking wait_dn " + wait_dn);
-                do {
-                    try {
-                        Thread.sleep(1000);
-                    } catch (InterruptedException e) {
-                        // restore the interrupted status
-                        Thread.currentThread().interrupt();
-                    }
-
-                    try {
-                        task = conn.read(wait_dn, (String[]) null);
-                        if (task != null) {
-                            LDAPAttribute attr = task.getAttribute("nsTaskExitCode");
-                            if (attr != null) {
-                                taskComplete = true;
-                                String val = (String) attr.getStringValues().nextElement();
-                                if (val.compareTo("0") != 0) {
-                                    CMS.debug("Error in populating local VLV indexes: nsTaskExitCode=" + val);
-                                }
-                            }
-                        }
-                    } catch (Exception le) {
-                        CMS.debug("Still checking wait_dn '" + wait_dn + "' (" + le.toString() + ")");
-                    }
-                } while (!taskComplete);
+                wait_for_task(conn, wait_dn);
             }
         } catch (Exception e) {
             CMS.debug("populateVLVIndexes(): Exception thrown: " + e);
diff --git a/base/server/cms/src/org/dogtagpki/server/rest/SystemConfigService.java b/base/server/cms/src/org/dogtagpki/server/rest/SystemConfigService.java
index 31891ca62..6e5414756 100644
--- a/base/server/cms/src/org/dogtagpki/server/rest/SystemConfigService.java
+++ b/base/server/cms/src/org/dogtagpki/server/rest/SystemConfigService.java
@@ -649,6 +649,7 @@ public class SystemConfigService extends PKIService implements SystemConfigResou
         cs.putString("preop.database.removeData", data.getRemoveData());
         cs.putBoolean("preop.database.createNewDB", data.getCreateNewDB());
         cs.putBoolean("preop.database.setupReplication", data.getSetupReplication());
+        cs.putBoolean("preop.database.reindexData", data.getReindexData());
     }
 
     public void initializeDatabase(ConfigurationRequest data) {
diff --git a/base/server/etc/default.cfg b/base/server/etc/default.cfg
index 26ffd0d38..ddd2d8367 100644
--- a/base/server/etc/default.cfg
+++ b/base/server/etc/default.cfg
@@ -194,6 +194,7 @@ pki_clone_replication_master_port=
 pki_clone_replication_clone_port=
 pki_clone_replication_security=None
 pki_clone_setup_replication=True
+pki_clone_reindex_data=False
 pki_master_hostname=%(pki_security_domain_hostname)s
 pki_master_https_port=%(pki_security_domain_https_port)s
 pki_clone_uri=https://%(pki_master_hostname)s:%(pki_master_https_port)s
diff --git a/base/server/man/man5/pki_default.cfg.5 b/base/server/man/man5/pki_default.cfg.5
index 17130aecf..4e2c13bfc 100644
--- a/base/server/man/man5/pki_default.cfg.5
+++ b/base/server/man/man5/pki_default.cfg.5
@@ -267,6 +267,14 @@ Location and password of the PKCS #12 file containing the system certificates fo
 .IP
 Defaults to True.  If set to False, the installer does not set up replication agreements from the master to the clone as part of the subsystem configuration.  In this case, it is expected that the top level suffix already exists, and that the data has already been replicated.  This option is useful if you want to use other tools to create and manage your replication topology, or if the baseDN is already replicated as part of a top-level suffix.
 .TP
+.B pki_clone_reindex_data
+.IP
+Defaults to False.  This parameter is only relevant when \fBpki_clone_setup_replication\fP is
+set to False.  In this case, it is expected that the database has been prepared and replicated
+as noted above.  Part of that preparation could involve adding indexes and indexing the data.
+If you would like the Dogtag installer to add the indexes and reindex the data instead, set
+\fBpki_clone_reindex_data\fP to True.
+.TP
 .B pki_clone_replication_master_port, pki_clone_replication_clone_port
 .IP
 Ports on which replication occurs.  These are the ports on the master and clone databases respectively.  Defaults to the internal database port. 
diff --git a/base/server/python/pki/server/deployment/pkihelper.py b/base/server/python/pki/server/deployment/pkihelper.py
index 93fa38494..b6ee61b27 100644
--- a/base/server/python/pki/server/deployment/pkihelper.py
+++ b/base/server/python/pki/server/deployment/pkihelper.py
@@ -4264,6 +4264,7 @@ class ConfigClient:
             data.cloneReplicationPort = \
                 self.mdict['pki_clone_replication_clone_port']
         data.setupReplication = self.mdict['pki_clone_setup_replication']
+        data.reindexData = self.mdict['pki_clone_reindex_data']
 
     def set_hierarchy_parameters(self, data):
         if self.subsystem == "CA":