Fixed hard-coded database name for TPS VLV indexes.

The vlv.ldif for TPS has been modified to remove the hard-coded
database name and to use customizable parameter instead.

The token and activity REST services have been modified to search
the database using VLV.

The existing database can be fixed using the following procedure:
http://pki.fedoraproject.org/wiki/Database_Upgrade_for_PKI_10.3.x#Relocating_VLV_indexes

https://fedorahosted.org/pki/ticket/2342

9 files changed, 68 insertions, 46 deletions

diff --git a/base/server/cmscore/src/com/netscape/cmscore/dbs/LDAPDatabase.java b/base/server/cmscore/src/com/netscape/cmscore/dbs/LDAPDatabase.java
index 3e9f25e95..48d15950e 100644
--- a/base/server/cmscore/src/com/netscape/cmscore/dbs/LDAPDatabase.java
+++ b/base/server/cmscore/src/com/netscape/cmscore/dbs/LDAPDatabase.java
@@ -16,6 +16,7 @@ import com.netscape.certsrv.dbs.IDBRegistry;
 import com.netscape.certsrv.dbs.IDBSSession;
 import com.netscape.certsrv.dbs.IDBSearchResults;
 import com.netscape.certsrv.dbs.IDBSubsystem;
+import com.netscape.certsrv.dbs.IDBVirtualList;
 import com.netscape.certsrv.dbs.Modification;
 import com.netscape.certsrv.dbs.ModificationSet;
 import com.netscape.cmsutil.ldap.LDAPUtil;
@@ -140,6 +141,25 @@ public abstract class LDAPDatabase<E extends IDBObj> extends Database<E> {
         }
     }
 
+    public IDBVirtualList<E> findRecords(String keyword, Map<String, String> attributes,
+            String sortKey, int pageSize) throws Exception {
+
+        CMS.debug("LDAPDatabase: findRecords()");
+
+        try (IDBSSession session = dbSubsystem.createSession()) {
+
+            String ldapFilter = createFilter(keyword, attributes);
+            CMS.debug("LDAPDatabase: searching " + baseDN + " with filter " + ldapFilter);
+
+            return session.<E>createVirtualList(
+                    baseDN,
+                    ldapFilter,
+                    null,
+                    sortKey,
+                    pageSize);
+        }
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public E getRecord(String id) throws Exception {
diff --git a/base/tps-client/scripts/addVLVIndexes.ldif b/base/tps-client/scripts/addVLVIndexes.ldif
index 9dc86ece1..0ce0437e2 100644
--- a/base/tps-client/scripts/addVLVIndexes.ldif
+++ b/base/tps-client/scripts/addVLVIndexes.ldif
@@ -18,23 +18,23 @@
 # All rights reserved.
 # --- END COPYRIGHT BLOCK ---
 #
-dn: cn=tus-listTokens-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
-cn: tus-listtokens-vlv
+dn: cn=listtokens,cn={database},cn=ldbm database,cn=plugins,cn=config
+cn: listtokens
 objectClass: top
 objectClass: vlvsearch
 vlvBase: ou=Tokens,$TOKENDB_ROOT
 vlvFilter: (&(cn=*)(tokenUserID=*))
 vlvScope: 2
 
-dn: cn=tus-listActivities-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
-cn: tus-listActivities-vlv
+dn: cn=listActivities,cn={database},cn=ldbm database,cn=plugins,cn=config
+cn: listActivities
 objectClass: top
 objectClass: vlvsearch
 vlvBase: ou=Activities,$TOKENDB_ROOT
 vlvFilter: (&(tokenID=*)(tokenUserID=*))
 vlvScope: 2
 
-dn: cn=listTokensIndex,cn=tus-listTokens-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+dn: cn=listTokensIndex,cn=listtokens,cn={database},cn=ldbm database,cn=plugins,cn=config
 cn: listTokensIndex
 objectClass: top
 objectClass: vlvindex
@@ -42,7 +42,7 @@ vlvSort: -dateOfModify
 vlvEnabled: 1
 vlvUses: 0
 
-dn: cn=listActivitiesIndex,cn=tus-listActivities-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+dn: cn=listActivitiesIndex,cn=listActivities,cn={database},cn=ldbm database,cn=plugins,cn=config
 cn: listActivitiesIndex
 objectClass: top
 objectClass: vlvindex
diff --git a/base/tps-client/scripts/vlvtasks.ldif b/base/tps-client/scripts/vlvtasks.ldif
index b1b93aabf..b82e9d38b 100644
--- a/base/tps-client/scripts/vlvtasks.ldif
+++ b/base/tps-client/scripts/vlvtasks.ldif
@@ -23,6 +23,6 @@ objectclass: top
 objectclass: extensibleObject
 cn: index1160528734
 ttl: 4
-nsinstance: userRoot
+nsinstance: {database}
 nsindexVLVAttribute: listTokensIndex
 nsindexVLVAttribute: listActivitiesIndex
diff --git a/base/tps/shared/conf/vlv.ldif b/base/tps/shared/conf/vlv.ldif
index db7988e36..2ea84689e 100644
--- a/base/tps/shared/conf/vlv.ldif
+++ b/base/tps/shared/conf/vlv.ldif
@@ -18,23 +18,23 @@
 # All rights reserved.
 # --- END COPYRIGHT BLOCK ---
 #
-dn: cn=tus-listTokens-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
-cn: tus-listtokens-vlv
+dn: cn=listTokens,cn={database},cn=ldbm database,cn=plugins,cn=config
+cn: listtokens
 objectClass: top
 objectClass: vlvsearch
 vlvBase: ou=Tokens,{rootSuffix}
 vlvFilter: (&(cn=*)(tokenUserID=*))
 vlvScope: 2
 
-dn: cn=tus-listActivities-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
-cn: tus-listActivities-vlv
+dn: cn=listActivities,cn={database},cn=ldbm database,cn=plugins,cn=config
+cn: listActivities
 objectClass: top
 objectClass: vlvsearch
 vlvBase: ou=Activities,{rootSuffix}
 vlvFilter: (&(tokenID=*)(tokenUserID=*))
 vlvScope: 2
 
-dn: cn=listTokensIndex,cn=tus-listTokens-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+dn: cn=listTokensIndex,cn=listTokens,cn={database},cn=ldbm database,cn=plugins,cn=config
 cn: listTokensIndex
 objectClass: top
 objectClass: vlvindex
@@ -42,7 +42,7 @@ vlvSort: -dateOfModify
 vlvEnabled: 1
 vlvUses: 0
 
-dn: cn=listActivitiesIndex,cn=tus-listActivities-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+dn: cn=listActivitiesIndex,cn=listActivities,cn={database},cn=ldbm database,cn=plugins,cn=config
 cn: listActivitiesIndex
 objectClass: top
 objectClass: vlvindex
diff --git a/base/tps/shared/conf/vlvtasks.ldif b/base/tps/shared/conf/vlvtasks.ldif
index b1b93aabf..b82e9d38b 100644
--- a/base/tps/shared/conf/vlvtasks.ldif
+++ b/base/tps/shared/conf/vlvtasks.ldif
@@ -23,6 +23,6 @@ objectclass: top
 objectclass: extensibleObject
 cn: index1160528734
 ttl: 4
-nsinstance: userRoot
+nsinstance: {database}
 nsindexVLVAttribute: listTokensIndex
 nsindexVLVAttribute: listActivitiesIndex
diff --git a/base/tps/src/org/dogtagpki/server/tps/dbs/ActivityDatabase.java b/base/tps/src/org/dogtagpki/server/tps/dbs/ActivityDatabase.java
index fdcd4eaff..46ab2128d 100644
--- a/base/tps/src/org/dogtagpki/server/tps/dbs/ActivityDatabase.java
+++ b/base/tps/src/org/dogtagpki/server/tps/dbs/ActivityDatabase.java
@@ -107,7 +107,7 @@ public class ActivityDatabase extends LDAPDatabase<ActivityRecord> {
         createFilter(sb, attributes);
 
         if (sb.length() == 0) {
-            sb.append("(id=*)");
+            sb.append("(&(tokenID=*)(userID=*))"); // listActivities VLV
         }
 
         return sb.toString();
diff --git a/base/tps/src/org/dogtagpki/server/tps/dbs/TokenDatabase.java b/base/tps/src/org/dogtagpki/server/tps/dbs/TokenDatabase.java
index 19f4830b3..f751c66c9 100644
--- a/base/tps/src/org/dogtagpki/server/tps/dbs/TokenDatabase.java
+++ b/base/tps/src/org/dogtagpki/server/tps/dbs/TokenDatabase.java
@@ -67,7 +67,7 @@ public class TokenDatabase extends LDAPDatabase<TokenRecord> {
         createFilter(sb, attributes);
 
         if (sb.length() == 0) {
-            sb.append("(id=*)");
+            sb.append("(&(id=*)(userID=*))"); // listTokens VLV
         }
 
         return sb.toString();
diff --git a/base/tps/src/org/dogtagpki/server/tps/rest/ActivityService.java b/base/tps/src/org/dogtagpki/server/tps/rest/ActivityService.java
index 6002e7a94..5fb3d1956 100644
--- a/base/tps/src/org/dogtagpki/server/tps/rest/ActivityService.java
+++ b/base/tps/src/org/dogtagpki/server/tps/rest/ActivityService.java
@@ -21,7 +21,6 @@ package org.dogtagpki.server.tps.rest;
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.net.URLEncoder;
-import java.util.Iterator;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.core.Context;
@@ -38,6 +37,7 @@ import org.jboss.resteasy.plugins.providers.atom.Link;
 import com.netscape.certsrv.apps.CMS;
 import com.netscape.certsrv.base.BadRequestException;
 import com.netscape.certsrv.base.PKIException;
+import com.netscape.certsrv.dbs.IDBVirtualList;
 import com.netscape.certsrv.logging.ActivityCollection;
 import com.netscape.certsrv.logging.ActivityData;
 import com.netscape.certsrv.logging.ActivityResource;
@@ -121,37 +121,39 @@ public class ActivityService extends PKIService implements ActivityResource {
             TPSSubsystem subsystem = (TPSSubsystem)CMS.getSubsystem(TPSSubsystem.ID);
             ActivityDatabase database = subsystem.getActivityDatabase();
 
-            Iterator<ActivityRecord> activities = database.findRecords(filter).iterator();
+            IDBVirtualList<ActivityRecord> list = database.findRecords(filter, null, "date", size);
+            int total = list.getSize();
 
             ActivityCollection response = new ActivityCollection();
-            int i = 0;
 
-            // skip to the start of the page
-            for ( ; i<start && activities.hasNext(); i++) activities.next();
+            // return entries in the requested page
+            for (int i = start; i < start + size && i < total; i++) {
+                ActivityRecord record = list.getElementAt(i);
 
-            // return entries up to the page size
-            for ( ; i<start+size && activities.hasNext(); i++) {
-                response.addEntry(createActivityData(activities.next()));
+                if (record == null) {
+                    CMS.debug("ActivityService: Activity record not found");
+                    throw new PKIException("Activity record not found");
+                }
+
+                response.addEntry(createActivityData(record));
             }
 
-            // count the total entries
-            for ( ; activities.hasNext(); i++) activities.next();
-            response.setTotal(i);
+            response.setTotal(total);
 
             if (start > 0) {
-                URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", Math.max(start-size, 0)).build();
+                URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", Math.max(start - size, 0)).build();
                 response.addLink(new Link("prev", uri));
             }
 
-            if (start+size < i) {
-                URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", start+size).build();
+            if (start+size < total) {
+                URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", start + size).build();
                 response.addLink(new Link("next", uri));
             }
 
             return createOKResponse(response);
 
         } catch (Exception e) {
-            e.printStackTrace();
+            CMS.debug(e);
             throw new PKIException(e.getMessage());
         }
     }
@@ -170,7 +172,7 @@ public class ActivityService extends PKIService implements ActivityResource {
             return createOKResponse(createActivityData(database.getRecord(activityID)));
 
         } catch (Exception e) {
-            e.printStackTrace();
+            CMS.debug(e);
             throw new PKIException(e.getMessage());
         }
     }
diff --git a/base/tps/src/org/dogtagpki/server/tps/rest/TokenService.java b/base/tps/src/org/dogtagpki/server/tps/rest/TokenService.java
index 2104f292b..40022a440 100644
--- a/base/tps/src/org/dogtagpki/server/tps/rest/TokenService.java
+++ b/base/tps/src/org/dogtagpki/server/tps/rest/TokenService.java
@@ -24,7 +24,6 @@ import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.ResourceBundle;
 
@@ -46,6 +45,7 @@ import com.netscape.certsrv.apps.CMS;
 import com.netscape.certsrv.base.BadRequestException;
 import com.netscape.certsrv.base.PKIException;
 import com.netscape.certsrv.dbs.EDBException;
+import com.netscape.certsrv.dbs.IDBVirtualList;
 import com.netscape.certsrv.ldap.LDAPExceptionConverter;
 import com.netscape.certsrv.logging.ILogger;
 import com.netscape.certsrv.tps.token.TokenCollection;
@@ -259,31 +259,31 @@ public class TokenService extends PKIService implements TokenResource {
             TPSSubsystem subsystem = (TPSSubsystem) CMS.getSubsystem(TPSSubsystem.ID);
             TokenDatabase database = subsystem.getTokenDatabase();
 
-            Iterator<TokenRecord> tokens = database.findRecords(filter, attributes).iterator();
+            IDBVirtualList<TokenRecord> list = database.findRecords(filter, null, "modifyTimestamp", size);
+            int total = list.getSize();
 
             TokenCollection response = new TokenCollection();
-            int i = 0;
-
-            // skip to the start of the page
-            for (; i < start && tokens.hasNext(); i++)
-                tokens.next();
 
             // return entries up to the page size
-            for (; i < start + size && tokens.hasNext(); i++) {
-                response.addEntry(createTokenData(tokens.next()));
+            for (int i = start; i < start + size && i < total; i++) {
+                TokenRecord record = list.getElementAt(i);
+
+                if (record == null) {
+                    CMS.debug("TokenService: Token record not found");
+                    throw new PKIException("Token record not found");
+                }
+
+                response.addEntry(createTokenData(record));
             }
 
-            // count the total entries
-            for (; tokens.hasNext(); i++)
-                tokens.next();
-            response.setTotal(i);
+            response.setTotal(total);
 
             if (start > 0) {
                 URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", Math.max(start - size, 0)).build();
                 response.addLink(new Link("prev", uri));
             }
 
-            if (start + size < i) {
+            if (start + size < total) {
                 URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", start + size).build();
                 response.addLink(new Link("next", uri));
             }