Fixed cert-find performance.

The CertService.searchCerts() has been modified to use the VLV
properly to retrieve just the entries in the requested page, thus
reducing the response time and memory requirement.

Some classes have been modified to clean up the debugging logs.

4 files changed, 130 insertions, 69 deletions

diff --git a/base/ca/src/org/dogtagpki/server/ca/rest/CertService.java b/base/ca/src/org/dogtagpki/server/ca/rest/CertService.java
index e43909bbb..440f756de 100644
--- a/base/ca/src/org/dogtagpki/server/ca/rest/CertService.java
+++ b/base/ca/src/org/dogtagpki/server/ca/rest/CertService.java
@@ -70,6 +70,7 @@ import com.netscape.certsrv.cert.CertSearchRequest;
 import com.netscape.certsrv.dbs.EDBRecordNotFoundException;
 import com.netscape.certsrv.dbs.certdb.CertId;
 import com.netscape.certsrv.dbs.certdb.ICertRecord;
+import com.netscape.certsrv.dbs.certdb.ICertRecordList;
 import com.netscape.certsrv.dbs.certdb.ICertificateRepository;
 import com.netscape.certsrv.logging.AuditFormat;
 import com.netscape.certsrv.logging.ILogger;
@@ -442,47 +443,44 @@ public class CertService extends PKIService implements CertResource {
     @Override
     public Response searchCerts(CertSearchRequest data, Integer start, Integer size) {
 
+        CMS.debug("CertService.searchCerts()");
+
         if (data == null) {
-            throw new BadRequestException("Search request is null.");
+            throw new BadRequestException("Search request is null");
         }
 
         start = start == null ? 0 : start;
         size = size == null ? DEFAULT_SIZE : size;
 
         String filter = createSearchFilter(data);
-        CMS.debug("CertService.searchCerts: filter: " + filter);
+        CMS.debug("CertService: filter: " + filter);
 
         CertDataInfos infos = new CertDataInfos();
         try {
-            Enumeration<ICertRecord> e = repo.findCertRecords(filter);
-            if (e == null) {
-                throw new EBaseException("search results are null");
-            }
+            ICertRecordList list = repo.findCertRecordsInList(filter, null, "serialno", size);
+            int total = list.getSize();
+            CMS.debug("CertService: total: " + total);
 
-            int i = 0;
+            // return entries in the requested page
+            for (int i = start; i < start + size && i < total; i++) {
+                ICertRecord record = list.getCertRecord(i);
 
-            // skip to the start of the page
-            for (; i < start && e.hasMoreElements(); i++)
-                e.nextElement();
+                if (record == null) {
+                    CMS.debug("CertService: Certificate record not found");
+                    throw new PKIException("Certificate record not found");
+                }
 
-            // return entries up to the page size
-            for (; i < start + size && e.hasMoreElements(); i++) {
-                ICertRecord user = e.nextElement();
-                infos.addEntry(createCertDataInfo(user));
+                infos.addEntry(createCertDataInfo(record));
             }
 
-            // count the total entries
-            for (; e.hasMoreElements(); i++)
-                e.nextElement();
-
-            infos.setTotal(i);
+            infos.setTotal(total);
 
             if (start > 0) {
                 URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", Math.max(start - size, 0)).build();
                 infos.addLink(new Link("prev", uri));
             }
 
-            if (start + size < i) {
+            if (start + size < total) {
                 URI uri = uriInfo.getRequestUriBuilder().replaceQueryParam("start", start + size).build();
                 infos.addLink(new Link("next", uri));
             }
diff --git a/base/server/cms/src/com/netscape/cms/servlet/cert/ListCerts.java b/base/server/cms/src/com/netscape/cms/servlet/cert/ListCerts.java
index 185e1fa8e..d0c3fe82a 100644
--- a/base/server/cms/src/com/netscape/cms/servlet/cert/ListCerts.java
+++ b/base/server/cms/src/com/netscape/cms/servlet/cert/ListCerts.java
@@ -40,6 +40,7 @@ import netscape.security.x509.X500Name;
 import netscape.security.x509.X509CertImpl;
 import netscape.security.x509.X509Key;
 
+import com.netscape.certsrv.apps.CMS;
 import com.netscape.certsrv.authentication.IAuthToken;
 import com.netscape.certsrv.authorization.AuthzToken;
 import com.netscape.certsrv.base.EBaseException;
@@ -352,7 +353,8 @@ public class ListCerts extends CMSServlet {
         }
     }
 
-    private void processCertFilter(CMSTemplateParams argSet,
+    private void processCertFilter(
+            CMSTemplateParams argSet,
             IArgBlock header,
             int maxCount,
             BigInteger sentinel,
@@ -364,6 +366,14 @@ public class ListCerts extends CMSServlet {
             String revokeAll,
             Locale locale
             ) throws EBaseException {
+
+        CMS.debug("ListCerts.processCertFilter()");
+        CMS.debug("ListCerts: max count: " + maxCount);
+        CMS.debug("ListCerts: sentinel: " + sentinel);
+        CMS.debug("ListCerts: total record count: " + totalRecordCount);
+        CMS.debug("ListCerts: serialTo: " + serialTo);
+        CMS.debug("ListCerts: filter: " + filter);
+
         BigInteger serialToVal = MINUS_ONE;
 
         try {
@@ -380,6 +390,10 @@ public class ListCerts extends CMSServlet {
         } catch (Exception e) {
         }
 
+        CMS.debug("ListCerts: serialToVal: " + serialToVal);
+        CMS.debug("ListCerts: mReverse: " + mReverse);
+        CMS.debug("ListCerts: mHardJumpTo: " + mHardJumpTo);
+
         String jumpTo = sentinel.toString();
         int pSize = 0;
         if (mReverse) {
@@ -390,10 +404,14 @@ public class ListCerts extends CMSServlet {
         } else
             pSize = maxCount;
 
+        CMS.debug("ListCerts: pSize: " + pSize);
+
+        CMS.debug("ListCerts: calling findCertRecordsInList() with jumpTo");
         ICertRecordList list = mCertDB.findCertRecordsInList(
                 filter, (String[]) null, jumpTo, mHardJumpTo, "serialno",
                 pSize);
         // retrive maxCount + 1 entries
+        CMS.debug("ListCerts: list size: " + list.getSize());
 
         Enumeration<ICertRecord> e = list.getCertRecords(0, maxCount);
 
@@ -403,18 +421,24 @@ public class ListCerts extends CMSServlet {
         if (!serialToVal.equals(MINUS_ONE)) {
             // if user specify a range, we need to
             // calculate the totalRecordCount
+            CMS.debug("ListCerts: calling findCertRecordsInList() with serialTo");
             tolist = mCertDB.findCertRecordsInList(
                         filter,
                         (String[]) null, serialTo,
                         "serialno", maxCount);
+            CMS.debug("ListCerts: tolist size: " + tolist.getSize());
+
             Enumeration<ICertRecord> en = tolist.getCertRecords(0, 0);
 
             if (en == null || (!en.hasMoreElements())) {
+                CMS.debug("ListCerts: no results");
                 toCurIndex = list.getSize() - 1;
+
             } else {
                 toCurIndex = tolist.getCurrentIndex();
                 ICertRecord rx = en.nextElement();
                 BigInteger curToSerial = rx.getSerialNumber();
+                CMS.debug("ListCerts: curToSerial: " + curToSerial);
 
                 if (curToSerial.compareTo(serialToVal) == -1) {
                     toCurIndex = list.getSize() - 1;
@@ -424,11 +448,12 @@ public class ListCerts extends CMSServlet {
                     }
                 }
             }
+            CMS.debug("ListCerts: toCurIndex: " + toCurIndex);
         }
 
         int curIndex = list.getCurrentIndex();
+        CMS.debug("ListCerts: curIndex: " + curIndex);
 
-        int count = 0;
         BigInteger firstSerial = new BigInteger("0");
         BigInteger curSerial = new BigInteger("0");
         ICertRecord[] recs = new ICertRecord[maxCount];
@@ -438,19 +463,22 @@ public class ListCerts extends CMSServlet {
             /* in reverse (page up), because the sentinel is the one after the
              * last item to be displayed, we need to skip it
              */
+            CMS.debug("ListCerts: records:");
+            int count = 0;
             while ((count < ((mReverse && !mHardJumpTo) ? (maxCount + 1) : maxCount)) && e.hasMoreElements()) {
                 ICertRecord rec = e.nextElement();
 
                 if (rec == null) {
-                    com.netscape.certsrv.apps.CMS.debug("record " + count + " is null");
+                    CMS.debug("ListCerts: * record " + count + " is null");
                     break;
                 }
                 curSerial = rec.getSerialNumber();
-                com.netscape.certsrv.apps.CMS.debug("record " + count + " is serial#" + curSerial);
+                CMS.debug("ListCerts: * record " + count + ": " + curSerial);
 
                 if (count == 0) {
                     firstSerial = curSerial;
                     if (mReverse && !mHardJumpTo) {//reverse got one more, skip
+                        CMS.debug("ListCerts:   skipping record");
                         count++;
                         continue;
                     }
@@ -468,32 +496,34 @@ public class ListCerts extends CMSServlet {
                 if (!serialToVal.equals(MINUS_ONE)) {
                     // check if we go over the limit
                     if (curSerial.compareTo(serialToVal) == 1) {
-                        com.netscape.certsrv.apps.CMS.debug("curSerial compare serialToVal 1 breaking...");
+                        CMS.debug("ListCerts: curSerial compare serialToVal 1 breaking...");
                         break;
                     }
                 }
 
                 if (mReverse) {
+                    CMS.debug("ListCerts: returning with rcount: " + rcount);
                     recs[rcount++] = rec;
-                } else {
-
-                    IArgBlock rarg = com.netscape.certsrv.apps.CMS.createArgBlock();
 
+                } else {
+                    CMS.debug("ListCerts: returning with arg block");
+                    IArgBlock rarg = CMS.createArgBlock();
                     fillRecordIntoArg(rec, rarg);
                     argSet.addRepeatRecord(rarg);
                 }
+
                 count++;
             }
         } else {
-            com.netscape.certsrv.apps.CMS.debug(
-                    "ListCerts::processCertFilter() - no Cert Records found!");
+            CMS.debug("ListCerts: no records found");
             return;
         }
 
         if (mReverse) {
-            // fill records into arg block and argSet
+            CMS.debug("ListCerts: fill records into arg block and argSet");
             for (int ii = rcount - 1; ii >= 0; ii--) {
                 if (recs[ii] != null) {
+                    CMS.debug("ListCerts: processing recs[" + ii + "]");
                     IArgBlock rarg = com.netscape.certsrv.apps.CMS.createArgBlock();
                     //com.netscape.certsrv.apps.CMS.debug("item "+ii+" is serial # "+ recs[ii].getSerialNumber());
                     fillRecordIntoArg(recs[ii], rarg);
@@ -507,56 +537,70 @@ public class ListCerts extends CMSServlet {
 
         if (e.hasMoreElements()) {
             nextRec = e.nextElement();
+            CMS.debug("ListCerts: next record: " + nextRec.getSerialNumber());
+
+        } else {
+            CMS.debug("ListCerts: no next record");
         }
 
         header.addStringValue("op", req.getParameter("op"));
         if (revokeAll != null)
             header.addStringValue("revokeAll", revokeAll);
+
         if (mAuthName != null)
             header.addStringValue("issuerName", mAuthName.toString());
+
         if (!serialToVal.equals(MINUS_ONE))
             header.addStringValue("serialTo", serialToVal.toString());
+
         header.addStringValue("serviceURL", req.getRequestURI());
         header.addStringValue("queryCertFilter", filter);
         header.addStringValue("templateName", "queryCert");
         header.addStringValue("queryFilter", filter);
         header.addIntegerValue("maxCount", maxCount);
+
         if (totalRecordCount == -1) {
             if (!serialToVal.equals(MINUS_ONE)) {
                 totalRecordCount = toCurIndex - curIndex + 1;
-                com.netscape.certsrv.apps.CMS.debug("totalRecordCount=" + totalRecordCount);
+                CMS.debug("ListCerts: totalRecordCount: " + totalRecordCount);
             } else {
                 totalRecordCount = list.getSize() -
                         list.getCurrentIndex();
-                com.netscape.certsrv.apps.CMS.debug("totalRecordCount=" + totalRecordCount);
+                CMS.debug("ListCerts: totalRecordCount: " + totalRecordCount);
             }
         }
 
+        int currentRecordCount = list.getSize() - list.getCurrentIndex();
+        CMS.debug("ListCerts: totalRecordCount: " + totalRecordCount);
+        CMS.debug("ListCerts: currentRecordCount: " + currentRecordCount);
+
         header.addIntegerValue("totalRecordCount", totalRecordCount);
-        header.addIntegerValue("currentRecordCount", list.getSize() -
-                list.getCurrentIndex());
+        header.addIntegerValue("currentRecordCount", currentRecordCount);
 
         String qs = "";
         if (mReverse)
             qs = "querySentinelUp";
         else
             qs = "querySentinelDown";
+        CMS.debug("ListCerts: qs: " + qs);
 
         if (mHardJumpTo) {
-            com.netscape.certsrv.apps.CMS.debug("curSerial added to querySentinelUp:" + curSerial.toString());
-
+            CMS.debug("ListCerts: curSerial added to querySentinelUp: " + curSerial);
             header.addStringValue("querySentinelUp", curSerial.toString());
+
         } else {
             if (nextRec == null) {
                 header.addStringValue(qs, null);
-                com.netscape.certsrv.apps.CMS.debug("nextRec is null");
-                if (mReverse) {
-                    com.netscape.certsrv.apps.CMS.debug("curSerial added to querySentinelUp:" + curSerial.toString());
+                CMS.debug("ListCerts: nextRec is null");
 
+                if (mReverse) {
+                    CMS.debug("ListCerts: curSerial added to querySentinelUp: " + curSerial);
                     header.addStringValue("querySentinelUp", curSerial.toString());
                 }
+
             } else {
                 BigInteger nextRecNo = nextRec.getSerialNumber();
+                CMS.debug("ListCerts: nextRecNo: " + nextRecNo);
 
                 if (serialToVal.equals(MINUS_ONE)) {
                     header.addStringValue(
@@ -570,7 +614,7 @@ public class ListCerts extends CMSServlet {
                                 null);
                     }
                 }
-                com.netscape.certsrv.apps.CMS.debug("querySentinel " + qs + " = " + nextRecNo.toString());
+                CMS.debug("ListCerts: querySentinel " + qs + ": " + nextRecNo);
             }
         } // !mHardJumpto
 
diff --git a/base/server/cmscore/src/com/netscape/cmscore/dbs/CertificateRepository.java b/base/server/cmscore/src/com/netscape/cmscore/dbs/CertificateRepository.java
index 7d626b93d..8d9626521 100644
--- a/base/server/cmscore/src/com/netscape/cmscore/dbs/CertificateRepository.java
+++ b/base/server/cmscore/src/com/netscape/cmscore/dbs/CertificateRepository.java
@@ -24,8 +24,8 @@ import java.util.Arrays;
 import java.util.Date;
 import java.util.Enumeration;
 import java.util.Hashtable;
-import java.util.Vector;
 import java.util.Random;
+import java.util.Vector;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
@@ -42,9 +42,9 @@ import netscape.security.x509.X509CertInfo;
 
 import com.netscape.certsrv.apps.CMS;
 import com.netscape.certsrv.base.EBaseException;
+import com.netscape.certsrv.base.IConfigStore;
 import com.netscape.certsrv.base.MetaInfo;
 import com.netscape.certsrv.base.SessionContext;
-import com.netscape.certsrv.base.IConfigStore;
 import com.netscape.certsrv.ca.ICRLIssuingPoint;
 import com.netscape.certsrv.dbs.EDBException;
 import com.netscape.certsrv.dbs.EDBRecordNotFoundException;
@@ -192,7 +192,7 @@ public class CertificateRepository extends Repository
         randomNumber = (randomNumber.multiply(mRangeSize)).shiftRight(mBitLength);
         CMS.debug("CertificateRepository: getRandomNumber  randomNumber="+randomNumber);
 
-        return randomNumber; 
+        return randomNumber;
     }
 
     private BigInteger getRandomSerialNumber(BigInteger randomNumber) throws EBaseException {
@@ -201,7 +201,7 @@ public class CertificateRepository extends Repository
         nextSerialNumber = randomNumber.add(mMinSerialNo);
         CMS.debug("CertificateRepository: getRandomSerialNumber  nextSerialNumber="+nextSerialNumber);
 
-        return nextSerialNumber; 
+        return nextSerialNumber;
     }
 
     private BigInteger checkSerialNumbers(BigInteger randomNumber, BigInteger serialNumber) throws EBaseException {
@@ -247,7 +247,7 @@ public class CertificateRepository extends Repository
             }
         } while (nextSerialNumber == null && i < n);
 
-        return nextSerialNumber; 
+        return nextSerialNumber;
     }
 
     private Object nextSerialNumberMonitor = new Object();
@@ -295,7 +295,7 @@ public class CertificateRepository extends Repository
             }
         }
 
-        return nextSerialNumber; 
+        return nextSerialNumber;
     }
 
     private void updateCounter() {
@@ -363,7 +363,7 @@ public class CertificateRepository extends Repository
         }
         CMS.debug("CertificateRepository: getInRangeCount  count=" + count);
 
-        return count; 
+        return count;
     }
 
     private BigInteger getInRangeCounter(BigInteger  minSerialNo, BigInteger maxSerialNo)
@@ -412,7 +412,7 @@ public class CertificateRepository extends Repository
         }
         CMS.debug("CertificateRepository: getInRangeCounter:  counter=" + counter);
 
-        return counter; 
+        return counter;
     }
 
     public BigInteger getLastSerialNumberInRange(BigInteger serial_low_bound, BigInteger serial_upper_bound)
@@ -455,7 +455,7 @@ public class CertificateRepository extends Repository
                 mEnableRandomSerialNumbers = !mEnableRandomSerialNumbers;
                 mDBConfig.putBoolean(PROP_ENABLE_RANDOM_SERIAL_NUMBERS, mEnableRandomSerialNumbers);
             }
-        }  
+        }
         if (mEnableRandomSerialNumbers && mCounter == null) {
             mCounter = getInRangeCounter(serial_low_bound, serial_upper_bound);
         } else {
@@ -484,7 +484,7 @@ public class CertificateRepository extends Repository
 
             BigInteger ret = new BigInteger(serial_low_bound.toString(10));
 
-            ret = ret.subtract(BigInteger.ONE); 
+            ret = ret.subtract(BigInteger.ONE);
             CMS.debug("CertificateRepository:getLastCertRecordSerialNo: returning " + ret);
             return ret;
         }
@@ -523,7 +523,7 @@ public class CertificateRepository extends Repository
 
         BigInteger ret = new BigInteger(serial_low_bound.toString(10));
 
-        ret = ret.subtract(BigInteger.ONE); 
+        ret = ret.subtract(BigInteger.ONE);
 
         CMS.debug("CertificateRepository:getLastCertRecordSerialNo: returning " + ret);
         if (modeChange && mEnableRandomSerialNumbers) {
@@ -1027,7 +1027,7 @@ public class CertificateRepository extends Repository
         } catch (Exception e) {
             throw new EBaseException(e.getMessage());
         } finally {
-            if (s != null) 
+            if (s != null)
                 s.close();
         }
         return exists;
@@ -1278,21 +1278,25 @@ public class CertificateRepository extends Repository
     public ICertRecordList findCertRecordsInList(String filter,
             String attrs[], String sortKey, int pageSize)
             throws EBaseException {
-        IDBSSession s = mDBService.createSession();
 
-        CMS.debug("In findCertRecordsInList");
-        CertRecordList list = null;
+        CMS.debug("CertificateRepository.findCertRecordsInList()");
+
+        IDBSSession session = mDBService.createSession();
 
         try {
-            IDBVirtualList<ICertRecord> vlist = s.<ICertRecord>createVirtualList(getDN(), filter, attrs,
-                    sortKey, pageSize);
+            IDBVirtualList<ICertRecord> list = session.<ICertRecord>createVirtualList(
+                    getDN(),
+                    filter,
+                    attrs,
+                    sortKey,
+                    pageSize);
+
+            return new CertRecordList(list);
 
-            list = new CertRecordList(vlist);
         } finally {
-            if (s != null)
-                s.close();
+            if (session != null)
+                session.close();
         }
-        return list;
     }
 
     public ICertRecordList findCertRecordsInList(String filter,
diff --git a/base/server/cmscore/src/com/netscape/cmscore/dbs/DBVirtualList.java b/base/server/cmscore/src/com/netscape/cmscore/dbs/DBVirtualList.java
index 51a9230ac..fde67c663 100644
--- a/base/server/cmscore/src/com/netscape/cmscore/dbs/DBVirtualList.java
+++ b/base/server/cmscore/src/com/netscape/cmscore/dbs/DBVirtualList.java
@@ -343,7 +343,11 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
      * since you'd better check if the index is out of bound first.
      */
     public int getSize() {
+
+        CMS.debug("DBVirtualList.getSize()");
+
         if (!mInitialized) {
+
             mInitialized = true;
             // Do an initial search to get the virtual list size
             // Keep one page before and one page after the start
@@ -361,10 +365,13 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
             LDAPVirtualListControl cont = null;
 
             if (mJumpTo == null) {
+                CMS.debug("DBVirtualList: searching for entry A");
                 cont = new LDAPVirtualListControl("A",
                             mBeforeCount,
                             mAfterCount);
+
             } else {
+                CMS.debug("DBVirtualList: searching for entry " + mJumpTo);
 
                 if (mPageSize < 0) {
                     mBeforeCount = mPageSize * -1;
@@ -374,11 +381,12 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
                             mBeforeCount,
                             mAfterCount);
             }
+
             mPageControls[1] = cont;
             getJumpToPage();
         }
 
-        CMS.debug("Getting Virtual List size: " + mSize);
+        CMS.debug("DBVirtualList: size: " + mSize);
         return mSize;
     }
 
@@ -412,6 +420,9 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
     }
 
     private synchronized boolean getEntries() {
+
+        CMS.debug("DBVirtualList.getEntries()");
+
         // Specify necessary controls for vlist
         // LDAPSearchConstraints cons = mConn.getSearchConstraints();
         LDAPSearchConstraints cons = new LDAPSearchConstraints();
@@ -501,7 +512,8 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
         //System.out.println( "Returning " + mEntries.size() +
         //       " entries" );
 
-        CMS.debug("getEntries returning " + mEntries.size());
+        CMS.debug("DBVirtualList: entries: " + mEntries.size());
+
         return true;
     }
 
@@ -532,7 +544,7 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
                 mSelectedIndex = nextCont.getFirstPosition() - 1;
                 mTop = Math.max(0, mSelectedIndex - mBeforeCount);
 
-                CMS.debug("mTop " + mTop);
+                CMS.debug("DBVirtualList: top: " + mTop);
                 if (mJumpTo != null) {
                     mJumpToInitialIndex = mTop;
                 }
@@ -564,7 +576,9 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
      * @param first the index of the first entry of the page you want to fetch
      */
     public boolean getPage(int first) {
-        CMS.debug("getPage " + first);
+
+        CMS.debug("DBVirtualList.getPage(" + first + ")");
+
         if (!mInitialized) {
             LDAPVirtualListControl cont = new LDAPVirtualListControl(0,
                     mBeforeCount,
@@ -658,14 +672,15 @@ public class DBVirtualList<E> implements IDBVirtualList<E> {
          * the caller should really check the index is within bound before this
          * but I'll take care of this just in case they are too irresponsible
          */
-        if (!mInitialized)
+        if (!mInitialized) {
             mSize = getSize();
+        }
 
-        CMS.debug("getElementAt: " + index + " mTop " + mTop);
+        CMS.debug("DBVirtualList: retrieving entry #" + index);
 
         //System.out.println( "need entry " + index );
         if ((index < 0) || (index >= mSize)) {
-            CMS.debug("returning null");
+            CMS.debug("DBVirtualList: returning null");
             return null;
         }