Bugzilla Bug 574942 - TPS database has performance problems with a large number of tokens

git-svn-id: svn+ssh://svn.fedorahosted.org/svn/pki/trunk@1317 c9f7a03b-bd48-0410-a16d-cbbf54688b0b

6 files changed, 256 insertions, 6 deletions

diff --git a/pki/base/tps/doc/CS.cfg b/pki/base/tps/doc/CS.cfg
index d9422c3f1..949f5cc69 100644
--- a/pki/base/tps/doc/CS.cfg
+++ b/pki/base/tps/doc/CS.cfg
@@ -339,6 +339,10 @@ applet.so_pin=000000000000
 applet.delete_old=true
 general.verifyProof=1
 general.applet_ext=ijc
+general.search.sizelimit.max=2000
+general.search.sizelimit.default=100
+general.search.timelimit.max=10
+general.search.timelimit.default=10
 channel._000=#########################################
 channel._001=# channel.encryption:
 channel._002=#
diff --git a/pki/base/tps/lib/perl/PKI/TPS/DatabasePanel.pm b/pki/base/tps/lib/perl/PKI/TPS/DatabasePanel.pm
index c4fc7a8f8..2e3a0bb47 100755
--- a/pki/base/tps/lib/perl/PKI/TPS/DatabasePanel.pm
+++ b/pki/base/tps/lib/perl/PKI/TPS/DatabasePanel.pm
@@ -174,7 +174,7 @@ sub update
 
     # add VLV indexes
     $tmp = "/tmp/addVLVIndexes-$$.ldif";
-    system("sed -e 's/userRoot/$database/g' " .
+    system("sed -e 's/userRoot/$database/g;s/\$TOKENDB_ROOT/$basedn/g' " .
               "/usr/share/$flavor/tps/scripts/addVLVIndexes.ldif > $tmp");
     system("$mozldap_path/ldapmodify -h '$host' -p '$port' -D '$binddn' " .
               "-w '$bindpwd' -a " .
diff --git a/pki/base/tps/scripts/addVLVIndexes.ldif b/pki/base/tps/scripts/addVLVIndexes.ldif
index e54688c17..9dc86ece1 100644
--- a/pki/base/tps/scripts/addVLVIndexes.ldif
+++ b/pki/base/tps/scripts/addVLVIndexes.ldif
@@ -22,7 +22,7 @@ dn: cn=tus-listTokens-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
 cn: tus-listtokens-vlv
 objectClass: top
 objectClass: vlvsearch
-vlvBase: ou=Tokens,{rootSuffix}
+vlvBase: ou=Tokens,$TOKENDB_ROOT
 vlvFilter: (&(cn=*)(tokenUserID=*))
 vlvScope: 2
 
@@ -30,8 +30,8 @@ dn: cn=tus-listActivities-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
 cn: tus-listActivities-vlv
 objectClass: top
 objectClass: vlvsearch
-vlvBase: ou=Activities,{rootSuffix}
-vlvFilter: (tokenID=*)
+vlvBase: ou=Activities,$TOKENDB_ROOT
+vlvFilter: (&(tokenID=*)(tokenUserID=*))
 vlvScope: 2
 
 dn: cn=listTokensIndex,cn=tus-listTokens-vlv,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
diff --git a/pki/base/tps/src/include/tus/tus_db.h b/pki/base/tps/src/include/tus/tus_db.h
index 441230b98..ac183c4d9 100644
--- a/pki/base/tps/src/include/tus/tus_db.h
+++ b/pki/base/tps/src/include/tus/tus_db.h
@@ -169,6 +169,7 @@ TPS_PUBLIC int add_default_tus_db_entry (const char *uid, const char *agentid, c
 TPS_PUBLIC int delete_tus_db_entry (char *userid, char *cn);
 TPS_PUBLIC int find_tus_db_entry (char *cn, int max, LDAPMessage **result);
 TPS_PUBLIC int find_tus_db_entries (const char *filter, int max, LDAPMessage **result);
+TPS_PUBLIC int find_tus_db_entries_pcontrol_1 (const char *filter, int max, int time_limit, int size_limit, LDAPMessage **result);
 TPS_PUBLIC int find_tus_token_entries (char *filter, int max, LDAPMessage **result, int order);
 TPS_PUBLIC int find_tus_token_entries_no_vlv (char *filter, LDAPMessage **result, int order);
 TPS_PUBLIC int tus_has_active_tokens(char *userid);
@@ -190,6 +191,7 @@ TPS_PUBLIC int add_certificate (char *tokenid, char *origin, char *tokenType, ch
 TPS_PUBLIC int add_tus_db_entry (char *cn, LDAPMod **mods);
 TPS_PUBLIC int add_new_tus_db_entry (const char *userid, char *cn, const char *uid, int flag, const char *status, char *applet_version, char *key_info, const char *token_type);
 TPS_PUBLIC int find_tus_activity_entries (char *filter, int max, LDAPMessage **result);
+TPS_PUBLIC int find_tus_activity_entries_pcontrol_1 (char *filter, int max, int time_limit, int size_limit, LDAPMessage **result);
 TPS_PUBLIC int find_tus_activity_entries_no_vlv (char *filter, LDAPMessage **result, int order);
 TPS_PUBLIC int get_number_of_entries (LDAPMessage *result);
 TPS_PUBLIC int free_results (LDAPMessage *results);
diff --git a/pki/base/tps/src/modules/tokendb/mod_tokendb.cpp b/pki/base/tps/src/modules/tokendb/mod_tokendb.cpp
index 39db8ab5d..28c56bd33 100644
--- a/pki/base/tps/src/modules/tokendb/mod_tokendb.cpp
+++ b/pki/base/tps/src/modules/tokendb/mod_tokendb.cpp
@@ -155,6 +155,32 @@ extern TOKENDB_PUBLIC char *nss_var_lookup( apr_pool_t *p, server_rec *s,
         } \
     }
 
+#define get_cfg_int(cname, vname) \
+    if( ( s = PL_strstr( buf, cname ) ) != NULL ) { \
+        s += PL_strlen( cname ); \
+        v = s; \
+        while( *s != '\x0D' && *s != '\x0A' && *s != '\0' && \
+               ( PRUint32 ) ( s - buf ) < size ) { \
+            s++; \
+        } \
+        n = s - v; \
+        s = PL_strndup( v, n ); \
+        if( s != NULL ) { \
+            char *endptr = NULL; \
+            errno = 0; \
+            vname = strtol(s, &endptr, 10);\
+            if ((errno == ERANGE && (vname == LONG_MAX || vname == LONG_MIN)) \
+              || (endptr == s)) { \
+                vname=0; \
+            } \
+            do_free(s); \
+        } else { \
+            do_free(buf); \
+            do_free(s); \
+            return 0; \
+        } \
+    }
+
 /**
  * Provide reasonable defaults for some defines.
  */
@@ -224,6 +250,10 @@ static char *editConfigTemplate              = NULL;
 static char *confirmConfigChangesTemplate    = NULL;
 static char *addConfigTemplate               = NULL;
 static char *confirmDeleteConfigTemplate     = NULL;
+static int maxSizeLimit                      = 0;
+static int defaultSizeLimit                  = 0;
+static int maxTimeLimit                      = 0;
+static int defaultTimeLimit                  = 0;
 
 static char *profileList                     = NULL;
 static char *transitionList                  = NULL;
@@ -2334,6 +2364,10 @@ int get_tus_config( char *name )
     get_cfg_string("tokendb.addConfigTemplate=", addConfigTemplate);
     get_cfg_string("tokendb.confirmDeleteConfigTemplate=", confirmDeleteConfigTemplate);
     get_cfg_string("target.Profiles.list=", profileList);
+    get_cfg_int("general.search.sizelimit.max=", maxSizeLimit);
+    get_cfg_int("general.search.sizelimit.default=", defaultSizeLimit);
+    get_cfg_int("general.search.timelimit.max=", maxTimeLimit);
+    get_cfg_int("general.search.timelimit.min=", defaultTimeLimit);
 
     if( buf != NULL ) {
         PR_Free( buf );
@@ -3213,6 +3247,40 @@ void parse_and_apply_changes(char* userid, char* ptype, char* pname, char *opera
     do_strfree(fixed_pattern);
 }
 
+static int get_time_limit(char *query)
+{
+  char *val = NULL;
+  int ret;
+
+  val  = get_field(query, "timeLimit=", SHORT_LEN);
+  if (val == NULL) {
+      return maxTimeLimit;
+  } 
+
+  ret = atoi(val);
+  if ((ret == 0) || (maxTimeLimit == NULL) || (ret > maxTimeLimit)) {
+      return maxTimeLimit;
+  } 
+  return ret;
+}
+
+static int get_size_limit(char *query)
+{
+  char *val = NULL;
+  int ret;
+
+  val  = get_field(query, "sizeLimit=", SHORT_LEN);
+  if (val == NULL) {
+      return maxSizeLimit;
+  }
+
+  ret = atoi(val);
+  if ((ret == 0) || (maxSizeLimit == NULL) || (ret > maxSizeLimit)) {
+      return maxSizeLimit;
+  }
+  return ret;
+}
+
 /**
  * mod_tokendb_handler handles the protocol between the tokendb and the RA
  */
@@ -5779,6 +5847,9 @@ mod_tokendb_handler( request_rec *rq )
         char *complete_filter = add_profile_filter(filter, auth_filter);
         do_free(auth_filter);
 
+        int time_limit = get_time_limit(query);
+        int size_limit = get_size_limit(query);
+
         tokendbDebug( "looking for filter:" );
         tokendbDebug( complete_filter );
         tokendbDebug( filter );
@@ -5805,7 +5876,12 @@ mod_tokendb_handler( request_rec *rq )
             status = find_tus_activity_entries_no_vlv( complete_filter, &result, 1 ); 
         } else if (( PL_strstr( query, "op=view_activity_admin" )) ||
             ( PL_strstr( query, "op=view_activity" ) )) {
-            status = find_tus_activity_entries( complete_filter, maxReturns, &result);
+            if (PL_strcmp(complete_filter, "(&(tokenID=*)(tokenUserID=*))") == 0) {
+                tokendbDebug("activity vlv search");
+                status = find_tus_activity_entries(complete_filter, maxReturns, &result);
+            } else {
+                status = find_tus_activity_entries_pcontrol_1( complete_filter, maxReturns, time_limit, size_limit, &result);
+            }
         } else if(( PL_strstr( query, "op=view_certificate_all" ) ) ||
             ( PL_strstr( query, "op=show_certificate") )) {
 
@@ -5837,7 +5913,12 @@ mod_tokendb_handler( request_rec *rq )
                    (PL_strstr (query, "op=edit_user" )))  {
             status = find_tus_user_entries_no_vlv( filter, &result, 0); 
         } else {
-            status = find_tus_db_entries( complete_filter, maxReturns, &result );
+            if (PL_strcmp(complete_filter, "(&(cn=*)(tokenUserID=*))") == 0) {
+                tokendbDebug("token vlv search");
+                status = find_tus_db_entries(complete_filter, maxReturns, &result);
+            } else {
+                status = find_tus_db_entries_pcontrol_1( complete_filter, maxReturns, time_limit, size_limit, &result );
+            }
         }
 
         if( status != LDAP_SUCCESS ) {
diff --git a/pki/base/tps/src/tus/tus_db.c b/pki/base/tps/src/tus/tus_db.c
index 3fd02a1ad..9ada4ffbf 100644
--- a/pki/base/tps/src/tus/tus_db.c
+++ b/pki/base/tps/src/tus/tus_db.c
@@ -43,6 +43,7 @@ extern "C"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <assert.h>
 
 #include "plstr.h"
 #include "prmem.h"
@@ -2613,6 +2614,117 @@ TPS_PUBLIC int find_tus_db_entries (const char *filter, int max, LDAPMessage **r
     return rc;
 }
 
+/* all of the paging functions below are adapted from openldap
+ * When we move to openldap, they can be removed, and the openldap 
+ * equivalent of the top level function can ldap_create_page_control
+ * can be used instead */
+
+#define LDAP_CONTROL_PAGE_OID  "1.2.840.113556.1.4.319"
+
+int ldap_create_page_control_cs(LDAP * ld,
+                          unsigned long pagesize,
+                          struct berval *cookiep,
+                          int iscritical, LDAPControl ** ctrlp) {
+    BerElement *ber;
+    int i, rc;
+
+    if ( (ld == NULL) || (ctrlp == NULL)) {
+        return LDAP_PARAM_ERROR;
+    }
+
+    /* create a ber package to hold the controlValue */
+    if ( ( nsldapi_alloc_ber_with_options( ld, &ber ) ) != LDAP_SUCCESS ) {
+        ldap_set_lderrno( ld, LDAP_NO_MEMORY, NULL, NULL );
+        return LDAP_NO_MEMORY;
+    }
+
+    /* encode the start of the sequence of sequences into the ber */
+    if ( ber_printf( ber, "{" ) == -1 ) {
+        goto encoding_error_exit;
+    }
+
+    if ( ber_printf( ber, "i", pagesize) == -1 ) {
+        goto encoding_error_exit;
+    }
+
+    if (cookiep == NULL) {
+        if (ber_printf(ber, "o", "", 0) == -1) {
+            goto encoding_error_exit;
+        }
+    } else {
+        if (ber_printf (ber, "O", cookiep) == -1) {
+            goto encoding_error_exit;
+        }
+    }
+
+    if (ber_printf (ber,  "n}") == -1) {
+        goto encoding_error_exit;
+    }
+
+    rc = nsldapi_build_control( LDAP_CONTROL_PAGE_OID, ber, 1,
+            iscritical, ctrlp );
+
+    ldap_set_lderrno( ld, rc, NULL, NULL );
+    return rc ;
+
+encoding_error_exit:
+    ldap_set_lderrno( ld, LDAP_ENCODING_ERROR, NULL, NULL );
+    ber_free( ber, 1 );
+    return LDAP_ENCODING_ERROR;
+}
+
+TPS_PUBLIC int find_tus_db_entries_pcontrol_1(const char *filter, int max, int time_limit, int size_limit, LDAPMessage **result)
+{
+    int  rc = LDAP_OTHER, tries = 0;
+
+    LDAPsortkey **sortKeyList;
+    LDAPControl *controls[3];
+    struct berval  *cookie=NULL;
+    struct timeval timeout;
+
+    timeout.tv_sec = time_limit;
+    timeout.tv_usec = 0;
+
+    tus_check_conn();
+    controls[0] = NULL;
+    controls[1] = NULL;
+    controls[2] = NULL;
+
+    rc = ldap_create_page_control_cs(ld, max, cookie, 0, &controls[0]);
+
+    ldap_create_sort_keylist(&sortKeyList, "-dateOfModify");
+    ldap_create_sort_control(ld, sortKeyList, 1 /* non-critical */,
+        &controls[1]);
+
+    for (tries = 0; tries < MAX_RETRIES; tries++) {
+        rc = ldap_search_ext_s (ld, baseDN, LDAP_SCOPE_SUBTREE, filter,
+                 NULL, 0, controls, NULL, 
+                 time_limit >0 ? &timeout : NULL, 
+                 size_limit, result);
+        if ((rc == LDAP_SUCCESS) || (rc == LDAP_PARTIAL_RESULTS)) {
+            break;
+        } else if (rc == LDAP_SERVER_DOWN || rc == LDAP_CONNECT_ERROR) {
+            rc = ldap_simple_bind_s (ld, bindDN, bindPass);
+            if (rc != LDAP_SUCCESS) {
+                bindStatus = rc;
+                break;
+            }
+        }
+    }
+
+    if (cookie != NULL) {
+        ber_bvfree(cookie);
+        cookie = NULL;
+    }
+
+    ldap_free_sort_keylist(sortKeyList);
+    
+    ldap_control_free(controls[0]);
+    ldap_control_free(controls[1]);
+
+    return rc;
+}
+
 static int sort_cmp(const char *v1, const char *v2)
 {
   return PL_strcasecmp(v1, v2);
@@ -3193,6 +3305,57 @@ int find_tus_activity_entries (char *filter, int max, LDAPMessage **result)
     return rc;
 }
 
+TPS_PUBLIC int find_tus_activity_entries_pcontrol_1(char *filter, int max, int time_limit, int size_limit, LDAPMessage **result)
+{
+    int  rc = LDAP_OTHER, tries = 0;
+    LDAPsortkey **sortKeyList;
+    LDAPControl *controls[3];
+    struct berval *cookie=NULL;
+    struct timeval timeout;
+
+    timeout.tv_sec = time_limit;
+    timeout.tv_usec = 0;
+
+    tus_check_conn();
+    controls[0] = NULL;
+    controls[1] = NULL;
+    controls[2] = NULL;
+
+    rc = ldap_create_page_control_cs(ld, max, cookie, 0, &controls[0]);
+
+    ldap_create_sort_keylist(&sortKeyList, "-dateOfCreate");
+    ldap_create_sort_control(ld, sortKeyList, 1 /* non-critical */,
+        &controls[1]);
+
+    for (tries = 0; tries < MAX_RETRIES; tries++) {
+        rc = ldap_search_ext_s (ld, activityBaseDN, LDAP_SCOPE_SUBTREE, filter,
+                 NULL, 0, controls, NULL,
+                 time_limit >0 ? &timeout : NULL,
+                 size_limit, result);
+        if ((rc == LDAP_SUCCESS) || (rc == LDAP_PARTIAL_RESULTS)) {
+            break;
+        } else if (rc == LDAP_SERVER_DOWN || rc == LDAP_CONNECT_ERROR) {
+            rc = ldap_simple_bind_s (ld, bindDN, bindPass);
+            if (rc != LDAP_SUCCESS) {
+                bindStatus = rc;
+                break;
+            }
+        }
+    }
+
+    if (cookie != NULL) {
+       ber_bvfree(cookie);
+       cookie = NULL;
+    }
+
+    ldap_free_sort_keylist(sortKeyList);
+
+    ldap_control_free(controls[0]);
+    ldap_control_free(controls[1]);
+
+    return rc;
+}
+
 int get_number_of_entries (LDAPMessage *result)
 {
     int  n = 0, rc = 0, tries = 0;