Improve caching refresh operation

4 files changed, 34 insertions, 6 deletions

diff --git a/kittystore/caching/email.py b/kittystore/caching/email.py
index 89440d3..527d1e3 100644
--- a/kittystore/caching/email.py
+++ b/kittystore/caching/email.py
@@ -53,7 +53,9 @@ class MailmanUser(CachedValue):
         # XXX: Storm-specific
         from kittystore.storm.model import Email
         try:
-            for message in store.db.find(Email, Email.user_id == None):
+            for num, message in enumerate(store.db.find(Email, Email.user_id == None)):
                 message.user_id = self._get_user_id(store, message)
+                if num % 1000 == 0:
+                    store.commit() # otherwise we'll blow up the memory
         except (HTTPError, mailmanclient.MailmanConnectionError):
             return # Can't refresh at this time
diff --git a/kittystore/caching/mlist.py b/kittystore/caching/mlist.py
index fae0df3..16e993d 100644
--- a/kittystore/caching/mlist.py
+++ b/kittystore/caching/mlist.py
@@ -8,7 +8,6 @@ from urllib2 import HTTPError
 
 import mailmanclient
 from dateutil.parser import parse as date_parse
-from mailman.interfaces.mailinglist import IMailingList
 from mailman.interfaces.archiver import ArchivePolicy
 
 from kittystore.caching import CachedValue
@@ -30,7 +29,7 @@ class CompatibleMList(object):
             except AttributeError:
                 value = mlist.settings[prop]
             if prop in self.converters:
-                value = converters[prop](value)
+                value = self.converters[prop](value)
             setattr(self, prop, value)
 
 
@@ -38,8 +37,7 @@ class ListProperties(CachedValue):
 
     def on_new_message(self, store, mlist, message):
         l = store.get_list(mlist.fqdn_listname)
-        if not IMailingList.providedBy(mlist):
-            # this is probably a List instance returned by mailmanclient
+        if isinstance(mlist, mailmanclient._client._List):
             mlist = CompatibleMList(mlist, l.mailman_props)
         for propname in l.mailman_props:
             setattr(l, propname, getattr(mlist, propname))
diff --git a/kittystore/caching/thread.py b/kittystore/caching/thread.py
index 1ec5600..62edb5d 100644
--- a/kittystore/caching/thread.py
+++ b/kittystore/caching/thread.py
@@ -20,10 +20,12 @@ class ThreadStats(CachedValue):
     def refresh(self, store):
         # XXX: Storm-specific
         from kittystore.storm.model import Thread
-        for thread in store.db.find(Thread):
+        for num, thread in enumerate(store.db.find(Thread)):
             thread.emails_count = None # reset it
             len(thread) # this will refill the cached value
             thread.participants_count = len(thread.participants)
+            if num % 1000 == 0:
+                store.commit() # otherwise we'll blow up the memory
 
 
 class ThreadSubject(CachedValue):
diff --git a/kittystore/storm/schema/patch_15.py b/kittystore/storm/schema/patch_15.py
new file mode 100644
index 0000000..c9d3f6b
--- /dev/null
+++ b/kittystore/storm/schema/patch_15.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+from .utils import get_db_type
+
+
+SQL = {
+    "sqlite": [
+        'CREATE INDEX "ix_thread_subject" ON "thread" (subject);',
+        ],
+    "postgres": [
+        'CREATE INDEX "ix_thread_subject" ON "thread" (subject);',
+        ],
+    "mysql": [
+        'CREATE INDEX `ix_thread_subject` ON `thread` (subject);',
+        ],
+    }
+
+
+def apply(store):
+    """Add indexes on thread.subject"""
+    dbtype = get_db_type(store)
+    for statement in SQL[dbtype]:
+        store.execute(statement)
+    store.commit()