Speed up page loading on long threads

Speed up page loading on threads with many replies by loading the
replies asynchronously.

Preserve search engine indexation by detecting bots and serving the
usual page in one go.

12 files changed, 190 insertions, 66 deletions

diff --git a/hyperkitty.spec b/hyperkitty.spec
index fc42e81..3b34b52 100644
--- a/hyperkitty.spec
+++ b/hyperkitty.spec
@@ -30,6 +30,7 @@ BuildRequires:  django-assets
 BuildRequires:  python-rjsmin
 BuildRequires:  python-cssmin
 BuildRequires:  python-mailman-client
+BuildRequires:  python-robot-detection
 %if 0%{fedora} && 0%{fedora} < 18
 BuildRequires:  Django
 BuildRequires:  Django-south
@@ -48,6 +49,7 @@ Requires:       django-assets
 Requires:       python-rjsmin
 Requires:       python-cssmin
 Requires:       python-mailman-client
+Requires:       python-robot-detection
 %if 0%{fedora} && 0%{fedora} < 18
 Requires:       Django >= 1.4
 Requires:       Django-south
diff --git a/hyperkitty/static/css/hyperkitty-common.css b/hyperkitty/static/css/hyperkitty-common.css
index 0a3e114..4309136 100644
--- a/hyperkitty/static/css/hyperkitty-common.css
+++ b/hyperkitty/static/css/hyperkitty-common.css
@@ -264,3 +264,10 @@ a.thread-new strong {
 .new-thread-form textarea {
     width: 90%;
 }
+
+
+/* AJAX */
+.ajaxloader {
+    display: block;
+    margin: 1em auto;
+}
diff --git a/hyperkitty/static/css/hyperkitty-message.css b/hyperkitty/static/css/hyperkitty-message.css
index 0870277..edc5cab 100644
--- a/hyperkitty/static/css/hyperkitty-message.css
+++ b/hyperkitty/static/css/hyperkitty-message.css
@@ -144,7 +144,7 @@
     margin: 1em 0;
 }
 
-#participants img {
+#participants img.gravatar {
     width: 20px;
     vertical-align: middle;
 }
diff --git a/hyperkitty/static/img/ajax-loader.gif b/hyperkitty/static/img/ajax-loader.gif
new file mode 100644
index 0000000..49b6d85
--- /dev/null
+++ b/hyperkitty/static/img/ajax-loader.gif
diff --git a/hyperkitty/static/js/hyperkitty.js b/hyperkitty/static/js/hyperkitty.js
index 79cec43..ddebd09 100644
--- a/hyperkitty/static/js/hyperkitty.js
+++ b/hyperkitty/static/js/hyperkitty.js
@@ -147,8 +147,43 @@ function setup_favorites() {
  * Replies
  */
 
-function setup_replies() {
-    $("a.reply").click(function(e) {
+function setup_emails_list(baseElem) {
+    if (!baseElem) {
+        baseElem = document;
+    }
+    // Attachements
+    $(baseElem).find(".email-info .attachments a.attachments").each(function() {
+        var att_list = $(this).next("ul.attachments-list");
+        var pos = $(this).position();
+        att_list.css("left", pos.left);
+        $(this).click(function() {
+            att_list.slideToggle('fast');
+        });
+    });
+    // Quotes
+    $(baseElem).find('div.email-body .quoted-switch a')
+        .click(function(e) {
+            e.preventDefault();
+            $(this).parent().next(".quoted-text").slideToggle('fast');
+        });
+    setup_replies(baseElem);
+}
+
+function fold_quotes(baseElem) {
+    $(baseElem).find('div.email-body .quoted-text').each(function() {
+        var linescount = $(this).text().split("\n").length;
+        if (linescount > 3) {
+            // hide if the quote is more than 3 lines long
+            $(this).hide();
+        }
+    });
+}
+
+function setup_replies(baseElem) {
+    if (!baseElem) {
+        baseElem = document;
+    }
+    $(baseElem).find("a.reply").click(function(e) {
         e.preventDefault();
         if (!$(this).hasClass("disabled")) {
             $(this).next().slideToggle("fast", function() {
@@ -158,7 +193,7 @@ function setup_replies() {
             });
         }
     });
-    $(".reply-form button[type='submit']").click(function(e) {
+    $(baseElem).find(".reply-form button[type='submit']").click(function(e) {
         e.preventDefault();
         var form = $(this).parents("form").first();
         // remove previous error messages
@@ -186,11 +221,11 @@ function setup_replies() {
             }
         });
     });
-    $(".reply-form a.cancel").click(function(e) {
+    $(baseElem).find(".reply-form a.cancel").click(function(e) {
         e.preventDefault();
         $(this).parents(".reply-form").first().slideUp();
     });
-    $(".reply-form a.quote").click(function(e) {
+    $(baseElem).find(".reply-form a.quote").click(function(e) {
         e.preventDefault();
         var quoted = $(this).parents(".email").first()
                         .find(".email-body").clone()
@@ -220,7 +255,7 @@ function setup_replies() {
             this_form.find("textarea").focus();
         }
     }
-    $(".reply-form input[name='newthread']").change(function() {
+    $(baseElem).find(".reply-form input[name='newthread']").change(function() {
         set_new_thread($(this));
     }).change();
 }
@@ -312,28 +347,36 @@ function activity_graph(elem_id, dates, counts, baseurl) {
       .text("Messages");
 }
 
+
 /*
- * Misc.
+ * Thread replies list
  */
-
-function setup_attachments() {
-    $(".email-info .attachments a.attachments").each(function() {
-        var att_list = $(this).next("ul.attachments-list");
-        var pos = $(this).position();
-        att_list.css("left", pos.left);
-        $(this).click(function() {
-            att_list.slideToggle('fast');
-        });
+function update_thread_replies(url) {
+    $.ajax({
+        dataType: "json",
+        url: url,
+        success: function(data) {
+            // replies
+            var newcontent = $(data.replies_html);
+            $(".replies").html(newcontent);
+            // re-bind events
+            setup_emails_list(newcontent);
+            fold_quotes(newcontent);
+            setup_disabled_tooltips(newcontent);
+            setup_vote(newcontent);
+            // participants list
+            $("#participants").html(data.participants_html);
+        },
+        error: function(jqXHR, textStatus, errorThrown) {
+            alert(jqXHR.responseText);
+        }
     });
 }
 
-function setup_quotes() {
-    $('div.email-body .quoted-switch a')
-        .click(function(e) {
-            e.preventDefault();
-            $(this).parent().next(".quoted-text").slideToggle('fast');
-        });
-}
+
+/*
+ * Misc.
+ */
 
 function setup_months_list() {
     var current = $("#months-list li.current").parent().prev();
@@ -345,8 +388,11 @@ function setup_months_list() {
     $("#months-list").accordion({ collapsible: true, active: current });
 }
 
-function setup_disabled_tooltips() {
-    $("a.disabled").tooltip().click(function (e) {
+function setup_disabled_tooltips(baseElem) {
+    if (!baseElem) {
+        baseElem = document;
+    }
+    $(baseElem).find("a.disabled").tooltip().click(function (e) {
         e.preventDefault();
     });
 }
@@ -362,12 +408,10 @@ function setup_flash_messages() {
 
 $(document).ready(function() {
     setup_vote();
-    setup_attachments();
     setup_add_tag();
-    setup_quotes();
     setup_months_list();
     setup_favorites();
-    setup_replies();
+    setup_emails_list();
     setup_disabled_tooltips();
     setup_flash_messages();
 });
diff --git a/hyperkitty/templates/thread.html b/hyperkitty/templates/thread.html
index 9de03b1..a198014 100644
--- a/hyperkitty/templates/thread.html
+++ b/hyperkitty/templates/thread.html
@@ -50,14 +50,11 @@
 			</p>
 
 			<div class="replies">
-				{% for email in replies %}
-				<div class="{% cycle 'even' 'odd' %}"
-					{% if email.level %}style="margin-left:{{ email.level|multiply:"2" }}em;"{% endif %}>
-					<!-- Start email -->
-					{% include 'messages/message.html' %}
-					<!-- End of email -->
-				</div>
-				{% endfor %}
+			{% if is_bot %}
+				{% include 'threads/replies.html' %}
+			{% else %}
+				<img alt="Loading..." class="ajaxloader" src="{{ STATIC_URL }}img/ajax-loader.gif" />
+			{% endif %}
 			</div>
 
 		</section>
@@ -80,14 +77,10 @@
 
 <script type="text/javascript">
 	$(document).ready(function() {
-		// hide quotes by default in the thread view
-		$('div.email-body .quoted-text').each(function() {
-			var linescount = $(this).text().split("\n").length;
-			if (linescount > 3) {
-				// hide if the quote is more than 3 lines long
-				$(this).hide();
-			}
-		});
+		// Hide quotes by default in the thread view
+		fold_quotes()
+		// Load the replies
+		update_thread_replies("{% url 'thread_replies' threadid=threadid mlist_fqdn=mlist.name %}?sort={{sort_mode}}");
 	});
 </script>
 
diff --git a/hyperkitty/templates/threads/participants.html b/hyperkitty/templates/threads/participants.html
new file mode 100644
index 0000000..b40445b
--- /dev/null
+++ b/hyperkitty/templates/threads/participants.html
@@ -0,0 +1,14 @@
+{% load url from future %}
+{% load gravatar %}
+{% load hk_generic %}
+
+		<span id="participants_title">participants</span> ({{participants|length}})
+		<ul>
+			{% for name, email in participants.items|sort %}
+			<li>
+				{% gravatar email 20 %}
+				{{ name|escapeemail }}
+			</li>
+			{% endfor %}
+		</ul>
+
diff --git a/hyperkitty/templates/threads/replies.html b/hyperkitty/templates/threads/replies.html
new file mode 100644
index 0000000..19ee0e2
--- /dev/null
+++ b/hyperkitty/templates/threads/replies.html
@@ -0,0 +1,11 @@
+{% load url from future %}
+{% load hk_generic %}
+
+				{% for email in replies %}
+				<div class="{% cycle 'even' 'odd' %}"
+					{% if email.level %}style="margin-left:{{ email.level|multiply:"2" }}em;"{% endif %}>
+					<!-- Start email -->
+					{% include 'messages/message.html' %}
+					<!-- End of email -->
+				</div>
+				{% endfor %}
diff --git a/hyperkitty/templates/threads/right_col.html b/hyperkitty/templates/threads/right_col.html
index cbc7772..8744f66 100644
--- a/hyperkitty/templates/threads/right_col.html
+++ b/hyperkitty/templates/threads/right_col.html
@@ -44,15 +44,12 @@
 		</form>
 	</div>
 	<div id="participants">
-		<span id="participants_title">participants</span> ({{participants|length}})
-		<ul>
-			{% for name, email in participants.items|sort %}
-			<li>
-				{% gravatar email 20 %}
-				{{ name|escapeemail }}
-			</li>
-			{% endfor %}
-		</ul>
+		{% if is_bot %}
+			{% include 'threads/participants.html' %}
+		{% else %}
+			<span id="participants_title">participants</span>
+			<img alt="Loading..." class="ajaxloader" src="{{ STATIC_URL }}img/ajax-loader.gif" />
+		{% endif %}
 	</div>
 </section>
 
diff --git a/hyperkitty/urls.py b/hyperkitty/urls.py
index 2ec92c9..9199de0 100644
--- a/hyperkitty/urls.py
+++ b/hyperkitty/urls.py
@@ -75,6 +75,8 @@ urlpatterns = patterns('hyperkitty.views',
     # Thread
     url(r'^list/(?P<mlist_fqdn>[^/@]+@[^/@]+)/thread/(?P<threadid>\w+)/$',
         'thread.thread_index', name='thread'),
+    url(r'^list/(?P<mlist_fqdn>[^/@]+@[^/@]+)/thread/(?P<threadid>\w+)/replies$',
+        'thread.replies', name='thread_replies'),
     url(r'^list/(?P<mlist_fqdn>[^/@]+@[^/@]+)/thread/(?P<threadid>\w+)/addtag$',
         'thread.add_tag', name='add_tag'),
     url(r'^list/(?P<mlist_fqdn>[^/@]+@[^/@]+)/thread/(?P<threadid>\w+)/favorite$',
diff --git a/hyperkitty/views/thread.py b/hyperkitty/views/thread.py
index 7929ac6..be92be7 100644
--- a/hyperkitty/views/thread.py
+++ b/hyperkitty/views/thread.py
@@ -31,6 +31,7 @@ from django.template import RequestContext, loader
 from django.shortcuts import render
 from django.core.urlresolvers import reverse
 from django.core.exceptions import SuspiciousOperation
+import robot_detection
 
 from hyperkitty.models import Tag, Favorite
 from forms import SearchForm, AddTagForm, ReplyForm
@@ -38,14 +39,10 @@ from hyperkitty.lib import get_months, get_store, stripped_subject
 from hyperkitty.lib.voting import set_message_votes
 
 
-def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
-    ''' Displays all the email for a given thread identifier '''
-    search_form = SearchForm(auto_id=False)
-    store = get_store(request)
-    thread = store.get_thread(mlist_fqdn, threadid)
+def _get_thread_replies(request, thread):
+    '''Get and sort the replies for a thread'''
     if not thread:
         raise Http404
-    prev_thread, next_thread = store.get_thread_neighbors(mlist_fqdn, threadid)
 
     if "sort" in request.GET and request.GET["sort"] == "date":
         sort_mode = "date"
@@ -54,7 +51,10 @@ def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
         sort_mode = "thread"
         emails = thread.emails_by_reply
 
-    participants = {}
+    # Don't forget to add the sender to the participants
+    participants = {thread.starting_email.sender_name:
+                    thread.starting_email.sender_email}
+    emails = list(emails)[1:] # only select replies
     for email in emails:
         # Extract all the votes for this message
         set_message_votes(email, request.user)
@@ -67,6 +67,21 @@ def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
             if email.level > 5:
                 email.level = 5
 
+    return {"replies": emails, "participants": participants}
+
+
+def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
+    ''' Displays all the email for a given thread identifier '''
+    search_form = SearchForm(auto_id=False)
+    store = get_store(request)
+    thread = store.get_thread(mlist_fqdn, threadid)
+    if not thread:
+        raise Http404
+    prev_thread, next_thread = store.get_thread_neighbors(mlist_fqdn, threadid)
+
+    sort_mode = request.GET.get("sort", "thread")
+    set_message_votes(thread.starting_email, request.user)
+
     from_url = reverse("thread", kwargs={"mlist_fqdn":mlist_fqdn,
                                          "threadid":threadid})
     # Tags
@@ -95,17 +110,22 @@ def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
     mlist = store.get_list(mlist_fqdn)
     subject = stripped_subject(mlist, thread.starting_email.subject)
 
+    # TODO: eventually move to a middleware ?
+    # http://djangosnippets.org/snippets/1865/
+    is_bot = True
+    user_agent = request.META.get('HTTP_USER_AGENT', None)
+    if user_agent:
+        is_bot = robot_detection.is_robot(user_agent)
+
     context = {
-        'mlist' : mlist,
-        'threadid' : threadid,
+        'mlist': mlist,
+        'threadid': threadid,
         'subject': subject,
-        'tags' : tags,
+        'tags': tags,
         'search_form': search_form,
         'addtag_form': tag_form,
         'month': thread.date_active,
-        'participants': participants,
         'first_mail': thread.starting_email,
-        'replies': list(emails)[1:],
         'neighbors': (prev_thread, next_thread),
         'months_list': get_months(store, mlist.name),
         'days_inactive': days_inactive.days,
@@ -113,10 +133,43 @@ def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
         'sort_mode': sort_mode,
         'fav_action': fav_action,
         'reply_form': ReplyForm(),
+        'is_bot': is_bot,
     }
+
+    if is_bot:
+        # Don't rely on AJAX to load the replies
+        thread_replies = _get_thread_replies(request, thread)
+        context["participants"] = thread_replies["participants"]
+        context["replies"] = thread_replies["replies"]
+
     return render(request, "thread.html", context)
 
 
+def replies(request, mlist_fqdn, threadid):
+    """Get JSON encoded lists with the replies and the participants"""
+    store = get_store(request)
+    thread = store.get_thread(mlist_fqdn, threadid)
+    thread_replies = _get_thread_replies(request, thread)
+    mlist = store.get_list(mlist_fqdn)
+    context = {
+        'mlist': mlist,
+        'threadid': threadid,
+        'reply_form': ReplyForm(),
+    }
+    context["participants"] = thread_replies["participants"]
+    context["replies"] = thread_replies["replies"]
+
+    replies_tpl = loader.get_template('threads/replies.html')
+    replies_html = replies_tpl.render(RequestContext(request, context))
+    participants_tpl = loader.get_template('threads/participants.html')
+    participants_html = participants_tpl.render(RequestContext(request, context))
+    response = {"replies_html": replies_html,
+                "participants_html": participants_html,
+               }
+    return HttpResponse(json.dumps(response),
+                        mimetype='application/javascript')
+
+
 def add_tag(request, mlist_fqdn, threadid):
     """ Add a tag to a given thread. """
     if not request.user.is_authenticated():
diff --git a/requirements.txt b/requirements.txt
index 9f38b3c..12aa50a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,3 +10,4 @@ django-assets
 rjsmin
 cssmin
 mailmanclient
+robot-detection