Adapt to the i18n filename bug

See: https://bugs.launchpad.net/mailman/+bug/1060951

3 files changed, 42 insertions, 12 deletions

diff --git a/kittystore/scrub.py b/kittystore/scrub.py
index 74aeed9..28e9651 100644
--- a/kittystore/scrub.py
+++ b/kittystore/scrub.py
@@ -248,7 +248,12 @@ class Scrubber(object):
         ctype = part.get_content_type()
         charset = get_charset(part, default=None, guess=False)
         # i18n file name is encoded
-        filename = oneline(part.get_filename(''), in_unicode=True)
+        try:
+            filename = oneline(part.get_filename(''), in_unicode=True)
+        except TypeError:
+            # Workaround for https://bugs.launchpad.net/mailman/+bug/1060951
+            # (accented filenames)
+            filename = "attachment.bin"
         filename, fnext = os.path.splitext(filename)
         # For safety, we should confirm this is valid ext for content-type
         # but we can use fnext if we introduce fnext filtering
@@ -281,7 +286,9 @@ class Scrubber(object):
             # Strip off leading dots
             filename = dre.sub('', filename)
             # Allow only alphanumerics, dash, underscore, and dot
-            #filename = sre.sub('', filename)
+            # i18n filenames are not supported yet,
+            # see https://bugs.launchpad.net/bugs/1060951
+            filename = sre.sub('', filename)
             # If the filename's extension doesn't match the type we guessed,
             # which one should we go with?  For now, let's go with the one we
             # guessed so attachments can't lie about their type.  Also, if the
diff --git a/kittystore/test/test_scrub.py b/kittystore/test/test_scrub.py
index 205167c..0504122 100644
--- a/kittystore/test/test_scrub.py
+++ b/kittystore/test/test_scrub.py
@@ -4,6 +4,7 @@ import unittest
 import email
 
 from mock import Mock
+from mailman.email.message import Message
 
 from kittystore.scrub import Scrubber
 from kittystore.test import get_test_file
@@ -13,7 +14,7 @@ class TestScrubber(unittest.TestCase):
 
     def test_attachment_1(self):
         with open(get_test_file("attachment-1.txt")) as email_file:
-            msg = email.message_from_file(email_file)
+            msg = email.message_from_file(email_file, _class=Message)
         store = Mock()
         scrubber = Scrubber("testlist@example.com", msg, store)
         contents = scrubber.scrub()
@@ -31,7 +32,7 @@ class TestScrubber(unittest.TestCase):
 
     def test_attachment_2(self):
         with open(get_test_file("attachment-2.txt")) as email_file:
-            msg = email.message_from_file(email_file)
+            msg = email.message_from_file(email_file, _class=Message)
         store = Mock()
         scrubber = Scrubber("testlist@example.com", msg, store)
         contents = scrubber.scrub()
@@ -52,7 +53,7 @@ class TestScrubber(unittest.TestCase):
 
     def test_attachment_3(self):
         with open(get_test_file("attachment-3.txt")) as email_file:
-            msg = email.message_from_file(email_file)
+            msg = email.message_from_file(email_file, _class=Message)
         store = Mock()
         scrubber = Scrubber("testlist@example.com", msg, store)
         contents = scrubber.scrub()
@@ -73,7 +74,7 @@ class TestScrubber(unittest.TestCase):
 
     def test_html_email_1(self):
         with open(get_test_file("html-email-1.txt")) as email_file:
-            msg = email.message_from_file(email_file)
+            msg = email.message_from_file(email_file, _class=Message)
         store = Mock()
         scrubber = Scrubber("testlist@example.com", msg, store)
         contents = scrubber.scrub()
@@ -93,7 +94,7 @@ class TestScrubber(unittest.TestCase):
         """Scrubber must handle non-ascii messages"""
         for enc in ["utf8", "iso8859"]:
             with open(get_test_file("payload-%s.txt" % enc)) as email_file:
-                msg = email.message_from_file(email_file)
+                msg = email.message_from_file(email_file, _class=Message)
             store = Mock()
             scrubber = Scrubber("testlist@example.com", msg, store)
             contents = scrubber.scrub()
@@ -103,7 +104,7 @@ class TestScrubber(unittest.TestCase):
 
     def test_attachment_4(self):
         with open(get_test_file("attachment-4.txt")) as email_file:
-            msg = email.message_from_file(email_file)
+            msg = email.message_from_file(email_file, _class=Message)
         store = Mock()
         scrubber = Scrubber("testlist@example.com", msg, store)
         contents = scrubber.scrub()
@@ -114,13 +115,33 @@ class TestScrubber(unittest.TestCase):
                 "CAHmoxtXXb3un1C=ZvYNtz-eYghm-GH925gDVHyjhvL2YEsZ-Yw@mail.gmail.com",
                 3, "attachment.html", "text/html", "iso-8859-1"))
         self.assertEqual(len(args_1[0][6]), 114)
-        # Image attachment
+        # text attachment
         self.assertEqual(args_2[0][0:6], ("testlist@example.com",
                 "CAHmoxtXXb3un1C=ZvYNtz-eYghm-GH925gDVHyjhvL2YEsZ-Yw@mail.gmail.com",
-                4, u"todo-déjeuner.txt", "text/plain", "utf-8"))
+                #4, u"todo-déjeuner.txt", "text/plain", "utf-8"))
+                4, u"todo-djeuner.txt", "text/plain", "utf-8"))
         self.assertEqual(len(args_2[0][6]), 112)
         # Scrubbed content
         self.assertEqual(contents, u'This is a test, HTML message with '
                 u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an '
                 u'attachment with an accented filename\r\n')
 
+    def test_attachment_5(self):
+        with open(get_test_file("attachment-5.txt")) as email_file:
+            msg = email.message_from_file(email_file, _class=Message)
+        store = Mock()
+        scrubber = Scrubber("testlist@example.com", msg, store)
+        contents = scrubber.scrub()
+        self.assertEqual(store.add_attachment.call_count, 1)
+        args = store.add_attachment.call_args_list[0][0]
+        # text attachment
+        self.assertEqual(args[0:6],
+                ("testlist@example.com", "506C3344.7020501@free.fr",
+                #2, u"todo-déjeuner.txt", "text/plain", "utf-8"))
+                2, u"attachment.bin", "text/plain", "utf-8"))
+        self.assertEqual(len(args[6]), 112)
+        # Scrubbed content
+        self.assertEqual(contents, u'This is a test, HTML message with '
+                u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an '
+                u'attachment with an accented filename\r\n\r\n\r\n\r\n')
+
diff --git a/kittystore/test/test_utils.py b/kittystore/test/test_utils.py
index 118d2e5..abea07f 100644
--- a/kittystore/test/test_utils.py
+++ b/kittystore/test/test_utils.py
@@ -6,6 +6,8 @@ import datetime
 import dateutil
 from mock import Mock
 
+from mailman.email.message import Message
+
 import kittystore.utils
 from kittystore.test import get_test_file
 
@@ -14,7 +16,7 @@ class TestUtils(unittest.TestCase):
 
     def test_ref_parsing(self):
         with open(get_test_file("strange-in-reply-to-header.txt")) as email_file:
-            msg = email.message_from_file(email_file)
+            msg = email.message_from_file(email_file, _class=Message)
         store = Mock()
         store.get_message_by_id_from_list.return_value = None
         ref_id, thread_id = kittystore.utils.get_ref_and_thread_id(
@@ -23,7 +25,7 @@ class TestUtils(unittest.TestCase):
 
     def test_wrong_reply_to_format(self):
         with open(get_test_file("wrong-in-reply-to-header.txt")) as email_file:
-            msg = email.message_from_file(email_file)
+            msg = email.message_from_file(email_file, _class=Message)
         store = Mock()
         store.get_message_by_id_from_list.return_value = None
         ref_id, thread_id = kittystore.utils.get_ref_and_thread_id(