From 61d74d67e870bce113a868043a96d1de56baf6bc Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Wed, 3 Oct 2012 17:12:43 +0200 Subject: Adapt to the i18n filename bug See: https://bugs.launchpad.net/mailman/+bug/1060951 --- kittystore/scrub.py | 11 +++++++++-- kittystore/test/test_scrub.py | 37 +++++++++++++++++++++++++++++-------- kittystore/test/test_utils.py | 6 ++++-- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/kittystore/scrub.py b/kittystore/scrub.py index 74aeed9..28e9651 100644 --- a/kittystore/scrub.py +++ b/kittystore/scrub.py @@ -248,7 +248,12 @@ class Scrubber(object): ctype = part.get_content_type() charset = get_charset(part, default=None, guess=False) # i18n file name is encoded - filename = oneline(part.get_filename(''), in_unicode=True) + try: + filename = oneline(part.get_filename(''), in_unicode=True) + except TypeError: + # Workaround for https://bugs.launchpad.net/mailman/+bug/1060951 + # (accented filenames) + filename = "attachment.bin" filename, fnext = os.path.splitext(filename) # For safety, we should confirm this is valid ext for content-type # but we can use fnext if we introduce fnext filtering @@ -281,7 +286,9 @@ class Scrubber(object): # Strip off leading dots filename = dre.sub('', filename) # Allow only alphanumerics, dash, underscore, and dot - #filename = sre.sub('', filename) + # i18n filenames are not supported yet, + # see https://bugs.launchpad.net/bugs/1060951 + filename = sre.sub('', filename) # If the filename's extension doesn't match the type we guessed, # which one should we go with? For now, let's go with the one we # guessed so attachments can't lie about their type. Also, if the diff --git a/kittystore/test/test_scrub.py b/kittystore/test/test_scrub.py index 205167c..0504122 100644 --- a/kittystore/test/test_scrub.py +++ b/kittystore/test/test_scrub.py @@ -4,6 +4,7 @@ import unittest import email from mock import Mock +from mailman.email.message import Message from kittystore.scrub import Scrubber from kittystore.test import get_test_file @@ -13,7 +14,7 @@ class TestScrubber(unittest.TestCase): def test_attachment_1(self): with open(get_test_file("attachment-1.txt")) as email_file: - msg = email.message_from_file(email_file) + msg = email.message_from_file(email_file, _class=Message) store = Mock() scrubber = Scrubber("testlist@example.com", msg, store) contents = scrubber.scrub() @@ -31,7 +32,7 @@ class TestScrubber(unittest.TestCase): def test_attachment_2(self): with open(get_test_file("attachment-2.txt")) as email_file: - msg = email.message_from_file(email_file) + msg = email.message_from_file(email_file, _class=Message) store = Mock() scrubber = Scrubber("testlist@example.com", msg, store) contents = scrubber.scrub() @@ -52,7 +53,7 @@ class TestScrubber(unittest.TestCase): def test_attachment_3(self): with open(get_test_file("attachment-3.txt")) as email_file: - msg = email.message_from_file(email_file) + msg = email.message_from_file(email_file, _class=Message) store = Mock() scrubber = Scrubber("testlist@example.com", msg, store) contents = scrubber.scrub() @@ -73,7 +74,7 @@ class TestScrubber(unittest.TestCase): def test_html_email_1(self): with open(get_test_file("html-email-1.txt")) as email_file: - msg = email.message_from_file(email_file) + msg = email.message_from_file(email_file, _class=Message) store = Mock() scrubber = Scrubber("testlist@example.com", msg, store) contents = scrubber.scrub() @@ -93,7 +94,7 @@ class TestScrubber(unittest.TestCase): """Scrubber must handle non-ascii messages""" for enc in ["utf8", "iso8859"]: with open(get_test_file("payload-%s.txt" % enc)) as email_file: - msg = email.message_from_file(email_file) + msg = email.message_from_file(email_file, _class=Message) store = Mock() scrubber = Scrubber("testlist@example.com", msg, store) contents = scrubber.scrub() @@ -103,7 +104,7 @@ class TestScrubber(unittest.TestCase): def test_attachment_4(self): with open(get_test_file("attachment-4.txt")) as email_file: - msg = email.message_from_file(email_file) + msg = email.message_from_file(email_file, _class=Message) store = Mock() scrubber = Scrubber("testlist@example.com", msg, store) contents = scrubber.scrub() @@ -114,13 +115,33 @@ class TestScrubber(unittest.TestCase): "CAHmoxtXXb3un1C=ZvYNtz-eYghm-GH925gDVHyjhvL2YEsZ-Yw@mail.gmail.com", 3, "attachment.html", "text/html", "iso-8859-1")) self.assertEqual(len(args_1[0][6]), 114) - # Image attachment + # text attachment self.assertEqual(args_2[0][0:6], ("testlist@example.com", "CAHmoxtXXb3un1C=ZvYNtz-eYghm-GH925gDVHyjhvL2YEsZ-Yw@mail.gmail.com", - 4, u"todo-déjeuner.txt", "text/plain", "utf-8")) + #4, u"todo-déjeuner.txt", "text/plain", "utf-8")) + 4, u"todo-djeuner.txt", "text/plain", "utf-8")) self.assertEqual(len(args_2[0][6]), 112) # Scrubbed content self.assertEqual(contents, u'This is a test, HTML message with ' u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an ' u'attachment with an accented filename\r\n') + def test_attachment_5(self): + with open(get_test_file("attachment-5.txt")) as email_file: + msg = email.message_from_file(email_file, _class=Message) + store = Mock() + scrubber = Scrubber("testlist@example.com", msg, store) + contents = scrubber.scrub() + self.assertEqual(store.add_attachment.call_count, 1) + args = store.add_attachment.call_args_list[0][0] + # text attachment + self.assertEqual(args[0:6], + ("testlist@example.com", "506C3344.7020501@free.fr", + #2, u"todo-déjeuner.txt", "text/plain", "utf-8")) + 2, u"attachment.bin", "text/plain", "utf-8")) + self.assertEqual(len(args[6]), 112) + # Scrubbed content + self.assertEqual(contents, u'This is a test, HTML message with ' + u'accented letters : \xe9 \xe8 \xe7 \xe0.\r\nAnd an ' + u'attachment with an accented filename\r\n\r\n\r\n\r\n') + diff --git a/kittystore/test/test_utils.py b/kittystore/test/test_utils.py index 118d2e5..abea07f 100644 --- a/kittystore/test/test_utils.py +++ b/kittystore/test/test_utils.py @@ -6,6 +6,8 @@ import datetime import dateutil from mock import Mock +from mailman.email.message import Message + import kittystore.utils from kittystore.test import get_test_file @@ -14,7 +16,7 @@ class TestUtils(unittest.TestCase): def test_ref_parsing(self): with open(get_test_file("strange-in-reply-to-header.txt")) as email_file: - msg = email.message_from_file(email_file) + msg = email.message_from_file(email_file, _class=Message) store = Mock() store.get_message_by_id_from_list.return_value = None ref_id, thread_id = kittystore.utils.get_ref_and_thread_id( @@ -23,7 +25,7 @@ class TestUtils(unittest.TestCase): def test_wrong_reply_to_format(self): with open(get_test_file("wrong-in-reply-to-header.txt")) as email_file: - msg = email.message_from_file(email_file) + msg = email.message_from_file(email_file, _class=Message) store = Mock() store.get_message_by_id_from_list.return_value = None ref_id, thread_id = kittystore.utils.get_ref_and_thread_id( -- cgit