diff --git a/modoboa/lib/email_utils.py b/modoboa/lib/email_utils.py index 97dbb3c00..1a16b6062 100644 --- a/modoboa/lib/email_utils.py +++ b/modoboa/lib/email_utils.py @@ -1,22 +1,29 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- from __future__ import unicode_literals -from email.header import Header, decode_header +import email +from email.header import Header from email.mime.text import MIMEText from email.utils import make_msgid, formatdate, parseaddr import re import smtplib import time -from django.conf import settings -from django.utils.encoding import smart_text -from django.utils.html import conditional_escape +from django.utils.encoding import smart_str, smart_text +from django.utils.html import conditional_escape, escape from django.template.loader import render_to_string -from django.utils import six from modoboa.lib import u2u_decode +import lxml.html +from lxml.html.clean import Cleaner + + +# used by Email() +_RE_REMOVE_EXTRA_WHITESPACE = re.compile(r"\n\s*\n") +_RE_CID = re.compile(r".*[ ]*cid=\"([^\"]*)\".*", re.I) + class EmailAddress(object): @@ -33,107 +40,100 @@ class EmailAddress(object): class Email(object): - def __init__(self, mailid, mformat="plain", dformat="plain", links=0): - self.attached_map = {} + _basic_headers = ( + "From", + "To", + "Cc", + "Date", + "Subject", + ) + + _image_mime_types = ( + "image/png", + "image/gif", + "image/jpeg", + ) + + def __init__(self, mailid, mformat="plain", dformat="plain", links=False): self.contents = {"html": "", "plain": ""} - self.headers = [] - self.attachments = {} self.mailid = mailid self.mformat = mformat self.dformat = dformat - self.links = links + + if isinstance(links, bool): + self.links = links + elif isinstance(links, int): + # TODO: modoboa-webmail currently sets links == 0 or 1, it needs + # updated to use True or False. + self.links = bool(links) + elif links == "0": + raise TypeError("links == \"0\" is not valid, did you mean True or " + "False?") + else: + raise TypeError("links should be a boolean value") + self._msg = None + self._headers = None self._body = None + self._images = {} + + # used by modoboa_webmail.lib.imapemail.ImapEmail + self.attachments = {} @property def msg(self): - """Return an email.message object. - """ - raise NotImplementedError + if self._msg is None: + mail_text = self._fetch_message() + assert isinstance(mail_text, bytes) + # python 2 expects str (bytes), python 3 expects str (unicode) + mail_text = smart_str(mail_text, errors="replace") + self._msg = email.message_from_string(mail_text) + + return self._msg + + @property + def headers(self): + if self._headers is None: + self._headers = [] + + # don't break modoboa_webmail.lib.imapemail.ImapEmail, it + # doesn't have self._msg it loads e-mails (and their headers) + # from the IMAP server. + if self._msg is not None: + for header in self._basic_headers: + value = self.get_header(self.msg, header) + self._headers.append({ + "name": header, + "value": value + }) + + return self._headers + + @headers.setter + def headers(self, value): + self._headers = value @property def body(self): - """Return email's body.""" if self._body is None: - self._body = ( - getattr(self, "viewmail_%s" % self.mformat) - (self.contents[self.mformat], links=self.links) - ) + self._parse(self.msg) + self._body = getattr(self, "viewmail_%s" % self.dformat)() + return self._body - def get_header(self, msg, hdrname): - """Look for a particular header. - - :param string hdrname: header name - :return: header avalue - """ - for name in [hdrname, hdrname.upper()]: - if name in msg: - return msg[name] + def get_header(self, msg, header): + # msg parameter to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if header in msg: + return "".join([smart_text(v, encoding=(e or 'ascii')) + for v, e in email.header.decode_header(msg[header]) + ]) return "" - def _parse_default(self, msg, level): - """Default parser + def _fetch_message(self): + raise NotImplementedError() - All parts handled by this parser will be consireded as - attachments. - """ - fname = msg.get_filename() - if fname is not None: - if isinstance(fname, six.text_type): - fname = fname.encode("utf-8") - decoded = decode_header(fname) - value = ( - decoded[0][0] if decoded[0][1] is None - else smart_text(decoded[0][0], decoded[0][1]) - ) - else: - value = "part_%s" % level - self.attachments[level] = value - - def _parse_text(self, msg, level): - """Displayable content parser - - text, html, calendar, etc. All those contents can be displayed - inside a navigator. - """ - if msg.get_content_subtype() not in ["plain", "html"]: - self._parse_default(msg, level) - target = "plain" - else: - target = msg.get_content_subtype() - self.contents[target] += decode(msg.get_payload(decode=True), - charset=msg.get_content_charset()) - - def _parse_image(self, msg, level): - """image/* parser - - The only reason to make a specific parser for images is that, - sometimes, messages embark inline images, which means they - must be displayed and not attached. - """ - if self.dformat == "html" and self.links \ - and "Content-Disposition" in msg: - if msg["Content-Disposition"].startswith("inline"): - cid = None - if "Content-ID" in msg: - m = re.match("<(.+)>", msg["Content-ID"]) - cid = m is not None and m.group(1) or msg["Content-ID"] - fname = msg.get_filename() - if fname is None: - if "Content-Location" in msg: - fname = msg["Content-Location"] - elif cid is not None: - fname = cid - else: - # I give up for now :p - return - self.attached_map[cid] = re.match("^http:", fname) and fname \ - or self.__save_image(fname, msg) - return - self._parse_default(msg, level) - - def _parse(self, msg, level=None): + def _parse(self, msg, level=0): """Recursive email parser A message structure can be complex. To correctly handle @@ -146,85 +146,121 @@ class Email(object): :param msg: message (or part) to parse :param level: current part number """ - if msg.is_multipart() and msg.get_content_maintype() != "message": - cpt = 1 - for part in msg.get_payload(): - nlevel = level is None and ("%d" % cpt) \ - or "%s.%d" % (level, cpt) - self._parse(part, nlevel) - cpt += 1 + content_type = msg.get_content_maintype() + is_attachment = "attachment" in msg.get("Content-Disposition", "") + + if content_type == "text" and not is_attachment: + self._parse_text(msg, level) + elif content_type == "multipart" and not is_attachment: + self._parse_multipart(msg, level) + else: + self._parse_inline_image(msg, level) + + if self.contents["plain"]: + self._post_process_plain() + if self.contents["html"]: + self._post_process_html() + + def _parse_text(self, msg, level=0): + content_type = msg.get_content_subtype() + + if content_type in ["plain", "html"]: + encoding = msg.get_content_charset() or "utf-8" + mail_text = smart_text(msg.get_payload(decode=True), encoding=encoding, errors="replace") + self.contents[content_type] += mail_text + + def _parse_multipart(self, msg, level=0): + level += 1 + for part in msg.walk(): + content_type = part.get_content_maintype() + if content_type == "text": + self._parse_text(part, level=level) + else: + # I'm a dumb mail parser and treat all non-text parts as attachments + self._parse_inline_image(part, level=level) + + def _parse_inline_image(self, msg, level=0): + content_type = msg.get_content_type() + cid = None + if "Content-ID" in msg: + cid = msg["Content-ID"] + if cid.startswith('<') and cid.endswith('>'): + cid = cid[1:-1] + else: + matches = _RE_CID.match(msg["Content-Type"]) + if matches is not None: + cid = matches.group(1) + transfer_encoding = msg["Content-Transfer-Encoding"] + if content_type not in self._image_mime_types or cid is None: + # I'm a dumb mail parser and I don't care ;p + return + if transfer_encoding != "base64": + # I don't deal with non base64 encoded images + return + if cid in self._images: + # Duplicate Content-ID return - if level is None: - level = "1" - try: - getattr(self, "_parse_%s" % msg.get_content_maintype())(msg, level) - except AttributeError: - self._parse_default(msg, level) + self._images[cid] = "data:%s;base64,%s" % ( + content_type, "".join(msg.get_payload().splitlines(False))) - def __save_image(self, fname, part): - """Save an inline image on the filesystem. + def _post_process_plain(self): + mail_text = _RE_REMOVE_EXTRA_WHITESPACE \ + .sub("\n\n", self.contents["plain"]).strip() + mail_text = escape(mail_text) + self.contents["plain"] = smart_text(mail_text) - Some HTML messages are using inline images (attached images - with links on them inside the body). In order to display them, - images are saved on the filesystem and links contained in the - message are modified. + def _post_process_html(self): + html = lxml.html.fromstring(self.contents["html"]) - :param fname: the image associated filename - :param part: the email part that contains the image payload - """ - if re.search(r"\.\.", fname): - return None - path = "/static/tmp/" + fname - fp = open(settings.MODOBOA_DIR + path, "wb") - fp.write(part.get_payload(decode=True)) - fp.close() - return path + if self.links: + html.rewrite_links(self._map_cid) - def map_cid(self, url): - """Replace attachment links. + for link in html.iterlinks(): + link[0].set("target", "_blank") + else: + html.rewrite_links(lambda x: None) + + cleaner = Cleaner( + scripts=True, + javascript=True, + links=True, + page_structure=True, + embedded=True, + frames=True, + add_nofollow=True) + mail_text = lxml.html.tostring(cleaner.clean_html(html)) + self.contents["html"] = smart_text(mail_text) + + def _map_cid(self, url): + if url.startswith("cid:"): + cid = url[4:] + if cid in self._images: + url = self._images[cid] - :param str url: original url - :rtype: string - :return: internal link - """ - match = re.match(".*cid:(.+)", url) - if match is not None: - if match.group(1) in self.attached_map: - return self.attached_map[match.group(1)] return url + def viewmail_plain(self, contents=None, **kwargs): + # contents and **kwargs parameters to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if contents is None: + contents = self.contents["plain"] + else: + contents = conditional_escape(contents) + return "
%s
" % contents + + def viewmail_html(self, contents=None, **kwargs): + # contents and **kwargs parameters to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if contents is None: + contents = self.contents["html"] + return contents + def render_headers(self, **kwargs): - return render_to_string("common/mailheaders.html", { + context = { "headers": self.headers, - }) - - def viewmail_plain(self, content, **kwargs): - content = conditional_escape(content) - return "
%s
" % content - - def viewmail_html(self, content, **kwargs): - import lxml.html - - if content is None or content == "": - return "" - links = kwargs.get("links", 0) - html = lxml.html.fromstring(content) - if not links: - html.rewrite_links(lambda x: None) - else: - html.rewrite_links(self.map_cid) - for link in html.iterlinks(): - link[0].set('target', '_blank') - body = html.find("body") - if body is None: - body = lxml.html.tostring(html) - else: - body = lxml.html.tostring(body) - body = re.sub( - "<(/?)body", lambda m: "<%sdiv" % m.group(1), - body.decode()) - return body + } + return render_to_string("common/mailheaders.html", context) def split_mailbox(mailbox, return_extension=False): diff --git a/modoboa/lib/tests.py b/modoboa/lib/tests/__init__.py similarity index 55% rename from modoboa/lib/tests.py rename to modoboa/lib/tests/__init__.py index e8c963af7..ace3d977e 100644 --- a/modoboa/lib/tests.py +++ b/modoboa/lib/tests/__init__.py @@ -1,4 +1,5 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- + """Testing utilities.""" from __future__ import unicode_literals @@ -13,8 +14,7 @@ from rest_framework.test import APITestCase from modoboa.core import models as core_models -from . import sysutils -from . import u2u_decode +from .. import sysutils try: s = socket.create_connection(('127.0.0.1', 25)) @@ -24,7 +24,7 @@ except socket.error: NO_SMTP = True try: - import ldap # NOQA + import ldap # noqa NO_LDAP = False except ImportError: NO_LDAP = True @@ -34,7 +34,7 @@ class ParametersMixin(object): """Add tools to manage parameters.""" @classmethod - def setUpTestData(cls): + def setUpTestData(cls): # noqa """Set LocalConfig instance.""" super(ParametersMixin, cls).setUpTestData() cls.localconfig = core_models.LocalConfig.objects.first() @@ -58,7 +58,7 @@ class ModoTestCase(ParametersMixin, TestCase): """All test cases must inherit from this one.""" @classmethod - def setUpTestData(cls): + def setUpTestData(cls): # noqa """Create a default user.""" super(ModoTestCase, cls).setUpTestData() management.call_command("load_initial_data") @@ -93,7 +93,7 @@ class ModoAPITestCase(ParametersMixin, APITestCase): """All test cases must inherit from this one.""" @classmethod - def setUpTestData(cls): + def setUpTestData(cls): # noqa """Create a default user.""" super(ModoAPITestCase, cls).setUpTestData() management.call_command("load_initial_data") @@ -104,54 +104,3 @@ class ModoAPITestCase(ParametersMixin, APITestCase): """Setup.""" super(ModoAPITestCase, self).setUp() self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token.key) - - -class U2UTestCase(TestCase): - """Test RFC1342 decoding utilities.""" - - def test_header_decoding(self): - """Simple decoding.""" - samples = [ - ("=?ISO-8859-15?Q?=20Profitez de tous les services en ligne sur " - "impots.gouv.fr?=", - "Profitez de tous les services en ligne sur impots.gouv.fr"), - ("=?ISO-8859-1?Q?Accus=E9?= de =?ISO-8859-1?Q?r=E9ception?= de " - "votre annonce", - "Accusé de réception de votre annonce"), - ("Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord", - "Sm\xf6rg\xe5sbord"), - # The following case currently fails because of the way we split - # encoded words to parse them separately, which can lead to - # unexpected unicode decode errors... I think it will work fine on - # Python3 - # ("=?utf-8?B?VMOpbMOpcMOpYWdlIFZJTkNJIEF1dG9yb3V0ZXMgLSBFeHDD?=\n" - # "=?utf-8?B?qWRpdGlvbiBkZSB2b3RyZSBjb21tYW5kZSBOwrAgMjAxNzEyMDcw" - # "MDA1?=\n=?utf-8?B?MyBkdSAwNy8xMi8yMDE3IDE0OjQ5OjQx?=", - # "") - ] - for sample in samples: - self.assertEqual(u2u_decode.u2u_decode(sample[0]), sample[1]) - - def test_address_header_decoding(self): - """Check address decoding.""" - mailsploit_sample = ( - "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=00?=" - "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com") - expected_result = ( - "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??=" - "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com") - self.assertEqual( - u2u_decode.decode_address(mailsploit_sample), - ("", expected_result) - ) - mailsploit_sample = ( - '"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?="\n' - '<=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?=' - '@mailsploit.com>') - expected_result = ( - 'potus@whitehouse.gov', - '=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??=' - '@mailsploit.com') - self.assertEqual( - u2u_decode.decode_address(mailsploit_sample), - expected_result) diff --git a/modoboa/lib/tests/sample_messages/multipart-input.txt b/modoboa/lib/tests/sample_messages/multipart-input.txt new file mode 100644 index 000000000..ee0b51430 --- /dev/null +++ b/modoboa/lib/tests/sample_messages/multipart-input.txt @@ -0,0 +1,130 @@ +Return-Path: +X-Original-To: someone@example.net +Delivered-To: me@example.net +Received: from mail.example.net (localhost [127.0.0.1]) + by mail.example.net (Postfix) with ESMTP id 4BD531AF130 + for ; Sun, 17 Dec 2017 03:11:30 +0000 (GMT) +Authentication-Results: mail.example.net (amavisd-new); + dkim=pass (1024-bit key) reason="pass (just generated, assumed good)" + header.d=example.net +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=example.net; h= + content-language:content-type:content-type:mime-version + :user-agent:date:date:message-id:subject:subject:from:from + :received:received; s=mail; t=1513480289; bh=wAmt0wg2Yk04321B + Q63uWL5f67PEBB2EEGONiFUrIvk=; b=UZUzPqH8TUk/bMZm1ofkjwatNCQftZKK + C91uIuX3mmjvngNwbVjx4m7jFZItZscNqigyGJN5HCOl8tC/LJmmWzRdhIisFtLA + uXRiBuWps82rbV4ciFU8N30i9UXvtWvMywwL6pYSESqixiwylzbddjMjKXXdyqaT + DTyR16ZK8IE= +X-Virus-Scanned: amavisd-new at example.net +X-Spam-Flag: NO +X-Spam-Score: -0.988 +X-Spam-Level: +X-Spam-Status: No, score=-0.988 tagged_above=-999 required=3.9 + tests=[ALL_TRUSTED=-1, HTML_MESSAGE=0.001, T_HTML_ATTACH=0.01, + URIBL_BLOCKED=0.001] autolearn=no autolearn_force=no +Received: from mail.example.net ([127.0.0.1]) + by mail.example.net (mail.example.net [127.0.0.1]) (amavisd-new, port 10026) + with LMTP id BSCmm59DxNvB for ; + Sun, 17 Dec 2017 03:11:29 +0000 (GMT) +Received: from [10.0.0.1] (my-pc.example.net [10.0.0.1]) + by mail.example.net (Postfix) with ESMTPSA id 641171AF12D + for ; Sun, 17 Dec 2017 03:11:29 +0000 (GMT) +To: Someone +From: Me +X-Clacks-Overhead: GNU Terry Pratchett +Subject: A test message +Message-ID: <5294ad7a-e966-f1a0-3431-8c4b1b76f5a3@example.net> +Date: Sun, 17 Dec 2017 03:11:33 +0000 +User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 + Thunderbird/52.5.0 +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="------------7EB9EDC10414C906CC3C2CEA" +Content-Language: en-GB + +This is a multi-part message in MIME format. +--------------7EB9EDC10414C906CC3C2CEA +Content-Type: multipart/alternative; + boundary="------------87B289699A48D5D0D61BABCC" + + +--------------87B289699A48D5D0D61BABCC +Content-Type: text/plain; charset=utf-8; format=flowed +Content-Transfer-Encoding: 7bit + +This is an *html* /test /_message._ + +:-) + + +-- +Someone +someone@example.net + + +--------------87B289699A48D5D0D61BABCC +Content-Type: multipart/related; + boundary="------------D585B241EEF60F0F44EC20E5" + + +--------------D585B241EEF60F0F44EC20E5 +Content-Type: text/html; charset=utf-8 +Content-Transfer-Encoding: 7bit + + + + + + + +

This is an html test message.

+ :-)
+
+
+ 
-- 
+someone
+someone@example.net
+ + + +--------------D585B241EEF60F0F44EC20E5 +Content-Type: image/png; + name="image.png" +Content-Transfer-Encoding: base64 +Content-ID: +Content-Disposition: inline; + filename="image.png" + +iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAACXBIWXMAAAsTAAALEwEAmpwY +AAAAF0lEQVQY02NgYGD4//8/ThKfHASMRN0AfaqVa6MQxIQAAAAASUVORK5CYII= +--------------D585B241EEF60F0F44EC20E5-- + +--------------87B289699A48D5D0D61BABCC-- + +--------------7EB9EDC10414C906CC3C2CEA +Content-Type: text/html; + name="test.html" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="test.html" + +//48AHMAcABhAG4APgA8AGIAPgBTAG8AbQBlACAAPABpAD4ASABUAE0ATAA8AC8AaQA+ACAA +dABlAHgAdAA8AC8AYgA+ACwAIABhACAAPABhACAAaAByAGUAZgA9ACIAaAB0AHQAcAA6AC8A +LwBlAHgAYQBtAHAAbABlAC4AbgBlAHQAIgAgAHQAYQByAGcAZQB0AD0AIgBfAGIAbABhAG4A +awAiACAAcgBlAGwAPQAiAG4AbwBmAG8AbABsAG8AdwAiAD4AbABpAG4AawA8AC8AYQA+ACwA +IABhAG4AJgAjADEAMwA7AA0ACgBlAG0AYgBlAGQAZABlAGQAIABpAG0AYQBnAGUAIABhAG4A +ZAAgAGEAbgAgAGUAeAB0AGUAcgBuAGEAbAAgAGkAbQBhAGcAZQAuADwAYgByAD4AJgAjADEA +MwA7AA0ACgA8AGkAbQBnACAAcwByAGMAPQAiAGQAYQB0AGEAOgBpAG0AYQBnAGUALwBwAG4A +ZwA7AGIAYQBzAGUANgA0ACwAaQBWAEIATwBSAHcAMABLAEcAZwBvAEEAQQBBAEEATgBTAFUA +aABFAFUAZwBBAEEAQQBBAG8AQQBBAEEAQQBLAEMAQQBJAEEAQQBBAEEAQwBVAEYAagBxAEEA +QQBBAEEAQwBYAEIASQBXAFgATQBBAEEAQQBzAFQAQQBBAEEATABFAHcARQBBAG0AcAB3AFkA +QQBBAEEAQQBGADAAbABFAFEAVgBRAFkAMAAyAE4AZwBZAEcARAA0AC8ALwA4AC8AVABoAEsA +ZgBIAEEAUwBNAFIATgAwAEEAZgBhAHEAVgBhADYATQBRAHgASQBRAEEAQQBBAEEAQQBTAFUA +VgBPAFIASwA1AEMAWQBJAEkAPQAiACAAdABhAHIAZwBlAHQAPQAiAF8AYgBsAGEAbgBrACIA +PgA8AGIAcgA+ACYAIwAxADMAOwANAAoAPABpAG0AZwAgAHMAcgBjAD0AIgBoAHQAdABwADoA +LwAvAGUAeABhAG0AcABsAGUALgBuAGUAdAAvAGkAbQBhAGcAZQAuAHAAbgBnACIAIAB0AGEA +cgBnAGUAdAA9ACIAXwBiAGwAYQBuAGsAIgA+ADwAYgByAD4AJgAjADEAMwA7AA0ACgBOAGkA +ZgB0AHkAIQA8AC8AcwBwAGEAbgA+AA0ACgA= +--------------7EB9EDC10414C906CC3C2CEA-- diff --git a/modoboa/lib/tests/sample_messages/multipart-output-html_links.txt b/modoboa/lib/tests/sample_messages/multipart-output-html_links.txt new file mode 100644 index 000000000..5c831152f --- /dev/null +++ b/modoboa/lib/tests/sample_messages/multipart-output-html_links.txt @@ -0,0 +1,16 @@ +
+ + + + + +

This is an html test message.

+ :-)
+
+
+ 
-- 
+someone
+someone@example.net
+ +

+��

\ No newline at end of file diff --git a/modoboa/lib/tests/sample_messages/multipart-output-html_nolinks.txt b/modoboa/lib/tests/sample_messages/multipart-output-html_nolinks.txt new file mode 100644 index 000000000..e14a409da --- /dev/null +++ b/modoboa/lib/tests/sample_messages/multipart-output-html_nolinks.txt @@ -0,0 +1,16 @@ +
+ + + + + +

This is an html test message.

+ :-)
+
+
+ 
-- 
+someone
+someone@example.net
+ +

+��

\ No newline at end of file diff --git a/modoboa/lib/tests/sample_messages/multipart-output-plain_nolinks.txt b/modoboa/lib/tests/sample_messages/multipart-output-plain_nolinks.txt new file mode 100644 index 000000000..8795edadc --- /dev/null +++ b/modoboa/lib/tests/sample_messages/multipart-output-plain_nolinks.txt @@ -0,0 +1,7 @@ +
This is an *html* /test /_message._
+
+:-)
+
+-- 
+Someone
+someone@example.net
\ No newline at end of file diff --git a/modoboa/lib/tests/sample_messages/text_plain-input.txt b/modoboa/lib/tests/sample_messages/text_plain-input.txt new file mode 100644 index 000000000..bfbf9aca8 --- /dev/null +++ b/modoboa/lib/tests/sample_messages/text_plain-input.txt @@ -0,0 +1,51 @@ +Return-Path: +X-Original-To: me@example.net Delivered-To: someone@example.net +Received: from mail.example.net (localhost [127.0.0.1]) + by mail.example.net (Postfix) with ESMTP id A142E1AF12D + for ; Sun, 17 Dec 2017 13:10:10 +0000 (GMT) +Authentication-Results: mail.example.net (amavisd-new); + dkim=pass (1024-bit key) reason="pass (just generated, assumed good)" + header.d=example.net +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=example.net; h= + content-transfer-encoding:content-language:content-type + :content-type:mime-version:user-agent:date:date:message-id + :subject:subject:from:from:received:received; s=mail; t= + 1513516210; bh=yyFHETi2jtS7bFMEIArr53VnJMNNjltuXCFTOqs/cL0=; b=X + nMo4pMsaH7EO9A55i0SOv5RGP4Sp4eukaXXQJgMWahcfUn4mM2+pSdc+8ZMaXX7y + ITQdCzI0LL9WGfX7bbMA49hYtIELjQ/AochqABTUsMxnq0Sgf1FE37bA3NS1pBtp + TBKehpGjr86y0V247Okmf2faEkO9dMvmwWz8QBI0N4= +X-Virus-Scanned: amavisd-new at example.net +X-Spam-Flag: NO +X-Spam-Score: -0.999 +X-Spam-Level: +X-Spam-Status: No, score=-0.999 tagged_above=-999 required=3.9 + tests=[ALL_TRUSTED=-1, URIBL_BLOCKED=0.001] + autolearn=no autolearn_force=no +Received: from mail.example.net ([127.0.0.1]) + by mail.example.net (mail.example.net [127.0.0.1]) (amavisd-new, port 10026) + with LMTP id j0UYMNn7ZprP for ; + Sun, 17 Dec 2017 13:10:10 +0000 (GMT) +Received: from [10.0.0.1] (my-pc.example.net [10.0.0.1]) + by mail.example.net (Postfix) with ESMTPSA id E66201AF132 + for ; Sun, 17 Dec 2017 13:10:09 +0000 (GMT) +To: Me +From: Someone +X-Clacks-Overhead: GNU Terry Pratchett +Subject: =?UTF-8?Q?T=c3=a9st_message?= +Message-ID: <8626caea-4c05-8dc5-c687-bcfb7a329d18@example.net> +Date: Sun, 17 Dec 2017 13:10:14 +0000 +User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 + Thunderbird/52.5.0 +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8; format=flowed +Content-Language: en-GB +Content-Transfer-Encoding: 7bit +X-rAnDoM-cAsE-hEaDeR: fOoBaR +X-UPPER-CASE-HEADER: FOOBAR +x-lower-case-header: foobar + +This is a plain text e-mail message. + +-- +Someone +someone@example.net diff --git a/modoboa/lib/tests/sample_messages/text_plain-output-plain_nolinks.txt b/modoboa/lib/tests/sample_messages/text_plain-output-plain_nolinks.txt new file mode 100644 index 000000000..e027961d8 --- /dev/null +++ b/modoboa/lib/tests/sample_messages/text_plain-output-plain_nolinks.txt @@ -0,0 +1,5 @@ +
This is a plain text e-mail message.
+
+-- 
+Someone
+someone@example.net
\ No newline at end of file diff --git a/modoboa/lib/tests/test_email_utils.py b/modoboa/lib/tests/test_email_utils.py new file mode 100644 index 000000000..e3e267059 --- /dev/null +++ b/modoboa/lib/tests/test_email_utils.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +"""Tests for email_utils.""" + +from __future__ import unicode_literals + +import os + +from django.test import SimpleTestCase +from django.utils.encoding import smart_bytes, smart_text + +from ..email_utils import Email + +SAMPLES_DIR = os.path.realpath( + os.path.join(os.path.dirname(__file__), "sample_messages")) + + +class EmailTestImplementation(Email): + + def _fetch_message(self): + message_path = os.path.join(SAMPLES_DIR, "%s-input.txt" % self.mailid) + assert os.path.isfile(message_path), "%s does not exist." % message_path + + with open(message_path, "rb") as fp: + mail_text = smart_bytes(fp.read()) + + return mail_text + + +class EmailTests(SimpleTestCase): + """Tests for modoboa.lib.email_utils.Email + + When writing new sample messages use the following naming convention for + the sample files stored in sample_messages: + + input: {message_id}-input.txt + output: {message_id}-output-{dformat}_{no,}links.txt + """ + + def _get_expected_output(self, message_id, **kwargs): + ext = kwargs["dformat"] if "dformat" in kwargs else "plain" + ext += "_links" if "links" in kwargs and kwargs["links"] else "_nolinks" + message_path = os.path.join(SAMPLES_DIR, + "%s-output-%s.txt" % (message_id, ext)) + assert os.path.isfile(message_path), "%s does not exist." % message_path + + with open(message_path, "rb") as fp: + # output should always be unicode (py2) or str (py3) + mail_text = smart_text(fp.read()) + + return mail_text + + def _test_email(self, message_id, **kwargs): + """Boiler plate code for testing e-mails.""" + expected_output = self._get_expected_output(message_id, **kwargs) + output = EmailTestImplementation(message_id, **kwargs).body + self.assertEqual(output, expected_output) + + def test_invalid_links_value(self): + """modoboa-amavis set links = "0"; in python "0" == True, fixed in PR 79 + https://github.com/modoboa/modoboa-amavis/pull/79""" + with self.assertRaises(TypeError) as cm: + EmailTestImplementation("text_plain", links="0") + + ex_message = cm.exception.messages + self.assertEqual(ex_message, + "links == \"0\" is not valid, did you mean True or " + "False?") + + def test_links_value_for_webmail(self): + """modoboa-webmail sets links = 0 or 1 + TODO: this should be fixed in modoboa-webmail to use True/False""" + email = EmailTestImplementation("text_plain", links=0) + self.assertFalse(email.links) + + email = EmailTestImplementation("text_plain", links=1) + self.assertTrue(email.links) + + def test_headers(self): + email = EmailTestImplementation("text_plain") + # test header names are case insensative + self.assertEqual(email.msg["x-upper-case-header"], "FOOBAR") + self.assertEqual(email.msg["X-Lower-Case-Header"], "foobar") + self.assertEqual(email.msg["x-random-case-header"], "fOoBaR") + + expected_output = [ + {"name": "From", "value": "Someone "}, + {"name": "To", "value": "Me "}, + {"name": "Cc", "value": ""}, + {"name": "Date", "value": "Sun, 17 Dec 2017 13:10:14 +0000"}, + {"name": "Subject", "value": "Tést message"}, + ] + self.assertEqual(email.headers, expected_output) + + def test_email_text_plain(self): + self._test_email("text_plain") + + def test_email_multipart_as_text(self): + """display the text/plain part of a multipart message""" + self._test_email("multipart") + + def test_email_multipart_with_links(self): + """display the text/html part of a multipart message""" + self._test_email("multipart", dformat="html", links=True) + + def test_email_multipart_without_links(self): + """display the text/html part of a multipart message with links removed""" + self._test_email("multipart", dformat="html", links=False) diff --git a/modoboa/lib/tests/test_u2u_decode.py b/modoboa/lib/tests/test_u2u_decode.py new file mode 100644 index 000000000..b06bf38c6 --- /dev/null +++ b/modoboa/lib/tests/test_u2u_decode.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +"""Tests for u2u_decode.""" + +from __future__ import unicode_literals + +from django.test import TestCase + +from .. import u2u_decode + + +class U2UTestCase(TestCase): + """Test RFC1342 decoding utilities.""" + + def test_header_decoding(self): + """Simple decoding.""" + samples = [ + ("=?ISO-8859-15?Q?=20Profitez de tous les services en ligne sur " + "impots.gouv.fr?=", + "Profitez de tous les services en ligne sur impots.gouv.fr"), + ("=?ISO-8859-1?Q?Accus=E9?= de =?ISO-8859-1?Q?r=E9ception?= de " + "votre annonce", + "Accusé de réception de votre annonce"), + ("Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord", + "Sm\xf6rg\xe5sbord"), + # The following case currently fails because of the way we split + # encoded words to parse them separately, which can lead to + # unexpected unicode decode errors... I think it will work fine on + # Python3 + # ("=?utf-8?B?VMOpbMOpcMOpYWdlIFZJTkNJIEF1dG9yb3V0ZXMgLSBFeHDD?=\n" + # "=?utf-8?B?qWRpdGlvbiBkZSB2b3RyZSBjb21tYW5kZSBOwrAgMjAxNzEyMDcw" + # "MDA1?=\n=?utf-8?B?MyBkdSAwNy8xMi8yMDE3IDE0OjQ5OjQx?=", + # "") + ] + for sample in samples: + self.assertEqual(u2u_decode.u2u_decode(sample[0]), sample[1]) + + def test_address_header_decoding(self): + """Check address decoding.""" + mailsploit_sample = ( + "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=00?=" + "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com") + expected_result = ( + "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??=" + "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com") + self.assertEqual( + u2u_decode.decode_address(mailsploit_sample), + ("", expected_result) + ) + mailsploit_sample = ( + '"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?="\n' + '<=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?=' + '@mailsploit.com>') + expected_result = ( + 'potus@whitehouse.gov', + '=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??=' + '@mailsploit.com') + self.assertEqual( + u2u_decode.decode_address(mailsploit_sample), + expected_result) diff --git a/requirements.txt b/requirements.txt index 94712f9db..74e0f4af7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ cryptography pytz requests rfc6266 +lxml