From 161570de7011e7d251d88e6d72f555ff463d464f Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Thu, 21 Dec 2017 12:18:39 +0000 Subject: [PATCH 1/3] split existing tests into tests directory --- modoboa/lib/{tests.py => tests/__init__.py} | 65 +++------------------ modoboa/lib/tests/test_u2u_decode.py | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+), 58 deletions(-) rename modoboa/lib/{tests.py => tests/__init__.py} (55%) create mode 100644 modoboa/lib/tests/test_u2u_decode.py diff --git a/modoboa/lib/tests.py b/modoboa/lib/tests/__init__.py similarity index 55% rename from modoboa/lib/tests.py rename to modoboa/lib/tests/__init__.py index e8c963af7..ace3d977e 100644 --- a/modoboa/lib/tests.py +++ b/modoboa/lib/tests/__init__.py @@ -1,4 +1,5 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- + """Testing utilities.""" from __future__ import unicode_literals @@ -13,8 +14,7 @@ from rest_framework.test import APITestCase from modoboa.core import models as core_models -from . import sysutils -from . import u2u_decode +from .. import sysutils try: s = socket.create_connection(('127.0.0.1', 25)) @@ -24,7 +24,7 @@ except socket.error: NO_SMTP = True try: - import ldap # NOQA + import ldap # noqa NO_LDAP = False except ImportError: NO_LDAP = True @@ -34,7 +34,7 @@ class ParametersMixin(object): """Add tools to manage parameters.""" @classmethod - def setUpTestData(cls): + def setUpTestData(cls): # noqa """Set LocalConfig instance.""" super(ParametersMixin, cls).setUpTestData() cls.localconfig = core_models.LocalConfig.objects.first() @@ -58,7 +58,7 @@ class ModoTestCase(ParametersMixin, TestCase): """All test cases must inherit from this one.""" @classmethod - def setUpTestData(cls): + def setUpTestData(cls): # noqa """Create a default user.""" super(ModoTestCase, cls).setUpTestData() management.call_command("load_initial_data") @@ -93,7 +93,7 @@ class ModoAPITestCase(ParametersMixin, APITestCase): """All test cases must inherit from this one.""" @classmethod - def setUpTestData(cls): + def setUpTestData(cls): # noqa """Create a default user.""" super(ModoAPITestCase, cls).setUpTestData() management.call_command("load_initial_data") @@ -104,54 +104,3 @@ class ModoAPITestCase(ParametersMixin, APITestCase): """Setup.""" super(ModoAPITestCase, self).setUp() self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token.key) - - -class U2UTestCase(TestCase): - """Test RFC1342 decoding utilities.""" - - def test_header_decoding(self): - """Simple decoding.""" - samples = [ - ("=?ISO-8859-15?Q?=20Profitez de tous les services en ligne sur " - "impots.gouv.fr?=", - "Profitez de tous les services en ligne sur impots.gouv.fr"), - ("=?ISO-8859-1?Q?Accus=E9?= de =?ISO-8859-1?Q?r=E9ception?= de " - "votre annonce", - "Accusé de réception de votre annonce"), - ("Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord", - "Sm\xf6rg\xe5sbord"), - # The following case currently fails because of the way we split - # encoded words to parse them separately, which can lead to - # unexpected unicode decode errors... I think it will work fine on - # Python3 - # ("=?utf-8?B?VMOpbMOpcMOpYWdlIFZJTkNJIEF1dG9yb3V0ZXMgLSBFeHDD?=\n" - # "=?utf-8?B?qWRpdGlvbiBkZSB2b3RyZSBjb21tYW5kZSBOwrAgMjAxNzEyMDcw" - # "MDA1?=\n=?utf-8?B?MyBkdSAwNy8xMi8yMDE3IDE0OjQ5OjQx?=", - # "") - ] - for sample in samples: - self.assertEqual(u2u_decode.u2u_decode(sample[0]), sample[1]) - - def test_address_header_decoding(self): - """Check address decoding.""" - mailsploit_sample = ( - "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=00?=" - "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com") - expected_result = ( - "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??=" - "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com") - self.assertEqual( - u2u_decode.decode_address(mailsploit_sample), - ("", expected_result) - ) - mailsploit_sample = ( - '"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?="\n' - '<=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?=' - '@mailsploit.com>') - expected_result = ( - 'potus@whitehouse.gov', - '=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??=' - '@mailsploit.com') - self.assertEqual( - u2u_decode.decode_address(mailsploit_sample), - expected_result) diff --git a/modoboa/lib/tests/test_u2u_decode.py b/modoboa/lib/tests/test_u2u_decode.py new file mode 100644 index 000000000..b06bf38c6 --- /dev/null +++ b/modoboa/lib/tests/test_u2u_decode.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +"""Tests for u2u_decode.""" + +from __future__ import unicode_literals + +from django.test import TestCase + +from .. import u2u_decode + + +class U2UTestCase(TestCase): + """Test RFC1342 decoding utilities.""" + + def test_header_decoding(self): + """Simple decoding.""" + samples = [ + ("=?ISO-8859-15?Q?=20Profitez de tous les services en ligne sur " + "impots.gouv.fr?=", + "Profitez de tous les services en ligne sur impots.gouv.fr"), + ("=?ISO-8859-1?Q?Accus=E9?= de =?ISO-8859-1?Q?r=E9ception?= de " + "votre annonce", + "Accusé de réception de votre annonce"), + ("Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord", + "Sm\xf6rg\xe5sbord"), + # The following case currently fails because of the way we split + # encoded words to parse them separately, which can lead to + # unexpected unicode decode errors... I think it will work fine on + # Python3 + # ("=?utf-8?B?VMOpbMOpcMOpYWdlIFZJTkNJIEF1dG9yb3V0ZXMgLSBFeHDD?=\n" + # "=?utf-8?B?qWRpdGlvbiBkZSB2b3RyZSBjb21tYW5kZSBOwrAgMjAxNzEyMDcw" + # "MDA1?=\n=?utf-8?B?MyBkdSAwNy8xMi8yMDE3IDE0OjQ5OjQx?=", + # "") + ] + for sample in samples: + self.assertEqual(u2u_decode.u2u_decode(sample[0]), sample[1]) + + def test_address_header_decoding(self): + """Check address decoding.""" + mailsploit_sample = ( + "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=00?=" + "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com") + expected_result = ( + "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??=" + "=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com") + self.assertEqual( + u2u_decode.decode_address(mailsploit_sample), + ("", expected_result) + ) + mailsploit_sample = ( + '"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?="\n' + '<=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?=' + '@mailsploit.com>') + expected_result = ( + 'potus@whitehouse.gov', + '=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??=' + '@mailsploit.com') + self.assertEqual( + u2u_decode.decode_address(mailsploit_sample), + expected_result) From c984596ae9f9301c3898c2145a37b4b2bad4599d Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Sat, 16 Dec 2017 00:54:17 +0000 Subject: [PATCH 2/3] improve modoboa.lib.email_utils.Email This is a refactor of moboboa.lib.email_utils.Email, currently it has trouble parsing messages containing certain attachment types. For html messages inline images are now base64 encoded and embedded into the tag rather than being saved to disk. This only effects modoboa-amavis as modoboa-webmail implements it's own parser for e-mails as they are retrieved differently using IMAP. This adds a new external dependency on lxml. --- modoboa/lib/email_utils.py | 350 ++++++++++++++++++++----------------- requirements.txt | 1 + 2 files changed, 194 insertions(+), 157 deletions(-) diff --git a/modoboa/lib/email_utils.py b/modoboa/lib/email_utils.py index 97dbb3c00..1a16b6062 100644 --- a/modoboa/lib/email_utils.py +++ b/modoboa/lib/email_utils.py @@ -1,22 +1,29 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- from __future__ import unicode_literals -from email.header import Header, decode_header +import email +from email.header import Header from email.mime.text import MIMEText from email.utils import make_msgid, formatdate, parseaddr import re import smtplib import time -from django.conf import settings -from django.utils.encoding import smart_text -from django.utils.html import conditional_escape +from django.utils.encoding import smart_str, smart_text +from django.utils.html import conditional_escape, escape from django.template.loader import render_to_string -from django.utils import six from modoboa.lib import u2u_decode +import lxml.html +from lxml.html.clean import Cleaner + + +# used by Email() +_RE_REMOVE_EXTRA_WHITESPACE = re.compile(r"\n\s*\n") +_RE_CID = re.compile(r".*[ ]*cid=\"([^\"]*)\".*", re.I) + class EmailAddress(object): @@ -33,107 +40,100 @@ class EmailAddress(object): class Email(object): - def __init__(self, mailid, mformat="plain", dformat="plain", links=0): - self.attached_map = {} + _basic_headers = ( + "From", + "To", + "Cc", + "Date", + "Subject", + ) + + _image_mime_types = ( + "image/png", + "image/gif", + "image/jpeg", + ) + + def __init__(self, mailid, mformat="plain", dformat="plain", links=False): self.contents = {"html": "", "plain": ""} - self.headers = [] - self.attachments = {} self.mailid = mailid self.mformat = mformat self.dformat = dformat - self.links = links + + if isinstance(links, bool): + self.links = links + elif isinstance(links, int): + # TODO: modoboa-webmail currently sets links == 0 or 1, it needs + # updated to use True or False. + self.links = bool(links) + elif links == "0": + raise TypeError("links == \"0\" is not valid, did you mean True or " + "False?") + else: + raise TypeError("links should be a boolean value") + self._msg = None + self._headers = None self._body = None + self._images = {} + + # used by modoboa_webmail.lib.imapemail.ImapEmail + self.attachments = {} @property def msg(self): - """Return an email.message object. - """ - raise NotImplementedError + if self._msg is None: + mail_text = self._fetch_message() + assert isinstance(mail_text, bytes) + # python 2 expects str (bytes), python 3 expects str (unicode) + mail_text = smart_str(mail_text, errors="replace") + self._msg = email.message_from_string(mail_text) + + return self._msg + + @property + def headers(self): + if self._headers is None: + self._headers = [] + + # don't break modoboa_webmail.lib.imapemail.ImapEmail, it + # doesn't have self._msg it loads e-mails (and their headers) + # from the IMAP server. + if self._msg is not None: + for header in self._basic_headers: + value = self.get_header(self.msg, header) + self._headers.append({ + "name": header, + "value": value + }) + + return self._headers + + @headers.setter + def headers(self, value): + self._headers = value @property def body(self): - """Return email's body.""" if self._body is None: - self._body = ( - getattr(self, "viewmail_%s" % self.mformat) - (self.contents[self.mformat], links=self.links) - ) + self._parse(self.msg) + self._body = getattr(self, "viewmail_%s" % self.dformat)() + return self._body - def get_header(self, msg, hdrname): - """Look for a particular header. - - :param string hdrname: header name - :return: header avalue - """ - for name in [hdrname, hdrname.upper()]: - if name in msg: - return msg[name] + def get_header(self, msg, header): + # msg parameter to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if header in msg: + return "".join([smart_text(v, encoding=(e or 'ascii')) + for v, e in email.header.decode_header(msg[header]) + ]) return "" - def _parse_default(self, msg, level): - """Default parser + def _fetch_message(self): + raise NotImplementedError() - All parts handled by this parser will be consireded as - attachments. - """ - fname = msg.get_filename() - if fname is not None: - if isinstance(fname, six.text_type): - fname = fname.encode("utf-8") - decoded = decode_header(fname) - value = ( - decoded[0][0] if decoded[0][1] is None - else smart_text(decoded[0][0], decoded[0][1]) - ) - else: - value = "part_%s" % level - self.attachments[level] = value - - def _parse_text(self, msg, level): - """Displayable content parser - - text, html, calendar, etc. All those contents can be displayed - inside a navigator. - """ - if msg.get_content_subtype() not in ["plain", "html"]: - self._parse_default(msg, level) - target = "plain" - else: - target = msg.get_content_subtype() - self.contents[target] += decode(msg.get_payload(decode=True), - charset=msg.get_content_charset()) - - def _parse_image(self, msg, level): - """image/* parser - - The only reason to make a specific parser for images is that, - sometimes, messages embark inline images, which means they - must be displayed and not attached. - """ - if self.dformat == "html" and self.links \ - and "Content-Disposition" in msg: - if msg["Content-Disposition"].startswith("inline"): - cid = None - if "Content-ID" in msg: - m = re.match("<(.+)>", msg["Content-ID"]) - cid = m is not None and m.group(1) or msg["Content-ID"] - fname = msg.get_filename() - if fname is None: - if "Content-Location" in msg: - fname = msg["Content-Location"] - elif cid is not None: - fname = cid - else: - # I give up for now :p - return - self.attached_map[cid] = re.match("^http:", fname) and fname \ - or self.__save_image(fname, msg) - return - self._parse_default(msg, level) - - def _parse(self, msg, level=None): + def _parse(self, msg, level=0): """Recursive email parser A message structure can be complex. To correctly handle @@ -146,85 +146,121 @@ class Email(object): :param msg: message (or part) to parse :param level: current part number """ - if msg.is_multipart() and msg.get_content_maintype() != "message": - cpt = 1 - for part in msg.get_payload(): - nlevel = level is None and ("%d" % cpt) \ - or "%s.%d" % (level, cpt) - self._parse(part, nlevel) - cpt += 1 + content_type = msg.get_content_maintype() + is_attachment = "attachment" in msg.get("Content-Disposition", "") + + if content_type == "text" and not is_attachment: + self._parse_text(msg, level) + elif content_type == "multipart" and not is_attachment: + self._parse_multipart(msg, level) + else: + self._parse_inline_image(msg, level) + + if self.contents["plain"]: + self._post_process_plain() + if self.contents["html"]: + self._post_process_html() + + def _parse_text(self, msg, level=0): + content_type = msg.get_content_subtype() + + if content_type in ["plain", "html"]: + encoding = msg.get_content_charset() or "utf-8" + mail_text = smart_text(msg.get_payload(decode=True), encoding=encoding, errors="replace") + self.contents[content_type] += mail_text + + def _parse_multipart(self, msg, level=0): + level += 1 + for part in msg.walk(): + content_type = part.get_content_maintype() + if content_type == "text": + self._parse_text(part, level=level) + else: + # I'm a dumb mail parser and treat all non-text parts as attachments + self._parse_inline_image(part, level=level) + + def _parse_inline_image(self, msg, level=0): + content_type = msg.get_content_type() + cid = None + if "Content-ID" in msg: + cid = msg["Content-ID"] + if cid.startswith('<') and cid.endswith('>'): + cid = cid[1:-1] + else: + matches = _RE_CID.match(msg["Content-Type"]) + if matches is not None: + cid = matches.group(1) + transfer_encoding = msg["Content-Transfer-Encoding"] + if content_type not in self._image_mime_types or cid is None: + # I'm a dumb mail parser and I don't care ;p + return + if transfer_encoding != "base64": + # I don't deal with non base64 encoded images + return + if cid in self._images: + # Duplicate Content-ID return - if level is None: - level = "1" - try: - getattr(self, "_parse_%s" % msg.get_content_maintype())(msg, level) - except AttributeError: - self._parse_default(msg, level) + self._images[cid] = "data:%s;base64,%s" % ( + content_type, "".join(msg.get_payload().splitlines(False))) - def __save_image(self, fname, part): - """Save an inline image on the filesystem. + def _post_process_plain(self): + mail_text = _RE_REMOVE_EXTRA_WHITESPACE \ + .sub("\n\n", self.contents["plain"]).strip() + mail_text = escape(mail_text) + self.contents["plain"] = smart_text(mail_text) - Some HTML messages are using inline images (attached images - with links on them inside the body). In order to display them, - images are saved on the filesystem and links contained in the - message are modified. + def _post_process_html(self): + html = lxml.html.fromstring(self.contents["html"]) - :param fname: the image associated filename - :param part: the email part that contains the image payload - """ - if re.search(r"\.\.", fname): - return None - path = "/static/tmp/" + fname - fp = open(settings.MODOBOA_DIR + path, "wb") - fp.write(part.get_payload(decode=True)) - fp.close() - return path + if self.links: + html.rewrite_links(self._map_cid) - def map_cid(self, url): - """Replace attachment links. + for link in html.iterlinks(): + link[0].set("target", "_blank") + else: + html.rewrite_links(lambda x: None) + + cleaner = Cleaner( + scripts=True, + javascript=True, + links=True, + page_structure=True, + embedded=True, + frames=True, + add_nofollow=True) + mail_text = lxml.html.tostring(cleaner.clean_html(html)) + self.contents["html"] = smart_text(mail_text) + + def _map_cid(self, url): + if url.startswith("cid:"): + cid = url[4:] + if cid in self._images: + url = self._images[cid] - :param str url: original url - :rtype: string - :return: internal link - """ - match = re.match(".*cid:(.+)", url) - if match is not None: - if match.group(1) in self.attached_map: - return self.attached_map[match.group(1)] return url + def viewmail_plain(self, contents=None, **kwargs): + # contents and **kwargs parameters to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if contents is None: + contents = self.contents["plain"] + else: + contents = conditional_escape(contents) + return "
%s
" % contents + + def viewmail_html(self, contents=None, **kwargs): + # contents and **kwargs parameters to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if contents is None: + contents = self.contents["html"] + return contents + def render_headers(self, **kwargs): - return render_to_string("common/mailheaders.html", { + context = { "headers": self.headers, - }) - - def viewmail_plain(self, content, **kwargs): - content = conditional_escape(content) - return "
%s
" % content - - def viewmail_html(self, content, **kwargs): - import lxml.html - - if content is None or content == "": - return "" - links = kwargs.get("links", 0) - html = lxml.html.fromstring(content) - if not links: - html.rewrite_links(lambda x: None) - else: - html.rewrite_links(self.map_cid) - for link in html.iterlinks(): - link[0].set('target', '_blank') - body = html.find("body") - if body is None: - body = lxml.html.tostring(html) - else: - body = lxml.html.tostring(body) - body = re.sub( - "<(/?)body", lambda m: "<%sdiv" % m.group(1), - body.decode()) - return body + } + return render_to_string("common/mailheaders.html", context) def split_mailbox(mailbox, return_extension=False): diff --git a/requirements.txt b/requirements.txt index 94712f9db..74e0f4af7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ cryptography pytz requests rfc6266 +lxml From ea0cf90d4826245d2af87f26412515d36b3edfa3 Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Sat, 16 Dec 2017 18:48:39 +0000 Subject: [PATCH 3/3] add tests for modoboa.lib.email_utils.Email --- .../tests/sample_messages/multipart-input.txt | 130 ++++++++++++++++++ .../multipart-output-html_links.txt | 16 +++ .../multipart-output-html_nolinks.txt | 16 +++ .../multipart-output-plain_nolinks.txt | 7 + .../sample_messages/text_plain-input.txt | 51 +++++++ .../text_plain-output-plain_nolinks.txt | 5 + modoboa/lib/tests/test_email_utils.py | 108 +++++++++++++++ 7 files changed, 333 insertions(+) create mode 100644 modoboa/lib/tests/sample_messages/multipart-input.txt create mode 100644 modoboa/lib/tests/sample_messages/multipart-output-html_links.txt create mode 100644 modoboa/lib/tests/sample_messages/multipart-output-html_nolinks.txt create mode 100644 modoboa/lib/tests/sample_messages/multipart-output-plain_nolinks.txt create mode 100644 modoboa/lib/tests/sample_messages/text_plain-input.txt create mode 100644 modoboa/lib/tests/sample_messages/text_plain-output-plain_nolinks.txt create mode 100644 modoboa/lib/tests/test_email_utils.py diff --git a/modoboa/lib/tests/sample_messages/multipart-input.txt b/modoboa/lib/tests/sample_messages/multipart-input.txt new file mode 100644 index 000000000..ee0b51430 --- /dev/null +++ b/modoboa/lib/tests/sample_messages/multipart-input.txt @@ -0,0 +1,130 @@ +Return-Path: +X-Original-To: someone@example.net +Delivered-To: me@example.net +Received: from mail.example.net (localhost [127.0.0.1]) + by mail.example.net (Postfix) with ESMTP id 4BD531AF130 + for ; Sun, 17 Dec 2017 03:11:30 +0000 (GMT) +Authentication-Results: mail.example.net (amavisd-new); + dkim=pass (1024-bit key) reason="pass (just generated, assumed good)" + header.d=example.net +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=example.net; h= + content-language:content-type:content-type:mime-version + :user-agent:date:date:message-id:subject:subject:from:from + :received:received; s=mail; t=1513480289; bh=wAmt0wg2Yk04321B + Q63uWL5f67PEBB2EEGONiFUrIvk=; b=UZUzPqH8TUk/bMZm1ofkjwatNCQftZKK + C91uIuX3mmjvngNwbVjx4m7jFZItZscNqigyGJN5HCOl8tC/LJmmWzRdhIisFtLA + uXRiBuWps82rbV4ciFU8N30i9UXvtWvMywwL6pYSESqixiwylzbddjMjKXXdyqaT + DTyR16ZK8IE= +X-Virus-Scanned: amavisd-new at example.net +X-Spam-Flag: NO +X-Spam-Score: -0.988 +X-Spam-Level: +X-Spam-Status: No, score=-0.988 tagged_above=-999 required=3.9 + tests=[ALL_TRUSTED=-1, HTML_MESSAGE=0.001, T_HTML_ATTACH=0.01, + URIBL_BLOCKED=0.001] autolearn=no autolearn_force=no +Received: from mail.example.net ([127.0.0.1]) + by mail.example.net (mail.example.net [127.0.0.1]) (amavisd-new, port 10026) + with LMTP id BSCmm59DxNvB for ; + Sun, 17 Dec 2017 03:11:29 +0000 (GMT) +Received: from [10.0.0.1] (my-pc.example.net [10.0.0.1]) + by mail.example.net (Postfix) with ESMTPSA id 641171AF12D + for ; Sun, 17 Dec 2017 03:11:29 +0000 (GMT) +To: Someone +From: Me +X-Clacks-Overhead: GNU Terry Pratchett +Subject: A test message +Message-ID: <5294ad7a-e966-f1a0-3431-8c4b1b76f5a3@example.net> +Date: Sun, 17 Dec 2017 03:11:33 +0000 +User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 + Thunderbird/52.5.0 +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="------------7EB9EDC10414C906CC3C2CEA" +Content-Language: en-GB + +This is a multi-part message in MIME format. +--------------7EB9EDC10414C906CC3C2CEA +Content-Type: multipart/alternative; + boundary="------------87B289699A48D5D0D61BABCC" + + +--------------87B289699A48D5D0D61BABCC +Content-Type: text/plain; charset=utf-8; format=flowed +Content-Transfer-Encoding: 7bit + +This is an *html* /test /_message._ + +:-) + + +-- +Someone +someone@example.net + + +--------------87B289699A48D5D0D61BABCC +Content-Type: multipart/related; + boundary="------------D585B241EEF60F0F44EC20E5" + + +--------------D585B241EEF60F0F44EC20E5 +Content-Type: text/html; charset=utf-8 +Content-Transfer-Encoding: 7bit + + + + + + + +

This is an html test message.

+ :-)
+
+
+ 
-- 
+someone
+someone@example.net
+ + + +--------------D585B241EEF60F0F44EC20E5 +Content-Type: image/png; + name="image.png" +Content-Transfer-Encoding: base64 +Content-ID: +Content-Disposition: inline; + filename="image.png" + +iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAACXBIWXMAAAsTAAALEwEAmpwY +AAAAF0lEQVQY02NgYGD4//8/ThKfHASMRN0AfaqVa6MQxIQAAAAASUVORK5CYII= +--------------D585B241EEF60F0F44EC20E5-- + +--------------87B289699A48D5D0D61BABCC-- + +--------------7EB9EDC10414C906CC3C2CEA +Content-Type: text/html; + name="test.html" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="test.html" + +//48AHMAcABhAG4APgA8AGIAPgBTAG8AbQBlACAAPABpAD4ASABUAE0ATAA8AC8AaQA+ACAA +dABlAHgAdAA8AC8AYgA+ACwAIABhACAAPABhACAAaAByAGUAZgA9ACIAaAB0AHQAcAA6AC8A +LwBlAHgAYQBtAHAAbABlAC4AbgBlAHQAIgAgAHQAYQByAGcAZQB0AD0AIgBfAGIAbABhAG4A +awAiACAAcgBlAGwAPQAiAG4AbwBmAG8AbABsAG8AdwAiAD4AbABpAG4AawA8AC8AYQA+ACwA +IABhAG4AJgAjADEAMwA7AA0ACgBlAG0AYgBlAGQAZABlAGQAIABpAG0AYQBnAGUAIABhAG4A +ZAAgAGEAbgAgAGUAeAB0AGUAcgBuAGEAbAAgAGkAbQBhAGcAZQAuADwAYgByAD4AJgAjADEA +MwA7AA0ACgA8AGkAbQBnACAAcwByAGMAPQAiAGQAYQB0AGEAOgBpAG0AYQBnAGUALwBwAG4A +ZwA7AGIAYQBzAGUANgA0ACwAaQBWAEIATwBSAHcAMABLAEcAZwBvAEEAQQBBAEEATgBTAFUA +aABFAFUAZwBBAEEAQQBBAG8AQQBBAEEAQQBLAEMAQQBJAEEAQQBBAEEAQwBVAEYAagBxAEEA +QQBBAEEAQwBYAEIASQBXAFgATQBBAEEAQQBzAFQAQQBBAEEATABFAHcARQBBAG0AcAB3AFkA +QQBBAEEAQQBGADAAbABFAFEAVgBRAFkAMAAyAE4AZwBZAEcARAA0AC8ALwA4AC8AVABoAEsA +ZgBIAEEAUwBNAFIATgAwAEEAZgBhAHEAVgBhADYATQBRAHgASQBRAEEAQQBBAEEAQQBTAFUA +VgBPAFIASwA1AEMAWQBJAEkAPQAiACAAdABhAHIAZwBlAHQAPQAiAF8AYgBsAGEAbgBrACIA +PgA8AGIAcgA+ACYAIwAxADMAOwANAAoAPABpAG0AZwAgAHMAcgBjAD0AIgBoAHQAdABwADoA +LwAvAGUAeABhAG0AcABsAGUALgBuAGUAdAAvAGkAbQBhAGcAZQAuAHAAbgBnACIAIAB0AGEA +cgBnAGUAdAA9ACIAXwBiAGwAYQBuAGsAIgA+ADwAYgByAD4AJgAjADEAMwA7AA0ACgBOAGkA +ZgB0AHkAIQA8AC8AcwBwAGEAbgA+AA0ACgA= +--------------7EB9EDC10414C906CC3C2CEA-- diff --git a/modoboa/lib/tests/sample_messages/multipart-output-html_links.txt b/modoboa/lib/tests/sample_messages/multipart-output-html_links.txt new file mode 100644 index 000000000..5c831152f --- /dev/null +++ b/modoboa/lib/tests/sample_messages/multipart-output-html_links.txt @@ -0,0 +1,16 @@ +
+ + + + + +

This is an html test message.

+ :-)
+
+
+ 
-- 
+someone
+someone@example.net
+ +

+��

\ No newline at end of file diff --git a/modoboa/lib/tests/sample_messages/multipart-output-html_nolinks.txt b/modoboa/lib/tests/sample_messages/multipart-output-html_nolinks.txt new file mode 100644 index 000000000..e14a409da --- /dev/null +++ b/modoboa/lib/tests/sample_messages/multipart-output-html_nolinks.txt @@ -0,0 +1,16 @@ +
+ + + + + +

This is an html test message.

+ :-)
+
+
+ 
-- 
+someone
+someone@example.net
+ +

+��

\ No newline at end of file diff --git a/modoboa/lib/tests/sample_messages/multipart-output-plain_nolinks.txt b/modoboa/lib/tests/sample_messages/multipart-output-plain_nolinks.txt new file mode 100644 index 000000000..8795edadc --- /dev/null +++ b/modoboa/lib/tests/sample_messages/multipart-output-plain_nolinks.txt @@ -0,0 +1,7 @@ +
This is an *html* /test /_message._
+
+:-)
+
+-- 
+Someone
+someone@example.net
\ No newline at end of file diff --git a/modoboa/lib/tests/sample_messages/text_plain-input.txt b/modoboa/lib/tests/sample_messages/text_plain-input.txt new file mode 100644 index 000000000..bfbf9aca8 --- /dev/null +++ b/modoboa/lib/tests/sample_messages/text_plain-input.txt @@ -0,0 +1,51 @@ +Return-Path: +X-Original-To: me@example.net Delivered-To: someone@example.net +Received: from mail.example.net (localhost [127.0.0.1]) + by mail.example.net (Postfix) with ESMTP id A142E1AF12D + for ; Sun, 17 Dec 2017 13:10:10 +0000 (GMT) +Authentication-Results: mail.example.net (amavisd-new); + dkim=pass (1024-bit key) reason="pass (just generated, assumed good)" + header.d=example.net +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=example.net; h= + content-transfer-encoding:content-language:content-type + :content-type:mime-version:user-agent:date:date:message-id + :subject:subject:from:from:received:received; s=mail; t= + 1513516210; bh=yyFHETi2jtS7bFMEIArr53VnJMNNjltuXCFTOqs/cL0=; b=X + nMo4pMsaH7EO9A55i0SOv5RGP4Sp4eukaXXQJgMWahcfUn4mM2+pSdc+8ZMaXX7y + ITQdCzI0LL9WGfX7bbMA49hYtIELjQ/AochqABTUsMxnq0Sgf1FE37bA3NS1pBtp + TBKehpGjr86y0V247Okmf2faEkO9dMvmwWz8QBI0N4= +X-Virus-Scanned: amavisd-new at example.net +X-Spam-Flag: NO +X-Spam-Score: -0.999 +X-Spam-Level: +X-Spam-Status: No, score=-0.999 tagged_above=-999 required=3.9 + tests=[ALL_TRUSTED=-1, URIBL_BLOCKED=0.001] + autolearn=no autolearn_force=no +Received: from mail.example.net ([127.0.0.1]) + by mail.example.net (mail.example.net [127.0.0.1]) (amavisd-new, port 10026) + with LMTP id j0UYMNn7ZprP for ; + Sun, 17 Dec 2017 13:10:10 +0000 (GMT) +Received: from [10.0.0.1] (my-pc.example.net [10.0.0.1]) + by mail.example.net (Postfix) with ESMTPSA id E66201AF132 + for ; Sun, 17 Dec 2017 13:10:09 +0000 (GMT) +To: Me +From: Someone +X-Clacks-Overhead: GNU Terry Pratchett +Subject: =?UTF-8?Q?T=c3=a9st_message?= +Message-ID: <8626caea-4c05-8dc5-c687-bcfb7a329d18@example.net> +Date: Sun, 17 Dec 2017 13:10:14 +0000 +User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 + Thunderbird/52.5.0 +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8; format=flowed +Content-Language: en-GB +Content-Transfer-Encoding: 7bit +X-rAnDoM-cAsE-hEaDeR: fOoBaR +X-UPPER-CASE-HEADER: FOOBAR +x-lower-case-header: foobar + +This is a plain text e-mail message. + +-- +Someone +someone@example.net diff --git a/modoboa/lib/tests/sample_messages/text_plain-output-plain_nolinks.txt b/modoboa/lib/tests/sample_messages/text_plain-output-plain_nolinks.txt new file mode 100644 index 000000000..e027961d8 --- /dev/null +++ b/modoboa/lib/tests/sample_messages/text_plain-output-plain_nolinks.txt @@ -0,0 +1,5 @@ +
This is a plain text e-mail message.
+
+-- 
+Someone
+someone@example.net
\ No newline at end of file diff --git a/modoboa/lib/tests/test_email_utils.py b/modoboa/lib/tests/test_email_utils.py new file mode 100644 index 000000000..e3e267059 --- /dev/null +++ b/modoboa/lib/tests/test_email_utils.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +"""Tests for email_utils.""" + +from __future__ import unicode_literals + +import os + +from django.test import SimpleTestCase +from django.utils.encoding import smart_bytes, smart_text + +from ..email_utils import Email + +SAMPLES_DIR = os.path.realpath( + os.path.join(os.path.dirname(__file__), "sample_messages")) + + +class EmailTestImplementation(Email): + + def _fetch_message(self): + message_path = os.path.join(SAMPLES_DIR, "%s-input.txt" % self.mailid) + assert os.path.isfile(message_path), "%s does not exist." % message_path + + with open(message_path, "rb") as fp: + mail_text = smart_bytes(fp.read()) + + return mail_text + + +class EmailTests(SimpleTestCase): + """Tests for modoboa.lib.email_utils.Email + + When writing new sample messages use the following naming convention for + the sample files stored in sample_messages: + + input: {message_id}-input.txt + output: {message_id}-output-{dformat}_{no,}links.txt + """ + + def _get_expected_output(self, message_id, **kwargs): + ext = kwargs["dformat"] if "dformat" in kwargs else "plain" + ext += "_links" if "links" in kwargs and kwargs["links"] else "_nolinks" + message_path = os.path.join(SAMPLES_DIR, + "%s-output-%s.txt" % (message_id, ext)) + assert os.path.isfile(message_path), "%s does not exist." % message_path + + with open(message_path, "rb") as fp: + # output should always be unicode (py2) or str (py3) + mail_text = smart_text(fp.read()) + + return mail_text + + def _test_email(self, message_id, **kwargs): + """Boiler plate code for testing e-mails.""" + expected_output = self._get_expected_output(message_id, **kwargs) + output = EmailTestImplementation(message_id, **kwargs).body + self.assertEqual(output, expected_output) + + def test_invalid_links_value(self): + """modoboa-amavis set links = "0"; in python "0" == True, fixed in PR 79 + https://github.com/modoboa/modoboa-amavis/pull/79""" + with self.assertRaises(TypeError) as cm: + EmailTestImplementation("text_plain", links="0") + + ex_message = cm.exception.messages + self.assertEqual(ex_message, + "links == \"0\" is not valid, did you mean True or " + "False?") + + def test_links_value_for_webmail(self): + """modoboa-webmail sets links = 0 or 1 + TODO: this should be fixed in modoboa-webmail to use True/False""" + email = EmailTestImplementation("text_plain", links=0) + self.assertFalse(email.links) + + email = EmailTestImplementation("text_plain", links=1) + self.assertTrue(email.links) + + def test_headers(self): + email = EmailTestImplementation("text_plain") + # test header names are case insensative + self.assertEqual(email.msg["x-upper-case-header"], "FOOBAR") + self.assertEqual(email.msg["X-Lower-Case-Header"], "foobar") + self.assertEqual(email.msg["x-random-case-header"], "fOoBaR") + + expected_output = [ + {"name": "From", "value": "Someone "}, + {"name": "To", "value": "Me "}, + {"name": "Cc", "value": ""}, + {"name": "Date", "value": "Sun, 17 Dec 2017 13:10:14 +0000"}, + {"name": "Subject", "value": "Tést message"}, + ] + self.assertEqual(email.headers, expected_output) + + def test_email_text_plain(self): + self._test_email("text_plain") + + def test_email_multipart_as_text(self): + """display the text/plain part of a multipart message""" + self._test_email("multipart") + + def test_email_multipart_with_links(self): + """display the text/html part of a multipart message""" + self._test_email("multipart", dformat="html", links=True) + + def test_email_multipart_without_links(self): + """display the text/html part of a multipart message with links removed""" + self._test_email("multipart", dformat="html", links=False)