mirror of
https://github.com/jwinterm/modoboa.git
synced 2026-11-15 09:23:58 +01:00
Merge pull request #1330 from fyfe/improved-email-parser
Improved email parser
This commit is contained in:
Antoine Nguyen
+193
-157
@@ -1,22 +1,29 @@
|
||||
# coding: utf-8
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from email.header import Header, decode_header
|
||||
import email
|
||||
from email.header import Header
|
||||
from email.mime.text import MIMEText
|
||||
from email.utils import make_msgid, formatdate, parseaddr
|
||||
import re
|
||||
import smtplib
|
||||
import time
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils.encoding import smart_text
|
||||
from django.utils.html import conditional_escape
|
||||
from django.utils.encoding import smart_str, smart_text
|
||||
from django.utils.html import conditional_escape, escape
|
||||
from django.template.loader import render_to_string
|
||||
from django.utils import six
|
||||
|
||||
from modoboa.lib import u2u_decode
|
||||
|
||||
import lxml.html
|
||||
from lxml.html.clean import Cleaner
|
||||
|
||||
|
||||
# used by Email()
|
||||
_RE_REMOVE_EXTRA_WHITESPACE = re.compile(r"\n\s*\n")
|
||||
_RE_CID = re.compile(r".*[ ]*cid=\"([^\"]*)\".*", re.I)
|
||||
|
||||
|
||||
class EmailAddress(object):
|
||||
|
||||
@@ -33,107 +40,100 @@ class EmailAddress(object):
|
||||
|
||||
class Email(object):
|
||||
|
||||
def __init__(self, mailid, mformat="plain", dformat="plain", links=0):
|
||||
self.attached_map = {}
|
||||
_basic_headers = (
|
||||
"From",
|
||||
"To",
|
||||
"Cc",
|
||||
"Date",
|
||||
"Subject",
|
||||
)
|
||||
|
||||
_image_mime_types = (
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/jpeg",
|
||||
)
|
||||
|
||||
def __init__(self, mailid, mformat="plain", dformat="plain", links=False):
|
||||
self.contents = {"html": "", "plain": ""}
|
||||
self.headers = []
|
||||
self.attachments = {}
|
||||
self.mailid = mailid
|
||||
self.mformat = mformat
|
||||
self.dformat = dformat
|
||||
self.links = links
|
||||
|
||||
if isinstance(links, bool):
|
||||
self.links = links
|
||||
elif isinstance(links, int):
|
||||
# TODO: modoboa-webmail currently sets links == 0 or 1, it needs
|
||||
# updated to use True or False.
|
||||
self.links = bool(links)
|
||||
elif links == "0":
|
||||
raise TypeError("links == \"0\" is not valid, did you mean True or "
|
||||
"False?")
|
||||
else:
|
||||
raise TypeError("links should be a boolean value")
|
||||
|
||||
self._msg = None
|
||||
self._headers = None
|
||||
self._body = None
|
||||
self._images = {}
|
||||
|
||||
# used by modoboa_webmail.lib.imapemail.ImapEmail
|
||||
self.attachments = {}
|
||||
|
||||
@property
|
||||
def msg(self):
|
||||
"""Return an email.message object.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
if self._msg is None:
|
||||
mail_text = self._fetch_message()
|
||||
assert isinstance(mail_text, bytes)
|
||||
# python 2 expects str (bytes), python 3 expects str (unicode)
|
||||
mail_text = smart_str(mail_text, errors="replace")
|
||||
self._msg = email.message_from_string(mail_text)
|
||||
|
||||
return self._msg
|
||||
|
||||
@property
|
||||
def headers(self):
|
||||
if self._headers is None:
|
||||
self._headers = []
|
||||
|
||||
# don't break modoboa_webmail.lib.imapemail.ImapEmail, it
|
||||
# doesn't have self._msg it loads e-mails (and their headers)
|
||||
# from the IMAP server.
|
||||
if self._msg is not None:
|
||||
for header in self._basic_headers:
|
||||
value = self.get_header(self.msg, header)
|
||||
self._headers.append({
|
||||
"name": header,
|
||||
"value": value
|
||||
})
|
||||
|
||||
return self._headers
|
||||
|
||||
@headers.setter
|
||||
def headers(self, value):
|
||||
self._headers = value
|
||||
|
||||
@property
|
||||
def body(self):
|
||||
"""Return email's body."""
|
||||
if self._body is None:
|
||||
self._body = (
|
||||
getattr(self, "viewmail_%s" % self.mformat)
|
||||
(self.contents[self.mformat], links=self.links)
|
||||
)
|
||||
self._parse(self.msg)
|
||||
self._body = getattr(self, "viewmail_%s" % self.dformat)()
|
||||
|
||||
return self._body
|
||||
|
||||
def get_header(self, msg, hdrname):
|
||||
"""Look for a particular header.
|
||||
|
||||
:param string hdrname: header name
|
||||
:return: header avalue
|
||||
"""
|
||||
for name in [hdrname, hdrname.upper()]:
|
||||
if name in msg:
|
||||
return msg[name]
|
||||
def get_header(self, msg, header):
|
||||
# msg parameter to maintain compatibility with
|
||||
# modoboa_webmail.lib.imapemail.ImapEmail
|
||||
if header in msg:
|
||||
return "".join([smart_text(v, encoding=(e or 'ascii'))
|
||||
for v, e in email.header.decode_header(msg[header])
|
||||
])
|
||||
return ""
|
||||
|
||||
def _parse_default(self, msg, level):
|
||||
"""Default parser
|
||||
def _fetch_message(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
All parts handled by this parser will be consireded as
|
||||
attachments.
|
||||
"""
|
||||
fname = msg.get_filename()
|
||||
if fname is not None:
|
||||
if isinstance(fname, six.text_type):
|
||||
fname = fname.encode("utf-8")
|
||||
decoded = decode_header(fname)
|
||||
value = (
|
||||
decoded[0][0] if decoded[0][1] is None
|
||||
else smart_text(decoded[0][0], decoded[0][1])
|
||||
)
|
||||
else:
|
||||
value = "part_%s" % level
|
||||
self.attachments[level] = value
|
||||
|
||||
def _parse_text(self, msg, level):
|
||||
"""Displayable content parser
|
||||
|
||||
text, html, calendar, etc. All those contents can be displayed
|
||||
inside a navigator.
|
||||
"""
|
||||
if msg.get_content_subtype() not in ["plain", "html"]:
|
||||
self._parse_default(msg, level)
|
||||
target = "plain"
|
||||
else:
|
||||
target = msg.get_content_subtype()
|
||||
self.contents[target] += decode(msg.get_payload(decode=True),
|
||||
charset=msg.get_content_charset())
|
||||
|
||||
def _parse_image(self, msg, level):
|
||||
"""image/* parser
|
||||
|
||||
The only reason to make a specific parser for images is that,
|
||||
sometimes, messages embark inline images, which means they
|
||||
must be displayed and not attached.
|
||||
"""
|
||||
if self.dformat == "html" and self.links \
|
||||
and "Content-Disposition" in msg:
|
||||
if msg["Content-Disposition"].startswith("inline"):
|
||||
cid = None
|
||||
if "Content-ID" in msg:
|
||||
m = re.match("<(.+)>", msg["Content-ID"])
|
||||
cid = m is not None and m.group(1) or msg["Content-ID"]
|
||||
fname = msg.get_filename()
|
||||
if fname is None:
|
||||
if "Content-Location" in msg:
|
||||
fname = msg["Content-Location"]
|
||||
elif cid is not None:
|
||||
fname = cid
|
||||
else:
|
||||
# I give up for now :p
|
||||
return
|
||||
self.attached_map[cid] = re.match("^http:", fname) and fname \
|
||||
or self.__save_image(fname, msg)
|
||||
return
|
||||
self._parse_default(msg, level)
|
||||
|
||||
def _parse(self, msg, level=None):
|
||||
def _parse(self, msg, level=0):
|
||||
"""Recursive email parser
|
||||
|
||||
A message structure can be complex. To correctly handle
|
||||
@@ -146,85 +146,121 @@ class Email(object):
|
||||
:param msg: message (or part) to parse
|
||||
:param level: current part number
|
||||
"""
|
||||
if msg.is_multipart() and msg.get_content_maintype() != "message":
|
||||
cpt = 1
|
||||
for part in msg.get_payload():
|
||||
nlevel = level is None and ("%d" % cpt) \
|
||||
or "%s.%d" % (level, cpt)
|
||||
self._parse(part, nlevel)
|
||||
cpt += 1
|
||||
content_type = msg.get_content_maintype()
|
||||
is_attachment = "attachment" in msg.get("Content-Disposition", "")
|
||||
|
||||
if content_type == "text" and not is_attachment:
|
||||
self._parse_text(msg, level)
|
||||
elif content_type == "multipart" and not is_attachment:
|
||||
self._parse_multipart(msg, level)
|
||||
else:
|
||||
self._parse_inline_image(msg, level)
|
||||
|
||||
if self.contents["plain"]:
|
||||
self._post_process_plain()
|
||||
if self.contents["html"]:
|
||||
self._post_process_html()
|
||||
|
||||
def _parse_text(self, msg, level=0):
|
||||
content_type = msg.get_content_subtype()
|
||||
|
||||
if content_type in ["plain", "html"]:
|
||||
encoding = msg.get_content_charset() or "utf-8"
|
||||
mail_text = smart_text(msg.get_payload(decode=True), encoding=encoding, errors="replace")
|
||||
self.contents[content_type] += mail_text
|
||||
|
||||
def _parse_multipart(self, msg, level=0):
|
||||
level += 1
|
||||
for part in msg.walk():
|
||||
content_type = part.get_content_maintype()
|
||||
if content_type == "text":
|
||||
self._parse_text(part, level=level)
|
||||
else:
|
||||
# I'm a dumb mail parser and treat all non-text parts as attachments
|
||||
self._parse_inline_image(part, level=level)
|
||||
|
||||
def _parse_inline_image(self, msg, level=0):
|
||||
content_type = msg.get_content_type()
|
||||
cid = None
|
||||
if "Content-ID" in msg:
|
||||
cid = msg["Content-ID"]
|
||||
if cid.startswith('<') and cid.endswith('>'):
|
||||
cid = cid[1:-1]
|
||||
else:
|
||||
matches = _RE_CID.match(msg["Content-Type"])
|
||||
if matches is not None:
|
||||
cid = matches.group(1)
|
||||
transfer_encoding = msg["Content-Transfer-Encoding"]
|
||||
if content_type not in self._image_mime_types or cid is None:
|
||||
# I'm a dumb mail parser and I don't care ;p
|
||||
return
|
||||
if transfer_encoding != "base64":
|
||||
# I don't deal with non base64 encoded images
|
||||
return
|
||||
if cid in self._images:
|
||||
# Duplicate Content-ID
|
||||
return
|
||||
|
||||
if level is None:
|
||||
level = "1"
|
||||
try:
|
||||
getattr(self, "_parse_%s" % msg.get_content_maintype())(msg, level)
|
||||
except AttributeError:
|
||||
self._parse_default(msg, level)
|
||||
self._images[cid] = "data:%s;base64,%s" % (
|
||||
content_type, "".join(msg.get_payload().splitlines(False)))
|
||||
|
||||
def __save_image(self, fname, part):
|
||||
"""Save an inline image on the filesystem.
|
||||
def _post_process_plain(self):
|
||||
mail_text = _RE_REMOVE_EXTRA_WHITESPACE \
|
||||
.sub("\n\n", self.contents["plain"]).strip()
|
||||
mail_text = escape(mail_text)
|
||||
self.contents["plain"] = smart_text(mail_text)
|
||||
|
||||
Some HTML messages are using inline images (attached images
|
||||
with links on them inside the body). In order to display them,
|
||||
images are saved on the filesystem and links contained in the
|
||||
message are modified.
|
||||
def _post_process_html(self):
|
||||
html = lxml.html.fromstring(self.contents["html"])
|
||||
|
||||
:param fname: the image associated filename
|
||||
:param part: the email part that contains the image payload
|
||||
"""
|
||||
if re.search(r"\.\.", fname):
|
||||
return None
|
||||
path = "/static/tmp/" + fname
|
||||
fp = open(settings.MODOBOA_DIR + path, "wb")
|
||||
fp.write(part.get_payload(decode=True))
|
||||
fp.close()
|
||||
return path
|
||||
if self.links:
|
||||
html.rewrite_links(self._map_cid)
|
||||
|
||||
def map_cid(self, url):
|
||||
"""Replace attachment links.
|
||||
for link in html.iterlinks():
|
||||
link[0].set("target", "_blank")
|
||||
else:
|
||||
html.rewrite_links(lambda x: None)
|
||||
|
||||
cleaner = Cleaner(
|
||||
scripts=True,
|
||||
javascript=True,
|
||||
links=True,
|
||||
page_structure=True,
|
||||
embedded=True,
|
||||
frames=True,
|
||||
add_nofollow=True)
|
||||
mail_text = lxml.html.tostring(cleaner.clean_html(html))
|
||||
self.contents["html"] = smart_text(mail_text)
|
||||
|
||||
def _map_cid(self, url):
|
||||
if url.startswith("cid:"):
|
||||
cid = url[4:]
|
||||
if cid in self._images:
|
||||
url = self._images[cid]
|
||||
|
||||
:param str url: original url
|
||||
:rtype: string
|
||||
:return: internal link
|
||||
"""
|
||||
match = re.match(".*cid:(.+)", url)
|
||||
if match is not None:
|
||||
if match.group(1) in self.attached_map:
|
||||
return self.attached_map[match.group(1)]
|
||||
return url
|
||||
|
||||
def viewmail_plain(self, contents=None, **kwargs):
|
||||
# contents and **kwargs parameters to maintain compatibility with
|
||||
# modoboa_webmail.lib.imapemail.ImapEmail
|
||||
if contents is None:
|
||||
contents = self.contents["plain"]
|
||||
else:
|
||||
contents = conditional_escape(contents)
|
||||
return "<pre>%s</pre>" % contents
|
||||
|
||||
def viewmail_html(self, contents=None, **kwargs):
|
||||
# contents and **kwargs parameters to maintain compatibility with
|
||||
# modoboa_webmail.lib.imapemail.ImapEmail
|
||||
if contents is None:
|
||||
contents = self.contents["html"]
|
||||
return contents
|
||||
|
||||
def render_headers(self, **kwargs):
|
||||
return render_to_string("common/mailheaders.html", {
|
||||
context = {
|
||||
"headers": self.headers,
|
||||
})
|
||||
|
||||
def viewmail_plain(self, content, **kwargs):
|
||||
content = conditional_escape(content)
|
||||
return "<pre>%s</pre>" % content
|
||||
|
||||
def viewmail_html(self, content, **kwargs):
|
||||
import lxml.html
|
||||
|
||||
if content is None or content == "":
|
||||
return ""
|
||||
links = kwargs.get("links", 0)
|
||||
html = lxml.html.fromstring(content)
|
||||
if not links:
|
||||
html.rewrite_links(lambda x: None)
|
||||
else:
|
||||
html.rewrite_links(self.map_cid)
|
||||
for link in html.iterlinks():
|
||||
link[0].set('target', '_blank')
|
||||
body = html.find("body")
|
||||
if body is None:
|
||||
body = lxml.html.tostring(html)
|
||||
else:
|
||||
body = lxml.html.tostring(body)
|
||||
body = re.sub(
|
||||
"<(/?)body", lambda m: "<%sdiv" % m.group(1),
|
||||
body.decode())
|
||||
return body
|
||||
}
|
||||
return render_to_string("common/mailheaders.html", context)
|
||||
|
||||
|
||||
def split_mailbox(mailbox, return_extension=False):
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# coding: utf-8
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""Testing utilities."""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
@@ -13,8 +14,7 @@ from rest_framework.test import APITestCase
|
||||
|
||||
from modoboa.core import models as core_models
|
||||
|
||||
from . import sysutils
|
||||
from . import u2u_decode
|
||||
from .. import sysutils
|
||||
|
||||
try:
|
||||
s = socket.create_connection(('127.0.0.1', 25))
|
||||
@@ -24,7 +24,7 @@ except socket.error:
|
||||
NO_SMTP = True
|
||||
|
||||
try:
|
||||
import ldap # NOQA
|
||||
import ldap # noqa
|
||||
NO_LDAP = False
|
||||
except ImportError:
|
||||
NO_LDAP = True
|
||||
@@ -34,7 +34,7 @@ class ParametersMixin(object):
|
||||
"""Add tools to manage parameters."""
|
||||
|
||||
@classmethod
|
||||
def setUpTestData(cls):
|
||||
def setUpTestData(cls): # noqa
|
||||
"""Set LocalConfig instance."""
|
||||
super(ParametersMixin, cls).setUpTestData()
|
||||
cls.localconfig = core_models.LocalConfig.objects.first()
|
||||
@@ -58,7 +58,7 @@ class ModoTestCase(ParametersMixin, TestCase):
|
||||
"""All test cases must inherit from this one."""
|
||||
|
||||
@classmethod
|
||||
def setUpTestData(cls):
|
||||
def setUpTestData(cls): # noqa
|
||||
"""Create a default user."""
|
||||
super(ModoTestCase, cls).setUpTestData()
|
||||
management.call_command("load_initial_data")
|
||||
@@ -93,7 +93,7 @@ class ModoAPITestCase(ParametersMixin, APITestCase):
|
||||
"""All test cases must inherit from this one."""
|
||||
|
||||
@classmethod
|
||||
def setUpTestData(cls):
|
||||
def setUpTestData(cls): # noqa
|
||||
"""Create a default user."""
|
||||
super(ModoAPITestCase, cls).setUpTestData()
|
||||
management.call_command("load_initial_data")
|
||||
@@ -104,54 +104,3 @@ class ModoAPITestCase(ParametersMixin, APITestCase):
|
||||
"""Setup."""
|
||||
super(ModoAPITestCase, self).setUp()
|
||||
self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token.key)
|
||||
|
||||
|
||||
class U2UTestCase(TestCase):
|
||||
"""Test RFC1342 decoding utilities."""
|
||||
|
||||
def test_header_decoding(self):
|
||||
"""Simple decoding."""
|
||||
samples = [
|
||||
("=?ISO-8859-15?Q?=20Profitez de tous les services en ligne sur "
|
||||
"impots.gouv.fr?=",
|
||||
"Profitez de tous les services en ligne sur impots.gouv.fr"),
|
||||
("=?ISO-8859-1?Q?Accus=E9?= de =?ISO-8859-1?Q?r=E9ception?= de "
|
||||
"votre annonce",
|
||||
"Accusé de réception de votre annonce"),
|
||||
("Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord",
|
||||
"Sm\xf6rg\xe5sbord"),
|
||||
# The following case currently fails because of the way we split
|
||||
# encoded words to parse them separately, which can lead to
|
||||
# unexpected unicode decode errors... I think it will work fine on
|
||||
# Python3
|
||||
# ("=?utf-8?B?VMOpbMOpcMOpYWdlIFZJTkNJIEF1dG9yb3V0ZXMgLSBFeHDD?=\n"
|
||||
# "=?utf-8?B?qWRpdGlvbiBkZSB2b3RyZSBjb21tYW5kZSBOwrAgMjAxNzEyMDcw"
|
||||
# "MDA1?=\n=?utf-8?B?MyBkdSAwNy8xMi8yMDE3IDE0OjQ5OjQx?=",
|
||||
# "")
|
||||
]
|
||||
for sample in samples:
|
||||
self.assertEqual(u2u_decode.u2u_decode(sample[0]), sample[1])
|
||||
|
||||
def test_address_header_decoding(self):
|
||||
"""Check address decoding."""
|
||||
mailsploit_sample = (
|
||||
"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=00?="
|
||||
"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com")
|
||||
expected_result = (
|
||||
"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??="
|
||||
"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com")
|
||||
self.assertEqual(
|
||||
u2u_decode.decode_address(mailsploit_sample),
|
||||
("", expected_result)
|
||||
)
|
||||
mailsploit_sample = (
|
||||
'"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?="\n'
|
||||
'<=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?='
|
||||
'@mailsploit.com>')
|
||||
expected_result = (
|
||||
'potus@whitehouse.gov',
|
||||
'=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??='
|
||||
'@mailsploit.com')
|
||||
self.assertEqual(
|
||||
u2u_decode.decode_address(mailsploit_sample),
|
||||
expected_result)
|
||||
@@ -0,0 +1,130 @@
|
||||
Return-Path: <me@example.net>
|
||||
X-Original-To: someone@example.net
|
||||
Delivered-To: me@example.net
|
||||
Received: from mail.example.net (localhost [127.0.0.1])
|
||||
by mail.example.net (Postfix) with ESMTP id 4BD531AF130
|
||||
for <someone@example.net>; Sun, 17 Dec 2017 03:11:30 +0000 (GMT)
|
||||
Authentication-Results: mail.example.net (amavisd-new);
|
||||
dkim=pass (1024-bit key) reason="pass (just generated, assumed good)"
|
||||
header.d=example.net
|
||||
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=example.net; h=
|
||||
content-language:content-type:content-type:mime-version
|
||||
:user-agent:date:date:message-id:subject:subject:from:from
|
||||
:received:received; s=mail; t=1513480289; bh=wAmt0wg2Yk04321B
|
||||
Q63uWL5f67PEBB2EEGONiFUrIvk=; b=UZUzPqH8TUk/bMZm1ofkjwatNCQftZKK
|
||||
C91uIuX3mmjvngNwbVjx4m7jFZItZscNqigyGJN5HCOl8tC/LJmmWzRdhIisFtLA
|
||||
uXRiBuWps82rbV4ciFU8N30i9UXvtWvMywwL6pYSESqixiwylzbddjMjKXXdyqaT
|
||||
DTyR16ZK8IE=
|
||||
X-Virus-Scanned: amavisd-new at example.net
|
||||
X-Spam-Flag: NO
|
||||
X-Spam-Score: -0.988
|
||||
X-Spam-Level:
|
||||
X-Spam-Status: No, score=-0.988 tagged_above=-999 required=3.9
|
||||
tests=[ALL_TRUSTED=-1, HTML_MESSAGE=0.001, T_HTML_ATTACH=0.01,
|
||||
URIBL_BLOCKED=0.001] autolearn=no autolearn_force=no
|
||||
Received: from mail.example.net ([127.0.0.1])
|
||||
by mail.example.net (mail.example.net [127.0.0.1]) (amavisd-new, port 10026)
|
||||
with LMTP id BSCmm59DxNvB for <someone@example.net>;
|
||||
Sun, 17 Dec 2017 03:11:29 +0000 (GMT)
|
||||
Received: from [10.0.0.1] (my-pc.example.net [10.0.0.1])
|
||||
by mail.example.net (Postfix) with ESMTPSA id 641171AF12D
|
||||
for <someone@example.net>; Sun, 17 Dec 2017 03:11:29 +0000 (GMT)
|
||||
To: Someone <someone@example.net>
|
||||
From: Me <me@example.net>
|
||||
X-Clacks-Overhead: GNU Terry Pratchett
|
||||
Subject: A test message
|
||||
Message-ID: <5294ad7a-e966-f1a0-3431-8c4b1b76f5a3@example.net>
|
||||
Date: Sun, 17 Dec 2017 03:11:33 +0000
|
||||
User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
|
||||
Thunderbird/52.5.0
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/mixed;
|
||||
boundary="------------7EB9EDC10414C906CC3C2CEA"
|
||||
Content-Language: en-GB
|
||||
|
||||
This is a multi-part message in MIME format.
|
||||
--------------7EB9EDC10414C906CC3C2CEA
|
||||
Content-Type: multipart/alternative;
|
||||
boundary="------------87B289699A48D5D0D61BABCC"
|
||||
|
||||
|
||||
--------------87B289699A48D5D0D61BABCC
|
||||
Content-Type: text/plain; charset=utf-8; format=flowed
|
||||
Content-Transfer-Encoding: 7bit
|
||||
|
||||
This is an *html* /test /_message._
|
||||
|
||||
:-)
|
||||
|
||||
|
||||
--
|
||||
Someone
|
||||
someone@example.net
|
||||
|
||||
|
||||
--------------87B289699A48D5D0D61BABCC
|
||||
Content-Type: multipart/related;
|
||||
boundary="------------D585B241EEF60F0F44EC20E5"
|
||||
|
||||
|
||||
--------------D585B241EEF60F0F44EC20E5
|
||||
Content-Type: text/html; charset=utf-8
|
||||
Content-Transfer-Encoding: 7bit
|
||||
|
||||
<html>
|
||||
<head>
|
||||
|
||||
<meta http-equiv="content-type" content="text/html; charset=utf-8">
|
||||
</head>
|
||||
<body text="#000000" bgcolor="#FFFFFF">
|
||||
<p>This is an <b>html</b> <i>test </i><u>message.</u></p>
|
||||
<span class="moz-smiley-s1"><span>:-)</span></span><br>
|
||||
<br>
|
||||
<img moz-do-not-send="false"
|
||||
src="cid:part1.D56823CE.7959DBD4@example.net" alt="" height="10"
|
||||
width="10"><br>
|
||||
<pre class="moz-signature" cols="72">--
|
||||
someone
|
||||
<a class="moz-txt-link-abbreviated" href="mailto:someone@example.net">someone@example.net</a></pre>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
--------------D585B241EEF60F0F44EC20E5
|
||||
Content-Type: image/png;
|
||||
name="image.png"
|
||||
Content-Transfer-Encoding: base64
|
||||
Content-ID: <part1.D56823CE.7959DBD4@example.net>
|
||||
Content-Disposition: inline;
|
||||
filename="image.png"
|
||||
|
||||
iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAACXBIWXMAAAsTAAALEwEAmpwY
|
||||
AAAAF0lEQVQY02NgYGD4//8/ThKfHASMRN0AfaqVa6MQxIQAAAAASUVORK5CYII=
|
||||
--------------D585B241EEF60F0F44EC20E5--
|
||||
|
||||
--------------87B289699A48D5D0D61BABCC--
|
||||
|
||||
--------------7EB9EDC10414C906CC3C2CEA
|
||||
Content-Type: text/html;
|
||||
name="test.html"
|
||||
Content-Transfer-Encoding: base64
|
||||
Content-Disposition: attachment;
|
||||
filename="test.html"
|
||||
|
||||
//48AHMAcABhAG4APgA8AGIAPgBTAG8AbQBlACAAPABpAD4ASABUAE0ATAA8AC8AaQA+ACAA
|
||||
dABlAHgAdAA8AC8AYgA+ACwAIABhACAAPABhACAAaAByAGUAZgA9ACIAaAB0AHQAcAA6AC8A
|
||||
LwBlAHgAYQBtAHAAbABlAC4AbgBlAHQAIgAgAHQAYQByAGcAZQB0AD0AIgBfAGIAbABhAG4A
|
||||
awAiACAAcgBlAGwAPQAiAG4AbwBmAG8AbABsAG8AdwAiAD4AbABpAG4AawA8AC8AYQA+ACwA
|
||||
IABhAG4AJgAjADEAMwA7AA0ACgBlAG0AYgBlAGQAZABlAGQAIABpAG0AYQBnAGUAIABhAG4A
|
||||
ZAAgAGEAbgAgAGUAeAB0AGUAcgBuAGEAbAAgAGkAbQBhAGcAZQAuADwAYgByAD4AJgAjADEA
|
||||
MwA7AA0ACgA8AGkAbQBnACAAcwByAGMAPQAiAGQAYQB0AGEAOgBpAG0AYQBnAGUALwBwAG4A
|
||||
ZwA7AGIAYQBzAGUANgA0ACwAaQBWAEIATwBSAHcAMABLAEcAZwBvAEEAQQBBAEEATgBTAFUA
|
||||
aABFAFUAZwBBAEEAQQBBAG8AQQBBAEEAQQBLAEMAQQBJAEEAQQBBAEEAQwBVAEYAagBxAEEA
|
||||
QQBBAEEAQwBYAEIASQBXAFgATQBBAEEAQQBzAFQAQQBBAEEATABFAHcARQBBAG0AcAB3AFkA
|
||||
QQBBAEEAQQBGADAAbABFAFEAVgBRAFkAMAAyAE4AZwBZAEcARAA0AC8ALwA4AC8AVABoAEsA
|
||||
ZgBIAEEAUwBNAFIATgAwAEEAZgBhAHEAVgBhADYATQBRAHgASQBRAEEAQQBBAEEAQQBTAFUA
|
||||
VgBPAFIASwA1AEMAWQBJAEkAPQAiACAAdABhAHIAZwBlAHQAPQAiAF8AYgBsAGEAbgBrACIA
|
||||
PgA8AGIAcgA+ACYAIwAxADMAOwANAAoAPABpAG0AZwAgAHMAcgBjAD0AIgBoAHQAdABwADoA
|
||||
LwAvAGUAeABhAG0AcABsAGUALgBuAGUAdAAvAGkAbQBhAGcAZQAuAHAAbgBnACIAIAB0AGEA
|
||||
cgBnAGUAdAA9ACIAXwBiAGwAYQBuAGsAIgA+ADwAYgByAD4AJgAjADEAMwA7AA0ACgBOAGkA
|
||||
ZgB0AHkAIQA8AC8AcwBwAGEAbgA+AA0ACgA=
|
||||
--------------7EB9EDC10414C906CC3C2CEA--
|
||||
@@ -0,0 +1,16 @@
|
||||
<div>
|
||||
|
||||
|
||||
|
||||
|
||||
<body>
|
||||
<p>This is an <b>html</b> <i>test </i><u>message.</u></p>
|
||||
<span class="moz-smiley-s1"><span>:-)</span></span><br>
|
||||
<br>
|
||||
<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAF0lEQVQY02NgYGD4//8/ThKfHASMRN0AfaqVa6MQxIQAAAAASUVORK5CYII=" alt="" height="10" width="10" target="_blank"><br>
|
||||
<pre class="moz-signature" cols="72">--
|
||||
someone
|
||||
<a class="moz-txt-link-abbreviated" href="mailto:someone@example.net" target="_blank" rel="nofollow">someone@example.net</a></pre>
|
||||
</body>
|
||||
<p>
|
||||
��</p></div>
|
||||
@@ -0,0 +1,16 @@
|
||||
<div>
|
||||
|
||||
|
||||
|
||||
|
||||
<body>
|
||||
<p>This is an <b>html</b> <i>test </i><u>message.</u></p>
|
||||
<span class="moz-smiley-s1"><span>:-)</span></span><br>
|
||||
<br>
|
||||
<img alt="" height="10" width="10"><br>
|
||||
<pre class="moz-signature" cols="72">--
|
||||
someone
|
||||
<a class="moz-txt-link-abbreviated">someone@example.net</a></pre>
|
||||
</body>
|
||||
<p>
|
||||
��</p></div>
|
||||
@@ -0,0 +1,7 @@
|
||||
<pre>This is an *html* /test /_message._
|
||||
|
||||
:-)
|
||||
|
||||
--
|
||||
Someone
|
||||
someone@example.net</pre>
|
||||
@@ -0,0 +1,51 @@
|
||||
Return-Path: <someone@example.net>
|
||||
X-Original-To: me@example.net Delivered-To: someone@example.net
|
||||
Received: from mail.example.net (localhost [127.0.0.1])
|
||||
by mail.example.net (Postfix) with ESMTP id A142E1AF12D
|
||||
for <me@example.net>; Sun, 17 Dec 2017 13:10:10 +0000 (GMT)
|
||||
Authentication-Results: mail.example.net (amavisd-new);
|
||||
dkim=pass (1024-bit key) reason="pass (just generated, assumed good)"
|
||||
header.d=example.net
|
||||
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=example.net; h=
|
||||
content-transfer-encoding:content-language:content-type
|
||||
:content-type:mime-version:user-agent:date:date:message-id
|
||||
:subject:subject:from:from:received:received; s=mail; t=
|
||||
1513516210; bh=yyFHETi2jtS7bFMEIArr53VnJMNNjltuXCFTOqs/cL0=; b=X
|
||||
nMo4pMsaH7EO9A55i0SOv5RGP4Sp4eukaXXQJgMWahcfUn4mM2+pSdc+8ZMaXX7y
|
||||
ITQdCzI0LL9WGfX7bbMA49hYtIELjQ/AochqABTUsMxnq0Sgf1FE37bA3NS1pBtp
|
||||
TBKehpGjr86y0V247Okmf2faEkO9dMvmwWz8QBI0N4=
|
||||
X-Virus-Scanned: amavisd-new at example.net
|
||||
X-Spam-Flag: NO
|
||||
X-Spam-Score: -0.999
|
||||
X-Spam-Level:
|
||||
X-Spam-Status: No, score=-0.999 tagged_above=-999 required=3.9
|
||||
tests=[ALL_TRUSTED=-1, URIBL_BLOCKED=0.001]
|
||||
autolearn=no autolearn_force=no
|
||||
Received: from mail.example.net ([127.0.0.1])
|
||||
by mail.example.net (mail.example.net [127.0.0.1]) (amavisd-new, port 10026)
|
||||
with LMTP id j0UYMNn7ZprP for <me@example.net>;
|
||||
Sun, 17 Dec 2017 13:10:10 +0000 (GMT)
|
||||
Received: from [10.0.0.1] (my-pc.example.net [10.0.0.1])
|
||||
by mail.example.net (Postfix) with ESMTPSA id E66201AF132
|
||||
for <me@example.net>; Sun, 17 Dec 2017 13:10:09 +0000 (GMT)
|
||||
To: Me <me@example.net>
|
||||
From: Someone <someone@example.net>
|
||||
X-Clacks-Overhead: GNU Terry Pratchett
|
||||
Subject: =?UTF-8?Q?T=c3=a9st_message?=
|
||||
Message-ID: <8626caea-4c05-8dc5-c687-bcfb7a329d18@example.net>
|
||||
Date: Sun, 17 Dec 2017 13:10:14 +0000
|
||||
User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
|
||||
Thunderbird/52.5.0
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=utf-8; format=flowed
|
||||
Content-Language: en-GB
|
||||
Content-Transfer-Encoding: 7bit
|
||||
X-rAnDoM-cAsE-hEaDeR: fOoBaR
|
||||
X-UPPER-CASE-HEADER: FOOBAR
|
||||
x-lower-case-header: foobar
|
||||
|
||||
This is a plain text e-mail message.
|
||||
|
||||
--
|
||||
Someone
|
||||
someone@example.net
|
||||
@@ -0,0 +1,5 @@
|
||||
<pre>This is a plain text e-mail message.
|
||||
|
||||
--
|
||||
Someone
|
||||
someone@example.net</pre>
|
||||
@@ -0,0 +1,108 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""Tests for email_utils."""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import os
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
from django.utils.encoding import smart_bytes, smart_text
|
||||
|
||||
from ..email_utils import Email
|
||||
|
||||
SAMPLES_DIR = os.path.realpath(
|
||||
os.path.join(os.path.dirname(__file__), "sample_messages"))
|
||||
|
||||
|
||||
class EmailTestImplementation(Email):
|
||||
|
||||
def _fetch_message(self):
|
||||
message_path = os.path.join(SAMPLES_DIR, "%s-input.txt" % self.mailid)
|
||||
assert os.path.isfile(message_path), "%s does not exist." % message_path
|
||||
|
||||
with open(message_path, "rb") as fp:
|
||||
mail_text = smart_bytes(fp.read())
|
||||
|
||||
return mail_text
|
||||
|
||||
|
||||
class EmailTests(SimpleTestCase):
|
||||
"""Tests for modoboa.lib.email_utils.Email
|
||||
|
||||
When writing new sample messages use the following naming convention for
|
||||
the sample files stored in sample_messages:
|
||||
|
||||
input: {message_id}-input.txt
|
||||
output: {message_id}-output-{dformat}_{no,}links.txt
|
||||
"""
|
||||
|
||||
def _get_expected_output(self, message_id, **kwargs):
|
||||
ext = kwargs["dformat"] if "dformat" in kwargs else "plain"
|
||||
ext += "_links" if "links" in kwargs and kwargs["links"] else "_nolinks"
|
||||
message_path = os.path.join(SAMPLES_DIR,
|
||||
"%s-output-%s.txt" % (message_id, ext))
|
||||
assert os.path.isfile(message_path), "%s does not exist." % message_path
|
||||
|
||||
with open(message_path, "rb") as fp:
|
||||
# output should always be unicode (py2) or str (py3)
|
||||
mail_text = smart_text(fp.read())
|
||||
|
||||
return mail_text
|
||||
|
||||
def _test_email(self, message_id, **kwargs):
|
||||
"""Boiler plate code for testing e-mails."""
|
||||
expected_output = self._get_expected_output(message_id, **kwargs)
|
||||
output = EmailTestImplementation(message_id, **kwargs).body
|
||||
self.assertEqual(output, expected_output)
|
||||
|
||||
def test_invalid_links_value(self):
|
||||
"""modoboa-amavis set links = "0"; in python "0" == True, fixed in PR 79
|
||||
https://github.com/modoboa/modoboa-amavis/pull/79"""
|
||||
with self.assertRaises(TypeError) as cm:
|
||||
EmailTestImplementation("text_plain", links="0")
|
||||
|
||||
ex_message = cm.exception.messages
|
||||
self.assertEqual(ex_message,
|
||||
"links == \"0\" is not valid, did you mean True or "
|
||||
"False?")
|
||||
|
||||
def test_links_value_for_webmail(self):
|
||||
"""modoboa-webmail sets links = 0 or 1
|
||||
TODO: this should be fixed in modoboa-webmail to use True/False"""
|
||||
email = EmailTestImplementation("text_plain", links=0)
|
||||
self.assertFalse(email.links)
|
||||
|
||||
email = EmailTestImplementation("text_plain", links=1)
|
||||
self.assertTrue(email.links)
|
||||
|
||||
def test_headers(self):
|
||||
email = EmailTestImplementation("text_plain")
|
||||
# test header names are case insensative
|
||||
self.assertEqual(email.msg["x-upper-case-header"], "FOOBAR")
|
||||
self.assertEqual(email.msg["X-Lower-Case-Header"], "foobar")
|
||||
self.assertEqual(email.msg["x-random-case-header"], "fOoBaR")
|
||||
|
||||
expected_output = [
|
||||
{"name": "From", "value": "Someone <someone@example.net>"},
|
||||
{"name": "To", "value": "Me <me@example.net>"},
|
||||
{"name": "Cc", "value": ""},
|
||||
{"name": "Date", "value": "Sun, 17 Dec 2017 13:10:14 +0000"},
|
||||
{"name": "Subject", "value": "Tést message"},
|
||||
]
|
||||
self.assertEqual(email.headers, expected_output)
|
||||
|
||||
def test_email_text_plain(self):
|
||||
self._test_email("text_plain")
|
||||
|
||||
def test_email_multipart_as_text(self):
|
||||
"""display the text/plain part of a multipart message"""
|
||||
self._test_email("multipart")
|
||||
|
||||
def test_email_multipart_with_links(self):
|
||||
"""display the text/html part of a multipart message"""
|
||||
self._test_email("multipart", dformat="html", links=True)
|
||||
|
||||
def test_email_multipart_without_links(self):
|
||||
"""display the text/html part of a multipart message with links removed"""
|
||||
self._test_email("multipart", dformat="html", links=False)
|
||||
@@ -0,0 +1,60 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""Tests for u2u_decode."""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.test import TestCase
|
||||
|
||||
from .. import u2u_decode
|
||||
|
||||
|
||||
class U2UTestCase(TestCase):
|
||||
"""Test RFC1342 decoding utilities."""
|
||||
|
||||
def test_header_decoding(self):
|
||||
"""Simple decoding."""
|
||||
samples = [
|
||||
("=?ISO-8859-15?Q?=20Profitez de tous les services en ligne sur "
|
||||
"impots.gouv.fr?=",
|
||||
"Profitez de tous les services en ligne sur impots.gouv.fr"),
|
||||
("=?ISO-8859-1?Q?Accus=E9?= de =?ISO-8859-1?Q?r=E9ception?= de "
|
||||
"votre annonce",
|
||||
"Accusé de réception de votre annonce"),
|
||||
("Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord",
|
||||
"Sm\xf6rg\xe5sbord"),
|
||||
# The following case currently fails because of the way we split
|
||||
# encoded words to parse them separately, which can lead to
|
||||
# unexpected unicode decode errors... I think it will work fine on
|
||||
# Python3
|
||||
# ("=?utf-8?B?VMOpbMOpcMOpYWdlIFZJTkNJIEF1dG9yb3V0ZXMgLSBFeHDD?=\n"
|
||||
# "=?utf-8?B?qWRpdGlvbiBkZSB2b3RyZSBjb21tYW5kZSBOwrAgMjAxNzEyMDcw"
|
||||
# "MDA1?=\n=?utf-8?B?MyBkdSAwNy8xMi8yMDE3IDE0OjQ5OjQx?=",
|
||||
# "")
|
||||
]
|
||||
for sample in samples:
|
||||
self.assertEqual(u2u_decode.u2u_decode(sample[0]), sample[1])
|
||||
|
||||
def test_address_header_decoding(self):
|
||||
"""Check address decoding."""
|
||||
mailsploit_sample = (
|
||||
"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=00?="
|
||||
"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com")
|
||||
expected_result = (
|
||||
"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??="
|
||||
"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?=@mailsploit.com")
|
||||
self.assertEqual(
|
||||
u2u_decode.decode_address(mailsploit_sample),
|
||||
("", expected_result)
|
||||
)
|
||||
mailsploit_sample = (
|
||||
'"=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?="\n'
|
||||
'<=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q?=0A=00?='
|
||||
'@mailsploit.com>')
|
||||
expected_result = (
|
||||
'potus@whitehouse.gov',
|
||||
'=?utf-8?b?cG90dXNAd2hpdGVob3VzZS5nb3Y=?==?utf-8?Q??='
|
||||
'@mailsploit.com')
|
||||
self.assertEqual(
|
||||
u2u_decode.decode_address(mailsploit_sample),
|
||||
expected_result)
|
||||
@@ -27,3 +27,4 @@ cryptography
|
||||
pytz
|
||||
requests
|
||||
rfc6266
|
||||
lxml
|
||||
|
||||
Reference in New Issue
Block a user