From c984596ae9f9301c3898c2145a37b4b2bad4599d Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Sat, 16 Dec 2017 00:54:17 +0000 Subject: [PATCH] improve modoboa.lib.email_utils.Email This is a refactor of moboboa.lib.email_utils.Email, currently it has trouble parsing messages containing certain attachment types. For html messages inline images are now base64 encoded and embedded into the tag rather than being saved to disk. This only effects modoboa-amavis as modoboa-webmail implements it's own parser for e-mails as they are retrieved differently using IMAP. This adds a new external dependency on lxml. --- modoboa/lib/email_utils.py | 350 ++++++++++++++++++++----------------- requirements.txt | 1 + 2 files changed, 194 insertions(+), 157 deletions(-) diff --git a/modoboa/lib/email_utils.py b/modoboa/lib/email_utils.py index 97dbb3c00..1a16b6062 100644 --- a/modoboa/lib/email_utils.py +++ b/modoboa/lib/email_utils.py @@ -1,22 +1,29 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- from __future__ import unicode_literals -from email.header import Header, decode_header +import email +from email.header import Header from email.mime.text import MIMEText from email.utils import make_msgid, formatdate, parseaddr import re import smtplib import time -from django.conf import settings -from django.utils.encoding import smart_text -from django.utils.html import conditional_escape +from django.utils.encoding import smart_str, smart_text +from django.utils.html import conditional_escape, escape from django.template.loader import render_to_string -from django.utils import six from modoboa.lib import u2u_decode +import lxml.html +from lxml.html.clean import Cleaner + + +# used by Email() +_RE_REMOVE_EXTRA_WHITESPACE = re.compile(r"\n\s*\n") +_RE_CID = re.compile(r".*[ ]*cid=\"([^\"]*)\".*", re.I) + class EmailAddress(object): @@ -33,107 +40,100 @@ class EmailAddress(object): class Email(object): - def __init__(self, mailid, mformat="plain", dformat="plain", links=0): - self.attached_map = {} + _basic_headers = ( + "From", + "To", + "Cc", + "Date", + "Subject", + ) + + _image_mime_types = ( + "image/png", + "image/gif", + "image/jpeg", + ) + + def __init__(self, mailid, mformat="plain", dformat="plain", links=False): self.contents = {"html": "", "plain": ""} - self.headers = [] - self.attachments = {} self.mailid = mailid self.mformat = mformat self.dformat = dformat - self.links = links + + if isinstance(links, bool): + self.links = links + elif isinstance(links, int): + # TODO: modoboa-webmail currently sets links == 0 or 1, it needs + # updated to use True or False. + self.links = bool(links) + elif links == "0": + raise TypeError("links == \"0\" is not valid, did you mean True or " + "False?") + else: + raise TypeError("links should be a boolean value") + self._msg = None + self._headers = None self._body = None + self._images = {} + + # used by modoboa_webmail.lib.imapemail.ImapEmail + self.attachments = {} @property def msg(self): - """Return an email.message object. - """ - raise NotImplementedError + if self._msg is None: + mail_text = self._fetch_message() + assert isinstance(mail_text, bytes) + # python 2 expects str (bytes), python 3 expects str (unicode) + mail_text = smart_str(mail_text, errors="replace") + self._msg = email.message_from_string(mail_text) + + return self._msg + + @property + def headers(self): + if self._headers is None: + self._headers = [] + + # don't break modoboa_webmail.lib.imapemail.ImapEmail, it + # doesn't have self._msg it loads e-mails (and their headers) + # from the IMAP server. + if self._msg is not None: + for header in self._basic_headers: + value = self.get_header(self.msg, header) + self._headers.append({ + "name": header, + "value": value + }) + + return self._headers + + @headers.setter + def headers(self, value): + self._headers = value @property def body(self): - """Return email's body.""" if self._body is None: - self._body = ( - getattr(self, "viewmail_%s" % self.mformat) - (self.contents[self.mformat], links=self.links) - ) + self._parse(self.msg) + self._body = getattr(self, "viewmail_%s" % self.dformat)() + return self._body - def get_header(self, msg, hdrname): - """Look for a particular header. - - :param string hdrname: header name - :return: header avalue - """ - for name in [hdrname, hdrname.upper()]: - if name in msg: - return msg[name] + def get_header(self, msg, header): + # msg parameter to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if header in msg: + return "".join([smart_text(v, encoding=(e or 'ascii')) + for v, e in email.header.decode_header(msg[header]) + ]) return "" - def _parse_default(self, msg, level): - """Default parser + def _fetch_message(self): + raise NotImplementedError() - All parts handled by this parser will be consireded as - attachments. - """ - fname = msg.get_filename() - if fname is not None: - if isinstance(fname, six.text_type): - fname = fname.encode("utf-8") - decoded = decode_header(fname) - value = ( - decoded[0][0] if decoded[0][1] is None - else smart_text(decoded[0][0], decoded[0][1]) - ) - else: - value = "part_%s" % level - self.attachments[level] = value - - def _parse_text(self, msg, level): - """Displayable content parser - - text, html, calendar, etc. All those contents can be displayed - inside a navigator. - """ - if msg.get_content_subtype() not in ["plain", "html"]: - self._parse_default(msg, level) - target = "plain" - else: - target = msg.get_content_subtype() - self.contents[target] += decode(msg.get_payload(decode=True), - charset=msg.get_content_charset()) - - def _parse_image(self, msg, level): - """image/* parser - - The only reason to make a specific parser for images is that, - sometimes, messages embark inline images, which means they - must be displayed and not attached. - """ - if self.dformat == "html" and self.links \ - and "Content-Disposition" in msg: - if msg["Content-Disposition"].startswith("inline"): - cid = None - if "Content-ID" in msg: - m = re.match("<(.+)>", msg["Content-ID"]) - cid = m is not None and m.group(1) or msg["Content-ID"] - fname = msg.get_filename() - if fname is None: - if "Content-Location" in msg: - fname = msg["Content-Location"] - elif cid is not None: - fname = cid - else: - # I give up for now :p - return - self.attached_map[cid] = re.match("^http:", fname) and fname \ - or self.__save_image(fname, msg) - return - self._parse_default(msg, level) - - def _parse(self, msg, level=None): + def _parse(self, msg, level=0): """Recursive email parser A message structure can be complex. To correctly handle @@ -146,85 +146,121 @@ class Email(object): :param msg: message (or part) to parse :param level: current part number """ - if msg.is_multipart() and msg.get_content_maintype() != "message": - cpt = 1 - for part in msg.get_payload(): - nlevel = level is None and ("%d" % cpt) \ - or "%s.%d" % (level, cpt) - self._parse(part, nlevel) - cpt += 1 + content_type = msg.get_content_maintype() + is_attachment = "attachment" in msg.get("Content-Disposition", "") + + if content_type == "text" and not is_attachment: + self._parse_text(msg, level) + elif content_type == "multipart" and not is_attachment: + self._parse_multipart(msg, level) + else: + self._parse_inline_image(msg, level) + + if self.contents["plain"]: + self._post_process_plain() + if self.contents["html"]: + self._post_process_html() + + def _parse_text(self, msg, level=0): + content_type = msg.get_content_subtype() + + if content_type in ["plain", "html"]: + encoding = msg.get_content_charset() or "utf-8" + mail_text = smart_text(msg.get_payload(decode=True), encoding=encoding, errors="replace") + self.contents[content_type] += mail_text + + def _parse_multipart(self, msg, level=0): + level += 1 + for part in msg.walk(): + content_type = part.get_content_maintype() + if content_type == "text": + self._parse_text(part, level=level) + else: + # I'm a dumb mail parser and treat all non-text parts as attachments + self._parse_inline_image(part, level=level) + + def _parse_inline_image(self, msg, level=0): + content_type = msg.get_content_type() + cid = None + if "Content-ID" in msg: + cid = msg["Content-ID"] + if cid.startswith('<') and cid.endswith('>'): + cid = cid[1:-1] + else: + matches = _RE_CID.match(msg["Content-Type"]) + if matches is not None: + cid = matches.group(1) + transfer_encoding = msg["Content-Transfer-Encoding"] + if content_type not in self._image_mime_types or cid is None: + # I'm a dumb mail parser and I don't care ;p + return + if transfer_encoding != "base64": + # I don't deal with non base64 encoded images + return + if cid in self._images: + # Duplicate Content-ID return - if level is None: - level = "1" - try: - getattr(self, "_parse_%s" % msg.get_content_maintype())(msg, level) - except AttributeError: - self._parse_default(msg, level) + self._images[cid] = "data:%s;base64,%s" % ( + content_type, "".join(msg.get_payload().splitlines(False))) - def __save_image(self, fname, part): - """Save an inline image on the filesystem. + def _post_process_plain(self): + mail_text = _RE_REMOVE_EXTRA_WHITESPACE \ + .sub("\n\n", self.contents["plain"]).strip() + mail_text = escape(mail_text) + self.contents["plain"] = smart_text(mail_text) - Some HTML messages are using inline images (attached images - with links on them inside the body). In order to display them, - images are saved on the filesystem and links contained in the - message are modified. + def _post_process_html(self): + html = lxml.html.fromstring(self.contents["html"]) - :param fname: the image associated filename - :param part: the email part that contains the image payload - """ - if re.search(r"\.\.", fname): - return None - path = "/static/tmp/" + fname - fp = open(settings.MODOBOA_DIR + path, "wb") - fp.write(part.get_payload(decode=True)) - fp.close() - return path + if self.links: + html.rewrite_links(self._map_cid) - def map_cid(self, url): - """Replace attachment links. + for link in html.iterlinks(): + link[0].set("target", "_blank") + else: + html.rewrite_links(lambda x: None) + + cleaner = Cleaner( + scripts=True, + javascript=True, + links=True, + page_structure=True, + embedded=True, + frames=True, + add_nofollow=True) + mail_text = lxml.html.tostring(cleaner.clean_html(html)) + self.contents["html"] = smart_text(mail_text) + + def _map_cid(self, url): + if url.startswith("cid:"): + cid = url[4:] + if cid in self._images: + url = self._images[cid] - :param str url: original url - :rtype: string - :return: internal link - """ - match = re.match(".*cid:(.+)", url) - if match is not None: - if match.group(1) in self.attached_map: - return self.attached_map[match.group(1)] return url + def viewmail_plain(self, contents=None, **kwargs): + # contents and **kwargs parameters to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if contents is None: + contents = self.contents["plain"] + else: + contents = conditional_escape(contents) + return "
%s
" % contents + + def viewmail_html(self, contents=None, **kwargs): + # contents and **kwargs parameters to maintain compatibility with + # modoboa_webmail.lib.imapemail.ImapEmail + if contents is None: + contents = self.contents["html"] + return contents + def render_headers(self, **kwargs): - return render_to_string("common/mailheaders.html", { + context = { "headers": self.headers, - }) - - def viewmail_plain(self, content, **kwargs): - content = conditional_escape(content) - return "
%s
" % content - - def viewmail_html(self, content, **kwargs): - import lxml.html - - if content is None or content == "": - return "" - links = kwargs.get("links", 0) - html = lxml.html.fromstring(content) - if not links: - html.rewrite_links(lambda x: None) - else: - html.rewrite_links(self.map_cid) - for link in html.iterlinks(): - link[0].set('target', '_blank') - body = html.find("body") - if body is None: - body = lxml.html.tostring(html) - else: - body = lxml.html.tostring(body) - body = re.sub( - "<(/?)body", lambda m: "<%sdiv" % m.group(1), - body.decode()) - return body + } + return render_to_string("common/mailheaders.html", context) def split_mailbox(mailbox, return_extension=False): diff --git a/requirements.txt b/requirements.txt index 94712f9db..74e0f4af7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ cryptography pytz requests rfc6266 +lxml