Forward a message AND its attachments.

Code cleanup.

see #545

modoboa/extensions/amavis/sql_listing.py

@@ -306,18 +306,16 @@ class SQLlisting(EmailListing):
 
 
 class SQLemail(Email):
-    def __init__(self, msg, *args, **kwargs):
-        super(SQLemail, self).__init__(msg, *args, **kwargs)
+    def __init__(self, *args, **kwargs):
+        super(SQLemail, self).__init__(*args, **kwargs)
         fields = ["From", "To", "Cc", "Date", "Subject"]
         for f in fields:
             label = f
-            self.headers += [{"name": label, "value": self.get_header(msg, f)}]
-            try:
-                label = re.sub("-", "_", label)
-                setattr(self, label, msg[f])
-            except:
-                pass
-        qreason = self.get_header(msg, "X-Amavis-Alert")
+            self.headers += [
+                {"name": label, "value": self.get_header(self.msg, f)}
+            ]
+            setattr(self, label, self.msg[f])
+        qreason = self.get_header(self.msg, "X-Amavis-Alert")
         self.qtype = ""
         self.qreason = ""
         if qreason != "":
@@ -328,11 +326,19 @@ class SQLemail(Email):
                 self.qtype = "BAD HEADER SECTION"
                 self.qreason = qreason[19:]
 
-    def get_header(self, msg, name):
-        for k in [name, name.upper()]:
-            if k in msg:
-                return msg[k]
-        return ""
+    @property
+    def msg(self):
+        """
+        """
+        import email
+
+        if self._msg is None:
+            qmails = get_wrapper().get_mail_content(self.mailid)
+            self._msg = email.message_from_string(
+                "".join([qm.mail_text for qm in qmails])
+            )
+            self._parse(self._msg)
+        return self._msg
 
     def render_headers(self, **kwargs):
         return render_to_string("amavis/mailheaders.html", {

modoboa/extensions/amavis/views.py

@@ -92,12 +92,7 @@ def index(request):
 
 
 def getmailcontent_selfservice(request, mail_id):
-    qmails = get_wrapper().get_mail_content(mail_id)
-    content = ""
-    for qm in qmails:
-        content += qm.mail_text
-    msg = email.message_from_string(content)
-    mail = SQLemail(msg, mformat="plain", links="0")
+    mail = SQLemail(mail_id, mformat="plain", links="0")
     return render(request, "common/viewmail.html", {
         "headers": mail.render_headers(),
         "mailbody": mail.body
@@ -106,12 +101,7 @@ def getmailcontent_selfservice(request, mail_id):
 
 @selfservice(getmailcontent_selfservice)
 def getmailcontent(request, mail_id):
-    qmails = get_wrapper().get_mail_content(mail_id)
-    content = ""
-    for qm in qmails:
-        content += qm.mail_text
-    msg = email.message_from_string(content)
-    mail = SQLemail(msg, mformat="plain", links="0")
+    mail = SQLemail(mail_id, mformat="plain", links="0")
     return render(request, "common/viewmail.html", {
         "headers": mail.render_headers(),
         "mailbody": mail.body

modoboa/extensions/webmail/forms.py

@@ -1,9 +1,84 @@
 # coding: utf-8
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+import lxml.html
+
 from django import forms
+from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
+from modoboa.lib import parameters
+from modoboa.lib.emailutils import set_email_headers
+from .lib import (
+    ImapEmail, create_mail_attachment
+)
+
+
+def html2plaintext(content):
+    """HTML to plain text translation.
+
+    :param content: some HTML content
+    """
+    if not content:
+        return ""
+    html = lxml.html.fromstring(content)
+    plaintext = ""
+    for ch in html.iter():
+        p = None
+        if ch.text is not None:
+            p = ch.text.strip('\r\t\n')
+        if ch.tag == "img":
+            p = ch.get("alt")
+        if p is None:
+            continue
+        plaintext += p + "\n"
+    return plaintext
+
+
+def make_body_images_inline(body):
+    """Looks for images inside the body and make them inline.
+
+    Before sending a message in HTML format, it is necessary to find
+    all img tags contained in the body in order to rewrite them. For
+    example, icons provided by CKeditor are stored on the server
+    filesystem and not accessible from the outside. We must embark
+    them as parts of the MIME message if we want recipients to
+    display them correctly.
+
+    :param body: the HTML body to parse
+    """
+    import os
+    from email.mime.image import MIMEImage
+    from urlparse import urlparse
+
+    html = lxml.html.fromstring(body)
+    parts = []
+    for tag in html.iter("img"):
+        src = tag.get("src")
+        if src is None:
+            continue
+        o = urlparse(src)
+        path = os.path.join(settings.MODOBOA_DIR, o.path[1:])
+        if not os.path.exists(path):
+            continue
+        fname = os.path.basename(path)
+        cid = "%s@modoboa" % os.path.splitext(fname)[0]
+        tag.set("src", "cid:%s" % cid)
+        with open(path, "rb") as fp:
+            part = MIMEImage(fp.read())
+        part["Content-ID"] = "<%s>" % cid
+        part.replace_header(
+            "Content-Type", '%s; name="%s"' % (part["Content-Type"], fname)
+        )
+        part["Content-Disposition"] = "inline"
+        parts.append(part)
+    return lxml.html.tostring(html), parts
+
 
 class ComposeMailForm(forms.Form):
+    """Compose mail form.
+    """
     to = forms.CharField(label=_("To"))
     cc = forms.CharField(label=_("Cc"), required=False)
     cci = forms.CharField(label=_("Cci"), required=False)
@@ -13,6 +88,103 @@ class ComposeMailForm(forms.Form):
     origmsgid = forms.CharField(
         label="", widget=forms.HiddenInput(), required=False
     )
+    body = forms.CharField(widget=forms.widgets.Textarea)
+
+    def _html_msg(self):
+        """Create a multipart message.
+
+        We attach two alternatives:
+        * text/html
+        * text/plain
+        """
+        msg = MIMEMultipart(_subtype="related")
+        submsg = MIMEMultipart(_subtype="alternative")
+        body = self.cleaned_data["body"]
+        charset = "utf-8"
+        if body:
+            tbody = html2plaintext(body)
+            body, images = make_body_images_inline(body)
+        else:
+            tbody = ""
+            images = []
+        submsg.attach(
+            MIMEText(tbody.encode(charset), _subtype="plain", _charset=charset)
+        )
+        submsg.attach(
+            MIMEText(body.encode(charset), _subtype="html", _charset=charset)
+        )
+        msg.attach(submsg)
+        for img in images:
+            msg.attach(img)
+        return msg
+
+    def _plain_msg(self):
+        """Create a simple text message.
+        """
+        charset = "utf-8"
+        text = MIMEText(self.cleaned_data["body"].encode(charset),
+                        _subtype="plain", _charset=charset)
+        return text
+
+    def _build_msg(self, request):
+        """Convert form's content to a MIME message.
+        """
+        editormode = parameters.get_user(request.user, "EDITOR")
+        msg = getattr(self, "_%s_msg" % editormode)()
+
+        if request.session["compose_mail"]["attachments"]:
+            wrapper = MIMEMultipart(_subtype="mixed")
+            wrapper.attach(msg)
+            for attdef in request.session["compose_mail"]["attachments"]:
+                wrapper.attach(create_mail_attachment(attdef))
+            msg = wrapper
+        return msg
+
+    def to_msg(self, request):
+        """Convert form's content to an object ready to send.
+
+        We set headers at the end to be sure no one will override
+        them.
+
+        """
+        msg = self._build_msg(request)
+        set_email_headers(
+            msg, self.cleaned_data["subject"], request.user.encoded_address,
+            self.cleaned_data['to']
+        )
+        origmsgid = self.cleaned_data.get("origmsgid", None)
+        if origmsgid:
+            msg["References"] = msg["In-Reply-To"] = origmsgid
+        return msg
+
+
+class ForwardMailForm(ComposeMailForm):
+    """Forward mail form.
+    """
+
+    def _build_msg(self, request):
+        """Convert form's content to a MIME message.
+
+        We also add original attachments (if any) to the new message.
+        """
+        from modoboa.extensions.webmail.lib import decode_payload
+
+        mbox = request.GET.get("mbox", None)
+        mailid = request.GET.get("mailid", None)
+        msg = super(ForwardMailForm, self)._build_msg(request)
+        origmsg = ImapEmail(request, False, "%s:%s" % (mbox, mailid))
+        if origmsg.attachments:
+            if not msg.is_multipart or not msg.get_content_subtype() == "mixed":
+                wrapper = MIMEMultipart(_subtype="mixed")
+                wrapper.attach(msg)
+                msg = wrapper
+            for attpart, fname in origmsg.attachments.items():
+                attdef, payload = origmsg.fetch_attachment(attpart)
+                attdef["fname"] = fname
+                msg.attach(create_mail_attachment(
+                    attdef, decode_payload(attdef["encoding"], payload)
+                ))
+        return msg
 
 
 class FolderForm(forms.Form):

modoboa/extensions/webmail/lib.py

@@ -1,773 +0,0 @@
-# coding: utf-8
-import os
-import re
-from datetime import datetime, timedelta
-import email
-import lxml.html
-import chardet
-from rfc6266 import build_header, parse_headers
-from django.core.files.uploadhandler import FileUploadHandler, SkipFile
-from django.utils.translation import ugettext as _, ugettext_lazy
-from django.conf import settings
-from modoboa.lib import u2u_decode, tables, parameters
-from modoboa.lib.webutils import size2integer, NavigationParameters
-from modoboa.lib.exceptions import InternalError
-from modoboa.lib.email_listing import EmailListing
-from modoboa.lib.emailutils import (
-    EmailAddress, Email, prepare_addresses, set_email_headers
-)
-from modoboa.extensions.webmail.exceptions import WebmailInternalError
-from modoboa.extensions.webmail.imaputils import (
-    IMAPconnector, get_imapconnector, BodyStructure
-)
-
-
-class WMtable(tables.Table):
-    tableid = "emails"
-    styles = "table-condensed"
-    idkey = "imapid"
-    drag = tables.ImgColumn(
-        "drag", cssclass="draggable left", width="1%",
-        defvalue="%spics/grippy.png" % settings.STATIC_URL,
-        header="<input type='checkbox' name='toggleselect' id='toggleselect' />"
-    )
-    selection = tables.SelectionColumn(
-        "selection", safe=True, width='1%', header=None, sortable=False
-    )
-    flags = tables.ImgColumn("flags", width="3%")
-    withatts = tables.ImgColumn("withatts", width="2%")
-    subject = tables.Column(
-        "subject", label=ugettext_lazy("Subject"), width="50%", limit=60,
-        cssclass="openable"
-    )
-    from_ = tables.Column(
-        "from", width="20%", label=ugettext_lazy("From"), limit=30,
-        cssclass="openable"
-    )
-    date = tables.Column(
-        "date", width="15%", label=ugettext_lazy("Date"), cssclass="openable"
-    )
-
-    cols_order = ["drag", "selection", "withatts", "flags", "subject", "from_", "date"]
-
-    def parse(self, header, value):
-        if value is None:
-            return ""
-        try:
-            value = getattr(IMAPheader, "parse_%s" % header)(value)
-        except AttributeError:
-            pass
-        return value
-
-
-class IMAPheader(object):
-    @staticmethod
-    def to_unicode(value):
-        if value is None or type(value) is unicode:
-            return value
-        try:
-            value = value.decode('utf-8')
-        except UnicodeDecodeError:
-            pass
-        else:
-            return value
-        try:
-            res = chardet.detect(value)
-        except UnicodeDecodeError:
-            return value
-        if res["encoding"] == "ascii":
-            return value
-        return value.decode(res["encoding"])
-
-    @staticmethod
-    def parse_address(value, **kwargs):
-        addr = EmailAddress(value)
-        if "full" in kwargs.keys() and kwargs["full"]:
-            return IMAPheader.to_unicode(addr.fulladdress)
-        result = addr.name and addr.name or addr.fulladdress
-        return IMAPheader.to_unicode(result)
-
-    @staticmethod
-    def parse_address_list(values, **kwargs):
-        lst = values.split(",")
-        result = ""
-        for addr in lst:
-            if result != "":
-                result += ", "
-            result += IMAPheader.parse_address(addr, **kwargs)
-        return result
-
-    @staticmethod
-    def parse_from(value, **kwargs):
-        return IMAPheader.parse_address(value, **kwargs)
-
-    @staticmethod
-    def parse_to(value, **kwargs):
-        return IMAPheader.parse_address_list(value, **kwargs)
-
-    @staticmethod
-    def parse_cc(value, **kwargs):
-        return IMAPheader.parse_address_list(value, **kwargs)
-
-    @staticmethod
-    def parse_reply_to(value, **kwargs):
-        return IMAPheader.parse_address_list(value, **kwargs)
-
-    @staticmethod
-    def parse_date(value, **kwargs):
-        tmp = email.utils.parsedate_tz(value)
-        if not tmp:
-            return value
-        ndate = datetime(*(tmp)[:7])
-        now = datetime.now()
-        try:
-            if now - ndate > timedelta(7):
-                return ndate.strftime("%d.%m.%Y %H:%M")
-            return ndate.strftime("%a %H:%M")
-        except ValueError:
-            return value
-
-    @staticmethod
-    def parse_message_id(value, **kwargs):
-        return value.strip('\n')
-
-    @staticmethod
-    def parse_subject(value, **kwargs):
-        try:
-            subject = u2u_decode.u2u_decode(value)
-        except UnicodeDecodeError:
-            subject = value
-        return IMAPheader.to_unicode(subject)
-
-
-class ImapListing(EmailListing):
-    tpl = "webmail/index.html"
-    tbltype = WMtable
-    deflocation = "INBOX/"
-    defcallback = "wm_updatelisting"
-    reset_wm_url = False
-
-    def __init__(self, user, password, **kwargs):
-        self.user = user
-        self.mbc = IMAPconnector(user=user.username, password=password)
-        if "pattern" in kwargs and kwargs["pattern"]:
-            self.parse_search_parameters(
-                kwargs["criteria"], kwargs["pattern"]
-            )
-        else:
-            self.mbc.criterions = []
-
-        super(ImapListing, self).__init__(**kwargs)
-        self.extravars["refreshrate"] = \
-            int(parameters.get_user(user, "REFRESH_INTERVAL")) * 1000
-
-    def parse_search_parameters(self, criterion, pattern):
-
-        def or_criterion(old, c):
-            if old == "":
-                return c
-            return "OR (%s) (%s)" % (old, c)
-
-        if criterion == u"both":
-            criterion = u"from_addr, subject"
-        criterions = ""
-        if type(pattern) is unicode:
-            pattern = pattern.encode("utf-8")
-        if type(criterion) is unicode:
-            criterion = criterion.encode("utf-8")
-        for c in criterion.split(','):
-            if c == "from_addr":
-                key = "FROM"
-            elif c == "subject":
-                key = "SUBJECT"
-            else:
-                continue
-            criterions = \
-                or_criterion(criterions, '(%s "%s")' % (key, pattern))
-        self.mbc.criterions = [criterions]
-
-    @staticmethod
-    def computequota(mbc):
-        try:
-            return int(float(mbc.quota_actual) \
-                / float(mbc.quota_limit) * 100)
-        except (AttributeError, TypeError):
-            return -1
-
-    def getquota(self):
-        return ImapListing.computequota(self.mbc)
-
-
-class ImapEmail(Email):
-    headernames = [
-        ('From', True),
-        ('To', True),
-        ('Cc', True),
-        ('Date', True),
-        ('Subject', True),
-    ]
-
-    def __init__(self, mbox, mailid, request, dformat="DISPLAYMODE", addrfull=False,
-                 links=0):
-        mformat = parameters.get_user(request.user, "DISPLAYMODE")
-        self.dformat = parameters.get_user(request.user, dformat)
-
-        self.headers = []
-        self.attachments = {}
-        self.imapc = get_imapconnector(request)
-        msg = self.imapc.fetchmail(mbox, mailid, readonly=False,
-                                   headers=self.headers_as_list)
-        self.mbox = mbox
-        self.mailid = mailid
-        headers = msg['BODY[HEADER.FIELDS (%s)]' % self.headers_as_text]
-        fallback_fmt = "html" if self.dformat == "plain" else "plain"
-
-        self.bs = BodyStructure(msg['BODYSTRUCTURE'])
-        data = None
-
-        mformat = self.dformat if self.dformat in self.bs.contents else fallback_fmt
-
-        if len(self.bs.contents):
-            bodyc = u''
-            for part in self.bs.contents[mformat]:
-                pnum = part['pnum']
-                data = self.imapc._cmd("FETCH", mailid, "(BODY.PEEK[%s])" % pnum)
-                content = decode_payload(part['encoding'],
-                                         data[int(mailid)]['BODY[%s]' % pnum])
-                charset = self._find_content_charset(part)
-                if charset is not None:
-                    try:
-                        content = content.decode(charset)
-                    except (UnicodeDecodeError, LookupError):
-                        result = chardet.detect(content)
-                        content = content.decode(result['encoding'])
-                bodyc += content
-
-            self._fetch_inlines()
-            self.body = \
-                getattr(self, "viewmail_%s" % mformat)(bodyc, links=links)
-        else:
-            self.body = None
-
-        self._find_attachments()
-
-        msg = email.message_from_string(headers)
-        for hdr in self.headernames:
-            label = hdr[0]
-            name = hdr[0]
-            if not name in msg.keys():
-                name = name.upper()
-                if not name in msg.keys():
-                    continue
-            try:
-                key = re.sub("-", "_", name).lower()
-                value = getattr(IMAPheader, "parse_%s" % key)(msg[name], full=addrfull)
-            except AttributeError:
-                value = msg[name]
-            if hdr[1]:
-                self.headers += [{"name": label, "value": value}]
-            try:
-                label = re.sub("-", "_", label)
-                setattr(self, label, value)
-            except:
-                pass
-
-    @property
-    def headers_as_list(self):
-        return [hdr[0].upper() for hdr in self.headernames]
-
-    @property
-    def headers_as_text(self):
-        return " ".join(self.headers_as_list)
-
-    def _find_content_charset(self, part):
-        for pos, elem in enumerate(part["params"]):
-            if elem == "charset":
-                return part["params"][pos + 1]
-        return None
-
-    def _find_attachments(self):
-        """Retrieve attachments from the parsed body structure.
-
-        We try to find and decode a file name for each attachment. If
-        we failed, a generic name will be used (ie. part_1, part_2, ...).
-        """
-        for att in self.bs.attachments:
-            attname = "part_%s" % att["pnum"]
-            if "params" in att and att["params"] != "NIL":
-                attname = u2u_decode.u2u_decode(att["params"][1]) \
-                    .strip("\r\t\n")
-            elif "disposition" in att and len(att["disposition"]) > 1:
-                for pos, value in enumerate(att["disposition"][1]):
-                    if not value.startswith("filename"):
-                        continue
-                    header = "%s; %s=%s" \
-                        % (att['disposition'][0],
-                           value,
-                           att["disposition"][1][pos + 1].strip("\r\t\n"))
-                    attname = parse_headers(header).filename_unsafe
-                    if attname is None:
-                        attname = u2u_decode.u2u_decode(
-                            att["disposition"][1][pos + 1]
-                        ).strip("\r\t\n")
-                    break
-            self.attachments[att["pnum"]] = attname
-
-    def _fetch_inlines(self):
-        for cid, params in self.bs.inlines.iteritems():
-            if re.search("\.\.", cid):
-                continue
-            fname = "webmail/%s_%s" % (self.mailid, cid)
-            path = os.path.join(settings.MEDIA_ROOT, fname)
-            params["fname"] = os.path.join(settings.MEDIA_URL, fname)
-            if os.path.exists(path):
-                continue
-
-            pdef, content = self.imapc.fetchpart(self.mailid, self.mbox, params["pnum"])
-            fp = open(path, "wb")
-            fp.write(decode_payload(params["encoding"], content))
-            fp.close()
-
-    def map_cid(self, url):
-        m = re.match(".*cid:(.+)", url)
-        if m:
-            if m.group(1) in self.bs.inlines:
-                return self.bs.inlines[m.group(1)]["fname"]
-        return url
-
-    def render_headers(self, **kwargs):
-        from django.template.loader import render_to_string
-
-        res = render_to_string("webmail/headers.html", {
-            "headers": self.headers,
-            "folder": kwargs["folder"], "mail_id": kwargs["mail_id"],
-            "attachments": self.attachments != {} and self.attachments or None
-        })
-        return res
-
-
-class Modifier(ImapEmail):
-    def __init__(self, *args, **kwargs):
-        super(Modifier, self).__init__(*args, **kwargs)
-        getattr(self, "_modify_%s" % self.dformat)()
-
-    def _modify_plain(self):
-        self.body = re.sub("</?pre>", "", self.body)
-
-    def _modify_html(self):
-        pass
-
-
-class ReplyModifier(Modifier):
-    headernames = ImapEmail.headernames + \
-        [("Reply-To", True),
-         ("Message-ID", False)]
-
-    def __init__(self, mbox, mailid, request, form,  **kwargs):
-        super(ReplyModifier, self).__init__(
-            mbox, mailid, request, dformat="EDITOR", **kwargs
-        )
-
-        self.textheader = "%s %s" % (self.From, _("wrote:"))
-
-        if hasattr(self, "Message_ID"):
-            form.fields["origmsgid"].initial = self.Message_ID
-        if not hasattr(self, "Reply_To"):
-            form.fields["to"].initial = self.From
-        else:
-            form.fields["to"].initial = self.Reply_To
-        if request.GET.get("all", "0") == "1":  # reply-all
-            form.fields["cc"].initial = ""
-            toparse = self.To.split(",")
-            if hasattr(self, 'Cc'):
-                toparse += self.Cc.split(",")
-            for addr in toparse:
-                tmp = EmailAddress(addr)
-                if tmp.address and tmp.address == request.user.username:
-                    continue
-                if form.fields["cc"].initial != "":
-                    form.fields["cc"].initial += ", "
-                form.fields["cc"].initial += tmp.fulladdress
-        m = re.match("re\s*:\s*.+", self.Subject.lower())
-        if m:
-            form.fields["subject"].initial = self.Subject
-        else:
-            form.fields["subject"].initial = "Re: %s" % self.Subject
-
-    def _modify_plain(self):
-        super(ReplyModifier, self)._modify_plain()
-        lines = self.body.split('\n')
-        body = ""
-        for l in lines:
-            if body != "":
-                body += "\n"
-            body += ">%s" % l
-        self.body = body
-
-
-class ForwardModifier(Modifier):
-    def __init__(self, mbox, mailid, request, form, **kwargs):
-        super(ForwardModifier, self).__init__(
-            mbox, mailid, request, dformat="EDITOR", **kwargs
-        )
-
-        self._header()
-        form.fields["subject"].initial = "Fwd: %s" % self.Subject
-
-    def __getfunc(self, name):
-        return getattr(self, "%s_%s" % (name, self.dformat))
-
-    def _header(self):
-        self.textheader = self.__getfunc("_header_begin")() + "\n"
-        self.textheader += \
-            self.__getfunc("_header_line")(_("Subject"), self.Subject) + "\n"
-        self.textheader += \
-            self.__getfunc("_header_line")(_("Date"), self.Date) + "\n"
-        for hdr in ["From", "To", "Reply-To"]:
-            try:
-                key = re.sub("-", "_", hdr)
-                value = getattr(self, key)
-                self.textheader += \
-                    self.__getfunc("_header_line")(_(hdr), value) + "\n"
-            except:
-                pass
-        self.textheader += self.__getfunc("_header_end")()
-
-    def _header_begin_plain(self):
-        return "----- %s -----" % _("Original message")
-
-    def _header_begin_html(self):
-        return  """----- %s -----<br/>
-<table border='0'>""" % _("Original message")
-
-    def _header_line_plain(self, key, value):
-        return "%s: %s" % (key, value)
-
-    def _header_line_html(self, key, value):
-        return "  <tr><td>%s</td><td>%s</td></tr>" % (key, value)
-
-    def _header_end_plain(self):
-        return "\n"
-
-    def _header_end_html(self):
-        return "</table>"
-
-
-class EmailSignature(object):
-    """User signature
-
-    :param user: User object
-    """
-    def __init__(self, user):
-        self._sig = ""
-        dformat = parameters.get_user(user, "EDITOR")
-        content = parameters.get_user(user, "SIGNATURE")
-        if len(content):
-            getattr(self, "_format_sig_%s" % dformat)(content)
-
-    def _format_sig_plain(self, content):
-        self._sig = """
----
-%s""" % content
-
-    def _format_sig_html(self, content):
-        content = re.sub("\n", "<br/>", content)
-        self._sig = """<br/>
----<br/>
-%s""" % content
-
-    def __repr__(self):
-        return self._sig
-
-
-def decode_payload(encoding, payload):
-    """Decode the payload according to the given encoding
-
-    Supported encodings: base64, quoted-printable.
-
-    :param encoding: the encoding's name
-    :param payload: the value to decode
-    :return: a string
-    """
-    encoding = encoding.lower()
-    if encoding == "base64":
-        import base64
-        return base64.b64decode(payload)
-    elif encoding == "quoted-printable":
-        import quopri
-        return quopri.decodestring(payload)
-    return payload
-
-
-def find_images_in_body(body):
-    """Looks for images inside a HTML body
-
-    Before sending a message in HTML format, it is necessary to find
-    all img tags contained in the body in order to rewrite them. For
-    example, icons provided by CKeditor are stored on the server
-    filesystem and not accessible from the outside. We must embark
-    them as parts off the MIME message if we want recipients to
-    display them correctly.
-
-    :param body: the HTML body to parse
-    """
-    from email.mime.image import MIMEImage
-    from urlparse import urlparse
-
-    html = lxml.html.fromstring(body)
-    parts = []
-    for tag in html.iter("img"):
-        src = tag.get("src")
-        if src is None:
-            continue
-        o = urlparse(src)
-        fname = os.path.basename(o.path)
-        path = os.path.join(settings.MEDIA_ROOT, "webmail", fname)
-        if not os.path.exists(path):
-            continue
-        cid = "%s@modoboa" % os.path.splitext(fname)[0]
-        tag.set("src", "cid:%s" % cid)
-        fp = open(path, "rb")
-        p = MIMEImage(fp.read())
-        fp.close()
-        p["Content-ID"] = "<%s>" % cid
-        ct = p["Content-Type"]
-        p.replace_header("Content-Type", '%s; name="%s"'
-                         % (ct, os.path.basename(fname)))
-        p["Content-Disposition"] = "inline"
-        parts.append(p)
-
-    return lxml.html.tostring(html), parts
-
-
-def set_compose_session(request):
-    """Initialize a new "compose" session.
-
-    It is used to keep track of attachments defined with a new
-    message. Each new message will be associated with a unique ID (in
-    order to avoid conflicts between users).
-
-    :param request: a Request object.
-    :return: the new unique ID.
-    """
-    import uuid
-    randid = str(uuid.uuid4()).replace("-", "")
-    request.session["compose_mail"] = {"id": randid, "attachments": []}
-    return randid
-
-
-def save_attachment(f):
-    """Save a new attachment to the filesystem.
-
-    The attachment is not saved using its own name to the
-    filesystem. To avoid conflicts, a random name is generated and
-    used instead.
-
-    :param f: an uploaded file object (see Django's documentation)
-    :return: the new random name
-    """
-    from tempfile import NamedTemporaryFile
-
-    dstdir = os.path.join(settings.MEDIA_ROOT, "webmail")
-    try:
-        fp = NamedTemporaryFile(dir=dstdir, delete=False)
-    except Exception as e:
-        raise InternalError(str(e))
-    for chunk in f.chunks():
-        fp.write(chunk)
-    fp.close()
-    return fp.name
-
-
-def clean_attachments(attlist):
-    """Remove all attachments from the filesystem
-
-    :param attlist: a list of 2-uple. Each element must contain the
-                    following information : (random name, real name).
-    """
-    for att in attlist:
-        fullpath = os.path.join(settings.MEDIA_ROOT, "webmail", att["tmpname"])
-        try:
-            os.remove(fullpath)
-        except OSError:
-            pass
-
-
-def html2plaintext(content):
-    """HTML to plain text translation
-
-    :param content: some HTML content
-    """
-    if not content:
-        return ""
-    html = lxml.html.fromstring(content)
-    plaintext = ""
-    for ch in html.iter():
-        p = None
-        if ch.text is not None:
-            p = ch.text.strip('\r\t\n')
-        if ch.tag == "img":
-            p = ch.get("alt")
-        if p is None:
-            continue
-        plaintext += p + "\n"
-
-    return plaintext
-
-
-def create_mail_attachment(attdef):
-    """Create the MIME part corresponding to the given attachment.
-
-    Mandatory keys: 'fname', 'tmpname', 'content-type'
-
-    :param attdef: a dictionary containing the attachment definition
-    :return: a MIMEBase object
-    """
-    from email import Encoders
-    from email.mime.base import MIMEBase
-
-    maintype, subtype = attdef["content-type"].split("/")
-    res = MIMEBase(maintype, subtype)
-    fp = open(os.path.join(settings.MEDIA_ROOT, "webmail", attdef["tmpname"]), "rb")
-    res.set_payload(fp.read())
-    fp.close()
-    Encoders.encode_base64(res)
-    res['Content-Disposition'] = build_header(attdef['fname'].decode('utf-8'))
-    return res
-
-
-def send_mail(request, posturl=None):
-    """Email verification and sending.
-
-    If the form does not present any error, a new MIME message is
-    constructed. Then, a connection is established with the defined
-    SMTP server and the message is finally sent.
-
-    :param request: a Request object
-    :param posturl: the url to post the message form to
-    :return: a 2-uple (True|False, HttpResponse)
-    """
-    from email.mime.multipart import MIMEMultipart
-    from .forms import ComposeMailForm
-    from modoboa.lib.webutils import _render_to_string
-    from modoboa.lib.cryptutils import get_password
-
-    form = ComposeMailForm(request.POST)
-    editormode = parameters.get_user(request.user, "EDITOR")
-    if form.is_valid():
-        from email.mime.text import MIMEText
-        import smtplib
-
-        body = request.POST["id_body"]
-        charset = "utf-8"
-
-        if editormode == "html":
-            msg = MIMEMultipart(_subtype="related")
-            submsg = MIMEMultipart(_subtype="alternative")
-            if body:
-                textbody = html2plaintext(body)
-                body, images = find_images_in_body(body)
-            else:
-                textbody = ""
-                images = []
-            submsg.attach(MIMEText(textbody.encode(charset),
-                                   _subtype="plain", _charset=charset))
-            submsg.attach(MIMEText(body.encode(charset), _subtype=editormode,
-                                   _charset=charset))
-            msg.attach(submsg)
-            for img in images:
-                msg.attach(img)
-        else:
-            text = MIMEText(body.encode(charset),
-                            _subtype=editormode, _charset=charset)
-            if len(request.session["compose_mail"]["attachments"]):
-                msg = MIMEMultipart()
-                msg.attach(text)
-            else:
-                msg = text
-
-        for attdef in request.session["compose_mail"]["attachments"]:
-            msg.attach(create_mail_attachment(attdef))
-
-        set_email_headers(
-            msg, form.cleaned_data["subject"], request.user.encoded_address,
-            form.cleaned_data['to']
-        )
-        if "origmsgid" in form.cleaned_data:
-            msg["References"] = msg["In-Reply-To"] = form.cleaned_data["origmsgid"]
-        rcpts = prepare_addresses(form.cleaned_data["to"], "envelope")
-        for hdr in ["cc", "cci"]:
-            if form.cleaned_data[hdr] != "":
-                msg[hdr.capitalize()] = prepare_addresses(form.cleaned_data[hdr])
-                rcpts += prepare_addresses(form.cleaned_data[hdr], "envelope")
-        try:
-            secmode = parameters.get_admin("SMTP_SECURED_MODE")
-            if secmode == "ssl":
-                s = smtplib.SMTP_SSL(parameters.get_admin("SMTP_SERVER"),
-                                     int(parameters.get_admin("SMTP_PORT")))
-            else:
-                s = smtplib.SMTP(parameters.get_admin("SMTP_SERVER"),
-                                 int(parameters.get_admin("SMTP_PORT")))
-                if secmode == "starttls":
-                    s.starttls()
-        except Exception as text:
-            raise WebmailInternalError(str(text))
-
-        if parameters.get_admin("SMTP_AUTHENTICATION") == "yes":
-            try:
-                s.login(request.user.username, get_password(request))
-            except smtplib.SMTPException, e:
-                raise WebmailInternalError(str(e))
-        try:
-            s.sendmail(request.user.email, rcpts, msg.as_string())
-            s.quit()
-        except smtplib.SMTPException, e:
-            raise WebmailInternalError(str(e))
-
-        sentfolder = parameters.get_user(request.user, "SENT_FOLDER")
-        IMAPconnector(user=request.user.username,
-                      password=request.session["password"]).push_mail(sentfolder, msg)
-        clean_attachments(request.session["compose_mail"]["attachments"])
-        del request.session["compose_mail"]
-        return True, {}
-
-    listing = _render_to_string(request, "webmail/compose.html",
-                                {"form": form, "noerrors": True,
-                                 "body": request.POST["id_body"].strip(),
-                                 "posturl": posturl})
-    return False, dict(status="ko", listing=listing, editor=editormode)
-
-
-class AttachmentUploadHandler(FileUploadHandler):
-    """
-    Simple upload handler to limit the size of the attachments users
-    can upload.
-    """
-
-    def __init__(self, request=None):
-        super(AttachmentUploadHandler, self).__init__(request)
-        self.total_upload = 0
-        self.toobig = False
-        self.maxsize = size2integer(parameters.get_admin("MAX_ATTACHMENT_SIZE"))
-
-    def receive_data_chunk(self, raw_data, start):
-        self.total_upload += len(raw_data)
-        if self.total_upload >= self.maxsize:
-            self.toobig = True
-            raise SkipFile()
-        return raw_data
-
-    def file_complete(self, file_size):
-        return None
-
-
-class WebmailNavigationParameters(NavigationParameters):
-    """
-    Specific NavigationParameters subclass for the webmail.
-    """
-    def __init__(self, request, defmailbox=None):
-        super(WebmailNavigationParameters, self).__init__(
-            request, 'webmail_navparams'
-        )
-        if defmailbox is not None:
-            self.parameters += [('mbox', defmailbox, False)]

modoboa/extensions/webmail/lib/__init__.py

@@ -0,0 +1,35 @@
+from .attachments import (
+    create_mail_attachment, save_attachment, clean_attachments,
+    set_compose_session, AttachmentUploadHandler
+)
+from .imapemail import ImapEmail, ReplyModifier, ForwardModifier
+from .imaputils import (
+    BodyStructure, IMAPconnector, get_imapconnector, separate_mailbox
+)
+from .listing import ImapListing
+from .signature import EmailSignature
+from .utils import decode_payload, WebmailNavigationParameters
+from .sendmail import send_mail
+from .fetch_parser import parse_fetch_response
+
+
+__all__ = [
+    'AttachmentUploadHandler',
+    'BodyStructure',
+    'EmailSignature',
+    'ForwardModifier',
+    'IMAPconnector',
+    'ImapEmail',
+    'ImapListing',
+    'ReplyModifier',
+    'WebmailNavigationParameters',
+    'clean_attachments',
+    'create_mail_attachment',
+    'decode_payload',
+    'get_imapconnector',
+    'parse_fetch_response',
+    'save_attachment',
+    'send_mail',
+    'separate_mailbox',
+    'set_compose_session',
+]

modoboa/extensions/webmail/lib/attachments.py

@@ -0,0 +1,112 @@
+import os
+from rfc6266 import build_header
+from django.conf import settings
+from django.core.files.uploadhandler import FileUploadHandler, SkipFile
+from modoboa.lib import parameters
+from modoboa.lib.exceptions import InternalError
+from modoboa.lib.webutils import size2integer
+
+
+def set_compose_session(request):
+    """Initialize a new "compose" session.
+
+    It is used to keep track of attachments defined with a new
+    message. Each new message will be associated with a unique ID (in
+    order to avoid conflicts between users).
+
+    :param request: a Request object.
+    :return: the new unique ID.
+    """
+    import uuid
+    randid = str(uuid.uuid4()).replace("-", "")
+    request.session["compose_mail"] = {"id": randid, "attachments": []}
+    return randid
+
+
+def save_attachment(f):
+    """Save a new attachment to the filesystem.
+
+    The attachment is not saved using its own name to the
+    filesystem. To avoid conflicts, a random name is generated and
+    used instead.
+
+    :param f: an uploaded file object (see Django's documentation)
+    :return: the new random name
+    """
+    from tempfile import NamedTemporaryFile
+
+    dstdir = os.path.join(settings.MEDIA_ROOT, "webmail")
+    try:
+        fp = NamedTemporaryFile(dir=dstdir, delete=False)
+    except Exception as e:
+        raise InternalError(str(e))
+    for chunk in f.chunks():
+        fp.write(chunk)
+    fp.close()
+    return fp.name
+
+
+def clean_attachments(attlist):
+    """Remove all attachments from the filesystem
+
+    :param attlist: a list of 2-uple. Each element must contain the
+                    following information : (random name, real name).
+    """
+    for att in attlist:
+        fullpath = os.path.join(settings.MEDIA_ROOT, "webmail", att["tmpname"])
+        try:
+            os.remove(fullpath)
+        except OSError:
+            pass
+
+
+def create_mail_attachment(attdef, payload=None):
+    """Create the MIME part corresponding to the given attachment.
+
+    Mandatory keys: 'fname', 'tmpname', 'content-type'
+
+    :param attdef: a dictionary containing the attachment definition
+    :return: a MIMEBase object
+    """
+    from email import Encoders
+    from email.mime.base import MIMEBase
+
+    if "content-type" in attdef:
+        maintype, subtype = attdef["content-type"].split("/")
+    elif "Content-Type" in attdef:
+        maintype, subtype = attdef["Content-Type"].split("/")
+    else:
+        return None
+    res = MIMEBase(maintype, subtype)
+    if payload is None:
+        with open(os.path.join(
+            settings.MEDIA_ROOT, "webmail", attdef["tmpname"]), "rb") as fp:
+            res.set_payload(fp.read())
+    else:
+        res.set_payload(payload)
+    Encoders.encode_base64(res)
+    res['Content-Disposition'] = build_header(attdef['fname'].decode('utf-8'))
+    return res
+
+
+class AttachmentUploadHandler(FileUploadHandler):
+    """
+    Simple upload handler to limit the size of the attachments users
+    can upload.
+    """
+
+    def __init__(self, request=None):
+        super(AttachmentUploadHandler, self).__init__(request)
+        self.total_upload = 0
+        self.toobig = False
+        self.maxsize = size2integer(parameters.get_admin("MAX_ATTACHMENT_SIZE"))
+
+    def receive_data_chunk(self, raw_data, start):
+        self.total_upload += len(raw_data)
+        if self.total_upload >= self.maxsize:
+            self.toobig = True
+            raise SkipFile()
+        return raw_data
+
+    def file_complete(self, file_size):
+        return None

modoboa/extensions/webmail/lib/imapemail.py

@@ -0,0 +1,307 @@
+"""
+Set of classes to manipulate/display emails inside the webmail.
+"""
+import os
+import re
+import email
+
+import chardet
+from rfc6266 import parse_headers
+
+from django.conf import settings
+from django.utils.translation import ugettext as _
+
+from modoboa.lib import parameters, u2u_decode
+from modoboa.lib.emailutils import Email, EmailAddress
+from .imaputils import (
+    get_imapconnector, BodyStructure
+)
+from .utils import decode_payload
+
+
+class ImapEmail(Email):
+    """
+    A class to represent an email fetched from an IMAP server.
+    """
+    headernames = [
+        ('From', True),
+        ('To', True),
+        ('Cc', True),
+        ('Date', True),
+        ('Subject', True),
+    ]
+
+    def __init__(self, request, addrfull, *args, **kwargs):
+        super(ImapEmail, self).__init__(*args, **kwargs)
+        self.request = request
+        self.addrfull = addrfull
+        self.imapc = get_imapconnector(request)
+        self.mbox, self.mailid = self.mailid.split(":")
+
+        headers = self.msg['BODY[HEADER.FIELDS (%s)]' % self.headers_as_text]
+        msg = email.message_from_string(headers)
+        for hdr in self.headernames:
+            label = hdr[0]
+            hdrvalue = self.get_header(msg, label)
+            if not hdrvalue:
+                continue
+            if hdr[1]:
+                self.headers += [{"name": label, "value": hdrvalue}]
+            label = re.sub("-", "_", label)
+            setattr(self, label, hdrvalue)
+
+    def get_header(self, msg, hdrname):
+        """Look for a particular header.
+
+        We also try to decode the default value.
+        """
+        from . import imapheader
+
+        hdrvalue = super(ImapEmail, self).get_header(msg, hdrname)
+        if not hdrvalue:
+            return ""
+        try:
+            key = re.sub("-", "_", hdrname).lower()
+            hdrvalue = getattr(imapheader, "parse_%s" % key)(
+                hdrvalue, full=self.addrfull
+            )
+        except AttributeError:
+            pass
+        return hdrvalue
+
+    @property
+    def msg(self):
+        """
+        """
+        if self._msg is None:
+            self._msg = self.imapc.fetchmail(
+                self.mbox, self.mailid, readonly=False,
+                headers=self.headers_as_list
+            )
+            self.bs = BodyStructure(self._msg['BODYSTRUCTURE'])
+            self._find_attachments()
+            if not self.dformat in ["plain", "html"]:
+                self.dformat = parameters.get_user(
+                    self.request.user, self.dformat
+                )
+            fallback_fmt = "html" if self.dformat == "plain" else "plain"
+            self.mformat = self.dformat \
+                if self.dformat in self.bs.contents else fallback_fmt
+        return self._msg
+
+    @property
+    def headers_as_list(self):
+        return [hdr[0].upper() for hdr in self.headernames]
+
+    @property
+    def headers_as_text(self):
+        return " ".join(self.headers_as_list)
+
+    @property
+    def body(self):
+        """Load email's body.
+
+        This operation has to be made "on demand" because it requires
+        a communication with the IMAP server.
+
+        """
+        if self._body is None and self.bs.contents:
+            bodyc = u''
+            for part in self.bs.contents[self.mformat]:
+                pnum = part['pnum']
+                data = self.imapc._cmd(
+                    "FETCH", self.mailid, "(BODY.PEEK[%s])" % pnum
+                )
+                content = decode_payload(
+                    part['encoding'], data[int(self.mailid)]['BODY[%s]' % pnum]
+                )
+                charset = self._find_content_charset(part)
+                if charset is not None:
+                    try:
+                        content = content.decode(charset)
+                    except (UnicodeDecodeError, LookupError):
+                        result = chardet.detect(content)
+                        content = content.decode(result['encoding'])
+                bodyc += content
+            self._fetch_inlines()
+            self._body = getattr(self, "viewmail_%s" % self.mformat)(
+                bodyc, links=self.links
+            )
+        return self._body
+
+    @body.setter
+    def body(self, value):
+        self._body = value
+
+    def _find_content_charset(self, part):
+        for pos, elem in enumerate(part["params"]):
+            if elem == "charset":
+                return part["params"][pos + 1]
+        return None
+
+    def _find_attachments(self):
+        """Retrieve attachments from the parsed body structure.
+
+        We try to find and decode a file name for each attachment. If
+        we failed, a generic name will be used (ie. part_1, part_2, ...).
+        """
+        for att in self.bs.attachments:
+            attname = "part_%s" % att["pnum"]
+            if "params" in att and att["params"] != "NIL":
+                attname = u2u_decode.u2u_decode(att["params"][1]) \
+                    .strip("\r\t\n")
+            elif "disposition" in att and len(att["disposition"]) > 1:
+                for pos, value in enumerate(att["disposition"][1]):
+                    if not value.startswith("filename"):
+                        continue
+                    header = "%s; %s=%s" \
+                        % (att['disposition'][0],
+                           value,
+                           att["disposition"][1][pos + 1].strip("\r\t\n"))
+                    attname = parse_headers(header).filename_unsafe
+                    if attname is None:
+                        attname = u2u_decode.u2u_decode(
+                            att["disposition"][1][pos + 1]
+                        ).strip("\r\t\n")
+                    break
+            self.attachments[att["pnum"]] = attname
+
+    def _fetch_inlines(self):
+        for cid, params in self.bs.inlines.iteritems():
+            if re.search(r"\.\.", cid):
+                continue
+            fname = "webmail/%s_%s" % (self.mailid, cid)
+            path = os.path.join(settings.MEDIA_ROOT, fname)
+            params["fname"] = os.path.join(settings.MEDIA_URL, fname)
+            if os.path.exists(path):
+                continue
+
+            pdef, content = self.imapc.fetchpart(
+                self.mailid, self.mbox, params["pnum"]
+            )
+            with open(path, "wb") as fpo:
+                fpo.write(decode_payload(params["encoding"], content))
+
+    def map_cid(self, url):
+        m = re.match(".*cid:(.+)", url)
+        if m:
+            if m.group(1) in self.bs.inlines:
+                return self.bs.inlines[m.group(1)]["fname"]
+        return url
+
+    def render_headers(self, **kwargs):
+        from django.template.loader import render_to_string
+
+        res = render_to_string("webmail/headers.html", {
+            "headers": self.headers,
+            "folder": kwargs["folder"], "mail_id": kwargs["mail_id"],
+            "attachments": self.attachments != {} and self.attachments or None
+        })
+        return res
+
+    def fetch_attachment(self, pnum):
+        """Fetch an attachment from the IMAP server.
+        """
+        return self.imapc.fetchpart(self.mailid, self.mbox, pnum)
+
+
+class Modifier(ImapEmail):
+    def __init__(self, form, *args, **kwargs):
+        kwargs["dformat"] = "EDITOR"
+        super(Modifier, self).__init__(*args, **kwargs)
+        self.form = form
+        getattr(self, "_modify_%s" % self.dformat)()
+
+    def _modify_plain(self):
+        self.body = re.sub("</?pre>", "", self.body)
+
+    def _modify_html(self):
+        pass
+
+
+class ReplyModifier(Modifier):
+    headernames = ImapEmail.headernames + \
+        [("Reply-To", True),
+         ("Message-ID", False)]
+
+    def __init__(self, *args, **kwargs):
+        super(ReplyModifier, self).__init__(*args, **kwargs)
+
+        self.textheader = "%s %s" % (self.From, _("wrote:"))
+        if hasattr(self, "Message_ID"):
+            self.form.fields["origmsgid"].initial = self.Message_ID
+        if not hasattr(self, "Reply_To"):
+            self.form.fields["to"].initial = self.From
+        else:
+            self.form.fields["to"].initial = self.Reply_To
+        if self.request.GET.get("all", "0") == "1":  # reply-all
+            self.form.fields["cc"].initial = ""
+            toparse = self.To.split(",")
+            if hasattr(self, 'Cc'):
+                toparse += self.Cc.split(",")
+            for addr in toparse:
+                tmp = EmailAddress(addr)
+                if tmp.address and tmp.address == self.request.user.username:
+                    continue
+                if self.form.fields["cc"].initial != "":
+                    self.form.fields["cc"].initial += ", "
+                self.form.fields["cc"].initial += tmp.fulladdress
+        m = re.match(r"re\s*:\s*.+", self.Subject.lower())
+        if m:
+            self.form.fields["subject"].initial = self.Subject
+        else:
+            self.form.fields["subject"].initial = "Re: %s" % self.Subject
+
+    def _modify_plain(self):
+        super(ReplyModifier, self)._modify_plain()
+        lines = self.body.split('\n')
+        body = ""
+        for l in lines:
+            if body != "":
+                body += "\n"
+            body += ">%s" % l
+        self.body = body
+
+
+class ForwardModifier(Modifier):
+    def __init__(self, *args, **kwargs):
+        super(ForwardModifier, self).__init__(*args, **kwargs)
+        self._header()
+        self.form.fields["subject"].initial = "Fwd: %s" % self.Subject
+
+    def __getfunc(self, name):
+        return getattr(self, "%s_%s" % (name, self.dformat))
+
+    def _header(self):
+        self.textheader = self.__getfunc("_header_begin")() + "\n"
+        self.textheader += \
+            self.__getfunc("_header_line")(_("Subject"), self.Subject)
+        self.textheader += \
+            self.__getfunc("_header_line")(_("Date"), self.Date)
+        for hdr in ["From", "To", "Reply-To"]:
+            try:
+                key = re.sub("-", "_", hdr)
+                value = getattr(self, key)
+                self.textheader += \
+                    self.__getfunc("_header_line")(_(hdr), value)
+            except AttributeError:
+                pass
+        self.textheader += self.__getfunc("_header_end")()
+
+    def _header_begin_plain(self):
+        return "----- %s -----" % _("Original message")
+
+    def _header_begin_html(self):
+        return  "----- %s -----" % _("Original message")
+
+    def _header_line_plain(self, key, value):
+        return "%s: %s\n" % (key, value)
+
+    def _header_line_html(self, key, value):
+        return "<p>%s: %s</p>" % (key, value)
+
+    def _header_end_plain(self):
+        return "\n"
+
+    def _header_end_html(self):
+        return ""

modoboa/extensions/webmail/lib/imapheader.py

@@ -0,0 +1,114 @@
+"""
+Set of functions used to parse and transform email headers.
+"""
+import chardet
+from modoboa.lib.emailutils import EmailAddress
+
+__all__ = [
+    'parse_from', 'parse_to', 'parse_message_id', 'parse_date',
+    'parse_reply_to', 'parse_cc', 'parse_subject'
+]
+
+
+def to_unicode(value):
+    """Try to convert a string to unicode.
+    """
+    if value is None or type(value) is unicode:
+        return value
+    try:
+        value = value.decode('utf-8')
+    except UnicodeDecodeError:
+        pass
+    else:
+        return value
+    try:
+        res = chardet.detect(value)
+    except UnicodeDecodeError:
+        return value
+    if res["encoding"] == "ascii":
+        return value
+    return value.decode(res["encoding"])
+
+
+def parse_address(value, **kwargs):
+    """Parse an email address.
+    """
+    addr = EmailAddress(value)
+    if "full" in kwargs.keys() and kwargs["full"]:
+        return to_unicode(addr.fulladdress)
+    result = addr.name and addr.name or addr.fulladdress
+    return to_unicode(result)
+
+
+def parse_address_list(values, **kwargs):
+    """Parse a list of email addresses.
+    """
+    lst = values.split(",")
+    result = ""
+    for addr in lst:
+        if result != "":
+            result += ", "
+        result += parse_address(addr, **kwargs)
+    return result
+
+
+def parse_from(value, **kwargs):
+    """Parse a From: header.
+    """
+    return parse_address(value, **kwargs)
+
+
+def parse_to(value, **kwargs):
+    """Parse a To: header.
+    """
+    return parse_address_list(value, **kwargs)
+
+
+def parse_cc(value, **kwargs):
+    """Parse a Cc: header.
+    """
+    return parse_address_list(value, **kwargs)
+
+
+def parse_reply_to(value, **kwargs):
+    """Parse a Reply-To: header.
+    """
+    return parse_address_list(value, **kwargs)
+
+
+def parse_date(value, **kwargs):
+    """Parse a Date: header.
+    """
+    import datetime
+    import email
+
+    tmp = email.utils.parsedate_tz(value)
+    if not tmp:
+        return value
+    ndate = datetime.datetime(*(tmp)[:7])
+    now = datetime.datetime.now()
+    try:
+        if now - ndate > datetime.timedelta(7):
+            return ndate.strftime("%d.%m.%Y %H:%M")
+        return ndate.strftime("%a %H:%M")
+    except ValueError:
+        return value
+
+
+def parse_message_id(value, **kwargs):
+    """Parse a Message-ID: header.
+    """
+    return value.strip('\n')
+
+
+def parse_subject(value, **kwargs):
+    """Parse a Subject: header.
+    """
+    from modoboa.lib import u2u_decode
+
+    try:
+        subject = u2u_decode.u2u_decode(value)
+    except UnicodeDecodeError:
+        subject = value
+    return to_unicode(subject)
+

modoboa/extensions/webmail/lib/imaputils.py

@@ -14,8 +14,8 @@ from modoboa.lib import parameters, imap_utf7
 from modoboa.lib.exceptions import InternalError
 from modoboa.lib.connections import ConnectionsManager
 from modoboa.lib.webutils import static_url
-from exceptions import ImapError, WebmailInternalError
-from fetch_parser import parse_fetch_response
+from ..exceptions import ImapError, WebmailInternalError
+from .fetch_parser import parse_fetch_response
 
 #imaplib.Debug = 4
 

modoboa/extensions/webmail/lib/listing.py

@@ -0,0 +1,107 @@
+from django.conf import settings
+from django.utils.translation import ugettext_lazy
+
+from modoboa.lib import parameters, tables
+from modoboa.lib.email_listing import EmailListing
+from . import IMAPconnector, imapheader
+
+
+class WMtable(tables.Table):
+    """
+    Webmail listing table.
+    """
+    tableid = "emails"
+    styles = "table-condensed"
+    idkey = "imapid"
+    drag = tables.ImgColumn(
+        "drag", cssclass="draggable left", width="1%",
+        defvalue="%spics/grippy.png" % settings.STATIC_URL,
+        header="<input type='checkbox' name='toggleselect' id='toggleselect' />"
+    )
+    selection = tables.SelectionColumn(
+        "selection", safe=True, width='1%', header=None, sortable=False
+    )
+    flags = tables.ImgColumn("flags", width="3%")
+    withatts = tables.ImgColumn("withatts", width="2%")
+    subject = tables.Column(
+        "subject", label=ugettext_lazy("Subject"), width="50%", limit=60,
+        cssclass="openable"
+    )
+    from_ = tables.Column(
+        "from", width="20%", label=ugettext_lazy("From"), limit=30,
+        cssclass="openable"
+    )
+    date = tables.Column(
+        "date", width="15%", label=ugettext_lazy("Date"), cssclass="openable"
+    )
+
+    cols_order = [
+        "drag", "selection", "withatts", "flags", "subject", "from_", "date"
+    ]
+
+    def parse(self, header, value):
+        if value is None:
+            return ""
+        try:
+            value = getattr(imapheader, "parse_%s" % header)(value)
+        except AttributeError:
+            pass
+        return value
+
+
+class ImapListing(EmailListing):
+    tpl = "webmail/index.html"
+    tbltype = WMtable
+    deflocation = "INBOX/"
+    defcallback = "wm_updatelisting"
+    reset_wm_url = False
+
+    def __init__(self, user, password, **kwargs):
+        self.user = user
+        self.mbc = IMAPconnector(user=user.username, password=password)
+        if "pattern" in kwargs and kwargs["pattern"]:
+            self.parse_search_parameters(
+                kwargs["criteria"], kwargs["pattern"]
+            )
+        else:
+            self.mbc.criterions = []
+
+        super(ImapListing, self).__init__(**kwargs)
+        self.extravars["refreshrate"] = \
+            int(parameters.get_user(user, "REFRESH_INTERVAL")) * 1000
+
+    def parse_search_parameters(self, criterion, pattern):
+
+        def or_criterion(old, c):
+            if old == "":
+                return c
+            return "OR (%s) (%s)" % (old, c)
+
+        if criterion == u"both":
+            criterion = u"from_addr, subject"
+        criterions = ""
+        if type(pattern) is unicode:
+            pattern = pattern.encode("utf-8")
+        if type(criterion) is unicode:
+            criterion = criterion.encode("utf-8")
+        for c in criterion.split(','):
+            if c == "from_addr":
+                key = "FROM"
+            elif c == "subject":
+                key = "SUBJECT"
+            else:
+                continue
+            criterions = \
+                or_criterion(criterions, '(%s "%s")' % (key, pattern))
+        self.mbc.criterions = [criterions]
+
+    @staticmethod
+    def computequota(mbc):
+        try:
+            return int(float(mbc.quota_actual) \
+                / float(mbc.quota_limit) * 100)
+        except (AttributeError, TypeError):
+            return -1
+
+    def getquota(self):
+        return ImapListing.computequota(self.mbc)

modoboa/extensions/webmail/lib/sendmail.py

@@ -0,0 +1,65 @@
+import smtplib
+from modoboa.lib import parameters
+from modoboa.lib.emailutils import prepare_addresses
+from modoboa.lib.webutils import _render_to_string
+from modoboa.lib.cryptutils import get_password
+from modoboa.extensions.webmail.exceptions import WebmailInternalError
+from . import get_imapconnector, clean_attachments
+
+
+def send_mail(request, form, posturl=None):
+    """Email verification and sending.
+
+    If the form does not present any error, a new MIME message is
+    constructed. Then, a connection is established with the defined
+    SMTP server and the message is finally sent.
+
+    :param request: a Request object
+    :param posturl: the url to post the message form to
+    :return: a 2-uple (True|False, HttpResponse)
+    """
+    if not form.is_valid():
+        editormode = parameters.get_user(request.user, "EDITOR")
+        listing = _render_to_string(
+            request, "webmail/compose.html",
+            {"form": form, "noerrors": True,
+             "body": form.cleaned_data.get("body", "").strip(),
+             "posturl": posturl}
+        )
+        return False, dict(status="ko", listing=listing, editor=editormode)
+
+    msg = form.to_msg(request)
+    rcpts = prepare_addresses(form.cleaned_data["to"], "envelope")
+    for hdr in ["cc", "cci"]:
+        if form.cleaned_data[hdr]:
+            msg[hdr.capitalize()] = prepare_addresses(form.cleaned_data[hdr])
+            rcpts += prepare_addresses(form.cleaned_data[hdr], "envelope")
+    try:
+        secmode = parameters.get_admin("SMTP_SECURED_MODE")
+        if secmode == "ssl":
+            s = smtplib.SMTP_SSL(parameters.get_admin("SMTP_SERVER"),
+                                 int(parameters.get_admin("SMTP_PORT")))
+        else:
+            s = smtplib.SMTP(parameters.get_admin("SMTP_SERVER"),
+                             int(parameters.get_admin("SMTP_PORT")))
+            if secmode == "starttls":
+                s.starttls()
+    except Exception as text:
+        raise WebmailInternalError(str(text))
+
+    if parameters.get_admin("SMTP_AUTHENTICATION") == "yes":
+        try:
+            s.login(request.user.username, get_password(request))
+        except smtplib.SMTPException as err:
+            raise WebmailInternalError(str(err))
+    try:
+        s.sendmail(request.user.email, rcpts, msg.as_string())
+        s.quit()
+    except smtplib.SMTPException as err:
+        raise WebmailInternalError(str(err))
+
+    sentfolder = parameters.get_user(request.user, "SENT_FOLDER")
+    get_imapconnector(request).push_mail(sentfolder, msg)
+    clean_attachments(request.session["compose_mail"]["attachments"])
+    del request.session["compose_mail"]
+    return True, {}

modoboa/extensions/webmail/lib/signature.py

@@ -0,0 +1,29 @@
+import re
+from modoboa.lib import parameters
+
+
+class EmailSignature(object):
+    """User signature
+
+    :param user: User object
+    """
+    def __init__(self, user):
+        self._sig = ""
+        dformat = parameters.get_user(user, "EDITOR")
+        content = parameters.get_user(user, "SIGNATURE")
+        if len(content):
+            getattr(self, "_format_sig_%s" % dformat)(content)
+
+    def _format_sig_plain(self, content):
+        self._sig = """
+---
+%s""" % content
+
+    def _format_sig_html(self, content):
+        content = re.sub("\n", "<br/>", content)
+        self._sig = """<br/>
+---<br/>
+%s""" % content
+
+    def __repr__(self):
+        return self._sig

modoboa/extensions/webmail/lib/tables.py

@@ -0,0 +1,48 @@
+from django.conf import settings
+from django.utils.translation import ugettext_lazy
+
+from modoboa.lib import tables
+from . import imapheader
+
+
+class WMtable(tables.Table):
+    """
+    The webmail listing.
+    """
+    tableid = "emails"
+    styles = "table-condensed"
+    idkey = "imapid"
+    drag = tables.ImgColumn(
+        "drag", cssclass="draggable left", width="1%",
+        defvalue="%spics/grippy.png" % settings.STATIC_URL,
+        header="<input type='checkbox' name='toggleselect' id='toggleselect' />"
+    )
+    selection = tables.SelectionColumn(
+        "selection", safe=True, width='1%', header=None, sortable=False
+    )
+    flags = tables.ImgColumn("flags", width="3%")
+    withatts = tables.ImgColumn("withatts", width="2%")
+    subject = tables.Column(
+        "subject", label=ugettext_lazy("Subject"), width="50%", limit=60,
+        cssclass="openable"
+    )
+    from_ = tables.Column(
+        "from", width="20%", label=ugettext_lazy("From"), limit=30,
+        cssclass="openable"
+    )
+    date = tables.Column(
+        "date", width="15%", label=ugettext_lazy("Date"), cssclass="openable"
+    )
+
+    cols_order = [
+        "drag", "selection", "withatts", "flags", "subject", "from_", "date"
+    ]
+
+    def parse(self, header, value):
+        if value is None:
+            return ""
+        try:
+            value = getattr(imapheader, "parse_%s" % header)(value)
+        except AttributeError:
+            pass
+        return value

modoboa/extensions/webmail/lib/utils.py

@@ -0,0 +1,35 @@
+"""
+Misc. utilities.
+"""
+from modoboa.lib.webutils import NavigationParameters
+
+
+def decode_payload(encoding, payload):
+    """Decode the payload according to the given encoding
+
+    Supported encodings: base64, quoted-printable.
+
+    :param encoding: the encoding's name
+    :param payload: the value to decode
+    :return: a string
+    """
+    encoding = encoding.lower()
+    if encoding == "base64":
+        import base64
+        return base64.b64decode(payload)
+    elif encoding == "quoted-printable":
+        import quopri
+        return quopri.decodestring(payload)
+    return payload
+
+
+class WebmailNavigationParameters(NavigationParameters):
+    """
+    Specific NavigationParameters subclass for the webmail.
+    """
+    def __init__(self, request, defmailbox=None):
+        super(WebmailNavigationParameters, self).__init__(
+            request, 'webmail_navparams'
+        )
+        if defmailbox is not None:
+            self.parameters += [('mbox', defmailbox, False)]

modoboa/extensions/webmail/static/webmail/js/webmail.js

@@ -753,7 +753,7 @@ Webmail.prototype = {
         }, this)).fail($.proxy(function(jqxhr) {
             var data = $.parseJSON(jqxhr.responseText);
             if (data != undefined) {
-                $("body").notify("error", data);
+                this.compose_callback(data);
             }
             $("a[name=sendmail]").attr("disabled", null);
         }, this));

modoboa/extensions/webmail/templates/webmail/compose.html

@@ -23,11 +23,11 @@
 
     <div id="body_container">
     {% if body %}
-    <textarea id="id_body" name="id_body" class="editor">{% if bodyheader %}{{ bodyheader }}{% endif %}
+    <textarea id="id_body" name="body" class="editor">{% if bodyheader %}{{ bodyheader }}{% endif %}
 {{ body }}
 </textarea>
     {% else %}
-    <textarea id="id_body" name="id_body" class="editor"></textarea>
+    <textarea id="id_body" name="body" class="editor"></textarea>
     {% endif %}
     </div>
   </div>

modoboa/extensions/webmail/templatetags/webmail_tags.py

@@ -5,7 +5,7 @@ from django.utils.translation import ugettext as _
 from django.core.urlresolvers import reverse
 from django.conf import settings
 from modoboa.extensions import webmail
-from modoboa.extensions.webmail.imaputils import separate_mailbox
+from modoboa.extensions.webmail.lib import separate_mailbox
 from modoboa.lib import parameters
 
 register = template.Library()

modoboa/extensions/webmail/views.py

@@ -17,13 +17,15 @@ from modoboa.lib.webutils import (
 )
 from modoboa.extensions.admin.lib import needs_mailbox
 from .exceptions import UnknownAction
-from .forms import FolderForm, AttachmentForm, ComposeMailForm
-from .imaputils import get_imapconnector, IMAPconnector, separate_mailbox
+from .forms import (
+    FolderForm, AttachmentForm, ComposeMailForm, ForwardMailForm
+)
 from .lib import (
     decode_payload, AttachmentUploadHandler,
     save_attachment, ImapListing, EmailSignature,
     clean_attachments, set_compose_session, send_mail,
-    ImapEmail, WebmailNavigationParameters, ReplyModifier, ForwardModifier
+    ImapEmail, WebmailNavigationParameters, ReplyModifier, ForwardModifier,
+    get_imapconnector, IMAPconnector, separate_mailbox
 )
 from templatetags import webmail_tags
 
@@ -382,43 +384,62 @@ def render_compose(request, form, posturl, email=None, insert_signature=False):
 def compose(request):
     url = "?action=compose"
     if request.method == "POST":
-        status, resp = send_mail(request, posturl=url)
+        form = ComposeMailForm(request.POST)
+        status, resp = send_mail(request, form, posturl=url)
         return resp
 
     form = ComposeMailForm()
     return render_compose(request, form, url, insert_signature=True)
 
 
-def compose_and_send(request, action, callback=None):
+def get_mail_info(request):
+    """Retrieve a mailbox and an email ID from a request.
+    """
     mbox = request.GET.get("mbox", None)
     mailid = request.GET.get("mailid", None)
     if mbox is None or mailid is None:
         raise BadRequest(_("Invalid request"))
-    url = "?action=%s&mbox=%s&mailid=%s" % (action, mbox, mailid)
-    if request.method == "POST":
-        status, resp = send_mail(request, url)
-        if status and callback:
-            callback(mbox, mailid)
-        return resp
+    return mbox, mailid
 
+
+def new_compose_form(request, action, mbox, mailid):
+    """Return a new composition form.
+
+    Valid for reply and forward actions only.
+    """
     form = ComposeMailForm()
     modclass = globals()["%sModifier" % action.capitalize()]
-    email = modclass(mbox, mailid, request, form, addrfull=True, links="1")
+    email = modclass(form, request, True, "%s:%s" % (mbox, mailid), links="1")
+    url = "?action=%s&mbox=%s&mailid=%s" % (action, mbox, mailid)
     return render_compose(request, form, url, email)
 
 
 def reply(request):
-    def msg_replied(mbox, mailid):
-        get_imapconnector(request).msg_answered(mbox, mailid)
-
-    return compose_and_send(request, "reply", msg_replied)
+    """Reply to email.
+    """
+    mbox, mailid = get_mail_info(request)
+    if request.method == "POST":
+        url = "?action=reply&mbox=%s&mailid=%s" % (mbox, mailid)
+        form = ComposeMailForm(request.POST)
+        status, resp = send_mail(request, form, url)
+        if status:
+            get_imapconnector(request).msg_answered(mbox, mailid)
+        return resp
+    return new_compose_form(request, "reply", mbox, mailid)
 
 
 def forward(request):
-    def msg_forwarded(mbox, mailid):
-        get_imapconnector(request).msg_forwarded(mbox, mailid)
-
-    return compose_and_send(request, "forward", msg_forwarded)
+    """Forward email.
+    """
+    mbox, mailid = get_mail_info(request)
+    if request.method == "POST":
+        url = "?action=forward&mbox=%s&mailid=%s" % (mbox, mailid)
+        form = ForwardMailForm(request.POST)
+        status, resp = send_mail(request, form, url)
+        if status:
+            get_imapconnector(request).msg_forwarded(mbox, mailid)
+        return resp
+    return new_compose_form(request, "forward", mbox, mailid)
 
 
 @login_required
@@ -428,7 +449,10 @@ def getmailcontent(request):
     mailid = request.GET.get("mailid", None)
     if mbox is None or mailid is None:
         raise BadRequest(_("Invalid request"))
-    email = ImapEmail(mbox, mailid, request, links=int(request.GET["links"]))
+    email = ImapEmail(
+        request, False, "%s:%s" % (mbox, mailid), dformat="DISPLAYMODE",
+        links=int(request.GET["links"])
+    )
     return render(request, "common/viewmail.html", {
         "headers": email.render_headers(folder=mbox, mail_id=mailid),
         "folder": mbox, "imapid": mailid,

modoboa/lib/emailutils.py

@@ -5,9 +5,10 @@ import smtplib
 from email.header import Header, decode_header
 from email.mime.text import MIMEText
 from email.utils import make_msgid, formatdate, parseaddr
+import lxml.html
 from django.template.loader import render_to_string
 from django.conf import settings
-import u2u_decode
+from modoboa.lib import u2u_decode
 
 
 class EmailAddress(object):
@@ -22,26 +23,49 @@ class EmailAddress(object):
 
 
 class Email(object):
-    def __init__(self, msg, mformat="plain", dformat="plain", links=0):
+    def __init__(self, mailid, mformat="plain", dformat="plain", links=0):
         self.attached_map = {}
         self.contents = {"html": "", "plain": ""}
         self.headers = []
         self.attachments = {}
+        self.mailid = mailid
         self.mformat = mformat
         self.dformat = dformat
         self.links = links
-
-        self.__parse(msg)
+        self._msg = None
+        self._body = None
 
         if not mformat in self.contents or self.contents[mformat] == "":
             # Fallback
             self.mformat = mformat == "html" and "plain" or "html"
 
-        self.body = \
-            getattr(self, "viewmail_%s" % self.mformat) \
-            (self.contents[self.mformat], links=links)
+    @property
+    def msg(self):
+        """Return an email.message object.
+        """
+        raise NotImplementedError
 
-    def __parse_default(self, msg, level):
+    @property
+    def body(self):
+        """Return email's body.
+        """
+        if self._body is None:
+            self._body = getattr(self, "viewmail_%s" % self.mformat) \
+                (self.contents[self.mformat], links=self.links)
+        return self._body
+
+    def get_header(self, msg, hdrname):
+        """Look for a particular header.
+
+        :param string hdrname: header name
+        :return: header avalue
+        """
+        for name in [hdrname, hdrname.upper()]:
+            if name in msg:
+                return msg[name]
+        return ""
+
+    def _parse_default(self, msg, level):
         """Default parser
 
         All parts handled by this parser will be consireded as
@@ -63,7 +87,7 @@ class Email(object):
         inside a navigator.
         """
         if msg.get_content_subtype() not in ["plain", "html"]:
-            self.__parse_default(msg, level)
+            self._parse_default(msg, level)
             target = "plain"
         else:
             target = msg.get_content_subtype()
@@ -96,9 +120,9 @@ class Email(object):
                 self.attached_map[cid] = re.match("^http:", fname) and fname \
                     or self.__save_image(fname, msg)
                 return
-        self.__parse_default(msg, level)
+        self._parse_default(msg, level)
 
-    def __parse(self, msg, level=None):
+    def _parse(self, msg, level=None):
         """Recursive email parser
 
         A message structure can be complex. To correctly handle
@@ -116,7 +140,7 @@ class Email(object):
             for part in msg.get_payload():
                 nlevel = level is None and ("%d" % cpt) \
                     or "%s.%d" % (level, cpt)
-                self.__parse(part, nlevel)
+                self._parse(part, nlevel)
                 cpt += 1
             return
 
@@ -125,7 +149,7 @@ class Email(object):
         try:
             getattr(self, "_parse_%s" % msg.get_content_maintype())(msg, level)
         except AttributeError:
-            self.__parse_default(msg, level)
+            self._parse_default(msg, level)
 
     def __save_image(self, fname, part):
         """Save an inline image on the filesystem.
@@ -138,7 +162,7 @@ class Email(object):
         :param fname: the image associated filename
         :param part: the email part that contains the image payload
         """
-        if re.search("\.\.", fname):
+        if re.search(r"\.\.", fname):
             return None
         path = "/static/tmp/" + fname
         fp = open(settings.MODOBOA_DIR + path, "wb")

modoboa/lib/u2u_decode.py

@@ -27,7 +27,10 @@ def decode_mime(m):
     """substitute matching encoded_word with unicode equiv.
     """
     h = decode_header(clean_spaces(m))
-    u = unicode(make_header(h))
+    try:
+        u = unicode(make_header(h))
+    except UnicodeDecodeError:
+        return m.group(0)
     return u