wip: Webmail big refactoring

* New fetch responses parser
* Better detection of attachments

extensions/webmail/fetch_parser.py

@@ -0,0 +1,173 @@
+# coding: utf-8
+"""
+:mod:`fetch_parser` --- Simple parser for FETCH responses
+---------------------------------------------------------
+
+The ``imaplib`` module doesn't parse IMAP responses, it returns raw
+values. This is pretty annoying when one FETCH command is issued to
+retrieve multiple attributes of a message.
+
+This simple module tries to fix that *problem*.
+"""
+
+class ParseError(Exception): pass
+
+def parse_next_token(buf):
+    """Look for the next *token*
+
+    By *token*, I mean: *literal*, *quoted* or anything else until the
+    next ' ' or ')' character (number, NIL and others should fall into
+    this last category).
+
+    :param buf: the buffer to parse
+    :return: the position of the token's last caracter into ``buf``
+    """
+    end = -1
+    if buf[0] == '{':
+        # Literal
+        end = buf.find('}')
+    elif buf[0] == '"':
+        # quoted
+        end = buf.find('"', 1)
+    else:
+        for pos, c in enumerate(buf):
+            if c in [' ', ')']:
+                end = pos - 1
+                break
+        if end == -1:
+            raise ParseError("End of buffer reached while looking for a token end")
+    return end
+
+def parse_bodystructure(buf, depth=0, prefix=""):
+    """Special parser for BODYSTRUCTURE response
+
+    This function tries to transform a BODYSTRUCTURE response sent by
+    the server into the corresponding list structure.
+
+    :param buf: the buffer to parse
+    :return: a list object and the position of the last scanned
+             character into ``buf``
+    """
+    ret = []
+    pos = 1 # skip the first ')'
+    nb_bodystruct = 0
+    pnum = 1
+    while pos < len(buf):
+        c = buf[pos]
+        if c == '(':
+            nprefix = "%s.%s" % (prefix, pnum) if prefix != "" else "%s" % pnum
+            subret, end = parse_bodystructure(buf[pos:], depth + 1, nprefix)
+            pnum += 1
+            if nb_bodystruct == 0:
+                ret.append(subret)
+            elif nb_bodystruct == 1:
+                newret = [ret.pop(-1)]
+                newret.append(subret)
+                ret.append(newret)
+            else:
+                ret[-1].append(subret)
+            pos += end + 1
+            nb_bodystruct += 1
+            continue
+
+        if c == ')':
+            partnum = None
+            if depth:
+                if type(ret[0]) == list or len(ret) >= 7:
+                    partnum = prefix
+            if partnum is not None:
+                return [dict(partnum=partnum, struct=ret)], pos
+            return ret, pos
+        if c == ' ':
+            pos += 1
+            continue
+        nb_bodystruct = 0
+        end = parse_next_token(buf[pos:])
+        ret.append(buf[pos:pos + end + 1].strip('"'))
+        pos += end + 1
+
+    raise ParseError("End of buffer reached while looking for a BODY/BODYSTRUCTURE end")
+
+def parse_fetch_response(data):
+    """Parse a FETCH response, previously issued by a UID command
+
+    We extract the message number, the UID and its value and consider
+    what remains as the response data.
+    
+    :param data: the data returned by the ``imaplib`` command
+    :return: a dictionnary
+    """
+    result = {}
+    for part in data[0:-1]:
+        if part == ')':
+            continue
+        buf = "".join(part)
+        parts = buf.split(' ', 3)
+        msgdef = result[int(parts[2])] = {}
+        response = parts[3]
+
+        while len(response):
+            if response.startswith('BODY') and response[4] == '[':
+                end = response.find(']', 5)
+                if response[end + 1] == '<':
+                    end = response.find('>', end + 1)
+                end += 1
+            else:
+                end = response.find(' ')
+            cmdname = response[:end]
+            response = response[end + 1:]
+
+            end = 0
+            if cmdname in ['BODY', 'BODYSTRUCTURE', 'FLAGS']:
+                parendepth = 0
+                for pos, c in enumerate(response):
+                    if c == '(':
+                        parendepth += 1
+                        continue
+                    if c == ')':
+                        parendepth -= 1
+                        if parendepth == 0:
+                            end = pos + 1
+                            break
+            else:
+                end = parse_next_token(response)
+                if response[0] == '{':
+                    datalen = int(response[1:end])
+                    response = response[end + 1:]
+                    end = datalen
+
+            msgdef[cmdname] = response[:end]
+            response = response[end + 1:]
+            try:
+                func = globals()["parse_%s" % cmdname.lower()]
+                msgdef[cmdname] = func(msgdef[cmdname])[0]
+            except KeyError:
+                pass
+
+    return result
+
+def dump_bodystructure(bs, depth=0):
+    if depth:
+        print " " * (depth * 4),
+    if type(bs[0]) == dict:
+        print "%s :" % bs[0]["partnum"],
+        struct = bs[0]["struct"]
+    else:
+        struct = bs
+
+    if type(struct[0]) == list:
+        print "multipart/%s" % struct[1]
+        for part in struct[0]:
+            dump_bodystructure(part, depth + 1)
+    else:
+        print "%s/%s" % (struct[0], struct[1])
+        
+
+if __name__ == "__main__":
+    resp = [('855 (UID 46931 BODYSTRUCTURE ((("text" "plain" ("charset" "iso-8859-1") NIL NIL "quoted-printable" 886 32 NIL NIL NIL NIL)("text" "html" ("charset" "us-ascii") NIL NIL "quoted-printable" 1208 16 NIL NIL NIL NIL) "alternative" ("boundary" "----=_NextPart_001_0003_01CCC564.B2F64FF0") NIL NIL NIL)("application" "octet-stream" ("name" "Carte Verte_2.pdf") NIL NIL "base64" 285610 NIL ("attachment" ("filename" "Carte Verte_2.pdf")) NIL NIL) "mixed" ("boundary" "----=_NextPart_000_0002_01CCC564.B2F64FF0") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {153}', 'From: <Service.client10@maaf.fr>\r\nTo: <TONIO@NGYN.ORG>\r\nCc: \r\nSubject: Notre contact du 28/12/2011 - 192175092\r\nDate: Wed, 28 Dec 2011 13:29:17 +0100\r\n\r\n'), ')']
+
+    #resp = [('856 (UID 46936 BODYSTRUCTURE (("text" "plain" ("charset" "ISO-8859-1") NIL NIL "quoted-printable" 724 22 NIL NIL NIL NIL)("text" "html" ("charset" "ISO-8859-1") NIL NIL "quoted-printable" 2662 48 NIL NIL NIL NIL) "alternative" ("boundary" "----=_Part_1326887_254624357.1325083973970") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {258}', 'Date: Wed, 28 Dec 2011 15:52:53 +0100 (CET)\r\nFrom: =?ISO-8859-1?Q?Malakoff_M=E9d=E9ric?= <communication@communication.malakoffmederic.com>\r\nTo: Antoine Nguyen <tonio@ngyn.org>\r\nSubject: =?ISO-8859-1?Q?Votre_inscription_au_grand_Jeu_Malakoff_M=E9d=E9ric?=\r\n\r\n'), ')']
+
+    resp = [('856 (UID 11111 BODYSTRUCTURE ((("text" "plain" ("charset" "UTF-8") NIL NIL "7bit" 0 0 NIL NIL NIL NIL) "mixed" ("boundary" "----=_Part_407172_3159001.1321948277321") NIL NIL NIL)("application" "octet-stream" ("name" "26274308.pdf") NIL NIL "base64" 14906 NIL ("attachment" ("filename" "26274308.pdf")) NIL NIL) "mixed" ("boundary" "----=_Part_407171_9686991.1321948277321") NIL NIL NIL)'), ')']
+
+    print parse_fetch_response(resp)

extensions/webmail/imaputils.py

@@ -11,6 +11,7 @@ from modoboa.lib import parameters
 from modoboa.lib.connections import *
 from modoboa.lib.webutils import static_url
 from exceptions import ImapError
+from fetch_parser import *
 
 #imaplib.Debug = 4
 
@@ -35,6 +36,86 @@ class capability(object):
 
         return wrapped_func
 
+class BodyStructure(object):
+    """
+    BODYSTRUCTURE response parser.
+
+    Just a simple class that tries to distinguish content parts from
+    attachments.
+    """
+    def __init__(self, definition=None):
+        self.is_multipart = False
+        self.contents = {}
+        self.attachments = []
+        self.inlines = {}
+
+        if definition is not None:
+            self.load_from_definition(definition)
+
+    def __store_part(self, definition, pnum, multisubtype):
+        """Store the given message part in the appropriate category.
+
+        This method sort parts in two categories:
+
+        * contents (what is going to be displayed)
+        * attachments
+
+        As there is no official definition about what is a content and
+        what is an attachment, the following rules are applied:
+
+        * If the MIME type is text/plain or text/html:
+
+         * If no previous part of this type has already been seen, it's a content
+         * Otherwise it's an attachment
+
+        * Else, if the multipart subtype is related, we consider this
+          part as content because it is certainly an embedded image
+
+        * Any other MIME type is considered as an attachment (for now)
+
+        :param definition: a part definition (list)
+        :param prefix: the part's number
+        :param multisubtype: the multipart subtype
+        """
+        pnum = "1" if pnum is None else pnum
+        params = dict(pnum=pnum, params=definition[2], cid=definition[3],
+                      description=definition[4], encoding=definition[5], 
+                      size=definition[6])
+        subtype = definition[1].lower()
+        mtype = "%s/%s" % (definition[0].lower(), subtype)
+        if mtype in ("text/plain", "text/html"):
+            if not self.contents.has_key(mtype):
+                self.contents[subtype] = params
+                return
+        elif multisubtype in ["related"]:
+            self.inlines[params["cid"].strip("<>")] = params
+            return
+
+        if len(definition) > 7:
+            params.update(md5=definition[7], disposition=definition[8], 
+                          lang=definition[9], location=definition[10])
+        self.attachments += [params]
+
+    def load_from_definition(self, definition, multisubtype=None):
+        if type(definition) == dict:
+            struct = definition["struct"]
+            pnum = definition["partnum"]
+        elif type(definition[0]) == dict:
+            struct = definition[0]["struct"]
+            pnum = definition[0]["partnum"]
+        else:
+            struct = definition
+            pnum = None
+
+        if type(struct[0]) == list:
+            for part in struct[0]:
+                self.load_from_definition(part, struct[1])
+            return
+        
+        self.__store_part(struct, pnum, multisubtype)
+
+    def has_attachments(self):
+        return len(self.attachments)
 
 class IMAPconnector(object):
     __metaclass__ = ConnectionsManager
@@ -105,13 +186,14 @@ class IMAPconnector(object):
             for part in bodystruct[0]:
                 nprefix = "%s" % cpt if prefix == "" else "%s.%d" % (prefix, cpt)
                 index, encoding, size = \
-                    self.__find_content_in_bodystruct(bodystruct[0], mtype, stype, nprefix)
+                    self.__find_content_in_bodystruct(part, mtype, stype, nprefix)
-                if index:
+                if index is not None:
                     return (index, encoding, size)
                 cpt += 1
-
+        else:
-        if bodystruct[0] == mtype and bodystruct[1] == stype:
+            if bodystruct[0].lower() == mtype and bodystruct[1].lower() == stype:
-            return ("1" if not len(prefix) else prefix, bodystruct[5], int(bodystruct[6]))
+                return ("1" if not len(prefix) else prefix, 
+                        bodystruct[5], int(bodystruct[6]))
         return (None, None, 0)
 
     def refresh(self, user, password):
@@ -401,14 +483,10 @@ class IMAPconnector(object):
             self.quota_limit = int(m.group(2))
             self.quota_actual = int(m.group(1))
 
-    def fetchpart(self, uid, folder, part):
+    def fetchpart(self, uid, mbox, partnum):
-        self.m.select(self._encodefolder(folder), True)
+        self.select_mailbox(mbox, False)
-        typ, data = self.m.fetch(uid, "(BODY[%(p)s.MIME] BODY[%(p)s])" \
+        data = self._cmd("FETCH", uid, "(BODY[%s])" % partnum)
-                                     % {"p" : part})
+        return data[int(uid)]["BODY[%s]" % partnum]
-        if typ != "OK":
-            return None
-        msg = email.message_from_string(data[0][1] + data[1][1])
-        return msg
 
     def fetch(self, start, stop=None, mbox=None, **kwargs):
         """Retrieve information about messages from the server
@@ -427,7 +505,7 @@ class IMAPconnector(object):
         else:
             submessages = [start]
             range = start
-        query = '(FLAGS BODY.PEEK[HEADER.FIELDS (DATE FROM TO CC SUBJECT)])'
+        query = '(FLAGS BODYSTRUCTURE BODY.PEEK[HEADER.FIELDS (DATE FROM TO CC SUBJECT)])'
         data = self._cmd("FETCH", range, query)
         result = []
         for uid in submessages:
@@ -437,10 +515,13 @@ class IMAPconnector(object):
                 msg['class'] = 'unseen'
             if r'\Answered' in data[int(uid)]['FLAGS']:
                 msg['img_flags'] = static_url('pics/answered.png')
+            bs = BodyStructure(data[int(uid)]['BODYSTRUCTURE'])
+            if bs.has_attachments():
+                msg['img_withatts'] = static_url('pics/attachment.png')
             result += [msg]
         return result
 
-    def fetchmail(self, mbox, mailid, fmt="plain"):
+    def fetchmail(self, mbox, mailid):
         """Retrieve information about a specific message
 
         Issue a FETCH command to retrieve a message's content from the
@@ -455,34 +536,15 @@ class IMAPconnector(object):
         """
         self.select_mailbox(mbox, False)
         data = self._cmd("FETCH", mailid, "(BODYSTRUCTURE BODY.PEEK[HEADER.FIELDS (DATE FROM TO CC SUBJECT)])")
-        content = data[int(mailid)]['BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)]']
+        return data[int(mailid)]
-
-        fallback_fmt = "html" if fmt == "plain" else "plain"
-        (pnum, encoding, size) = (None, None, 0)
-        for f in [fmt, fallback_fmt]:
-            pnum, encoding, size = \
-                self.__find_content_in_bodystruct(data[int(mailid)]['BODYSTRUCTURE'],
-                                                  "text", f)
-            if pnum is not None and size:
-                break
-        if pnum is None:
-            return None
-
-        data = self._cmd("FETCH", mailid, "(BODY.PEEK[%s])" % pnum)
-
-        data = data[int(mailid)]['BODY[%s]' % pnum]        
-        if encoding == "base64":
-            import base64
-            content += base64.b64decode(data)
-        elif encoding == "quoted-printable":
-            import quopri
-            content += quopri.decodestring(data)
-        else:
-            content += data
-
-        return email.message_from_string(content)
 
 def get_imapconnector(request):
     imapc = IMAPconnector(user=request.user.username, 
                           password=request.session["password"])
     return imapc
+
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()
+

extensions/webmail/lib.py

@@ -7,7 +7,6 @@ sys.setdefaultencoding("utf-8")
 import re
 import time
 import socket
-import imaplib
 import ssl
 from datetime import datetime, timedelta
 import email
@@ -31,12 +30,13 @@ class WMtable(tables.Table):
     select = tables.ImgColumn("select", cssclass="draggable left", width="1%", 
                               defvalue="%spics/grippy.png" % settings.MEDIA_URL,
                               header="<input type='checkbox' name='toggleselect' id='toggleselect' />")
-    flags = tables.ImgColumn("flags", width="4%")
+    flags = tables.ImgColumn("flags", width="2%")
+    withatts = tables.ImgColumn("withatts", width="2%")
     subject = tables.Column("subject", label=__("Subject"), width="50%")
     from_ = tables.Column("from", width="20%", label=__("From"))
     date = tables.Column("date", width="20%", label=__("Date"))
 
-    cols_order = ["select", "flags", "subject", "from_", "date"]
+    cols_order = ["select", "withatts", "flags", "subject", "from_", "date"]
 
     def parse(self, header, value):
         try:
@@ -148,12 +148,40 @@ class ImapListing(EmailListing):
         return ImapListing.computequota(self.mbc)
 
 class ImapEmail(Email):
-    def __init__(self, msg, user, dformat="DISPLAYMODE", addrfull=False, 
+    def __init__(self, mbox, mailid, request, dformat="DISPLAYMODE", addrfull=False, 
-                 *args, **kwargs):
+                 links=0):
-        mformat = parameters.get_user(user, "DISPLAYMODE")
+        mformat = parameters.get_user(request.user, "DISPLAYMODE")
-        dformat = parameters.get_user(user, dformat)
+        dformat = parameters.get_user(request.user, dformat)
-        Email.__init__(self, msg, mformat, dformat, *args, **kwargs)
 
+        self.headers = []
+        self.attachments = {}
+        self.imapc = get_imapconnector(request)
+        msg = self.imapc.fetchmail(mbox, mailid)
+        self.mbox = mbox
+        self.mailid = mailid
+        headers = msg['BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)]']
+        fallback_fmt = "html" if dformat == "plain" else "plain"
+        
+        self.bs = BodyStructure(msg['BODYSTRUCTURE'])
+        data = None
+
+        mformat = dformat if self.bs.contents.has_key(dformat) else fallback_fmt
+
+        pnum = self.bs.contents[mformat]['pnum']
+        data = self.imapc._cmd("FETCH", mailid, "(BODY.PEEK[%s])" % pnum)
+        content = self._decode_content(self.bs.contents[mformat]['encoding'],
+                                       data[int(mailid)]['BODY[%s]' % pnum])
+        charset = self._find_content_charset(mformat)
+        if charset is not None:
+            content = content.decode(charset)
+
+        self._find_attachments()
+        self._fetch_inlines()
+
+        self.body = \
+            getattr(self, "viewmail_%s" % mformat)(content, links=links)
+
+        msg = email.message_from_string(headers)
         fields = ["Subject", "From", "To", "Reply-To", "Cc", "Date"]
         for f in fields:
             label = f
@@ -173,6 +201,68 @@ class ImapEmail(Email):
             except:
                 pass
 
+    def _find_content_charset(self, subtype):
+        for pos, elem in enumerate(self.bs.contents[subtype]["params"]):
+            if elem == "charset":
+                return self.bs.contents[subtype]["params"][pos + 1]
+        return None
+
+    def _decode_content(self, encoding, content):
+        encoding = encoding.lower()
+        if encoding == "base64":
+            import base64
+            return base64.b64decode(content)
+        elif encoding == "quoted-printable":
+            import quopri
+            return quopri.decodestring(content)
+        return content
+
+    def _find_attachments(self):
+        for att in self.bs.attachments:
+            attname = "part_%s" % att["pnum"]
+            params = None
+            key = None
+            if att.has_key("params") and att["params"] != "NIL":
+                params = att["params"]
+                key = "name"
+
+            if key is None and \
+                    att.has_key("disposition") and len(att["disposition"]) > 1:
+                params = att["disposition"][1]
+                key = "filename"
+            
+            if key and params:
+                for pos, value in enumerate(params):
+                    if value == key:
+                        attname = u2u_decode.u2u_decode(params[pos + 1]).strip("\r\t\n")
+                        break
+            self.attachments[att["pnum"]] = attname
+
+    def _fetch_inlines(self):
+        for cid, params in self.bs.inlines.iteritems():
+            if re.search("\.\.", cid):
+                continue
+            fname = "static/tmp/%s_%s" % (self.mailid, cid)
+            path = os.path.join(settings.MODOBOA_DIR, fname)
+            params["fname"] = "/%s" % fname
+            if os.path.exists(path):
+                continue
+
+            content = self.imapc.fetchpart(self.mailid, self.mbox, params["pnum"])
+            fp = open(path, "wb")
+            fp.write(self._decode_content(params["encoding"], content))
+            fp.close()
+
+
+    def map_cid(self, url):
+        import re
+
+        m = re.match(".*cid:(.+)", url)
+        if m:
+            if self.bs.inlines.has_key(m.group(1)):
+                return self.bs.inlines[m.group(1)]["fname"]
+        return url
+
     def render_headers(self, **kwargs):
         from django.template.loader import render_to_string
 

extensions/webmail/views.py

@@ -166,13 +166,13 @@ def getmailcontent(request):
     if mbox is None or mailid is None:
         raise WebmailError(_("Invalid request"))
 
-    content = fetchmail(request, mbox, mailid, True)
+    #content = fetchmail(request, mbox, mailid, True)
 
     # if "class" in msg.keys() and msg["class"] == "unseen":
     #     IMAPconnector(user=request.user.username,
     #                   password=request.session["password"]).msg_read(folder, mail_id)
 
-    email = ImapEmail(content, request.user, links=request.GET["links"])
+    email = ImapEmail(mbox, mailid, request, links=request.GET["links"])
     return _render(request, "common/viewmail.html", {
             "headers" : email.render_headers(folder=mbox, mail_id=mailid), 
             "folder" : mbox, "imapid" : mailid, "mailbody" : email.body
@@ -659,10 +659,9 @@ def newindex(request):
     json = request.GET.get("json", False)
 
     if action is not None:
-        try:
+        if not globals().has_key(action):
-            response = globals()[action](request)
-        except KeyError:
             raise WebmailError(_("Undefined action"))
+        response = globals()[action](request)            
     else:
         if json:
             raise WebmailError(_("Bad request"))

lib/emailutils.py

@@ -166,7 +166,7 @@ class Email(object):
 
     def viewmail_html(self, content, **kwargs):
         if content is None or content == "":
-            return (False, "")
+            return ""
         links = kwargs.has_key("links") and kwargs["links"] or "0"
         html = lxml.html.fromstring(content) 
         if links == "0":