diff --git a/extensions/webmail/fetch_parser.py b/extensions/webmail/fetch_parser.py new file mode 100644 index 000000000..7ebcb9c0e --- /dev/null +++ b/extensions/webmail/fetch_parser.py @@ -0,0 +1,173 @@ +# coding: utf-8 +""" +:mod:`fetch_parser` --- Simple parser for FETCH responses +--------------------------------------------------------- + +The ``imaplib`` module doesn't parse IMAP responses, it returns raw +values. This is pretty annoying when one FETCH command is issued to +retrieve multiple attributes of a message. + +This simple module tries to fix that *problem*. +""" + +class ParseError(Exception): pass + +def parse_next_token(buf): + """Look for the next *token* + + By *token*, I mean: *literal*, *quoted* or anything else until the + next ' ' or ')' character (number, NIL and others should fall into + this last category). + + :param buf: the buffer to parse + :return: the position of the token's last caracter into ``buf`` + """ + end = -1 + if buf[0] == '{': + # Literal + end = buf.find('}') + elif buf[0] == '"': + # quoted + end = buf.find('"', 1) + else: + for pos, c in enumerate(buf): + if c in [' ', ')']: + end = pos - 1 + break + if end == -1: + raise ParseError("End of buffer reached while looking for a token end") + return end + +def parse_bodystructure(buf, depth=0, prefix=""): + """Special parser for BODYSTRUCTURE response + + This function tries to transform a BODYSTRUCTURE response sent by + the server into the corresponding list structure. + + :param buf: the buffer to parse + :return: a list object and the position of the last scanned + character into ``buf`` + """ + ret = [] + pos = 1 # skip the first ')' + nb_bodystruct = 0 + pnum = 1 + while pos < len(buf): + c = buf[pos] + if c == '(': + nprefix = "%s.%s" % (prefix, pnum) if prefix != "" else "%s" % pnum + subret, end = parse_bodystructure(buf[pos:], depth + 1, nprefix) + pnum += 1 + if nb_bodystruct == 0: + ret.append(subret) + elif nb_bodystruct == 1: + newret = [ret.pop(-1)] + newret.append(subret) + ret.append(newret) + else: + ret[-1].append(subret) + pos += end + 1 + nb_bodystruct += 1 + continue + + if c == ')': + partnum = None + if depth: + if type(ret[0]) == list or len(ret) >= 7: + partnum = prefix + if partnum is not None: + return [dict(partnum=partnum, struct=ret)], pos + return ret, pos + if c == ' ': + pos += 1 + continue + nb_bodystruct = 0 + end = parse_next_token(buf[pos:]) + ret.append(buf[pos:pos + end + 1].strip('"')) + pos += end + 1 + + raise ParseError("End of buffer reached while looking for a BODY/BODYSTRUCTURE end") + +def parse_fetch_response(data): + """Parse a FETCH response, previously issued by a UID command + + We extract the message number, the UID and its value and consider + what remains as the response data. + + :param data: the data returned by the ``imaplib`` command + :return: a dictionnary + """ + result = {} + for part in data[0:-1]: + if part == ')': + continue + buf = "".join(part) + parts = buf.split(' ', 3) + msgdef = result[int(parts[2])] = {} + response = parts[3] + + while len(response): + if response.startswith('BODY') and response[4] == '[': + end = response.find(']', 5) + if response[end + 1] == '<': + end = response.find('>', end + 1) + end += 1 + else: + end = response.find(' ') + cmdname = response[:end] + response = response[end + 1:] + + end = 0 + if cmdname in ['BODY', 'BODYSTRUCTURE', 'FLAGS']: + parendepth = 0 + for pos, c in enumerate(response): + if c == '(': + parendepth += 1 + continue + if c == ')': + parendepth -= 1 + if parendepth == 0: + end = pos + 1 + break + else: + end = parse_next_token(response) + if response[0] == '{': + datalen = int(response[1:end]) + response = response[end + 1:] + end = datalen + + msgdef[cmdname] = response[:end] + response = response[end + 1:] + try: + func = globals()["parse_%s" % cmdname.lower()] + msgdef[cmdname] = func(msgdef[cmdname])[0] + except KeyError: + pass + + return result + +def dump_bodystructure(bs, depth=0): + if depth: + print " " * (depth * 4), + if type(bs[0]) == dict: + print "%s :" % bs[0]["partnum"], + struct = bs[0]["struct"] + else: + struct = bs + + if type(struct[0]) == list: + print "multipart/%s" % struct[1] + for part in struct[0]: + dump_bodystructure(part, depth + 1) + else: + print "%s/%s" % (struct[0], struct[1]) + + +if __name__ == "__main__": + resp = [('855 (UID 46931 BODYSTRUCTURE ((("text" "plain" ("charset" "iso-8859-1") NIL NIL "quoted-printable" 886 32 NIL NIL NIL NIL)("text" "html" ("charset" "us-ascii") NIL NIL "quoted-printable" 1208 16 NIL NIL NIL NIL) "alternative" ("boundary" "----=_NextPart_001_0003_01CCC564.B2F64FF0") NIL NIL NIL)("application" "octet-stream" ("name" "Carte Verte_2.pdf") NIL NIL "base64" 285610 NIL ("attachment" ("filename" "Carte Verte_2.pdf")) NIL NIL) "mixed" ("boundary" "----=_NextPart_000_0002_01CCC564.B2F64FF0") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {153}', 'From: \r\nTo: \r\nCc: \r\nSubject: Notre contact du 28/12/2011 - 192175092\r\nDate: Wed, 28 Dec 2011 13:29:17 +0100\r\n\r\n'), ')'] + + #resp = [('856 (UID 46936 BODYSTRUCTURE (("text" "plain" ("charset" "ISO-8859-1") NIL NIL "quoted-printable" 724 22 NIL NIL NIL NIL)("text" "html" ("charset" "ISO-8859-1") NIL NIL "quoted-printable" 2662 48 NIL NIL NIL NIL) "alternative" ("boundary" "----=_Part_1326887_254624357.1325083973970") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {258}', 'Date: Wed, 28 Dec 2011 15:52:53 +0100 (CET)\r\nFrom: =?ISO-8859-1?Q?Malakoff_M=E9d=E9ric?= \r\nTo: Antoine Nguyen \r\nSubject: =?ISO-8859-1?Q?Votre_inscription_au_grand_Jeu_Malakoff_M=E9d=E9ric?=\r\n\r\n'), ')'] + + resp = [('856 (UID 11111 BODYSTRUCTURE ((("text" "plain" ("charset" "UTF-8") NIL NIL "7bit" 0 0 NIL NIL NIL NIL) "mixed" ("boundary" "----=_Part_407172_3159001.1321948277321") NIL NIL NIL)("application" "octet-stream" ("name" "26274308.pdf") NIL NIL "base64" 14906 NIL ("attachment" ("filename" "26274308.pdf")) NIL NIL) "mixed" ("boundary" "----=_Part_407171_9686991.1321948277321") NIL NIL NIL)'), ')'] + + print parse_fetch_response(resp) diff --git a/extensions/webmail/imaputils.py b/extensions/webmail/imaputils.py index 1750ac7f6..a29851b60 100644 --- a/extensions/webmail/imaputils.py +++ b/extensions/webmail/imaputils.py @@ -11,6 +11,7 @@ from modoboa.lib import parameters from modoboa.lib.connections import * from modoboa.lib.webutils import static_url from exceptions import ImapError +from fetch_parser import * #imaplib.Debug = 4 @@ -35,6 +36,86 @@ class capability(object): return wrapped_func +class BodyStructure(object): + """ + BODYSTRUCTURE response parser. + + Just a simple class that tries to distinguish content parts from + attachments. + """ + def __init__(self, definition=None): + self.is_multipart = False + self.contents = {} + self.attachments = [] + self.inlines = {} + + if definition is not None: + self.load_from_definition(definition) + + def __store_part(self, definition, pnum, multisubtype): + """Store the given message part in the appropriate category. + + This method sort parts in two categories: + + * contents (what is going to be displayed) + * attachments + + As there is no official definition about what is a content and + what is an attachment, the following rules are applied: + + * If the MIME type is text/plain or text/html: + + * If no previous part of this type has already been seen, it's a content + * Otherwise it's an attachment + + * Else, if the multipart subtype is related, we consider this + part as content because it is certainly an embedded image + + * Any other MIME type is considered as an attachment (for now) + + :param definition: a part definition (list) + :param prefix: the part's number + :param multisubtype: the multipart subtype + """ + pnum = "1" if pnum is None else pnum + params = dict(pnum=pnum, params=definition[2], cid=definition[3], + description=definition[4], encoding=definition[5], + size=definition[6]) + subtype = definition[1].lower() + mtype = "%s/%s" % (definition[0].lower(), subtype) + if mtype in ("text/plain", "text/html"): + if not self.contents.has_key(mtype): + self.contents[subtype] = params + return + elif multisubtype in ["related"]: + self.inlines[params["cid"].strip("<>")] = params + return + + if len(definition) > 7: + params.update(md5=definition[7], disposition=definition[8], + lang=definition[9], location=definition[10]) + self.attachments += [params] + + def load_from_definition(self, definition, multisubtype=None): + if type(definition) == dict: + struct = definition["struct"] + pnum = definition["partnum"] + elif type(definition[0]) == dict: + struct = definition[0]["struct"] + pnum = definition[0]["partnum"] + else: + struct = definition + pnum = None + + if type(struct[0]) == list: + for part in struct[0]: + self.load_from_definition(part, struct[1]) + return + + self.__store_part(struct, pnum, multisubtype) + + def has_attachments(self): + return len(self.attachments) class IMAPconnector(object): __metaclass__ = ConnectionsManager @@ -105,13 +186,14 @@ class IMAPconnector(object): for part in bodystruct[0]: nprefix = "%s" % cpt if prefix == "" else "%s.%d" % (prefix, cpt) index, encoding, size = \ - self.__find_content_in_bodystruct(bodystruct[0], mtype, stype, nprefix) - if index: + self.__find_content_in_bodystruct(part, mtype, stype, nprefix) + if index is not None: return (index, encoding, size) cpt += 1 - - if bodystruct[0] == mtype and bodystruct[1] == stype: - return ("1" if not len(prefix) else prefix, bodystruct[5], int(bodystruct[6])) + else: + if bodystruct[0].lower() == mtype and bodystruct[1].lower() == stype: + return ("1" if not len(prefix) else prefix, + bodystruct[5], int(bodystruct[6])) return (None, None, 0) def refresh(self, user, password): @@ -401,14 +483,10 @@ class IMAPconnector(object): self.quota_limit = int(m.group(2)) self.quota_actual = int(m.group(1)) - def fetchpart(self, uid, folder, part): - self.m.select(self._encodefolder(folder), True) - typ, data = self.m.fetch(uid, "(BODY[%(p)s.MIME] BODY[%(p)s])" \ - % {"p" : part}) - if typ != "OK": - return None - msg = email.message_from_string(data[0][1] + data[1][1]) - return msg + def fetchpart(self, uid, mbox, partnum): + self.select_mailbox(mbox, False) + data = self._cmd("FETCH", uid, "(BODY[%s])" % partnum) + return data[int(uid)]["BODY[%s]" % partnum] def fetch(self, start, stop=None, mbox=None, **kwargs): """Retrieve information about messages from the server @@ -427,7 +505,7 @@ class IMAPconnector(object): else: submessages = [start] range = start - query = '(FLAGS BODY.PEEK[HEADER.FIELDS (DATE FROM TO CC SUBJECT)])' + query = '(FLAGS BODYSTRUCTURE BODY.PEEK[HEADER.FIELDS (DATE FROM TO CC SUBJECT)])' data = self._cmd("FETCH", range, query) result = [] for uid in submessages: @@ -437,10 +515,13 @@ class IMAPconnector(object): msg['class'] = 'unseen' if r'\Answered' in data[int(uid)]['FLAGS']: msg['img_flags'] = static_url('pics/answered.png') + bs = BodyStructure(data[int(uid)]['BODYSTRUCTURE']) + if bs.has_attachments(): + msg['img_withatts'] = static_url('pics/attachment.png') result += [msg] return result - def fetchmail(self, mbox, mailid, fmt="plain"): + def fetchmail(self, mbox, mailid): """Retrieve information about a specific message Issue a FETCH command to retrieve a message's content from the @@ -455,34 +536,15 @@ class IMAPconnector(object): """ self.select_mailbox(mbox, False) data = self._cmd("FETCH", mailid, "(BODYSTRUCTURE BODY.PEEK[HEADER.FIELDS (DATE FROM TO CC SUBJECT)])") - content = data[int(mailid)]['BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)]'] - - fallback_fmt = "html" if fmt == "plain" else "plain" - (pnum, encoding, size) = (None, None, 0) - for f in [fmt, fallback_fmt]: - pnum, encoding, size = \ - self.__find_content_in_bodystruct(data[int(mailid)]['BODYSTRUCTURE'], - "text", f) - if pnum is not None and size: - break - if pnum is None: - return None - - data = self._cmd("FETCH", mailid, "(BODY.PEEK[%s])" % pnum) - - data = data[int(mailid)]['BODY[%s]' % pnum] - if encoding == "base64": - import base64 - content += base64.b64decode(data) - elif encoding == "quoted-printable": - import quopri - content += quopri.decodestring(data) - else: - content += data - - return email.message_from_string(content) + return data[int(mailid)] def get_imapconnector(request): imapc = IMAPconnector(user=request.user.username, password=request.session["password"]) return imapc + + +if __name__ == "__main__": + import doctest + doctest.testmod() + diff --git a/extensions/webmail/lib.py b/extensions/webmail/lib.py index 3cd6d3bfa..f420689d7 100644 --- a/extensions/webmail/lib.py +++ b/extensions/webmail/lib.py @@ -7,7 +7,6 @@ sys.setdefaultencoding("utf-8") import re import time import socket -import imaplib import ssl from datetime import datetime, timedelta import email @@ -31,12 +30,13 @@ class WMtable(tables.Table): select = tables.ImgColumn("select", cssclass="draggable left", width="1%", defvalue="%spics/grippy.png" % settings.MEDIA_URL, header="") - flags = tables.ImgColumn("flags", width="4%") + flags = tables.ImgColumn("flags", width="2%") + withatts = tables.ImgColumn("withatts", width="2%") subject = tables.Column("subject", label=__("Subject"), width="50%") from_ = tables.Column("from", width="20%", label=__("From")) date = tables.Column("date", width="20%", label=__("Date")) - cols_order = ["select", "flags", "subject", "from_", "date"] + cols_order = ["select", "withatts", "flags", "subject", "from_", "date"] def parse(self, header, value): try: @@ -148,12 +148,40 @@ class ImapListing(EmailListing): return ImapListing.computequota(self.mbc) class ImapEmail(Email): - def __init__(self, msg, user, dformat="DISPLAYMODE", addrfull=False, - *args, **kwargs): - mformat = parameters.get_user(user, "DISPLAYMODE") - dformat = parameters.get_user(user, dformat) - Email.__init__(self, msg, mformat, dformat, *args, **kwargs) + def __init__(self, mbox, mailid, request, dformat="DISPLAYMODE", addrfull=False, + links=0): + mformat = parameters.get_user(request.user, "DISPLAYMODE") + dformat = parameters.get_user(request.user, dformat) + self.headers = [] + self.attachments = {} + self.imapc = get_imapconnector(request) + msg = self.imapc.fetchmail(mbox, mailid) + self.mbox = mbox + self.mailid = mailid + headers = msg['BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)]'] + fallback_fmt = "html" if dformat == "plain" else "plain" + + self.bs = BodyStructure(msg['BODYSTRUCTURE']) + data = None + + mformat = dformat if self.bs.contents.has_key(dformat) else fallback_fmt + + pnum = self.bs.contents[mformat]['pnum'] + data = self.imapc._cmd("FETCH", mailid, "(BODY.PEEK[%s])" % pnum) + content = self._decode_content(self.bs.contents[mformat]['encoding'], + data[int(mailid)]['BODY[%s]' % pnum]) + charset = self._find_content_charset(mformat) + if charset is not None: + content = content.decode(charset) + + self._find_attachments() + self._fetch_inlines() + + self.body = \ + getattr(self, "viewmail_%s" % mformat)(content, links=links) + + msg = email.message_from_string(headers) fields = ["Subject", "From", "To", "Reply-To", "Cc", "Date"] for f in fields: label = f @@ -173,6 +201,68 @@ class ImapEmail(Email): except: pass + def _find_content_charset(self, subtype): + for pos, elem in enumerate(self.bs.contents[subtype]["params"]): + if elem == "charset": + return self.bs.contents[subtype]["params"][pos + 1] + return None + + def _decode_content(self, encoding, content): + encoding = encoding.lower() + if encoding == "base64": + import base64 + return base64.b64decode(content) + elif encoding == "quoted-printable": + import quopri + return quopri.decodestring(content) + return content + + def _find_attachments(self): + for att in self.bs.attachments: + attname = "part_%s" % att["pnum"] + params = None + key = None + if att.has_key("params") and att["params"] != "NIL": + params = att["params"] + key = "name" + + if key is None and \ + att.has_key("disposition") and len(att["disposition"]) > 1: + params = att["disposition"][1] + key = "filename" + + if key and params: + for pos, value in enumerate(params): + if value == key: + attname = u2u_decode.u2u_decode(params[pos + 1]).strip("\r\t\n") + break + self.attachments[att["pnum"]] = attname + + def _fetch_inlines(self): + for cid, params in self.bs.inlines.iteritems(): + if re.search("\.\.", cid): + continue + fname = "static/tmp/%s_%s" % (self.mailid, cid) + path = os.path.join(settings.MODOBOA_DIR, fname) + params["fname"] = "/%s" % fname + if os.path.exists(path): + continue + + content = self.imapc.fetchpart(self.mailid, self.mbox, params["pnum"]) + fp = open(path, "wb") + fp.write(self._decode_content(params["encoding"], content)) + fp.close() + + + def map_cid(self, url): + import re + + m = re.match(".*cid:(.+)", url) + if m: + if self.bs.inlines.has_key(m.group(1)): + return self.bs.inlines[m.group(1)]["fname"] + return url + def render_headers(self, **kwargs): from django.template.loader import render_to_string diff --git a/extensions/webmail/views.py b/extensions/webmail/views.py index 327e72c99..c9b0a1d0c 100644 --- a/extensions/webmail/views.py +++ b/extensions/webmail/views.py @@ -166,13 +166,13 @@ def getmailcontent(request): if mbox is None or mailid is None: raise WebmailError(_("Invalid request")) - content = fetchmail(request, mbox, mailid, True) + #content = fetchmail(request, mbox, mailid, True) # if "class" in msg.keys() and msg["class"] == "unseen": # IMAPconnector(user=request.user.username, # password=request.session["password"]).msg_read(folder, mail_id) - email = ImapEmail(content, request.user, links=request.GET["links"]) + email = ImapEmail(mbox, mailid, request, links=request.GET["links"]) return _render(request, "common/viewmail.html", { "headers" : email.render_headers(folder=mbox, mail_id=mailid), "folder" : mbox, "imapid" : mailid, "mailbody" : email.body @@ -659,10 +659,9 @@ def newindex(request): json = request.GET.get("json", False) if action is not None: - try: - response = globals()[action](request) - except KeyError: + if not globals().has_key(action): raise WebmailError(_("Undefined action")) + response = globals()[action](request) else: if json: raise WebmailError(_("Bad request")) diff --git a/lib/emailutils.py b/lib/emailutils.py index 2099734bd..3d7b3d820 100644 --- a/lib/emailutils.py +++ b/lib/emailutils.py @@ -166,7 +166,7 @@ class Email(object): def viewmail_html(self, content, **kwargs): if content is None or content == "": - return (False, "") + return "" links = kwargs.has_key("links") and kwargs["links"] or "0" html = lxml.html.fromstring(content) if links == "0":