merge

INSTALL

@@ -15,10 +15,16 @@ Download Mailng's tarball and extract it::
   tar xzf mailng-<version>.tar.gz
   mv mailng-<version> mailng
 
-Enter the mailng directory, create a settings.py file (you can use 
-settings-sample.py as a model). Specify database relative informations. You 
-must choose between mysql and postgresql, other types are not supported by 
-major email projects like postfix or dovecot.
+Settings file
+
+  Enter the mailng directory, create a settings.py file (you can use
+  settings-sample.py as a model).
+  Specify database relative informations.
+  You must choose between mysql and postgresql, other types are not supported by
+  major email projects like postfix or dovecot.
+  Create a database with the name you specified in settings.py.
+  you will probably need python database module backend like MySQLdb.
+  Modify TEMPLATE_DIRS.
 
 Initialise the database by running the following command::
 
@@ -378,4 +384,15 @@ Replace mailng_dir with the installation path of MailNG.
 
 By default, messages older than 14 days are automatically purged. You
 can modify this value by changing the variable MAX_MESSAGES_AGE inside
-settings.py.
+settings.py.
+
+Statistics plugin setup
+-----------------------
+
+Early version of that plugin only support parsing of postfix log as syslog format.
+Please make sure you have a valid /var/log/maillog postfix log file.
+Don't forget to create a link from /templates/stats to extensions/stats/template
+FIXME : 
+Parsing for domain is not fully supported, soon in a next version.
+User priviledges are not handled properly. So you understand 
+it is a really eraly pluggin  version!! Enjoy

extensions/stats/mailrrd.py

@@ -0,0 +1,381 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+Interface with rrdtool to generate appropriate postfix graph statistics
+sent, recv, bounced, rejected messages are tracked.
+"""
+import sys, os, rrdtool, re, time
+import pdb
+from optparse import OptionParser
+try:
+    import django.conf
+except:
+    print "[rrd] Warning no Django conf found"
+rrdstep           = 60
+xpoints           = 540
+points_per_sample = 3
+rrd_inited        = False
+this_minute       = 0
+
+rrd_file = "postfix.rrd"
+png_file = "postfix.png"
+log_file = "/var/log/maillog"
+tmp_path     = "static/tmp/" 
+
+months_map = {
+    'Jan' : 0, 'Feb' : 1, 'Mar' : 2,
+    'Apr' : 3, 'May' : 4, 'Jun' : 5,
+    'Jul' : 6, 'Aug' : 7, 'Sep' : 8,
+    'Oct' : 9, 'Nov' :10, 'Dec' :11,
+    'jan' : 0, 'feb' : 1, 'mar' : 2,
+    'apr' : 3, 'may' : 4, 'jun' : 5,
+    'jul' : 6, 'aug' : 7, 'sep' : 8,
+    'oct' : 9, 'nov' :10, 'dec' :11,
+}
+
+def str2Time(y,M,d,h,m,s):
+    """str2Time
+
+    return epoch time from Year Month Day Hour:Minute:Second time format
+    """
+    try:
+        local = time.strptime("%s %s %s %s:%s:%s" %(y,M,d,h,m,s), \
+                          "%Y %b %d %H:%M:%S")
+    except:
+        print "[rrd] ERROR unrecognized %s time format" %(y,M,d,h,m,s)
+        exit(0)
+    return int(time.mktime(local))
+
+
+class parser():
+    """parser
+
+    Parse a mail log in syslog format:
+    Month day HH:MM:SS host prog[pid]: log message...
+
+    For each epoch record the parser stores number of
+    'sent', 'received', 'bounced','rejected' messages over
+    a rrdstep period of time.
+
+    parameters : logfile, and starting year
+    
+    """
+    def __init__(self,domain,logfile=log_file,rrdfile=rrd_file,imgFile=png_file,
+                 create=False,year=None,debug=False,verbose=False,graph=None):
+        """constructor
+        """
+        self.domain = str(domain) #.replace(".","_")
+        self.enable_year_decrement = None
+        self.logfile = logfile
+        self.rrdfile = tmp_path + self.domain+"_"+rrdfile
+        self.create = create
+        self.year = year
+        self.debug = debug
+        self.verbose = verbose
+        self.graph = graph
+        self.imgFile = self.domain+"_"+imgFile
+        self.types = ['AVERAGE','MAX']
+        self.natures = ['sent_recv','boun_reje']
+        self.legend = {'sent_recv' : ['sent messages','received messages'],
+                       'boun_reje' : ['bounced messages','rejected messages']}
+        try:
+            self.f = open(logfile)
+        except IOError as (errno, strerror):
+            print "I/O error({0}): {1} ".format(errno, strerror)+logfile
+        self.data = {}
+        self.last_month = None
+        if not self.year:
+            self.year = time.localtime().tm_year
+        self.first_minute = 0
+        self.last_minute  = 0
+
+        if self.create and os.path.exists(logfile):
+            print "[rrd] force new RRD"
+            os.system("rm -f %s" %rrdfile)
+
+        elif os.path.exists(self.rrdfile):
+            self.last_minute = rrdtool.last(self.rrdfile)
+            self.first_minute = rrdtool.first(self.rrdfile)
+            if self.debug:
+                print "[rrd] DEBUG updating rrd from %s"\
+                      %time.asctime(time.localtime(self.last_minute))
+
+
+        self.process_log()
+
+    def year_increment(self,month):
+        """year_increment
+        """
+        if month == 0:
+            if self.last_month and self.last_month == 1:
+                self.year += 1
+                self.enable_year_decrement = True
+        elif month == 11:
+            if self.enable_year_decrement \
+               and self.last_month and self.last_month != 1:
+                self.year -= 1
+        else:
+            self.enable_year_decrement = False
+
+        self.last_month = month
+
+    def process_line(self,text):
+        """process_line
+
+        Parse a log line and look for
+        'sent','received','bounced',rejected' messages event
+
+        Return True if data are up-to-date
+        else False
+        """
+        # get date
+        ret = False
+
+        m = re.match("(\w+)\s+(\d+)\s+(\d+):(\d+):(\d+)\s+(\w+)\s+(\w+)/?\w*[[](\d+)[]]:\s+(.*)",text)
+        if not m : return ret
+
+        mo = m.group(1)
+        da = m.group(2)
+        ho = m.group(3)
+        mi = m.group(4)
+        se = m.group(5)
+        host = m.group(6)
+        prog = m.group(7)
+        pid  = m.group(8)
+        log  = m.group(9)
+
+        sent = 0
+        recv = 0
+        boun  = 0
+        reje  = 0
+
+        self.year_increment(months_map[mo])
+        se = int(int(se)/rrdstep)            # rrd step is one-minute => se = 0
+        cur_t = str2Time(self.year,mo,da,ho,mi,se)
+
+        if not self.data.has_key(cur_t):
+            self.data[cur_t] = {'sent':0,'recv':0, 'boun':0,'reje':0}
+
+        # watch events
+        if re.match(".*\s+status=sent\s+.*",text):
+            sent = 1
+
+        elif re.match(".*\s+status=bounced\s+.*",text):
+            boun = 1
+
+        elif re.match(".*[0-9A-Z]+: client=(\S+).*",text):
+            recv = 1
+
+        elif re.match(".*(?:[0-9A-Z]+: |NOQUEUE: )?reject: .*",text):
+            reje = 1
+
+        elif re.match(".*(?:[0-9A-Z]+: |NOQUEUE: )?milter-reject: .*",text):
+            if not re.match(".*Blocked by SpamAssassin.*",text):
+                reje = 1
+            #else regiter spam
+
+        # register events
+        if sent or recv or boun or reje:
+            self.data[cur_t]['sent'] = self.data[cur_t]['sent'] + sent
+            self.data[cur_t]['recv'] = self.data[cur_t]['recv'] + recv
+            self.data[cur_t]['boun'] = self.data[cur_t]['boun'] + boun
+            self.data[cur_t]['reje'] = self.data[cur_t]['reje'] + reje
+            return True
+
+        return False
+
+    def init_rrd(self,m):
+        """init_rrd
+
+        Set-up Data Sources (DS)
+        Set-up Round Robin Archives (RRA):
+        - day,week,month and year archives
+        - 2 types : AVERAGE and MAX
+
+        parameter : start time
+        return    : last epoch recorded
+        """
+        ds_type = 'ABSOLUTE'
+        rows = xpoints / points_per_sample
+
+        realrows = int(rows * 1.1)    # ensure that the full range is covered
+        day_steps = int(3600*24/(rrdstep * rows))
+        week_steps = day_steps * 7
+        month_steps = week_steps * 5
+        year_steps = month_steps * 12
+
+        # Set up data sources for our RRD
+        ds1 = 'DS:sent:%s:%s:0:U' %(ds_type,rrdstep*2)
+        ds2 = 'DS:recv:%s:%s:0:U' %(ds_type,rrdstep*2)
+        ds3 = 'DS:boun:%s:%s:0:U' %(ds_type,rrdstep*2)
+        ds4 = 'DS:reje:%s:%s:0:U' %(ds_type,rrdstep*2)
+
+        # Set up RRD to archive data
+        rra1 = 'RRA:%s:0.5:%s:%s' %('AVERAGE',day_steps,realrows)   # day
+        rra2 = 'RRA:%s:0.5:%s:%s' %('AVERAGE',week_steps,realrows)  # week
+        rra3 = 'RRA:%s:0.5:%s:%s' %('AVERAGE',month_steps,realrows) # month
+        rra4 = 'RRA:%s:0.5:%s:%s' %('AVERAGE',year_steps,realrows)  # year
+        rra5 = 'RRA:%s:0.5:%s:%s' %('MAX',day_steps,realrows)
+        rra6 = 'RRA:%s:0.5:%s:%s' %('MAX',week_steps,realrows)
+        rra7 = 'RRA:%s:0.5:%s:%s' %('MAX',month_steps,realrows)
+        rra8 = 'RRA:%s:0.5:%s:%s' %('MAX',year_steps,realrows)
+
+        # With those setup, we can now created the RRD
+        if not os.path.exists(self.rrdfile):
+            rrdtool.create(self.rrdfile,
+                           '--start',str(m),
+                           '--step',str(rrdstep),
+                           ds1,ds2,ds3,ds4,
+                           rra1,rra2,rra3,rra4,
+                           rra5,rra6,rra7,rra8)
+            this_minute = m
+        else:
+            this_minute = rrdtool.last(self.rrdfile) + rrdstep
+
+        return this_minute
+
+
+    def update_rrd(self,t):
+        """update_rrd
+
+        Update RRD with records at t time.
+
+        True  : if data are up-to-date for current minute
+        False : syslog may have probably been already recorded
+        or something wrong
+        """
+        m = t - (t % rrdstep)
+
+        if not self.last_minute :
+            self.last_minute = self.init_rrd(m)
+            print "[rrd] create new RRD file"
+
+        if m < self.last_minute :
+            if self.verbose:
+                print "[rrd] VERBOSE events at %s already recorded in RRD" %m
+            return False
+        elif m == self.last_minute :
+            return True
+
+        # Missing some RRD steps
+        if m > self.last_minute + rrdstep:
+            for p in range(self.last_minute+rrdstep,m,rrdstep):
+                if self.verbose:
+                    print "[rrd] VERBOSE update %s:%s:%s:%s:%s (SKIP)" \
+                          %(p,'0','0','0','0')
+                rrdtool.update(self.rrdfile,"%s:%s:%s:%s:%s" \
+                               %(p,'0','0','0','0'))
+
+        if self.verbose:
+            print "[rrd] VERBOSE update %s:%s:%s:%s:%s" \
+                  %(m,self.data[m]['sent'], self.data[m]['recv'],\
+                    self.data[m]['boun'], self.data[m]['reje'])
+
+        rrdtool.update(self.rrdfile,"%s:%s:%s:%s:%s" \
+                       %(m,self.data[m]['sent'], self.data[m]['recv'],\
+                         self.data[m]['boun'], self.data[m]['reje']))
+
+        self.last_minute = m
+        return True
+
+
+    def process_log(self):
+        """process_log
+
+        Go through entire log file
+        """
+        for line in self.f.readlines():
+            evt = self.process_line(line)
+
+        # Sort everything by time
+        sortedData = {}
+        sortedData = [ (i,self.data[i]) for i in sorted(self.data.keys()) ]
+
+        for t, dict in sortedData:
+            if self.update_rrd(t):
+                if not self.first_minute:
+                    self.first_minute = t
+                self.last_minute = t
+
+
+    def plot_rrd(self,f=None,color1 = "#990033", color2 = "#330099",
+                 year = None, start = None, end = None, t =None, n = None):
+        """plot_rrd
+
+        Graph rrd from start to end epoch
+        """
+        if f:
+            self.imgFile = f
+        start = str(self.first_minute)
+        end   = str(self.last_minute)
+        if t:
+            self.types = t
+        if n:
+            self.natures = n
+
+        if self.graph:
+            try:
+                start = int(time.mktime(time.strptime(self.graph[0], \
+                                                      "%Y %b %d %H:%M:%S")))
+                end = int(time.mktime(time.strptime(self.graph[1], \
+                                                      "%Y %b %d %H:%M:%S")))
+            except:
+                print "[rrd] ERROR bad time format for option --graph"
+
+
+        print "[rrd] plot rrd graph from %s to %s" \
+              %(time.asctime(time.localtime(float(start))),\
+                time.asctime(time.localtime(float(end))))
+        if not year: year = time.localtime().tm_year
+        self.imgFile
+        if not os.path.exists(tmp_path):
+            os.mkdir(tmp_path)
+        for n in self.natures:
+            for t in self.types:
+                path = '%s%s_%s_%s'%(tmp_path,n,t,os.path.basename(self.imgFile))
+                ds1 = n.split('_')[0]
+                ds2 = n.split('_')[1]
+                print 'DEF:%s=%s:%s:%s:' %(ds1,ds1, self.rrdfile,t)
+                print 'DEF:%s=%s:%s:%s:' %(ds2,ds2, self.rrdfile,t)
+                print 'LINE:%s%s:%s messages' %(ds1,color1,self.legend[n][0])
+                print 'LINE:%s%s:%s messages' %(ds2,color2,self.legend[n][1])
+                rrdtool.graph(
+                    path,
+                    '--imgformat','PNG',
+                    '--width','540',
+                    '--height','100',
+                    '--start', str(start),
+                    '--end', str(end),
+                    '--vertical-label', '%s message' %t.lower(),
+                    '--title', '%s message flow per minute' %t.lower(),
+                    '--lower-limit', '0',
+                    'DEF:%s=%s:%s:%s:' %(ds1,self.rrdfile,ds1,t),
+                    'DEF:%s=%s:%s:%s:' %(ds2,self.rrdfile,ds2,t),
+                    'LINE:%s%s:%s' %(ds1,color1,self.legend[n][0]),
+                    'LINE:%s%s:%s' %(ds2,color2,self.legend[n][1])
+                    )
+
+if __name__ == "__main__":
+    parser = OptionParser()
+
+    parser.add_option("-f","--file",dest="filename",default=rrdfile,
+                      help="write rrd data base to FILE",metavar="FILE")
+    parser.add_option("-i","--imgFile",default=png_file,
+                      help="graph rrd stats to FILE",metavar="FILE")
+    parser.add_option("-c","--create",default=False,action="store_true",dest="create",
+                      help="force creation of new RRD")
+    parser.add_option("-g","--graph",nargs=2,dest="graph",
+                      help="generate graph in between time period (y M d YY:MM:SS)", metavar="START STOP")
+    parser.parse_args(['--graph', 'start', 'stop'])
+    parser.add_option("-l","--logFile",default="maillog",
+                      help="postfix log in syslog format",metavar="FILE")
+    parser.add_option("-v","--verbose",default=False,action="store_true",dest="verbose",
+                      help="set verbose mode")
+    parser.add_option("-d","--debug",default=False,action="store_true",dest="debug",
+                      help="set debug mode")
+
+    (options,args) = parseOption()
+    P = parser(logfile=options.logFile,rrdfile=options.file,
+               debug=options.debug,verbose=options.verbose,
+               create=options.create,graph=options.graph,imgFile=options.imgFile)
+    P.plot_rrd()

extensions/stats/main.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+"""
+graph plugin do display rrd stats
+
+This module provides rrdtool support to retreive statistics
+from postfix log : sent, received, bounced, rejected
+
+"""
+from django.core.urlresolvers import reverse
+from django.utils.translation import ugettext as _
+from django.conf.urls.defaults import *
+from mailng.lib import events, _render
+from mailng.admin.views import is_superuser
+from mailng.admin.models import Domain, Mailbox
+from mailng.extensions.stats.mailrrd import parser
+from django.contrib.auth.decorators \
+    import login_required
+#from models import Stats
+import pdb
+
+graph_nature = ["sent_recv","boun_reje"]
+graph_types  = ["AVERAGE","MAX"]
+tmp_path     = "static/tmp"
+
+def init():
+    events.register("UserMenuDisplay", menu)
+
+def urls():
+    return (r'^mailng/stats/', 
+            include('mailng.extensions.stats.urls'))
+
+
+def menu(**kwargs):
+    if kwargs["target"] != "user_menu_box":
+        return []
+
+    return [
+        {"name" : _("Statistics"),
+         "url" : reverse(index),
+         "img" : "/static/pics/graph.png"}
+        ]
+
+@login_required
+def domain(request,dom_id):
+    domains = []
+    errors = []
+    return graph_display(request,dom_id,graph_types)
+
+#@login_required
+def graph_display(request,dom_id,graph_t):
+    domains = []
+    errors = []
+    graph_type = None
+    if type(graph_t) == type([]):
+        graph_type = graph_t
+    else :
+        graph_type = [graph_t]
+
+    if dom_id:
+        domain = Domain.objects.get(pk=dom_id)
+        if domain:
+            domains.append(domain)
+
+    if domains == []:
+        errors.append(_("No Domain defined"))
+
+
+    for d in domains:
+        P = parser(d.name)
+        P.plot_rrd()
+
+    return _render(request, 'stats/index.html', {
+        "page" : "Domain statistics", "graph"   : graph_nature,
+        "domains" : domains, "messages" : "graph display" ,
+        "types" :  graph_type, "tmp_path" : tmp_path})
+
+
+@login_required
+def index(request, message=None):
+    domains = []
+    errors = []
+    if not is_superuser(request.user):
+        if not request.user.has_perm("admin.view_domains"):
+            if request.user.has_perm("admin.view_mailboxes"):
+                mb = Mailbox.objects.get(user=request.user.id)
+                dom_id=mb.domain.id
+                domain = Domain.objects.get(pk=dom_id)
+                if domain:
+                    domains.append(domain)
+    else:
+        domains = Domain.objects.all()
+
+    if domains == []:
+        errors.append(_("No Domain defined"))
+    return _render(request, 'stats/index.html', {
+        "page" : "Statistics",
+        "domains" : domains, "messages" : errors ,"types" : graph_types})

extensions/stats/models.py

@@ -0,0 +1,10 @@
+from django.db import models
+from mailng.admin.models import Domain, Mailbox
+
+"""
+class Stats(models.Model):
+    domain   = models.CharField(max_length=300,unique=True)
+    #rrd_file = models.CharField(max_length=100)
+    first    = models.DateTimeField()
+    last     = models.DateTimeField()
+"""

extensions/stats/templates/index.html

@@ -0,0 +1,82 @@
+{% extends "layout.html" %}
+{% load i18n %}
+{% block html_head %}
+<script src="/static/js/mootools-more.js" type="text/javascript"></script>
+<link rel="stylesheet" type="text/css" href="/static/stylesheets/box.css" />
+<style type="text/css">
+  form {
+  margin: 0 auto;
+  width: 98%;
+  }
+
+  td {
+  font-size: smaller;
+  height: 40px;
+  }
+
+  errors {
+  text-align: center;
+  width: 100%;
+  margin: 20px 20px;
+  color: red;
+  }
+
+  #graphcontent {
+  margin:60px auto 10px;
+  width:90%;
+  }
+
+</style>
+
+<script type="text/javascript">
+</script>
+{% endblock %}
+
+{% block middletitle %}
+  {{ page }}
+{% endblock %}
+
+{% block menu %}
+{% endblock %}
+{% block content %}
+<div id="graphcontent">
+  <table>
+  {% if graph %}
+    {% for d in domains %}
+    {% for g in graph %}
+        {% for t in types %}
+        <tr>
+          <th class="{% cycle 'odd' 'even' %}">{{ d }} > {{ t|lower }} > {{ g }}</th>
+        </tr>
+        <tr>
+        <td>
+          <img src="/{{ tmp_path }}/{{ g }}_{{ t }}_{{ d }}_postfix.png" alt="/{{ tmp_path }}/{{ g }}_{{ t }}_{{ d }}_postfix.png"/>
+        </td>
+        </tr>
+        {% endfor %}
+    {% endfor %}
+    {% endfor %}
+  {% else %}
+  {% if domains %}
+    <tr>
+      <th width="50px">{% trans "Graph type" %}</th>
+      <th width="80px">{% trans "Domain name" %}</th>
+    </tr>
+
+    {% for dom in domains %}
+    <tr class="{% cycle 'odd' 'even' %}">
+      <td>
+        {% for t in types %}
+          {% url extensions.stats.main.graph_display dom.id,t as graphurl %}
+          <a name="loadGraph" href="{{ graphurl }}">{{ t|lower }} </a>
+        {% endfor %}
+      </td>
+        {% url extensions.stats.main.domain dom.id as domurl %}
+        <td><a name="loadDom" href="{{ domurl }}">{{ dom.name }}</a></td>
+    </tr>
+  {% endfor %}
+  {% endif %}
+  {% endif %}
+  </table>
+</div>
+{% endblock %}

extensions/stats/urls.py

@@ -0,0 +1,10 @@
+from django.conf.urls.defaults import *
+
+urlpatterns = patterns('mailng.extensions.stats.main',
+                       (r'^$', 'index'),
+                       (r'^index.html$', 'index'),
+                       (r'^display/(?P<dom_id>\d+)/(?P<graph_t>\w+)/$', 'graph_display'),
+                       (r'^domain/(?P<dom_id>\d+)/$', 'domain'),
+                       )
+
+

static/stylesheets/basic.css

@@ -193,7 +193,7 @@ body {
 
 .menu2 li.selected {
 /*    text-decoration: underline;*/
-    background: #ffffcc;
+    background: #ffff77;
 }
 
 .menu2 a:link, .menu2 a:visited {
@@ -248,11 +248,11 @@ body {
 }
 
 #body tr.odd {
-    background: #F7F9Fb;
+    background: white;
 }
 
 #body tr.even {
-    background: white;
+    background: #F4F6F8;
 }
 
 #body table th {