diff --git a/INSTALL b/INSTALL index 560e0ad41..9d2c4cdaa 100644 --- a/INSTALL +++ b/INSTALL @@ -15,10 +15,16 @@ Download Mailng's tarball and extract it:: tar xzf mailng-.tar.gz mv mailng- mailng -Enter the mailng directory, create a settings.py file (you can use -settings-sample.py as a model). Specify database relative informations. You -must choose between mysql and postgresql, other types are not supported by -major email projects like postfix or dovecot. +Settings file + + Enter the mailng directory, create a settings.py file (you can use + settings-sample.py as a model). + Specify database relative informations. + You must choose between mysql and postgresql, other types are not supported by + major email projects like postfix or dovecot. + Create a database with the name you specified in settings.py. + you will probably need python database module backend like MySQLdb. + Modify TEMPLATE_DIRS. Initialise the database by running the following command:: @@ -378,4 +384,15 @@ Replace mailng_dir with the installation path of MailNG. By default, messages older than 14 days are automatically purged. You can modify this value by changing the variable MAX_MESSAGES_AGE inside -settings.py. \ No newline at end of file +settings.py. + +Statistics plugin setup +----------------------- + +Early version of that plugin only support parsing of postfix log as syslog format. +Please make sure you have a valid /var/log/maillog postfix log file. +Don't forget to create a link from /templates/stats to extensions/stats/template +FIXME : +Parsing for domain is not fully supported, soon in a next version. +User priviledges are not handled properly. So you understand +it is a really eraly pluggin version!! Enjoy diff --git a/extensions/stats/__init__.py b/extensions/stats/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/extensions/stats/mailrrd.py b/extensions/stats/mailrrd.py new file mode 100755 index 000000000..00ce79505 --- /dev/null +++ b/extensions/stats/mailrrd.py @@ -0,0 +1,381 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Interface with rrdtool to generate appropriate postfix graph statistics +sent, recv, bounced, rejected messages are tracked. +""" +import sys, os, rrdtool, re, time +import pdb +from optparse import OptionParser +try: + import django.conf +except: + print "[rrd] Warning no Django conf found" +rrdstep = 60 +xpoints = 540 +points_per_sample = 3 +rrd_inited = False +this_minute = 0 + +rrd_file = "postfix.rrd" +png_file = "postfix.png" +log_file = "/var/log/maillog" +tmp_path = "static/tmp/" + +months_map = { + 'Jan' : 0, 'Feb' : 1, 'Mar' : 2, + 'Apr' : 3, 'May' : 4, 'Jun' : 5, + 'Jul' : 6, 'Aug' : 7, 'Sep' : 8, + 'Oct' : 9, 'Nov' :10, 'Dec' :11, + 'jan' : 0, 'feb' : 1, 'mar' : 2, + 'apr' : 3, 'may' : 4, 'jun' : 5, + 'jul' : 6, 'aug' : 7, 'sep' : 8, + 'oct' : 9, 'nov' :10, 'dec' :11, +} + +def str2Time(y,M,d,h,m,s): + """str2Time + + return epoch time from Year Month Day Hour:Minute:Second time format + """ + try: + local = time.strptime("%s %s %s %s:%s:%s" %(y,M,d,h,m,s), \ + "%Y %b %d %H:%M:%S") + except: + print "[rrd] ERROR unrecognized %s time format" %(y,M,d,h,m,s) + exit(0) + return int(time.mktime(local)) + + +class parser(): + """parser + + Parse a mail log in syslog format: + Month day HH:MM:SS host prog[pid]: log message... + + For each epoch record the parser stores number of + 'sent', 'received', 'bounced','rejected' messages over + a rrdstep period of time. + + parameters : logfile, and starting year + + """ + def __init__(self,domain,logfile=log_file,rrdfile=rrd_file,imgFile=png_file, + create=False,year=None,debug=False,verbose=False,graph=None): + """constructor + """ + self.domain = str(domain) #.replace(".","_") + self.enable_year_decrement = None + self.logfile = logfile + self.rrdfile = tmp_path + self.domain+"_"+rrdfile + self.create = create + self.year = year + self.debug = debug + self.verbose = verbose + self.graph = graph + self.imgFile = self.domain+"_"+imgFile + self.types = ['AVERAGE','MAX'] + self.natures = ['sent_recv','boun_reje'] + self.legend = {'sent_recv' : ['sent messages','received messages'], + 'boun_reje' : ['bounced messages','rejected messages']} + try: + self.f = open(logfile) + except IOError as (errno, strerror): + print "I/O error({0}): {1} ".format(errno, strerror)+logfile + self.data = {} + self.last_month = None + if not self.year: + self.year = time.localtime().tm_year + self.first_minute = 0 + self.last_minute = 0 + + if self.create and os.path.exists(logfile): + print "[rrd] force new RRD" + os.system("rm -f %s" %rrdfile) + + elif os.path.exists(self.rrdfile): + self.last_minute = rrdtool.last(self.rrdfile) + self.first_minute = rrdtool.first(self.rrdfile) + if self.debug: + print "[rrd] DEBUG updating rrd from %s"\ + %time.asctime(time.localtime(self.last_minute)) + + + self.process_log() + + def year_increment(self,month): + """year_increment + """ + if month == 0: + if self.last_month and self.last_month == 1: + self.year += 1 + self.enable_year_decrement = True + elif month == 11: + if self.enable_year_decrement \ + and self.last_month and self.last_month != 1: + self.year -= 1 + else: + self.enable_year_decrement = False + + self.last_month = month + + def process_line(self,text): + """process_line + + Parse a log line and look for + 'sent','received','bounced',rejected' messages event + + Return True if data are up-to-date + else False + """ + # get date + ret = False + + m = re.match("(\w+)\s+(\d+)\s+(\d+):(\d+):(\d+)\s+(\w+)\s+(\w+)/?\w*[[](\d+)[]]:\s+(.*)",text) + if not m : return ret + + mo = m.group(1) + da = m.group(2) + ho = m.group(3) + mi = m.group(4) + se = m.group(5) + host = m.group(6) + prog = m.group(7) + pid = m.group(8) + log = m.group(9) + + sent = 0 + recv = 0 + boun = 0 + reje = 0 + + self.year_increment(months_map[mo]) + se = int(int(se)/rrdstep) # rrd step is one-minute => se = 0 + cur_t = str2Time(self.year,mo,da,ho,mi,se) + + if not self.data.has_key(cur_t): + self.data[cur_t] = {'sent':0,'recv':0, 'boun':0,'reje':0} + + # watch events + if re.match(".*\s+status=sent\s+.*",text): + sent = 1 + + elif re.match(".*\s+status=bounced\s+.*",text): + boun = 1 + + elif re.match(".*[0-9A-Z]+: client=(\S+).*",text): + recv = 1 + + elif re.match(".*(?:[0-9A-Z]+: |NOQUEUE: )?reject: .*",text): + reje = 1 + + elif re.match(".*(?:[0-9A-Z]+: |NOQUEUE: )?milter-reject: .*",text): + if not re.match(".*Blocked by SpamAssassin.*",text): + reje = 1 + #else regiter spam + + # register events + if sent or recv or boun or reje: + self.data[cur_t]['sent'] = self.data[cur_t]['sent'] + sent + self.data[cur_t]['recv'] = self.data[cur_t]['recv'] + recv + self.data[cur_t]['boun'] = self.data[cur_t]['boun'] + boun + self.data[cur_t]['reje'] = self.data[cur_t]['reje'] + reje + return True + + return False + + def init_rrd(self,m): + """init_rrd + + Set-up Data Sources (DS) + Set-up Round Robin Archives (RRA): + - day,week,month and year archives + - 2 types : AVERAGE and MAX + + parameter : start time + return : last epoch recorded + """ + ds_type = 'ABSOLUTE' + rows = xpoints / points_per_sample + + realrows = int(rows * 1.1) # ensure that the full range is covered + day_steps = int(3600*24/(rrdstep * rows)) + week_steps = day_steps * 7 + month_steps = week_steps * 5 + year_steps = month_steps * 12 + + # Set up data sources for our RRD + ds1 = 'DS:sent:%s:%s:0:U' %(ds_type,rrdstep*2) + ds2 = 'DS:recv:%s:%s:0:U' %(ds_type,rrdstep*2) + ds3 = 'DS:boun:%s:%s:0:U' %(ds_type,rrdstep*2) + ds4 = 'DS:reje:%s:%s:0:U' %(ds_type,rrdstep*2) + + # Set up RRD to archive data + rra1 = 'RRA:%s:0.5:%s:%s' %('AVERAGE',day_steps,realrows) # day + rra2 = 'RRA:%s:0.5:%s:%s' %('AVERAGE',week_steps,realrows) # week + rra3 = 'RRA:%s:0.5:%s:%s' %('AVERAGE',month_steps,realrows) # month + rra4 = 'RRA:%s:0.5:%s:%s' %('AVERAGE',year_steps,realrows) # year + rra5 = 'RRA:%s:0.5:%s:%s' %('MAX',day_steps,realrows) + rra6 = 'RRA:%s:0.5:%s:%s' %('MAX',week_steps,realrows) + rra7 = 'RRA:%s:0.5:%s:%s' %('MAX',month_steps,realrows) + rra8 = 'RRA:%s:0.5:%s:%s' %('MAX',year_steps,realrows) + + # With those setup, we can now created the RRD + if not os.path.exists(self.rrdfile): + rrdtool.create(self.rrdfile, + '--start',str(m), + '--step',str(rrdstep), + ds1,ds2,ds3,ds4, + rra1,rra2,rra3,rra4, + rra5,rra6,rra7,rra8) + this_minute = m + else: + this_minute = rrdtool.last(self.rrdfile) + rrdstep + + return this_minute + + + def update_rrd(self,t): + """update_rrd + + Update RRD with records at t time. + + True : if data are up-to-date for current minute + False : syslog may have probably been already recorded + or something wrong + """ + m = t - (t % rrdstep) + + if not self.last_minute : + self.last_minute = self.init_rrd(m) + print "[rrd] create new RRD file" + + if m < self.last_minute : + if self.verbose: + print "[rrd] VERBOSE events at %s already recorded in RRD" %m + return False + elif m == self.last_minute : + return True + + # Missing some RRD steps + if m > self.last_minute + rrdstep: + for p in range(self.last_minute+rrdstep,m,rrdstep): + if self.verbose: + print "[rrd] VERBOSE update %s:%s:%s:%s:%s (SKIP)" \ + %(p,'0','0','0','0') + rrdtool.update(self.rrdfile,"%s:%s:%s:%s:%s" \ + %(p,'0','0','0','0')) + + if self.verbose: + print "[rrd] VERBOSE update %s:%s:%s:%s:%s" \ + %(m,self.data[m]['sent'], self.data[m]['recv'],\ + self.data[m]['boun'], self.data[m]['reje']) + + rrdtool.update(self.rrdfile,"%s:%s:%s:%s:%s" \ + %(m,self.data[m]['sent'], self.data[m]['recv'],\ + self.data[m]['boun'], self.data[m]['reje'])) + + self.last_minute = m + return True + + + def process_log(self): + """process_log + + Go through entire log file + """ + for line in self.f.readlines(): + evt = self.process_line(line) + + # Sort everything by time + sortedData = {} + sortedData = [ (i,self.data[i]) for i in sorted(self.data.keys()) ] + + for t, dict in sortedData: + if self.update_rrd(t): + if not self.first_minute: + self.first_minute = t + self.last_minute = t + + + def plot_rrd(self,f=None,color1 = "#990033", color2 = "#330099", + year = None, start = None, end = None, t =None, n = None): + """plot_rrd + + Graph rrd from start to end epoch + """ + if f: + self.imgFile = f + start = str(self.first_minute) + end = str(self.last_minute) + if t: + self.types = t + if n: + self.natures = n + + if self.graph: + try: + start = int(time.mktime(time.strptime(self.graph[0], \ + "%Y %b %d %H:%M:%S"))) + end = int(time.mktime(time.strptime(self.graph[1], \ + "%Y %b %d %H:%M:%S"))) + except: + print "[rrd] ERROR bad time format for option --graph" + + + print "[rrd] plot rrd graph from %s to %s" \ + %(time.asctime(time.localtime(float(start))),\ + time.asctime(time.localtime(float(end)))) + if not year: year = time.localtime().tm_year + self.imgFile + if not os.path.exists(tmp_path): + os.mkdir(tmp_path) + for n in self.natures: + for t in self.types: + path = '%s%s_%s_%s'%(tmp_path,n,t,os.path.basename(self.imgFile)) + ds1 = n.split('_')[0] + ds2 = n.split('_')[1] + print 'DEF:%s=%s:%s:%s:' %(ds1,ds1, self.rrdfile,t) + print 'DEF:%s=%s:%s:%s:' %(ds2,ds2, self.rrdfile,t) + print 'LINE:%s%s:%s messages' %(ds1,color1,self.legend[n][0]) + print 'LINE:%s%s:%s messages' %(ds2,color2,self.legend[n][1]) + rrdtool.graph( + path, + '--imgformat','PNG', + '--width','540', + '--height','100', + '--start', str(start), + '--end', str(end), + '--vertical-label', '%s message' %t.lower(), + '--title', '%s message flow per minute' %t.lower(), + '--lower-limit', '0', + 'DEF:%s=%s:%s:%s:' %(ds1,self.rrdfile,ds1,t), + 'DEF:%s=%s:%s:%s:' %(ds2,self.rrdfile,ds2,t), + 'LINE:%s%s:%s' %(ds1,color1,self.legend[n][0]), + 'LINE:%s%s:%s' %(ds2,color2,self.legend[n][1]) + ) + +if __name__ == "__main__": + parser = OptionParser() + + parser.add_option("-f","--file",dest="filename",default=rrdfile, + help="write rrd data base to FILE",metavar="FILE") + parser.add_option("-i","--imgFile",default=png_file, + help="graph rrd stats to FILE",metavar="FILE") + parser.add_option("-c","--create",default=False,action="store_true",dest="create", + help="force creation of new RRD") + parser.add_option("-g","--graph",nargs=2,dest="graph", + help="generate graph in between time period (y M d YY:MM:SS)", metavar="START STOP") + parser.parse_args(['--graph', 'start', 'stop']) + parser.add_option("-l","--logFile",default="maillog", + help="postfix log in syslog format",metavar="FILE") + parser.add_option("-v","--verbose",default=False,action="store_true",dest="verbose", + help="set verbose mode") + parser.add_option("-d","--debug",default=False,action="store_true",dest="debug", + help="set debug mode") + + (options,args) = parseOption() + P = parser(logfile=options.logFile,rrdfile=options.file, + debug=options.debug,verbose=options.verbose, + create=options.create,graph=options.graph,imgFile=options.imgFile) + P.plot_rrd() diff --git a/extensions/stats/main.py b/extensions/stats/main.py new file mode 100644 index 000000000..67cfea8e0 --- /dev/null +++ b/extensions/stats/main.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +""" +graph plugin do display rrd stats + +This module provides rrdtool support to retreive statistics +from postfix log : sent, received, bounced, rejected + +""" +from django.core.urlresolvers import reverse +from django.utils.translation import ugettext as _ +from django.conf.urls.defaults import * +from mailng.lib import events, _render +from mailng.admin.views import is_superuser +from mailng.admin.models import Domain, Mailbox +from mailng.extensions.stats.mailrrd import parser +from django.contrib.auth.decorators \ + import login_required +#from models import Stats +import pdb + +graph_nature = ["sent_recv","boun_reje"] +graph_types = ["AVERAGE","MAX"] +tmp_path = "static/tmp" + +def init(): + events.register("UserMenuDisplay", menu) + +def urls(): + return (r'^mailng/stats/', + include('mailng.extensions.stats.urls')) + + +def menu(**kwargs): + if kwargs["target"] != "user_menu_box": + return [] + + return [ + {"name" : _("Statistics"), + "url" : reverse(index), + "img" : "/static/pics/graph.png"} + ] + +@login_required +def domain(request,dom_id): + domains = [] + errors = [] + return graph_display(request,dom_id,graph_types) + +#@login_required +def graph_display(request,dom_id,graph_t): + domains = [] + errors = [] + graph_type = None + if type(graph_t) == type([]): + graph_type = graph_t + else : + graph_type = [graph_t] + + if dom_id: + domain = Domain.objects.get(pk=dom_id) + if domain: + domains.append(domain) + + if domains == []: + errors.append(_("No Domain defined")) + + + for d in domains: + P = parser(d.name) + P.plot_rrd() + + return _render(request, 'stats/index.html', { + "page" : "Domain statistics", "graph" : graph_nature, + "domains" : domains, "messages" : "graph display" , + "types" : graph_type, "tmp_path" : tmp_path}) + + +@login_required +def index(request, message=None): + domains = [] + errors = [] + if not is_superuser(request.user): + if not request.user.has_perm("admin.view_domains"): + if request.user.has_perm("admin.view_mailboxes"): + mb = Mailbox.objects.get(user=request.user.id) + dom_id=mb.domain.id + domain = Domain.objects.get(pk=dom_id) + if domain: + domains.append(domain) + else: + domains = Domain.objects.all() + + if domains == []: + errors.append(_("No Domain defined")) + return _render(request, 'stats/index.html', { + "page" : "Statistics", + "domains" : domains, "messages" : errors ,"types" : graph_types}) diff --git a/extensions/stats/models.py b/extensions/stats/models.py new file mode 100644 index 000000000..6916509f4 --- /dev/null +++ b/extensions/stats/models.py @@ -0,0 +1,10 @@ +from django.db import models +from mailng.admin.models import Domain, Mailbox + +""" +class Stats(models.Model): + domain = models.CharField(max_length=300,unique=True) + #rrd_file = models.CharField(max_length=100) + first = models.DateTimeField() + last = models.DateTimeField() +""" diff --git a/extensions/stats/templates/index.html b/extensions/stats/templates/index.html new file mode 100644 index 000000000..ce160cadf --- /dev/null +++ b/extensions/stats/templates/index.html @@ -0,0 +1,82 @@ +{% extends "layout.html" %} +{% load i18n %} +{% block html_head %} + + + + + +{% endblock %} + +{% block middletitle %} + {{ page }} +{% endblock %} + +{% block menu %} +{% endblock %} +{% block content %} +
+ + {% if graph %} + {% for d in domains %} + {% for g in graph %} + {% for t in types %} + + + + + + + {% endfor %} + {% endfor %} + {% endfor %} + {% else %} + {% if domains %} + + + + + + {% for dom in domains %} + + + {% url extensions.stats.main.domain dom.id as domurl %} + + + {% endfor %} + {% endif %} + {% endif %} +
{{ d }} > {{ t|lower }} > {{ g }}
+ /{{ tmp_path }}/{{ g }}_{{ t }}_{{ d }}_postfix.png +
{% trans "Graph type" %}{% trans "Domain name" %}
+ {% for t in types %} + {% url extensions.stats.main.graph_display dom.id,t as graphurl %} + {{ t|lower }} + {% endfor %} + {{ dom.name }}
+
+{% endblock %} diff --git a/extensions/stats/urls.py b/extensions/stats/urls.py new file mode 100644 index 000000000..ce6759077 --- /dev/null +++ b/extensions/stats/urls.py @@ -0,0 +1,10 @@ +from django.conf.urls.defaults import * + +urlpatterns = patterns('mailng.extensions.stats.main', + (r'^$', 'index'), + (r'^index.html$', 'index'), + (r'^display/(?P\d+)/(?P\w+)/$', 'graph_display'), + (r'^domain/(?P\d+)/$', 'domain'), + ) + + diff --git a/static/stylesheets/basic.css b/static/stylesheets/basic.css index 923f5ba43..96f36dacd 100644 --- a/static/stylesheets/basic.css +++ b/static/stylesheets/basic.css @@ -193,7 +193,7 @@ body { .menu2 li.selected { /* text-decoration: underline;*/ - background: #ffffcc; + background: #ffff77; } .menu2 a:link, .menu2 a:visited { @@ -248,11 +248,11 @@ body { } #body tr.odd { - background: #F7F9Fb; + background: white; } #body tr.even { - background: white; + background: #F4F6F8; } #body table th {