wip: big refactoring

stats extension OK.

modoboa/core/models.py

@@ -19,6 +19,7 @@ from django.contrib.auth.models import (
 from django.contrib.contenttypes.models import ContentType
 from modoboa.lib import events, md5crypt, parameters
 from modoboa.lib.exceptions import PermDeniedException
+from modoboa.lib.sysutils import exec_cmd
 from modoboa.core.extensions import exts_pool
 from modoboa.core.exceptions import AdminError
 

modoboa/extensions/stats/__init__.py

@@ -10,7 +10,7 @@ sent, received, bounced, rejected
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _, ugettext_lazy
 from modoboa.lib import events, parameters
-from modoboa.extensions import ModoExtension, exts_pool
+from modoboa.core.extensions import ModoExtension, exts_pool
 
 
 class Stats(ModoExtension):
@@ -21,9 +21,11 @@ class Stats(ModoExtension):
     needs_media = True
 
     def load(self):
-        from app_settings import ParametersForm
+        from modoboa.extensions.stats.app_settings import ParametersForm
         events.declare(["GetGraphSets"])
-        parameters.register(ParametersForm, ugettext_lazy("Graphical statistics"))
+        parameters.register(
+            ParametersForm, ugettext_lazy("Graphical statistics")
+        )
 
     def destroy(self):
         events.unregister("AdminMenuDisplay", menu)
@@ -44,7 +46,7 @@ def menu(target, user):
 
 @events.observe("GetGraphSets")
 def get_default_graph_sets():
-    from graph_templates import MailTraffic
+    from modoboa.extensions.stats.graph_templates import MailTraffic
 
     gset = MailTraffic()
     return {gset.html_id: gset}

modoboa/extensions/stats/graph_templates.py

@@ -2,6 +2,7 @@
 
 from django.utils.translation import ugettext_lazy
 
+
 class Graph(object):
     width = 540
     height = 120
@@ -16,42 +17,45 @@ class Graph(object):
     def display_name(self):
         return self.__class__.__name__.lower()
 
+
 class AvgTraffic(Graph):
     title = ugettext_lazy('Average normal traffic')
     vertlabel = ugettext_lazy('msgs/min')
-    
+
     def __init__(self):
         super(AvgTraffic, self).__init__(
-            sent={'type' : 'AREA', 'color' : '#00FF00', 
-                  "legend" : ugettext_lazy("sent messages")},
-            recv={"type" : "AREA", "color" : "#0000FF", 
-                  "legend" : ugettext_lazy("received messages")}
-            )
+            sent={'type': 'AREA', 'color': '#00FF00',
+                  "legend": ugettext_lazy("sent messages")},
+            recv={"type": "AREA", "color": "#0000FF",
+                  "legend": ugettext_lazy("received messages")}
+        )
+
 
 class AvgBadTraffic(Graph):
     title = ugettext_lazy('Average bad traffic')
     vertlabel = ugettext_lazy('msgs/min')
-    
+
     def __init__(self):
         super(AvgBadTraffic, self).__init__(
-            bounced={'type' : 'AREA', 'color' : '#FFFF00', 
-                     "legend" : ugettext_lazy("bounced messages")},
-            reject={"type" : "AREA", "color" : "#FF0000", 
-                    "legend" : ugettext_lazy("rejected messages")}
-            )
+            bounced={'type': 'AREA', 'color': '#FFFF00',
+                     "legend": ugettext_lazy("bounced messages")},
+            reject={"type": "AREA", "color": "#FF0000",
+                    "legend": ugettext_lazy("rejected messages")}
+        )
+
 
 class AvgTrafficSize(Graph):
     title = ugettext_lazy('Average normal traffic size')
     vertlabel = ugettext_lazy('bytes/min')
-    
+
     def __init__(self):
         super(AvgTrafficSize, self).__init__(
-            size_recv={'type' : 'AREA', 'color' : '#FF9900', 
-                       "legend" : ugettext_lazy("received size")},
-            size_sent={"type" : "AREA", "color" : "#339999", 
-                       "legend" : ugettext_lazy("sent size")}
-            )
-            
+            size_recv={'type': 'AREA', 'color': '#FF9900',
+                       "legend": ugettext_lazy("received size")},
+            size_sent={"type": "AREA", "color": "#339999",
+                       "legend": ugettext_lazy("sent size")}
+        )
+
 
 class GraphSet(object):
     title = None
@@ -63,11 +67,12 @@ class GraphSet(object):
 
     def get_graphs(self):
         if not hasattr(self, '__graph_instances'):
-            self.__graph_instances =  map(lambda gc: gc(), self.graphs)
+            self.__graph_instances = [gc() for gc in self.graphs]
         return self.__graph_instances
 
     def get_graph_names(self):
-        return map(lambda g: g.display_name, self.get_graphs())
+        return [g.display_name for g in self.get_graphs()]
+
 
 class MailTraffic(GraphSet):
     title = ugettext_lazy('Mail traffic')

modoboa/extensions/stats/grapher.py

@@ -1,15 +1,17 @@
 # coding: utf-8
 
-import sys, os
+import sys
+import os
 import time
 import datetime
 from django.utils.translation import ugettext as _, ugettext_lazy
 from modoboa.lib import parameters
 
-periods = [{"name" : "day", "label" : ugettext_lazy("Day")},
-           {"name" : "week", "label" : ugettext_lazy("Week")},
-           {"name" : "month", "label" : ugettext_lazy("Month")},
-           {"name" : "year", "label" : ugettext_lazy("Year")}]
+periods = [{"name": "day", "label": ugettext_lazy("Day")},
+           {"name": "week", "label": ugettext_lazy("Week")},
+           {"name": "month", "label": ugettext_lazy("Month")},
+           {"name": "year", "label": ugettext_lazy("Year")}]
+
 
 def str2Time(y, M, d, h="00", m="00", s="00"):
     """Date conversion
@@ -20,17 +22,18 @@ def str2Time(y, M, d, h="00", m="00", s="00"):
     """
     try:
         if not M.isdigit():
-            local = time.strptime("%s %s %s %s:%s:%s" % (y, M, d, h, m, s), \
-                                      "%Y %b %d %H:%M:%S")
+            local = time.strptime("%s %s %s %s:%s:%s" % (y, M, d, h, m, s),
+                                  "%Y %b %d %H:%M:%S")
         else:
-            local = time.strptime("%s %s %s %s:%s:%s" % (y, M, d, h, m, s), \
-                                      "%Y %m %d %H:%M:%S")
+            local = time.strptime("%s %s %s %s:%s:%s" % (y, M, d, h, m, s),
+                                  "%Y %m %d %H:%M:%S")
     except ValueError:
-        print >>sys.stderr, "Error: failed to convert date and time"
+        print >> sys.stderr, "Error: failed to convert date and time"
         return 0
 
     return int(time.mktime(local))
 
+
 class Grapher(object):
     def __init__(self):
         self.rrd_rootdir = parameters.get_admin("RRD_ROOTDIR")
@@ -53,7 +56,7 @@ class Grapher(object):
             defs += [str('DEF:%s=%s:%s:%s' % (v, rrdfile, v, graph_tpl.cf)),
                      str('CDEF:%spm=%s,60,*' % (v, v)),
                      str('VDEF:%s_total=%s,TOTAL' % (v, v))]
-            type = d.has_key("type") and d["type"] or "LINE"
+            type = d["type"] if "type" in d else "LINE"
             lines += [str("%s:%spm%s:%s:STACK" % (type, v, d["color"],
                                                   (d["legend"].encode("utf8")).ljust(20)))]
             lines.append('GPRINT:%s_total:%s%%7.0lf%%s\\l' % (v, _("Total").encode('utf-8')))
@@ -64,23 +67,25 @@ class Grapher(object):
         defs.append('GPRINT:%s_first:%s %%c:strftime' % (first_ds, "From"))
         defs.append('GPRINT:%s_last:%s %%c\\c:strftime' % (first_ds, "to"))
         defs.append('COMMENT:\\s')
-        
+
         params = defs + lines
-        rrdtool.graph(str(path),
-                      "--imgformat", "PNG",
-                      "--width", str(graph_tpl.width),
-                      "--height", str(graph_tpl.height),
-                      "--start", start,
-                      "--end", end,
-                      "--vertical-label", graph_tpl.vertlabel.encode("utf-8"),
-                      "--lower-limit", "0",
-                      "--slope-mode",
-                      "--units-exponent", "0",
-                      "--title", str("%s: %s" % (target, graph_tpl.title.encode("utf-8"))),
-                      *params)
+        rrdtool.graph(
+            str(path),
+            "--imgformat", "PNG",
+            "--width", str(graph_tpl.width),
+            "--height", str(graph_tpl.height),
+            "--start", start,
+            "--end", end,
+            "--vertical-label", graph_tpl.vertlabel.encode("utf-8"),
+            "--lower-limit", "0",
+            "--slope-mode",
+            "--units-exponent", "0",
+            "--title", str("%s: %s" % (target, graph_tpl.title.encode("utf-8"))),
+            *params
+        )
 
         if not os.path.exists(path):
-            print "[graph] Impossible to create %s graph" %path
+            print "[graph] Impossible to create %s graph" % path
 
     def make_defaults(self, target, graph_tpl):
         end = "%d" % int(time.mktime(time.localtime()))
@@ -88,5 +93,3 @@ class Grapher(object):
         self.process(target, "week", "-1week", end, graph_tpl)
         self.process(target, "month", "-1month", end, graph_tpl)
         self.process(target, "year", "-1year", end, graph_tpl)
-        
-        

modoboa/extensions/stats/management/commands/logparser.py

@@ -28,7 +28,7 @@ from optparse import make_option
 from modoboa.lib import parameters
 from modoboa.admin.models import Domain
 from modoboa.extensions.stats import Stats
-from modoboa.extensions.stats.grapher import *
+from modoboa.extensions.stats.grapher import str2Time, Grapher
 from modoboa.extensions.stats.graph_templates import MailTraffic
 
 rrdstep = 60
@@ -37,6 +37,7 @@ points_per_sample = 3
 variables = ["sent", "recv", "bounced", "reject", "spam", "virus",
              "size_sent", "size_recv"]
 
+
 class LogParser(object):
     def __init__(self, options, workdir, year=None):
         self.logfile = options["logfile"]
@@ -94,7 +95,6 @@ class LogParser(object):
             params += ['DS:%s:%s:%s:0:U' % (v, ds_type, rrdstep * 2)]
 
         # Set up RRD to archive data
-        rras = []
         for cf in ['AVERAGE', 'MAX']:
             for step in [day_steps, week_steps, month_steps, year_steps]:
                 params += ['RRA:%s:0.5:%s:%s' % (cf, step, realrows)]
@@ -120,14 +120,14 @@ class LogParser(object):
         if not os.path.exists(fname):
             self.lupdates[fname] = self.init_rrd(fname, m)
             if self.debug:
-                print "[rrd] create new RRD file %s" %fname
+                print "[rrd] create new RRD file %s" % fname
         else:
-            if not self.lupdates.has_key(fname):
+            if not fname in self.lupdates:
                 self.lupdates[fname] = rrdtool.last(str(fname))
 
         if m <= self.lupdates[fname]:
             if self.verbose:
-                print "[rrd] VERBOSE events at %s already recorded in RRD" %m
+                print "[rrd] VERBOSE events at %s already recorded in RRD" % m
             return False
 
         tpl = ""
@@ -140,7 +140,8 @@ class LogParser(object):
         if m > self.lupdates[fname] + rrdstep:
             values = ""
             for v in variables:
-                if values != "": values += ":"
+                if values != "":
+                    values += ":"
                 values += "0"
             for p in range(self.lupdates[fname] + rrdstep, m, rrdstep):
                 if self.verbose:
@@ -165,16 +166,17 @@ class LogParser(object):
 
     def initcounters(self, dom, cur_t):
         init = {}
-        for v in variables: init[v] = 0
+        for v in variables:
+            init[v] = 0
         self.data[dom][cur_t] = init
 
     def inc_counter(self, dom, cur_t, counter, val=1):
         if dom is not None and dom in self.domains:
-            if not self.data[dom].has_key(cur_t):
+            if not cur_t in self.data[dom]:
                 self.initcounters(dom, cur_t)
             self.data[dom][cur_t][counter] += val
 
-        if not self.data["global"].has_key(cur_t):
+        if not cur_t in self.data["global"]:
             self.initcounters("global", cur_t)
         self.data["global"][cur_t][counter] += val
 
@@ -211,7 +213,7 @@ class LogParser(object):
             if not m:
                 continue
             (mo, da, ho, mi, se, host, prog, pid, log) = m.groups()
-            se = int(int(se) / rrdstep)            # rrd step is one-minute => se = 0
+            se = int(int(se) / rrdstep)  # rrd step is one-minute => se = 0
 
             if prev_se != se or prev_mi != mi or prev_ho != ho:
                 cur_t = str2Time(self.year(mo), mo, da, ho, mi, se)
@@ -225,18 +227,18 @@ class LogParser(object):
 
                 m = re.search("message-id=<([^>]*)>", line_log)
                 if m:
-                    self.workdict[line_id] = {'from' : m.group(1), 'size' : 0}
+                    self.workdict[line_id] = {'from': m.group(1), 'size': 0}
                     continue
 
                 m = re.search("from=<([^>]*)>, size=(\d+)", line_log)
                 if m:
-                    self.workdict[line_id] = {'from' : m.group(1),
-                                             'size' : string.atoi(m.group(2))}
+                    self.workdict[line_id] = {'from': m.group(1),
+                                              'size': string.atoi(m.group(2))}
                     continue
 
                 m = re.search("to=<([^>]*)>.*status=(\S+)", line_log)
                 if m:
-                    if not self.workdict.has_key(line_id):
+                    if not line_id in self.workdict:
                         if self.debug:
                             print "Inconsistent mail (%s: %s), skipping" % (line_id, m.group(1))
                         continue
@@ -276,13 +278,14 @@ class LogParser(object):
         G = Grapher()
         for dom, data in self.data.iteritems():
             if self.debug:
-                print "[rrd] dealing with domain %s" %dom
+                print "[rrd] dealing with domain %s" % dom
             for t in sorted(data.keys()):
                 self.update_rrd(dom, t)
 
             for graph_tpl in MailTraffic().get_graphs():
                 G.make_defaults(dom, graph_tpl)
 
+
 class Command(BaseCommand):
     help = 'Log file parser'
 
@@ -293,7 +296,7 @@ class Command(BaseCommand):
                     dest="verbose", help="Set verbose mode"),
         make_option("--debug", default=False, action="store_true",
                     help="Set debug mode")
-        )
+    )
 
     def handle(self, *args, **options):
         Stats().load()

modoboa/extensions/stats/templates/stats/index.html

@@ -7,7 +7,7 @@
 <script type="text/javascript" src="{{ STATIC_URL }}stats/js/stats.js"></script>
 <script type="text/javascript">
 function get_domains_list(container) {
-    $.getJSON("{% url 'modoboa.admin.views.domains_list' %}", function(data) {
+    $.getJSON("{% url 'modoboa.extensions.admin.views.domain.domains_list' %}", function(data) {
         container.value = data;
     });
 }

modoboa/extensions/stats/views.py

@@ -1,13 +1,17 @@
 # coding: utf-8
+from django.shortcuts import render
 from django.utils.translation import ugettext as _, ugettext_lazy
-from modoboa.lib.webutils import _render, _render_to_string, ajax_simple_response
-from modoboa.admin.models import Domain, Mailbox
-from django.contrib.auth.decorators \
-    import login_required, user_passes_test, permission_required
-from modoboa.extensions.stats.grapher import *
-from modoboa.lib.exceptions import *
+from django.contrib.auth.decorators import (
+    login_required, user_passes_test, permission_required
+)
 from modoboa.lib import events
-from graph_templates import *
+from modoboa.lib.exceptions import ModoboaException, PermDeniedException
+from modoboa.lib.webutils import (
+    _render_to_string, ajax_simple_response
+)
+from modoboa.extensions.admin.models import Domain, Mailbox
+from modoboa.extensions.stats.grapher import periods, str2Time, Grapher
+
 
 @login_required
 @permission_required("admin.view_mailboxes")
@@ -19,16 +23,17 @@ def index(request):
     if not request.user.is_superuser:
         if not len(request.user.get_domains()):
             raise ModoboaException(_("No statistics available"))
-        
+
     period = request.GET.get("period", "day")
     graph_sets = events.raiseDictEvent('GetGraphSets')
-    return _render(request, 'stats/index.html', {
-            "periods" : periods,
-            "period" : period,
-            "selection" : "stats",
-            "deflocation" : deflocation,
-            "graph_sets" : graph_sets
-            })
+    return render(request, 'stats/index.html', {
+        "periods": periods,
+        "period": period,
+        "selection": "stats",
+        "deflocation": deflocation,
+        "graph_sets": graph_sets
+    })
+
 
 @login_required
 @user_passes_test(lambda u: u.group != "SimpleUsers")
@@ -42,31 +47,34 @@ def graphs(request):
     tplvars = dict(graphs=[], period=period)
     if searchq in [None, "global"]:
         if not request.user.is_superuser:
-            if len(request.user.get_domains()):
-                tplvars.update(domain=request.user.get_domains()[0].name)
-            else:
-                return ajax_simple_response({"status" : "ok"})
+            if not Domain.objects.get_for_admin(request.user).count():
+                return ajax_simple_response({"status": "ok"})
+            tplvars.update(
+                domain=Domain.objects.get_for_admin(request.user)[0].name
+            )
         else:
             tplvars.update(domain="global")
     else:
         domain = Domain.objects.filter(name__contains=searchq)
-        if len(domain) != 1:
-            return ajax_simple_response({"status" : "ok"})
+        if domain.count() != 1:
+            return ajax_simple_response({"status": "ok"})
         if not request.user.can_access(domain[0]):
             raise PermDeniedException
         tplvars.update(domain=domain[0].name)
 
     if period == "custom":
-        if not request.GET.has_key("start") or not request.GET.has_key("end"):
+        if not "start" in request.GET or not "end" in request.GET:
             raise ModoboaException(_("Bad custom period"))
         start = request.GET["start"]
         end = request.GET["end"]
         G = Grapher()
-        period_name = "%s_%s" % (start.replace('-',''), end.replace('-',''))
+        period_name = "%s_%s" % (start.replace('-', ''), end.replace('-', ''))
         for tpl in gsets[gset].get_graphs():
             tplvars['graphs'].append(tpl.display_name)
-            G.process(tplvars["domain"], period_name, str2Time(*start.split('-')),
-                      str2Time(*end.split('-')), tpl)
+            G.process(
+                tplvars["domain"], period_name, str2Time(*start.split('-')),
+                str2Time(*end.split('-')), tpl
+            )
         tplvars["period_name"] = period_name
         tplvars["start"] = start
         tplvars["end"] = end
@@ -74,6 +82,6 @@ def graphs(request):
         tplvars['graphs'] = gsets[gset].get_graph_names()
 
     return ajax_simple_response(dict(
-            status="ok",  
-            content=_render_to_string(request, "stats/graphs.html", tplvars)
-            ))
+        status="ok",
+        content=_render_to_string(request, "stats/graphs.html", tplvars)
+    ))