From c28f635db2da23c8f9be25e3cd9a93750802915d Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Thu, 22 Aug 2013 16:48:07 +0200 Subject: [PATCH] wip: big refactoring stats extension OK. --- modoboa/core/models.py | 1 + modoboa/extensions/stats/__init__.py | 10 +-- modoboa/extensions/stats/graph_templates.py | 47 ++++++++------ modoboa/extensions/stats/grapher.py | 57 +++++++++-------- .../stats/management/commands/logparser.py | 35 +++++----- .../stats/templates/stats/index.html | 2 +- modoboa/extensions/stats/views.py | 64 +++++++++++-------- 7 files changed, 119 insertions(+), 97 deletions(-) diff --git a/modoboa/core/models.py b/modoboa/core/models.py index ccfc3624e..d4e4b277d 100644 --- a/modoboa/core/models.py +++ b/modoboa/core/models.py @@ -19,6 +19,7 @@ from django.contrib.auth.models import ( from django.contrib.contenttypes.models import ContentType from modoboa.lib import events, md5crypt, parameters from modoboa.lib.exceptions import PermDeniedException +from modoboa.lib.sysutils import exec_cmd from modoboa.core.extensions import exts_pool from modoboa.core.exceptions import AdminError diff --git a/modoboa/extensions/stats/__init__.py b/modoboa/extensions/stats/__init__.py index 61bb62101..7197cc01c 100644 --- a/modoboa/extensions/stats/__init__.py +++ b/modoboa/extensions/stats/__init__.py @@ -10,7 +10,7 @@ sent, received, bounced, rejected from django.core.urlresolvers import reverse from django.utils.translation import ugettext as _, ugettext_lazy from modoboa.lib import events, parameters -from modoboa.extensions import ModoExtension, exts_pool +from modoboa.core.extensions import ModoExtension, exts_pool class Stats(ModoExtension): @@ -21,9 +21,11 @@ class Stats(ModoExtension): needs_media = True def load(self): - from app_settings import ParametersForm + from modoboa.extensions.stats.app_settings import ParametersForm events.declare(["GetGraphSets"]) - parameters.register(ParametersForm, ugettext_lazy("Graphical statistics")) + parameters.register( + ParametersForm, ugettext_lazy("Graphical statistics") + ) def destroy(self): events.unregister("AdminMenuDisplay", menu) @@ -44,7 +46,7 @@ def menu(target, user): @events.observe("GetGraphSets") def get_default_graph_sets(): - from graph_templates import MailTraffic + from modoboa.extensions.stats.graph_templates import MailTraffic gset = MailTraffic() return {gset.html_id: gset} diff --git a/modoboa/extensions/stats/graph_templates.py b/modoboa/extensions/stats/graph_templates.py index 5350a4c0d..cbe8ab161 100644 --- a/modoboa/extensions/stats/graph_templates.py +++ b/modoboa/extensions/stats/graph_templates.py @@ -2,6 +2,7 @@ from django.utils.translation import ugettext_lazy + class Graph(object): width = 540 height = 120 @@ -16,42 +17,45 @@ class Graph(object): def display_name(self): return self.__class__.__name__.lower() + class AvgTraffic(Graph): title = ugettext_lazy('Average normal traffic') vertlabel = ugettext_lazy('msgs/min') - + def __init__(self): super(AvgTraffic, self).__init__( - sent={'type' : 'AREA', 'color' : '#00FF00', - "legend" : ugettext_lazy("sent messages")}, - recv={"type" : "AREA", "color" : "#0000FF", - "legend" : ugettext_lazy("received messages")} - ) + sent={'type': 'AREA', 'color': '#00FF00', + "legend": ugettext_lazy("sent messages")}, + recv={"type": "AREA", "color": "#0000FF", + "legend": ugettext_lazy("received messages")} + ) + class AvgBadTraffic(Graph): title = ugettext_lazy('Average bad traffic') vertlabel = ugettext_lazy('msgs/min') - + def __init__(self): super(AvgBadTraffic, self).__init__( - bounced={'type' : 'AREA', 'color' : '#FFFF00', - "legend" : ugettext_lazy("bounced messages")}, - reject={"type" : "AREA", "color" : "#FF0000", - "legend" : ugettext_lazy("rejected messages")} - ) + bounced={'type': 'AREA', 'color': '#FFFF00', + "legend": ugettext_lazy("bounced messages")}, + reject={"type": "AREA", "color": "#FF0000", + "legend": ugettext_lazy("rejected messages")} + ) + class AvgTrafficSize(Graph): title = ugettext_lazy('Average normal traffic size') vertlabel = ugettext_lazy('bytes/min') - + def __init__(self): super(AvgTrafficSize, self).__init__( - size_recv={'type' : 'AREA', 'color' : '#FF9900', - "legend" : ugettext_lazy("received size")}, - size_sent={"type" : "AREA", "color" : "#339999", - "legend" : ugettext_lazy("sent size")} - ) - + size_recv={'type': 'AREA', 'color': '#FF9900', + "legend": ugettext_lazy("received size")}, + size_sent={"type": "AREA", "color": "#339999", + "legend": ugettext_lazy("sent size")} + ) + class GraphSet(object): title = None @@ -63,11 +67,12 @@ class GraphSet(object): def get_graphs(self): if not hasattr(self, '__graph_instances'): - self.__graph_instances = map(lambda gc: gc(), self.graphs) + self.__graph_instances = [gc() for gc in self.graphs] return self.__graph_instances def get_graph_names(self): - return map(lambda g: g.display_name, self.get_graphs()) + return [g.display_name for g in self.get_graphs()] + class MailTraffic(GraphSet): title = ugettext_lazy('Mail traffic') diff --git a/modoboa/extensions/stats/grapher.py b/modoboa/extensions/stats/grapher.py index 02f14c67b..e19f42bf6 100644 --- a/modoboa/extensions/stats/grapher.py +++ b/modoboa/extensions/stats/grapher.py @@ -1,15 +1,17 @@ # coding: utf-8 -import sys, os +import sys +import os import time import datetime from django.utils.translation import ugettext as _, ugettext_lazy from modoboa.lib import parameters -periods = [{"name" : "day", "label" : ugettext_lazy("Day")}, - {"name" : "week", "label" : ugettext_lazy("Week")}, - {"name" : "month", "label" : ugettext_lazy("Month")}, - {"name" : "year", "label" : ugettext_lazy("Year")}] +periods = [{"name": "day", "label": ugettext_lazy("Day")}, + {"name": "week", "label": ugettext_lazy("Week")}, + {"name": "month", "label": ugettext_lazy("Month")}, + {"name": "year", "label": ugettext_lazy("Year")}] + def str2Time(y, M, d, h="00", m="00", s="00"): """Date conversion @@ -20,17 +22,18 @@ def str2Time(y, M, d, h="00", m="00", s="00"): """ try: if not M.isdigit(): - local = time.strptime("%s %s %s %s:%s:%s" % (y, M, d, h, m, s), \ - "%Y %b %d %H:%M:%S") + local = time.strptime("%s %s %s %s:%s:%s" % (y, M, d, h, m, s), + "%Y %b %d %H:%M:%S") else: - local = time.strptime("%s %s %s %s:%s:%s" % (y, M, d, h, m, s), \ - "%Y %m %d %H:%M:%S") + local = time.strptime("%s %s %s %s:%s:%s" % (y, M, d, h, m, s), + "%Y %m %d %H:%M:%S") except ValueError: - print >>sys.stderr, "Error: failed to convert date and time" + print >> sys.stderr, "Error: failed to convert date and time" return 0 return int(time.mktime(local)) + class Grapher(object): def __init__(self): self.rrd_rootdir = parameters.get_admin("RRD_ROOTDIR") @@ -53,7 +56,7 @@ class Grapher(object): defs += [str('DEF:%s=%s:%s:%s' % (v, rrdfile, v, graph_tpl.cf)), str('CDEF:%spm=%s,60,*' % (v, v)), str('VDEF:%s_total=%s,TOTAL' % (v, v))] - type = d.has_key("type") and d["type"] or "LINE" + type = d["type"] if "type" in d else "LINE" lines += [str("%s:%spm%s:%s:STACK" % (type, v, d["color"], (d["legend"].encode("utf8")).ljust(20)))] lines.append('GPRINT:%s_total:%s%%7.0lf%%s\\l' % (v, _("Total").encode('utf-8'))) @@ -64,23 +67,25 @@ class Grapher(object): defs.append('GPRINT:%s_first:%s %%c:strftime' % (first_ds, "From")) defs.append('GPRINT:%s_last:%s %%c\\c:strftime' % (first_ds, "to")) defs.append('COMMENT:\\s') - + params = defs + lines - rrdtool.graph(str(path), - "--imgformat", "PNG", - "--width", str(graph_tpl.width), - "--height", str(graph_tpl.height), - "--start", start, - "--end", end, - "--vertical-label", graph_tpl.vertlabel.encode("utf-8"), - "--lower-limit", "0", - "--slope-mode", - "--units-exponent", "0", - "--title", str("%s: %s" % (target, graph_tpl.title.encode("utf-8"))), - *params) + rrdtool.graph( + str(path), + "--imgformat", "PNG", + "--width", str(graph_tpl.width), + "--height", str(graph_tpl.height), + "--start", start, + "--end", end, + "--vertical-label", graph_tpl.vertlabel.encode("utf-8"), + "--lower-limit", "0", + "--slope-mode", + "--units-exponent", "0", + "--title", str("%s: %s" % (target, graph_tpl.title.encode("utf-8"))), + *params + ) if not os.path.exists(path): - print "[graph] Impossible to create %s graph" %path + print "[graph] Impossible to create %s graph" % path def make_defaults(self, target, graph_tpl): end = "%d" % int(time.mktime(time.localtime())) @@ -88,5 +93,3 @@ class Grapher(object): self.process(target, "week", "-1week", end, graph_tpl) self.process(target, "month", "-1month", end, graph_tpl) self.process(target, "year", "-1year", end, graph_tpl) - - diff --git a/modoboa/extensions/stats/management/commands/logparser.py b/modoboa/extensions/stats/management/commands/logparser.py index 49473ae03..cbc5c2151 100644 --- a/modoboa/extensions/stats/management/commands/logparser.py +++ b/modoboa/extensions/stats/management/commands/logparser.py @@ -28,7 +28,7 @@ from optparse import make_option from modoboa.lib import parameters from modoboa.admin.models import Domain from modoboa.extensions.stats import Stats -from modoboa.extensions.stats.grapher import * +from modoboa.extensions.stats.grapher import str2Time, Grapher from modoboa.extensions.stats.graph_templates import MailTraffic rrdstep = 60 @@ -37,6 +37,7 @@ points_per_sample = 3 variables = ["sent", "recv", "bounced", "reject", "spam", "virus", "size_sent", "size_recv"] + class LogParser(object): def __init__(self, options, workdir, year=None): self.logfile = options["logfile"] @@ -94,7 +95,6 @@ class LogParser(object): params += ['DS:%s:%s:%s:0:U' % (v, ds_type, rrdstep * 2)] # Set up RRD to archive data - rras = [] for cf in ['AVERAGE', 'MAX']: for step in [day_steps, week_steps, month_steps, year_steps]: params += ['RRA:%s:0.5:%s:%s' % (cf, step, realrows)] @@ -120,14 +120,14 @@ class LogParser(object): if not os.path.exists(fname): self.lupdates[fname] = self.init_rrd(fname, m) if self.debug: - print "[rrd] create new RRD file %s" %fname + print "[rrd] create new RRD file %s" % fname else: - if not self.lupdates.has_key(fname): + if not fname in self.lupdates: self.lupdates[fname] = rrdtool.last(str(fname)) if m <= self.lupdates[fname]: if self.verbose: - print "[rrd] VERBOSE events at %s already recorded in RRD" %m + print "[rrd] VERBOSE events at %s already recorded in RRD" % m return False tpl = "" @@ -140,7 +140,8 @@ class LogParser(object): if m > self.lupdates[fname] + rrdstep: values = "" for v in variables: - if values != "": values += ":" + if values != "": + values += ":" values += "0" for p in range(self.lupdates[fname] + rrdstep, m, rrdstep): if self.verbose: @@ -165,16 +166,17 @@ class LogParser(object): def initcounters(self, dom, cur_t): init = {} - for v in variables: init[v] = 0 + for v in variables: + init[v] = 0 self.data[dom][cur_t] = init def inc_counter(self, dom, cur_t, counter, val=1): if dom is not None and dom in self.domains: - if not self.data[dom].has_key(cur_t): + if not cur_t in self.data[dom]: self.initcounters(dom, cur_t) self.data[dom][cur_t][counter] += val - if not self.data["global"].has_key(cur_t): + if not cur_t in self.data["global"]: self.initcounters("global", cur_t) self.data["global"][cur_t][counter] += val @@ -211,7 +213,7 @@ class LogParser(object): if not m: continue (mo, da, ho, mi, se, host, prog, pid, log) = m.groups() - se = int(int(se) / rrdstep) # rrd step is one-minute => se = 0 + se = int(int(se) / rrdstep) # rrd step is one-minute => se = 0 if prev_se != se or prev_mi != mi or prev_ho != ho: cur_t = str2Time(self.year(mo), mo, da, ho, mi, se) @@ -225,18 +227,18 @@ class LogParser(object): m = re.search("message-id=<([^>]*)>", line_log) if m: - self.workdict[line_id] = {'from' : m.group(1), 'size' : 0} + self.workdict[line_id] = {'from': m.group(1), 'size': 0} continue m = re.search("from=<([^>]*)>, size=(\d+)", line_log) if m: - self.workdict[line_id] = {'from' : m.group(1), - 'size' : string.atoi(m.group(2))} + self.workdict[line_id] = {'from': m.group(1), + 'size': string.atoi(m.group(2))} continue m = re.search("to=<([^>]*)>.*status=(\S+)", line_log) if m: - if not self.workdict.has_key(line_id): + if not line_id in self.workdict: if self.debug: print "Inconsistent mail (%s: %s), skipping" % (line_id, m.group(1)) continue @@ -276,13 +278,14 @@ class LogParser(object): G = Grapher() for dom, data in self.data.iteritems(): if self.debug: - print "[rrd] dealing with domain %s" %dom + print "[rrd] dealing with domain %s" % dom for t in sorted(data.keys()): self.update_rrd(dom, t) for graph_tpl in MailTraffic().get_graphs(): G.make_defaults(dom, graph_tpl) + class Command(BaseCommand): help = 'Log file parser' @@ -293,7 +296,7 @@ class Command(BaseCommand): dest="verbose", help="Set verbose mode"), make_option("--debug", default=False, action="store_true", help="Set debug mode") - ) + ) def handle(self, *args, **options): Stats().load() diff --git a/modoboa/extensions/stats/templates/stats/index.html b/modoboa/extensions/stats/templates/stats/index.html index 325e7b9da..b70947d06 100644 --- a/modoboa/extensions/stats/templates/stats/index.html +++ b/modoboa/extensions/stats/templates/stats/index.html @@ -7,7 +7,7 @@