From 415cefa92833b22be582a938fae2d4b6b1d65da2 Mon Sep 17 00:00:00 2001 From: tonio Date: Fri, 11 Jun 2010 18:35:40 +0200 Subject: [PATCH] Big stats extension update! Tickets #24 and #31 are fixed. --- INSTALL | 2 +- extensions/amavis_quarantine/cleanup.py | 3 + extensions/stats/grapher.py | 26 +++- extensions/stats/{ => scripts}/logparser.py | 52 +++---- extensions/stats/templates/adminindex.html | 55 +------ extensions/stats/templates/calendar.html | 29 ---- extensions/stats/templates/content.html | 113 ++++++++++++++ extensions/stats/templates/graphs.html | 5 +- extensions/stats/templates/index.html | 41 +---- extensions/stats/templatetags/__init__.py | 0 extensions/stats/templatetags/stats_extra.py | 14 -- extensions/stats/urls.py | 5 +- extensions/stats/views.py | 156 ++++++------------- 13 files changed, 214 insertions(+), 287 deletions(-) rename extensions/stats/{ => scripts}/logparser.py (86%) delete mode 100644 extensions/stats/templates/calendar.html create mode 100644 extensions/stats/templates/content.html delete mode 100644 extensions/stats/templatetags/__init__.py delete mode 100644 extensions/stats/templatetags/stats_extra.py diff --git a/INSTALL b/INSTALL index af2961874..7e3ea7bd5 100644 --- a/INSTALL +++ b/INSTALL @@ -473,7 +473,7 @@ To finish, you need to collect information periodically in order to feed rrd files. To do so, just add the following lines into root's crontab:: - */5 * * * * PYTHONPATH= DJANGO_SETTINGS_MODULE=mailng.settings /extensions/stats/logparser.py + */5 * * * * PYTHONPATH= DJANGO_SETTINGS_MODULE=mailng.settings /scripts/stats/logparser.py Replace mailng_dir with the installation path of MailNG. diff --git a/extensions/amavis_quarantine/cleanup.py b/extensions/amavis_quarantine/cleanup.py index 7a5dd6d4f..3a80d4b9d 100755 --- a/extensions/amavis_quarantine/cleanup.py +++ b/extensions/amavis_quarantine/cleanup.py @@ -4,6 +4,9 @@ from mailng.lib import db, parameters if __name__ == "__main__": + from mailng.extensions.amavis_quarantine import main + + main.init() conn = db.getconnection("amavis_quarantine") max_messages_age = parameters.get_admin("amavis_quarantine", "MAX_MESSAGES_AGE") diff --git a/extensions/stats/grapher.py b/extensions/stats/grapher.py index d70ecc6e5..db4520638 100644 --- a/extensions/stats/grapher.py +++ b/extensions/stats/grapher.py @@ -53,6 +53,24 @@ tpl = {'traffic':traffic_avg_template, 'badtraffic':badtraffic_avg_template, 'size':size_avg_template} +def str2Time(y, M, d, h ="0", m ="0", s="0"): + """str2Time + + return epoch time from Year Month Day Hour:Minute:Second time format + """ + try: + local = time.strptime("%s %s %s %s:%s:%s" %(y, M, d, h, m, s), \ + "%Y %b %d %H:%M:%S") + except: + # try with 01-12 month format + try: + local = time.strptime("%s %s %s %s:%s:%s" %(y, M, d, h, m, s), \ + "%Y %m %d %H:%M:%S") + except: + print "[rrd] ERROR unrecognized %s time format" %(y, M, d, h, m, s) + return 0 + return int(time.mktime(local)) + class Grapher(object): def __init__(self): self.rrd_rootdir = parameters.get_admin("stats", "RRD_ROOTDIR") @@ -71,11 +89,11 @@ class Grapher(object): defs = [] lines = [] for v, d in tpl["vars"].iteritems(): - defs += ['DEF:%s=%s:%s:%s' % (v, rrdfile, v, tpl['cf']), - 'CDEF:%spm=%s,60,*' % (v, v)] + defs += [str('DEF:%s=%s:%s:%s' % (v, rrdfile, v, tpl['cf'])), + str('CDEF:%spm=%s,60,*' % (v, v))] type = d.has_key("type") and d["type"] or "LINE" - lines += ["%s:%spm%s:%s" % (type, v, d["color"], - d["legend"].encode("utf-8"))] + lines += [str("%s:%spm%s:%s" % (type, v, d["color"], + d["legend"].encode("utf-8")))] params = defs + lines rrdtool.graph(str(path), "--imgformat", "PNG", diff --git a/extensions/stats/logparser.py b/extensions/stats/scripts/logparser.py similarity index 86% rename from extensions/stats/logparser.py rename to extensions/stats/scripts/logparser.py index ddc0899a0..65565a546 100755 --- a/extensions/stats/logparser.py +++ b/extensions/stats/scripts/logparser.py @@ -35,39 +35,20 @@ points_per_sample = 3 variables = ["sent", "recv", "bounced", "reject", "spam", "virus", "size_sent", "size_recv"] -def str2Time(y, M, d, h ="0", m ="0", s="0"): - """str2Time - - return epoch time from Year Month Day Hour:Minute:Second time format - """ - try: - local = time.strptime("%s %s %s %s:%s:%s" %(y, M, d, h, m, s), \ - "%Y %b %d %H:%M:%S") - except: - # try with 01-12 month format - try: - local = time.strptime("%s %s %s %s:%s:%s" %(y, M, d, h, m, s), \ - "%Y %m %d %H:%M:%S") - except: - print "[rrd] ERROR unrecognized %s time format" %(y, M, d, h, m, s) - return 0 - return int(time.mktime(local)) - class LogParser(object): - def __init__(self, logfile, workdir, - year=None, debug=False, verbose=False): - self.logfile = logfile + def __init__(self, options, workdir, year=None): + self.logfile = options.logfile try: - self.f = open(logfile) + self.f = open(self.logfile) except IOError, errno: - if self.debug: + if options.debug: print "%s" % errno sys.exit(1) self.workdir = workdir self.year = year - self.debug = debug - self.verbose = verbose + self.debug = options.debug + self.verbose = options.verbose self.cfs = ['AVERAGE', 'MAX'] self.last_month = None @@ -118,7 +99,7 @@ class LogParser(object): params += ['RRA:%s:0.5:%s:%s' % (cf, step, realrows)] # With those setup, we can now created the RRD - rrdtool.create(fname, + rrdtool.create(str(fname), '--start', str(m), '--step', str(rrdstep), *params) @@ -141,7 +122,7 @@ class LogParser(object): print "[rrd] create new RRD file %s" %fname else: if not self.lupdates.has_key(fname): - self.lupdates[fname] = rrdtool.last(fname) + self.lupdates[fname] = rrdtool.last(str(fname)) if m <= self.lupdates[fname]: if self.verbose: @@ -164,7 +145,7 @@ class LogParser(object): if self.verbose: print "[rrd] VERBOSE update -t %s %s:%s (SKIP)" \ % (tpl, p, values) - rrdtool.update(fname, "-t", tpl, "%s:%s" % (p, values)) + rrdtool.update(str(fname), "-t", tpl, "%s:%s" % (p, values)) values = "%s" % m tpl = "" @@ -177,7 +158,7 @@ class LogParser(object): if self.verbose: print "[rrd] VERBOSE update -t %s %s" % (tpl, values) - rrdtool.update(fname, "-t", tpl, values) + rrdtool.update(str(fname), "-t", tpl, values) self.lupdates[fname] = m return True @@ -208,7 +189,7 @@ class LogParser(object): se = int(int(se) / rrdstep) # rrd step is one-minute => se = 0 if prev_se != se or prev_mi != mi or prev_ho != ho: - cur_t = str2Time(self.year, mo, da, ho, mi, se) + cur_t = grapher.str2Time(self.year, mo, da, ho, mi, se) cur_t = cur_t - cur_t % rrdstep prev_mi = mi prev_ho = ho @@ -276,18 +257,21 @@ class LogParser(object): G.make_defaults(dom, tpl=grapher.size_avg_template) if __name__ == "__main__": + from mailng.extensions.stats import main + + main.init() + log_file = parameters.get_admin("stats", "LOGFILE") rrd_rootdir = parameters.get_admin("stats", "RRD_ROOTDIR") parser = OptionParser() - parser.add_option("-l","--logFile", default=log_file, + parser.add_option("-l","--logfile", default=log_file, help="postfix log in syslog format", metavar="FILE") parser.add_option("-v","--verbose", default=False, action="store_true", dest="verbose", help="set verbose mode") parser.add_option("-d","--debug", default=False, action="store_true", - dest="debug", help="set debug mode") + help="set debug mode") (options, args) = parser.parse_args() - P = LogParser(options.logFile, rrd_rootdir, - debug=options.debug, verbose=options.verbose) + P = LogParser(options, rrd_rootdir) P.process() diff --git a/extensions/stats/templates/adminindex.html b/extensions/stats/templates/adminindex.html index 31bba8802..75d44111c 100644 --- a/extensions/stats/templates/adminindex.html +++ b/extensions/stats/templates/adminindex.html @@ -1,61 +1,8 @@ {% extends "nlayout.html" %} {% load i18n %} {% load admin_extras %} -{% load stats_extra %} -{% block html_head %} - - - - -{% endblock %} {% block middletitle %}{% trans "Global statistics" %}{% endblock %} {% block menu %}{% domains_menu "stats" user %}{% endblock %} {% block content %} -
-
- - - - - - {% ifequal period "Custom" %} - {% include "stats/calendar.html" %} - {% endifequal %} -
- {% if start or end %} -

{% trans "Custom period : from" %} {{ start }} {% trans "to" %} {{ end }}

- {% endif %} -
- {% include "stats/graphs.html" %} -
-
+{% include "stats/content.html" %} {% endblock %} diff --git a/extensions/stats/templates/calendar.html b/extensions/stats/templates/calendar.html deleted file mode 100644 index 9add75625..000000000 --- a/extensions/stats/templates/calendar.html +++ /dev/null @@ -1,29 +0,0 @@ -{% load i18n %} -
- {% url extensions.stats.main.custom_period as c_stats_url %} -
- {% trans "Start" %}: - {% trans "End" %}: - -
-
- - - diff --git a/extensions/stats/templates/content.html b/extensions/stats/templates/content.html new file mode 100644 index 000000000..952e4b833 --- /dev/null +++ b/extensions/stats/templates/content.html @@ -0,0 +1,113 @@ +{% load i18n %} + + + + + + +
+ {% if domains %} +
+ + +
+ {% endif %} +
+ + +
+
+ {% url extensions.stats.main.custom_period as c_stats_url %} + {% trans "Start" %}: + {% trans "End" %}: + +
+
+ +
+
+ {% include "stats/graphs.html" %} +
diff --git a/extensions/stats/templates/graphs.html b/extensions/stats/templates/graphs.html index c3d6a2a19..1779fff3b 100644 --- a/extensions/stats/templates/graphs.html +++ b/extensions/stats/templates/graphs.html @@ -1,6 +1,9 @@ {% load i18n %} +{% if start or end %} +

{% trans "Custom period : from" %} {{ start }} {% trans "to" %} {{ end }}

+{% endif %} {% for elems in graphs %}

{{ elems.label }}

-

{% trans 'No statistics available' %}

{% endfor %} diff --git a/extensions/stats/templates/index.html b/extensions/stats/templates/index.html index cfb2eb61b..33d8dff90 100644 --- a/extensions/stats/templates/index.html +++ b/extensions/stats/templates/index.html @@ -1,47 +1,8 @@ {% extends "admin/domain_header.html" %} -{% load i18n %} -{% block html_head %} - - - - - -{% endblock %} - {% block menu %} -{% if domain %} {% load admin_extras %} {% domain_menu domain.id "stats" user %} -{% endif %} {% endblock %} {% block domcontent %} -
- - - {% ifequal period "Custom" %} - {% include "stats/calendar.html" %} - {% endifequal %} -
- {% if start or end %} -

{% trans "Custom period : from" %} {{ start }} {% trans "to" %} {{ end }}

- {% endif %} -
- {% include "stats/graphs.html" %} -
- +{% include "stats/content.html" %} {% endblock %} diff --git a/extensions/stats/templatetags/__init__.py b/extensions/stats/templatetags/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/extensions/stats/templatetags/stats_extra.py b/extensions/stats/templatetags/stats_extra.py deleted file mode 100644 index 3f4222c73..000000000 --- a/extensions/stats/templatetags/stats_extra.py +++ /dev/null @@ -1,14 +0,0 @@ -from django import template -from django.utils.translation import ugettext as _ - -register = template.Library() - -@register.simple_tag -def getLabel(var): - labels = { - "day" : _("Day"), - "week" : _("Week"), - "month" : _("Month"), - "year" : _("Year") - } - return labels.has_key(var) and labels[var] or var diff --git a/extensions/stats/urls.py b/extensions/stats/urls.py index e22eaaa2f..11fabce95 100644 --- a/extensions/stats/urls.py +++ b/extensions/stats/urls.py @@ -2,10 +2,9 @@ from django.conf.urls.defaults import * urlpatterns = patterns('mailng.extensions.stats.views', url(r'^$', 'adminindex', name='fullindex'), - url(r'^custom_period$', 'custom_period'), url(r'^(?P\d+)/$', 'index', name='domindex'), - (r'^display/(?P\d+)/(?P\w+)/$', 'graph_display'), - (r'^domain/(?P\d+)/$', 'domain'), + url(r'^graph/$', "getgraph", name="getgraph_root"), + url(r'^graph/(?P\w+)/$', "getgraph", name="getgraph"), ) diff --git a/extensions/stats/views.py b/extensions/stats/views.py index 783f4b399..8b5a2451b 100644 --- a/extensions/stats/views.py +++ b/extensions/stats/views.py @@ -1,65 +1,30 @@ # -*- coding: utf-8 -*- -import calendar +from django.http import HttpResponse from django.core.urlresolvers import reverse from django.utils.translation import ugettext as _ -from mailng.lib import parameters, _render +from django.utils import simplejson +from django.template.loader import render_to_string +from mailng.lib import parameters, _render, getctx from mailng.admin.views import good_domain from mailng.admin.models import Domain, Mailbox from django.contrib.auth.decorators \ import login_required, user_passes_test, permission_required -from extensions.stats.grapher import Grapher, tpl -from extensions.stats.logparser import str2Time +from extensions.stats.grapher import * graph_types = ['AVERAGE', 'MAX'] graph_list = [{"name" : "traffic", "label" : _("Average normal traffic")}, {"name" : "badtraffic", "label" : _("Average bad traffic")}, {"name" : "size", "label" : _("Average normal traffic size")}] - -@login_required -def domain(request, dom_id): - domains = [] - errors = [] - return graph_display(request,dom_id,graph_types) - -@login_required -def graph_display(request,dom_id,graph_t=graph_types): - domains = [] - errors = [] - graph_type = None - domain=None - if type(graph_t) == type([]): - graph_type = graph_t - else : - graph_type = [graph_t] - - if dom_id: - domain = Domain.objects.get(pk=dom_id) - if domain: - domains.append(domain) - - if domains == []: - errors.append(_("No Domain defined")) - - - for d in domains: - P = parser(d.name) - if type(P) == type('err'): - errors.append(P) - else: - P.plot_rrd() - - return _render(request, 'stats/index.html', { - "page" : "Domain statistics", "graph" : graph_nature, - "domain": domain, "domains" : domains, "messages" : errors, - "types" : graph_type, "tmp_path" : tmp_path, - "img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")}) +periods = [{"name" : "day", "label" : _("Day")}, + {"name" : "week", "label" : _("Week")}, + {"name" : "month", "label" : _("Month")}, + {"name" : "year", "label" : _("Year")}, + {"name" : "custom", "label" : _("Custom")}] @login_required @user_passes_test(lambda u: u.is_superuser) def adminindex(request): - CH = calendar.HTMLCalendar(calendar.MONDAY) - CH_print = CH.formatmonth(2009,11) period = request.GET.has_key("period") and request.GET["period"] or "day" domain = request.GET.has_key("domain") and request.GET["domain"] or "global" if domain != "global": @@ -73,65 +38,8 @@ def adminindex(request): return _render(request, 'stats/adminindex.html', { "domains" : domains, "domain" : domain, "graphs" : graph_list, - "periods" : ["day", "week", "month", "year","Custom"], - "period" : period, "cal" : CH_print, - "img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")}) - -@login_required -@user_passes_test(lambda u: u.is_superuser) -def custom_period(request): - start = None - end = None - start_ref = None - end_ref = None - domains = Domain.objects.all() - domain = request.GET.has_key("domain") and request.GET["domain"] or "global" - period = request.GET.has_key("period") and request.GET["period"] or "Custom" - - if domain != "global": - domain = Domain.objects.get(pk=domain) - domain = domain.name - try: - start_ref = str(request.POST.get('start')) - start = start_ref.split(', ')[1] - end_ref = str(request.POST.get('end')) - end = end_ref.split(', ')[1] - except: - pass - - if not end or not start: - return _render(request, 'stats/adminindex.html', { - "domains" : domains, "domain" : domain, - "graphs" : graph_list, - "period" : period, - "periods" : ["day", "week", "month", "year","Custom"], - "period_name":None, - "start" : start_ref, - "end" : end_ref, - "messages" : ["Custom period not selected"], - "img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")}) - - period_name = "%s_%s" %(start.replace('/',''),end.replace('/','')) - G = Grapher() - for tpl_name in graph_list: - G.process(domain, - period_name, - str2Time(*start.split('/')), - str2Time(*end.split('/')), - tpl[tpl_name['name']]) - - print "[stats] ", period - return _render(request, 'stats/adminindex.html', { - "domains" : domains, "domain" : domain, - "graphs" : graph_list, - "period" : period, - "periods" : ["day", "week", "month", "year","Custom"], - "period_name": period_name, - "start" : start_ref, - "end" : end_ref, - "messages" : None, - "img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")}) - + "periods" : periods, + "period" : period}) @login_required @good_domain @@ -145,6 +53,40 @@ def index(request, dom_id=None): return _render(request, 'stats/index.html', { "domain" : domain, "graphs" : graph_list, - "periods" : ["day", "week", "month", "year","Custom"], - "period" : period, - "img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")}) + "periods" : periods, + "period" : period + }) + +@login_required +@good_domain +@permission_required("admin.view_mailboxes") +def getgraph(request, dom_id): + period = request.GET.has_key("period") and request.GET["period"] or "day" + if dom_id == "global": + domain = dom_id + else: + domain = Domain.objects.get(pk=dom_id).name + ctx = None + tplvars = {"graphs" : graph_list, "period" : period, "domain" : domain} + if period == "custom": + if not request.GET.has_key("start") or not request.GET.has_key("end"): + ctx = getctx("ko", error=_("Bad custom period")) + else: + start = request.GET["start"] + end = request.GET["end"] + G = Grapher() + period_name = "%s_%s" % (start.replace('-',''), end.replace('-','')) + for tpl_name in graph_list: + G.process(domain, + period_name, + str2Time(*start.split('-')), + str2Time(*end.split('-')), + tpl[tpl_name['name']]) + tplvars["period_name"] = period_name + tplvars["start"] = start + tplvars["end"] = end + if ctx is None: + content = render_to_string("stats/graphs.html", tplvars) + ctx = getctx("ok", content=content) + + return HttpResponse(simplejson.dumps(ctx), mimetype="application/json")