github-jwinterm/modoboa
1
0
Fork 0
mirror of https://github.com/jwinterm/modoboa.git synced 2026-10-18 05:51:26 +02:00
Code Issues Packages Projects Releases Wiki Activity

Big stats extension update!

Browse Source 
Tickets #24 and #31 are fixed.
This commit is contained in:
tonio
2010-06-11 18:35:40 +02:00
parent 060dc98f0a
commit 415cefa928
13 changed files with 214 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
INSTALL
+1 -1
View File
@@ -473,7 +473,7 @@ To finish, you need to collect information periodically in order to
feed rrd files. To do so, just add the following lines into root's
crontab::
*/5 * * * * PYTHONPATH=<mailng_dir/..> DJANGO_SETTINGS_MODULE=mailng.settings <mailng_dir>/extensions/stats/logparser.py
*/5 * * * * PYTHONPATH=<mailng_dir/..> DJANGO_SETTINGS_MODULE=mailng.settings <mailng_dir>/scripts/stats/logparser.py
Replace mailng_dir with the installation path of MailNG.
extensions/amavis_quarantine/cleanup.py
+3
View File
@@ -4,6 +4,9 @@
from mailng.lib import db, parameters
if __name__ == "__main__":
from mailng.extensions.amavis_quarantine import main
main.init()
conn = db.getconnection("amavis_quarantine")
max_messages_age = parameters.get_admin("amavis_quarantine", "MAX_MESSAGES_AGE")
extensions/stats/grapher.py
+22 -4
View File
@@ -53,6 +53,24 @@ tpl = {'traffic':traffic_avg_template,
'badtraffic':badtraffic_avg_template,
'size':size_avg_template}
def str2Time(y, M, d, h ="0", m ="0", s="0"):
"""str2Time
return epoch time from Year Month Day Hour:Minute:Second time format
"""
try:
local = time.strptime("%s %s %s %s:%s:%s" %(y, M, d, h, m, s), \
"%Y %b %d %H:%M:%S")
except:
# try with 01-12 month format
try:
local = time.strptime("%s %s %s %s:%s:%s" %(y, M, d, h, m, s), \
"%Y %m %d %H:%M:%S")
except:
print "[rrd] ERROR unrecognized %s time format" %(y, M, d, h, m, s)
return 0
return int(time.mktime(local))
class Grapher(object):
def __init__(self):
self.rrd_rootdir = parameters.get_admin("stats", "RRD_ROOTDIR")
@@ -71,11 +89,11 @@ class Grapher(object):
defs = []
lines = []
for v, d in tpl["vars"].iteritems():
defs += ['DEF:%s=%s:%s:%s' % (v, rrdfile, v, tpl['cf']),
'CDEF:%spm=%s,60,*' % (v, v)]
defs += [str('DEF:%s=%s:%s:%s' % (v, rrdfile, v, tpl['cf'])),
str('CDEF:%spm=%s,60,*' % (v, v))]
type = d.has_key("type") and d["type"] or "LINE"
lines += ["%s:%spm%s:%s" % (type, v, d["color"],
d["legend"].encode("utf-8"))]
lines += [str("%s:%spm%s:%s" % (type, v, d["color"],
d["legend"].encode("utf-8")))]
params = defs + lines
rrdtool.graph(str(path),
"--imgformat", "PNG",
extensions/stats/logparser.py → extensions/stats/scripts/logparser.py
+18 -34
View File
@@ -35,39 +35,20 @@ points_per_sample = 3
variables = ["sent", "recv", "bounced", "reject", "spam", "virus",
"size_sent", "size_recv"]
def str2Time(y, M, d, h ="0", m ="0", s="0"):
"""str2Time
return epoch time from Year Month Day Hour:Minute:Second time format
"""
try:
local = time.strptime("%s %s %s %s:%s:%s" %(y, M, d, h, m, s), \
"%Y %b %d %H:%M:%S")
except:
# try with 01-12 month format
try:
local = time.strptime("%s %s %s %s:%s:%s" %(y, M, d, h, m, s), \
"%Y %m %d %H:%M:%S")
except:
print "[rrd] ERROR unrecognized %s time format" %(y, M, d, h, m, s)
return 0
return int(time.mktime(local))
class LogParser(object):
def __init__(self, logfile, workdir,
year=None, debug=False, verbose=False):
self.logfile = logfile
def __init__(self, options, workdir, year=None):
self.logfile = options.logfile
try:
self.f = open(logfile)
self.f = open(self.logfile)
except IOError, errno:
if self.debug:
if options.debug:
print "%s" % errno
sys.exit(1)
self.workdir = workdir
self.year = year
self.debug = debug
self.verbose = verbose
self.debug = options.debug
self.verbose = options.verbose
self.cfs = ['AVERAGE', 'MAX']
self.last_month = None
@@ -118,7 +99,7 @@ class LogParser(object):
params += ['RRA:%s:0.5:%s:%s' % (cf, step, realrows)]
# With those setup, we can now created the RRD
rrdtool.create(fname,
rrdtool.create(str(fname),
'--start', str(m),
'--step', str(rrdstep),
*params)
@@ -141,7 +122,7 @@ class LogParser(object):
print "[rrd] create new RRD file %s" %fname
else:
if not self.lupdates.has_key(fname):
self.lupdates[fname] = rrdtool.last(fname)
self.lupdates[fname] = rrdtool.last(str(fname))
if m <= self.lupdates[fname]:
if self.verbose:
@@ -164,7 +145,7 @@ class LogParser(object):
if self.verbose:
print "[rrd] VERBOSE update -t %s %s:%s (SKIP)" \
% (tpl, p, values)
rrdtool.update(fname, "-t", tpl, "%s:%s" % (p, values))
rrdtool.update(str(fname), "-t", tpl, "%s:%s" % (p, values))
values = "%s" % m
tpl = ""
@@ -177,7 +158,7 @@ class LogParser(object):
if self.verbose:
print "[rrd] VERBOSE update -t %s %s" % (tpl, values)
rrdtool.update(fname, "-t", tpl, values)
rrdtool.update(str(fname), "-t", tpl, values)
self.lupdates[fname] = m
return True
@@ -208,7 +189,7 @@ class LogParser(object):
se = int(int(se) / rrdstep) # rrd step is one-minute => se = 0
if prev_se != se or prev_mi != mi or prev_ho != ho:
cur_t = str2Time(self.year, mo, da, ho, mi, se)
cur_t = grapher.str2Time(self.year, mo, da, ho, mi, se)
cur_t = cur_t - cur_t % rrdstep
prev_mi = mi
prev_ho = ho
@@ -276,18 +257,21 @@ class LogParser(object):
G.make_defaults(dom, tpl=grapher.size_avg_template)
if __name__ == "__main__":
from mailng.extensions.stats import main
main.init()
log_file = parameters.get_admin("stats", "LOGFILE")
rrd_rootdir = parameters.get_admin("stats", "RRD_ROOTDIR")
parser = OptionParser()
parser.add_option("-l","--logFile", default=log_file,
parser.add_option("-l","--logfile", default=log_file,
help="postfix log in syslog format", metavar="FILE")
parser.add_option("-v","--verbose", default=False, action="store_true",
dest="verbose", help="set verbose mode")
parser.add_option("-d","--debug", default=False, action="store_true",
dest="debug", help="set debug mode")
help="set debug mode")
(options, args) = parser.parse_args()
P = LogParser(options.logFile, rrd_rootdir,
debug=options.debug, verbose=options.verbose)
P = LogParser(options, rrd_rootdir)
P.process()
extensions/stats/templates/adminindex.html
+1 -54
View File
@@ -1,61 +1,8 @@
{% extends "nlayout.html" %}
{% load i18n %}
{% load admin_extras %}
{% load stats_extra %}
{% block html_head %}
<style type="text/css">
#graphcontent {
margin: 25px auto 10px;
width:90%;
}
#selectors {
float: none;
}
</style>
<script type="text/javascript">
window.addEvent("domready", function() {
$("period").addEvent("change", function(event) {
{% url extensions.stats.main.adminindex as stats_url %}
location.href = "{{ stats_url }}" + "?period=" + $("period").value;
});
$("domain").addEvent("change", function(event) {
{% url extensions.stats.main.adminindex as stats_url %}
location.href = "{{ stats_url }}" + "?domain=" + $("domain").value;
});
});
</script>
<script type="text/javascript" src="/static/js/calendar.js"></script>
{% endblock %}
{% block middletitle %}{% trans "Global statistics" %}{% endblock %}
{% block menu %}{% domains_menu "stats" user %}{% endblock %}
{% block content %}
<div id="graphcontent">
<div id="selectors">
<label for="domain">{% trans "Domain:" %}</label>
<select id="domain" name="domain">
<option value="global" {% ifequal domain "global" %}selected{% endifequal %}>{% trans "Global" %}</option>
{% for d in domains %}
<option value="{{ d.id }}" {% ifequal d.name domain %}selected{% endifequal %}>{{ d.name }}</option>
{% endfor %}
</select>
<label for="period">{% trans "Period:" %}</label>
<select id="period" name="period">
{% for p in periods %}
<option value="{{ p }}" {% ifequal p period %}selected{% endifequal %}>{% getLabel p %}</option>
{% endfor %}
</select>
{% ifequal period "Custom" %}
{% include "stats/calendar.html" %}
{% endifequal %}
</div>
{% if start or end %}
<h3>{% trans "Custom period : from" %} {{ start }} {% trans "to" %} {{ end }}</h3>
{% endif %}
<div style="float:None;">
{% include "stats/graphs.html" %}
</div>
</div>
{% include "stats/content.html" %}
{% endblock %}
extensions/stats/templates/calendar.html
-29
View File
@@ -1,29 +0,0 @@
{% load i18n %}
<div style="border:1px solid #dddddd;float:right;padding:3px;margin-top:-3px;">
{% url extensions.stats.main.custom_period as c_stats_url %}
<form class="calendarForm" action="{{ c_stats_url }}" method="POST">
{% trans "Start" %}: <input id="dateTravelStart" name="start" value="{{ start }}" type="text"/>
{% trans "End" %}: <input id="dateTravelEnd" name="end" value="{{ end }}" type="text" />
<input TYPE=SUBMIT VALUE="ok">
</form>
</div>
<script type="text/javascript">
myCal = new Calendar(
{
dateTravelStart: 'D, Y/m/d',
dateTravelEnd : 'D, Y/m/d'
},
{
direction: 0, // -1 past, 0 past + future, 1 future
pad:1,
offset: 1, // first day of the week: 0 = sunday, 1 = monday, etc..
tweak: {x: -147, y: 18} // tweak calendar positioning
});
</script>
<!--
<form name="calendar">
<textarea type="text" rows="0" cols="5">{{ sdate }}</textarea>
<textarea type="text" rows="0" cols="5">{{ edate }}</textarea>
</select>
-->
extensions/stats/templates/content.html
+113
View File
@@ -0,0 +1,113 @@
{% load i18n %}
<link rel="stylesheet" type="text/css" href="/static/stylesheets/calendar.css" />
<style type="text/css">
#selectors {
}
#periodbox, #domainbox {
float: left;
margin-right: 20px;
}
#graphs {
margin-top: 15px;
float: none;
}
#customperiod {
display: none;
border: 1px solid #dddddd;
float: left;
padding: 3px;
margin-top: -3px;
}
</style>
<script type="text/javascript">
window.addEvent("domready", function() {
var myCal = new Calendar({
dateTravelStart: 'Y-m-d',
dateTravelEnd : 'Y-m-d'
}, {
direction: 0, // -1 past, 0 past + future, 1 future
pad: 1,
offset: 1, // first day of the week: 0 = sunday, 1 = monday, etc..
tweak: {x: -147, y: 18} // tweak calendar positioning
});
getgraph_callback = function(resp) {
if (resp.status == "ok") {
$("graphs").set("html", resp.content);
} else {
infobox.error(resp.error);
}
}
get_period = function() {
if ($("period").value != "custom") {
$("customperiod").setStyle("display", "none");
return "period=" + $("period").value;
}
$("customperiod").setStyle("display", "block");
if ($("dateTravelStart").value == "" || $("dateTravelEnd").value == "") {
return "";
}
return "period=custom"
+ "&start=" + $("dateTravelStart").value
+ "&end=" + $("dateTravelEnd").value;
}
get_graph = function(evt) {
{% if domains %}
var url = "{% url getgraph_root %}" + $("domain").value + "/";
{% else %}
var url = "{% url getgraph domain.id %}";
{% endif %}
var p = get_period();
if (p == "") {
return;
}
new Request.JSON({url : url + "?" + p, onSuccess : getgraph_callback}).get();
}
{% if domains %}
$("domain").addEvent("change", get_graph);
{% endif %}
$("period").addEvent("change", get_graph);
$("customsend").addEvent("click", get_graph);
});
</script>
<script type="text/javascript" src="/static/js/calendar.js"></script>
<div id="selectors">
{% if domains %}
<div id="domainbox">
<label for="domain">{% trans "Domain:" %}</label>
<select id="domain" name="domain">
<option value="global" {% ifequal domain "global" %}selected{% endifequal %}>{% trans "Global" %}</option>
{% for d in domains %}
<option value="{{ d.id }}" {% ifequal d.name domain %}selected{% endifequal %}>{{ d.name }}</option>
{% endfor %}
</select>
</div>
{% endif %}
<div id="periodbox">
<label for="period">{% trans "Period:" %}</label>
<select id="period" name="period">
{% for p in periods %}
<option value="{{ p.name }}" {% ifequal p.name period %}selected{% endifequal %}>{{ p.label }}</option>
{% endfor %}
</select>
</div>
<div id="customperiod">
{% url extensions.stats.main.custom_period as c_stats_url %}
{% trans "Start" %}: <input id="dateTravelStart" name="start"
value="{{ start }}" type="text"/>
{% trans "End" %}: <input id="dateTravelEnd" name="end"
value="{{ end }}" type="text" />
<input type="button" id="customsend" value="ok">
</div>
</div>
<div class="spacer"></div>
<div id="graphs">
{% include "stats/graphs.html" %}
</div>
extensions/stats/templates/graphs.html
+4 -1
View File
@@ -1,6 +1,9 @@
{% load i18n %}
{% if start or end %}
<h2>{% trans "Custom period : from" %} {{ start }} {% trans "to" %} {{ end }}</h2>
{% endif %}
{% for elems in graphs %}
<h3>{{ elems.label }}</h3>
<p><img src="/img/stats/{{ elems.name }}_{{ domain }}_AVERAGE_{% ifequal period 'Custom' %}{{ period_name }}{% else %}{{ period }}{% endifequal %}.png"
<p><img src="/img/stats/{{ elems.name }}_{{ domain }}_AVERAGE_{% ifequal period 'custom' %}{{ period_name }}{% else %}{{ period }}{% endifequal %}.png"
alt="{% trans 'No statistics available' %}" /></p>
{% endfor %}
extensions/stats/templates/index.html
+1 -40
View File
@@ -1,47 +1,8 @@
{% extends "admin/domain_header.html" %}
{% load i18n %}
{% block html_head %}
<link rel="stylesheet" type="text/css" href="/static/stylesheets/calendar.css" />
<style type="text/css">
#selectors {
float: none;
}
</style>
<script type="text/javascript">
window.addEvent("domready", function() {
$("period").addEvent("change", function(event) {
{% url extensions.stats.main.index domain.id as stats_url %}
location.href = "{{ stats_url }}" + "?period=" + $("period").value;
});
});
</script>
<script type="text/javascript" src="/static/js/calendar.js"></script>
{% endblock %}
{% block menu %}
{% if domain %}
{% load admin_extras %}
{% domain_menu domain.id "stats" user %}
{% endif %}
{% endblock %}
{% block domcontent %}
<div id="selectors">
<label for="period">{% trans "Period:" %}</label>
<select id="period" name="period">
{% for p in periods %}
<option value="{{ p }}" {% ifequal p period %}selected{% endifequal %}>{{ p }}</option>
{% endfor %}
</select>
{% ifequal period "Custom" %}
{% include "stats/calendar.html" %}
{% endifequal %}
</div>
{% if start or end %}
<h3>{% trans "Custom period : from" %} {{ start }} {% trans "to" %} {{ end }}</h3>
{% endif %}
<div style="float:None;">
{% include "stats/graphs.html" %}
</div>
</div>
{% include "stats/content.html" %}
{% endblock %}
extensions/stats/templatetags/__init__.py
View File
extensions/stats/templatetags/stats_extra.py
-14
View File
@@ -1,14 +0,0 @@
from django import template
from django.utils.translation import ugettext as _
register = template.Library()
@register.simple_tag
def getLabel(var):
labels = {
"day" : _("Day"),
"week" : _("Week"),
"month" : _("Month"),
"year" : _("Year")
}
return labels.has_key(var) and labels[var] or var
extensions/stats/urls.py
+2 -3
View File
@@ -2,10 +2,9 @@ from django.conf.urls.defaults import *
urlpatterns = patterns('mailng.extensions.stats.views',
url(r'^$', 'adminindex', name='fullindex'),
url(r'^custom_period$', 'custom_period'),
url(r'^(?P<dom_id>\d+)/$', 'index', name='domindex'),
(r'^display/(?P<dom_id>\d+)/(?P<graph_t>\w+)/$', 'graph_display'),
(r'^domain/(?P<dom_id>\d+)/$', 'domain'),
url(r'^graph/$', "getgraph", name="getgraph_root"),
url(r'^graph/(?P<dom_id>\w+)/$', "getgraph", name="getgraph"),
)
extensions/stats/views.py
+49 -107
View File
@@ -1,65 +1,30 @@
# -*- coding: utf-8 -*-
import calendar
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _
from mailng.lib import parameters, _render
from django.utils import simplejson
from django.template.loader import render_to_string
from mailng.lib import parameters, _render, getctx
from mailng.admin.views import good_domain
from mailng.admin.models import Domain, Mailbox
from django.contrib.auth.decorators \
import login_required, user_passes_test, permission_required
from extensions.stats.grapher import Grapher, tpl
from extensions.stats.logparser import str2Time
from extensions.stats.grapher import *
graph_types = ['AVERAGE', 'MAX']
graph_list = [{"name" : "traffic", "label" : _("Average normal traffic")},
{"name" : "badtraffic", "label" : _("Average bad traffic")},
{"name" : "size", "label" : _("Average normal traffic size")}]
@login_required
def domain(request, dom_id):
domains = []
errors = []
return graph_display(request,dom_id,graph_types)
@login_required
def graph_display(request,dom_id,graph_t=graph_types):
domains = []
errors = []
graph_type = None
domain=None
if type(graph_t) == type([]):
graph_type = graph_t
else :
graph_type = [graph_t]
if dom_id:
domain = Domain.objects.get(pk=dom_id)
if domain:
domains.append(domain)
if domains == []:
errors.append(_("No Domain defined"))
for d in domains:
P = parser(d.name)
if type(P) == type('err'):
errors.append(P)
else:
P.plot_rrd()
return _render(request, 'stats/index.html', {
"page" : "Domain statistics", "graph" : graph_nature,
"domain": domain, "domains" : domains, "messages" : errors,
"types" : graph_type, "tmp_path" : tmp_path,
"img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")})
periods = [{"name" : "day", "label" : _("Day")},
{"name" : "week", "label" : _("Week")},
{"name" : "month", "label" : _("Month")},
{"name" : "year", "label" : _("Year")},
{"name" : "custom", "label" : _("Custom")}]
@login_required
@user_passes_test(lambda u: u.is_superuser)
def adminindex(request):
CH = calendar.HTMLCalendar(calendar.MONDAY)
CH_print = CH.formatmonth(2009,11)
period = request.GET.has_key("period") and request.GET["period"] or "day"
domain = request.GET.has_key("domain") and request.GET["domain"] or "global"
if domain != "global":
@@ -73,65 +38,8 @@ def adminindex(request):
return _render(request, 'stats/adminindex.html', {
"domains" : domains, "domain" : domain,
"graphs" : graph_list,
"periods" : ["day", "week", "month", "year","Custom"],
"period" : period, "cal" : CH_print,
"img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")})
@login_required
@user_passes_test(lambda u: u.is_superuser)
def custom_period(request):
start = None
end = None
start_ref = None
end_ref = None
domains = Domain.objects.all()
domain = request.GET.has_key("domain") and request.GET["domain"] or "global"
period = request.GET.has_key("period") and request.GET["period"] or "Custom"
if domain != "global":
domain = Domain.objects.get(pk=domain)
domain = domain.name
try:
start_ref = str(request.POST.get('start'))
start = start_ref.split(', ')[1]
end_ref = str(request.POST.get('end'))
end = end_ref.split(', ')[1]
except:
pass
if not end or not start:
return _render(request, 'stats/adminindex.html', {
"domains" : domains, "domain" : domain,
"graphs" : graph_list,
"period" : period,
"periods" : ["day", "week", "month", "year","Custom"],
"period_name":None,
"start" : start_ref,
"end" : end_ref,
"messages" : ["Custom period not selected"],
"img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")})
period_name = "%s_%s" %(start.replace('/',''),end.replace('/',''))
G = Grapher()
for tpl_name in graph_list:
G.process(domain,
period_name,
str2Time(*start.split('/')),
str2Time(*end.split('/')),
tpl[tpl_name['name']])
print "[stats] ", period
return _render(request, 'stats/adminindex.html', {
"domains" : domains, "domain" : domain,
"graphs" : graph_list,
"period" : period,
"periods" : ["day", "week", "month", "year","Custom"],
"period_name": period_name,
"start" : start_ref,
"end" : end_ref,
"messages" : None,
"img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")})
"periods" : periods,
"period" : period})
@login_required
@good_domain
@@ -145,6 +53,40 @@ def index(request, dom_id=None):
return _render(request, 'stats/index.html', {
"domain" : domain,
"graphs" : graph_list,
"periods" : ["day", "week", "month", "year","Custom"],
"period" : period,
"img_rootdir" : parameters.get_admin("stats", "IMG_ROOTDIR")})
"periods" : periods,
"period" : period
})
@login_required
@good_domain
@permission_required("admin.view_mailboxes")
def getgraph(request, dom_id):
period = request.GET.has_key("period") and request.GET["period"] or "day"
if dom_id == "global":
domain = dom_id
else:
domain = Domain.objects.get(pk=dom_id).name
ctx = None
tplvars = {"graphs" : graph_list, "period" : period, "domain" : domain}
if period == "custom":
if not request.GET.has_key("start") or not request.GET.has_key("end"):
ctx = getctx("ko", error=_("Bad custom period"))
else:
start = request.GET["start"]
end = request.GET["end"]
G = Grapher()
period_name = "%s_%s" % (start.replace('-',''), end.replace('-',''))
for tpl_name in graph_list:
G.process(domain,
period_name,
str2Time(*start.split('-')),
str2Time(*end.split('-')),
tpl[tpl_name['name']])
tplvars["period_name"] = period_name
tplvars["start"] = start
tplvars["end"] = end
if ctx is None:
content = render_to_string("stats/graphs.html", tplvars)
ctx = getctx("ok", content=content)
return HttpResponse(simplejson.dumps(ctx), mimetype="application/json")