From 53df75f045d4bf043b7cd14bd46c52f41df97f32 Mon Sep 17 00:00:00 2001
From: tonio <devnull@localhost>
Date: Mon, 2 Nov 2009 18:27:23 +0100
Subject: [PATCH] Some work about statistics plugin. It is almost finished (for
 a basic version)

---
 extensions/stats/grapher.py           | 76 +++++++++++++++++++++++++++
 extensions/stats/logparser.py         | 31 ++++++-----
 extensions/stats/main.py              |  8 +--
 extensions/stats/templates/index.html | 31 +++++------
 lib/__init__.py                       |  8 +++
 5 files changed, 119 insertions(+), 35 deletions(-)
 create mode 100644 extensions/stats/grapher.py

diff --git a/extensions/stats/grapher.py b/extensions/stats/grapher.py
new file mode 100644
index 000000000..4f33e817c
--- /dev/null
+++ b/extensions/stats/grapher.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+import os
+import time
+import datetime
+import rrdtool
+from mailng.lib import getoption
+
+default_avg_template = {
+    'width'  : '540',
+    'height' : '100',
+    'title'  : 'average message flow per minute',
+    'vertlabel' : 'msgs/min',
+    'cf' : 'AVERAGE',
+    'vars' : {
+        'sent' : { "type" : "AREA", "color" : "#00FF00", "legend" : "sent messages" },
+        'recv' : { "type" : "AREA", "color" : "#0000FF", "legend" : "received messages" },
+#         'bounced' : { "color" : "#FFFF00", "legend" : "bounced messages" },
+#         'reject' : { "color" : "#FF0000", "legend" : "rejected messages" },
+        }
+}
+
+class Grapher(object):
+    def __init__(self):
+        self.rrd_rootdir = getoption("RRD_ROOTDIR", "/tmp")
+        self.img_rootdir = getoption("IMG_ROOTDIR", "/tmp")
+
+    def process(self, target, suffix, start, end, tpl=default_avg_template):
+        rrdfile = "%s/%s.rrd" % (self.rrd_rootdir, target)
+        if not os.path.exists(rrdfile):
+            return
+        ext = "png"
+        path = "%s/%s_%s_%s.%s" % (self.img_rootdir, target, tpl['cf'], suffix, ext)
+        start = str(start)
+        end = str(end)
+        defs = []
+        lines = []
+        for v, d in tpl["vars"].iteritems():
+            defs += ['DEF:%s=%s:%s:%s' % (v, rrdfile, v, tpl['cf']),
+                     'CDEF:%spm=%s,60,*' % (v, v)]
+            type = d.has_key("type") and d["type"] or "LINE"
+            lines += ["%s:%spm%s:%s" % (type, v, d["color"], d["legend"])]
+        params = defs + lines
+        rrdtool.graph(path,
+                      "--imgformat", "PNG",
+                      "--width", tpl["width"],
+                      "--height", tpl["height"],
+                      "--start", start,
+                      "--end", end,
+                      "--vertical-label", tpl["vertlabel"],
+                      "--title", tpl["title"],
+                      "--lower-limit", "0",
+                      "--slope-mode",
+                      *params)
+
+    def make_defaults(self, target):
+        #now = int(time.time())
+        #start = now - (now % 86400)
+        #end = start + 86400
+        end = "now"
+        self.process(target, "day", "-1day", end)
+
+        #today = datetime.date.today()
+        #day = today.day - today.weekday()
+        #weekbegin = datetime.date(today.year, today.month, day)
+        #start = weekbegin.strftime("%Y%m%d")
+        #end = "+1week"
+        self.process(target, "week", "-1week", end)
+
+#         monthbegin = datetime.date(today.year, today.month, 1)
+#         start = monthbegin.strftime("%Y%m%d")
+#         end = "+1month"
+        self.process(target, "month", "-1month", end)
+        self.process(target, "year", "-1year", end)
+        
+        
diff --git a/extensions/stats/logparser.py b/extensions/stats/logparser.py
index b0059bf18..40b656fba 100755
--- a/extensions/stats/logparser.py
+++ b/extensions/stats/logparser.py
@@ -6,8 +6,9 @@ import os
 import re
 import rrdtool
 from optparse import OptionParser
-from django.conf import settings
+from mailng.lib import getoption
 from mailng.admin.models import Domain
+import grapher
 
 """
 Postfix log parser.
@@ -85,7 +86,7 @@ class LogParser(object):
 
         # Set up data sources for our RRD
         params = []
-        for v in ["sent", "recv", "bounced", "reje"]:
+        for v in ["sent", "recv", "bounced", "reject"]:
             params += ['DS:%s:%s:%s:0:U' % (v, ds_type, rrdstep * 2)]
 
         # Set up RRD to archive data
@@ -137,25 +138,25 @@ class LogParser(object):
         if self.verbose:
             print "[rrd] VERBOSE update %s:%s:%s:%s:%s" \
                   %(m, self.data[dom][m]['sent'], self.data[dom][m]['recv'],\
-                    self.data[dom][m]['bounced'], self.data[dom][m]['reje'])
+                    self.data[dom][m]['bounced'], self.data[dom][m]['reject'])
 
         rrdtool.update(fname, "%s:%s:%s:%s:%s" \
                            % (m, self.data[dom][m]['sent'],  
                               self.data[dom][m]['recv'],
                               self.data[dom][m]['bounced'], 
-                              self.data[dom][m]['reje']))
+                              self.data[dom][m]['reject']))
         self.lupdates[fname] = m
         return True
 
     def inc_counter(self, dom, cur_t, counter):
         if not self.data[dom].has_key(cur_t):
             self.data[dom][cur_t] = \
-                {'sent' : 0, 'recv' : 0, 'bounced' : 0, 'reje' : 0}
+                {'sent' : 0, 'recv' : 0, 'bounced' : 0, 'reject' : 0}
         self.data[dom][cur_t][counter] += 1
 
         if not self.data["global"].has_key(cur_t):
             self.data["global"][cur_t] = \
-                {'sent' : 0, 'recv' : 0, 'bounced' : 0, 'reje' : 0}
+                {'sent' : 0, 'recv' : 0, 'bounced' : 0, 'reject' : 0}
         self.data["global"][cur_t][counter] += 1
 
     def process(self):
@@ -188,21 +189,23 @@ class LogParser(object):
                         self.inc_counter(addrto.group(2), cur_t, 'recv')
                     else:
                         self.inc_counter(addrto.group(2), cur_t, m.group(3))
+                continue
+            
+            m = re.search("NOQUEUE: reject: .*from=<(.*)> to=<([^>]*)>", log)
+            if m:
+                addrto = re.match("([^@]+)@(.+)", m.group(2))
+                if addrto and addrto.group(2) in self.domains:
+                    self.inc_counter(addrto.group(2), cur_t, 'reject')
+                continue
         
         # Sort everything by time
+        G = grapher.Grapher()
         for dom, data in self.data.iteritems():
             sortedData = {}
             sortedData = [ (i, data[i]) for i in sorted(data.keys()) ]
             for t, dict in sortedData:
                 self.update_rrd(dom, t)
-
-def getoption(name, default=None):
-    res = None
-    try:
-        res = getattr(settings, name)
-    except AttributeError:
-        res = default
-    return res
+            G.make_defaults(dom)
 
 if __name__ == "__main__":
     log_file = getoption("LOGFILE", "/var/log/maillog")
diff --git a/extensions/stats/main.py b/extensions/stats/main.py
index 9efe617f6..75ceb9dda 100644
--- a/extensions/stats/main.py
+++ b/extensions/stats/main.py
@@ -13,7 +13,7 @@ from django.conf.urls.defaults import *
 from mailng.lib import events, _render
 from mailng.admin.views import is_superuser
 from mailng.admin.models import Domain, Mailbox
-from mailng.extensions.stats.mailrrd import parser
+#from mailng.extensions.stats.mailrrd import parser
 from django.contrib.auth.decorators \
     import login_required
 import pdb
@@ -88,7 +88,7 @@ def index(request, dom_id=None):
     domains = []
     errors = []
     domain = None
-    if not is_superuser(request.user):
+    if dom_id:
         if request.user.has_perm("admin.view_mailboxes"):
             domain = Domain.objects.get(pk=dom_id)
             if domain:
@@ -102,4 +102,6 @@ def index(request, dom_id=None):
         errors.append(_("No Domain defined"))
     return _render(request, 'stats/index.html', {
         "page" : "Statistics",
-        "domain" : domain, "domains": domains, "messages" : errors ,"types" : graph_types})
+        "domain" : domain, "domains": domains, "messages" : errors , 
+        "periods" : ["day", "week", "month", "year"]
+        })
diff --git a/extensions/stats/templates/index.html b/extensions/stats/templates/index.html
index d5e0efb24..90f43880b 100644
--- a/extensions/stats/templates/index.html
+++ b/extensions/stats/templates/index.html
@@ -38,29 +38,24 @@
 {% domain_menu domain.id "stats" perms %}
 {% endif %}
 {% endblock %}
-
 {% block middletitle %}
-  {{ page }}
+{% if domain %}
+  {% trans "Statistics for" %} {{ domain.name }}
+{% else %}
+  {% trans "Global statistics" %}
+{% endif %}
 {% endblock %}
-
 {% block content %}
 <div id="graphcontent">
   <table>
-  {% if graph %}
-    {% for d in domains %}
-    {% for g in graph %}
-        {% for t in types %}
-        <tr>
-          <th class="{% cycle 'odd' 'even' %}">{{ d }} > {{ t|lower }} > {{ g }}</th>
-        </tr>
-        <tr>
-        <td>
-          <img src="/{{ tmp_path }}/{{ g }}_{{ t }}_{{ d }}_postfix.png" alt="No statistics available"/>
-        </td>
-        </tr>
-        {% endfor %}
-    {% endfor %}
-    {% endfor %}
+  {% if domain %}
+  {% for p in periods %}
+  <tr>
+    <td>
+      <img src="/static/graphs/{{ domain.name }}_AVERAGE_{{ p }}.png" alt="No statistics available" />
+    </td>
+  </tr>
+  {% endfor %}
   {% else %}
   {% if domains %}
     <tr>
diff --git a/lib/__init__.py b/lib/__init__.py
index 3a1c2e212..ff9fab9b6 100644
--- a/lib/__init__.py
+++ b/lib/__init__.py
@@ -70,3 +70,11 @@ def decode(s, encodings=('ascii', 'utf8', 'latin1')):
         except UnicodeDecodeError:
             pass
     return s.decode('ascii', 'ignore')
+
+def getoption(name, default=None):
+    res = None
+    try:
+        res = getattr(settings, name)
+    except AttributeError:
+        res = default
+    return res