Merge pull request #888 from tonioo/feature/dnsbl_check_879

Feature: Dnsbl check

modoboa/admin/app_settings.py

@@ -51,6 +51,16 @@ ADMIN_EVENTS = [
 class AdminParametersForm(parameters.AdminParametersForm):
     app = "admin"
 
+    dom_sep = SeparatorField(label=ugettext_lazy("Domains"))
+
+    enable_dnsbl_checks = YesNoField(
+        label=ugettext_lazy("Enable DNSBL checks"),
+        initial="yes",
+        help_text=ugettext_lazy(
+            "Check every domain against major DNSBL providers"
+        )
+    )
+
     mbsep = SeparatorField(label=ugettext_lazy("Mailboxes"))
 
     handle_mailboxes = YesNoField(

modoboa/admin/constants.py

@@ -0,0 +1,83 @@
+"""Admin constants."""
+
+DNSBL_PROVIDERS = [
+    "aspews.ext.sorbs.net",
+    "b.barracudacentral.org",
+    "bad.psky.me",
+    "bl.deadbeef.com",
+    "bl.emailbasura.org",
+    "bl.spamcannibal.org",
+    "bl.spamcop.net",
+    "blackholes.five-ten-sg.com",
+    "blacklist.woody.ch",
+    "bogons.cymru.com",
+    "cbl.abuseat.org",
+    "cdl.anti-spam.org.cn",
+    "combined.abuse.ch",
+    "combined.rbl.msrbl.net",
+    "db.wpbl.info",
+    "dnsbl-1.uceprotect.net",
+    "dnsbl-2.uceprotect.net",
+    "dnsbl-3.uceprotect.net",
+    "dnsbl.cyberlogic.net",
+    "dnsbl.inps.de",
+    "dnsbl.njabl.org",
+    "dnsbl.sorbs.net",
+    "drone.abuse.ch",
+    "duinv.aupads.org",
+    "dul.dnsbl.sorbs.net",
+    "dul.ru",
+    "dyna.spamrats.com",
+    "dynip.rothen.com",
+    "http.dnsbl.sorbs.net"
+    "images.rbl.msrbl.net",
+    "ips.backscatterer.org",
+    "ix.dnsbl.manitu.net",
+    "korea.services.net",
+    "misc.dnsbl.sorbs.net",
+    "noptr.spamrats.com",
+    "ohps.dnsbl.net.au",
+    "omrs.dnsbl.net.au",
+    "orvedb.aupads.org",
+    "osps.dnsbl.net.au",
+    "osrs.dnsbl.net.au",
+    "owfs.dnsbl.net.au",
+    "owps.dnsbl.net.au"
+    "pbl.spamhaus.org",
+    "phishing.rbl.msrbl.net",
+    "probes.dnsbl.net.au"
+    "proxy.bl.gweep.ca",
+    "proxy.block.transip.nl",
+    "psbl.surriel.com",
+    "rbl.interserver.net",
+    "rdts.dnsbl.net.au",
+    "relays.bl.gweep.ca",
+    "relays.bl.kundenserver.de",
+    "relays.nether.net",
+    "residential.block.transip.nl",
+    "ricn.dnsbl.net.au",
+    "rmst.dnsbl.net.au",
+    "sbl.spamhaus.org",
+    "short.rbl.jp",
+    "smtp.dnsbl.sorbs.net",
+    "socks.dnsbl.sorbs.net",
+    "spam.abuse.ch",
+    "spam.dnsbl.sorbs.net",
+    "spam.rbl.msrbl.net",
+    "spam.spamrats.com",
+    "spamlist.or.kr",
+    "spamrbl.imp.ch",
+    "t3direct.dnsbl.net.au",
+    "tor.dnsbl.sectoor.de",
+    "torserver.tor.dnsbl.sectoor.de",
+    "ubl.lashback.com",
+    "ubl.unsubscore.com",
+    "virbl.bit.nl",
+    "virus.rbl.jp",
+    "virus.rbl.msrbl.net",
+    "web.dnsbl.sorbs.net",
+    "wormrbl.imp.ch",
+    "xbl.spamhaus.org",
+    "zen.spamhaus.org",
+    "zombie.dnsbl.sorbs.net",
+]

modoboa/admin/lib.py

@@ -84,7 +84,8 @@ def get_domains(user, domfilter=None, searchquery=None, **extrafilters):
     :rtype: list
     :return: a list of domains and/or relay domains
     """
-    domains = Domain.objects.get_for_admin(user)
+    domains = (
+        Domain.objects.get_for_admin(user).prefetch_related("domainalias_set"))
     if domfilter:
         domains = domains.filter(type=domfilter)
     if searchquery is not None:

modoboa/admin/management/commands/modo.py

@@ -2,6 +2,7 @@
 
 from subcommand.base import SubcommandCommand
 
+from .subcommands._dnsbl import CheckDNSBLCommand
 from .subcommands._export import ExportCommand
 from .subcommands._import import ImportCommand
 
@@ -17,4 +18,5 @@ class Command(SubcommandCommand):
     subcommands = {
         "export": ExportCommand,
         "import": ImportCommand,
+        "check_dnsbl": CheckDNSBLCommand,
     }

modoboa/admin/management/commands/subcommands/_dnsbl.py

@@ -0,0 +1,137 @@
+"""Management command to check defined domains against DNSBL providers."""
+
+import ipaddress
+
+import dns.resolver
+import gevent
+from gevent import socket
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.template.loader import render_to_string
+from django.utils.functional import cached_property
+from django.utils.translation import ugettext as _
+
+from modoboa.admin import constants
+from modoboa.admin import models
+from modoboa.lib import parameters
+from modoboa.lib import email_utils
+
+
+class CheckDNSBLCommand(BaseCommand):
+    """Command class."""
+
+    help = "Check defined domains against DNSBL providers."
+
+    @cached_property
+    def providers(self):
+        """Return a list of DNSBL providers."""
+        if not hasattr(settings, "DNSBL_PROVIDERS"):
+            return constants.DNSBL_PROVIDERS
+        return settings.DNSBL_PROVIDERS
+
+    @cached_property
+    def sender(self):
+        """Return sender address for notifications."""
+        return parameters.get_admin("SENDER_ADDRESS", app="core")
+
+    def add_arguments(self, parser):
+        """Add extra arguments to command."""
+        parser.add_argument(
+            "--timeout", type=int, default=3,
+            help="Timeout used for queries.")
+
+    def query(self, ip_list, provider):
+        """Check given IP against given provider."""
+        results = {}
+        for ip in ip_list:
+            reverse = ".".join(reversed(ip.split(".")))
+            pattern = "{}.{}".format(reverse, provider)
+            try:
+                results[ip] = socket.gethostbyname(pattern)
+            except socket.gaierror:
+                results[ip] = False
+        return provider, results
+
+    def store_domain_result(self, domain, provider, results):
+        """Store provider results for domain."""
+        alerts = {}
+        to_create = []
+        for ip in results.keys():
+            result = "" if not results[ip] else results[ip]
+            dnsbl_result = models.DNSBLResult.objects.filter(
+                domain=domain, provider=provider, mx=ip).first()
+            if dnsbl_result is None:
+                to_create.append(
+                    models.DNSBLResult(
+                        domain=domain, provider=provider, mx=ip,
+                        status=result)
+                )
+            else:
+                if not dnsbl_result.status and result:
+                    if domain not in alerts:
+                        alerts[domain] = []
+                    alerts[domain].append((provider, ip))
+                dnsbl_result.status = result
+                dnsbl_result.save()
+        models.DNSBLResult.objects.bulk_create(to_create)
+        if not alerts:
+            return
+        content = render_to_string(
+            "admin/notifications/domain_in_dnsbl.html", {
+                "domain": domain, "alerts": alerts
+            })
+        subject = _("[modoboa] DNSBL issue(s) for domain {}").format(
+            domain.name)
+        for admin in domain.admins:
+            if not admin.email:
+                continue
+            status, msg = email_utils.sendmail_simple(
+                self.sender, admin.email,
+                domainsubject=subject, content=content)
+            if not status:
+                print msg
+
+    def check_domain(self, domain, timeout):
+        """Check specified domain."""
+        resolver = dns.resolver.Resolver()
+        try:
+            answers = resolver.query(domain.name, "MX")
+        except dns.resolver.NoAnswer:
+            return
+        ip_list = []
+        for answer in answers:
+            address = None
+            try:
+                ipaddress.ip_address(str(answer.exchange))
+            except ValueError:
+                try:
+                    address = socket.gethostbyname(str(answer.exchange))
+                except socket.gaierror:
+                    pass
+            else:
+                address = str(answer.exchange)
+            finally:
+                if address is not None:
+                    ip_list.append(address)
+        if len(ip_list) == 0:
+            return
+        jobs = [
+            gevent.spawn(self.query, ip_list, provider)
+            for provider in self.providers]
+        gevent.joinall(jobs, timeout)
+        for job in jobs:
+            if not job.successful():
+                continue
+            provider, results = job.value
+            self.store_domain_result(domain, provider, results)
+
+    def handle(self, *args, **options):
+        """Command entry point."""
+        if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no":
+            return
+        # Remove deprecated records first
+        models.DNSBLResult.objects.exclude(
+            provider__in=self.providers).delete()
+        for domain in models.Domain.objects.filter(enabled=True):
+            self.check_domain(domain, options.get("timeout"))

modoboa/admin/migrations/0004_auto_20160608_1423.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-06-08 12:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('admin', '0003_auto_20151118_1215'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DNSBLResult',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('provider', models.CharField(max_length=254, db_index=True)),
+                ('mx', models.GenericIPAddressField()),
+                ('status', models.CharField(blank=True, max_length=45, db_index=True)),
+                ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.Domain')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='dnsblresult',
+            unique_together=set([('domain', 'provider', 'mx')]),
+        ),
+    ]

modoboa/admin/models/__init__.py

@@ -1,10 +1,10 @@
 from .base import AdminObject
-from .domain import Domain
+from .domain import Domain, DNSBLResult
 from .domain_alias import DomainAlias
 from .mailbox import Mailbox, Quota, MailboxOperation
 from .alias import Alias, AliasRecipient
 
 __all__ = [
-    'Domain', 'DomainAlias', 'Mailbox', 'Quota', 'Alias',
+    'Domain', 'DNSBLResult', 'DomainAlias', 'Mailbox', 'Quota', 'Alias',
     'MailboxOperation', 'AdminObject', 'AliasRecipient'
 ]

modoboa/admin/models/domain.py

@@ -3,6 +3,7 @@
 from django.db import models
 from django.db.models.manager import Manager
 from django.utils.encoding import python_2_unicode_compatible, smart_text
+from django.utils.functional import cached_property
 from django.utils.translation import ugettext as _, ugettext_lazy
 
 from django.contrib.contenttypes.fields import GenericRelation
@@ -101,6 +102,16 @@ class Domain(AdminObject):
     def aliases(self):
         return self.domainalias_set
 
+    @cached_property
+    def dnsbl_status_color(self):
+        """Shortcut to DNSBL results."""
+        if not self.dnsblresult_set.exists():
+            return "warning"
+        elif self.dnsblresult_set.blacklisted().exists():
+            return "danger"
+        else:
+            return "success"
+
     def add_admin(self, account):
         """Add a new administrator to this domain.
 
@@ -207,3 +218,26 @@ class Domain(AdminObject):
             domalias.post_create(creator)
 
 reversion.register(Domain)
+
+
+class DNSBLQuerySet(models.QuerySet):
+    """Custom manager for DNSBLResultManager."""
+
+    def blacklisted(self):
+        """Return blacklisted results."""
+        return self.exclude(status="")
+
+
+class DNSBLResult(models.Model):
+    """Store a DNSBL query result."""
+
+    domain = models.ForeignKey(Domain)
+    provider = models.CharField(max_length=254, db_index=True)
+    mx = models.GenericIPAddressField()
+    status = models.CharField(max_length=45, blank=True, db_index=True)
+
+    objects = models.Manager.from_queryset(DNSBLQuerySet)()
+
+    class Meta:
+        app_label = "admin"
+        unique_together = [("domain", "provider", "mx")]

modoboa/admin/templates/admin/dnsbl_domain_detail.html

@@ -0,0 +1,35 @@
+{% extends "common/generic_modal.html" %}
+
+{% load i18n %}
+
+{% block modalbody %}
+  {% if domain.dnsbl_status_color == "danger" %}
+    <table class="table">
+      <thead>
+        <tr>
+          <th>{% trans "Provider" %}</th>
+          <th>{% trans "Address" %}</th>
+          <th>{% trans "Status" %}</th>
+        </tr>
+      </thead>
+      <tbody>
+      {% for result in domain.dnsblresult_set.blacklisted %}
+        <tr>
+          <td>{{ result.provider }}</td>
+          <td>{{ result.mx }}</td>
+          <td>
+            {% if result.status %}<span class="label label-danger">{% trans "listed" %}</span>{% else %}<span class="label label-success"></span>{% endif %}</td>
+        </tr>
+      {% endfor %}
+      </tbody>
+    </table>
+  {% else %}
+    {% if domain.dnsbl_status_color == "success" %}
+    <div class="alert alert-success">
+      {% trans "Everything is fine, this domain is not listed." %}</div>
+    {% else %}
+    <div class="alert alert-warning">
+      {% trans "No information available for this domain." %}</div>
+    {% endif %}
+  {% endif %}
+{% endblock %}

modoboa/admin/templates/admin/domain_detail.html

@@ -17,9 +17,17 @@
 
   <div class="col-xs-12 col-sm-6">
     <div class="panel panel-info">
-      <div class="panel-heading"><h3 class="panel-title">{% trans "Information" %}</h3></div>
+      <div class="panel-heading">
+        <h3 class="panel-title">{% trans "Information" %}</h3>
+      </div>
       <div class="panel-body">
-        <div class="row"><span class="title">{% trans "Creation date" %}</span><span>{{ domain.creation }}</span></div>
+        <div class="row">
+          <span class="title">{% trans "Authentication" %}</span>
+          <span class="label label-{% if domain.is_in_dnsbl %}danger{% else %}success{% endif %}"><a href="{% url 'admin:dnsbl_domain_detail' domain.pk %}" data-toggle="ajaxmodal">DNSBL</a></span>
+        </div>
+        <div class="row">
+          <span class="title">{% trans "Creation date" %}</span><span>{{ domain.creation }}</span>
+        </div>
         <div class="row"><span class="title">{% trans "Last modification date" %}</span><span>{{ domain.last_modification }}</span></div>
         <div class="row"><span class="title">{% trans "Domain aliases" %}</span><span>{{ domain.domainalias_count }}</span></div>
         <div class="row"><span class="title"><a href="{% url 'admin:identity_list' %}#list/?searchquery=@{{ object.name }}">{% trans "Mailboxes" %}</a></span><span>{{ domain.mailbox_count }}</span></div>

modoboa/admin/templates/admin/domains.html

@@ -25,5 +25,6 @@ $(document).ready(function() {
 <th id="name" class="sortable" data-sort_order="name" width="">{% trans "Name" %}</th>
 <th id="aliases" width="">{% trans "Alias(es)" %}</th>
 <th id="tags" class="sortable" data-sort_order="tags" width="">{% trans "Tags" %}</th>
+<th id="tags" width="">{% trans "Authentication" %}</th>
 <th id="actions" width="">{% trans "Actions" %}</th>
 {% endblock %}

modoboa/admin/templates/admin/domains_table.html

@@ -5,6 +5,11 @@
   <td name="name"><a href="{% url 'admin:domain_detail' domain.id %}">{{ domain }}</a>{% if not domain.enabled %} ({% trans "disabled" %}){% endif %}</td>
   <td name="aliases">{% domain_aliases domain %} </td>
   <td name="tags">{% render_tags domain.tags %}</td>
+  <td name="auth">
+    <small>
+      <span class="label label-mini label-{{ domain.dnsbl_status_color }}"><a href="{% url 'admin:dnsbl_domain_detail' domain.pk %}" data-toggle="ajaxmodal">DNSBL</a></span>
+    </small>
+  </td>
   <td name="actions">{% domain_actions user domain %}</td>
 </tr>
 {% endfor %}

modoboa/admin/templates/admin/notifications/domain_in_dnsbl.html

@@ -0,0 +1,15 @@
+{% load i18n %}
+
+{% blocktrans trimmed domain=domain.name %}
+  Modoboa detected that domain {{ domain }} is listed by the following DNSBL providers:
+{% endblocktrans %}
+
+{% for provider, ip in alerts %}
+  {{ provider }}: {{ ip }}
+{% endfor %}
+
+{% blocktrans trimmed %}
+  The domain's reputation will be affected and there is a chance that
+  emails coming from it are considered as spam. You should contact
+  those providers and ask them to unlist detected IP address(es).
+{% endblocktrans %}

modoboa/admin/tests/test_dnsbl.py

@@ -0,0 +1,32 @@
+"""DNSBL related tests."""
+
+from django.core import management
+from django.core.urlresolvers import reverse
+from django.test import override_settings
+
+from modoboa.lib.tests import ModoTestCase
+
+from .. import factories
+from .. import models
+
+
+class DNSBLTestCase(ModoTestCase):
+    """TestCase for DNSBL related features."""
+
+    @classmethod
+    def setUpTestData(cls):
+        """Create some data."""
+        super(DNSBLTestCase, cls).setUpTestData()
+        cls.domain = factories.DomainFactory(name="modoboa.org")
+        factories.DomainFactory(name="pouet.com")  # should not exist
+
+    @override_settings(DNSBL_PROVIDERS=["zen.spamhaus.org"])
+    def test_management_command(self):
+        """Check that command works fine."""
+        self.assertEqual(models.DNSBLResult.objects.count(), 0)
+        management.call_command("modo", "check_dnsbl")
+        self.assertTrue(
+            models.DNSBLResult.objects.filter(domain=self.domain).exists())
+        response = self.client.get(
+            reverse("admin:dnsbl_domain_detail", args=[self.domain.pk]))
+        self.assertEqual(response.status_code, 200)

modoboa/admin/urls.py

@@ -12,6 +12,8 @@ urlpatterns = patterns(
     url(r'^domains/$', 'domains', name="domain_list"),
     url(r'^domains/(?P<pk>\d+)/$', views.DomainDetailView.as_view(),
         name="domain_detail"),
+    url(r'^domains/(?P<pk>\d+)/dnsbl/$', views.DNSBLDomainDetailView.as_view(),
+        name="dnsbl_domain_detail"),
     url(r'^domains/list/$', '_domains', name="_domain_list"),
     url(r'^domains/flatlist/$', 'domains_list', name="domain_flat_list"),
     url(r'^domains/stats/$', 'domain_statistics', name="domain_statistics"),

modoboa/admin/views/__init__.py

@@ -1,8 +1,10 @@
 """Shortcuts."""
 
+from .dnsbl import DNSBLDomainDetailView
 from .domain import DomainDetailView
 
 
 __all__ = [
+    "DNSBLDomainDetailView",
     "DomainDetailView",
 ]

modoboa/admin/views/dnsbl.py

@@ -0,0 +1,32 @@
+"""DNSBL related views."""
+
+from django.views import generic
+
+from django.utils.translation import ugettext as _
+
+from django.contrib.auth import mixins as auth_mixins
+
+from .. import models
+
+
+class DNSBLDomainDetailView(
+        auth_mixins.PermissionRequiredMixin, generic.DetailView):
+    """DetailView for Domain."""
+
+    model = models.Domain
+    permission_required = "admin.view_domain"
+    template_name = "admin/dnsbl_domain_detail.html"
+
+    def get_queryset(self):
+        """Add some prefetching."""
+        return (
+            super(DNSBLDomainDetailView, self).get_queryset()
+            .prefetch_related("dnsblresult_set"))
+
+    def get_context_data(self, **kwargs):
+        """Add extra variables."""
+        context = super(DNSBLDomainDetailView, self).get_context_data(**kwargs)
+        context.update({
+            "title": _("DNSBL summary for {}").format(self.object.name)
+        })
+        return context

modoboa/admin/views/domain.py

@@ -164,6 +164,13 @@ class DomainDetailView(
     model = Domain
     permission_required = "admin.view_domain"
 
+    def get_queryset(self):
+        """Add some prefetching."""
+        return (
+            super(DomainDetailView, self).get_queryset()
+            .prefetch_related("domainalias_set", "mailbox_set", "alias_set")
+        )
+
     def get_context_data(self, **kwargs):
         """Include extra widgets."""
         context = super(DomainDetailView, self).get_context_data(**kwargs)

modoboa/core/app_settings.py

@@ -4,6 +4,7 @@ from django.conf import settings
 from django.utils.translation import ugettext as _, ugettext_lazy
 
 from modoboa.lib import parameters
+from modoboa.lib import fields as lib_fields
 from modoboa.lib.cryptutils import random_key
 from modoboa.lib.form_utils import (
     YesNoField, SeparatorField, InlineRadioSelect
@@ -209,6 +210,16 @@ class GeneralParametersForm(parameters.AdminParametersForm):
         required=False
     )
 
+    notif_sep = SeparatorField(label=ugettext_lazy("Notifications"))
+
+    sender_address = lib_fields.UTF8EmailField(
+        label=_("Sender address"),
+        initial="noreply@modoboa.org",
+        help_text=_(
+            "Email address used to send notifications."
+        )
+    )
+
     sep3 = SeparatorField(label=ugettext_lazy("Miscellaneous"))
 
     top_notifications_check_interval = forms.IntegerField(

requirements.txt

@@ -15,4 +15,6 @@ progressbar==2.3
 djangorestframework==3.3.1
 django-rest-swagger==0.3.5
 django-subcommand2==0.1.1
-
+gevent==1.1.1
+ipaddress
+dnspython==1.12.0