From 7f233b2f269ddbfbc1b4031476e7970c953e093c Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Jun 2016 17:08:08 +0200 Subject: [PATCH 1/8] DNSBL checks for domains. see #879 --- modoboa/admin/constants.py | 83 +++++++++++++++++++ modoboa/admin/lib.py | 3 +- modoboa/admin/management/commands/modo.py | 2 + .../management/commands/subcommands/_dnsbl.py | 75 +++++++++++++++++ .../migrations/0004_auto_20160608_1423.py | 30 +++++++ modoboa/admin/models/__init__.py | 4 +- modoboa/admin/models/domain.py | 27 ++++++ .../templates/admin/dnsbl_domain_detail.html | 30 +++++++ .../admin/templates/admin/domain_detail.html | 12 ++- modoboa/admin/templates/admin/domains.html | 1 + .../admin/templates/admin/domains_table.html | 5 ++ modoboa/admin/urls.py | 2 + modoboa/admin/views/__init__.py | 2 + modoboa/admin/views/dnsbl.py | 32 +++++++ modoboa/admin/views/domain.py | 7 ++ requirements.txt | 3 +- 16 files changed, 312 insertions(+), 6 deletions(-) create mode 100644 modoboa/admin/constants.py create mode 100644 modoboa/admin/management/commands/subcommands/_dnsbl.py create mode 100644 modoboa/admin/migrations/0004_auto_20160608_1423.py create mode 100644 modoboa/admin/templates/admin/dnsbl_domain_detail.html create mode 100644 modoboa/admin/views/dnsbl.py diff --git a/modoboa/admin/constants.py b/modoboa/admin/constants.py new file mode 100644 index 000000000..ce41092be --- /dev/null +++ b/modoboa/admin/constants.py @@ -0,0 +1,83 @@ +"""Admin constants.""" + +DNSBL_PROVIDERS = [ + "aspews.ext.sorbs.net", + "b.barracudacentral.org", + "bad.psky.me", + "bl.deadbeef.com", + "bl.emailbasura.org", + "bl.spamcannibal.org", + "bl.spamcop.net", + "blackholes.five-ten-sg.com", + "blacklist.woody.ch", + "bogons.cymru.com", + "cbl.abuseat.org", + "cdl.anti-spam.org.cn", + "combined.abuse.ch", + "combined.rbl.msrbl.net", + "db.wpbl.info", + "dnsbl-1.uceprotect.net", + "dnsbl-2.uceprotect.net", + "dnsbl-3.uceprotect.net", + "dnsbl.cyberlogic.net", + "dnsbl.inps.de", + "dnsbl.njabl.org", + "dnsbl.sorbs.net", + "drone.abuse.ch", + "duinv.aupads.org", + "dul.dnsbl.sorbs.net", + "dul.ru", + "dyna.spamrats.com", + "dynip.rothen.com", + "http.dnsbl.sorbs.net" + "images.rbl.msrbl.net", + "ips.backscatterer.org", + "ix.dnsbl.manitu.net", + "korea.services.net", + "misc.dnsbl.sorbs.net", + "noptr.spamrats.com", + "ohps.dnsbl.net.au", + "omrs.dnsbl.net.au", + "orvedb.aupads.org", + "osps.dnsbl.net.au", + "osrs.dnsbl.net.au", + "owfs.dnsbl.net.au", + "owps.dnsbl.net.au" + "pbl.spamhaus.org", + "phishing.rbl.msrbl.net", + "probes.dnsbl.net.au" + "proxy.bl.gweep.ca", + "proxy.block.transip.nl", + "psbl.surriel.com", + "rbl.interserver.net", + "rdts.dnsbl.net.au", + "relays.bl.gweep.ca", + "relays.bl.kundenserver.de", + "relays.nether.net", + "residential.block.transip.nl", + "ricn.dnsbl.net.au", + "rmst.dnsbl.net.au", + "sbl.spamhaus.org", + "short.rbl.jp", + "smtp.dnsbl.sorbs.net", + "socks.dnsbl.sorbs.net", + "spam.abuse.ch", + "spam.dnsbl.sorbs.net", + "spam.rbl.msrbl.net", + "spam.spamrats.com", + "spamlist.or.kr", + "spamrbl.imp.ch", + "t3direct.dnsbl.net.au", + "tor.dnsbl.sectoor.de", + "torserver.tor.dnsbl.sectoor.de", + "ubl.lashback.com", + "ubl.unsubscore.com", + "virbl.bit.nl", + "virus.rbl.jp", + "virus.rbl.msrbl.net", + "web.dnsbl.sorbs.net", + "wormrbl.imp.ch", + "xbl.spamhaus.org", + "zen.spamhaus.org", + "zombie.dnsbl.sorbs.net", +] diff --git a/modoboa/admin/lib.py b/modoboa/admin/lib.py index 075663131..932a1504e 100644 --- a/modoboa/admin/lib.py +++ b/modoboa/admin/lib.py @@ -84,7 +84,8 @@ def get_domains(user, domfilter=None, searchquery=None, **extrafilters): :rtype: list :return: a list of domains and/or relay domains """ - domains = Domain.objects.get_for_admin(user) + domains = ( + Domain.objects.get_for_admin(user).prefetch_related("domainalias_set")) if domfilter: domains = domains.filter(type=domfilter) if searchquery is not None: diff --git a/modoboa/admin/management/commands/modo.py b/modoboa/admin/management/commands/modo.py index 36f4469ac..b5aba2f6d 100644 --- a/modoboa/admin/management/commands/modo.py +++ b/modoboa/admin/management/commands/modo.py @@ -2,6 +2,7 @@ from subcommand.base import SubcommandCommand +from .subcommands._dnsbl import CheckDNSBLCommand from .subcommands._export import ExportCommand from .subcommands._import import ImportCommand @@ -17,4 +18,5 @@ class Command(SubcommandCommand): subcommands = { "export": ExportCommand, "import": ImportCommand, + "check_dnsbl": CheckDNSBLCommand, } diff --git a/modoboa/admin/management/commands/subcommands/_dnsbl.py b/modoboa/admin/management/commands/subcommands/_dnsbl.py new file mode 100644 index 000000000..79a824586 --- /dev/null +++ b/modoboa/admin/management/commands/subcommands/_dnsbl.py @@ -0,0 +1,75 @@ +"""Management command to check defined domains against DNSBL providers.""" + +import ipaddress + +import dns.resolver +import gevent +from gevent import socket + +from django.core.management.base import BaseCommand + +from modoboa.admin import constants +from modoboa.admin import models + + +class CheckDNSBLCommand(BaseCommand): + """Command class.""" + + help = "Check defined domains against DNSBL providers." + + def add_arguments(self, parser): + """Add extra arguments to command.""" + parser.add_argument( + "--timeout", type=int, default=3, + help="Timeout used for queries.") + + def query(self, ip_list, provider): + """Check given IP against given provider.""" + results = {} + for ip in ip_list: + reverse = ".".join(reversed(ip.split("."))) + pattern = "{}.{}".format(reverse, provider) + try: + results[ip] = socket.gethostbyname(pattern) + except socket.gaierror: + results[ip] = False + return provider, results + + def check_domain(self, domain, timeout): + """Check specified domain.""" + resolver = dns.resolver.Resolver() + answers = resolver.query(domain.name, "MX") + ip_list = [] + for answer in answers: + address = None + try: + ipaddress.ip_address(str(answer.exchange)) + except ValueError: + try: + address = socket.gethostbyname(str(answer.exchange)) + except socket.gaierror: + pass + else: + address = str(answer.exchange) + finally: + if address is not None: + ip_list.append(address) + + jobs = [ + gevent.spawn(self.query, ip_list, provider) + for provider in constants.DNSBL_PROVIDERS] + gevent.joinall(jobs, timeout) + for job in jobs: + if not job.successful(): + continue + provider, results = job.value + for ip in results.keys(): + result = "" if not results[ip] else results[ip] + models.DNSBLResult.objects.update_or_create( + domain=domain, provider=provider, mx=ip, + status=result) + + def handle(self, *args, **options): + """Command entry point.""" + for domain in models.Domain.objects.filter(enabled=True): + self.check_domain(domain, options.get("timeout")) diff --git a/modoboa/admin/migrations/0004_auto_20160608_1423.py b/modoboa/admin/migrations/0004_auto_20160608_1423.py new file mode 100644 index 000000000..98072136c --- /dev/null +++ b/modoboa/admin/migrations/0004_auto_20160608_1423.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-06-08 12:23 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('admin', '0003_auto_20151118_1215'), + ] + + operations = [ + migrations.CreateModel( + name='DNSBLResult', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('provider', models.CharField(max_length=254)), + ('mx', models.GenericIPAddressField()), + ('status', models.CharField(blank=True, max_length=45)), + ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.Domain')), + ], + ), + migrations.AlterUniqueTogether( + name='dnsblresult', + unique_together=set([('domain', 'provider', 'mx')]), + ), + ] diff --git a/modoboa/admin/models/__init__.py b/modoboa/admin/models/__init__.py index c2c223752..0fdc800d3 100644 --- a/modoboa/admin/models/__init__.py +++ b/modoboa/admin/models/__init__.py @@ -1,10 +1,10 @@ from .base import AdminObject -from .domain import Domain +from .domain import Domain, DNSBLResult from .domain_alias import DomainAlias from .mailbox import Mailbox, Quota, MailboxOperation from .alias import Alias, AliasRecipient __all__ = [ - 'Domain', 'DomainAlias', 'Mailbox', 'Quota', 'Alias', + 'Domain', 'DNSBLResult', 'DomainAlias', 'Mailbox', 'Quota', 'Alias', 'MailboxOperation', 'AdminObject', 'AliasRecipient' ] diff --git a/modoboa/admin/models/domain.py b/modoboa/admin/models/domain.py index 8134fe401..2dc24b73d 100644 --- a/modoboa/admin/models/domain.py +++ b/modoboa/admin/models/domain.py @@ -101,6 +101,10 @@ class Domain(AdminObject): def aliases(self): return self.domainalias_set + def is_in_dnsbl(self): + """Shortcut to DNSBL results.""" + return self.dnsblresult_set.blacklisted().exists() + def add_admin(self, account): """Add a new administrator to this domain. @@ -207,3 +211,26 @@ class Domain(AdminObject): domalias.post_create(creator) reversion.register(Domain) + + +class DNSBLQuerySet(models.QuerySet): + """Custom manager for DNSBLResultManager.""" + + def blacklisted(self): + """Return blacklisted results.""" + return self.exclude(status="") + + +class DNSBLResult(models.Model): + """Store a DNSBL query result.""" + + domain = models.ForeignKey(Domain) + provider = models.CharField(max_length=254) + mx = models.GenericIPAddressField() + status = models.CharField(max_length=45, blank=True) + + objects = models.Manager.from_queryset(DNSBLQuerySet)() + + class Meta: + app_label = "admin" + unique_together = [("domain", "provider", "mx")] diff --git a/modoboa/admin/templates/admin/dnsbl_domain_detail.html b/modoboa/admin/templates/admin/dnsbl_domain_detail.html new file mode 100644 index 000000000..d68210708 --- /dev/null +++ b/modoboa/admin/templates/admin/dnsbl_domain_detail.html @@ -0,0 +1,30 @@ +{% extends "common/generic_modal.html" %} + +{% load i18n %} + +{% block modalbody %} + {% if domain.is_in_dnsbl %} + + + + + + + + + + {% for result in domain.dnsblresult_set.blacklisted %} + + + + + + {% endfor %} + +
{% trans "Provider" %}{% trans "Address" %}{% trans "Status" %}
{{ result.provider }}{{ result.mx }} + {% if result.status %}{% trans "listed" %}{% else %}{% endif %}
+ {% else %} +
+ {% trans "Everything is fine, this domain is not listed." %}
+ {% endif %} +{% endblock %} diff --git a/modoboa/admin/templates/admin/domain_detail.html b/modoboa/admin/templates/admin/domain_detail.html index adfc9eef4..cbac67289 100644 --- a/modoboa/admin/templates/admin/domain_detail.html +++ b/modoboa/admin/templates/admin/domain_detail.html @@ -17,9 +17,17 @@
-

{% trans "Information" %}

+
+

{% trans "Information" %}

+
-
{% trans "Creation date" %}{{ domain.creation }}
+
+ {% trans "Authentication" %} + DNSBL +
+
+ {% trans "Creation date" %}{{ domain.creation }} +
{% trans "Last modification date" %}{{ domain.last_modification }}
{% trans "Domain aliases" %}{{ domain.domainalias_count }}
{% trans "Mailboxes" %}{{ domain.mailbox_count }}
diff --git a/modoboa/admin/templates/admin/domains.html b/modoboa/admin/templates/admin/domains.html index ec4025f0c..1e0b5e390 100644 --- a/modoboa/admin/templates/admin/domains.html +++ b/modoboa/admin/templates/admin/domains.html @@ -25,5 +25,6 @@ $(document).ready(function() { {% trans "Name" %} {% trans "Alias(es)" %} {% trans "Tags" %} +{% trans "Authentication" %} {% trans "Actions" %} {% endblock %} diff --git a/modoboa/admin/templates/admin/domains_table.html b/modoboa/admin/templates/admin/domains_table.html index b579c4070..c209dee64 100644 --- a/modoboa/admin/templates/admin/domains_table.html +++ b/modoboa/admin/templates/admin/domains_table.html @@ -5,6 +5,11 @@ {{ domain }}{% if not domain.enabled %} ({% trans "disabled" %}){% endif %} {% domain_aliases domain %} {% render_tags domain.tags %} + + + DNSBL + + {% domain_actions user domain %} {% endfor %} diff --git a/modoboa/admin/urls.py b/modoboa/admin/urls.py index cffbc927b..fb5ae7376 100644 --- a/modoboa/admin/urls.py +++ b/modoboa/admin/urls.py @@ -12,6 +12,8 @@ urlpatterns = patterns( url(r'^domains/$', 'domains', name="domain_list"), url(r'^domains/(?P\d+)/$', views.DomainDetailView.as_view(), name="domain_detail"), + url(r'^domains/(?P\d+)/dnsbl/$', views.DNSBLDomainDetailView.as_view(), + name="dnsbl_domain_detail"), url(r'^domains/list/$', '_domains', name="_domain_list"), url(r'^domains/flatlist/$', 'domains_list', name="domain_flat_list"), url(r'^domains/stats/$', 'domain_statistics', name="domain_statistics"), diff --git a/modoboa/admin/views/__init__.py b/modoboa/admin/views/__init__.py index dacf92012..008fb6858 100644 --- a/modoboa/admin/views/__init__.py +++ b/modoboa/admin/views/__init__.py @@ -1,8 +1,10 @@ """Shortcuts.""" +from .dnsbl import DNSBLDomainDetailView from .domain import DomainDetailView __all__ = [ + "DNSBLDomainDetailView", "DomainDetailView", ] diff --git a/modoboa/admin/views/dnsbl.py b/modoboa/admin/views/dnsbl.py new file mode 100644 index 000000000..dad479483 --- /dev/null +++ b/modoboa/admin/views/dnsbl.py @@ -0,0 +1,32 @@ +"""DNSBL related views.""" + +from django.views import generic + +from django.utils.translation import ugettext as _ + +from django.contrib.auth import mixins as auth_mixins + +from .. import models + + +class DNSBLDomainDetailView( + auth_mixins.PermissionRequiredMixin, generic.DetailView): + """DetailView for Domain.""" + + model = models.Domain + permission_required = "admin.view_domain" + template_name = "admin/dnsbl_domain_detail.html" + + def get_queryset(self): + """Add some prefetching.""" + return ( + super(DNSBLDomainDetailView, self).get_queryset() + .prefetch_related("dnsblresult_set")) + + def get_context_data(self, **kwargs): + """Add extra variables.""" + context = super(DNSBLDomainDetailView, self).get_context_data(**kwargs) + context.update({ + "title": _("DNSBL summary for {}").format(self.object.name) + }) + return context diff --git a/modoboa/admin/views/domain.py b/modoboa/admin/views/domain.py index 622d529b5..4bc5039cf 100644 --- a/modoboa/admin/views/domain.py +++ b/modoboa/admin/views/domain.py @@ -164,6 +164,13 @@ class DomainDetailView( model = Domain permission_required = "admin.view_domain" + def get_queryset(self): + """Add some prefetching.""" + return ( + super(DomainDetailView, self).get_queryset() + .prefetch_related("domainalias_set", "mailbox_set", "alias_set") + ) + def get_context_data(self, **kwargs): """Include extra widgets.""" context = super(DomainDetailView, self).get_context_data(**kwargs) diff --git a/requirements.txt b/requirements.txt index f8abf92e7..5f8ef1932 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,5 @@ progressbar==2.3 djangorestframework==3.3.1 django-rest-swagger==0.3.5 django-subcommand2==0.1.1 - +gevent==1.1.1 +ipaddress From c7f48cc3e27b5a1a6e64ea8cf0bf1f2dcfff2462 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Jun 2016 17:08:57 +0200 Subject: [PATCH 2/8] Add requirement. see #879 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5f8ef1932..9f02704e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ django-rest-swagger==0.3.5 django-subcommand2==0.1.1 gevent==1.1.1 ipaddress +dnspython==1.12.0 From 3c4c7b2bbc69b2fa3b9c8460434feee429bab4d3 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Jun 2016 17:12:39 +0200 Subject: [PATCH 3/8] Allow provider list customization. see #879 --- modoboa/admin/management/commands/subcommands/_dnsbl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modoboa/admin/management/commands/subcommands/_dnsbl.py b/modoboa/admin/management/commands/subcommands/_dnsbl.py index 79a824586..bb94488b8 100644 --- a/modoboa/admin/management/commands/subcommands/_dnsbl.py +++ b/modoboa/admin/management/commands/subcommands/_dnsbl.py @@ -6,6 +6,7 @@ import dns.resolver import gevent from gevent import socket +from django.conf import settings from django.core.management.base import BaseCommand from modoboa.admin import constants @@ -54,10 +55,13 @@ class CheckDNSBLCommand(BaseCommand): finally: if address is not None: ip_list.append(address) - + if not hasattr(settings, "DNSBL_PROVIDERS"): + providers = constants.DNSBL_PROVIDERS + else: + providers = settings.DNSBL_PROVIDERS jobs = [ gevent.spawn(self.query, ip_list, provider) - for provider in constants.DNSBL_PROVIDERS] + for provider in providers] gevent.joinall(jobs, timeout) for job in jobs: if not job.successful(): From 2aed5d8ce6a7f682f5beac17add18be2bb5fc64d Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Jun 2016 17:27:58 +0200 Subject: [PATCH 4/8] Add unit test. see #879 --- modoboa/admin/management/commands/subcommands/_dnsbl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modoboa/admin/management/commands/subcommands/_dnsbl.py b/modoboa/admin/management/commands/subcommands/_dnsbl.py index bb94488b8..c2819502e 100644 --- a/modoboa/admin/management/commands/subcommands/_dnsbl.py +++ b/modoboa/admin/management/commands/subcommands/_dnsbl.py @@ -39,7 +39,10 @@ class CheckDNSBLCommand(BaseCommand): def check_domain(self, domain, timeout): """Check specified domain.""" resolver = dns.resolver.Resolver() - answers = resolver.query(domain.name, "MX") + try: + answers = resolver.query(domain.name, "MX") + except dns.resolver.NoAnswer: + return ip_list = [] for answer in answers: address = None From 951620b05b9a7faba8185183b0447da5aade895f Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Jun 2016 17:36:28 +0200 Subject: [PATCH 5/8] Add unit test. see #879 --- modoboa/admin/tests/test_dnsbl.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 modoboa/admin/tests/test_dnsbl.py diff --git a/modoboa/admin/tests/test_dnsbl.py b/modoboa/admin/tests/test_dnsbl.py new file mode 100644 index 000000000..c42142db1 --- /dev/null +++ b/modoboa/admin/tests/test_dnsbl.py @@ -0,0 +1,27 @@ +"""DNSBL related tests.""" + +from django.core import management +from django.test import override_settings + +from modoboa.lib.tests import ModoTestCase + +from .. import factories +from .. import models + + +class DNSBLTestCase(ModoTestCase): + """TestCase for DNSBL related features.""" + + @classmethod + def setUpTestData(cls): + """Create some data.""" + super(DNSBLTestCase, cls).setUpTestData() + factories.DomainFactory(name="modoboa.org") + + @override_settings(DNSBL_PROVIDERS=["zen.spamhaus.org"]) + def test_management_command(self): + """Check that command works fine.""" + management.call_command("modo", "check_dnsbl") + self.assertTrue( + models.DNSBLResult.objects.filter( + domain__name="modoboa.org").exists()) From 4db98c0739ddb06efb8b31fae1140b811b2fbee3 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Jun 2016 18:16:12 +0200 Subject: [PATCH 6/8] Increase coverage. see #879 --- modoboa/admin/tests/test_dnsbl.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modoboa/admin/tests/test_dnsbl.py b/modoboa/admin/tests/test_dnsbl.py index c42142db1..57a5e9772 100644 --- a/modoboa/admin/tests/test_dnsbl.py +++ b/modoboa/admin/tests/test_dnsbl.py @@ -1,6 +1,7 @@ """DNSBL related tests.""" from django.core import management +from django.core.urlresolvers import reverse from django.test import override_settings from modoboa.lib.tests import ModoTestCase @@ -16,12 +17,16 @@ class DNSBLTestCase(ModoTestCase): def setUpTestData(cls): """Create some data.""" super(DNSBLTestCase, cls).setUpTestData() - factories.DomainFactory(name="modoboa.org") + cls.domain = factories.DomainFactory(name="modoboa.org") + factories.DomainFactory(name="pouet.com") # should not exist @override_settings(DNSBL_PROVIDERS=["zen.spamhaus.org"]) def test_management_command(self): """Check that command works fine.""" + self.assertEqual(models.DNSBLResult.objects.count(), 0) management.call_command("modo", "check_dnsbl") self.assertTrue( - models.DNSBLResult.objects.filter( - domain__name="modoboa.org").exists()) + models.DNSBLResult.objects.filter(domain=self.domain).exists()) + response = self.client.get( + reverse("admin:dnsbl_domain_detail", args=[self.domain.pk])) + self.assertEqual(response.status_code, 200) From f2c8736d0271588c4385c1e71679e20f7a47a803 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Jun 2016 18:37:32 +0200 Subject: [PATCH 7/8] Increase coverage. see #879 --- modoboa/admin/app_settings.py | 10 ++++++++++ .../admin/management/commands/subcommands/_dnsbl.py | 6 ++++++ modoboa/admin/models/domain.py | 11 +++++++++-- .../admin/templates/admin/dnsbl_domain_detail.html | 7 ++++++- modoboa/admin/templates/admin/domains_table.html | 2 +- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/modoboa/admin/app_settings.py b/modoboa/admin/app_settings.py index 7a8678e59..efe660930 100644 --- a/modoboa/admin/app_settings.py +++ b/modoboa/admin/app_settings.py @@ -51,6 +51,16 @@ ADMIN_EVENTS = [ class AdminParametersForm(parameters.AdminParametersForm): app = "admin" + dom_sep = SeparatorField(label=ugettext_lazy("Domains")) + + enable_dnsbl_checks = YesNoField( + label=ugettext_lazy("Enable DNSBL checks"), + initial="yes", + help_text=ugettext_lazy( + "Check every domain against major DNSBL providers" + ) + ) + mbsep = SeparatorField(label=ugettext_lazy("Mailboxes")) handle_mailboxes = YesNoField( diff --git a/modoboa/admin/management/commands/subcommands/_dnsbl.py b/modoboa/admin/management/commands/subcommands/_dnsbl.py index c2819502e..ff8695334 100644 --- a/modoboa/admin/management/commands/subcommands/_dnsbl.py +++ b/modoboa/admin/management/commands/subcommands/_dnsbl.py @@ -11,6 +11,7 @@ from django.core.management.base import BaseCommand from modoboa.admin import constants from modoboa.admin import models +from modoboa.lib import parameters class CheckDNSBLCommand(BaseCommand): @@ -58,6 +59,9 @@ class CheckDNSBLCommand(BaseCommand): finally: if address is not None: ip_list.append(address) + print domain.name, ip_list + if len(ip_list) == 0: + return if not hasattr(settings, "DNSBL_PROVIDERS"): providers = constants.DNSBL_PROVIDERS else: @@ -78,5 +82,7 @@ class CheckDNSBLCommand(BaseCommand): def handle(self, *args, **options): """Command entry point.""" + if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no": + return for domain in models.Domain.objects.filter(enabled=True): self.check_domain(domain, options.get("timeout")) diff --git a/modoboa/admin/models/domain.py b/modoboa/admin/models/domain.py index 2dc24b73d..277db2d40 100644 --- a/modoboa/admin/models/domain.py +++ b/modoboa/admin/models/domain.py @@ -3,6 +3,7 @@ from django.db import models from django.db.models.manager import Manager from django.utils.encoding import python_2_unicode_compatible, smart_text +from django.utils.functional import cached_property from django.utils.translation import ugettext as _, ugettext_lazy from django.contrib.contenttypes.fields import GenericRelation @@ -101,9 +102,15 @@ class Domain(AdminObject): def aliases(self): return self.domainalias_set - def is_in_dnsbl(self): + @cached_property + def dnsbl_status_color(self): """Shortcut to DNSBL results.""" - return self.dnsblresult_set.blacklisted().exists() + if not self.dnsblresult_set.exists(): + return "warning" + elif self.dnsblresult_set.blacklisted().exists(): + return "danger" + else: + return "success" def add_admin(self, account): """Add a new administrator to this domain. diff --git a/modoboa/admin/templates/admin/dnsbl_domain_detail.html b/modoboa/admin/templates/admin/dnsbl_domain_detail.html index d68210708..e69e42f79 100644 --- a/modoboa/admin/templates/admin/dnsbl_domain_detail.html +++ b/modoboa/admin/templates/admin/dnsbl_domain_detail.html @@ -3,7 +3,7 @@ {% load i18n %} {% block modalbody %} - {% if domain.is_in_dnsbl %} + {% if domain.dnsbl_status_color == "danger" %} @@ -24,7 +24,12 @@
{% else %} + {% if domain.dnsbl_status_color == "success" %}
{% trans "Everything is fine, this domain is not listed." %}
+ {% else %} +
+ {% trans "No information available for this domain." %}
+ {% endif %} {% endif %} {% endblock %} diff --git a/modoboa/admin/templates/admin/domains_table.html b/modoboa/admin/templates/admin/domains_table.html index c209dee64..c2a5b1de8 100644 --- a/modoboa/admin/templates/admin/domains_table.html +++ b/modoboa/admin/templates/admin/domains_table.html @@ -7,7 +7,7 @@ {% render_tags domain.tags %} - DNSBL + DNSBL {% domain_actions user domain %} From 27dd4411117fa2a131eceb12be41028860435d96 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Thu, 9 Jun 2016 14:40:17 +0200 Subject: [PATCH 8/8] Send notifications. see #879 --- .../management/commands/subcommands/_dnsbl.py | 71 ++++++++++++++++--- .../migrations/0004_auto_20160608_1423.py | 4 +- modoboa/admin/models/domain.py | 4 +- .../admin/notifications/domain_in_dnsbl.html | 15 ++++ modoboa/core/app_settings.py | 11 +++ 5 files changed, 90 insertions(+), 15 deletions(-) create mode 100644 modoboa/admin/templates/admin/notifications/domain_in_dnsbl.html diff --git a/modoboa/admin/management/commands/subcommands/_dnsbl.py b/modoboa/admin/management/commands/subcommands/_dnsbl.py index ff8695334..ff0a5d24c 100644 --- a/modoboa/admin/management/commands/subcommands/_dnsbl.py +++ b/modoboa/admin/management/commands/subcommands/_dnsbl.py @@ -8,10 +8,14 @@ from gevent import socket from django.conf import settings from django.core.management.base import BaseCommand +from django.template.loader import render_to_string +from django.utils.functional import cached_property +from django.utils.translation import ugettext as _ from modoboa.admin import constants from modoboa.admin import models from modoboa.lib import parameters +from modoboa.lib import email_utils class CheckDNSBLCommand(BaseCommand): @@ -19,6 +23,18 @@ class CheckDNSBLCommand(BaseCommand): help = "Check defined domains against DNSBL providers." + @cached_property + def providers(self): + """Return a list of DNSBL providers.""" + if not hasattr(settings, "DNSBL_PROVIDERS"): + return constants.DNSBL_PROVIDERS + return settings.DNSBL_PROVIDERS + + @cached_property + def sender(self): + """Return sender address for notifications.""" + return parameters.get_admin("SENDER_ADDRESS", app="core") + def add_arguments(self, parser): """Add extra arguments to command.""" parser.add_argument( @@ -37,6 +53,45 @@ class CheckDNSBLCommand(BaseCommand): results[ip] = False return provider, results + def store_domain_result(self, domain, provider, results): + """Store provider results for domain.""" + alerts = {} + to_create = [] + for ip in results.keys(): + result = "" if not results[ip] else results[ip] + dnsbl_result = models.DNSBLResult.objects.filter( + domain=domain, provider=provider, mx=ip).first() + if dnsbl_result is None: + to_create.append( + models.DNSBLResult( + domain=domain, provider=provider, mx=ip, + status=result) + ) + else: + if not dnsbl_result.status and result: + if domain not in alerts: + alerts[domain] = [] + alerts[domain].append((provider, ip)) + dnsbl_result.status = result + dnsbl_result.save() + models.DNSBLResult.objects.bulk_create(to_create) + if not alerts: + return + content = render_to_string( + "admin/notifications/domain_in_dnsbl.html", { + "domain": domain, "alerts": alerts + }) + subject = _("[modoboa] DNSBL issue(s) for domain {}").format( + domain.name) + for admin in domain.admins: + if not admin.email: + continue + status, msg = email_utils.sendmail_simple( + self.sender, admin.email, + domainsubject=subject, content=content) + if not status: + print msg + def check_domain(self, domain, timeout): """Check specified domain.""" resolver = dns.resolver.Resolver() @@ -59,30 +114,24 @@ class CheckDNSBLCommand(BaseCommand): finally: if address is not None: ip_list.append(address) - print domain.name, ip_list if len(ip_list) == 0: return - if not hasattr(settings, "DNSBL_PROVIDERS"): - providers = constants.DNSBL_PROVIDERS - else: - providers = settings.DNSBL_PROVIDERS jobs = [ gevent.spawn(self.query, ip_list, provider) - for provider in providers] + for provider in self.providers] gevent.joinall(jobs, timeout) for job in jobs: if not job.successful(): continue provider, results = job.value - for ip in results.keys(): - result = "" if not results[ip] else results[ip] - models.DNSBLResult.objects.update_or_create( - domain=domain, provider=provider, mx=ip, - status=result) + self.store_domain_result(domain, provider, results) def handle(self, *args, **options): """Command entry point.""" if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no": return + # Remove deprecated records first + models.DNSBLResult.objects.exclude( + provider__in=self.providers).delete() for domain in models.Domain.objects.filter(enabled=True): self.check_domain(domain, options.get("timeout")) diff --git a/modoboa/admin/migrations/0004_auto_20160608_1423.py b/modoboa/admin/migrations/0004_auto_20160608_1423.py index 98072136c..177e0a78c 100644 --- a/modoboa/admin/migrations/0004_auto_20160608_1423.py +++ b/modoboa/admin/migrations/0004_auto_20160608_1423.py @@ -17,9 +17,9 @@ class Migration(migrations.Migration): name='DNSBLResult', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('provider', models.CharField(max_length=254)), + ('provider', models.CharField(max_length=254, db_index=True)), ('mx', models.GenericIPAddressField()), - ('status', models.CharField(blank=True, max_length=45)), + ('status', models.CharField(blank=True, max_length=45, db_index=True)), ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.Domain')), ], ), diff --git a/modoboa/admin/models/domain.py b/modoboa/admin/models/domain.py index 277db2d40..d51a95d8d 100644 --- a/modoboa/admin/models/domain.py +++ b/modoboa/admin/models/domain.py @@ -232,9 +232,9 @@ class DNSBLResult(models.Model): """Store a DNSBL query result.""" domain = models.ForeignKey(Domain) - provider = models.CharField(max_length=254) + provider = models.CharField(max_length=254, db_index=True) mx = models.GenericIPAddressField() - status = models.CharField(max_length=45, blank=True) + status = models.CharField(max_length=45, blank=True, db_index=True) objects = models.Manager.from_queryset(DNSBLQuerySet)() diff --git a/modoboa/admin/templates/admin/notifications/domain_in_dnsbl.html b/modoboa/admin/templates/admin/notifications/domain_in_dnsbl.html new file mode 100644 index 000000000..337a18e6e --- /dev/null +++ b/modoboa/admin/templates/admin/notifications/domain_in_dnsbl.html @@ -0,0 +1,15 @@ +{% load i18n %} + +{% blocktrans trimmed domain=domain.name %} + Modoboa detected that domain {{ domain }} is listed by the following DNSBL providers: +{% endblocktrans %} + +{% for provider, ip in alerts %} + {{ provider }}: {{ ip }} +{% endfor %} + +{% blocktrans trimmed %} + The domain's reputation will be affected and there is a chance that + emails coming from it are considered as spam. You should contact + those providers and ask them to unlist detected IP address(es). +{% endblocktrans %} diff --git a/modoboa/core/app_settings.py b/modoboa/core/app_settings.py index ed93cc03a..c8ac1c080 100644 --- a/modoboa/core/app_settings.py +++ b/modoboa/core/app_settings.py @@ -4,6 +4,7 @@ from django.conf import settings from django.utils.translation import ugettext as _, ugettext_lazy from modoboa.lib import parameters +from modoboa.lib import fields as lib_fields from modoboa.lib.cryptutils import random_key from modoboa.lib.form_utils import ( YesNoField, SeparatorField, InlineRadioSelect @@ -209,6 +210,16 @@ class GeneralParametersForm(parameters.AdminParametersForm): required=False ) + notif_sep = SeparatorField(label=ugettext_lazy("Notifications")) + + sender_address = lib_fields.UTF8EmailField( + label=_("Sender address"), + initial="noreply@modoboa.org", + help_text=_( + "Email address used to send notifications." + ) + ) + sep3 = SeparatorField(label=ugettext_lazy("Miscellaneous")) top_notifications_check_interval = forms.IntegerField(