merge mx/dnsbl commands

modoboa/admin/lib.py

@@ -2,15 +2,9 @@
 
 """Internal library for admin."""
 
-import socket
-import ipaddress
-from datetime import datetime
-from datetime import timedelta
 from functools import wraps
 from itertools import chain
 
-import dns.resolver
-
 from django.contrib.contenttypes.models import ContentType
 from django.db.models import Q
 from django.utils.translation import ugettext as _
@@ -20,7 +14,7 @@ from modoboa.core import signals as core_signals
 from modoboa.lib import events
 from modoboa.lib.exceptions import PermDeniedException
 
-from .models import Domain, DomainAlias, Alias, MXRecord
+from .models import Domain, DomainAlias, Alias
 
 
 def needs_mailbox():
@@ -106,42 +100,6 @@ def get_domains(user, domfilter=None, searchquery=None, **extrafilters):
     return domains
 
 
-def get_mx_records_for_domain(domain, ttl=7200):
-    try:
-        answers = dns.resolver.query(domain.name, "MX")
-    except dns.resolver.NoAnswer:
-        return
-    now = datetime.now()
-    delta = timedelta(seconds=ttl)
-    records = MXRecord.objects.filter(domain=domain,
-                                      updated__gte=now - delta)
-    if records.count():
-        print('skip update')
-        for record in records:
-            yield record
-        raise StopIteration()
-    MXRecord.objects.filter(domain=domain).delete()
-    for answer in answers:
-        address = None
-        try:
-            ipaddress.ip_address(str(answer.exchange))
-        except ValueError:
-            try:
-                address = socket.gethostbyname(str(answer.exchange))
-            except socket.gaierror:
-                pass
-        else:
-            address = str(answer.exchange)
-        finally:
-            if address is not None:
-                record = MXRecord.objects.create(
-                    domain=domain,
-                    name=str(answer.exchange).strip('.'),
-                    address=address,
-                    updated=now)
-                yield record
-
-
 def check_if_domain_exists(name, extra_checks=None):
     """Check if a domain already exists.
 

modoboa/admin/management/commands/modo.py

@@ -2,7 +2,6 @@
 
 from subcommand.base import SubcommandCommand
 
-from .subcommands._dnsbl import CheckDNSBLCommand
 from .subcommands._mx import CheckMXRecords
 from .subcommands._export import ExportCommand
 from .subcommands._import import ImportCommand
@@ -20,5 +19,4 @@ class Command(SubcommandCommand):
         "export": ExportCommand,
         "import": ImportCommand,
         "check_mx": CheckMXRecords,
-        "check_dnsbl": CheckDNSBLCommand,
     }

modoboa/admin/management/commands/subcommands/_dnsbl.py

@@ -1,114 +0,0 @@
-"""Management command to check defined domains against DNSBL providers."""
-
-import gevent
-from gevent import socket
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-from django.template.loader import render_to_string
-from django.utils.functional import cached_property
-from django.utils.translation import ugettext as _
-
-from modoboa.admin import constants
-from modoboa.admin import models
-from modoboa.admin.lib import get_mx_records_for_domain
-from modoboa.lib import parameters
-from modoboa.lib import email_utils
-
-
-class CheckDNSBLCommand(BaseCommand):
-    """Command class."""
-
-    help = "Check defined domains against DNSBL providers."
-
-    @cached_property
-    def providers(self):
-        """Return a list of DNSBL providers."""
-        if not hasattr(settings, "DNSBL_PROVIDERS"):
-            return constants.DNSBL_PROVIDERS
-        return settings.DNSBL_PROVIDERS
-
-    @cached_property
-    def sender(self):
-        """Return sender address for notifications."""
-        return parameters.get_admin("SENDER_ADDRESS", app="core")
-
-    def add_arguments(self, parser):
-        """Add extra arguments to command."""
-        parser.add_argument(
-            "--timeout", type=int, default=3,
-            help="Timeout used for queries.")
-
-    def query(self, mx_list, provider):
-        """Check given IP against given provider."""
-        results = {}
-        for mx in mx_list:
-            reverse = ".".join(reversed(mx.address.split(".")))
-            pattern = "{}.{}".format(reverse, provider)
-            try:
-                results[mx] = socket.gethostbyname(pattern)
-            except socket.gaierror:
-                results[mx] = False
-        return provider, results
-
-    def store_domain_result(self, domain, provider, results):
-        """Store provider results for domain."""
-        alerts = {}
-        to_create = []
-        for mx in results.keys():
-            result = "" if not results[mx] else results[mx]
-            dnsbl_result = models.DNSBLResult.objects.filter(
-                domain=domain, provider=provider, mx=mx).first()
-            if dnsbl_result is None:
-                to_create.append(
-                    models.DNSBLResult(
-                        domain=domain, provider=provider, mx=mx,
-                        status=result)
-                )
-            else:
-                if not dnsbl_result.status and result:
-                    if domain not in alerts:
-                        alerts[domain] = []
-                    alerts[domain].append((provider, mx))
-                dnsbl_result.status = result
-                dnsbl_result.save()
-        models.DNSBLResult.objects.bulk_create(to_create)
-        if not alerts:
-            return
-        content = render_to_string(
-            "admin/notifications/domain_in_dnsbl.html", {
-                "domain": domain, "alerts": alerts
-            })
-        subject = _("[modoboa] DNSBL issue(s) for domain {}").format(
-            domain.name)
-        for admin in domain.admins:
-            if not admin.email:
-                continue
-            status, msg = email_utils.sendmail_simple(
-                self.sender, admin.email,
-                domainsubject=subject, content=content)
-            if not status:
-                print(msg)
-
-    def check_domain(self, domain, timeout):
-        """Check specified domain."""
-        mx_list = get_mx_records_for_domain(domain)
-        jobs = [
-            gevent.spawn(self.query, mx_list, provider)
-            for provider in self.providers]
-        gevent.joinall(jobs, timeout)
-        for job in jobs:
-            if not job.successful():
-                continue
-            provider, results = job.value
-            self.store_domain_result(domain, provider, results)
-
-    def handle(self, *args, **options):
-        """Command entry point."""
-        if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no":
-            return
-        # Remove deprecated records first
-        models.DNSBLResult.objects.exclude(
-            provider__in=self.providers).delete()
-        for domain in models.Domain.objects.filter(enabled=True):
-            self.check_domain(domain, options.get("timeout"))

modoboa/admin/management/commands/subcommands/_mx.py

@@ -1,19 +1,165 @@
-# -*- coding: utf-8 -*-
-"""Management command to check defined domains against MX records."""
+"""Management command to check defined domains."""
 
+import ipaddress
+from datetime import datetime
+from datetime import timedelta
+
+import dns.resolver
+
+import gevent
+from gevent import socket
+
+from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.template.loader import render_to_string
+from django.utils.functional import cached_property
+from django.utils.translation import ugettext as _
 
+from modoboa.admin import constants
 from modoboa.admin import models
-from modoboa.admin.lib import get_mx_records_for_domain
+from modoboa.lib import parameters
+from modoboa.lib import email_utils
 
 
 class CheckMXRecords(BaseCommand):
     """Command class."""
 
-    help = "Check defined domains against MX records."
+    help = "Check defined domains."
+
+    @cached_property
+    def providers(self):
+        """Return a list of DNSBL providers."""
+        if not hasattr(settings, "DNSBL_PROVIDERS"):
+            return constants.DNSBL_PROVIDERS
+        return settings.DNSBL_PROVIDERS
+
+    @cached_property
+    def sender(self):
+        """Return sender address for notifications."""
+        return parameters.get_admin("SENDER_ADDRESS", app="core")
+
+    def add_arguments(self, parser):
+        """Add extra arguments to command."""
+        parser.add_argument(
+            "--no-dnsbl", action='store_true', default=False,
+            help="Skip DNSBL queries.")
+        parser.add_argument(
+            "--timeout", type=int, default=3,
+            help="Timeout used for queries.")
+        parser.add_argument(
+            "--ttl", type=int, default=7200,
+            help="TTL for dns query.")
+
+    def get_mx_records_for_domain(self, domain, ttl=7200):
+        try:
+            answers = dns.resolver.query(domain.name, "MX")
+        except dns.resolver.NoAnswer:
+            return
+        now = datetime.now()
+        delta = timedelta(seconds=ttl)
+        records = models.MXRecord.objects.filter(domain=domain,
+                                                 updated__gte=now - delta)
+        if records.count():
+            for record in records:
+                yield record
+            raise StopIteration()
+        models.MXRecord.objects.filter(domain=domain).delete()
+        for answer in answers:
+            address = None
+            try:
+                ipaddress.ip_address(str(answer.exchange))
+            except ValueError:
+                try:
+                    address = socket.gethostbyname(str(answer.exchange))
+                except socket.gaierror:
+                    pass
+            else:
+                address = str(answer.exchange)
+            finally:
+                if address is not None:
+                    record = models.MXRecord.objects.create(
+                        domain=domain,
+                        name=str(answer.exchange).strip('.'),
+                        address=address,
+                        updated=now)
+                    yield record
+
+    def query_dnsbl(self, mx_list, provider):
+        """Check given IP against given provider."""
+        results = {}
+        for mx in mx_list:
+            reverse = ".".join(reversed(mx.address.split(".")))
+            pattern = "{}.{}".format(reverse, provider)
+            try:
+                results[mx] = socket.gethostbyname(pattern)
+            except socket.gaierror:
+                results[mx] = False
+        return provider, results
+
+    def store_domain_result(self, domain, provider, results):
+        """Store provider results for domain."""
+        alerts = {}
+        to_create = []
+        for mx in results.keys():
+            result = "" if not results[mx] else results[mx]
+            dnsbl_result = models.DNSBLResult.objects.filter(
+                domain=domain, provider=provider, mx=mx).first()
+            if dnsbl_result is None:
+                to_create.append(
+                    models.DNSBLResult(
+                        domain=domain, provider=provider, mx=mx,
+                        status=result)
+                )
+            else:
+                if not dnsbl_result.status and result:
+                    if domain not in alerts:
+                        alerts[domain] = []
+                    alerts[domain].append((provider, mx))
+                dnsbl_result.status = result
+                dnsbl_result.save()
+        models.DNSBLResult.objects.bulk_create(to_create)
+        if not alerts:
+            return
+        content = render_to_string(
+            "admin/notifications/domain_in_dnsbl.html", {
+                "domain": domain, "alerts": alerts
+            })
+        subject = _("[modoboa] DNSBL issue(s) for domain {}").format(
+            domain.name)
+        for admin in domain.admins:
+            if not admin.email:
+                continue
+            status, msg = email_utils.sendmail_simple(
+                self.sender, admin.email,
+                domainsubject=subject, content=content)
+            if not status:
+                print(msg)
+
+    def check_domain(self, domain, timeout=3, ttl=7200, **options):
+        """Check specified domain."""
+
+        mx_list = self.get_mx_records_for_domain(domain, ttl=ttl)
+
+        if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no":
+            return
+        elif options['no_dnsbl'] is True:
+            return
+        else:
+            jobs = [
+                gevent.spawn(self.query_dnsbl, mx_list, provider)
+                for provider in self.providers]
+            gevent.joinall(jobs, timeout)
+            for job in jobs:
+                if not job.successful():
+                    continue
+                provider, results = job.value
+                self.store_domain_result(domain, provider, results)
 
     def handle(self, *args, **options):
         """Command entry point."""
-        for domain in models.Domain.objects.all():
-            for record in get_mx_records_for_domain(domain):
-                print(record)
+        # Remove deprecated records first
+        models.DNSBLResult.objects.exclude(
+            provider__in=self.providers).delete()
+
+        for domain in models.Domain.objects.filter(enabled=True):
+            self.check_domain(domain, **options)

modoboa/admin/tests/test_mx.py

@@ -24,9 +24,20 @@ class DNSBLTestCase(ModoTestCase):
     def test_management_command(self):
         """Check that command works fine."""
         self.assertEqual(models.DNSBLResult.objects.count(), 0)
-        management.call_command("modo", "check_dnsbl")
+        management.call_command("modo", "check_mx")
         self.assertTrue(
             models.DNSBLResult.objects.filter(domain=self.domain).exists())
         response = self.client.get(
             reverse("admin:dnsbl_domain_detail", args=[self.domain.pk]))
         self.assertEqual(response.status_code, 200)
+
+    @override_settings(DNSBL_PROVIDERS=["zen.spamhaus.org"])
+    def test_management_command_no_dnsbl(self):
+        """Check that command works fine without dnsbl."""
+        self.assertEqual(models.DNSBLResult.objects.count(), 0)
+        management.call_command("modo", "check_mx", "--no-dnsbl")
+        self.assertFalse(
+            models.DNSBLResult.objects.filter(domain=self.domain).exists())
+        response = self.client.get(
+            reverse("admin:dnsbl_domain_detail", args=[self.domain.pk]))
+        self.assertEqual(response.status_code, 200)