From dd9b63eead6277a13c2b2894c694a0ac97554526 Mon Sep 17 00:00:00 2001 From: Gael Pasgrimaud Date: Tue, 13 Sep 2016 17:21:26 +0200 Subject: [PATCH] merge mx/dnsbl commands --- modoboa/admin/lib.py | 44 +---- modoboa/admin/management/commands/modo.py | 2 - .../management/commands/subcommands/_dnsbl.py | 114 ------------- .../management/commands/subcommands/_mx.py | 160 +++++++++++++++++- .../admin/tests/{test_dnsbl.py => test_mx.py} | 13 +- 5 files changed, 166 insertions(+), 167 deletions(-) delete mode 100644 modoboa/admin/management/commands/subcommands/_dnsbl.py rename modoboa/admin/tests/{test_dnsbl.py => test_mx.py} (63%) diff --git a/modoboa/admin/lib.py b/modoboa/admin/lib.py index 93785064d..932a1504e 100644 --- a/modoboa/admin/lib.py +++ b/modoboa/admin/lib.py @@ -2,15 +2,9 @@ """Internal library for admin.""" -import socket -import ipaddress -from datetime import datetime -from datetime import timedelta from functools import wraps from itertools import chain -import dns.resolver - from django.contrib.contenttypes.models import ContentType from django.db.models import Q from django.utils.translation import ugettext as _ @@ -20,7 +14,7 @@ from modoboa.core import signals as core_signals from modoboa.lib import events from modoboa.lib.exceptions import PermDeniedException -from .models import Domain, DomainAlias, Alias, MXRecord +from .models import Domain, DomainAlias, Alias def needs_mailbox(): @@ -106,42 +100,6 @@ def get_domains(user, domfilter=None, searchquery=None, **extrafilters): return domains -def get_mx_records_for_domain(domain, ttl=7200): - try: - answers = dns.resolver.query(domain.name, "MX") - except dns.resolver.NoAnswer: - return - now = datetime.now() - delta = timedelta(seconds=ttl) - records = MXRecord.objects.filter(domain=domain, - updated__gte=now - delta) - if records.count(): - print('skip update') - for record in records: - yield record - raise StopIteration() - MXRecord.objects.filter(domain=domain).delete() - for answer in answers: - address = None - try: - ipaddress.ip_address(str(answer.exchange)) - except ValueError: - try: - address = socket.gethostbyname(str(answer.exchange)) - except socket.gaierror: - pass - else: - address = str(answer.exchange) - finally: - if address is not None: - record = MXRecord.objects.create( - domain=domain, - name=str(answer.exchange).strip('.'), - address=address, - updated=now) - yield record - - def check_if_domain_exists(name, extra_checks=None): """Check if a domain already exists. diff --git a/modoboa/admin/management/commands/modo.py b/modoboa/admin/management/commands/modo.py index 1edebd35e..e5b7d3b40 100644 --- a/modoboa/admin/management/commands/modo.py +++ b/modoboa/admin/management/commands/modo.py @@ -2,7 +2,6 @@ from subcommand.base import SubcommandCommand -from .subcommands._dnsbl import CheckDNSBLCommand from .subcommands._mx import CheckMXRecords from .subcommands._export import ExportCommand from .subcommands._import import ImportCommand @@ -20,5 +19,4 @@ class Command(SubcommandCommand): "export": ExportCommand, "import": ImportCommand, "check_mx": CheckMXRecords, - "check_dnsbl": CheckDNSBLCommand, } diff --git a/modoboa/admin/management/commands/subcommands/_dnsbl.py b/modoboa/admin/management/commands/subcommands/_dnsbl.py deleted file mode 100644 index 7b9f09612..000000000 --- a/modoboa/admin/management/commands/subcommands/_dnsbl.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Management command to check defined domains against DNSBL providers.""" - -import gevent -from gevent import socket - -from django.conf import settings -from django.core.management.base import BaseCommand -from django.template.loader import render_to_string -from django.utils.functional import cached_property -from django.utils.translation import ugettext as _ - -from modoboa.admin import constants -from modoboa.admin import models -from modoboa.admin.lib import get_mx_records_for_domain -from modoboa.lib import parameters -from modoboa.lib import email_utils - - -class CheckDNSBLCommand(BaseCommand): - """Command class.""" - - help = "Check defined domains against DNSBL providers." - - @cached_property - def providers(self): - """Return a list of DNSBL providers.""" - if not hasattr(settings, "DNSBL_PROVIDERS"): - return constants.DNSBL_PROVIDERS - return settings.DNSBL_PROVIDERS - - @cached_property - def sender(self): - """Return sender address for notifications.""" - return parameters.get_admin("SENDER_ADDRESS", app="core") - - def add_arguments(self, parser): - """Add extra arguments to command.""" - parser.add_argument( - "--timeout", type=int, default=3, - help="Timeout used for queries.") - - def query(self, mx_list, provider): - """Check given IP against given provider.""" - results = {} - for mx in mx_list: - reverse = ".".join(reversed(mx.address.split("."))) - pattern = "{}.{}".format(reverse, provider) - try: - results[mx] = socket.gethostbyname(pattern) - except socket.gaierror: - results[mx] = False - return provider, results - - def store_domain_result(self, domain, provider, results): - """Store provider results for domain.""" - alerts = {} - to_create = [] - for mx in results.keys(): - result = "" if not results[mx] else results[mx] - dnsbl_result = models.DNSBLResult.objects.filter( - domain=domain, provider=provider, mx=mx).first() - if dnsbl_result is None: - to_create.append( - models.DNSBLResult( - domain=domain, provider=provider, mx=mx, - status=result) - ) - else: - if not dnsbl_result.status and result: - if domain not in alerts: - alerts[domain] = [] - alerts[domain].append((provider, mx)) - dnsbl_result.status = result - dnsbl_result.save() - models.DNSBLResult.objects.bulk_create(to_create) - if not alerts: - return - content = render_to_string( - "admin/notifications/domain_in_dnsbl.html", { - "domain": domain, "alerts": alerts - }) - subject = _("[modoboa] DNSBL issue(s) for domain {}").format( - domain.name) - for admin in domain.admins: - if not admin.email: - continue - status, msg = email_utils.sendmail_simple( - self.sender, admin.email, - domainsubject=subject, content=content) - if not status: - print(msg) - - def check_domain(self, domain, timeout): - """Check specified domain.""" - mx_list = get_mx_records_for_domain(domain) - jobs = [ - gevent.spawn(self.query, mx_list, provider) - for provider in self.providers] - gevent.joinall(jobs, timeout) - for job in jobs: - if not job.successful(): - continue - provider, results = job.value - self.store_domain_result(domain, provider, results) - - def handle(self, *args, **options): - """Command entry point.""" - if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no": - return - # Remove deprecated records first - models.DNSBLResult.objects.exclude( - provider__in=self.providers).delete() - for domain in models.Domain.objects.filter(enabled=True): - self.check_domain(domain, options.get("timeout")) diff --git a/modoboa/admin/management/commands/subcommands/_mx.py b/modoboa/admin/management/commands/subcommands/_mx.py index 32ee6395f..dc0d491d6 100644 --- a/modoboa/admin/management/commands/subcommands/_mx.py +++ b/modoboa/admin/management/commands/subcommands/_mx.py @@ -1,19 +1,165 @@ -# -*- coding: utf-8 -*- -"""Management command to check defined domains against MX records.""" +"""Management command to check defined domains.""" +import ipaddress +from datetime import datetime +from datetime import timedelta + +import dns.resolver + +import gevent +from gevent import socket + +from django.conf import settings from django.core.management.base import BaseCommand +from django.template.loader import render_to_string +from django.utils.functional import cached_property +from django.utils.translation import ugettext as _ +from modoboa.admin import constants from modoboa.admin import models -from modoboa.admin.lib import get_mx_records_for_domain +from modoboa.lib import parameters +from modoboa.lib import email_utils class CheckMXRecords(BaseCommand): """Command class.""" - help = "Check defined domains against MX records." + help = "Check defined domains." + + @cached_property + def providers(self): + """Return a list of DNSBL providers.""" + if not hasattr(settings, "DNSBL_PROVIDERS"): + return constants.DNSBL_PROVIDERS + return settings.DNSBL_PROVIDERS + + @cached_property + def sender(self): + """Return sender address for notifications.""" + return parameters.get_admin("SENDER_ADDRESS", app="core") + + def add_arguments(self, parser): + """Add extra arguments to command.""" + parser.add_argument( + "--no-dnsbl", action='store_true', default=False, + help="Skip DNSBL queries.") + parser.add_argument( + "--timeout", type=int, default=3, + help="Timeout used for queries.") + parser.add_argument( + "--ttl", type=int, default=7200, + help="TTL for dns query.") + + def get_mx_records_for_domain(self, domain, ttl=7200): + try: + answers = dns.resolver.query(domain.name, "MX") + except dns.resolver.NoAnswer: + return + now = datetime.now() + delta = timedelta(seconds=ttl) + records = models.MXRecord.objects.filter(domain=domain, + updated__gte=now - delta) + if records.count(): + for record in records: + yield record + raise StopIteration() + models.MXRecord.objects.filter(domain=domain).delete() + for answer in answers: + address = None + try: + ipaddress.ip_address(str(answer.exchange)) + except ValueError: + try: + address = socket.gethostbyname(str(answer.exchange)) + except socket.gaierror: + pass + else: + address = str(answer.exchange) + finally: + if address is not None: + record = models.MXRecord.objects.create( + domain=domain, + name=str(answer.exchange).strip('.'), + address=address, + updated=now) + yield record + + def query_dnsbl(self, mx_list, provider): + """Check given IP against given provider.""" + results = {} + for mx in mx_list: + reverse = ".".join(reversed(mx.address.split("."))) + pattern = "{}.{}".format(reverse, provider) + try: + results[mx] = socket.gethostbyname(pattern) + except socket.gaierror: + results[mx] = False + return provider, results + + def store_domain_result(self, domain, provider, results): + """Store provider results for domain.""" + alerts = {} + to_create = [] + for mx in results.keys(): + result = "" if not results[mx] else results[mx] + dnsbl_result = models.DNSBLResult.objects.filter( + domain=domain, provider=provider, mx=mx).first() + if dnsbl_result is None: + to_create.append( + models.DNSBLResult( + domain=domain, provider=provider, mx=mx, + status=result) + ) + else: + if not dnsbl_result.status and result: + if domain not in alerts: + alerts[domain] = [] + alerts[domain].append((provider, mx)) + dnsbl_result.status = result + dnsbl_result.save() + models.DNSBLResult.objects.bulk_create(to_create) + if not alerts: + return + content = render_to_string( + "admin/notifications/domain_in_dnsbl.html", { + "domain": domain, "alerts": alerts + }) + subject = _("[modoboa] DNSBL issue(s) for domain {}").format( + domain.name) + for admin in domain.admins: + if not admin.email: + continue + status, msg = email_utils.sendmail_simple( + self.sender, admin.email, + domainsubject=subject, content=content) + if not status: + print(msg) + + def check_domain(self, domain, timeout=3, ttl=7200, **options): + """Check specified domain.""" + + mx_list = self.get_mx_records_for_domain(domain, ttl=ttl) + + if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no": + return + elif options['no_dnsbl'] is True: + return + else: + jobs = [ + gevent.spawn(self.query_dnsbl, mx_list, provider) + for provider in self.providers] + gevent.joinall(jobs, timeout) + for job in jobs: + if not job.successful(): + continue + provider, results = job.value + self.store_domain_result(domain, provider, results) def handle(self, *args, **options): """Command entry point.""" - for domain in models.Domain.objects.all(): - for record in get_mx_records_for_domain(domain): - print(record) + # Remove deprecated records first + models.DNSBLResult.objects.exclude( + provider__in=self.providers).delete() + + for domain in models.Domain.objects.filter(enabled=True): + self.check_domain(domain, **options) diff --git a/modoboa/admin/tests/test_dnsbl.py b/modoboa/admin/tests/test_mx.py similarity index 63% rename from modoboa/admin/tests/test_dnsbl.py rename to modoboa/admin/tests/test_mx.py index 57a5e9772..413d3ced0 100644 --- a/modoboa/admin/tests/test_dnsbl.py +++ b/modoboa/admin/tests/test_mx.py @@ -24,9 +24,20 @@ class DNSBLTestCase(ModoTestCase): def test_management_command(self): """Check that command works fine.""" self.assertEqual(models.DNSBLResult.objects.count(), 0) - management.call_command("modo", "check_dnsbl") + management.call_command("modo", "check_mx") self.assertTrue( models.DNSBLResult.objects.filter(domain=self.domain).exists()) response = self.client.get( reverse("admin:dnsbl_domain_detail", args=[self.domain.pk])) self.assertEqual(response.status_code, 200) + + @override_settings(DNSBL_PROVIDERS=["zen.spamhaus.org"]) + def test_management_command_no_dnsbl(self): + """Check that command works fine without dnsbl.""" + self.assertEqual(models.DNSBLResult.objects.count(), 0) + management.call_command("modo", "check_mx", "--no-dnsbl") + self.assertFalse( + models.DNSBLResult.objects.filter(domain=self.domain).exists()) + response = self.client.get( + reverse("admin:dnsbl_domain_detail", args=[self.domain.pk])) + self.assertEqual(response.status_code, 200)