Merge pull request #913 from bearstech/mx_record

introduce mx record
This commit is contained in:
Antoine Nguyen
2016-09-14 14:39:57 +02:00
committed by GitHub
13 changed files with 413 additions and 256 deletions
+8
View File
@@ -53,6 +53,14 @@ class AdminParametersForm(parameters.AdminParametersForm):
dom_sep = SeparatorField(label=ugettext_lazy("Domains"))
enable_mx_checks = YesNoField(
label=ugettext_lazy("Enable MX checks"),
initial="yes",
help_text=ugettext_lazy(
"Check that every domain has a valid MX record"
)
)
enable_dnsbl_checks = YesNoField(
label=ugettext_lazy("Enable DNSBL checks"),
initial="yes",
+2 -2
View File
@@ -2,7 +2,7 @@
from subcommand.base import SubcommandCommand
from .subcommands._dnsbl import CheckDNSBLCommand
from .subcommands._mx import CheckMXRecords
from .subcommands._export import ExportCommand
from .subcommands._import import ImportCommand
@@ -18,5 +18,5 @@ class Command(SubcommandCommand):
subcommands = {
"export": ExportCommand,
"import": ImportCommand,
"check_dnsbl": CheckDNSBLCommand,
"check_mx": CheckMXRecords,
}
@@ -1,137 +0,0 @@
"""Management command to check defined domains against DNSBL providers."""
import ipaddress
import dns.resolver
import gevent
from gevent import socket
from django.conf import settings
from django.core.management.base import BaseCommand
from django.template.loader import render_to_string
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from modoboa.admin import constants
from modoboa.admin import models
from modoboa.lib import parameters
from modoboa.lib import email_utils
class CheckDNSBLCommand(BaseCommand):
"""Command class."""
help = "Check defined domains against DNSBL providers."
@cached_property
def providers(self):
"""Return a list of DNSBL providers."""
if not hasattr(settings, "DNSBL_PROVIDERS"):
return constants.DNSBL_PROVIDERS
return settings.DNSBL_PROVIDERS
@cached_property
def sender(self):
"""Return sender address for notifications."""
return parameters.get_admin("SENDER_ADDRESS", app="core")
def add_arguments(self, parser):
"""Add extra arguments to command."""
parser.add_argument(
"--timeout", type=int, default=3,
help="Timeout used for queries.")
def query(self, ip_list, provider):
"""Check given IP against given provider."""
results = {}
for ip in ip_list:
reverse = ".".join(reversed(ip.split(".")))
pattern = "{}.{}".format(reverse, provider)
try:
results[ip] = socket.gethostbyname(pattern)
except socket.gaierror:
results[ip] = False
return provider, results
def store_domain_result(self, domain, provider, results):
"""Store provider results for domain."""
alerts = {}
to_create = []
for ip in results.keys():
result = "" if not results[ip] else results[ip]
dnsbl_result = models.DNSBLResult.objects.filter(
domain=domain, provider=provider, mx=ip).first()
if dnsbl_result is None:
to_create.append(
models.DNSBLResult(
domain=domain, provider=provider, mx=ip,
status=result)
)
else:
if not dnsbl_result.status and result:
if domain not in alerts:
alerts[domain] = []
alerts[domain].append((provider, ip))
dnsbl_result.status = result
dnsbl_result.save()
models.DNSBLResult.objects.bulk_create(to_create)
if not alerts:
return
content = render_to_string(
"admin/notifications/domain_in_dnsbl.html", {
"domain": domain, "alerts": alerts
})
subject = _("[modoboa] DNSBL issue(s) for domain {}").format(
domain.name)
for admin in domain.admins:
if not admin.email:
continue
status, msg = email_utils.sendmail_simple(
self.sender, admin.email,
domainsubject=subject, content=content)
if not status:
print msg
def check_domain(self, domain, timeout):
"""Check specified domain."""
resolver = dns.resolver.Resolver()
try:
answers = resolver.query(domain.name, "MX")
except dns.resolver.NoAnswer:
return
ip_list = []
for answer in answers:
address = None
try:
ipaddress.ip_address(str(answer.exchange))
except ValueError:
try:
address = socket.gethostbyname(str(answer.exchange))
except socket.gaierror:
pass
else:
address = str(answer.exchange)
finally:
if address is not None:
ip_list.append(address)
if len(ip_list) == 0:
return
jobs = [
gevent.spawn(self.query, ip_list, provider)
for provider in self.providers]
gevent.joinall(jobs, timeout)
for job in jobs:
if not job.successful():
continue
provider, results = job.value
self.store_domain_result(domain, provider, results)
def handle(self, *args, **options):
"""Command entry point."""
if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no":
return
# Remove deprecated records first
models.DNSBLResult.objects.exclude(
provider__in=self.providers).delete()
for domain in models.Domain.objects.filter(enabled=True):
self.check_domain(domain, options.get("timeout"))
@@ -0,0 +1,230 @@
"""Management command to check defined domains."""
from datetime import datetime
from datetime import timedelta
import ipaddress
import dns.resolver
import gevent
from gevent import socket
from django.conf import settings
from django.core.management.base import BaseCommand
from django.template.loader import render_to_string
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from modoboa.admin import constants
from modoboa.admin import models
from modoboa.lib import parameters
from modoboa.lib import email_utils
class CheckMXRecords(BaseCommand):
"""Command class."""
help = "Check defined domains."
@cached_property
def providers(self):
"""Return a list of DNSBL providers."""
if not hasattr(settings, "DNSBL_PROVIDERS"):
return constants.DNSBL_PROVIDERS
return settings.DNSBL_PROVIDERS
@cached_property
def sender(self):
"""Return sender address for notifications."""
return parameters.get_admin("SENDER_ADDRESS", app="core")
def add_arguments(self, parser):
"""Add extra arguments to command."""
parser.add_argument(
"--valid-mx", type=str, action='append', default=[],
help="Valid MX ip(s) or subnet(s) used to check that domain's MX "
"are your own.")
parser.add_argument(
"--no-dnsbl", action='store_true', default=False,
help="Skip DNSBL queries.")
parser.add_argument(
"--email", type=str, action='append', default=[],
help="One or more email to notify")
parser.add_argument(
"--skip-admin-emails", action='store_true',
default=False,
help="Skip domain's admins email notification.")
parser.add_argument(
"--timeout", type=int, default=3,
help="Timeout used for queries.")
parser.add_argument(
"--ttl", type=int, default=7200,
help="TTL for dns query.")
def get_mx_records_for_domain(self, domain, ttl=7200):
"""Return one or more `models.MXRecord` for `domain`. DNS queries are
not performed while `ttl` (in seconds) is still valid"""
now = datetime.now()
records = models.MXRecord.objects.filter(domain=domain,
updated__gt=now)
if records.count():
for record in records:
yield record
raise StopIteration()
else:
models.MXRecord.objects.filter(domain=domain).delete()
try:
answers = dns.resolver.query(domain.name, "MX")
except dns.resolver.NoAnswer:
raise StopIteration()
delta = timedelta(seconds=ttl)
for answer in answers:
address = None
try:
ipaddress.ip_address(str(answer.exchange))
except ValueError:
try:
address = socket.gethostbyname(str(answer.exchange))
except socket.gaierror:
pass
else:
address = str(answer.exchange)
finally:
if address is not None:
record = models.MXRecord.objects.create(
domain=domain,
name=str(answer.exchange).strip('.'),
address=address,
updated=now + delta)
yield record
def query_dnsbl(self, mx_list, provider):
"""Check given IP against given DNSBL provider."""
results = {}
for mx in mx_list:
reverse = ".".join(reversed(mx.address.split(".")))
pattern = "{}.{}".format(reverse, provider)
try:
results[mx] = socket.gethostbyname(pattern)
except socket.gaierror:
results[mx] = False
return provider, results
def store_dnsbl_result(self, domain, provider, results, **options):
"""Store DNSBL provider results for domain."""
alerts = {}
to_create = []
for mx in results.keys():
result = "" if not results[mx] else results[mx]
dnsbl_result = models.DNSBLResult.objects.filter(
domain=domain, provider=provider, mx=mx).first()
if dnsbl_result is None:
to_create.append(
models.DNSBLResult(
domain=domain, provider=provider, mx=mx,
status=result)
)
else:
if not dnsbl_result.status and result:
if domain not in alerts:
alerts[domain] = []
alerts[domain].append((provider, mx))
dnsbl_result.status = result
dnsbl_result.save()
models.DNSBLResult.objects.bulk_create(to_create)
if not alerts:
return
emails = options['email']
if not options['skip_admin_emails']:
emails.extend([a.email for a in domain.admins if a.email])
if emails:
content = render_to_string(
"admin/notifications/domain_in_dnsbl.html", {
"domain": domain, "alerts": alerts
})
subject = _("[modoboa] DNSBL issue(s) for domain {}").format(
domain.name)
for email in emails:
status, msg = email_utils.sendmail_simple(
self.sender, email,
subject=subject, content=content)
if not status:
print(msg)
def check_valid_mx(self, domain, mx_list, valid_mx=None, **options):
"""Check that domain's MX record exist and is contains in `valid_mx` if
the option is provided"""
alerts = []
check = False
mxs = [ipaddress.ip_address(u"%s" % mx.address) for mx in mx_list]
if valid_mx:
for subnet in valid_mx:
check = True in [mx in subnet for mx in mxs]
if check is True:
break
if not mxs:
alerts.append(_("Domain {} as no MX record").format(domain))
elif valid_mx and check is False:
mx_names = ["{0.name} ({0.address})".format(mx) for mx in mx_list]
args = (domain, ", ".join(mx_names))
alerts.append(
_("Domain {0} got an invalid MX record. Got {1}").format(*args)
)
if not alerts:
return
emails = options['email']
if not options['skip_admin_emails']:
emails.extend([a.email for a in domain.admins if a.email])
if emails:
content = render_to_string(
"admin/notifications/domain_invalid_mx.html", {
"domain": domain, "valid_mx": valid_mx, "alerts": alerts
})
subject = _("[modoboa] MX issue(s) for domain {}").format(
domain.name)
for email in emails:
status, msg = email_utils.sendmail_simple(
self.sender, email,
subject=subject, content=content)
if not status:
print(msg)
def check_domain(self, domain, timeout=3, ttl=7200, **options):
"""Check specified domain."""
mx_list = list(self.get_mx_records_for_domain(domain, ttl=ttl))
if parameters.get_admin("ENABLE_MX_CHECKS") != "no":
self.check_valid_mx(domain, mx_list, **options)
if parameters.get_admin("ENABLE_DNSBL_CHECKS") == "no":
return
elif options['no_dnsbl'] is True:
return
elif mx_list:
jobs = [
gevent.spawn(self.query_dnsbl, mx_list, provider)
for provider in self.providers]
gevent.joinall(jobs, timeout)
for job in jobs:
if not job.successful():
continue
provider, results = job.value
self.store_dnsbl_result(domain, provider, results, **options)
def handle(self, *args, **options):
"""Command entry point."""
# Check that user provide valid network addresses
valid_mx = options['valid_mx']
options['valid_mx'] = [ipaddress.ip_network(u"%s" % v)
for v in valid_mx]
# Remove deprecated records first
models.DNSBLResult.objects.exclude(
provider__in=self.providers).delete()
for domain in models.Domain.objects.filter(enabled=True):
self.check_domain(domain, **options)
@@ -1,30 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.5 on 2016-06-08 12:23
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('admin', '0003_auto_20151118_1215'),
]
operations = [
migrations.CreateModel(
name='DNSBLResult',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('provider', models.CharField(max_length=254, db_index=True)),
('mx', models.GenericIPAddressField()),
('status', models.CharField(blank=True, max_length=45, db_index=True)),
('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.Domain')),
],
),
migrations.AlterUniqueTogether(
name='dnsblresult',
unique_together=set([('domain', 'provider', 'mx')]),
),
]
@@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.9 on 2016-09-14 09:07
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('admin', '0003_auto_20151118_1215'),
]
operations = [
migrations.CreateModel(
name='DNSBLResult',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('provider', models.CharField(db_index=True, max_length=254)),
('status', models.CharField(blank=True, db_index=True, max_length=45)),
('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.Domain')),
],
),
migrations.CreateModel(
name='MXRecord',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=254)),
('address', models.GenericIPAddressField()),
('managed', models.BooleanField(default=False)),
('updated', models.DateTimeField()),
('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.Domain')),
],
),
migrations.CreateModel(
name='SenderAddress',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('address', models.EmailField(max_length=254)),
('mailbox', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.Mailbox')),
],
),
migrations.AddField(
model_name='alias',
name='description',
field=models.TextField(blank=True),
),
migrations.AddField(
model_name='alias',
name='expire_at',
field=models.DateTimeField(blank=True, null=True),
),
migrations.AddField(
model_name='dnsblresult',
name='mx',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.MXRecord'),
),
migrations.AlterUniqueTogether(
name='senderaddress',
unique_together=set([('address', 'mailbox')]),
),
migrations.AlterUniqueTogether(
name='dnsblresult',
unique_together=set([('domain', 'provider', 'mx')]),
),
]
@@ -1,25 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.5 on 2016-07-19 07:03
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('admin', '0004_auto_20160608_1423'),
]
operations = [
migrations.AddField(
model_name='alias',
name='description',
field=models.TextField(blank=True),
),
migrations.AddField(
model_name='alias',
name='expire_at',
field=models.DateTimeField(blank=True, null=True),
),
]
@@ -1,28 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.5 on 2016-09-10 08:30
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('admin', '0005_auto_20160719_0903'),
]
operations = [
migrations.CreateModel(
name='SenderAddress',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('address', models.EmailField(max_length=254)),
('mailbox', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='admin.Mailbox')),
],
),
migrations.AlterUniqueTogether(
name='senderaddress',
unique_together=set([('address', 'mailbox')]),
),
]
+2 -1
View File
@@ -1,7 +1,7 @@
"""Admin models."""
from .base import AdminObject
from .domain import Domain, DNSBLResult
from .domain import Domain, MXRecord, DNSBLResult
from .domain_alias import DomainAlias
from .mailbox import Mailbox, Quota, MailboxOperation, SenderAddress
from .alias import Alias, AliasRecipient
@@ -15,6 +15,7 @@ __all__ = [
"DomainAlias",
"Mailbox",
"MailboxOperation",
"MXRecord",
"Quota",
"SenderAddress",
]
+17 -1
View File
@@ -102,6 +102,10 @@ class Domain(AdminObject):
def aliases(self):
return self.domainalias_set
@property
def mx_records(self):
self.mxrecord_set
@cached_property
def dnsbl_status_color(self):
"""Shortcut to DNSBL results."""
@@ -220,6 +224,18 @@ class Domain(AdminObject):
reversion.register(Domain)
class MXRecord(models.Model):
domain = models.ForeignKey(Domain)
name = models.CharField(max_length=254)
address = models.GenericIPAddressField()
managed = models.BooleanField(default=False)
updated = models.DateTimeField()
def __unicode__(self):
return u'{0.name} ({0.address}) for {0.domain} '.format(self)
class DNSBLQuerySet(models.QuerySet):
"""Custom manager for DNSBLResultManager."""
@@ -233,7 +249,7 @@ class DNSBLResult(models.Model):
domain = models.ForeignKey(Domain)
provider = models.CharField(max_length=254, db_index=True)
mx = models.GenericIPAddressField()
mx = models.ForeignKey(MXRecord)
status = models.CharField(max_length=45, blank=True, db_index=True)
objects = models.Manager.from_queryset(DNSBLQuerySet)()
@@ -0,0 +1,9 @@
{% load i18n %}
{% blocktrans trimmed with domain=domain.name %}
Modoboa detected that domain {{ domain }} has no valid MX record:
{% endblocktrans %}
{% for alert in alerts %}
- {{ alert }}
{% endfor %}
-32
View File
@@ -1,32 +0,0 @@
"""DNSBL related tests."""
from django.core import management
from django.core.urlresolvers import reverse
from django.test import override_settings
from modoboa.lib.tests import ModoTestCase
from .. import factories
from .. import models
class DNSBLTestCase(ModoTestCase):
"""TestCase for DNSBL related features."""
@classmethod
def setUpTestData(cls):
"""Create some data."""
super(DNSBLTestCase, cls).setUpTestData()
cls.domain = factories.DomainFactory(name="modoboa.org")
factories.DomainFactory(name="pouet.com") # should not exist
@override_settings(DNSBL_PROVIDERS=["zen.spamhaus.org"])
def test_management_command(self):
"""Check that command works fine."""
self.assertEqual(models.DNSBLResult.objects.count(), 0)
management.call_command("modo", "check_dnsbl")
self.assertTrue(
models.DNSBLResult.objects.filter(domain=self.domain).exists())
response = self.client.get(
reverse("admin:dnsbl_domain_detail", args=[self.domain.pk]))
self.assertEqual(response.status_code, 200)
+78
View File
@@ -0,0 +1,78 @@
"""DNSBL related tests."""
from django.core import management
from django.core.urlresolvers import reverse
from django.test import override_settings
from modoboa.lib.tests import ModoTestCase
from .. import factories
from .. import models
class MXTestCase(ModoTestCase):
"""TestCase for DNSBL related features."""
@classmethod
def setUpTestData(cls):
"""Create some data."""
super(MXTestCase, cls).setUpTestData()
cls.domain = factories.DomainFactory(name="modoboa.org")
factories.DomainFactory(name="pouet.com") # should not exist
def test_management_command(self):
"""Check that command works fine."""
self.assertEqual(models.MXRecord.objects.count(), 0)
management.call_command("modo", "check_mx", "--no-dnsbl",
"--valid-mx=127.0.0.1", "--ttl=0")
self.assertTrue(
models.MXRecord.objects.filter(domain=self.domain).exists())
# we passed a ttl to 0. this will reset the cache this time
qs = models.MXRecord.objects.filter(domain=self.domain)
id = qs[0].id
management.call_command("modo", "check_mx", "--no-dnsbl",
"--valid-mx=127.0.0.1", "--ttl=7200")
qs = models.MXRecord.objects.filter(domain=self.domain)
self.assertNotEqual(id, qs[0].id)
id = qs[0].id
# assume that mxrecords ids are the same. means that we taking care of
# ttl
management.call_command("modo", "check_mx", "--no-dnsbl",
"--valid-mx=127.0.0.1")
qs = models.MXRecord.objects.filter(domain=self.domain)
self.assertEqual(id, qs[0].id)
class DNSBLTestCase(ModoTestCase):
"""TestCase for DNSBL related features."""
@classmethod
def setUpTestData(cls):
"""Create some data."""
super(DNSBLTestCase, cls).setUpTestData()
cls.domain = factories.DomainFactory(name="modoboa.org")
factories.DomainFactory(name="pouet.com") # should not exist
@override_settings(DNSBL_PROVIDERS=["zen.spamhaus.org"])
def test_management_command(self):
"""Check that command works fine."""
self.assertEqual(models.DNSBLResult.objects.count(), 0)
management.call_command("modo", "check_mx")
self.assertTrue(
models.DNSBLResult.objects.filter(domain=self.domain).exists())
response = self.client.get(
reverse("admin:dnsbl_domain_detail", args=[self.domain.pk]))
self.assertEqual(response.status_code, 200)
@override_settings(DNSBL_PROVIDERS=["zen.spamhaus.org"])
def test_management_command_no_dnsbl(self):
"""Check that command works fine without dnsbl."""
self.assertEqual(models.DNSBLResult.objects.count(), 0)
management.call_command("modo", "check_mx", "--no-dnsbl")
self.assertFalse(
models.DNSBLResult.objects.filter(domain=self.domain).exists())
response = self.client.get(
reverse("admin:dnsbl_domain_detail", args=[self.domain.pk]))
self.assertEqual(response.status_code, 200)