github-jwinterm/modoboa
1
0
Fork 0
mirror of https://github.com/jwinterm/modoboa.git synced 2026-10-17 05:41:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3505 from modoboa/feature/use-rq

Browse Source 
Use RQ instead of cron
This commit is contained in:
Antoine Nguyen
2026-01-13 10:35:16 +01:00
committed by GitHub
parent e094dce2db 0ea42129ef
commit 062d2ccb62
29 changed files with 439 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.yml
+16 -1
View File
@@ -38,7 +38,22 @@ services:
PYTHONPATH: ".."
PYTHONUNBUFFERED: 1
command: >
sh -c "cd test_project; python3 manage.py rqworker dkim modoboa"
sh -c "cd test_project; python3 manage.py rqworker dkim modoboa dovecot"
volumes:
- .:/code
rq-scheduler:
build:
context: .
dockerfile: docker/Dockerfile.dev
environment:
DB: "SQLITE"
DEBUG: 1
REDIS_HOST: redis
PYTHONPATH: ".."
PYTHONUNBUFFERED: 1
command: >
sh -c "cd test_project; python3 manage.py rqcron test_project/cron_config.py"
volumes:
- .:/code
modoboa/admin/api/v2/serializers.py
+1
View File
@@ -223,6 +223,7 @@ class AdminGlobalParametersSerializer(serializers.Serializer):
enable_dkim_checks = serializers.BooleanField(default=True)
enable_dmarc_checks = serializers.BooleanField(default=True)
enable_autoconfig_checks = serializers.BooleanField(default=True)
enable_dns_notifications = serializers.BooleanField(default=True)
custom_dns_server = serializers.IPAddressField(allow_blank=True, allow_null=True)
enable_dnsbl_checks = serializers.BooleanField(default=True)
dkim_keys_storage_dir = serializers.CharField(allow_blank=True, allow_null=True)
modoboa/admin/app_settings.py
+12
View File
@@ -110,6 +110,18 @@ GLOBAL_PARAMETERS_STRUCT = collections.OrderedDict(
),
},
),
(
"enable_dns_notifications",
{
"label": gettext_lazy(
"Notify domain admins about DNS issues"
),
"help_text": gettext_lazy(
"Send a notification by email to domain administrators "
"when DNS issues are encountered"
),
},
),
(
"custom_dns_server",
{
modoboa/admin/management/commands/subcommands/_mx.py → modoboa/admin/dns_checker.py
+41 -111
View File
@@ -1,16 +1,10 @@
"""Management command to check defined domains."""
import ipaddress
import gevent
from gevent import socket
import socket
from django.conf import settings
from django.core.mail import EmailMessage
from django.core.management.base import BaseCommand
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils.encoding import smart_str
from django.utils.functional import cached_property
from django.utils.translation import gettext as _
@@ -19,10 +13,7 @@ from modoboa.dnstools import models as dns_models
from modoboa.parameters import tools as param_tools
class CheckMXRecords(BaseCommand):
"""Command class."""
help = "Check defined domains." # NOQA:A003
class DNSChecker:
@cached_property
def providers(self):
@@ -36,52 +27,24 @@ class CheckMXRecords(BaseCommand):
"""Return sender address for notifications."""
return param_tools.get_global_parameter("sender_address", app="core")
@cached_property
def config(self) -> dict:
return dict(param_tools.get_global_parameters("admin"))
@cached_property
def valid_mxs(self):
"""Return valid MXs set in admin."""
valid_mxs = param_tools.get_global_parameter("valid_mxs")
valid_mxs = self.config["valid_mxs"]
return [
ipaddress.ip_network(smart_str(v.strip()))
for v in valid_mxs.split()
if v.strip()
ipaddress.ip_network(str(v.strip())) for v in valid_mxs.split() if v.strip()
]
def add_arguments(self, parser):
"""Add extra arguments to command."""
parser.add_argument(
"--no-dnsbl", action="store_true", default=False, help="Skip DNSBL queries."
)
parser.add_argument(
"--email",
type=str,
action="append",
default=[],
help="One or more email to notify",
)
parser.add_argument(
"--skip-admin-emails",
action="store_true",
default=False,
help="Skip domain's admins email notification.",
)
parser.add_argument(
"--domain",
type=str,
action="append",
default=[],
help="Domain name or id to update.",
)
parser.add_argument(
"--timeout", type=int, default=3, help="Timeout used for queries."
)
parser.add_argument("--ttl", type=int, default=7200, help="TTL for dns query.")
def query_dnsbl(self, mx_list, provider):
"""Check given IP against given DNSBL provider."""
results = {}
for ip, mxs in mx_list.items():
try:
ip = ipaddress.ip_address(smart_str(ip))
ip = ipaddress.ip_address(str(ip))
except ValueError:
continue
else:
@@ -99,9 +62,9 @@ class CheckMXRecords(BaseCommand):
result = False
for mx in mxs:
results[mx] = result
return provider, results
return results
def store_dnsbl_result(self, domain, provider, results, **options):
def store_dnsbl_result(self, domain, provider, results):
"""Store DNSBL provider results for domain.
Return a list of alerts.
@@ -142,23 +105,24 @@ class CheckMXRecords(BaseCommand):
models.DNSBLResult.objects.bulk_create(to_create)
return alerts
def send_alert_notifications(self, domain, alerts, subject, tpl, **options):
def send_alert_notifications(
self, domain: models.Domain, alerts: list, subject: str, tpl: str
) -> None:
"""Send email notifications about given alerts."""
emails = list(options["email"])
if not options["skip_admin_emails"]:
emails.extend(
domain.admins.exclude(mailbox__isnull=True).values_list(
"email", flat=True
)
)
if not self.config["enable_dns_notifications"]:
return
emails = domain.admins.exclude(mailbox__isnull=True).values_list(
"email", flat=True
)
if not len(emails):
return
content = render_to_string(tpl, {"domain": domain, "alerts": alerts})
msg = EmailMessage(subject, content.strip(), self.sender, emails)
msg.send()
def check_valid_mx(self, domain, mx_list, **options):
"""Check that domain's MX record exist.
def check_valid_mx(self, domain: models.Domain, mx_list: list) -> None:
"""
Check that domain's MX record exist.
If `valid_mx` is provided, retrieved MX records must be
contained in it.
@@ -212,26 +176,25 @@ class CheckMXRecords(BaseCommand):
return
subject = _("[modoboa] MX issue(s) for domain {}").format(domain.name)
tpl = "admin/notifications/domain_invalid_mx.html"
self.send_alert_notifications(domain, alerts, subject, tpl, **options)
self.send_alert_notifications(domain, alerts, subject, tpl)
def run(self, domain: models.Domain, ttl: int = 7200):
# Remove deprecated records first
domain.dnsblresult_set.exclude(provider__in=self.providers).delete()
def check_domain(self, domain, timeout=3, ttl=7200, **options):
"""Check specified domain."""
mx_list = list(models.MXRecord.objects.get_or_create_for_domain(domain, ttl))
if param_tools.get_global_parameter("enable_mx_checks"):
self.check_valid_mx(domain, mx_list, **options)
if self.config["enable_mx_checks"]:
self.check_valid_mx(domain, mx_list)
if param_tools.get_global_parameter("enable_spf_checks"):
if self.config["enable_spf_checks"]:
dns_models.DNSRecord.objects.get_or_create_for_domain(domain, "spf", ttl)
condition = (
param_tools.get_global_parameter("enable_dkim_checks")
and domain.dkim_public_key
)
condition = self.config["enable_dkim_checks"] and domain.dkim_public_key
if condition:
dns_models.DNSRecord.objects.get_or_create_for_domain(domain, "dkim", ttl)
if param_tools.get_global_parameter("enable_dmarc_checks"):
if self.config["enable_dmarc_checks"]:
dns_models.DNSRecord.objects.get_or_create_for_domain(domain, "dmarc", ttl)
if param_tools.get_global_parameter("enable_autoconfig_checks"):
if self.config["enable_autoconfig_checks"]:
dns_models.DNSRecord.objects.get_or_create_for_domain(
domain, "autoconfig", ttl
)
@@ -239,11 +202,8 @@ class CheckMXRecords(BaseCommand):
domain, "autodiscover", ttl
)
condition = (
not param_tools.get_global_parameter("enable_dnsbl_checks")
or options["no_dnsbl"] is True
)
if condition or not mx_list:
condition = not self.config["enable_dnsbl_checks"] or not mx_list
if condition:
return
mx_by_ip = {}
@@ -253,44 +213,14 @@ class CheckMXRecords(BaseCommand):
elif mx not in mx_by_ip[mx.address]:
mx_by_ip[mx.address].append(mx)
jobs = [
gevent.spawn(self.query_dnsbl, mx_by_ip, provider)
for provider in self.providers
]
gevent.joinall(jobs, timeout)
alerts = []
for job in jobs:
if not job.successful():
continue
provider, results = job.value
alerts += self.store_dnsbl_result(domain, provider, results, **options)
for provider in self.providers:
results = self.query_dnsbl(mx_by_ip, provider)
alerts += self.store_dnsbl_result(domain, provider, results)
if not alerts:
return
subject = _("[modoboa] DNSBL issue(s) for domain {}").format(domain.name)
tpl = "admin/notifications/domain_in_dnsbl.html"
self.send_alert_notifications(domain, alerts, subject, tpl, **options)
def handle(self, *args, **options):
"""Command entry point."""
# Remove deprecated records first
models.DNSBLResult.objects.exclude(provider__in=self.providers).delete()
if options["domain"]:
domains = []
for domain in options["domain"]:
try:
if domain.isdigit():
domains.append(models.Domain.objects.get(pk=domain))
else:
domains.append(models.Domain.objects.get(name=domain))
except models.Domain.DoesNotExist:
pass
else:
domains = models.Domain.objects.filter(enabled=True, enable_dns_checks=True)
options.pop("domain")
for domain in domains:
if domain.uses_a_reserved_tld:
continue
self.check_domain(domain, **options)
self.send_alert_notifications(domain, alerts, subject, tpl)
modoboa/admin/jobs.py
+86
View File
@@ -0,0 +1,86 @@
"""Async jobs definition."""
import logging
import os
import shutil
from django.db.models import F
from django.utils import timezone
import django_rq
from modoboa.admin import models
from modoboa.admin.app_settings import load_admin_settings
from modoboa.admin.dns_checker import DNSChecker
from modoboa.lib.sysutils import exec_cmd
from modoboa.parameters import tools as param_tools
logger = logging.getLogger("modoboa.jobs")
def rename_mailbox(operation):
"""Rename the mailbox folder through a RQ Job."""
if not os.path.exists(operation.argument):
logger.error(f"Failed to rename {operation.argument}, folder not found")
operation.delete()
return
new_mail_home = operation.mailbox.mail_home
dirname = os.path.dirname(new_mail_home)
if not os.path.exists(dirname):
try:
os.makedirs(dirname)
except OSError as e:
reason = str(e).decode("utf-8")
logger.critical(
f"renaming of {operation.argument} to {new_mail_home} failed (reason: {reason})"
)
return
code, output = exec_cmd(f"mv {operation.argument} {new_mail_home}")
if code:
logger.critical(f"Renaming of {new_mail_home} failed (reason: {output})")
return
operation.delete()
def delete_mailbox(operation):
"""Delete the mailbox folder through a RQ Job."""
if not os.path.exists(operation.argument):
logger.error(f"Failed to delete {operation.argument}, folder not found")
operation.delete()
return
def onerror(function, path, excinfo):
"""Handle errors."""
logger.critical(f"delete failed (reason: {excinfo})")
operation.delete()
shutil.rmtree(operation.argument, False, onerror)
operation.delete()
def handle_mailbox_operations():
load_admin_settings()
if not param_tools.get_global_parameter("handle_mailboxes"):
return
for ope in models.MailboxOperation.objects.all():
if ope.type == "rename":
rename_mailbox(ope)
elif ope.type == "delete":
delete_mailbox(ope)
def launch_domain_dns_checks(domain_id: int):
domain = models.Domain.objects.get(id=domain_id)
DNSChecker().run(domain)
def handle_dns_checks():
"""Launch DNS checks for every possible domain."""
minute = timezone.now().minute
queue = django_rq.get_queue("modoboa")
for domain in models.Domain.objects.annotate(slot=F("id") % 60).filter(
enable_dns_checks=True, slot=minute
):
if domain.uses_a_reserved_tld:
continue
queue.enqueue(launch_domain_dns_checks, domain.id)
modoboa/admin/management/commands/handle_mailbox_operations.py
-103
View File
@@ -1,103 +0,0 @@
"""A management command to apply mailbox operations."""
import logging
import os
import shutil
from django.core.management.base import BaseCommand
from modoboa.lib.exceptions import InternalError
from modoboa.lib.sysutils import exec_cmd
from modoboa.parameters import tools as param_tools
from ...app_settings import load_admin_settings
from ...models import MailboxOperation
class OperationError(Exception):
"""Custom exception."""
pass
class Command(BaseCommand):
"""Command definition"""
help = "Handles rename and delete operations on mailboxes" # NOQA:A003
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = logging.getLogger("modoboa.admin")
def add_arguments(self, parser):
"""Add extra arguments to command line."""
parser.add_argument(
"--pidfile",
default="/tmp/handle_mailbox_operations.pid",
help="Path to the file that will contain the PID of this process",
)
def rename_mailbox(self, operation):
if not os.path.exists(operation.argument):
return
new_mail_home = operation.mailbox.mail_home
dirname = os.path.dirname(new_mail_home)
if not os.path.exists(dirname):
try:
os.makedirs(dirname)
except OSError as e:
raise OperationError(str(e)) from None
code, output = exec_cmd(f"mv {operation.argument} {new_mail_home}")
if code:
raise OperationError(output)
def delete_mailbox(self, operation):
"""Try to delete a mailbox tree on filesystem."""
if not os.path.exists(operation.argument):
return
def onerror(function, path, excinfo):
"""Handle errors."""
self.logger.critical(f"delete failed (reason: {excinfo})")
shutil.rmtree(operation.argument, False, onerror)
def check_pidfile(self, path):
"""Check if this command is already running
:param str path: path to the file containing the PID
:return: a boolean, True means we can go further
"""
if os.path.exists(path):
with open(path) as fp:
pid = fp.read().strip()
code, output = exec_cmd(
f"grep handle_mailbox_operations /proc/{pid}/cmdline"
)
if not code:
return False
with open(path, "w") as fp:
print(os.getpid(), file=fp)
return True
def handle(self, *args, **options):
"""Command entry point."""
load_admin_settings()
if not param_tools.get_global_parameter("handle_mailboxes"):
return
if not self.check_pidfile(options["pidfile"]):
return
for ope in MailboxOperation.objects.all():
try:
f = getattr(self, f"{ope.type}_mailbox")
except AttributeError:
continue
try:
f(ope)
except (OperationError, InternalError) as e:
self.logger.critical(
"%s failed (reason: %s)", ope, str(e).decode("utf-8")
)
else:
self.logger.info("%s succeed", ope)
ope.delete()
os.unlink(options["pidfile"])
modoboa/admin/management/commands/modo.py
-2
View File
@@ -5,7 +5,6 @@ from django.core.management.base import BaseCommand
from .subcommands._export import ExportCommand
from .subcommands._import import ImportCommand
from .subcommands._manage_dkim_keys import ManageDKIMKeys
from .subcommands._mx import CheckMXRecords
from .subcommands._repair import Repair
@@ -20,7 +19,6 @@ class Command(BaseCommand):
subcommands = {
"export": ExportCommand,
"import": ImportCommand,
"check_mx": CheckMXRecords,
"manage_dkim_keys": ManageDKIMKeys,
"repair": Repair,
}
modoboa/admin/tests/test_mailbox_operations.py
+4 -4
View File
@@ -4,10 +4,10 @@ import os
import shutil
from unittest import mock
from django.core.management import call_command
from django.test import override_settings
from django.urls import reverse
from modoboa.admin import jobs
from modoboa.lib.tests import ModoAPITestCase
from .. import factories, models
@@ -45,7 +45,7 @@ class MailboxOperationTestCase(ModoAPITestCase):
self.client.post(
reverse("v2:account-delete", args=[mb.user.pk]), {}, format="json"
)
call_command("handle_mailbox_operations")
jobs.handle_mailbox_operations()
self.assertFalse(models.MailboxOperation.objects.exists())
self.assertFalse(os.path.exists(mb.mail_home))
@@ -71,7 +71,7 @@ class MailboxOperationTestCase(ModoAPITestCase):
self.assertEqual(response.status_code, 200)
path = f"{self.workdir}/test.com/admin2"
mail_home_mock.__get__ = mock.Mock(return_value=path)
call_command("handle_mailbox_operations")
jobs.handle_mailbox_operations()
self.assertFalse(models.MailboxOperation.objects.exists())
self.assertTrue(os.path.exists(mb.mail_home))
@@ -84,6 +84,6 @@ class MailboxOperationTestCase(ModoAPITestCase):
self.client.post(
reverse("v2:domain-delete", args=[domain.pk]), {}, format="json"
)
call_command("handle_mailbox_operations")
jobs.handle_mailbox_operations()
self.assertFalse(models.MailboxOperation.objects.exists())
self.assertFalse(os.path.exists(path))
modoboa/admin/tests/test_mx.py
+58 -62
View File
@@ -3,12 +3,18 @@
from unittest import mock
import dns.resolver
from freezegun import freeze_time
from rq import SimpleWorker
from testfixtures import LogCapture
from django.core import mail, management
from django.core import mail
from django.test import override_settings
from django.utils.translation import gettext as _
import django_rq
from modoboa.admin import jobs
from modoboa.admin.dns_checker import DNSChecker
from modoboa.core import models as core_models, factories as core_factories
from modoboa.lib.tests import ModoTestCase
from . import utils
@@ -44,26 +50,26 @@ class MXTestCase(ModoTestCase):
cls.localconfig.save()
models.MXRecord.objects.all().delete()
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@mock.patch.object(dns.resolver.Resolver, "resolve")
def test_management_command(
self, mock_query, mock_getaddrinfo, mock_g_gethostbyname
):
def test_dns_checker(self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
"""Check that command works fine."""
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
self.set_global_parameter("enable_dnsbl_checks", False)
self.assertEqual(models.MXRecord.objects.count(), 0)
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=0")
DNSChecker().run(self.domain, ttl=0)
self.assertTrue(models.MXRecord.objects.filter(domain=self.domain).exists())
# we passed a ttl to 0. this will reset the cache this time
qs = models.MXRecord.objects.filter(domain=self.domain)
id_ = qs[0].id
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=7200")
DNSChecker().run(self.domain)
qs = models.MXRecord.objects.filter(domain=self.domain)
self.assertNotEqual(id_, qs[0].id)
id_ = qs[0].id
@@ -71,40 +77,11 @@ class MXTestCase(ModoTestCase):
# assume that mxrecords ids are the same. means that we taking care of
# ttl
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl")
DNSChecker().run(self.domain)
qs = models.MXRecord.objects.filter(domain=self.domain)
self.assertEqual(id_, qs[0].id)
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@mock.patch.object(dns.resolver.Resolver, "resolve")
def test_single_domain_update(
self, mock_query, mock_getaddrinfo, mock_g_gethostbyname
):
"""Update only one domain."""
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--domain", self.domain.name)
self.assertTrue(models.MXRecord.objects.filter(domain=self.domain).exists())
self.assertFalse(
models.MXRecord.objects.filter(domain=self.bad_domain).exists()
)
with LogCapture("modoboa.dns"):
management.call_command(
"modo", "check_mx", "--domain", str(self.bad_domain.pk)
)
self.assertFalse(
models.MXRecord.objects.filter(domain=self.bad_domain).exists()
)
self.assertEqual(len(mail.outbox), 1)
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--domain", "toto.com")
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@mock.patch.object(dns.resolver.Resolver, "resolve")
def test_invalid_mx(self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
@@ -112,6 +89,8 @@ class MXTestCase(ModoTestCase):
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
self.set_global_parameter("enable_dnsbl_checks", False)
domain = factories.DomainFactory(name="invalid-mx.com")
# Add domain admin with mailbox
mb = factories.MailboxFactory(
@@ -122,14 +101,10 @@ class MXTestCase(ModoTestCase):
)
domain.add_admin(mb.user)
with LogCapture("modoboa.dns"):
management.call_command(
"modo", "check_mx", "--domain", domain.name, "--no-dnsbl"
)
DNSChecker().run(domain)
self.assertEqual(domain.alarms.count(), 1)
with LogCapture("modoboa.dns"):
management.call_command(
"modo", "check_mx", "--domain", domain.name, "--no-dnsbl"
)
DNSChecker().run(domain)
self.assertEqual(domain.alarms.count(), 1)
self.assertEqual(len(mail.outbox), 1)
@@ -226,7 +201,16 @@ class DNSBLTestCase(ModoTestCase):
"""Create some data."""
super().setUpTestData()
cls.domain = factories.DomainFactory(name="modoboa.org")
factories.DomainFactory(name="does-not-exist.example.com")
# Add domain admin with mailbox
mb = factories.MailboxFactory(
address="admin",
domain=cls.domain,
user__username="admin@modoboa.org",
user__groups=("DomainAdmins",),
)
cls.domain.add_admin(mb.user)
cls.domain4 = factories.DomainFactory(name="does-not-exist.example.com")
cls.domain2 = factories.DomainFactory(
name="test.localhost"
) # Should not be checked
@@ -235,19 +219,29 @@ class DNSBLTestCase(ModoTestCase):
)
models.DNSBLResult.objects.all().delete()
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@mock.patch.object(dns.resolver.Resolver, "resolve")
def test_management_command(
self, mock_query, mock_getaddrinfo, mock_g_gethostbyname
):
def test_job(self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
"""Check that command works fine."""
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
self.assertEqual(models.DNSBLResult.objects.count(), 0)
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx")
with freeze_time(f"2026-01-12 14:{self.domain.id % 60:02}"):
jobs.handle_dns_checks()
with freeze_time(f"2026-01-12 14:{self.domain2.id % 60:02}"):
jobs.handle_dns_checks()
with freeze_time(f"2026-01-12 14:{self.domain3.id % 60:02}"):
jobs.handle_dns_checks()
with freeze_time(f"2026-01-12 14:{self.domain4.id % 60:02}"):
jobs.handle_dns_checks()
queue = django_rq.get_queue("modoboa")
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
self.assertTrue(models.DNSBLResult.objects.filter(domain=self.domain).exists())
self.assertFalse(
models.DNSBLResult.objects.filter(domain=self.domain3).exists()
@@ -255,7 +249,7 @@ class DNSBLTestCase(ModoTestCase):
self.assertFalse(self.domain.uses_a_reserved_tld)
self.assertTrue(self.domain2.uses_a_reserved_tld)
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@mock.patch.object(dns.resolver.Resolver, "resolve")
def test_notifications(self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
@@ -264,24 +258,24 @@ class DNSBLTestCase(ModoTestCase):
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "127.0.0.4"
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--email", "user@example.test")
self.assertEqual(len(mail.outbox), 2)
DNSChecker().run(self.domain)
self.assertEqual(len(mail.outbox), 1)
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@mock.patch.object(dns.resolver.Resolver, "resolve")
def test_notifications_wrong_dnsbl_response(
self, mock_query, mock_getaddrinfo, mock_g_gethostbyname
):
"""Check notifications."""
self.set_global_parameter("enable_dnsbl_checks", True)
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "127.255.255.254" # <--Spamhaus response when querying from an open resolver
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--email", "user@example.test")
self.assertEqual(len(mail.outbox), 1)
DNSChecker().run(self.domain)
self.assertEqual(len(mail.outbox), 0)
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@mock.patch.object(dns.resolver.Resolver, "resolve")
def test_management_command_no_dnsbl(
@@ -291,9 +285,10 @@ class DNSBLTestCase(ModoTestCase):
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
self.set_global_parameter("enable_dnsbl_checks", False)
self.assertEqual(models.DNSBLResult.objects.count(), 0)
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl")
DNSChecker().run(self.domain)
self.assertFalse(models.DNSBLResult.objects.filter(domain=self.domain).exists())
@@ -306,7 +301,7 @@ class DNSChecksTestCase(ModoTestCase):
super().setUpTestData()
cls.domain = factories.DomainFactory(name="dns-checks.com")
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@mock.patch.object(dns.resolver.Resolver, "resolve")
def test_management_command(
@@ -316,13 +311,14 @@ class DNSChecksTestCase(ModoTestCase):
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
self.set_global_parameter("enable_dnsbl_checks", False)
self.domain.enable_dkim = True
self.domain.dkim_public_key = "XXXXX"
self.domain.save(update_fields=["enable_dkim", "dkim_public_key"])
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=0")
DNSChecker().run(self.domain, ttl=0)
self.assertIsNot(self.domain.spf_record, None)
self.assertIsNot(self.domain.dkim_record, None)
modoboa/amavis/jobs.py
+11
View File
@@ -0,0 +1,11 @@
"""Async jobs definition."""
from django.core.management import call_command
def amnotify():
call_command("amnotify")
def qcleanup():
call_command("qcleanup")
modoboa/amavis/tests/test_jobs.py
+18
View File
@@ -0,0 +1,18 @@
"""Tests cases for RQ jobs."""
from modoboa.lib.tests import ModoTestCase
from modoboa.amavis import factories, jobs, models
class JobsTestCase(ModoTestCase):
databases = "__all__"
def test_qcleanup(self):
factories.create_spam("user@test.com", rs="D")
jobs.qcleanup()
self.assertEqual(models.Quarantine.objects.count(), 0)
self.assertEqual(models.Msgs.objects.count(), 0)
self.assertEqual(models.Maddr.objects.count(), 0)
self.assertEqual(models.Msgrcpt.objects.count(), 0)
modoboa/amavis/tests/test_viewsets.py
+2 -3
View File
@@ -4,14 +4,13 @@ from unittest import mock
from rq import SimpleWorker
from django.core import mail
from django.core.management import call_command
from django.test import override_settings
from django.urls import reverse
import django_rq
from modoboa.admin import factories as admin_factories, models as admin_models
from modoboa.amavis import factories, models
from modoboa.amavis import factories, jobs, models
from modoboa.amavis.utils import smart_str
from modoboa.core import models as core_models
from modoboa.lib.tests import ModoAPITestCase
@@ -204,7 +203,7 @@ class QuarantineViewSetTestCase(TestDataMixin, ModoAPITestCase):
self.assertEqual(self.msgrcpt.rs, "p")
# Send notification to admins
call_command("amnotify")
jobs.amnotify()
self.assertEqual(len(mail.outbox), 1)
def _test_mark_selection(self, action, status):
modoboa/calendars/jobs.py
+7
View File
@@ -0,0 +1,7 @@
"""Async jobs definition."""
from django.core.management import call_command
def generate_rights():
call_command("generate_rights")
modoboa/calendars/tests.py
+2 -1
View File
@@ -18,6 +18,7 @@ from modoboa.lib.tests import ModoAPITestCase
from modoboa.admin.factories import populate_database
from . import factories
from . import jobs
from . import models
from . import mocks
@@ -106,7 +107,7 @@ class AccessRuleTestCase(ModoAPITestCase):
self.assertEqual(cfg.get(section, "permissions"), "Rr")
# Call a second time
management.call_command("generate_rights", verbosity=False)
jobs.generate_rights()
def test_rights_file_generation_with_admin(self):
self.set_global_parameter(
modoboa/core/commands/deploy.py
+13
View File
@@ -3,6 +3,7 @@
import getpass
import os
from os.path import isfile
import random
import shutil
import subprocess
import sys
@@ -259,6 +260,18 @@ class DeployCommand(Command):
shutil.copyfile(f"{self._templates_dir}/urls.py.tpl", f"{path}/urls.py")
os.mkdir(f"{parsed_args.name}/media")
hour = random.randint(0, 6)
tpl = self._render_template(
f"{self._templates_dir}/cron_config.py.tpl",
{
"minute": random.randint(1, 59),
"hour_start": hour,
"hour_end": hour + 12,
},
)
with open(f"{path}/cron_config.py", "w") as fp:
fp.write(tpl)
if isfile(f"{path}/settings.pyc"):
os.unlink(f"{path}/settings.pyc")
self._exec_django_command("migrate", parsed_args.name, "--noinput")
modoboa/core/commands/templates/cron_config.py.tpl
+30
View File
@@ -0,0 +1,30 @@
from rq.cron import register
from django.conf import settings
from modoboa.amavis import jobs as amavis_jobs
from modoboa.calendars import jobs as calendars_jobs
from modoboa.core import jobs as core_jobs
from modoboa.admin import jobs as admin_jobs
from modoboa.maillog import jobs as maillog_jobs
register(core_jobs.clean_logs, queue_name="modoboa", cron="0 0 * * *")
register(
core_jobs.communicate_with_public_api,
queue_name="modoboa",
cron="{{ minute }} {{ hour_start }},{{ hour_end }} * * *",
)
register(admin_jobs.handle_mailbox_operations, queue_name="dovecot", cron="* * * * *")
register(admin_jobs.handle_dns_checks, queue_name="modoboa", cron="* * * * *")
register(maillog_jobs.logparser, queue_name="modoboa", cron="*/5 * * * *")
register(maillog_jobs.update_statistics, queue_name="modoboa", cron="0 * * * *")
register(calendars_jobs.generate_rights, queue_name="modoboa", cron="*/2 * * * *")
if "modoboa.amavis" in settings.MODOBOA_APPS:
register(amavis_jobs.qcleanup, queue_name="modoboa", cron="0 0 * * *")
register(amavis_jobs.amnotify, queue_name="modoboa", cron="0 12 * * *")
modoboa/core/commands/templates/settings.py.tpl
+6
View File
@@ -296,6 +296,12 @@ RQ_QUEUES = {
'DB': REDIS_QUOTA_DB,
'URL': REDIS_URL,
},
'dovecot': {
'HOST': REDIS_HOST,
'PORT': REDIS_PORT,
'DB': REDIS_QUOTA_DB,
'URL': REDIS_URL,
},
}
# CACHE
modoboa/core/jobs.py
+34
View File
@@ -0,0 +1,34 @@
"""Async jobs definition."""
import datetime
import logging
from django.core.management import call_command
from modoboa.core.models import Log
from modoboa.maillog.models import Maillog
from modoboa.parameters import tools as param_tools
from django.utils import timezone
logger = logging.getLogger("modoboa.jobs")
def clean_logs():
log_maximum_age = param_tools.get_global_parameter("log_maximum_age")
logger.info(f"Deleting logs older than {log_maximum_age} days...")
limit = timezone.now() - datetime.timedelta(log_maximum_age)
Log.objects.filter(date_created__lt=limit).delete()
message_history_maximum_age = param_tools.get_global_parameter(
"message_history_maximum_age"
)
logger.info(
f"Deleting messages in history older than {message_history_maximum_age} days..."
)
limit = timezone.now() - datetime.timedelta(message_history_maximum_age)
Maillog.objects.filter(date__lt=limit).delete()
def communicate_with_public_api():
call_command("communicate_with_public_api")
modoboa/core/management/commands/cleanlogs.py
-53
View File
@@ -1,53 +0,0 @@
import datetime
import logging
from django.core.management.base import BaseCommand
from django.utils import timezone
from modoboa.core.models import Log
from modoboa.maillog.models import Maillog
from modoboa.parameters import tools as param_tools
class Command(BaseCommand):
"""Command class."""
help = "Log table cleanup" # NOQA:A003
def add_arguments(self, parser):
"""Add extra arguments to command line."""
parser.add_argument(
"--debug", action="store_true", default=False, help="Activate debug output"
)
parser.add_argument(
"--verbose",
action="store_true",
default=False,
help="Display informational messages",
)
def __vprint(self, msg):
if not self.verbose:
return
print(msg)
def handle(self, *args, **options):
if options["debug"]:
log = logging.getLogger("django.db.backends")
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler())
self.verbose = options["verbose"]
log_maximum_age = param_tools.get_global_parameter("log_maximum_age")
self.__vprint(f"Deleting logs older than {log_maximum_age} days...")
limit = timezone.now() - datetime.timedelta(log_maximum_age)
Log.objects.filter(date_created__lt=limit).delete()
message_history_maximum_age = param_tools.get_global_parameter(
"message_history_maximum_age"
)
self.__vprint(
f"Deleting messages in history older than {message_history_maximum_age} days..."
)
limit = timezone.now() - datetime.timedelta(message_history_maximum_age)
Maillog.objects.filter(date__lt=limit).delete()
self.__vprint("Done.")
modoboa/core/password_hashers/base.py
+4 -1
View File
@@ -5,7 +5,10 @@ Contains weak hashers (the original ones) available with Modoboa.
"""
import base64
import hashlib
from random import Random
import string
from typing import Optional
from django.utils.crypto import constant_time_compare
@@ -49,7 +52,7 @@ class PasswordHasher(metaclass=MetaHasher):
def __init__(self, target="local"):
self._target = target
def _encrypt(self, clearvalue: str, salt: Optional[str] = None) -> str: # NOQA
def _encrypt(self, clearvalue: str, salt: Optional[str] = None) -> str:
raise NotImplementedError
def _b64encode(self, pwhash):
modoboa/core/tests/test_core.py
-14
View File
@@ -16,7 +16,6 @@ from django.urls import reverse
from django.utils import timezone
from modoboa.lib.tests import ModoTestCase, SimpleModoTestCase
from modoboa.maillog import factories as maillog_factories, models as maillog_models
from .. import factories, mocks, models
@@ -59,19 +58,6 @@ class ManagementCommandsTestCase(SimpleModoTestCase):
management.call_command("load_initial_data", "--admin-username", "modoadmin")
self.assertTrue(self.client.login(username="modoadmin", password="password"))
def test_clean_logs(self):
"""Run cleanlogs command."""
management.call_command("load_initial_data", "--no-frontend")
log1 = factories.LogFactory()
factories.LogFactory()
log1.date_created -= relativedelta(years=2)
log1.save(update_fields=["date_created"])
maillog_factories.MaillogFactory(date=timezone.now() - relativedelta(days=190))
maillog_factories.MaillogFactory()
management.call_command("cleanlogs")
self.assertEqual(models.Log.objects.count(), 1)
self.assertEqual(maillog_models.Maillog.objects.count(), 1)
def test_init_data_non_duplicate_client_creation(self):
"""Test that the load_initial_data command does not create duplicates
client for the frontend deployment."""
modoboa/core/tests/test_jobs.py
+40
View File
@@ -0,0 +1,40 @@
"""Tests cases for RQ jobs."""
from dateutil.relativedelta import relativedelta
import httmock
from django.core import mail
from django.core import management
from django.utils import timezone
from modoboa.core import factories, jobs, mocks, models
from modoboa.lib.tests import SimpleModoTestCase
from modoboa.maillog import factories as maillog_factories, models as maillog_models
class JobsTestCase(SimpleModoTestCase):
def test_clean_logs(self):
"""Check clean_logs job."""
management.call_command("load_initial_data", "--no-frontend")
log1 = factories.LogFactory()
factories.LogFactory()
log1.date_created -= relativedelta(years=2)
log1.save(update_fields=["date_created"])
maillog_factories.MaillogFactory(date=timezone.now() - relativedelta(days=190))
maillog_factories.MaillogFactory()
jobs.clean_logs()
self.assertEqual(models.Log.objects.count(), 1)
self.assertEqual(maillog_models.Maillog.objects.count(), 1)
def test_communicate_with_public_api(self):
"""Check that job starts fine."""
with httmock.HTTMock(
mocks.modo_api_instance_search,
mocks.modo_api_instance_create,
mocks.modo_api_instance_update,
mocks.modo_api_versions,
):
jobs.communicate_with_public_api()
self.assertEqual(models.LocalConfig.objects.first().api_pk, 100)
self.assertEqual(len(mail.outbox), 0)
modoboa/maillog/jobs.py
+11
View File
@@ -0,0 +1,11 @@
"""Async jobs definition."""
from django.core.management import call_command
def logparser():
call_command("logparser")
def update_statistics():
call_command("update_statistics")
modoboa/maillog/tests/test_views.py
+5 -4
View File
@@ -11,6 +11,7 @@ from django.test import override_settings
from modoboa.admin import factories as admin_factories
from modoboa.lib.tests import SimpleModoTestCase
from modoboa.maillog import jobs
class RunCommandsMixin:
@@ -36,14 +37,14 @@ class RunCommandsMixin:
with open(path, "w") as fp:
fp.write(content)
self.set_global_parameter("logfile", path)
call_command("logparser")
jobs.logparser()
def run_update_statistics(self, rebuild=False):
"""Run update_statistics command."""
args = []
if rebuild:
args.append("--rebuild")
call_command("update_statistics", *args)
call_command("update_statistics", ["--rebuild"])
else:
jobs.update_statistics()
@override_settings(RRDTOOL_TEST_MODE=True)
pyproject.toml
+2 -2
View File
@@ -48,7 +48,6 @@ dependencies = [
"dnspython==2.8.0",
"feedparser==6.0.12",
"fido2==2.0.0",
"gevent==25.9.1",
"progressbar33==2.4",
"python-dateutil",
"cryptography",
@@ -70,7 +69,7 @@ dependencies = [
"defusedxml>=0.6.0",
"python-magic>=0.4.24",
# RQ
"rq>=1.13.0",
"rq>=2.4.0",
"django-rq",
# Sieve
"sievelib>=1.4.1",
@@ -102,6 +101,7 @@ test = [
"factory-boy<3.4.0",
"httmock==1.4.0",
"testfixtures==8.3.0",
"freezegun",
"tox",
]
ldap = ["django-auth-ldap<6.0.0"]
test_project/.coveragerc
+1 -1
View File
@@ -1,5 +1,5 @@
[run]
branch = True
concurrency = multiprocessing, greenlet
concurrency = multiprocessing
parallel = True
source = ../modoboa
test_project/test_project/cron_config.py
+31
View File
@@ -0,0 +1,31 @@
from rq.cron import register
from django.conf import settings
from modoboa.amavis import jobs as amavis_jobs
from modoboa.calendars import jobs as calendars_jobs
from modoboa.core import jobs as core_jobs
from modoboa.admin import jobs as admin_jobs
from modoboa.maillog import jobs as maillog_jobs
register(core_jobs.clean_logs, queue_name="modoboa", cron="0 0 * * *")
# Random start for this one to avoid a DDoS on the API...
register(
core_jobs.communicate_with_public_api,
queue_name="modoboa",
cron="12 2,14 * * *",
)
register(admin_jobs.handle_mailbox_operations, queue_name="dovecot", cron="* * * * *")
register(admin_jobs.handle_dns_checks, queue_name="modoboa", cron="* * * * *")
register(maillog_jobs.logparser, queue_name="modoboa", cron="*/5 * * * *")
register(maillog_jobs.update_statistics, queue_name="modoboa", cron="0 * * * *")
register(calendars_jobs.generate_rights, queue_name="modoboa", cron="*/2 * * * *")
if "modoboa.amavis" in settings.MODOBOA_APPS:
register(amavis_jobs.qcleanup, queue_name="modoboa", cron="0 0 * * *")
register(amavis_jobs.amnotify, queue_name="modoboa", cron="0 12 * * *")
test_project/test_project/settings.py
+4
View File
@@ -303,6 +303,9 @@ RQ_QUEUES = {
"modoboa": {
"URL": REDIS_URL,
},
"dovecot": {
"URL": REDIS_URL,
},
}
# CACHE
@@ -382,6 +385,7 @@ LOGGING = {
},
"modoboa.admin": {"handlers": ["modoboa"], "level": "INFO", "propagate": False},
"modoboa.dns": {"handlers": ["syslog"], "level": "INFO", "propagate": False},
"modoboa.jobs": {"handlers": ["syslog"], "level": "INFO", "propagate": False},
"django.server": {
"handlers": ["django.server"],
"level": "INFO",
update_deps.sh
-8
View File
@@ -1,8 +0,0 @@
#!/bin/sh
pip-compile --output-file=requirements.txt pyproject.toml
pip-compile --extra=mysql --output-file=mysql-requirements.txt pyproject.toml
pip-compile --extra=postgresql --output-file=postgresql-requirements.txt pyproject.toml
pip-compile --extra=ldap --output-file=ldap-requirements.txt pyproject.toml
pip-compile --extra=dev --output-file=dev-requirements.txt pyproject.toml
pip-compile --extra=test --output-file=test-requirements.txt pyproject.toml