diff --git a/docker-compose.yml b/docker-compose.yml index 738fa68e1..4b07aa5bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,22 @@ services: PYTHONPATH: ".." PYTHONUNBUFFERED: 1 command: > - sh -c "cd test_project; python3 manage.py rqworker dkim modoboa" + sh -c "cd test_project; python3 manage.py rqworker dkim modoboa dovecot" + volumes: + - .:/code + + rq-scheduler: + build: + context: . + dockerfile: docker/Dockerfile.dev + environment: + DB: "SQLITE" + DEBUG: 1 + REDIS_HOST: redis + PYTHONPATH: ".." + PYTHONUNBUFFERED: 1 + command: > + sh -c "cd test_project; python3 manage.py rqcron test_project/cron_config.py" volumes: - .:/code diff --git a/modoboa/admin/jobs.py b/modoboa/admin/jobs.py index 4526a5aa9..836940dab 100644 --- a/modoboa/admin/jobs.py +++ b/modoboa/admin/jobs.py @@ -1,26 +1,19 @@ -"""Script to register RQ jobs.""" +"""Async jobs definition.""" + +import logging +import os +import shutil + +from modoboa.admin.app_settings import load_admin_settings +from modoboa.admin.models.mailbox import MailboxOperation +from modoboa.lib.sysutils import exec_cmd +from modoboa.parameters import tools as param_tools + +logger = logging.getLogger("modoboa.jobs") -def rename_mailbox(op_id): +def rename_mailbox(operation): """Rename the mailbox folder through a RQ Job.""" - import logging - import os - - from modoboa.admin.app_settings import load_admin_settings - from modoboa.admin.models.mailbox import MailboxOperation - from modoboa.lib.sysutils import exec_cmd - from modoboa.parameters import tools as param_tools - - logger = logging.getLogger("modoboa.admin") - - load_admin_settings() - if not param_tools.get_global_parameter("handle_mailboxes"): - return - try: - operation = MailboxOperation.objects.get(pk=op_id) - except MailboxOperation.DoesNotExist: - logger.error(f"Failed to retrieve operation id {op_id} for renaming") - return if not os.path.exists(operation.argument): logger.error(f"Failed to rename {operation.argument}, folder not found") operation.delete() @@ -43,38 +36,15 @@ def rename_mailbox(op_id): operation.delete() -def delete_mailbox(op_id): +def delete_mailbox(operation): """Delete the mailbox folder through a RQ Job.""" - import logging - import os - import shutil - - from modoboa.admin.app_settings import load_admin_settings - from modoboa.admin.models.mailbox import MailboxOperation - from modoboa.parameters import tools as param_tools - - logger = logging.getLogger("modoboa.admin") - - load_admin_settings() - if not param_tools.get_global_parameter("handle_mailboxes"): - return - - try: - operation = MailboxOperation.objects.get(pk=op_id) - except MailboxOperation.DoesNotExist: - logger.error(f"Failed to retrieve operation id {op_id} for deletion") - return - if not os.path.exists(operation.argument): logger.error(f"Failed to delete {operation.argument}, folder not found") operation.delete() return def onerror(function, path, excinfo): - import logging - """Handle errors.""" - logger = logging.getLogger("modoboa.admin") logger.critical(f"delete failed (reason: {excinfo})") operation.delete() @@ -82,24 +52,12 @@ def delete_mailbox(op_id): operation.delete() -def handle_remaining_operation(): - import logging - - import django_rq - - from modoboa.parameters import tools as param_tools - from modoboa.admin.app_settings import load_admin_settings - from modoboa.admin.models.mailbox import MailboxOperation - +def handle_mailbox_operations(): load_admin_settings() if not param_tools.get_global_parameter("handle_mailboxes"): return - queue = django_rq.get_queue("dovecot") - if queue.started_job_registry.count > 0 or queue.scheduled_job_registry.count > 0: - return - logger = logging.getLogger("modoboa.admin") for ope in MailboxOperation.objects.all(): if ope.type == "rename": - rename_mailbox(ope.pk) + rename_mailbox(ope) elif ope.type == "delete": - delete_mailbox(ope.pk) + delete_mailbox(ope) diff --git a/modoboa/admin/models/mailbox.py b/modoboa/admin/models/mailbox.py index 113e83cfe..3d667b790 100644 --- a/modoboa/admin/models/mailbox.py +++ b/modoboa/admin/models/mailbox.py @@ -9,15 +9,12 @@ from django.db.models.manager import Manager from django.utils.encoding import smart_str, force_str from django.utils.translation import gettext as _, gettext_lazy -from modoboa.admin import jobs from modoboa.core.models import User from modoboa.lib import exceptions as lib_exceptions from modoboa.lib.email_utils import split_mailbox from modoboa.lib.sysutils import doveadm_cmd from modoboa.parameters import tools as param_tools -import django_rq - from .base import AdminObject from .domain import Domain from . import mixins @@ -195,12 +192,9 @@ class Mailbox(mixins.MessageLimitMixin, AdminObject): hm = param_tools.get_global_parameter("handle_mailboxes", raise_exception=False) if not hm: return - mail_op = MailboxOperation.objects.create( + MailboxOperation.objects.create( mailbox=self, type="rename", argument=old_mail_home ) - mail_op.save() - dovecot_queue = django_rq.get_queue("dovecot") - dovecot_queue.enqueue(jobs.rename_mailbox, mail_op.pk) def rename(self, address, domain): """Rename the mailbox. @@ -229,12 +223,7 @@ class Mailbox(mixins.MessageLimitMixin, AdminObject): hm = param_tools.get_global_parameter("handle_mailboxes", raise_exception=False) if not hm: return - mail_op = MailboxOperation.objects.create( - type="delete", argument=self.mail_home - ) - mail_op.save() - dovecot_queue = django_rq.get_queue("dovecot") - dovecot_queue.enqueue(jobs.delete_mailbox, mail_op.pk) + MailboxOperation.objects.create(type="delete", argument=self.mail_home) def set_quota(self, value=None, override_rules=False): """Set or update quota value for this mailbox. diff --git a/modoboa/core/jobs.py b/modoboa/core/jobs.py index 2c8a1c621..62ecb8f83 100644 --- a/modoboa/core/jobs.py +++ b/modoboa/core/jobs.py @@ -1,14 +1,28 @@ -"""Core Jobs""" +"""Async jobs definition.""" + +import datetime +import logging + +from modoboa.core.models import Log +from modoboa.maillog.models import Maillog +from modoboa.parameters import tools as param_tools + +from django.utils import timezone + +logger = logging.getLogger("modoboa.jobs") -def clean_log(): - import datetime - - from modoboa.core.models import Log - from modoboa.parameters import tools as param_tools - - from django.utils import timezone - +def clean_logs(): log_maximum_age = param_tools.get_global_parameter("log_maximum_age") + logger.info(f"Deleting logs older than {log_maximum_age} days...") limit = timezone.now() - datetime.timedelta(log_maximum_age) Log.objects.filter(date_created__lt=limit).delete() + + message_history_maximum_age = param_tools.get_global_parameter( + "message_history_maximum_age" + ) + logger.info( + f"Deleting messages in history older than {message_history_maximum_age} days..." + ) + limit = timezone.now() - datetime.timedelta(message_history_maximum_age) + Maillog.objects.filter(date__lt=limit).delete() diff --git a/pyproject.toml b/pyproject.toml index 7a6deed71..42ef78220 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,9 +70,8 @@ dependencies = [ "defusedxml>=0.6.0", "python-magic>=0.4.24", # RQ - "rq>=1.13.0", + "rq>=2.4.0", "django-rq", - "rq-scheduler", # Sieve "sievelib>=1.4.1", # User diff --git a/test_project/test_project/cron_config.py b/test_project/test_project/cron_config.py new file mode 100644 index 000000000..d98b3abdf --- /dev/null +++ b/test_project/test_project/cron_config.py @@ -0,0 +1,8 @@ +from rq.cron import register + +from modoboa.core import jobs as core_jobs +from modoboa.admin import jobs as admin_jobs + + +register(admin_jobs.handle_mailbox_operations, queue_name="dovecot", cron="* * * * *") +register(core_jobs.clean_logs, queue_name="modoboa", cron="0 0 * * *") diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py index 8d3ffb235..1576326df 100644 --- a/test_project/test_project/settings.py +++ b/test_project/test_project/settings.py @@ -385,6 +385,7 @@ LOGGING = { }, "modoboa.admin": {"handlers": ["modoboa"], "level": "INFO", "propagate": False}, "modoboa.dns": {"handlers": ["syslog"], "level": "INFO", "propagate": False}, + "modoboa.jobs": {"handlers": ["syslog"], "level": "INFO", "propagate": False}, "django.server": { "handlers": ["django.server"], "level": "INFO",