Use RQ instead of cron - mailbox ops

modoboa/admin/jobs.py

@@ -0,0 +1,105 @@
+"""Script to register RQ jobs."""
+
+
+def rename_mailbox(op_id):
+    """Rename the mailbox folder through a RQ Job."""
+    import logging
+    import os
+
+    from modoboa.admin.app_settings import load_admin_settings
+    from modoboa.admin.models.mailbox import MailboxOperation
+    from modoboa.lib.sysutils import exec_cmd
+    from modoboa.parameters import tools as param_tools
+
+    logger = logging.getLogger("modoboa.admin")
+
+    load_admin_settings()
+    if not param_tools.get_global_parameter("handle_mailboxes"):
+        return
+    try:
+        operation = MailboxOperation.objects.get(pk=op_id)
+    except MailboxOperation.DoesNotExist:
+        logger.error(f"Failed to retrieve operation id {op_id} for renaming")
+        return
+    if not os.path.exists(operation.argument):
+        logger.error(f"Failed to rename {operation.argument}, folder not found")
+        operation.delete()
+        return
+    new_mail_home = operation.mailbox.mail_home
+    dirname = os.path.dirname(new_mail_home)
+    if not os.path.exists(dirname):
+        try:
+            os.makedirs(dirname)
+        except OSError as e:
+            reason = str(e).decode("utf-8")
+            logger.critical(
+                f"renaming of {operation.argument} to {new_mail_home} failed (reason: {reason})"
+            )
+            return
+    code, output = exec_cmd(f"mv {operation.argument} {new_mail_home}")
+    if code:
+        logger.critical(f"Renaming of {new_mail_home} failed (reason: {output})")
+        return
+    operation.delete()
+
+
+def delete_mailbox(op_id):
+    """Delete the mailbox folder through a RQ Job."""
+    import logging
+    import os
+    import shutil
+
+    from modoboa.admin.app_settings import load_admin_settings
+    from modoboa.admin.models.mailbox import MailboxOperation
+    from modoboa.parameters import tools as param_tools
+
+    logger = logging.getLogger("modoboa.admin")
+
+    load_admin_settings()
+    if not param_tools.get_global_parameter("handle_mailboxes"):
+        return
+
+    try:
+        operation = MailboxOperation.objects.get(pk=op_id)
+    except MailboxOperation.DoesNotExist:
+        logger.error(f"Failed to retrieve operation id {op_id} for deletion")
+        return
+
+    if not os.path.exists(operation.argument):
+        logger.error(f"Failed to delete {operation.argument}, folder not found")
+        operation.delete()
+        return
+
+    def onerror(function, path, excinfo):
+        import logging
+
+        """Handle errors."""
+        logger = logging.getLogger("modoboa.admin")
+        logger.critical(f"delete failed (reason: {excinfo})")
+        operation.delete()
+
+    shutil.rmtree(operation.argument, False, onerror)
+    operation.delete()
+
+
+def handle_remaining_operation():
+    import logging
+
+    import django_rq
+
+    from modoboa.parameters import tools as param_tools
+    from modoboa.admin.app_settings import load_admin_settings
+    from modoboa.admin.models.mailbox import MailboxOperation
+
+    load_admin_settings()
+    if not param_tools.get_global_parameter("handle_mailboxes"):
+        return
+    queue = django_rq.get_queue("dovecot")
+    if queue.started_job_registry.count > 0 or queue.scheduled_job_registry.count > 0:
+        return
+    logger = logging.getLogger("modoboa.admin")
+    for ope in MailboxOperation.objects.all():
+        if ope.type == "rename":
+            rename_mailbox(ope.pk)
+        elif ope.type == "delete":
+            delete_mailbox(ope.pk)

modoboa/admin/management/commands/handle_mailbox_operations.py

@@ -1,103 +0,0 @@
-"""A management command to apply mailbox operations."""
-
-import logging
-import os
-import shutil
-
-from django.core.management.base import BaseCommand
-
-from modoboa.lib.exceptions import InternalError
-from modoboa.lib.sysutils import exec_cmd
-from modoboa.parameters import tools as param_tools
-from ...app_settings import load_admin_settings
-from ...models import MailboxOperation
-
-
-class OperationError(Exception):
-    """Custom exception."""
-
-    pass
-
-
-class Command(BaseCommand):
-    """Command definition"""
-
-    help = "Handles rename and delete operations on mailboxes"  # NOQA:A003
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.logger = logging.getLogger("modoboa.admin")
-
-    def add_arguments(self, parser):
-        """Add extra arguments to command line."""
-        parser.add_argument(
-            "--pidfile",
-            default="/tmp/handle_mailbox_operations.pid",
-            help="Path to the file that will contain the PID of this process",
-        )
-
-    def rename_mailbox(self, operation):
-        if not os.path.exists(operation.argument):
-            return
-        new_mail_home = operation.mailbox.mail_home
-        dirname = os.path.dirname(new_mail_home)
-        if not os.path.exists(dirname):
-            try:
-                os.makedirs(dirname)
-            except OSError as e:
-                raise OperationError(str(e)) from None
-        code, output = exec_cmd(f"mv {operation.argument} {new_mail_home}")
-        if code:
-            raise OperationError(output)
-
-    def delete_mailbox(self, operation):
-        """Try to delete a mailbox tree on filesystem."""
-        if not os.path.exists(operation.argument):
-            return
-
-        def onerror(function, path, excinfo):
-            """Handle errors."""
-            self.logger.critical(f"delete failed (reason: {excinfo})")
-
-        shutil.rmtree(operation.argument, False, onerror)
-
-    def check_pidfile(self, path):
-        """Check if this command is already running
-
-        :param str path: path to the file containing the PID
-        :return: a boolean, True means we can go further
-        """
-        if os.path.exists(path):
-            with open(path) as fp:
-                pid = fp.read().strip()
-            code, output = exec_cmd(
-                f"grep handle_mailbox_operations /proc/{pid}/cmdline"
-            )
-            if not code:
-                return False
-        with open(path, "w") as fp:
-            print(os.getpid(), file=fp)
-        return True
-
-    def handle(self, *args, **options):
-        """Command entry point."""
-        load_admin_settings()
-        if not param_tools.get_global_parameter("handle_mailboxes"):
-            return
-        if not self.check_pidfile(options["pidfile"]):
-            return
-        for ope in MailboxOperation.objects.all():
-            try:
-                f = getattr(self, f"{ope.type}_mailbox")
-            except AttributeError:
-                continue
-            try:
-                f(ope)
-            except (OperationError, InternalError) as e:
-                self.logger.critical(
-                    "%s failed (reason: %s)", ope, str(e).decode("utf-8")
-                )
-            else:
-                self.logger.info("%s succeed", ope)
-                ope.delete()
-        os.unlink(options["pidfile"])

modoboa/admin/models/mailbox.py

@@ -9,12 +9,15 @@ from django.db.models.manager import Manager
 from django.utils.encoding import smart_str, force_str
 from django.utils.translation import gettext as _, gettext_lazy
 
+from modoboa.admin import jobs
 from modoboa.core.models import User
 from modoboa.lib import exceptions as lib_exceptions
 from modoboa.lib.email_utils import split_mailbox
 from modoboa.lib.sysutils import doveadm_cmd
 from modoboa.parameters import tools as param_tools
 
+import django_rq
+
 from .base import AdminObject
 from .domain import Domain
 from . import mixins
@@ -192,9 +195,12 @@ class Mailbox(mixins.MessageLimitMixin, AdminObject):
         hm = param_tools.get_global_parameter("handle_mailboxes", raise_exception=False)
         if not hm:
             return
-        MailboxOperation.objects.create(
+        mail_op = MailboxOperation.objects.create(
             mailbox=self, type="rename", argument=old_mail_home
         )
+        mail_op.save()
+        dovecot_queue = django_rq.get_queue("dovecot")
+        dovecot_queue.enqueue(jobs.rename_mailbox, mail_op.pk)
 
     def rename(self, address, domain):
         """Rename the mailbox.
@@ -223,7 +229,12 @@ class Mailbox(mixins.MessageLimitMixin, AdminObject):
         hm = param_tools.get_global_parameter("handle_mailboxes", raise_exception=False)
         if not hm:
             return
-        MailboxOperation.objects.create(type="delete", argument=self.mail_home)
+        mail_op = MailboxOperation.objects.create(
+            type="delete", argument=self.mail_home
+        )
+        mail_op.save()
+        dovecot_queue = django_rq.get_queue("dovecot")
+        dovecot_queue.enqueue(jobs.delete_mailbox, mail_op.pk)
 
     def set_quota(self, value=None, override_rules=False):
         """Set or update quota value for this mailbox.

test_project/test_project/settings.py

@@ -303,6 +303,9 @@ RQ_QUEUES = {
     "modoboa": {
         "URL": REDIS_URL,
     },
+    "dovecot": {
+        "URL": REDIS_URL,
+    },
 }
 
 # CACHE