Merge pull request #2983 from modoboa/rq-integration

RQ implementation

.github/workflows/modoboa.yml

@@ -52,6 +52,11 @@ jobs:
         image: redis
         ports:
           - 6379/tcp
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
       openldap:
         image: docker.pkg.github.com/modoboa/docker-openldap/docker-openldap
         ports:
@@ -79,12 +84,17 @@ jobs:
         run: |
           sudo apt-get update -y \
           && sudo apt-get install -y \
-            postfix librrd-dev rrdtool libldap2-dev libsasl2-dev libssl-dev
+            postfix librrd-dev rrdtool libldap2-dev libsasl2-dev libssl-dev redis-server
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r ldap-requirements.txt
           pip install -r test-requirements.txt
           python setup.py develop
+          echo "Testing redis connection"
+          redis-cli -h $REDIS_HOST -p $REDIS_PORT ping
+        env:
+          REDIS_HOST: localhost
+          REDIS_PORT: ${{ job.services.redis.ports[6379] }}
       - name: Install postgres requirements
         if: ${{ matrix.database == 'postgres' }}
         run: |
@@ -109,6 +119,7 @@ jobs:
           MYSQL_HOST: 127.0.0.1
           MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port
           MYSQL_USER: root
+          REDIS_HOST: localhost
           REDIS_PORT: ${{ job.services.redis.ports[6379] }}
           LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }}
 
@@ -128,6 +139,7 @@ jobs:
           MYSQL_HOST: 127.0.0.1
           MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port
           MYSQL_USER: root
+          REDIS_HOST: localhost
           REDIS_PORT: ${{ job.services.redis.ports[6379] }}
           LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }}
       - name: Upload coverage result

doc/manual_installation/modoboa.rst

@@ -248,7 +248,7 @@ following content inside:
    Please note that above crontab might not be ideal on high load systems.
    If you receive a fairly high amount of emails per day, you may want to
    run modoboas logparser tasks *once per night*.
-   
+
    This has the down side that the statistic graph and message log within
    the UI are updated once per day only.
 
@@ -286,4 +286,34 @@ But we recommend an automatic start using ``systemd`` or
 It will listen by default on ``127.0.0.1`` and port ``9999``. The
 policy daemon won't do anything unless you tell :ref:`postfix <policyd_config>` to use it.
 
+
+RQ daemon
+---------
+
+Modoboa uses `RQ <https://python-rq.org>`_ as a job handler for async task.
+
+A `redis server <https://redis.io/>`_ is required to run this new daemon.
+
+A worker needs to be launched in the venv.
+
+You can launch it manually using the following command:
+
+.. sourcecode:: bash
+
+   (env)> rq worker high default low
+
+But we recommend an automatic start using ``systemd`` or
+``supervisor``. Here is a configuration example for ``supervisor``:
+
+.. sourcecode:: ini
+
+   [program:modoboa-worker]
+   autostart=true
+   autorestart=true
+   command=/srv/modoboa/env/bin/python /srv/modoboa/instance/manage.py worker high default low
+   directory=/srv/modoboa
+   numprocs=1
+   stopsignal=TERM
+
+
 Now you can continue to the :ref:`webserver` section.

doc/upgrade.rst

@@ -5,6 +5,11 @@ Upgrade
 Modoboa
 *******
 
+.. note::
+   In this doc, ``by default`` mean that you used `modoboa installer
+   <https://github.com/modoboa/modoboa-installer>`_ to install modoboa
+   and that your didn't change your configuration.
+
 .. warning::
 
    The new version you are going to install may need to modify your
@@ -124,6 +129,69 @@ If you use Postgresql, you need to install pyscopg3+:
 
    pip install psycopg[binary]>=3.1
 
+RQ has been added. This should replace the use of cron jobs in the future.
+For now, only ``manage_dkim_keys`` has been migrated. This will make the dkim
+key generation asynchronious but the task will be started as soon as the generation
+is required.
+
+Follow these instructions to perform the update in case you used supervisord for the
+installation (this apply if you used ``modoboa-installer``):
+
+1. Edit settings.py (``/srv/modoboa/instance/instance/settings.py`` by default) and add:
+
+.. sourcecode:: python
+   INSTALLED_APPS = (
+   ...,
+   'django_rq',
+   )
+
+and add the RQ section bellow the ``#REDIS`` section:
+
+.. sourcecode:: python
+   # RQ
+
+   RQ_QUEUES = {
+      'default': {
+         'HOST': REDIS_HOST,
+         'PORT': REDIS_PORT,
+         'DB': 0,
+      },
+      'high': {
+         'HOST': REDIS_HOST,
+         'PORT': REDIS_PORT,
+         'DB': 0,
+      },
+      'low': {
+         'HOST': REDIS_HOST,
+         'PORT': REDIS_PORT,
+         'DB': 0,
+      }
+   }
+
+Then by default you will need to restart ``uwsgi`` service. ``systemctl restart uwsgi`` on Debian.
+
+2. Create a new supervisord config (``/etc/supervisor/conf.d/modoboaworkerd.conf`` by default) :
+
+.. sourcecode:: ini
+   [program:modoboa-worker]
+   autostart=true
+   autorestart=true
+   command={%python env path%} {% manage.py instance path%} worker high default low
+   directory={%modoboa home dir%}
+   numprocs=1
+   stopsignal=TERM
+
+``python env path`` : Python executable located in your virtual environment created for modoboa.
+You will find it here  ``/srv/modoboa/venv/bin/python`` by default.
+``manage.py instance path``: Path to manage.py of your modoboa instance.
+You will find it here : ``/srv/modoboa/instance/manage.py``by default.
+``Modoboa home dir``: Home dir of the user running modooba.
+You will find it here ``/srv/modoboa/`` by default.
+
+You can help you with ``/etc/supervisor/conf.d/policyd.conf`` (by default).
+
+Then restart supervisor. ``#> supervisorctl reread && supervisorctl update`` on Debian.
+
 
 2.1.0
 =====

docker-compose.yml

@@ -1,6 +1,25 @@
 version: '3'
 
 services:
+  redis:
+    image: redis:5-alpine
+    ports:
+      - 6379:6379
+
+  rq:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.dev
+    environment:
+      PYTHONPATH: ".."
+      DB: "SQLITE"
+      DEBUG: "True"
+      REDIS_HOST: redis
+    command: >
+      sh -c "cd test_project; python3 manage.py rqworker high default low"
+    volumes:
+      - .:/code
+
   api:
     container_name: modoboa_api
     restart: unless-stopped
@@ -13,6 +32,7 @@ services:
       PYTHONPATH: ".."
       DB: "SQLITE"
       DEBUG: "True"
+      REDIS_HOST: redis
     command: >
       sh -c "cd test_project
              python3 manage.py migrate

docker/Dockerfile.dev

@@ -18,7 +18,7 @@ RUN pip install -r requirements.txt -r test-requirements.txt
 
 FROM base as run
 COPY --from=build $VIRTUAL_ENV $VIRTUAL_ENV
-RUN apk add --no-cache --update librrd libxslt libjpeg libressl git rrdtool libmagic \
+RUN apk add --no-cache --update openssl librrd libxslt libjpeg libressl git rrdtool libmagic \
     && rm -rf /var/cache/apk/*
 RUN mkdir /code
 WORKDIR /code

modoboa/admin/handlers.py

@@ -1,11 +1,14 @@
 """Django signal handlers for admin."""
 
 from django.contrib.contenttypes.models import ContentType
+from django.core.management import call_command
 from django.db.models import signals
 from django.dispatch import receiver
 from django.urls import reverse
 from django.utils.translation import gettext as _
 
+import django_rq
+
 from modoboa.core import models as core_models, signals as core_signals
 from modoboa.lib import exceptions, permissions, signals as lib_signals
 from modoboa.lib.cryptutils import encrypt
@@ -44,6 +47,16 @@ def update_domain_mxs_and_mailboxes(sender, instance, **kwargs):
         alias.save(update_fields=["address"])
 
 
+@receiver(signals.post_save, sender=models.Domain)
+def create_dkim_key(sender, instance, **kwargs):
+    if not instance.enable_dkim:
+        return
+    django_rq.enqueue(call_command,
+                      "modo",
+                      "manage_dkim_keys",
+                      f"--domain={instance.name}")
+
+
 @receiver(signals.post_save, sender=models.DomainAlias)
 def create_alias_for_domainalias(sender, instance, **kwargs):
     """Create a dedicated alias for domain alias."""

modoboa/admin/management/commands/subcommands/_manage_dkim_keys.py

@@ -56,10 +56,28 @@ class ManageDKIMKeys(BaseCommand):
         domain.dkim_public_key = public_key
         domain.save(update_fields=["dkim_public_key", "dkim_private_key_path"])
 
+    def add_arguments(self, parser):
+        """Add arguments to command."""
+        parser.add_argument(
+            "--domain", type=str, dest="domain", default="",
+            help="Domain target for keys generation."
+        )
+
     def handle(self, *args, **options):
         """Entry point."""
         self.default_key_length = param_tools.get_global_parameter(
             "dkim_default_key_length")
+
+        if options["domain"] != "":
+            domain = models.Domain.objects.filter(name=options["domain"],
+                                                  enable_dkim=True,
+                                                  dkim_private_key_path="")
+            if domain.exists():
+                self.create_new_dkim_key(domain[0])
+                signals.new_dkim_keys.send(sender=self.__class__,
+                                           domains=domain)
+            return
+
         qset = models.Domain.objects.filter(
             enable_dkim=True, dkim_private_key_path="")
         for domain in qset:

modoboa/admin/tests/test_domain.py

@@ -470,12 +470,21 @@ class DKIMTestCase(ModoTestCase):
             constants.ALARM_CLOSED)
         key_path = os.path.join(self.workdir, "{}.pem".format(values["name"]))
         self.assertTrue(os.path.exists(key_path))
-        domain = Domain.objects.get(name=values["name"])
+
+        domain.refresh_from_db()
         url = reverse("admin:domain_detail", args=[domain.pk])
         response = self.client.get(url)
         self.assertContains(
             response, escape(domain.bind_format_dkim_public_key))
 
+        # Try generating DKIM key for a targetted domain
+        domain.dkim_private_key_path = ""
+        domain.save()
+
+        call_command("modo", "manage_dkim_keys", f"--domain={domain.name}")
+        domain.refresh_from_db()
+        self.assertNotEqual(domain.dkim_private_key_path, "")
+
     def test_dkim_key_length_modification(self):
         """ """
         self.set_global_parameter("dkim_keys_storage_dir", self.workdir)

modoboa/admin/tests/utils.py

@@ -2,6 +2,8 @@
 
 import socket
 
+from django.conf import settings
+
 from dns.name import Name
 from dns.rdtypes.IN.A import A
 from dns.rdtypes.ANY.MX import MX
@@ -67,6 +69,7 @@ _POSSIBLE_DNS_RESULTS_NO_MX = {
     "mx3.example.com": [_A_RECORD]
 }
 _POSSIBLE_IP_RESULTS = {
+    settings.REDIS_HOST: socket.getaddrinfo(settings.REDIS_HOST, settings.REDIS_PORT),
     "test3.com": [_IPV4_RECORD_2],
     "mx2.example.com": [_IPV4_RECORD_2],
     "bad-response.example.com": [_BAD_IP_RECORD],

modoboa/core/commands/templates/settings.py.tpl

@@ -70,6 +70,7 @@ INSTALLED_APPS = (
     'django_otp.plugins.otp_totp',
     'django_otp.plugins.otp_static',
     'django_rename_app',
+    'django_rq',
 {% if devmode %}    'djangobower',{% endif %}
 )
 
@@ -253,6 +254,26 @@ REDIS_PORT = 6379
 REDIS_QUOTA_DB = 0
 REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
 
+# RQ
+
+RQ_QUEUES = {
+    'default': {
+        'HOST': REDIS_HOST,
+        'PORT': REDIS_PORT,
+        'DB': 0,
+    },
+    'high': {
+        'HOST': REDIS_HOST,
+        'PORT': REDIS_PORT,
+        'DB': 0,
+    },
+    'low': {
+        'HOST': REDIS_HOST,
+        'PORT': REDIS_PORT,
+        'DB': 0,
+    }
+}
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 

modoboa/core/commands/templates/urls.py.tpl

@@ -1,5 +1,5 @@
-from django.conf.urls import include, url
+from django.conf.urls import include, path
 
 urlpatterns = [
-    url(r'', include('modoboa.urls')),
+    path(r'', include('modoboa.urls')),
 ]

requirements.txt

@@ -9,7 +9,7 @@ django-rename-app==0.1.5
 
 dj-database-url
 
-djangorestframework>=3.11,<3.15
+djangorestframework>=3.14,<3.15
 drf-spectacular
 uritemplate
 djangorestframework-simplejwt==5.2.2
@@ -46,3 +46,7 @@ reportlab
 tldextract>=2.0.2
 defusedxml>=0.6.0
 python-magic>=0.4.24
+
+# RQ
+rq>=1.13.0
+django-rq

test_project/test_project/settings.py

@@ -73,6 +73,7 @@ INSTALLED_APPS = (
     'django_otp.plugins.otp_totp',
     'django_otp.plugins.otp_static',
     'django_rename_app',
+    'django_rq',
 )
 
 # A dedicated place to register Modoboa applications
@@ -231,11 +232,25 @@ PID_FILE_STORAGE_PATH = '/tmp'
 
 # REDIS
 
-REDIS_HOST = 'localhost'
+REDIS_HOST = os.environ.get('REDIS_HOST', '127.0.0.1')
 REDIS_PORT = os.environ.get('REDIS_PORT', 6379)
 REDIS_QUOTA_DB = 0
 REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
 
+# RQ
+
+RQ_QUEUES = {
+    'default': {
+        'URL': REDIS_URL,
+    },
+    'high': {
+         'URL': REDIS_URL,
+    },
+    'low': {
+         'URL': REDIS_URL,
+    }
+}
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 

test_project/test_project/urls.py

@@ -1,8 +1,5 @@
-from __future__ import unicode_literals
-
-from django.conf.urls import include
-from django.urls import re_path
+from django.urls import include, path
 
 urlpatterns = [
-    re_path(r"", include("modoboa.urls")),
+    path("", include("modoboa.urls")),
 ]