added rq

Co-Authored-By: Antoine Nguyen <tonio@ngyn.org>

.github/workflows/modoboa.yml

@@ -52,6 +52,11 @@ jobs:
         image: redis
         ports:
           - 6379/tcp
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
       openldap:
         image: docker.pkg.github.com/modoboa/docker-openldap/docker-openldap
         ports:
@@ -79,12 +84,17 @@ jobs:
         run: |
           sudo apt-get update -y \
           && sudo apt-get install -y \
-            postfix librrd-dev rrdtool libldap2-dev libsasl2-dev libssl-dev
+            postfix librrd-dev rrdtool libldap2-dev libsasl2-dev libssl-dev redis-server
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r ldap-requirements.txt
           pip install -r test-requirements.txt
           python setup.py develop
+          echo "Testing redis connection"
+          redis-cli -h $REDIS_HOST -p $REDIS_PORT ping
+        env:
+          REDIS_HOST: localhost
+          REDIS_PORT: ${{ job.services.redis.ports[6379] }}
       - name: Install postgres requirements
         if: ${{ matrix.database == 'postgres' }}
         run: |
@@ -109,6 +119,7 @@ jobs:
           MYSQL_HOST: 127.0.0.1
           MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port
           MYSQL_USER: root
+          REDIS_HOST: localhost
           REDIS_PORT: ${{ job.services.redis.ports[6379] }}
           LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }}
 
@@ -128,6 +139,7 @@ jobs:
           MYSQL_HOST: 127.0.0.1
           MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port
           MYSQL_USER: root
+          REDIS_HOST: localhost
           REDIS_PORT: ${{ job.services.redis.ports[6379] }}
           LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }}
       - name: Upload coverage result

doc/manual_installation/modoboa.rst

@@ -248,7 +248,7 @@ following content inside:
    Please note that above crontab might not be ideal on high load systems.
    If you receive a fairly high amount of emails per day, you may want to
    run modoboas logparser tasks *once per night*.
-   
+
    This has the down side that the statistic graph and message log within
    the UI are updated once per day only.
 
@@ -286,4 +286,34 @@ But we recommend an automatic start using ``systemd`` or
 It will listen by default on ``127.0.0.1`` and port ``9999``. The
 policy daemon won't do anything unless you tell :ref:`postfix <policyd_config>` to use it.
 
+
+RQ daemon
+---------
+
+Modoboa uses `RQ <https://python-rq.org>`_ as a job handler for async task.
+
+A `redis server <https://redis.io/>`_ is required to run this new daemon.
+
+A worker needs to be launched in the venv.
+
+You can launch it manually using the following command:
+
+.. sourcecode:: bash
+
+   (env)> rq worker high default low
+
+But we recommend an automatic start using ``systemd`` or
+``supervisor``. Here is a configuration example for ``supervisor``:
+
+.. sourcecode:: ini
+
+   [program:modoboa-worker]
+   autostart=true
+   autorestart=true
+   command=/srv/modoboa/env/bin/rq worker high default low
+   directory=/srv/modoboa
+   numprocs=1
+   stopsignal=TERM
+
+
 Now you can continue to the :ref:`webserver` section.

docker-compose.yml

@@ -1,6 +1,25 @@
 version: '3'
 
 services:
+  redis:
+    image: redis:5-alpine
+    ports:
+      - 6379:6379
+
+  rq:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.dev
+    environment:
+      PYTHONPATH: ".."
+      DB: "SQLITE"
+      DEBUG: "True"
+      REDIS_HOST: redis
+    command: >
+      sh -c "cd test_project; python3 manage.py rqworker high default low"
+    volumes:
+      - .:/code
+
   api:
     container_name: modoboa_api
     restart: unless-stopped
@@ -13,6 +32,7 @@ services:
       PYTHONPATH: ".."
       DB: "SQLITE"
       DEBUG: "True"
+      REDIS_HOST: redis
     command: >
       sh -c "cd test_project
              python3 manage.py migrate

docker/Dockerfile.dev

@@ -18,7 +18,7 @@ RUN pip install -r requirements.txt -r test-requirements.txt
 
 FROM base as run
 COPY --from=build $VIRTUAL_ENV $VIRTUAL_ENV
-RUN apk add --no-cache --update librrd libxslt libjpeg libressl git rrdtool libmagic \
+RUN apk add --no-cache --update openssl librrd libxslt libjpeg libressl git rrdtool libmagic \
     && rm -rf /var/cache/apk/*
 RUN mkdir /code
 WORKDIR /code

modoboa/admin/handlers.py

@@ -1,11 +1,14 @@
 """Django signal handlers for admin."""
 
 from django.contrib.contenttypes.models import ContentType
+from django.core.management import call_command
 from django.db.models import signals
 from django.dispatch import receiver
 from django.urls import reverse
 from django.utils.translation import gettext as _
 
+import django_rq
+
 from modoboa.core import models as core_models, signals as core_signals
 from modoboa.lib import exceptions, permissions, signals as lib_signals
 from modoboa.lib.cryptutils import encrypt
@@ -44,6 +47,16 @@ def update_domain_mxs_and_mailboxes(sender, instance, **kwargs):
         alias.save(update_fields=["address"])
 
 
+@receiver(signals.post_save, sender=models.Domain)
+def create_dkim_key(sender, instance, **kwargs):
+    if not instance.enable_dkim:
+        return
+    django_rq.enqueue(call_command,
+                      "modo",
+                      "manage_dkim_keys",
+                      f"--domain={instance.name}")
+
+
 @receiver(signals.post_save, sender=models.DomainAlias)
 def create_alias_for_domainalias(sender, instance, **kwargs):
     """Create a dedicated alias for domain alias."""

modoboa/admin/management/commands/subcommands/_manage_dkim_keys.py

@@ -56,8 +56,25 @@ class ManageDKIMKeys(BaseCommand):
         domain.dkim_public_key = public_key
         domain.save(update_fields=["dkim_public_key", "dkim_private_key_path"])
 
+    def add_arguments(self, parser):
+        """Add arguments to command."""
+        parser.add_argument(
+            "--domain", type=str, dest="domain", default="",
+            help="Domain target for keys generation."
+        )
+
     def handle(self, *args, **options):
         """Entry point."""
+        if options["domain"] != "":
+            domain = models.Domain.objects.filter(name=options["domain"],
+                                                  enable_dkim=True,
+                                                  dkim_private_key_path="")
+            if domain.exists():
+                self.create_new_dkim_key(domain[0])
+                signals.new_dkim_keys.send(sender=self.__class__,
+                                           domains=domain)
+            return
+
         self.default_key_length = param_tools.get_global_parameter(
             "dkim_default_key_length")
         qset = models.Domain.objects.filter(

modoboa/admin/tests/utils.py

@@ -2,6 +2,8 @@
 
 import socket
 
+from django.conf import settings
+
 from dns.name import Name
 from dns.rdtypes.IN.A import A
 from dns.rdtypes.ANY.MX import MX
@@ -67,6 +69,7 @@ _POSSIBLE_DNS_RESULTS_NO_MX = {
     "mx3.example.com": [_A_RECORD]
 }
 _POSSIBLE_IP_RESULTS = {
+    settings.REDIS_HOST: socket.getaddrinfo(settings.REDIS_HOST, settings.REDIS_PORT),
     "test3.com": [_IPV4_RECORD_2],
     "mx2.example.com": [_IPV4_RECORD_2],
     "bad-response.example.com": [_BAD_IP_RECORD],

modoboa/core/commands/templates/settings.py.tpl

@@ -70,6 +70,7 @@ INSTALLED_APPS = (
     'django_otp.plugins.otp_totp',
     'django_otp.plugins.otp_static',
     'django_rename_app',
+    'django_rq',
 {% if devmode %}    'djangobower',{% endif %}
 )
 
@@ -253,6 +254,26 @@ REDIS_PORT = 6379
 REDIS_QUOTA_DB = 0
 REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
 
+# RQ
+
+RQ_QUEUES = {
+    'default': {
+        'HOST': REDIS_HOST,
+        'PORT': REDIS_PORT,
+        'DB': 0,
+    },
+    'high': {
+        'HOST': REDIS_HOST,
+        'PORT': REDIS_PORT,
+        'DB': 0,
+    },
+    'low': {
+        'HOST': REDIS_HOST,
+        'PORT': REDIS_PORT,
+        'DB': 0,
+    }
+}
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 

modoboa/core/commands/templates/urls.py.tpl

@@ -1,5 +1,5 @@
-from django.conf.urls import include, url
+from django.conf.urls import include, path
 
 urlpatterns = [
-    url(r'', include('modoboa.urls')),
+    path(r'', include('modoboa.urls')),
 ]

requirements.txt

@@ -9,7 +9,7 @@ django-rename-app==0.1.5
 
 dj-database-url
 
-djangorestframework>=3.11,<3.15
+djangorestframework>=3.14,<3.15
 drf-spectacular
 uritemplate
 djangorestframework-simplejwt==5.2.2
@@ -46,3 +46,7 @@ reportlab
 tldextract>=2.0.2
 defusedxml>=0.6.0
 python-magic>=0.4.24
+
+# RQ
+rq>=1.13.0
+django-rq

test_project/test_project/settings.py

@@ -73,6 +73,7 @@ INSTALLED_APPS = (
     'django_otp.plugins.otp_totp',
     'django_otp.plugins.otp_static',
     'django_rename_app',
+    'django_rq',
 )
 
 # A dedicated place to register Modoboa applications
@@ -231,11 +232,19 @@ PID_FILE_STORAGE_PATH = '/tmp'
 
 # REDIS
 
-REDIS_HOST = 'localhost'
+REDIS_HOST = os.environ.get('REDIS_HOST', '127.0.0.1')
 REDIS_PORT = os.environ.get('REDIS_PORT', 6379)
 REDIS_QUOTA_DB = 0
 REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
 
+# RQ
+
+RQ_QUEUES = {
+    'default': {
+        'URL': REDIS_URL,
+    },
+}
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 

test_project/test_project/urls.py

@@ -1,8 +1,5 @@
-from __future__ import unicode_literals
-
-from django.conf.urls import include
-from django.urls import re_path
+from django.urls import include, path
 
 urlpatterns = [
-    re_path(r"", include("modoboa.urls")),
+    path("", include("modoboa.urls")),
 ]