From f05478bccd74c920d6a6a0db5632a23ff9d892eb Mon Sep 17 00:00:00 2001 From: Spitap Date: Fri, 23 Jun 2023 17:10:59 +0200 Subject: [PATCH] added rq Co-Authored-By: Antoine Nguyen --- .github/workflows/modoboa.yml | 14 +++++++- doc/manual_installation/modoboa.rst | 32 ++++++++++++++++++- docker-compose.yml | 20 ++++++++++++ docker/Dockerfile.dev | 2 +- modoboa/admin/handlers.py | 13 ++++++++ .../commands/subcommands/_manage_dkim_keys.py | 17 ++++++++++ modoboa/admin/tests/utils.py | 3 ++ .../core/commands/templates/settings.py.tpl | 21 ++++++++++++ modoboa/core/commands/templates/urls.py.tpl | 4 +-- requirements.txt | 6 +++- test_project/test_project/settings.py | 11 ++++++- test_project/test_project/urls.py | 7 ++-- 12 files changed, 138 insertions(+), 12 deletions(-) diff --git a/.github/workflows/modoboa.yml b/.github/workflows/modoboa.yml index edeaf3677..86bd2c383 100644 --- a/.github/workflows/modoboa.yml +++ b/.github/workflows/modoboa.yml @@ -52,6 +52,11 @@ jobs: image: redis ports: - 6379/tcp + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 openldap: image: docker.pkg.github.com/modoboa/docker-openldap/docker-openldap ports: @@ -79,12 +84,17 @@ jobs: run: | sudo apt-get update -y \ && sudo apt-get install -y \ - postfix librrd-dev rrdtool libldap2-dev libsasl2-dev libssl-dev + postfix librrd-dev rrdtool libldap2-dev libsasl2-dev libssl-dev redis-server python -m pip install --upgrade pip pip install -r requirements.txt pip install -r ldap-requirements.txt pip install -r test-requirements.txt python setup.py develop + echo "Testing redis connection" + redis-cli -h $REDIS_HOST -p $REDIS_PORT ping + env: + REDIS_HOST: localhost + REDIS_PORT: ${{ job.services.redis.ports[6379] }} - name: Install postgres requirements if: ${{ matrix.database == 'postgres' }} run: | @@ -109,6 +119,7 @@ jobs: MYSQL_HOST: 127.0.0.1 MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port MYSQL_USER: root + REDIS_HOST: localhost REDIS_PORT: ${{ job.services.redis.ports[6379] }} LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }} @@ -128,6 +139,7 @@ jobs: MYSQL_HOST: 127.0.0.1 MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port MYSQL_USER: root + REDIS_HOST: localhost REDIS_PORT: ${{ job.services.redis.ports[6379] }} LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }} - name: Upload coverage result diff --git a/doc/manual_installation/modoboa.rst b/doc/manual_installation/modoboa.rst index f78ec8aef..463f59d28 100644 --- a/doc/manual_installation/modoboa.rst +++ b/doc/manual_installation/modoboa.rst @@ -248,7 +248,7 @@ following content inside: Please note that above crontab might not be ideal on high load systems. If you receive a fairly high amount of emails per day, you may want to run modoboas logparser tasks *once per night*. - + This has the down side that the statistic graph and message log within the UI are updated once per day only. @@ -286,4 +286,34 @@ But we recommend an automatic start using ``systemd`` or It will listen by default on ``127.0.0.1`` and port ``9999``. The policy daemon won't do anything unless you tell :ref:`postfix ` to use it. + +RQ daemon +--------- + +Modoboa uses `RQ `_ as a job handler for async task. + +A `redis server `_ is required to run this new daemon. + +A worker needs to be launched in the venv. + +You can launch it manually using the following command: + +.. sourcecode:: bash + + (env)> rq worker high default low + +But we recommend an automatic start using ``systemd`` or +``supervisor``. Here is a configuration example for ``supervisor``: + +.. sourcecode:: ini + + [program:modoboa-worker] + autostart=true + autorestart=true + command=/srv/modoboa/env/bin/rq worker high default low + directory=/srv/modoboa + numprocs=1 + stopsignal=TERM + + Now you can continue to the :ref:`webserver` section. diff --git a/docker-compose.yml b/docker-compose.yml index 06011ad22..7de4e402e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,25 @@ version: '3' services: + redis: + image: redis:5-alpine + ports: + - 6379:6379 + + rq: + build: + context: . + dockerfile: docker/Dockerfile.dev + environment: + PYTHONPATH: ".." + DB: "SQLITE" + DEBUG: "True" + REDIS_HOST: redis + command: > + sh -c "cd test_project; python3 manage.py rqworker high default low" + volumes: + - .:/code + api: container_name: modoboa_api restart: unless-stopped @@ -13,6 +32,7 @@ services: PYTHONPATH: ".." DB: "SQLITE" DEBUG: "True" + REDIS_HOST: redis command: > sh -c "cd test_project python3 manage.py migrate diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 8130514fd..ce827da36 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -18,7 +18,7 @@ RUN pip install -r requirements.txt -r test-requirements.txt FROM base as run COPY --from=build $VIRTUAL_ENV $VIRTUAL_ENV -RUN apk add --no-cache --update librrd libxslt libjpeg libressl git rrdtool libmagic \ +RUN apk add --no-cache --update openssl librrd libxslt libjpeg libressl git rrdtool libmagic \ && rm -rf /var/cache/apk/* RUN mkdir /code WORKDIR /code diff --git a/modoboa/admin/handlers.py b/modoboa/admin/handlers.py index d7bfc1361..78ac894e9 100644 --- a/modoboa/admin/handlers.py +++ b/modoboa/admin/handlers.py @@ -1,11 +1,14 @@ """Django signal handlers for admin.""" from django.contrib.contenttypes.models import ContentType +from django.core.management import call_command from django.db.models import signals from django.dispatch import receiver from django.urls import reverse from django.utils.translation import gettext as _ +import django_rq + from modoboa.core import models as core_models, signals as core_signals from modoboa.lib import exceptions, permissions, signals as lib_signals from modoboa.lib.cryptutils import encrypt @@ -44,6 +47,16 @@ def update_domain_mxs_and_mailboxes(sender, instance, **kwargs): alias.save(update_fields=["address"]) +@receiver(signals.post_save, sender=models.Domain) +def create_dkim_key(sender, instance, **kwargs): + if not instance.enable_dkim: + return + django_rq.enqueue(call_command, + "modo", + "manage_dkim_keys", + f"--domain={instance.name}") + + @receiver(signals.post_save, sender=models.DomainAlias) def create_alias_for_domainalias(sender, instance, **kwargs): """Create a dedicated alias for domain alias.""" diff --git a/modoboa/admin/management/commands/subcommands/_manage_dkim_keys.py b/modoboa/admin/management/commands/subcommands/_manage_dkim_keys.py index d3ccdda67..c1b9a0f3d 100644 --- a/modoboa/admin/management/commands/subcommands/_manage_dkim_keys.py +++ b/modoboa/admin/management/commands/subcommands/_manage_dkim_keys.py @@ -56,8 +56,25 @@ class ManageDKIMKeys(BaseCommand): domain.dkim_public_key = public_key domain.save(update_fields=["dkim_public_key", "dkim_private_key_path"]) + def add_arguments(self, parser): + """Add arguments to command.""" + parser.add_argument( + "--domain", type=str, dest="domain", default="", + help="Domain target for keys generation." + ) + def handle(self, *args, **options): """Entry point.""" + if options["domain"] != "": + domain = models.Domain.objects.filter(name=options["domain"], + enable_dkim=True, + dkim_private_key_path="") + if domain.exists(): + self.create_new_dkim_key(domain[0]) + signals.new_dkim_keys.send(sender=self.__class__, + domains=domain) + return + self.default_key_length = param_tools.get_global_parameter( "dkim_default_key_length") qset = models.Domain.objects.filter( diff --git a/modoboa/admin/tests/utils.py b/modoboa/admin/tests/utils.py index b15939d30..0bce09b2c 100644 --- a/modoboa/admin/tests/utils.py +++ b/modoboa/admin/tests/utils.py @@ -2,6 +2,8 @@ import socket +from django.conf import settings + from dns.name import Name from dns.rdtypes.IN.A import A from dns.rdtypes.ANY.MX import MX @@ -67,6 +69,7 @@ _POSSIBLE_DNS_RESULTS_NO_MX = { "mx3.example.com": [_A_RECORD] } _POSSIBLE_IP_RESULTS = { + settings.REDIS_HOST: socket.getaddrinfo(settings.REDIS_HOST, settings.REDIS_PORT), "test3.com": [_IPV4_RECORD_2], "mx2.example.com": [_IPV4_RECORD_2], "bad-response.example.com": [_BAD_IP_RECORD], diff --git a/modoboa/core/commands/templates/settings.py.tpl b/modoboa/core/commands/templates/settings.py.tpl index db2d58e01..7fc8a4a7b 100644 --- a/modoboa/core/commands/templates/settings.py.tpl +++ b/modoboa/core/commands/templates/settings.py.tpl @@ -70,6 +70,7 @@ INSTALLED_APPS = ( 'django_otp.plugins.otp_totp', 'django_otp.plugins.otp_static', 'django_rename_app', + 'django_rq', {% if devmode %} 'djangobower',{% endif %} ) @@ -253,6 +254,26 @@ REDIS_PORT = 6379 REDIS_QUOTA_DB = 0 REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB) +# RQ + +RQ_QUEUES = { + 'default': { + 'HOST': REDIS_HOST, + 'PORT': REDIS_PORT, + 'DB': 0, + }, + 'high': { + 'HOST': REDIS_HOST, + 'PORT': REDIS_PORT, + 'DB': 0, + }, + 'low': { + 'HOST': REDIS_HOST, + 'PORT': REDIS_PORT, + 'DB': 0, + } +} + # Password validation # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators diff --git a/modoboa/core/commands/templates/urls.py.tpl b/modoboa/core/commands/templates/urls.py.tpl index 1a4d5cea7..e3df451c4 100644 --- a/modoboa/core/commands/templates/urls.py.tpl +++ b/modoboa/core/commands/templates/urls.py.tpl @@ -1,5 +1,5 @@ -from django.conf.urls import include, url +from django.conf.urls import include, path urlpatterns = [ - url(r'', include('modoboa.urls')), + path(r'', include('modoboa.urls')), ] diff --git a/requirements.txt b/requirements.txt index 52397ed22..540969d4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ django-rename-app==0.1.5 dj-database-url -djangorestframework>=3.11,<3.15 +djangorestframework>=3.14,<3.15 drf-spectacular uritemplate djangorestframework-simplejwt==5.2.2 @@ -46,3 +46,7 @@ reportlab tldextract>=2.0.2 defusedxml>=0.6.0 python-magic>=0.4.24 + +# RQ +rq>=1.13.0 +django-rq diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py index 78b36070f..0d01a4c13 100644 --- a/test_project/test_project/settings.py +++ b/test_project/test_project/settings.py @@ -73,6 +73,7 @@ INSTALLED_APPS = ( 'django_otp.plugins.otp_totp', 'django_otp.plugins.otp_static', 'django_rename_app', + 'django_rq', ) # A dedicated place to register Modoboa applications @@ -231,11 +232,19 @@ PID_FILE_STORAGE_PATH = '/tmp' # REDIS -REDIS_HOST = 'localhost' +REDIS_HOST = os.environ.get('REDIS_HOST', '127.0.0.1') REDIS_PORT = os.environ.get('REDIS_PORT', 6379) REDIS_QUOTA_DB = 0 REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB) +# RQ + +RQ_QUEUES = { + 'default': { + 'URL': REDIS_URL, + }, +} + # Password validation # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators diff --git a/test_project/test_project/urls.py b/test_project/test_project/urls.py index 7e9306eed..4fcd37753 100644 --- a/test_project/test_project/urls.py +++ b/test_project/test_project/urls.py @@ -1,8 +1,5 @@ -from __future__ import unicode_literals - -from django.conf.urls import include -from django.urls import re_path +from django.urls import include, path urlpatterns = [ - re_path(r"", include("modoboa.urls")), + path("", include("modoboa.urls")), ]