Merge branch '2.3.x'

.github/workflows/modoboa.yml

@@ -5,7 +5,7 @@ name: Modoboa App
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, 2.3.x ]
     paths-ignore:
       - 'modoboa/**.po'
       - 'frontend/**'
@@ -104,7 +104,7 @@ jobs:
         run: |
           pip install -e .[mysql]
           echo "DB=mysql" >> $GITHUB_ENV
-      - name: Test with pytest
+      - name: Run tests
         if: ${{ matrix.python-version != '3.12' || matrix.database != 'postgres' }}
         run: |
           python ./tests.py
@@ -121,7 +121,26 @@ jobs:
           REDIS_PORT: ${{ job.services.redis.ports[6379] }}
           LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }}
 
-      - name: Test with pytest and coverage
+      - name: Start redis sentinel
+        if: ${{ matrix.python-version == '3.12' && matrix.database == 'postgres' }}
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: "./docker/compose.redis-sentinel.yml"
+
+      - name: Run tests againt redis sentinel
+        if: ${{ matrix.python-version == '3.12' && matrix.database == 'postgres' }}
+        run: |
+          cd test_project
+          coverage run manage.py test modoboa.policyd
+        env:
+          POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
+          REDIS_SENTINEL: 1
+          REDIS_SENTINEL_HOST: localhost
+          REDIS_SENTINEL_PORT: 26379
+          REDIS_MASTER: mymaster
+          OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_PRIVATE_KEY }}
+
+      - name: Run tests and coverage
         if: ${{ matrix.python-version == '3.12' && matrix.database == 'postgres' }}
         run: |
           python ./tests.py

doc/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Modoboa documentation build configuration file, created by
 # sphinx-quickstart on Mon Jan  3 22:29:25 2011.

doc/manual_installation/modoboa.rst

@@ -276,6 +276,7 @@ You can launch it manually using the following command:
 
 But we recommend an automatic start using ``systemd`` or
 ``supervisor``. Here is a configuration example for ``supervisor``:
+
 .. sourcecode:: ini
 
    [program:policyd]

docker-compose.yml

@@ -1,3 +1,7 @@
+networks:
+  redis-net:
+    external: true
+
 services:
   redis:
     image: redis:5-alpine
@@ -45,6 +49,9 @@ services:
       - radicale
     volumes:
       - .:/code
+    networks:
+      - default
+      - redis-net
 
   api-unsecured:
     container_name: modoboa_api_unsecured
@@ -105,3 +112,28 @@ services:
       dockerfile: docker/Dockerfile.radicale.dev
     volumes:
       - ./docker/radicale_config:/etc/radicale
+
+  policyd:
+    container_name: modoboa_policyd
+    restart: unless-stopped
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.dev
+    ports:
+      - 9999:9999
+    env_file:
+      - test_project/oidc_key
+    environment:
+      DB: "SQLITE"
+      DEBUG: "True"
+      REDIS_HOST: redis
+      PYTHONPATH: ".."
+      PYTHONUNBUFFERED: 1
+    command: >
+      sh -c "cd test_project
+             python3 manage.py policy_daemon --host 0.0.0.0 --port 9999 --debug"
+    volumes:
+      - .:/code
+    networks:
+      - default
+      - redis-net

docker/compose.redis-sentinel.yml

@@ -0,0 +1,25 @@
+services:
+  redis-master:
+    image: redis:7-alpine
+
+  redis-slave:
+    image: redis:7-alpine
+    command: redis-server --slaveof redis-master 6379
+    depends_on:
+      - redis-master
+
+  redis-sentinel:
+    image: redis:7-alpine
+    depends_on:
+      - redis-master
+      - redis-slave
+    ports:
+      - "26379:26379"
+    command: >
+      sh -c 'echo "bind 0.0.0.0" > /etc/sentinel.conf &&
+            echo "sentinel monitor mymaster redis-master 6379 2" >> /etc/sentinel.conf &&
+            echo "sentinel resolve-hostnames yes" >> /etc/sentinel.conf &&
+            echo "sentinel down-after-milliseconds mymaster 10000" >> /etc/sentinel.conf &&
+            echo "sentinel failover-timeout mymaster 10000" >> /etc/sentinel.conf &&
+            echo "sentinel parallel-syncs mymaster 1" >> /etc/sentinel.conf &&
+            redis-sentinel /etc/sentinel.conf'

modoboa/policyd/core.py

@@ -31,6 +31,15 @@ SUCCESS_ACTION = b"dunno"
 FAILURE_ACTION = b"defer_if_permit Daily limit reached, retry later"
 
 
+def get_redis_client():
+    if not getattr(settings, "REDIS_SENTINEL", False):
+        return aioredis.from_url(
+            settings.REDIS_URL, encoding="utf-8", decode_responses=True
+        )
+    sentinel = aioredis.sentinel.Sentinel(settings.REDIS_SENTINELS, socket_timeout=0.1)
+    return sentinel.master_for(settings.REDIS_MASTER, socket_timeout=0.1)
+
+
 def close_db_connections(func, *args, **kwargs):
     """
     Make sure to close all connections to DB.
@@ -144,9 +153,7 @@ async def apply_policies(attributes):
     sasl_username = attributes.get("sasl_username")
     if not sasl_username:
         return SUCCESS_ACTION
-    rclient = aioredis.from_url(
-        settings.REDIS_URL, encoding="utf-8", decode_responses=True
-    )
+    rclient = get_redis_client()
     decr_domain = False
     decr_user = False
     localpart, domain = split_mailbox(sasl_username)
@@ -200,16 +207,14 @@ async def handle_connection(reader, writer):
 
 
 async def new_connection(reader, writer):
-    try:
-        await asyncio.wait_for(handle_connection(reader, writer), timeout=5)
-    except asyncio.TimeoutError as err:
-        logger.warning("Timeout received while handling connection: %s", err)
-    finally:
-        writer.close()
-        if hasattr(writer, "wait_closed"):
-            # Python 3.7+ only
-            await writer.wait_closed()
-        logger.info("exit")
+    # try:
+    await asyncio.wait_for(handle_connection(reader, writer), timeout=5)
+    # except asyncio.TimeoutError as err:
+    #    logger.warning(f"Timeout received while handling connection: {err}")
+    # finally:
+    writer.close()
+    await writer.wait_closed()
+    logger.info("exit")
 
 
 def get_next_execution_dt():
@@ -255,9 +260,7 @@ def get_mailboxes_to_reset():
 
 async def reset_counters():
     """Reset all counters."""
-    rclient = aioredis.from_url(
-        settings.REDIS_URL, encoding="utf-8", decode_responses=True
-    )
+    rclient = get_redis_client()
     logger.info("Resetting all counters")
     for domain in await get_domains_to_reset():
         await rclient.hset(constants.REDIS_HASHNAME, domain.name, domain.message_limit)

modoboa/policyd/handlers.py

@@ -1,14 +1,12 @@
 """App related signal handlers."""
 
-import redis
-
-from django.conf import settings
 from django.db.models import signals
 from django.dispatch import receiver
 
 from modoboa.admin import models as admin_models
 
 from . import constants
+from . import utils
 
 
 def set_message_limit(instance, key):
@@ -16,9 +14,7 @@ def set_message_limit(instance, key):
     old_message_limit = instance._loaded_values.get("message_limit")
     if old_message_limit == instance.message_limit:
         return
-    rclient = redis.Redis(
-        host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_QUOTA_DB
-    )
+    rclient = utils.get_redis_connection()
     if instance.message_limit is None:
         # delete existing key
         if rclient.hexists(constants.REDIS_HASHNAME, key):

modoboa/policyd/tests.py

@@ -7,10 +7,8 @@ from multiprocessing import Process
 import socket
 import time
 
-import redis
 
 from django import db
-from django.conf import settings
 from django.core.management import call_command
 from django.test import TransactionTestCase
 from django.urls import reverse
@@ -22,6 +20,7 @@ from modoboa.lib.tests import ModoTestCase, ParametersMixin
 from modoboa.policyd import core as policyd_core
 
 from . import constants
+from . import utils
 
 
 def start_policy_daemon():
@@ -33,12 +32,7 @@ class RedisTestCaseMixin:
 
     def setUp(self):
         super().setUp()
-        self.rclient = redis.Redis(
-            host=settings.REDIS_HOST,
-            port=settings.REDIS_PORT,
-            db=settings.REDIS_QUOTA_DB,
-        )
-        self.rclient.set_response_callback("HGET", int)
+        self.rclient = utils.get_redis_connection()
         self.rclient.delete(constants.REDIS_HASHNAME)
 
 

modoboa/policyd/utils.py

@@ -9,9 +9,15 @@ from . import constants
 
 def get_redis_connection():
     """Return a client connection to Redis server."""
-    rclient = redis.Redis(
-        host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_QUOTA_DB
-    )
+    if not getattr(settings, "REDIS_SENTINEL", False):
+        rclient = redis.Redis(
+            host=settings.REDIS_HOST,
+            port=settings.REDIS_PORT,
+            db=settings.REDIS_QUOTA_DB,
+        )
+    else:
+        sentinel = redis.sentinel.Sentinel(settings.REDIS_SENTINELS, socket_timeout=0.1)
+        rclient = sentinel.master_for(settings.REDIS_MASTER, socket_timeout=0.1)
     rclient.set_response_callback("HGET", int)
     return rclient
 

test_project/envfile

@@ -0,0 +1,2 @@
+WEBMAIL_DEV_USERNAME=tonio@ngyn.org
+WEBMAIL_DEV_PASSWORD=Aka/12ctogael

test_project/manage.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 
-from __future__ import unicode_literals
 
 import os
 import sys
@@ -20,6 +19,6 @@ if __name__ == "__main__":
                 "Couldn't import Django. Are you sure it's installed and "
                 "available on your PYTHONPATH environment variable? Did you "
                 "forget to activate a virtual environment?"
-            )
+            ) from None
         raise
     execute_from_command_line(sys.argv)

test_project/test_project/settings.py

@@ -274,10 +274,19 @@ PID_FILE_STORAGE_PATH = "/tmp"
 
 # REDIS
 
+REDIS_SENTINEL = bool(os.environ.get("REDIS_SENTINEL", False))
+REDIS_SENTINELS = [
+    (
+        os.environ.get("REDIS_SENTINEL_HOST", "127.0.0.1"),
+        os.environ.get("REDIS_SENTINEL_PORT", 26379),
+    )
+]
+REDIS_MASTER = os.environ.get("REDIS_MASTER", "mymaster")
+
 REDIS_HOST = os.environ.get("REDIS_HOST", "127.0.0.1")
 REDIS_PORT = os.environ.get("REDIS_PORT", 6379)
 REDIS_QUOTA_DB = 0
-REDIS_URL = "redis://{}:{}/{}".format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
+REDIS_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_QUOTA_DB}"
 
 # RQ
 

test_project/test_project/wsgi.py

@@ -7,8 +7,6 @@ For more information on this file, see
 https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
 """
 
-from __future__ import unicode_literals
-
 import os
 
 from django.core.wsgi import get_wsgi_application