diff --git a/.github/workflows/modoboa.yml b/.github/workflows/modoboa.yml index c7c3c4c7d..a411935b0 100644 --- a/.github/workflows/modoboa.yml +++ b/.github/workflows/modoboa.yml @@ -5,7 +5,7 @@ name: Modoboa App on: push: - branches: [ master ] + branches: [ master, 2.3.x ] paths-ignore: - 'modoboa/**.po' - 'frontend/**' @@ -104,7 +104,7 @@ jobs: run: | pip install -e .[mysql] echo "DB=mysql" >> $GITHUB_ENV - - name: Test with pytest + - name: Run tests if: ${{ matrix.python-version != '3.12' || matrix.database != 'postgres' }} run: | python ./tests.py @@ -121,7 +121,26 @@ jobs: REDIS_PORT: ${{ job.services.redis.ports[6379] }} LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }} - - name: Test with pytest and coverage + - name: Start redis sentinel + if: ${{ matrix.python-version == '3.12' && matrix.database == 'postgres' }} + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: "./docker/compose.redis-sentinel.yml" + + - name: Run tests againt redis sentinel + if: ${{ matrix.python-version == '3.12' && matrix.database == 'postgres' }} + run: | + cd test_project + coverage run manage.py test modoboa.policyd + env: + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + REDIS_SENTINEL: 1 + REDIS_SENTINEL_HOST: localhost + REDIS_SENTINEL_PORT: 26379 + REDIS_MASTER: mymaster + OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_PRIVATE_KEY }} + + - name: Run tests and coverage if: ${{ matrix.python-version == '3.12' && matrix.database == 'postgres' }} run: | python ./tests.py diff --git a/doc/conf.py b/doc/conf.py index 7abc815ee..871548c14 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Modoboa documentation build configuration file, created by # sphinx-quickstart on Mon Jan 3 22:29:25 2011. diff --git a/doc/manual_installation/modoboa.rst b/doc/manual_installation/modoboa.rst index 9cfea6a91..a691af193 100644 --- a/doc/manual_installation/modoboa.rst +++ b/doc/manual_installation/modoboa.rst @@ -276,6 +276,7 @@ You can launch it manually using the following command: But we recommend an automatic start using ``systemd`` or ``supervisor``. Here is a configuration example for ``supervisor``: + .. sourcecode:: ini [program:policyd] diff --git a/docker-compose.yml b/docker-compose.yml index b37488bee..f187ef2b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,7 @@ +networks: + redis-net: + external: true + services: redis: image: redis:5-alpine @@ -45,6 +49,9 @@ services: - radicale volumes: - .:/code + networks: + - default + - redis-net api-unsecured: container_name: modoboa_api_unsecured @@ -105,3 +112,28 @@ services: dockerfile: docker/Dockerfile.radicale.dev volumes: - ./docker/radicale_config:/etc/radicale + + policyd: + container_name: modoboa_policyd + restart: unless-stopped + build: + context: . + dockerfile: docker/Dockerfile.dev + ports: + - 9999:9999 + env_file: + - test_project/oidc_key + environment: + DB: "SQLITE" + DEBUG: "True" + REDIS_HOST: redis + PYTHONPATH: ".." + PYTHONUNBUFFERED: 1 + command: > + sh -c "cd test_project + python3 manage.py policy_daemon --host 0.0.0.0 --port 9999 --debug" + volumes: + - .:/code + networks: + - default + - redis-net diff --git a/docker/compose.redis-sentinel.yml b/docker/compose.redis-sentinel.yml new file mode 100644 index 000000000..3fed7b41b --- /dev/null +++ b/docker/compose.redis-sentinel.yml @@ -0,0 +1,25 @@ +services: + redis-master: + image: redis:7-alpine + + redis-slave: + image: redis:7-alpine + command: redis-server --slaveof redis-master 6379 + depends_on: + - redis-master + + redis-sentinel: + image: redis:7-alpine + depends_on: + - redis-master + - redis-slave + ports: + - "26379:26379" + command: > + sh -c 'echo "bind 0.0.0.0" > /etc/sentinel.conf && + echo "sentinel monitor mymaster redis-master 6379 2" >> /etc/sentinel.conf && + echo "sentinel resolve-hostnames yes" >> /etc/sentinel.conf && + echo "sentinel down-after-milliseconds mymaster 10000" >> /etc/sentinel.conf && + echo "sentinel failover-timeout mymaster 10000" >> /etc/sentinel.conf && + echo "sentinel parallel-syncs mymaster 1" >> /etc/sentinel.conf && + redis-sentinel /etc/sentinel.conf' diff --git a/modoboa/policyd/core.py b/modoboa/policyd/core.py index e914ae24f..4d4bfcfe7 100644 --- a/modoboa/policyd/core.py +++ b/modoboa/policyd/core.py @@ -31,6 +31,15 @@ SUCCESS_ACTION = b"dunno" FAILURE_ACTION = b"defer_if_permit Daily limit reached, retry later" +def get_redis_client(): + if not getattr(settings, "REDIS_SENTINEL", False): + return aioredis.from_url( + settings.REDIS_URL, encoding="utf-8", decode_responses=True + ) + sentinel = aioredis.sentinel.Sentinel(settings.REDIS_SENTINELS, socket_timeout=0.1) + return sentinel.master_for(settings.REDIS_MASTER, socket_timeout=0.1) + + def close_db_connections(func, *args, **kwargs): """ Make sure to close all connections to DB. @@ -144,9 +153,7 @@ async def apply_policies(attributes): sasl_username = attributes.get("sasl_username") if not sasl_username: return SUCCESS_ACTION - rclient = aioredis.from_url( - settings.REDIS_URL, encoding="utf-8", decode_responses=True - ) + rclient = get_redis_client() decr_domain = False decr_user = False localpart, domain = split_mailbox(sasl_username) @@ -200,16 +207,14 @@ async def handle_connection(reader, writer): async def new_connection(reader, writer): - try: - await asyncio.wait_for(handle_connection(reader, writer), timeout=5) - except asyncio.TimeoutError as err: - logger.warning("Timeout received while handling connection: %s", err) - finally: - writer.close() - if hasattr(writer, "wait_closed"): - # Python 3.7+ only - await writer.wait_closed() - logger.info("exit") + # try: + await asyncio.wait_for(handle_connection(reader, writer), timeout=5) + # except asyncio.TimeoutError as err: + # logger.warning(f"Timeout received while handling connection: {err}") + # finally: + writer.close() + await writer.wait_closed() + logger.info("exit") def get_next_execution_dt(): @@ -255,9 +260,7 @@ def get_mailboxes_to_reset(): async def reset_counters(): """Reset all counters.""" - rclient = aioredis.from_url( - settings.REDIS_URL, encoding="utf-8", decode_responses=True - ) + rclient = get_redis_client() logger.info("Resetting all counters") for domain in await get_domains_to_reset(): await rclient.hset(constants.REDIS_HASHNAME, domain.name, domain.message_limit) diff --git a/modoboa/policyd/handlers.py b/modoboa/policyd/handlers.py index d4e05465f..04d8e7e1c 100644 --- a/modoboa/policyd/handlers.py +++ b/modoboa/policyd/handlers.py @@ -1,14 +1,12 @@ """App related signal handlers.""" -import redis - -from django.conf import settings from django.db.models import signals from django.dispatch import receiver from modoboa.admin import models as admin_models from . import constants +from . import utils def set_message_limit(instance, key): @@ -16,9 +14,7 @@ def set_message_limit(instance, key): old_message_limit = instance._loaded_values.get("message_limit") if old_message_limit == instance.message_limit: return - rclient = redis.Redis( - host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_QUOTA_DB - ) + rclient = utils.get_redis_connection() if instance.message_limit is None: # delete existing key if rclient.hexists(constants.REDIS_HASHNAME, key): diff --git a/modoboa/policyd/tests.py b/modoboa/policyd/tests.py index 000e541f3..84066b212 100644 --- a/modoboa/policyd/tests.py +++ b/modoboa/policyd/tests.py @@ -7,10 +7,8 @@ from multiprocessing import Process import socket import time -import redis from django import db -from django.conf import settings from django.core.management import call_command from django.test import TransactionTestCase from django.urls import reverse @@ -22,6 +20,7 @@ from modoboa.lib.tests import ModoTestCase, ParametersMixin from modoboa.policyd import core as policyd_core from . import constants +from . import utils def start_policy_daemon(): @@ -33,12 +32,7 @@ class RedisTestCaseMixin: def setUp(self): super().setUp() - self.rclient = redis.Redis( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - db=settings.REDIS_QUOTA_DB, - ) - self.rclient.set_response_callback("HGET", int) + self.rclient = utils.get_redis_connection() self.rclient.delete(constants.REDIS_HASHNAME) diff --git a/modoboa/policyd/utils.py b/modoboa/policyd/utils.py index ca4b88c17..5c2a06b1b 100644 --- a/modoboa/policyd/utils.py +++ b/modoboa/policyd/utils.py @@ -9,9 +9,15 @@ from . import constants def get_redis_connection(): """Return a client connection to Redis server.""" - rclient = redis.Redis( - host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_QUOTA_DB - ) + if not getattr(settings, "REDIS_SENTINEL", False): + rclient = redis.Redis( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + db=settings.REDIS_QUOTA_DB, + ) + else: + sentinel = redis.sentinel.Sentinel(settings.REDIS_SENTINELS, socket_timeout=0.1) + rclient = sentinel.master_for(settings.REDIS_MASTER, socket_timeout=0.1) rclient.set_response_callback("HGET", int) return rclient diff --git a/test_project/envfile b/test_project/envfile new file mode 100644 index 000000000..915208d47 --- /dev/null +++ b/test_project/envfile @@ -0,0 +1,2 @@ +WEBMAIL_DEV_USERNAME=tonio@ngyn.org +WEBMAIL_DEV_PASSWORD=Aka/12ctogael diff --git a/test_project/manage.py b/test_project/manage.py index ba4bba75c..43e3cadd9 100755 --- a/test_project/manage.py +++ b/test_project/manage.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import unicode_literals import os import sys @@ -20,6 +19,6 @@ if __name__ == "__main__": "Couldn't import Django. Are you sure it's installed and " "available on your PYTHONPATH environment variable? Did you " "forget to activate a virtual environment?" - ) + ) from None raise execute_from_command_line(sys.argv) diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py index 2094a6976..d26270a75 100644 --- a/test_project/test_project/settings.py +++ b/test_project/test_project/settings.py @@ -274,10 +274,19 @@ PID_FILE_STORAGE_PATH = "/tmp" # REDIS +REDIS_SENTINEL = bool(os.environ.get("REDIS_SENTINEL", False)) +REDIS_SENTINELS = [ + ( + os.environ.get("REDIS_SENTINEL_HOST", "127.0.0.1"), + os.environ.get("REDIS_SENTINEL_PORT", 26379), + ) +] +REDIS_MASTER = os.environ.get("REDIS_MASTER", "mymaster") + REDIS_HOST = os.environ.get("REDIS_HOST", "127.0.0.1") REDIS_PORT = os.environ.get("REDIS_PORT", 6379) REDIS_QUOTA_DB = 0 -REDIS_URL = "redis://{}:{}/{}".format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB) +REDIS_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_QUOTA_DB}" # RQ diff --git a/test_project/test_project/wsgi.py b/test_project/test_project/wsgi.py index fce40e063..4c5770158 100644 --- a/test_project/test_project/wsgi.py +++ b/test_project/test_project/wsgi.py @@ -7,8 +7,6 @@ For more information on this file, see https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ """ -from __future__ import unicode_literals - import os from django.core.wsgi import get_wsgi_application