diff --git a/.github/workflows/modoboa.yml b/.github/workflows/modoboa.yml index 3faf976b0..867480493 100644 --- a/.github/workflows/modoboa.yml +++ b/.github/workflows/modoboa.yml @@ -39,15 +39,15 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 mysql: - image: mysql:8.0 + image: mariadb:11 env: - MYSQL_ROOT_PASSWORD: root - MYSQL_USER: modoboa - MYSQL_PASSWORD: modoboa - MYSQL_DATABASE: modoboa + MARIADB_ROOT_PASSWORD: root + MARIADB_USER: modoboa + MARIADB_PASSWORD: modoboa + MARIADB_DATABASE: modoboa ports: - 3306/tcp - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + options: --health-cmd="mariadb-admin ping" --health-interval=10s --health-timeout=5s --health-retries=3 redis: image: redis ports: diff --git a/modoboa/admin/constants.py b/modoboa/admin/constants.py index 56924eef3..7a3595de9 100644 --- a/modoboa/admin/constants.py +++ b/modoboa/admin/constants.py @@ -44,3 +44,5 @@ ALARM_STATUSES = [ (ALARM_OPENED, _("Opened")), (ALARM_CLOSED, _("Closed")), ] + +REDIS_ALARM = "redis_connection_error" diff --git a/modoboa/admin/migrations/0022_alter_alarm_domain.py b/modoboa/admin/migrations/0022_alter_alarm_domain.py new file mode 100644 index 000000000..727db11f3 --- /dev/null +++ b/modoboa/admin/migrations/0022_alter_alarm_domain.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.9 on 2024-02-07 17:18 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("admin", "0021_mailbox_is_send_only"), + ] + + operations = [ + migrations.AlterField( + model_name="alarm", + name="domain", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="alarms", + to="admin.domain", + ), + ), + ] diff --git a/modoboa/admin/models/alarm.py b/modoboa/admin/models/alarm.py index 03509f32a..1e3575663 100644 --- a/modoboa/admin/models/alarm.py +++ b/modoboa/admin/models/alarm.py @@ -20,7 +20,8 @@ class Alarm(models.Model): """A simple alarm to attach to a domain and/or mailbox.""" domain = models.ForeignKey( - "admin.Domain", on_delete=models.CASCADE, related_name="alarms") + "admin.Domain", on_delete=models.CASCADE, related_name="alarms", + blank=True, null=True) mailbox = models.ForeignKey( "admin.Mailbox", on_delete=models.SET_NULL, related_name="alarms", diff --git a/modoboa/core/api/v2/tests.py b/modoboa/core/api/v2/tests.py index ecc051fd4..b2fa8b7a7 100644 --- a/modoboa/core/api/v2/tests.py +++ b/modoboa/core/api/v2/tests.py @@ -10,10 +10,10 @@ from django.test import override_settings from django.urls import reverse from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_encode -from django_rq import get_worker, get_queue -from modoboa.admin import factories -from modoboa.core import models +from modoboa.admin import (factories, models as admin_models, + constants as admin_constants) +from modoboa.core import models, constants from modoboa.core.tests import utils from modoboa.lib.tests import ModoAPITestCase from rest_framework.authtoken.models import Token @@ -78,19 +78,18 @@ CORE_SETTINGS = { class ParametersAPITestCase(ModoAPITestCase): + def setUp(self): + super().setUp() + cache.delete("password_scheme_choice") + @override_settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH], DOVECOT_USER=DOVECOT_USER) def test_update(self): - cache.delete("password_scheme_choice") - queue = get_queue("modoboa", is_async=False) - queue.empty() - url = reverse("v2:parameter-detail", args=["core"]) data = copy.copy(CORE_SETTINGS) resp = self.client.put(url, data, format="json") self.assertEqual(resp.status_code, 200) - get_worker("modoboa").work(burst=True) self.assertIsNotNone(cache.get("password_scheme_choice")) self.assertIn(('plain', 'plain (weak)'), cache.get("password_scheme_choice") @@ -134,6 +133,50 @@ class ParametersAPITestCase(ModoAPITestCase): resp = self.client.put(url, data, format="json") self.assertEqual(resp.status_code, 200) + def test_doveadm_alarm(self): + """ Test that an alarm is opened, closed or reopened + depending on the result of the doveadm command for the password scheme + """ + # Test case where doveadm command fails + url = reverse("v2:parameter-detail", args=["core"]) + data = copy.copy(CORE_SETTINGS) + data["password_scheme"] = "plain" + + resp = self.client.put(url, data, format="json") + self.assertEqual(resp.status_code, 200) + + self.assertIsNone(cache.get("password_scheme_choice")) + doveadm_alarm = admin_models.Alarm.objects.filter( + internal_name=constants.DOVEADM_PASS_SCHEME_ALARM) + self.assertEqual(doveadm_alarm.count(), 1) + cache.delete("job_cache_available_password_hasher") + + with self.settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH], + DOVECOT_USER=DOVECOT_USER): + # The command should work and close the alarm + resp = self.client.put(url, data, format="json") + self.assertEqual(resp.status_code, 200) + + self.assertIsNotNone(cache.get("password_scheme_choice")) + doveadm_alarm = admin_models.Alarm.objects.filter( + internal_name=constants.DOVEADM_PASS_SCHEME_ALARM) + self.assertEqual(doveadm_alarm.count(), 1) + self.assertEqual(doveadm_alarm.first().status, + admin_constants.ALARM_CLOSED) + cache.delete("job_cache_available_password_hasher") + + # And lastly check that the alarm is reopened if the issue starts again + # Simulate that the cache has expired + cache.delete("password_scheme_choice") + resp = self.client.put(url, data, format="json") + self.assertEqual(resp.status_code, 200) + + doveadm_alarm = admin_models.Alarm.objects.filter( + internal_name=constants.DOVEADM_PASS_SCHEME_ALARM) + self.assertEqual(doveadm_alarm.count(), 1) + self.assertEqual(doveadm_alarm.first().status, + admin_constants.ALARM_OPENED) + class AccountViewSetTestCase(ModoAPITestCase): diff --git a/modoboa/core/app_settings.py b/modoboa/core/app_settings.py index 48a5c6cb1..c2804e1df 100644 --- a/modoboa/core/app_settings.py +++ b/modoboa/core/app_settings.py @@ -8,13 +8,12 @@ from django.conf import settings from django.contrib.auth import password_validation from django.utils.translation import gettext as _, gettext_lazy -from django_rq import get_queue - from redis.exceptions import ConnectionError -from modoboa.core.jobs import job_retrieve_available_hashers -from modoboa.core.password_hashers import get_dovecot_schemes -from modoboa.core.password_hashers.base import PasswordHasher +from modoboa.admin.models import Alarm +from modoboa.admin.constants import REDIS_ALARM, ALARM_OPENED + +from modoboa.core.password_hashers.utils import cache_available_password_hasher from modoboa.lib import fields as lib_fields from modoboa.lib.form_utils import ( HorizontalRadioSelect, SeparatorField, YesNoField @@ -37,25 +36,20 @@ def enabled_applications(): return sorted(result, key=lambda e: e[0]) -def get_default_password_scheme(): - available_schemes = get_dovecot_schemes() - return [ - (hasher.name, gettext_lazy(hasher.label)) - for hasher in PasswordHasher.get_password_hashers() - if hasher().scheme in available_schemes - ] - - def get_password_scheme(): try: available_schemes = cache.get("password_scheme_choice") if available_schemes is None: - get_queue("modoboa").enqueue(job_retrieve_available_hashers) - return get_default_password_scheme() + available_schemes = cache_available_password_hasher() return available_schemes except ConnectionError: - # TODO : notification/email to admin - return get_default_password_scheme() + redis_alarm = Alarm.objects.filter(internal_name=REDIS_ALARM) + if redis_alarm.first() is None: + Alarm.objects.create(internal_name=REDIS_ALARM, + title="Redis seems misconfigured") + elif redis_alarm.first().status == ALARM_OPENED: + redis_alarm.first().reopen() + return cache_available_password_hasher(bypass_cache=True) class GeneralParametersForm(param_forms.AdminParametersForm): diff --git a/modoboa/core/constants.py b/modoboa/core/constants.py index cc9a8b11b..20e7a2150 100644 --- a/modoboa/core/constants.py +++ b/modoboa/core/constants.py @@ -113,3 +113,5 @@ if settings.DEBUG: SMS_BACKENDS.insert(1, ("dummy", gettext_lazy("Dummy"))) TFA_DEVICE_TOKEN_KEY = "otp_device_id" + +DOVEADM_PASS_SCHEME_ALARM = "doveadm_password_scheme_fail" diff --git a/modoboa/core/jobs.py b/modoboa/core/jobs.py deleted file mode 100644 index 259b7413e..000000000 --- a/modoboa/core/jobs.py +++ /dev/null @@ -1,9 +0,0 @@ -from rq import get_current_job - -from modoboa.core.password_hashers import cache_available_password_hasher - - -def job_retrieve_available_hashers(): - if get_current_job() is None: - return - cache_available_password_hasher() diff --git a/modoboa/core/password_hashers/__init__.py b/modoboa/core/password_hashers/__init__.py index afc166cb2..a0e29dda8 100644 --- a/modoboa/core/password_hashers/__init__.py +++ b/modoboa/core/password_hashers/__init__.py @@ -2,10 +2,6 @@ Password hashers for Modoboa. """ -from django.core.cache import cache -from django.conf import settings -from django.utils.encoding import smart_str - from modoboa.core.password_hashers.advanced import ( # NOQA:F401 BLFCRYPTHasher, MD5CRYPTHasher, SHA256CRYPTHasher, SHA512CRYPTHasher, ARGON2IDHasher, SSHAHasher @@ -13,7 +9,6 @@ from modoboa.core.password_hashers.advanced import ( # NOQA:F401 from modoboa.core.password_hashers.base import ( # NOQA:F401 CRYPTHasher, MD5Hasher, PLAINHasher, SHA256Hasher, PasswordHasher ) -from modoboa.lib.sysutils import doveadm_cmd def get_password_hasher(scheme): @@ -31,40 +26,3 @@ def get_password_hasher(scheme): except KeyError: hasher = PLAINHasher return hasher - - -def get_dovecot_schemes(): - """Return schemes supported by Dovecot. - - It first try to get supported schemes fron the settings, then from the - doveadm output, and fallback to {MD5-CRYPT} and {PLAIN} if the command - is not found. - - :return: A list of supported '{SCHEME}' - """ - schemes = getattr(settings, "DOVECOT_SUPPORTED_SCHEMES", None) - default_schemes = "MD5-CRYPT PLAIN" - - if not schemes: - try: - retcode, schemes = doveadm_cmd("pw -l") - except OSError: - schemes = default_schemes - else: - if retcode: - schemes = default_schemes - - return ["{{{}}}".format(smart_str(scheme)) - for scheme in schemes.split()] - - -def cache_available_password_hasher(): - available_schemes = get_dovecot_schemes() - password_scheme_choice = [ - (hasher.name, hasher.label) - for hasher in PasswordHasher.get_password_hashers() - if hasher().scheme in available_schemes - ] - cache.set("password_scheme_choice", - password_scheme_choice, - 2592000) diff --git a/modoboa/core/password_hashers/utils.py b/modoboa/core/password_hashers/utils.py new file mode 100644 index 000000000..356c77823 --- /dev/null +++ b/modoboa/core/password_hashers/utils.py @@ -0,0 +1,77 @@ +""" Utils for password schemes caching/retrieval. """ + +from django.conf import settings +from django.core.cache import cache +from django.utils.encoding import smart_str + +from modoboa.admin.models.alarm import Alarm +from modoboa.admin.constants import ALARM_CLOSED, ALARM_OPENED + +from modoboa.core.constants import DOVEADM_PASS_SCHEME_ALARM + +from modoboa.lib.sysutils import doveadm_cmd + +from .base import PasswordHasher + + +def get_dovecot_schemes(): + """Return schemes supported by Dovecot. + + It first try to get supported schemes fron the settings, then from the + doveadm output, and fallback to {MD5-CRYPT} and {PLAIN} if the command + is not found. + + :return: A tuple with [0] : a list of supported '{SCHEME}' + [1] : the status (0=actual schemes, 1=settings schemes, 2=default schemes) + """ + schemes = getattr(settings, "DOVECOT_SUPPORTED_SCHEMES", None) + default_schemes = "MD5-CRYPT PLAIN" + status = 1 + + if not schemes: + try: + retcode, schemes = doveadm_cmd("pw -l") + except OSError: + schemes = default_schemes + status = 2 + else: + if retcode: + schemes = default_schemes + status = 2 + else: + status = 0 + + # Manage alarms if needed + doveadm_alarms = Alarm.objects.filter( + internal_name=DOVEADM_PASS_SCHEME_ALARM) + doveadm_alarm = doveadm_alarms.first() + if status == 0: + condition = (doveadm_alarm is not None and + doveadm_alarm.status == ALARM_OPENED) + if condition: + doveadm_alarm.close() + elif status == 2: + if doveadm_alarm is not None: + if doveadm_alarm.status == ALARM_CLOSED: + doveadm_alarm.reopen() + else: + Alarm.objects.create( + title="Failed to retrieve available dovecot schemes", + internal_name=DOVEADM_PASS_SCHEME_ALARM) + + return ["{{{}}}".format(smart_str(scheme)) + for scheme in schemes.split()], status + + +def cache_available_password_hasher(bypass_cache=False): + available_schemes, status = get_dovecot_schemes() + password_scheme_choice = [ + (hasher.name, hasher.label) + for hasher in PasswordHasher.get_password_hashers() + if hasher().scheme in available_schemes + ] + if status == 0 and not bypass_cache: + cache.set("password_scheme_choice", + password_scheme_choice, + 2592000) + return password_scheme_choice diff --git a/modoboa/core/tests/test_authentication.py b/modoboa/core/tests/test_authentication.py index a1cf87f0c..425b3e11e 100644 --- a/modoboa/core/tests/test_authentication.py +++ b/modoboa/core/tests/test_authentication.py @@ -13,9 +13,8 @@ try: except ImportError: argon2 = None -from modoboa.core.password_hashers import ( - get_password_hasher, get_dovecot_schemes -) +from modoboa.core.password_hashers import get_password_hasher +from modoboa.core.password_hashers.utils import get_dovecot_schemes from modoboa.lib.tests import NO_SMTP, ModoTestCase from .. import factories, models @@ -187,7 +186,7 @@ class AuthenticationTestCase(ModoTestCase): "DIGEST-MD5 PLAIN-MD4 PLAIN-MD5 LDAP-MD5 LANMAN NTLM OTP SKEY RPA " "PBKDF2 CRYPT SHA256-CRYPT SHA512-CRYPT" ) - supported_schemes = get_dovecot_schemes() + supported_schemes = get_dovecot_schemes()[0] self.assertEqual(supported_schemes, ["{MD5}", "{MD5-CRYPT}", @@ -224,12 +223,12 @@ class AuthenticationTestCase(ModoTestCase): @override_settings(DOVECOT_SUPPORTED_SCHEMES="SHA1 SHA512-CRYPT") def test_dovecot_supported_schemes_from_settings(self): """Validate dovecot supported schemes from the settings.""" - supported_schemes = get_dovecot_schemes() + supported_schemes = get_dovecot_schemes()[0] self.assertEqual(supported_schemes, ["{SHA1}", "{SHA512-CRYPT}"]) def test_dovecot_default_schemes(self): """Check default scheme if doveadm is not found.""" - supported_schemes = get_dovecot_schemes() + supported_schemes = get_dovecot_schemes()[0] self.assertEqual(supported_schemes, ["{MD5-CRYPT}", "{PLAIN}"])