github-jwinterm/modoboa
1
0
Fork 0
mirror of https://github.com/jwinterm/modoboa.git synced 2026-10-17 05:41:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3182 from modoboa/caching-improvement

Browse Source 
Nullable domain on alarms, improved password scheme caching
This commit is contained in:
Antoine Nguyen
2024-02-13 16:59:07 +01:00
committed by GitHub
parent c45a0dced1 d9ee525218
commit 84d01d49eb
11 changed files with 174 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/modoboa.yml
+6 -6
View File
@@ -39,15 +39,15 @@ jobs:
# needed because the postgres container does not provide a healthcheck
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
mysql:
image: mysql:8.0
image: mariadb:11
env:
MYSQL_ROOT_PASSWORD: root
MYSQL_USER: modoboa
MYSQL_PASSWORD: modoboa
MYSQL_DATABASE: modoboa
MARIADB_ROOT_PASSWORD: root
MARIADB_USER: modoboa
MARIADB_PASSWORD: modoboa
MARIADB_DATABASE: modoboa
ports:
- 3306/tcp
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
options: --health-cmd="mariadb-admin ping" --health-interval=10s --health-timeout=5s --health-retries=3
redis:
image: redis
ports:
modoboa/admin/constants.py
+2
View File
@@ -44,3 +44,5 @@ ALARM_STATUSES = [
(ALARM_OPENED, _("Opened")),
(ALARM_CLOSED, _("Closed")),
]
REDIS_ALARM = "redis_connection_error"
modoboa/admin/migrations/0022_alter_alarm_domain.py
+24
View File
@@ -0,0 +1,24 @@
# Generated by Django 4.2.9 on 2024-02-07 17:18
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("admin", "0021_mailbox_is_send_only"),
]
operations = [
migrations.AlterField(
model_name="alarm",
name="domain",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="alarms",
to="admin.domain",
),
),
]
modoboa/admin/models/alarm.py
+2 -1
View File
@@ -20,7 +20,8 @@ class Alarm(models.Model):
"""A simple alarm to attach to a domain and/or mailbox."""
domain = models.ForeignKey(
"admin.Domain", on_delete=models.CASCADE, related_name="alarms")
"admin.Domain", on_delete=models.CASCADE, related_name="alarms",
blank=True, null=True)
mailbox = models.ForeignKey(
"admin.Mailbox", on_delete=models.SET_NULL,
related_name="alarms",
modoboa/core/api/v2/tests.py
+51 -8
View File
@@ -10,10 +10,10 @@ from django.test import override_settings
from django.urls import reverse
from django.utils.encoding import force_bytes
from django.utils.http import urlsafe_base64_encode
from django_rq import get_worker, get_queue
from modoboa.admin import factories
from modoboa.core import models
from modoboa.admin import (factories, models as admin_models,
constants as admin_constants)
from modoboa.core import models, constants
from modoboa.core.tests import utils
from modoboa.lib.tests import ModoAPITestCase
from rest_framework.authtoken.models import Token
@@ -78,19 +78,18 @@ CORE_SETTINGS = {
class ParametersAPITestCase(ModoAPITestCase):
def setUp(self):
super().setUp()
cache.delete("password_scheme_choice")
@override_settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH],
DOVECOT_USER=DOVECOT_USER)
def test_update(self):
cache.delete("password_scheme_choice")
queue = get_queue("modoboa", is_async=False)
queue.empty()
url = reverse("v2:parameter-detail", args=["core"])
data = copy.copy(CORE_SETTINGS)
resp = self.client.put(url, data, format="json")
self.assertEqual(resp.status_code, 200)
get_worker("modoboa").work(burst=True)
self.assertIsNotNone(cache.get("password_scheme_choice"))
self.assertIn(('plain', 'plain (weak)'),
cache.get("password_scheme_choice")
@@ -134,6 +133,50 @@ class ParametersAPITestCase(ModoAPITestCase):
resp = self.client.put(url, data, format="json")
self.assertEqual(resp.status_code, 200)
def test_doveadm_alarm(self):
""" Test that an alarm is opened, closed or reopened
depending on the result of the doveadm command for the password scheme
"""
# Test case where doveadm command fails
url = reverse("v2:parameter-detail", args=["core"])
data = copy.copy(CORE_SETTINGS)
data["password_scheme"] = "plain"
resp = self.client.put(url, data, format="json")
self.assertEqual(resp.status_code, 200)
self.assertIsNone(cache.get("password_scheme_choice"))
doveadm_alarm = admin_models.Alarm.objects.filter(
internal_name=constants.DOVEADM_PASS_SCHEME_ALARM)
self.assertEqual(doveadm_alarm.count(), 1)
cache.delete("job_cache_available_password_hasher")
with self.settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH],
DOVECOT_USER=DOVECOT_USER):
# The command should work and close the alarm
resp = self.client.put(url, data, format="json")
self.assertEqual(resp.status_code, 200)
self.assertIsNotNone(cache.get("password_scheme_choice"))
doveadm_alarm = admin_models.Alarm.objects.filter(
internal_name=constants.DOVEADM_PASS_SCHEME_ALARM)
self.assertEqual(doveadm_alarm.count(), 1)
self.assertEqual(doveadm_alarm.first().status,
admin_constants.ALARM_CLOSED)
cache.delete("job_cache_available_password_hasher")
# And lastly check that the alarm is reopened if the issue starts again
# Simulate that the cache has expired
cache.delete("password_scheme_choice")
resp = self.client.put(url, data, format="json")
self.assertEqual(resp.status_code, 200)
doveadm_alarm = admin_models.Alarm.objects.filter(
internal_name=constants.DOVEADM_PASS_SCHEME_ALARM)
self.assertEqual(doveadm_alarm.count(), 1)
self.assertEqual(doveadm_alarm.first().status,
admin_constants.ALARM_OPENED)
class AccountViewSetTestCase(ModoAPITestCase):
modoboa/core/app_settings.py
+5 -25
View File
@@ -8,13 +8,7 @@ from django.conf import settings
from django.contrib.auth import password_validation
from django.utils.translation import gettext as _, gettext_lazy
from django_rq import get_queue
from redis.exceptions import ConnectionError
from modoboa.core.jobs import job_retrieve_available_hashers
from modoboa.core.password_hashers import get_dovecot_schemes
from modoboa.core.password_hashers.base import PasswordHasher
from modoboa.core.password_hashers.utils import cache_available_password_hasher
from modoboa.lib import fields as lib_fields
from modoboa.lib.form_utils import (
HorizontalRadioSelect, SeparatorField, YesNoField
@@ -37,25 +31,11 @@ def enabled_applications():
return sorted(result, key=lambda e: e[0])
def get_default_password_scheme():
available_schemes = get_dovecot_schemes()
return [
(hasher.name, gettext_lazy(hasher.label))
for hasher in PasswordHasher.get_password_hashers()
if hasher().scheme in available_schemes
]
def get_password_scheme():
try:
available_schemes = cache.get("password_scheme_choice")
if available_schemes is None:
get_queue("modoboa").enqueue(job_retrieve_available_hashers)
return get_default_password_scheme()
return available_schemes
except ConnectionError:
# TODO : notification/email to admin
return get_default_password_scheme()
available_schemes = cache.get("password_scheme_choice")
if available_schemes is None:
available_schemes = cache_available_password_hasher()
return available_schemes
class GeneralParametersForm(param_forms.AdminParametersForm):
modoboa/core/constants.py
+2
View File
@@ -113,3 +113,5 @@ if settings.DEBUG:
SMS_BACKENDS.insert(1, ("dummy", gettext_lazy("Dummy")))
TFA_DEVICE_TOKEN_KEY = "otp_device_id"
DOVEADM_PASS_SCHEME_ALARM = "doveadm_password_scheme_fail"
modoboa/core/jobs.py
-9
View File
@@ -1,9 +0,0 @@
from rq import get_current_job
from modoboa.core.password_hashers import cache_available_password_hasher
def job_retrieve_available_hashers():
if get_current_job() is None:
return
cache_available_password_hasher()
modoboa/core/password_hashers/__init__.py
-42
View File
@@ -2,10 +2,6 @@
Password hashers for Modoboa.
"""
from django.core.cache import cache
from django.conf import settings
from django.utils.encoding import smart_str
from modoboa.core.password_hashers.advanced import ( # NOQA:F401
BLFCRYPTHasher, MD5CRYPTHasher, SHA256CRYPTHasher, SHA512CRYPTHasher,
ARGON2IDHasher, SSHAHasher
@@ -13,7 +9,6 @@ from modoboa.core.password_hashers.advanced import ( # NOQA:F401
from modoboa.core.password_hashers.base import ( # NOQA:F401
CRYPTHasher, MD5Hasher, PLAINHasher, SHA256Hasher, PasswordHasher
)
from modoboa.lib.sysutils import doveadm_cmd
def get_password_hasher(scheme):
@@ -31,40 +26,3 @@ def get_password_hasher(scheme):
except KeyError:
hasher = PLAINHasher
return hasher
def get_dovecot_schemes():
"""Return schemes supported by Dovecot.
It first try to get supported schemes fron the settings, then from the
doveadm output, and fallback to {MD5-CRYPT} and {PLAIN} if the command
is not found.
:return: A list of supported '{SCHEME}'
"""
schemes = getattr(settings, "DOVECOT_SUPPORTED_SCHEMES", None)
default_schemes = "MD5-CRYPT PLAIN"
if not schemes:
try:
retcode, schemes = doveadm_cmd("pw -l")
except OSError:
schemes = default_schemes
else:
if retcode:
schemes = default_schemes
return ["{{{}}}".format(smart_str(scheme))
for scheme in schemes.split()]
def cache_available_password_hasher():
available_schemes = get_dovecot_schemes()
password_scheme_choice = [
(hasher.name, hasher.label)
for hasher in PasswordHasher.get_password_hashers()
if hasher().scheme in available_schemes
]
cache.set("password_scheme_choice",
password_scheme_choice,
2592000)
modoboa/core/password_hashers/utils.py
+77
View File
@@ -0,0 +1,77 @@
""" Utils for password schemes caching/retrieval. """
from django.conf import settings
from django.core.cache import cache
from django.utils.encoding import smart_str
from modoboa.admin.models.alarm import Alarm
from modoboa.admin.constants import ALARM_CLOSED, ALARM_OPENED
from modoboa.core.constants import DOVEADM_PASS_SCHEME_ALARM
from modoboa.lib.sysutils import doveadm_cmd
from .base import PasswordHasher
def get_dovecot_schemes():
"""Return schemes supported by Dovecot.
It first try to get supported schemes fron the settings, then from the
doveadm output, and fallback to {MD5-CRYPT} and {PLAIN} if the command
is not found.
:return: A tuple with [0] : a list of supported '{SCHEME}'
[1] : the status (0=actual schemes, 1=settings schemes, 2=default schemes)
"""
schemes = getattr(settings, "DOVECOT_SUPPORTED_SCHEMES", None)
default_schemes = "MD5-CRYPT PLAIN"
status = 1
if not schemes:
try:
retcode, schemes = doveadm_cmd("pw -l")
except OSError:
schemes = default_schemes
status = 2
else:
if retcode:
schemes = default_schemes
status = 2
else:
status = 0
# Manage alarms if needed
doveadm_alarms = Alarm.objects.filter(
internal_name=DOVEADM_PASS_SCHEME_ALARM)
doveadm_alarm = doveadm_alarms.first()
if status == 0:
condition = (doveadm_alarm is not None and
doveadm_alarm.status == ALARM_OPENED)
if condition:
doveadm_alarm.close()
elif status == 2:
if doveadm_alarm is not None:
if doveadm_alarm.status == ALARM_CLOSED:
doveadm_alarm.reopen()
else:
Alarm.objects.create(
title="Failed to retrieve available dovecot schemes",
internal_name=DOVEADM_PASS_SCHEME_ALARM)
return ["{{{}}}".format(smart_str(scheme))
for scheme in schemes.split()], status
def cache_available_password_hasher(bypass_cache=False):
available_schemes, status = get_dovecot_schemes()
password_scheme_choice = [
(hasher.name, hasher.label)
for hasher in PasswordHasher.get_password_hashers()
if hasher().scheme in available_schemes
]
if status == 0 and not bypass_cache:
cache.set("password_scheme_choice",
password_scheme_choice,
2592000)
return password_scheme_choice
modoboa/core/tests/test_authentication.py
+5 -6
View File
@@ -13,9 +13,8 @@ try:
except ImportError:
argon2 = None
from modoboa.core.password_hashers import (
get_password_hasher, get_dovecot_schemes
)
from modoboa.core.password_hashers import get_password_hasher
from modoboa.core.password_hashers.utils import get_dovecot_schemes
from modoboa.lib.tests import NO_SMTP, ModoTestCase
from .. import factories, models
@@ -187,7 +186,7 @@ class AuthenticationTestCase(ModoTestCase):
"DIGEST-MD5 PLAIN-MD4 PLAIN-MD5 LDAP-MD5 LANMAN NTLM OTP SKEY RPA "
"PBKDF2 CRYPT SHA256-CRYPT SHA512-CRYPT"
)
supported_schemes = get_dovecot_schemes()
supported_schemes = get_dovecot_schemes()[0]
self.assertEqual(supported_schemes,
["{MD5}",
"{MD5-CRYPT}",
@@ -224,12 +223,12 @@ class AuthenticationTestCase(ModoTestCase):
@override_settings(DOVECOT_SUPPORTED_SCHEMES="SHA1 SHA512-CRYPT")
def test_dovecot_supported_schemes_from_settings(self):
"""Validate dovecot supported schemes from the settings."""
supported_schemes = get_dovecot_schemes()
supported_schemes = get_dovecot_schemes()[0]
self.assertEqual(supported_schemes, ["{SHA1}", "{SHA512-CRYPT}"])
def test_dovecot_default_schemes(self):
"""Check default scheme if doveadm is not found."""
supported_schemes = get_dovecot_schemes()
supported_schemes = get_dovecot_schemes()[0]
self.assertEqual(supported_schemes, ["{MD5-CRYPT}", "{PLAIN}"])