From 7da30177455aa21e13145122d30f04de33f2d63f Mon Sep 17 00:00:00 2001 From: Spitap Date: Fri, 20 Oct 2023 10:44:22 +0200 Subject: [PATCH] squashed old attempt Refactor Improved test Revert "Use SimpleWorker for the job test" This reverts commit f7ea07d19a43ba0455d3a251c216c50f94a1ac66. Force job to run Empty queue before bursting jobs Added test Updated test Switched json encoder Removed call to doveadm on user login Fixed issues, updated doc Reduce doveadm calls --- doc/upgrade.rst | 51 ++++++++++++++++ docker-compose.yml | 2 +- docker/Dockerfile.dev | 6 +- docker/Dockerfile.front.dev | 5 +- docker/doveadm | 3 + frontend/package.json | 4 +- frontend/yarn.lock | 6 +- modoboa/core/api/v2/serializers.py | 37 +++++++---- modoboa/core/api/v2/tests.py | 55 ++++++++++++++++- modoboa/core/app_settings.py | 32 +++++++--- .../core/commands/templates/settings.py.tpl | 5 ++ modoboa/core/jobs.py | 19 ++++++ modoboa/core/migrations/0008_localconfig.py | 3 +- .../migrations/0010_auto_20161026_1011.py | 7 +-- .../migrations/0011_auto_20161104_2217.py | 7 +-- ..._alter_localconfig__parameters_and_more.py | 55 +++++++++++++++++ modoboa/core/models.py | 13 ++-- modoboa/core/password_hashers/__init__.py | 18 +++++- modoboa/core/tests/utils.py | 4 ++ modoboa/lib/json_field_utils.py | 32 ++++++++++ modoboa/parameters/cache_tools.py | 61 +++++++++++++++++++ modoboa/transport/migrations/0001_initial.py | 4 +- .../0003_alter_transport__settings.py | 24 ++++++++ modoboa/transport/models.py | 4 +- requirements.txt | 1 - test_project/test_project/settings.py | 5 ++ 26 files changed, 413 insertions(+), 50 deletions(-) create mode 100755 docker/doveadm create mode 100644 modoboa/core/jobs.py create mode 100644 modoboa/core/migrations/0026_alter_localconfig__parameters_and_more.py create mode 100644 modoboa/core/tests/utils.py create mode 100644 modoboa/lib/json_field_utils.py create mode 100644 modoboa/parameters/cache_tools.py create mode 100644 modoboa/transport/migrations/0003_alter_transport__settings.py diff --git a/doc/upgrade.rst b/doc/upgrade.rst index 1b9bc598f..3d8847a8c 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -131,6 +131,57 @@ Rebuild Virtual Environment Specific instructions ********************* + +2.2.3 +===== + +* Multiple security fixes +* Fixed a CRSF error when logging multiple time on v1 +* Reduced the number of call to doveadm + +You to setup a new worker : + +1. Create a new supervisord config (``/etc/supervisor/conf.d/modoboa-base-worker.conf`` by default) : + +.. sourcecode:: ini + + [program:modoboa-base-worker] + autostart=true + autorestart=true + command={%python env path%} {% manage.py instance path%} rqworker modoboa + directory={%modoboa home dir%} + user=%{modoboa user} + numprocs=1 + stopsignal=TERM + +``python env path`` : Python executable located in your virtual environment created for modoboa. +You will find it here ``/srv/modoboa/venv/bin/python`` by default. +``manage.py instance path``: Path to manage.py of your modoboa instance. +You will find it here : ``/srv/modoboa/instance/manage.py``by default. +``Modoboa home dir``: Home dir of the user running modooba. +You will find it here ``/srv/modoboa/`` by default. +``modoboa user`` : User managing dkim signing (modoboa by default). + +You can help you with ``/etc/supervisor/conf.d/policyd.conf`` (by default). + +Then restart supervisor. ``#> supervisorctl reread && supervisorctl update`` on Debian. + +2. Update the RQ_QUEUES section bellow the ``#REDIS`` section: + +.. sourcecode:: python + + # RQ + RQ_QUEUES = { + ... + }, + 'modoboa': { + 'HOST': REDIS_HOST, + 'PORT': REDIS_PORT, + 'DB': 0, + }, + } + + 2.2.0 ===== diff --git a/docker-compose.yml b/docker-compose.yml index 7a98b2d29..d5a21a4c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: DEBUG: "True" REDIS_HOST: redis command: > - sh -c "cd test_project; python3 manage.py rqworker dkim" + sh -c "cd test_project; python3 manage.py rqworker dkim modoboa" volumes: - .:/code diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index d0187cf93..cd426dca6 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM python:3.9-alpine as base +FROM python:3.12-alpine as base MAINTAINER Antoine Nguyen ENV VIRTUAL_ENV=/opt/venv @@ -17,8 +17,10 @@ RUN python -m pip install -U pip RUN pip install -r requirements.txt -r test-requirements.txt FROM base as run +COPY docker/doveadm /usr/local/bin +RUN chmod +x /usr/local/bin/doveadm COPY --from=build $VIRTUAL_ENV $VIRTUAL_ENV -RUN apk add --no-cache --update openssl librrd libxslt libjpeg libressl git rrdtool libmagic \ +RUN apk add --no-cache --update openssl librrd libxslt libjpeg libressl git rrdtool libmagic sudo \ && rm -rf /var/cache/apk/* RUN mkdir /code WORKDIR /code diff --git a/docker/Dockerfile.front.dev b/docker/Dockerfile.front.dev index 8149784bc..5e335639f 100644 --- a/docker/Dockerfile.front.dev +++ b/docker/Dockerfile.front.dev @@ -1,7 +1,10 @@ -FROM node:16.10-alpine +FROM node:16.10-alpine ENV HOME=/home/node/ +RUN apk add --no-cache --update gettext \ + && rm -rf /var/cache/apk/* + WORKDIR $HOME RUN yarn install diff --git a/docker/doveadm b/docker/doveadm new file mode 100755 index 000000000..c8e8845fb --- /dev/null +++ b/docker/doveadm @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "SHA1 SSHA512 SCRAM-SHA-256 BLF-CRYPT PLAIN HMAC-MD5 OTP SHA512 SHA DES-CRYPT CRYPT SSHA MD5-CRYPT PLAIN-MD4 PLAIN-MD5 SCRAM-SHA-1 SHA512-CRYPT CLEAR CLEARTEXT ARGON2I ARGON2ID SSHA256 MD5 PBKDF2 SHA256 CRAM-MD5 PLAIN-TRUNC SHA256-CRYPT SMD5 DIGEST-MD5 LDAP-MD5" diff --git a/frontend/package.json b/frontend/package.json index 5aacece55..b82b971f1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,6 +16,7 @@ "debounce": "^1.2.1", "js-cookie": "^2.2.1", "luxon": "^1.28.1", + "qrcode.vue": "~1.7.0", "vee-validate": "^3.4.5", "vue": "^2.6.14", "vue-apexcharts": "^1.6.0", @@ -23,8 +24,7 @@ "vue-gettext": "^2.1.12", "vue-router": "^3.2.0", "vuetify": "^2.6.10", - "vuex": "^3.4.0", - "qrcode.vue": "~1.7.0" + "vuex": "^3.4.0" }, "devDependencies": { "@babel/core": "^7.12.16", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 27325a57f..a3e82686a 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -2515,9 +2515,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001400, caniuse-lite@^1.0.30001407: - version "1.0.30001481" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001481.tgz" - integrity sha512-KCqHwRnaa1InZBtqXzP98LPg0ajCVujMKjqKDhZEthIpAsJl/YEIa3YvXjGXPVqzZVguccuu7ga9KOE1J9rKPQ== + version "1.0.30001553" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001553.tgz" + integrity sha512-N0ttd6TrFfuqKNi+pMgWJTb9qrdJu4JSpgPFLe/lrD19ugC6fZgF0pUewRowDwzdDnb9V41mFcdlYgl/PyKf4A== case-sensitive-paths-webpack-plugin@^2.3.0: version "2.4.0" diff --git a/modoboa/core/api/v2/serializers.py b/modoboa/core/api/v2/serializers.py index 56cf3866d..0f78cb4fd 100644 --- a/modoboa/core/api/v2/serializers.py +++ b/modoboa/core/api/v2/serializers.py @@ -21,7 +21,9 @@ from django.utils.translation import gettext_lazy, gettext as _ from rest_framework import serializers, status from rest_framework.exceptions import APIException -from modoboa.core import constants, models, sms_backends, app_settings, context_processors +from modoboa.core import ( + constants, models, sms_backends, app_settings, context_processors +) from modoboa.core.models import User from modoboa.lib import fields as lib_fields, cryptutils @@ -62,21 +64,15 @@ class CoreGlobalParametersSerializer(serializers.Serializer): default="local" ) password_scheme = serializers.ChoiceField( - choices=[("sha512crypt", "sha512crypt"), - ("sha256crypt", "sha256crypt"), - ("blfcrypt", "bcrypt"), - ("md5crypt", gettext_lazy("md5crypt (weak)")), - ("sha256", gettext_lazy("sha256 (weak)")), - ("md5", gettext_lazy("md5 (weak)")), - ("crypt", gettext_lazy("crypt (weak)")), - ("plain", gettext_lazy("plain (weak)"))], + choices=[("sha512crypt", "sha512crypt")], default="sha512crypt" ) rounds_number = serializers.IntegerField(default=70000) update_scheme = serializers.BooleanField(default=True) - default_password = serializers.CharField(default="password") + default_password = serializers.CharField(default="ChangeMe1!") random_password_length = serializers.IntegerField(min_value=8, default=8) - update_password_url = serializers.URLField(required=False, allow_blank=True) + update_password_url = serializers.URLField( + required=False, allow_blank=True) password_recovery_msg = serializers.CharField( required=False, allow_blank=True) sms_password_recovery = serializers.BooleanField(default=False) @@ -173,8 +169,27 @@ class CoreGlobalParametersSerializer(serializers.Serializer): for field, definition in sms_backend_fields.items(): self.fields[field] = definition["type"]( **definition["attrs"]) + + try: + self.saved_password_scheme = args[0].get("password_scheme") + self.saved_default_top_redirection = args[0].get("default_top_redirection") + except IndexError: + self.saved_password_scheme = None + self.saved_default_top_redirection = None # Choices serializer for default_top_redirection field self.fields["default_top_redirection"].choices = app_settings.enabled_applications() + # Populate choices of password_scheme + localconfig = models.LocalConfig.objects.first() + self.fields["password_scheme"].choices = app_settings.get_password_scheme(localconfig) + + @property + def data(self): + ret = super().data + if self.saved_password_scheme is not None: + ret["password_scheme"] = self.saved_password_scheme + if self.saved_default_top_redirection is not None: + ret["default_top_redirection"] = self.saved_default_top_redirection + return ret def validate_ldap_user_dn_template(self, value): try: diff --git a/modoboa/core/api/v2/tests.py b/modoboa/core/api/v2/tests.py index cc0ccf019..aeadabccf 100644 --- a/modoboa/core/api/v2/tests.py +++ b/modoboa/core/api/v2/tests.py @@ -1,18 +1,26 @@ """Core API related tests.""" import copy +import datetime +import getpass import oath from unittest import mock +from django.test import override_settings from django.urls import reverse from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_encode +from django_rq import get_worker, get_queue from modoboa.admin import factories -from modoboa.core import models +from modoboa.core import models, jobs +from modoboa.core.tests import utils from modoboa.lib.tests import ModoAPITestCase from rest_framework.authtoken.models import Token +DOVEADM_TEST_PATH = utils.get_doveadm_test_path() +DOVECOT_USER = getpass.getuser() + CORE_SETTINGS = { "authentication_type": "local", "password_scheme": "sha512crypt", @@ -70,6 +78,8 @@ CORE_SETTINGS = { class ParametersAPITestCase(ModoAPITestCase): + @override_settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH], + DOVECOT_USER=DOVECOT_USER) def test_update(self): url = reverse("v2:parameter-detail", args=["core"]) data = copy.copy(CORE_SETTINGS) @@ -115,6 +125,49 @@ class ParametersAPITestCase(ModoAPITestCase): resp = self.client.put(url, data, format="json") self.assertEqual(resp.status_code, 200) + @override_settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH], + DOVECOT_USER=DOVECOT_USER) + def test_cache(self): + """Launch the password scheme job manually and check the result.""" + queue = get_queue("modoboa", is_async=False) + queue.empty() + job = queue.enqueue(jobs.job_retrieve_available_hashers, + bypass_condition=True) + get_worker("modoboa").work(burst=True) + job_result = job.latest_result() + self.assertEqual(job_result.type, job_result.Type.SUCCESSFUL) + localconfig = models.LocalConfig.objects.first() + cache = localconfig.cache + self.assertIn(['plain', 'plain (weak)'], + cache.get_value("password_scheme_choice", "core")) + entry = cache.get_cache_entry("password_scheme_choice", "core") + self.assertEqual(entry["last_update"], + cache.get_last_mod("password_scheme_choice", "core")) + entry_guess_extension = cache.get_cache_entry("password_scheme_choice") + self.assertEqual(entry, entry_guess_extension) + cache.set_cache_entry("test_cache", "test") + self.assertEqual( + cache.get_value("test_cache", "core"), + "test") + + # Check that the cache will be updated once a month + localconfig._cache["core"]["password_scheme_choice"]["value"] = "outofdate!" + localconfig.save() + job = queue.enqueue(jobs.job_retrieve_available_hashers) + get_worker("modoboa").work(burst=True) + localconfig = models.LocalConfig.objects.first() + value = localconfig.cache.get_value("password_scheme_choice") + self.assertEqual(value, "outofdate!") + + old_datetime = datetime.datetime.utcnow() - datetime.timedelta(weeks=5) + localconfig._cache["core"]["password_scheme_choice"]["last_update"] = old_datetime + localconfig.save() + job = queue.enqueue(jobs.job_retrieve_available_hashers) + get_worker("modoboa").work(burst=True) + localconfig = models.LocalConfig.objects.first() + value = localconfig.cache.get_value("password_scheme_choice") + self.assertFalse(value == "outofdate!") + class AccountViewSetTestCase(ModoAPITestCase): diff --git a/modoboa/core/app_settings.py b/modoboa/core/app_settings.py index ad817c961..5a80717ba 100644 --- a/modoboa/core/app_settings.py +++ b/modoboa/core/app_settings.py @@ -6,7 +6,9 @@ from django import forms from django.conf import settings from django.contrib.auth import password_validation from django.utils.translation import gettext as _, gettext_lazy +from django_rq import get_queue +from modoboa.core.jobs import job_retrieve_available_hashers from modoboa.core.password_hashers import get_dovecot_schemes from modoboa.core.password_hashers.base import PasswordHasher from modoboa.lib import fields as lib_fields @@ -31,6 +33,27 @@ def enabled_applications(): return sorted(result, key=lambda e: e[0]) +def get_default_password_scheme(): + available_schemes = get_dovecot_schemes() + return [ + (hasher.name, gettext_lazy(hasher.label)) + for hasher in PasswordHasher.get_password_hashers() + if hasher().scheme in available_schemes + ] + + +def get_password_scheme(localconfig): + if localconfig is None: + return get_default_password_scheme() + available_schemes = localconfig.cache.get_value( + "password_scheme_choice", "core", False) + get_queue("modoboa").enqueue( + job_retrieve_available_hashers) + if available_schemes is None: + return get_default_password_scheme() + return available_schemes + + class GeneralParametersForm(param_forms.AdminParametersForm): """General parameters.""" @@ -73,7 +96,7 @@ class GeneralParametersForm(param_forms.AdminParametersForm): default_password = forms.CharField( label=gettext_lazy("Default password"), - initial="password", + initial="ChangeMe1!", help_text=gettext_lazy( "Default password for automatically created accounts.") ) @@ -535,12 +558,7 @@ class GeneralParametersForm(param_forms.AdminParametersForm): self._add_visibilty_rules( sms_backends.get_all_backend_visibility_rules() ) - available_schemes = get_dovecot_schemes() - self.fields["password_scheme"].choices = [ - (hasher.name, gettext_lazy(hasher.label)) - for hasher in PasswordHasher.get_password_hashers() - if hasher().scheme in available_schemes - ] + self.fields["password_scheme"].choices = get_password_scheme(self.localconfig) def _add_dynamic_fields(self): new_fields = collections.OrderedDict() diff --git a/modoboa/core/commands/templates/settings.py.tpl b/modoboa/core/commands/templates/settings.py.tpl index c9e3874ee..6bfbb8c37 100644 --- a/modoboa/core/commands/templates/settings.py.tpl +++ b/modoboa/core/commands/templates/settings.py.tpl @@ -262,6 +262,11 @@ RQ_QUEUES = { 'PORT': REDIS_PORT, 'DB': 0, }, + 'modoboa': { + 'HOST': REDIS_HOST, + 'PORT': REDIS_PORT, + 'DB': 0, + }, } # Password validation diff --git a/modoboa/core/jobs.py b/modoboa/core/jobs.py new file mode 100644 index 000000000..4da42016b --- /dev/null +++ b/modoboa/core/jobs.py @@ -0,0 +1,19 @@ +from datetime import timedelta, datetime + +from modoboa.core import models +from modoboa.core.password_hashers import cache_available_passowrd_hasher + + +def job_retrieve_available_hashers(bypass_condition=False): + localconfig = models.LocalConfig.objects.first() + cache = localconfig.cache + password_scheme_last_mod = cache.get_last_mod( + "password_scheme_choice", "core", False) + cur_dt = datetime.utcnow() + condition = (bypass_condition or + password_scheme_last_mod is None or + password_scheme_last_mod < cur_dt - timedelta(weeks=4) + ) + if condition: + cache_available_passowrd_hasher() + diff --git a/modoboa/core/migrations/0008_localconfig.py b/modoboa/core/migrations/0008_localconfig.py index 9dae77efb..a9df6ff8d 100644 --- a/modoboa/core/migrations/0008_localconfig.py +++ b/modoboa/core/migrations/0008_localconfig.py @@ -1,7 +1,6 @@ # Generated by Django 1.9.5 on 2016-06-15 18:12 from django.db import migrations, models import django.db.models.deletion -import jsonfield.fields class Migration(migrations.Migration): @@ -17,7 +16,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('api_pk', models.PositiveIntegerField(null=True)), - ('api_versions', jsonfield.fields.JSONField()), + ('api_versions', models.JSONField()), ('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sites.Site')), ], ), diff --git a/modoboa/core/migrations/0010_auto_20161026_1011.py b/modoboa/core/migrations/0010_auto_20161026_1011.py index e681c79b8..930f252a9 100644 --- a/modoboa/core/migrations/0010_auto_20161026_1011.py +++ b/modoboa/core/migrations/0010_auto_20161026_1011.py @@ -1,6 +1,5 @@ # Generated by Django 1.9.5 on 2016-10-26 08:11 -from django.db import migrations -import jsonfield.fields +from django.db import migrations, models def set_parameter(store, parameter): @@ -51,13 +50,13 @@ class Migration(migrations.Migration): migrations.AddField( model_name='localconfig', name='_parameters', - field=jsonfield.fields.JSONField(default={}), + field=models.JSONField(default=dict), preserve_default=False, ), migrations.AddField( model_name='user', name='_parameters', - field=jsonfield.fields.JSONField(default={}), + field=models.JSONField(default=dict), preserve_default=False, ), migrations.RunPython(move_parameters, clear_parameters) diff --git a/modoboa/core/migrations/0011_auto_20161104_2217.py b/modoboa/core/migrations/0011_auto_20161104_2217.py index f64b2f545..f8b165155 100644 --- a/modoboa/core/migrations/0011_auto_20161104_2217.py +++ b/modoboa/core/migrations/0011_auto_20161104_2217.py @@ -1,6 +1,5 @@ # Generated by Django 1.9.5 on 2016-11-04 21:17 -from django.db import migrations -import jsonfield.fields +from django.db import migrations, models class Migration(migrations.Migration): @@ -13,11 +12,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='localconfig', name='_parameters', - field=jsonfield.fields.JSONField(default={}), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='user', name='_parameters', - field=jsonfield.fields.JSONField(default={}), + field=models.JSONField(default=dict), ), ] diff --git a/modoboa/core/migrations/0026_alter_localconfig__parameters_and_more.py b/modoboa/core/migrations/0026_alter_localconfig__parameters_and_more.py new file mode 100644 index 000000000..229b76704 --- /dev/null +++ b/modoboa/core/migrations/0026_alter_localconfig__parameters_and_more.py @@ -0,0 +1,55 @@ +# Generated by Django 4.2.6 on 2023-10-24 17:14 + +import django.core.serializers.json +from django.db import migrations, models +import modoboa.lib.json_field_utils + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0025_rename_user_email_is_active_core_user_email_c0c03f_idx'), + ] + + operations = [ + migrations.AlterField( + model_name='localconfig', + name='_parameters', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='localconfig', + name='api_versions', + field=models.JSONField(), + ), + migrations.AlterField( + model_name='user', + name='_parameters', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='localconfig', + name='api_versions', + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name='localconfig', + name='_cache', + field=models.JSONField(default=dict, encoder=modoboa.lib.json_field_utils.JSONDatetimeEncoder), + ), + migrations.AlterField( + model_name='localconfig', + name='_parameters', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + migrations.AlterField( + model_name='localconfig', + name='api_versions', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + migrations.AlterField( + model_name='user', + name='_parameters', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + ] diff --git a/modoboa/core/models.py b/modoboa/core/models.py index efff9e820..6f761ff77 100644 --- a/modoboa/core/models.py +++ b/modoboa/core/models.py @@ -3,6 +3,7 @@ import re from email.header import Header +from django.core.serializers.json import DjangoJSONEncoder from django.conf import settings from django.db import models from django.urls import reverse @@ -14,7 +15,6 @@ from django.contrib.auth.models import AbstractUser, Group from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType -import jsonfield from phonenumber_field.modelfields import PhoneNumberField from reversion import revisions as reversion @@ -22,7 +22,8 @@ from modoboa.core.password_hashers import get_password_hasher from modoboa.lib.exceptions import ( BadRequest, Conflict, InternalError, PermDeniedException ) -from modoboa.parameters import tools as param_tools +from modoboa.lib.json_field_utils import JSONDatetimeEncoder +from modoboa.parameters import tools as param_tools, cache_tools from . import constants, signals try: @@ -74,7 +75,7 @@ class User(AbstractUser): tfa_enabled = models.BooleanField(default=False) - _parameters = jsonfield.JSONField(default={}) + _parameters = models.JSONField(default=dict, encoder=DjangoJSONEncoder) class Meta(object): ordering = ["username"] @@ -447,9 +448,10 @@ class LocalConfig(models.Model): site = models.ForeignKey("sites.Site", on_delete=models.CASCADE) # API results cache - api_versions = jsonfield.JSONField() + api_versions = models.JSONField(default=dict, encoder=DjangoJSONEncoder) - _parameters = jsonfield.JSONField(default={}) + _parameters = models.JSONField(default=dict, encoder=DjangoJSONEncoder) + _cache = models.JSONField(default=dict, encoder=JSONDatetimeEncoder) # Dovecot LDAP update need_dovecot_update = models.BooleanField(default=False) @@ -458,6 +460,7 @@ class LocalConfig(models.Model): """Load parameter manager.""" super(LocalConfig, self).__init__(*args, **kwargs) self.parameters = param_tools.Manager("global", self._parameters) + self.cache = cache_tools.CacheManager(self._cache) class ExtensionUpdateHistory(models.Model): diff --git a/modoboa/core/password_hashers/__init__.py b/modoboa/core/password_hashers/__init__.py index aadf96bca..6a5e99ec4 100644 --- a/modoboa/core/password_hashers/__init__.py +++ b/modoboa/core/password_hashers/__init__.py @@ -10,8 +10,10 @@ from modoboa.core.password_hashers.advanced import ( # NOQA:F401 ARGON2IDHasher, SSHAHasher ) from modoboa.core.password_hashers.base import ( # NOQA:F401 - CRYPTHasher, MD5Hasher, PLAINHasher, SHA256Hasher + CRYPTHasher, MD5Hasher, PLAINHasher, SHA256Hasher, PasswordHasher ) +from modoboa.core import models + from modoboa.lib.sysutils import doveadm_cmd @@ -55,3 +57,17 @@ def get_dovecot_schemes(): return ["{{{}}}".format(smart_str(scheme)) for scheme in schemes.split()] + + +def cache_available_passowrd_hasher(): + available_schemes = get_dovecot_schemes() + password_scheme_choice = [ + (hasher.name, hasher.label) + for hasher in PasswordHasher.get_password_hashers() + if hasher().scheme in available_schemes + ] + localconfig = models.LocalConfig.objects.first() + localconfig.cache.set_cache_entry("password_scheme_choice", + password_scheme_choice, + "core") + localconfig.save(update_fields=["_cache"]) diff --git a/modoboa/core/tests/utils.py b/modoboa/core/tests/utils.py new file mode 100644 index 000000000..b9c841f64 --- /dev/null +++ b/modoboa/core/tests/utils.py @@ -0,0 +1,4 @@ +import os + +def get_doveadm_test_path(): + return "{}/doveadm".format(os.path.dirname(__file__)) diff --git a/modoboa/lib/json_field_utils.py b/modoboa/lib/json_field_utils.py new file mode 100644 index 000000000..83ea1ee8f --- /dev/null +++ b/modoboa/lib/json_field_utils.py @@ -0,0 +1,32 @@ +"""Utils regarding time management.""" + +import json + +from django.core.serializers.json import DjangoJSONEncoder + +from datetime import datetime, timezone + + +def datetime_to_json_tz_unaware(datetime_object=None): + """Take a datetime object and convert it to YYYY-MM-DDT-HH:MM:SS.""" + if datetime_object is None: + datetime_object = datetime.utcnow() + datetime_object = datetime_object.astimezone(timezone.utc) + return datetime_object.strftime("%Y-%m-%dT%H:%M:%S") + + +def json_to_datetime(datetime_string): + """Take a string in the YYYY-MM-DDT-HH:MM:SS form + and convert it to a datetime object.""" + if isinstance(datetime_string, datetime): + return datetime_string + return datetime.strptime(datetime_string, + "%Y-%m-%dT%H:%M:%S") + + +class JSONDatetimeEncoder(DjangoJSONEncoder): + def default(self, obj): + if isinstance(obj, datetime): + return datetime_to_json_tz_unaware(obj) + else: + return super().default(obj) diff --git a/modoboa/parameters/cache_tools.py b/modoboa/parameters/cache_tools.py new file mode 100644 index 000000000..20f4d2203 --- /dev/null +++ b/modoboa/parameters/cache_tools.py @@ -0,0 +1,61 @@ +"""Cache management.""" + +from datetime import datetime + +from modoboa.lib.json_field_utils import json_to_datetime +from modoboa.lib.sysutils import guess_extension_name + + +class CacheManager: + """A manager for cache.""" + + def __init__(self, cache): + """Constructor.""" + self._cache = cache + + def get_value(self, cache_label, app=None, raise_exception=True): + """Return the value associated to the specified parameter.""" + if app is None: + app = guess_extension_name() + cache_entry = self.get_cache_entry( + cache_label, app, raise_exception) + if cache_entry is None: + return None + return cache_entry.get("value") + + def get_last_mod(self, cache_label, app=None, raise_exception=True): + """Returns the last modification date for the specified parameter.""" + if app is None: + app = guess_extension_name() + cache_entry = self.get_cache_entry( + cache_label, app, raise_exception) + if cache_entry is None: + return None + return cache_entry.get("last_update") + + def get_cache_entry(self, cache_label, app=None, raise_exception=True): + """Return the entry associated to the specified parameter.""" + if app is None: + app = guess_extension_name() + # Compat. with the old way... + parameter = cache_label.lower() + try: + entry = self._cache[app][parameter] + entry["last_update"] = json_to_datetime(entry["last_update"]) + except KeyError: + if raise_exception: + raise + return None + + return entry + + def set_cache_entry(self, parameter, value, app=None): + """Set parameter for the given app.""" + if app is None: + app = guess_extension_name() + if self._cache.get(app) is None: + self._cache[app] = {} + self._cache[app][parameter] = { + "value": value, + "last_update": datetime.utcnow() + } diff --git a/modoboa/transport/migrations/0001_initial.py b/modoboa/transport/migrations/0001_initial.py index cd34ad3bd..450dc0d69 100644 --- a/modoboa/transport/migrations/0001_initial.py +++ b/modoboa/transport/migrations/0001_initial.py @@ -1,8 +1,6 @@ # Generated by Django 1.10.7 on 2017-11-24 11:16 from django.db import migrations, models -import jsonfield.fields - class Migration(migrations.Migration): @@ -19,7 +17,7 @@ class Migration(migrations.Migration): ('pattern', models.CharField(max_length=254, unique=True, verbose_name='pattern')), ('service', models.CharField(max_length=30, verbose_name='service')), ('next_hop', models.CharField(blank=True, max_length=100, verbose_name='next hop')), - ('_settings', jsonfield.fields.JSONField(default={})), + ('_settings', models.JSONField(default=dict)), ], options={ 'ordering': ['pattern'], diff --git a/modoboa/transport/migrations/0003_alter_transport__settings.py b/modoboa/transport/migrations/0003_alter_transport__settings.py new file mode 100644 index 000000000..79982d0d7 --- /dev/null +++ b/modoboa/transport/migrations/0003_alter_transport__settings.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.6 on 2023-10-24 17:14 + +import django.core.serializers.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transport', '0002_auto_20180928_1520'), + ] + + operations = [ + migrations.AlterField( + model_name='transport', + name='_settings', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='transport', + name='_settings', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + ] diff --git a/modoboa/transport/models.py b/modoboa/transport/models.py index 6356293b7..edd96fb58 100644 --- a/modoboa/transport/models.py +++ b/modoboa/transport/models.py @@ -1,8 +1,8 @@ """Transport models.""" -import jsonfield from reversion import revisions as reversion +from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.utils.translation import gettext_lazy as _ @@ -17,7 +17,7 @@ class Transport(models.Model): service = models.CharField(_("service"), max_length=30) next_hop = models.CharField(_("next hop"), max_length=100, blank=True) - _settings = jsonfield.JSONField(default={}) + _settings = models.JSONField(default=dict, encoder=DjangoJSONEncoder) class Meta: ordering = ["pattern"] diff --git a/requirements.txt b/requirements.txt index 214dd1b2a..4c3719175 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,6 @@ asgiref dnspython==2.4.2 feedparser==6.0.10 gevent==23.9.1 -jsonfield==3.1.0 Pillow # Optional dependency for django-ckeditor progressbar33==2.4 python-dateutil diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py index aa02ff62e..b173279a3 100644 --- a/test_project/test_project/settings.py +++ b/test_project/test_project/settings.py @@ -225,6 +225,8 @@ SPECTACULAR_SETTINGS = { # MODOBOA_CUSTOM_LOGO = os.path.join(MEDIA_URL, "custom_logo.png") # DOVECOT_LOOKUP_PATH = ('/path/to/dovecot', ) +DOVECOT_USER="root" + MODOBOA_API_URL = 'https://api.modoboa.org/1/' @@ -243,6 +245,9 @@ RQ_QUEUES = { 'dkim': { 'URL': REDIS_URL, }, + 'modoboa': { + 'URL': REDIS_URL, + }, } # Password validation