diff --git a/doc/upgrade.rst b/doc/upgrade.rst index 1b9bc598f..3d8847a8c 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -131,6 +131,57 @@ Rebuild Virtual Environment Specific instructions ********************* + +2.2.3 +===== + +* Multiple security fixes +* Fixed a CRSF error when logging multiple time on v1 +* Reduced the number of call to doveadm + +You to setup a new worker : + +1. Create a new supervisord config (``/etc/supervisor/conf.d/modoboa-base-worker.conf`` by default) : + +.. sourcecode:: ini + + [program:modoboa-base-worker] + autostart=true + autorestart=true + command={%python env path%} {% manage.py instance path%} rqworker modoboa + directory={%modoboa home dir%} + user=%{modoboa user} + numprocs=1 + stopsignal=TERM + +``python env path`` : Python executable located in your virtual environment created for modoboa. +You will find it here ``/srv/modoboa/venv/bin/python`` by default. +``manage.py instance path``: Path to manage.py of your modoboa instance. +You will find it here : ``/srv/modoboa/instance/manage.py``by default. +``Modoboa home dir``: Home dir of the user running modooba. +You will find it here ``/srv/modoboa/`` by default. +``modoboa user`` : User managing dkim signing (modoboa by default). + +You can help you with ``/etc/supervisor/conf.d/policyd.conf`` (by default). + +Then restart supervisor. ``#> supervisorctl reread && supervisorctl update`` on Debian. + +2. Update the RQ_QUEUES section bellow the ``#REDIS`` section: + +.. sourcecode:: python + + # RQ + RQ_QUEUES = { + ... + }, + 'modoboa': { + 'HOST': REDIS_HOST, + 'PORT': REDIS_PORT, + 'DB': 0, + }, + } + + 2.2.0 ===== diff --git a/docker-compose.yml b/docker-compose.yml index 7a98b2d29..d5a21a4c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: DEBUG: "True" REDIS_HOST: redis command: > - sh -c "cd test_project; python3 manage.py rqworker dkim" + sh -c "cd test_project; python3 manage.py rqworker dkim modoboa" volumes: - .:/code diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index d0187cf93..cd426dca6 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM python:3.9-alpine as base +FROM python:3.12-alpine as base MAINTAINER Antoine Nguyen ENV VIRTUAL_ENV=/opt/venv @@ -17,8 +17,10 @@ RUN python -m pip install -U pip RUN pip install -r requirements.txt -r test-requirements.txt FROM base as run +COPY docker/doveadm /usr/local/bin +RUN chmod +x /usr/local/bin/doveadm COPY --from=build $VIRTUAL_ENV $VIRTUAL_ENV -RUN apk add --no-cache --update openssl librrd libxslt libjpeg libressl git rrdtool libmagic \ +RUN apk add --no-cache --update openssl librrd libxslt libjpeg libressl git rrdtool libmagic sudo \ && rm -rf /var/cache/apk/* RUN mkdir /code WORKDIR /code diff --git a/docker/Dockerfile.front.dev b/docker/Dockerfile.front.dev index 8149784bc..5e335639f 100644 --- a/docker/Dockerfile.front.dev +++ b/docker/Dockerfile.front.dev @@ -1,7 +1,10 @@ -FROM node:16.10-alpine +FROM node:16.10-alpine ENV HOME=/home/node/ +RUN apk add --no-cache --update gettext \ + && rm -rf /var/cache/apk/* + WORKDIR $HOME RUN yarn install diff --git a/docker/doveadm b/docker/doveadm new file mode 100755 index 000000000..c8e8845fb --- /dev/null +++ b/docker/doveadm @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "SHA1 SSHA512 SCRAM-SHA-256 BLF-CRYPT PLAIN HMAC-MD5 OTP SHA512 SHA DES-CRYPT CRYPT SSHA MD5-CRYPT PLAIN-MD4 PLAIN-MD5 SCRAM-SHA-1 SHA512-CRYPT CLEAR CLEARTEXT ARGON2I ARGON2ID SSHA256 MD5 PBKDF2 SHA256 CRAM-MD5 PLAIN-TRUNC SHA256-CRYPT SMD5 DIGEST-MD5 LDAP-MD5" diff --git a/frontend/package.json b/frontend/package.json index 5aacece55..b82b971f1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,6 +16,7 @@ "debounce": "^1.2.1", "js-cookie": "^2.2.1", "luxon": "^1.28.1", + "qrcode.vue": "~1.7.0", "vee-validate": "^3.4.5", "vue": "^2.6.14", "vue-apexcharts": "^1.6.0", @@ -23,8 +24,7 @@ "vue-gettext": "^2.1.12", "vue-router": "^3.2.0", "vuetify": "^2.6.10", - "vuex": "^3.4.0", - "qrcode.vue": "~1.7.0" + "vuex": "^3.4.0" }, "devDependencies": { "@babel/core": "^7.12.16", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 27325a57f..a3e82686a 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -2515,9 +2515,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001400, caniuse-lite@^1.0.30001407: - version "1.0.30001481" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001481.tgz" - integrity sha512-KCqHwRnaa1InZBtqXzP98LPg0ajCVujMKjqKDhZEthIpAsJl/YEIa3YvXjGXPVqzZVguccuu7ga9KOE1J9rKPQ== + version "1.0.30001553" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001553.tgz" + integrity sha512-N0ttd6TrFfuqKNi+pMgWJTb9qrdJu4JSpgPFLe/lrD19ugC6fZgF0pUewRowDwzdDnb9V41mFcdlYgl/PyKf4A== case-sensitive-paths-webpack-plugin@^2.3.0: version "2.4.0" diff --git a/modoboa/core/api/v2/serializers.py b/modoboa/core/api/v2/serializers.py index 56cf3866d..0f78cb4fd 100644 --- a/modoboa/core/api/v2/serializers.py +++ b/modoboa/core/api/v2/serializers.py @@ -21,7 +21,9 @@ from django.utils.translation import gettext_lazy, gettext as _ from rest_framework import serializers, status from rest_framework.exceptions import APIException -from modoboa.core import constants, models, sms_backends, app_settings, context_processors +from modoboa.core import ( + constants, models, sms_backends, app_settings, context_processors +) from modoboa.core.models import User from modoboa.lib import fields as lib_fields, cryptutils @@ -62,21 +64,15 @@ class CoreGlobalParametersSerializer(serializers.Serializer): default="local" ) password_scheme = serializers.ChoiceField( - choices=[("sha512crypt", "sha512crypt"), - ("sha256crypt", "sha256crypt"), - ("blfcrypt", "bcrypt"), - ("md5crypt", gettext_lazy("md5crypt (weak)")), - ("sha256", gettext_lazy("sha256 (weak)")), - ("md5", gettext_lazy("md5 (weak)")), - ("crypt", gettext_lazy("crypt (weak)")), - ("plain", gettext_lazy("plain (weak)"))], + choices=[("sha512crypt", "sha512crypt")], default="sha512crypt" ) rounds_number = serializers.IntegerField(default=70000) update_scheme = serializers.BooleanField(default=True) - default_password = serializers.CharField(default="password") + default_password = serializers.CharField(default="ChangeMe1!") random_password_length = serializers.IntegerField(min_value=8, default=8) - update_password_url = serializers.URLField(required=False, allow_blank=True) + update_password_url = serializers.URLField( + required=False, allow_blank=True) password_recovery_msg = serializers.CharField( required=False, allow_blank=True) sms_password_recovery = serializers.BooleanField(default=False) @@ -173,8 +169,27 @@ class CoreGlobalParametersSerializer(serializers.Serializer): for field, definition in sms_backend_fields.items(): self.fields[field] = definition["type"]( **definition["attrs"]) + + try: + self.saved_password_scheme = args[0].get("password_scheme") + self.saved_default_top_redirection = args[0].get("default_top_redirection") + except IndexError: + self.saved_password_scheme = None + self.saved_default_top_redirection = None # Choices serializer for default_top_redirection field self.fields["default_top_redirection"].choices = app_settings.enabled_applications() + # Populate choices of password_scheme + localconfig = models.LocalConfig.objects.first() + self.fields["password_scheme"].choices = app_settings.get_password_scheme(localconfig) + + @property + def data(self): + ret = super().data + if self.saved_password_scheme is not None: + ret["password_scheme"] = self.saved_password_scheme + if self.saved_default_top_redirection is not None: + ret["default_top_redirection"] = self.saved_default_top_redirection + return ret def validate_ldap_user_dn_template(self, value): try: diff --git a/modoboa/core/api/v2/tests.py b/modoboa/core/api/v2/tests.py index cc0ccf019..aeadabccf 100644 --- a/modoboa/core/api/v2/tests.py +++ b/modoboa/core/api/v2/tests.py @@ -1,18 +1,26 @@ """Core API related tests.""" import copy +import datetime +import getpass import oath from unittest import mock +from django.test import override_settings from django.urls import reverse from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_encode +from django_rq import get_worker, get_queue from modoboa.admin import factories -from modoboa.core import models +from modoboa.core import models, jobs +from modoboa.core.tests import utils from modoboa.lib.tests import ModoAPITestCase from rest_framework.authtoken.models import Token +DOVEADM_TEST_PATH = utils.get_doveadm_test_path() +DOVECOT_USER = getpass.getuser() + CORE_SETTINGS = { "authentication_type": "local", "password_scheme": "sha512crypt", @@ -70,6 +78,8 @@ CORE_SETTINGS = { class ParametersAPITestCase(ModoAPITestCase): + @override_settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH], + DOVECOT_USER=DOVECOT_USER) def test_update(self): url = reverse("v2:parameter-detail", args=["core"]) data = copy.copy(CORE_SETTINGS) @@ -115,6 +125,49 @@ class ParametersAPITestCase(ModoAPITestCase): resp = self.client.put(url, data, format="json") self.assertEqual(resp.status_code, 200) + @override_settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH], + DOVECOT_USER=DOVECOT_USER) + def test_cache(self): + """Launch the password scheme job manually and check the result.""" + queue = get_queue("modoboa", is_async=False) + queue.empty() + job = queue.enqueue(jobs.job_retrieve_available_hashers, + bypass_condition=True) + get_worker("modoboa").work(burst=True) + job_result = job.latest_result() + self.assertEqual(job_result.type, job_result.Type.SUCCESSFUL) + localconfig = models.LocalConfig.objects.first() + cache = localconfig.cache + self.assertIn(['plain', 'plain (weak)'], + cache.get_value("password_scheme_choice", "core")) + entry = cache.get_cache_entry("password_scheme_choice", "core") + self.assertEqual(entry["last_update"], + cache.get_last_mod("password_scheme_choice", "core")) + entry_guess_extension = cache.get_cache_entry("password_scheme_choice") + self.assertEqual(entry, entry_guess_extension) + cache.set_cache_entry("test_cache", "test") + self.assertEqual( + cache.get_value("test_cache", "core"), + "test") + + # Check that the cache will be updated once a month + localconfig._cache["core"]["password_scheme_choice"]["value"] = "outofdate!" + localconfig.save() + job = queue.enqueue(jobs.job_retrieve_available_hashers) + get_worker("modoboa").work(burst=True) + localconfig = models.LocalConfig.objects.first() + value = localconfig.cache.get_value("password_scheme_choice") + self.assertEqual(value, "outofdate!") + + old_datetime = datetime.datetime.utcnow() - datetime.timedelta(weeks=5) + localconfig._cache["core"]["password_scheme_choice"]["last_update"] = old_datetime + localconfig.save() + job = queue.enqueue(jobs.job_retrieve_available_hashers) + get_worker("modoboa").work(burst=True) + localconfig = models.LocalConfig.objects.first() + value = localconfig.cache.get_value("password_scheme_choice") + self.assertFalse(value == "outofdate!") + class AccountViewSetTestCase(ModoAPITestCase): diff --git a/modoboa/core/app_settings.py b/modoboa/core/app_settings.py index ad817c961..5a80717ba 100644 --- a/modoboa/core/app_settings.py +++ b/modoboa/core/app_settings.py @@ -6,7 +6,9 @@ from django import forms from django.conf import settings from django.contrib.auth import password_validation from django.utils.translation import gettext as _, gettext_lazy +from django_rq import get_queue +from modoboa.core.jobs import job_retrieve_available_hashers from modoboa.core.password_hashers import get_dovecot_schemes from modoboa.core.password_hashers.base import PasswordHasher from modoboa.lib import fields as lib_fields @@ -31,6 +33,27 @@ def enabled_applications(): return sorted(result, key=lambda e: e[0]) +def get_default_password_scheme(): + available_schemes = get_dovecot_schemes() + return [ + (hasher.name, gettext_lazy(hasher.label)) + for hasher in PasswordHasher.get_password_hashers() + if hasher().scheme in available_schemes + ] + + +def get_password_scheme(localconfig): + if localconfig is None: + return get_default_password_scheme() + available_schemes = localconfig.cache.get_value( + "password_scheme_choice", "core", False) + get_queue("modoboa").enqueue( + job_retrieve_available_hashers) + if available_schemes is None: + return get_default_password_scheme() + return available_schemes + + class GeneralParametersForm(param_forms.AdminParametersForm): """General parameters.""" @@ -73,7 +96,7 @@ class GeneralParametersForm(param_forms.AdminParametersForm): default_password = forms.CharField( label=gettext_lazy("Default password"), - initial="password", + initial="ChangeMe1!", help_text=gettext_lazy( "Default password for automatically created accounts.") ) @@ -535,12 +558,7 @@ class GeneralParametersForm(param_forms.AdminParametersForm): self._add_visibilty_rules( sms_backends.get_all_backend_visibility_rules() ) - available_schemes = get_dovecot_schemes() - self.fields["password_scheme"].choices = [ - (hasher.name, gettext_lazy(hasher.label)) - for hasher in PasswordHasher.get_password_hashers() - if hasher().scheme in available_schemes - ] + self.fields["password_scheme"].choices = get_password_scheme(self.localconfig) def _add_dynamic_fields(self): new_fields = collections.OrderedDict() diff --git a/modoboa/core/commands/templates/settings.py.tpl b/modoboa/core/commands/templates/settings.py.tpl index c9e3874ee..6bfbb8c37 100644 --- a/modoboa/core/commands/templates/settings.py.tpl +++ b/modoboa/core/commands/templates/settings.py.tpl @@ -262,6 +262,11 @@ RQ_QUEUES = { 'PORT': REDIS_PORT, 'DB': 0, }, + 'modoboa': { + 'HOST': REDIS_HOST, + 'PORT': REDIS_PORT, + 'DB': 0, + }, } # Password validation diff --git a/modoboa/core/jobs.py b/modoboa/core/jobs.py new file mode 100644 index 000000000..4da42016b --- /dev/null +++ b/modoboa/core/jobs.py @@ -0,0 +1,19 @@ +from datetime import timedelta, datetime + +from modoboa.core import models +from modoboa.core.password_hashers import cache_available_passowrd_hasher + + +def job_retrieve_available_hashers(bypass_condition=False): + localconfig = models.LocalConfig.objects.first() + cache = localconfig.cache + password_scheme_last_mod = cache.get_last_mod( + "password_scheme_choice", "core", False) + cur_dt = datetime.utcnow() + condition = (bypass_condition or + password_scheme_last_mod is None or + password_scheme_last_mod < cur_dt - timedelta(weeks=4) + ) + if condition: + cache_available_passowrd_hasher() + diff --git a/modoboa/core/migrations/0008_localconfig.py b/modoboa/core/migrations/0008_localconfig.py index 9dae77efb..a9df6ff8d 100644 --- a/modoboa/core/migrations/0008_localconfig.py +++ b/modoboa/core/migrations/0008_localconfig.py @@ -1,7 +1,6 @@ # Generated by Django 1.9.5 on 2016-06-15 18:12 from django.db import migrations, models import django.db.models.deletion -import jsonfield.fields class Migration(migrations.Migration): @@ -17,7 +16,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('api_pk', models.PositiveIntegerField(null=True)), - ('api_versions', jsonfield.fields.JSONField()), + ('api_versions', models.JSONField()), ('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sites.Site')), ], ), diff --git a/modoboa/core/migrations/0010_auto_20161026_1011.py b/modoboa/core/migrations/0010_auto_20161026_1011.py index e681c79b8..930f252a9 100644 --- a/modoboa/core/migrations/0010_auto_20161026_1011.py +++ b/modoboa/core/migrations/0010_auto_20161026_1011.py @@ -1,6 +1,5 @@ # Generated by Django 1.9.5 on 2016-10-26 08:11 -from django.db import migrations -import jsonfield.fields +from django.db import migrations, models def set_parameter(store, parameter): @@ -51,13 +50,13 @@ class Migration(migrations.Migration): migrations.AddField( model_name='localconfig', name='_parameters', - field=jsonfield.fields.JSONField(default={}), + field=models.JSONField(default=dict), preserve_default=False, ), migrations.AddField( model_name='user', name='_parameters', - field=jsonfield.fields.JSONField(default={}), + field=models.JSONField(default=dict), preserve_default=False, ), migrations.RunPython(move_parameters, clear_parameters) diff --git a/modoboa/core/migrations/0011_auto_20161104_2217.py b/modoboa/core/migrations/0011_auto_20161104_2217.py index f64b2f545..f8b165155 100644 --- a/modoboa/core/migrations/0011_auto_20161104_2217.py +++ b/modoboa/core/migrations/0011_auto_20161104_2217.py @@ -1,6 +1,5 @@ # Generated by Django 1.9.5 on 2016-11-04 21:17 -from django.db import migrations -import jsonfield.fields +from django.db import migrations, models class Migration(migrations.Migration): @@ -13,11 +12,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='localconfig', name='_parameters', - field=jsonfield.fields.JSONField(default={}), + field=models.JSONField(default=dict), ), migrations.AlterField( model_name='user', name='_parameters', - field=jsonfield.fields.JSONField(default={}), + field=models.JSONField(default=dict), ), ] diff --git a/modoboa/core/migrations/0026_alter_localconfig__parameters_and_more.py b/modoboa/core/migrations/0026_alter_localconfig__parameters_and_more.py new file mode 100644 index 000000000..229b76704 --- /dev/null +++ b/modoboa/core/migrations/0026_alter_localconfig__parameters_and_more.py @@ -0,0 +1,55 @@ +# Generated by Django 4.2.6 on 2023-10-24 17:14 + +import django.core.serializers.json +from django.db import migrations, models +import modoboa.lib.json_field_utils + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0025_rename_user_email_is_active_core_user_email_c0c03f_idx'), + ] + + operations = [ + migrations.AlterField( + model_name='localconfig', + name='_parameters', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='localconfig', + name='api_versions', + field=models.JSONField(), + ), + migrations.AlterField( + model_name='user', + name='_parameters', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='localconfig', + name='api_versions', + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name='localconfig', + name='_cache', + field=models.JSONField(default=dict, encoder=modoboa.lib.json_field_utils.JSONDatetimeEncoder), + ), + migrations.AlterField( + model_name='localconfig', + name='_parameters', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + migrations.AlterField( + model_name='localconfig', + name='api_versions', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + migrations.AlterField( + model_name='user', + name='_parameters', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + ] diff --git a/modoboa/core/models.py b/modoboa/core/models.py index efff9e820..6f761ff77 100644 --- a/modoboa/core/models.py +++ b/modoboa/core/models.py @@ -3,6 +3,7 @@ import re from email.header import Header +from django.core.serializers.json import DjangoJSONEncoder from django.conf import settings from django.db import models from django.urls import reverse @@ -14,7 +15,6 @@ from django.contrib.auth.models import AbstractUser, Group from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType -import jsonfield from phonenumber_field.modelfields import PhoneNumberField from reversion import revisions as reversion @@ -22,7 +22,8 @@ from modoboa.core.password_hashers import get_password_hasher from modoboa.lib.exceptions import ( BadRequest, Conflict, InternalError, PermDeniedException ) -from modoboa.parameters import tools as param_tools +from modoboa.lib.json_field_utils import JSONDatetimeEncoder +from modoboa.parameters import tools as param_tools, cache_tools from . import constants, signals try: @@ -74,7 +75,7 @@ class User(AbstractUser): tfa_enabled = models.BooleanField(default=False) - _parameters = jsonfield.JSONField(default={}) + _parameters = models.JSONField(default=dict, encoder=DjangoJSONEncoder) class Meta(object): ordering = ["username"] @@ -447,9 +448,10 @@ class LocalConfig(models.Model): site = models.ForeignKey("sites.Site", on_delete=models.CASCADE) # API results cache - api_versions = jsonfield.JSONField() + api_versions = models.JSONField(default=dict, encoder=DjangoJSONEncoder) - _parameters = jsonfield.JSONField(default={}) + _parameters = models.JSONField(default=dict, encoder=DjangoJSONEncoder) + _cache = models.JSONField(default=dict, encoder=JSONDatetimeEncoder) # Dovecot LDAP update need_dovecot_update = models.BooleanField(default=False) @@ -458,6 +460,7 @@ class LocalConfig(models.Model): """Load parameter manager.""" super(LocalConfig, self).__init__(*args, **kwargs) self.parameters = param_tools.Manager("global", self._parameters) + self.cache = cache_tools.CacheManager(self._cache) class ExtensionUpdateHistory(models.Model): diff --git a/modoboa/core/password_hashers/__init__.py b/modoboa/core/password_hashers/__init__.py index aadf96bca..6a5e99ec4 100644 --- a/modoboa/core/password_hashers/__init__.py +++ b/modoboa/core/password_hashers/__init__.py @@ -10,8 +10,10 @@ from modoboa.core.password_hashers.advanced import ( # NOQA:F401 ARGON2IDHasher, SSHAHasher ) from modoboa.core.password_hashers.base import ( # NOQA:F401 - CRYPTHasher, MD5Hasher, PLAINHasher, SHA256Hasher + CRYPTHasher, MD5Hasher, PLAINHasher, SHA256Hasher, PasswordHasher ) +from modoboa.core import models + from modoboa.lib.sysutils import doveadm_cmd @@ -55,3 +57,17 @@ def get_dovecot_schemes(): return ["{{{}}}".format(smart_str(scheme)) for scheme in schemes.split()] + + +def cache_available_passowrd_hasher(): + available_schemes = get_dovecot_schemes() + password_scheme_choice = [ + (hasher.name, hasher.label) + for hasher in PasswordHasher.get_password_hashers() + if hasher().scheme in available_schemes + ] + localconfig = models.LocalConfig.objects.first() + localconfig.cache.set_cache_entry("password_scheme_choice", + password_scheme_choice, + "core") + localconfig.save(update_fields=["_cache"]) diff --git a/modoboa/core/tests/utils.py b/modoboa/core/tests/utils.py new file mode 100644 index 000000000..b9c841f64 --- /dev/null +++ b/modoboa/core/tests/utils.py @@ -0,0 +1,4 @@ +import os + +def get_doveadm_test_path(): + return "{}/doveadm".format(os.path.dirname(__file__)) diff --git a/modoboa/lib/json_field_utils.py b/modoboa/lib/json_field_utils.py new file mode 100644 index 000000000..83ea1ee8f --- /dev/null +++ b/modoboa/lib/json_field_utils.py @@ -0,0 +1,32 @@ +"""Utils regarding time management.""" + +import json + +from django.core.serializers.json import DjangoJSONEncoder + +from datetime import datetime, timezone + + +def datetime_to_json_tz_unaware(datetime_object=None): + """Take a datetime object and convert it to YYYY-MM-DDT-HH:MM:SS.""" + if datetime_object is None: + datetime_object = datetime.utcnow() + datetime_object = datetime_object.astimezone(timezone.utc) + return datetime_object.strftime("%Y-%m-%dT%H:%M:%S") + + +def json_to_datetime(datetime_string): + """Take a string in the YYYY-MM-DDT-HH:MM:SS form + and convert it to a datetime object.""" + if isinstance(datetime_string, datetime): + return datetime_string + return datetime.strptime(datetime_string, + "%Y-%m-%dT%H:%M:%S") + + +class JSONDatetimeEncoder(DjangoJSONEncoder): + def default(self, obj): + if isinstance(obj, datetime): + return datetime_to_json_tz_unaware(obj) + else: + return super().default(obj) diff --git a/modoboa/parameters/cache_tools.py b/modoboa/parameters/cache_tools.py new file mode 100644 index 000000000..20f4d2203 --- /dev/null +++ b/modoboa/parameters/cache_tools.py @@ -0,0 +1,61 @@ +"""Cache management.""" + +from datetime import datetime + +from modoboa.lib.json_field_utils import json_to_datetime +from modoboa.lib.sysutils import guess_extension_name + + +class CacheManager: + """A manager for cache.""" + + def __init__(self, cache): + """Constructor.""" + self._cache = cache + + def get_value(self, cache_label, app=None, raise_exception=True): + """Return the value associated to the specified parameter.""" + if app is None: + app = guess_extension_name() + cache_entry = self.get_cache_entry( + cache_label, app, raise_exception) + if cache_entry is None: + return None + return cache_entry.get("value") + + def get_last_mod(self, cache_label, app=None, raise_exception=True): + """Returns the last modification date for the specified parameter.""" + if app is None: + app = guess_extension_name() + cache_entry = self.get_cache_entry( + cache_label, app, raise_exception) + if cache_entry is None: + return None + return cache_entry.get("last_update") + + def get_cache_entry(self, cache_label, app=None, raise_exception=True): + """Return the entry associated to the specified parameter.""" + if app is None: + app = guess_extension_name() + # Compat. with the old way... + parameter = cache_label.lower() + try: + entry = self._cache[app][parameter] + entry["last_update"] = json_to_datetime(entry["last_update"]) + except KeyError: + if raise_exception: + raise + return None + + return entry + + def set_cache_entry(self, parameter, value, app=None): + """Set parameter for the given app.""" + if app is None: + app = guess_extension_name() + if self._cache.get(app) is None: + self._cache[app] = {} + self._cache[app][parameter] = { + "value": value, + "last_update": datetime.utcnow() + } diff --git a/modoboa/transport/migrations/0001_initial.py b/modoboa/transport/migrations/0001_initial.py index cd34ad3bd..450dc0d69 100644 --- a/modoboa/transport/migrations/0001_initial.py +++ b/modoboa/transport/migrations/0001_initial.py @@ -1,8 +1,6 @@ # Generated by Django 1.10.7 on 2017-11-24 11:16 from django.db import migrations, models -import jsonfield.fields - class Migration(migrations.Migration): @@ -19,7 +17,7 @@ class Migration(migrations.Migration): ('pattern', models.CharField(max_length=254, unique=True, verbose_name='pattern')), ('service', models.CharField(max_length=30, verbose_name='service')), ('next_hop', models.CharField(blank=True, max_length=100, verbose_name='next hop')), - ('_settings', jsonfield.fields.JSONField(default={})), + ('_settings', models.JSONField(default=dict)), ], options={ 'ordering': ['pattern'], diff --git a/modoboa/transport/migrations/0003_alter_transport__settings.py b/modoboa/transport/migrations/0003_alter_transport__settings.py new file mode 100644 index 000000000..79982d0d7 --- /dev/null +++ b/modoboa/transport/migrations/0003_alter_transport__settings.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.6 on 2023-10-24 17:14 + +import django.core.serializers.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transport', '0002_auto_20180928_1520'), + ] + + operations = [ + migrations.AlterField( + model_name='transport', + name='_settings', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='transport', + name='_settings', + field=models.JSONField(default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder), + ), + ] diff --git a/modoboa/transport/models.py b/modoboa/transport/models.py index 6356293b7..edd96fb58 100644 --- a/modoboa/transport/models.py +++ b/modoboa/transport/models.py @@ -1,8 +1,8 @@ """Transport models.""" -import jsonfield from reversion import revisions as reversion +from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.utils.translation import gettext_lazy as _ @@ -17,7 +17,7 @@ class Transport(models.Model): service = models.CharField(_("service"), max_length=30) next_hop = models.CharField(_("next hop"), max_length=100, blank=True) - _settings = jsonfield.JSONField(default={}) + _settings = models.JSONField(default=dict, encoder=DjangoJSONEncoder) class Meta: ordering = ["pattern"] diff --git a/requirements.txt b/requirements.txt index 214dd1b2a..4c3719175 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,6 @@ asgiref dnspython==2.4.2 feedparser==6.0.10 gevent==23.9.1 -jsonfield==3.1.0 Pillow # Optional dependency for django-ckeditor progressbar33==2.4 python-dateutil diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py index aa02ff62e..b173279a3 100644 --- a/test_project/test_project/settings.py +++ b/test_project/test_project/settings.py @@ -225,6 +225,8 @@ SPECTACULAR_SETTINGS = { # MODOBOA_CUSTOM_LOGO = os.path.join(MEDIA_URL, "custom_logo.png") # DOVECOT_LOOKUP_PATH = ('/path/to/dovecot', ) +DOVECOT_USER="root" + MODOBOA_API_URL = 'https://api.modoboa.org/1/' @@ -243,6 +245,9 @@ RQ_QUEUES = { 'dkim': { 'URL': REDIS_URL, }, + 'modoboa': { + 'URL': REDIS_URL, + }, } # Password validation