From 4ef53a8cd7450079554f2ec0969c48303d740176 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 20 Apr 2022 11:39:43 +0200 Subject: [PATCH 1/4] Django 3.2 support --- modoboa/admin/api/v1/serializers.py | 12 ++++++++---- modoboa/admin/api/v1/tests.py | 8 +++++--- modoboa/admin/apps.py | 2 +- modoboa/admin/models/domain_alias.py | 3 +-- modoboa/core/apps.py | 1 + modoboa/dnstools/apps.py | 1 + modoboa/dnstools/models.py | 2 -- modoboa/lib/templatetags/form_tags.py | 4 ++-- modoboa/limits/apps.py | 2 +- modoboa/maillog/apps.py | 1 + modoboa/parameters/apps.py | 2 +- modoboa/relaydomains/apps.py | 2 +- modoboa/transport/apps.py | 1 + requirements.txt | 4 ++-- 14 files changed, 26 insertions(+), 19 deletions(-) diff --git a/modoboa/admin/api/v1/serializers.py b/modoboa/admin/api/v1/serializers.py index 1027a04ca..6ef08418f 100644 --- a/modoboa/admin/api/v1/serializers.py +++ b/modoboa/admin/api/v1/serializers.py @@ -163,7 +163,14 @@ class MailboxSerializer(serializers.ModelSerializer): def validate_full_address(self, value): """Lower case address.""" - return value.lower() + value = value.lower() + address, domain_name = email_utils.split_mailbox(value) + domain = get_object_or_404( + admin_models.Domain, name=domain_name) + creator = self.context["request"].user + if not creator.can_access(domain): + raise serializers.ValidationError(_("Permission denied.")) + return value def validate_quota(self, value): """Convert quota to MB.""" @@ -353,9 +360,6 @@ class WritableAccountSerializer(AccountSerializer): address, domain_name = email_utils.split_mailbox(full_address) domain = get_object_or_404( admin_models.Domain, name=domain_name) - if not creator.can_access(domain): - raise serializers.ValidationError({ - "domain": _("Permission denied.")}) try: core_signals.can_create_object.send( sender=self.__class__, context=creator, diff --git a/modoboa/admin/api/v1/tests.py b/modoboa/admin/api/v1/tests.py index 2617cedd3..9545014ca 100644 --- a/modoboa/admin/api/v1/tests.py +++ b/modoboa/admin/api/v1/tests.py @@ -249,7 +249,7 @@ class AccountAPITestCase(ModoAPITestCase): def setUp(self): """Test setup.""" - super(AccountAPITestCase, self).setUp() + super().setUp() self.set_global_parameters({ "enable_admin_limits": False, "enable_domain_limits": False @@ -400,15 +400,17 @@ class AccountAPITestCase(ModoAPITestCase): data["mailbox"]["quota"] = 1000 url = reverse("v1:account-list") response = self.client.post(url, data, format="json") + # Should fail because domain quota is exceeded self.assertEqual(response.status_code, 400) + data["username"] = "fromapi@test2.com" data["mailbox"].update({"full_address": "fromapi@test2.com", "quota": 10}) - url = reverse("v1:account-list") response = self.client.post(url, data, format="json") + # Should fail because admin has no access to @test2.com domain self.assertEqual(response.status_code, 400) errors = response.json() - self.assertIn("domain", errors) + self.assertIn("mailbox", errors) def test_create_account_bad_master_user(self): """Try to create a new account.""" diff --git a/modoboa/admin/apps.py b/modoboa/admin/apps.py index e39753524..31e7e2808 100644 --- a/modoboa/admin/apps.py +++ b/modoboa/admin/apps.py @@ -6,9 +6,9 @@ from .app_settings import load_admin_settings class AdminConfig(AppConfig): - """App configuration.""" + default_auto_field = "django.db.models.BigAutoField" name = "modoboa.admin" verbose_name = "Modoboa admin console" diff --git a/modoboa/admin/models/domain_alias.py b/modoboa/admin/models/domain_alias.py index 6caa0a9cd..6f6e2d8e6 100644 --- a/modoboa/admin/models/domain_alias.py +++ b/modoboa/admin/models/domain_alias.py @@ -4,7 +4,7 @@ from reversion import revisions as reversion from django.contrib.contenttypes.fields import GenericRelation from django.db import models -from django.utils.encoding import python_2_unicode_compatible, smart_text +from django.utils.encoding import smart_text from django.utils.translation import ugettext as _, ugettext_lazy from modoboa.core import models as core_models, signals as core_signals @@ -26,7 +26,6 @@ class DomainAliasManager(models.Manager): return self.get_queryset().filter(owners__user=admin) -@python_2_unicode_compatible class DomainAlias(AdminObject): """Domain aliases.""" diff --git a/modoboa/core/apps.py b/modoboa/core/apps.py index 515174dce..4ab3218bd 100644 --- a/modoboa/core/apps.py +++ b/modoboa/core/apps.py @@ -26,6 +26,7 @@ def load_core_settings(): class CoreConfig(AppConfig): """App configuration.""" + default_auto_field = "django.db.models.BigAutoField" name = "modoboa.core" verbose_name = "Modoboa core" diff --git a/modoboa/dnstools/apps.py b/modoboa/dnstools/apps.py index cc7cbcead..c520d7b2d 100644 --- a/modoboa/dnstools/apps.py +++ b/modoboa/dnstools/apps.py @@ -2,4 +2,5 @@ from django.apps import AppConfig class DnstoolsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' name = 'modoboa.dnstools' diff --git a/modoboa/dnstools/models.py b/modoboa/dnstools/models.py index a2bd43a37..9c341773a 100644 --- a/modoboa/dnstools/models.py +++ b/modoboa/dnstools/models.py @@ -4,7 +4,6 @@ import datetime from django.db import models from django.utils import timezone -from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext as _ from . import constants @@ -35,7 +34,6 @@ class DNSRecordManager(models.Manager): return record -@python_2_unicode_compatible class DNSRecord(models.Model): """A model to store DNS records for Domain.""" diff --git a/modoboa/lib/templatetags/form_tags.py b/modoboa/lib/templatetags/form_tags.py index b7d7e3e4b..fad4637bb 100644 --- a/modoboa/lib/templatetags/form_tags.py +++ b/modoboa/lib/templatetags/form_tags.py @@ -56,7 +56,7 @@ def render_field( @register.simple_tag def render_fields_group(form, pattern): """Render a group of fields.""" - first = forms.forms.BoundField(form, form.fields[pattern], pattern) + first = forms.BoundField(form, form.fields[pattern], pattern) configure_field_classes(first) label = first.label group = [first] @@ -66,7 +66,7 @@ def render_fields_group(form, pattern): fname = "%s_%d" % (pattern, cpt) if fname not in form.fields: break - bfield = forms.forms.BoundField(form, form.fields[fname], fname) + bfield = forms.BoundField(form, form.fields[fname], fname) if len(bfield.errors): haserror = True configure_field_classes(bfield) diff --git a/modoboa/limits/apps.py b/modoboa/limits/apps.py index fa419ac58..8d3705f1b 100644 --- a/modoboa/limits/apps.py +++ b/modoboa/limits/apps.py @@ -18,9 +18,9 @@ def load_limits_settings(): class LimitsConfig(AppConfig): - """App configuration.""" + default_auto_field = "django.db.models.BigAutoField" name = "modoboa.limits" verbose_name = "Modoboa admin limits" diff --git a/modoboa/maillog/apps.py b/modoboa/maillog/apps.py index c4f62c1a5..f93bb1450 100644 --- a/modoboa/maillog/apps.py +++ b/modoboa/maillog/apps.py @@ -8,6 +8,7 @@ from .forms import load_settings class MaillogConfig(AppConfig): """App configuration.""" + default_auto_field = "django.db.models.BigAutoField" name = "modoboa.maillog" verbose_name = "Modoboa graphical statistics" diff --git a/modoboa/parameters/apps.py b/modoboa/parameters/apps.py index b3ed16382..0e49f0bef 100644 --- a/modoboa/parameters/apps.py +++ b/modoboa/parameters/apps.py @@ -2,4 +2,4 @@ from django.apps import AppConfig class ParametersConfig(AppConfig): - name = "parameters" + name = "modoboa.parameters" diff --git a/modoboa/relaydomains/apps.py b/modoboa/relaydomains/apps.py index 49ffe5fc8..1c3746c5c 100644 --- a/modoboa/relaydomains/apps.py +++ b/modoboa/relaydomains/apps.py @@ -4,9 +4,9 @@ from django.apps import AppConfig class RelayDomainsConfig(AppConfig): - """App configuration.""" + default_auto_field = "django.db.models.BigAutoField" name = "modoboa.relaydomains" verbose_name = "Modoboa relay domains" diff --git a/modoboa/transport/apps.py b/modoboa/transport/apps.py index 5beae6d6d..539e5c047 100644 --- a/modoboa/transport/apps.py +++ b/modoboa/transport/apps.py @@ -2,6 +2,7 @@ from django.apps import AppConfig class TransportConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" name = "modoboa.transport" def ready(self): diff --git a/requirements.txt b/requirements.txt index 4dacf67fe..a021429e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -django>=2.2,<2.3 +django<4.0 django-ckeditor==6.3.0 django-phonenumber-field[phonenumbers]==6.1.0 -django-reversion==3.0.9 +django-reversion==5.0.0 django-xforwardedfor-middleware==2.0 django-otp==1.1.3 django-filter From ed4ab0f8899e9f59152e9060cd1d67f81586cce1 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 20 Apr 2022 14:11:08 +0200 Subject: [PATCH 2/4] Fixed remaining issues. --- modoboa/admin/api/v1/serializers.py | 42 ++++++++++++++--------------- modoboa/policyd/core.py | 21 +++++---------- 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/modoboa/admin/api/v1/serializers.py b/modoboa/admin/api/v1/serializers.py index 6ef08418f..39adde055 100644 --- a/modoboa/admin/api/v1/serializers.py +++ b/modoboa/admin/api/v1/serializers.py @@ -163,14 +163,7 @@ class MailboxSerializer(serializers.ModelSerializer): def validate_full_address(self, value): """Lower case address.""" - value = value.lower() - address, domain_name = email_utils.split_mailbox(value) - domain = get_object_or_404( - admin_models.Domain, name=domain_name) - creator = self.context["request"].user - if not creator.can_access(domain): - raise serializers.ValidationError(_("Permission denied.")) - return value + return value.lower() def validate_quota(self, value): """Convert quota to MB.""" @@ -320,6 +313,24 @@ class WritableAccountSerializer(AccountSerializer): raise serializers.ValidationError({ "username": _("Must be equal to mailbox full_address") }) + if "mailbox" in data: + self.address, domain_name = email_utils.split_mailbox(data["mailbox"]["full_address"]) + self.domain = get_object_or_404( + admin_models.Domain, name=domain_name) + creator = self.context["request"].user + if not creator.can_access(self.domain): + raise serializers.ValidationError({"mailbox": _("Permission denied.")}) + if not self.instance: + try: + core_signals.can_create_object.send( + sender=self.__class__, context=creator, + klass=admin_models.Mailbox) + core_signals.can_create_object.send( + sender=self.__class__, context=self.domain, + object_type="mailboxes") + except lib_exceptions.ModoboaException as inst: + raise serializers.ValidationError({ + "mailbox": force_text(inst)}) condition = ( not data.get("random_password") and ( data.get("password") or @@ -357,22 +368,9 @@ class WritableAccountSerializer(AccountSerializer): def _create_mailbox(self, creator, account, data): """Create a new Mailbox instance.""" full_address = data.pop("full_address") - address, domain_name = email_utils.split_mailbox(full_address) - domain = get_object_or_404( - admin_models.Domain, name=domain_name) - try: - core_signals.can_create_object.send( - sender=self.__class__, context=creator, - klass=admin_models.Mailbox) - core_signals.can_create_object.send( - sender=self.__class__, context=domain, - object_type="mailboxes") - except lib_exceptions.ModoboaException as inst: - raise serializers.ValidationError({ - "domain": force_text(inst)}) quota = data.pop("quota", None) mb = admin_models.Mailbox( - user=account, address=address, domain=domain, **data) + user=account, address=self.address, domain=self.domain, **data) mb.set_quota(quota, creator.has_perm("admin.add_domain")) default_msg_limit = param_tools.get_global_parameter( "default_mailbox_message_limit") diff --git a/modoboa/policyd/core.py b/modoboa/policyd/core.py index 917d98234..23ae7d433 100644 --- a/modoboa/policyd/core.py +++ b/modoboa/policyd/core.py @@ -1,5 +1,6 @@ """Core components of the policy daemon.""" +from asgiref.sync import sync_to_async import asyncio import concurrent.futures from email.message import EmailMessage @@ -216,7 +217,7 @@ def get_next_execution_dt(): hour=0, minute=0, second=0) -@close_db_connections +@sync_to_async def get_domains_to_reset(): """ Return a list of domain to reset. @@ -230,10 +231,10 @@ def get_domains_to_reset(): ).update( status=admin_constants.ALARM_CLOSED, closed=timezone.now() ) - return qset + return list(qset) -@close_db_connections +@sync_to_async def get_mailboxes_to_reset(): """ Return a list of mailboxes to reset. @@ -250,25 +251,17 @@ def get_mailboxes_to_reset(): ).update( status=admin_constants.ALARM_CLOSED, closed=timezone.now() ) - return qset + return list(qset) async def reset_counters(): """Reset all counters.""" rclient = await aioredis.create_redis_pool(settings.REDIS_URL) logger.info("Resetting all counters") - # We're going to execute sync code so we need an executor - executor = concurrent.futures.ThreadPoolExecutor(max_workers=3) - loop = asyncio.get_event_loop() - futures = [ - loop.run_in_executor(executor, get_domains_to_reset), - loop.run_in_executor(executor, get_mailboxes_to_reset) - ] - domains, mboxes = await asyncio.gather(*futures) - for domain in domains: + for domain in await get_domains_to_reset(): rclient.hset( constants.REDIS_HASHNAME, domain.name, domain.message_limit) - for mb in mboxes: + for mb in await get_mailboxes_to_reset(): rclient.hset( constants.REDIS_HASHNAME, mb.full_address, mb.message_limit) rclient.close() From 6ee9074bee3e07bf565e3a2cf0ec232c9e7e96f4 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 20 Apr 2022 14:30:08 +0200 Subject: [PATCH 3/4] Fixed API v2 tests --- modoboa/admin/api/v2/serializers.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/modoboa/admin/api/v2/serializers.py b/modoboa/admin/api/v2/serializers.py index 99c300d8e..be352a1bc 100644 --- a/modoboa/admin/api/v2/serializers.py +++ b/modoboa/admin/api/v2/serializers.py @@ -5,6 +5,7 @@ import os from django.conf import settings from django.core.exceptions import ValidationError from django.core import validators as dj_validators +from django.shortcuts import get_object_or_404 from django.utils.translation import ugettext as _ from django.contrib.auth import password_validation @@ -13,6 +14,7 @@ from rest_framework import serializers from modoboa.admin.api.v1 import serializers as v1_serializers from modoboa.core import models as core_models, signals as core_signals +from modoboa.lib import email_utils from modoboa.lib import exceptions as lib_exceptions from modoboa.lib import fields as lib_fields from modoboa.lib import validators, web_utils @@ -402,6 +404,25 @@ class WritableAccountSerializer(v1_serializers.WritableAccountSerializer): data["mailbox"] = { "use_domain_quota": True } + if "mailbox" in data: + self.address, domain_name = email_utils.split_mailbox(data["username"]) + self.domain = get_object_or_404( + models.Domain, name=domain_name) + creator = self.context["request"].user + if not creator.can_access(self.domain): + raise serializers.ValidationError({"mailbox": _("Permission denied.")}) + if not self.instance: + try: + core_signals.can_create_object.send( + sender=self.__class__, context=creator, + klass=models.Mailbox) + core_signals.can_create_object.send( + sender=self.__class__, context=self.domain, + object_type="mailboxes") + except lib_exceptions.ModoboaException as inst: + raise serializers.ValidationError({ + "mailbox": force_text(inst) + }) if data.get("password") or not self.partial: password = data.get("password") if password: From c5765155bc77e63264eb620933a0522844630a60 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 20 Apr 2022 14:55:14 +0200 Subject: [PATCH 4/4] Reintroduced connections closing --- modoboa/policyd/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modoboa/policyd/core.py b/modoboa/policyd/core.py index 23ae7d433..0f8f7c20b 100644 --- a/modoboa/policyd/core.py +++ b/modoboa/policyd/core.py @@ -218,6 +218,7 @@ def get_next_execution_dt(): @sync_to_async +@close_db_connections def get_domains_to_reset(): """ Return a list of domain to reset. @@ -235,6 +236,7 @@ def get_domains_to_reset(): @sync_to_async +@close_db_connections def get_mailboxes_to_reset(): """ Return a list of mailboxes to reset.