Merge pull request #3337 from modoboa/chore/update_python_versions

Updated test matrix

.github/workflows/modoboa.yml

@@ -72,7 +72,7 @@ jobs:
     strategy:
       matrix:
         database: ['postgres', 'mysql']
-        python-version: [3.8, 3.9, '3.10', '3.11']
+        python-version: [3.9, '3.10', '3.11', '3.12']
       fail-fast: false
 
     steps:
@@ -109,7 +109,7 @@ jobs:
           pip install -r mysql-requirements.txt
           echo "DB=mysql" >> $GITHUB_ENV
       - name: Test with pytest
-        if: ${{ matrix.python-version != '3.11' || matrix.database != 'postgres' }}
+        if: ${{ matrix.python-version != '3.12' || matrix.database != 'postgres' }}
         run: |
           python ./tests.py
           cd test_project
@@ -127,7 +127,7 @@ jobs:
           OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_PRIVATE_KEY }}
 
       - name: Test with pytest and coverage
-        if: ${{ matrix.python-version == '3.11' && matrix.database == 'postgres' }}
+        if: ${{ matrix.python-version == '3.12' && matrix.database == 'postgres' }}
         run: |
           python ./tests.py
           cd test_project
@@ -148,7 +148,7 @@ jobs:
           OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_PRIVATE_KEY }}
 
       - name: Upload coverage result
-        if: ${{ matrix.python-version == '3.11' && matrix.database == 'postgres' }}
+        if: ${{ matrix.python-version == '3.12' && matrix.database == 'postgres' }}
         uses: actions/upload-artifact@v4
         with:
           name: coverage-results
@@ -182,10 +182,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
       - name: Build frontend
         shell: bash -l {0}
         run: |

docker/Dockerfile.dev

@@ -1,10 +1,10 @@
-FROM python:3.12-alpine as base
+FROM python:3.12-alpine AS base
 
-MAINTAINER Antoine Nguyen <tonio@ngyn.org>
+LABEL org.opencontainers.image.authors="Antoine Nguyen <tonio@ngyn.org>"
 ENV VIRTUAL_ENV=/opt/venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-FROM base as build
+FROM base AS build
 
 RUN apk add --update openssl librrd python3-dev libffi-dev gcc g++ musl-dev libxml2-dev libxslt-dev \
     libressl-dev jpeg-dev rrdtool-dev file make gettext freetype-dev cairo-dev cargo \
@@ -16,7 +16,8 @@ COPY test-requirements.txt /tmp
 COPY dev-requirements.txt /tmp
 RUN python -m pip install -U pip
 RUN pip install -r requirements.txt -r test-requirements.txt -r dev-requirements.txt
-FROM base as run
+
+FROM base AS run
 COPY docker/doveadm /usr/local/bin
 RUN chmod +x /usr/local/bin/doveadm
 COPY --from=build $VIRTUAL_ENV $VIRTUAL_ENV

modoboa/core/checks.py

@@ -3,7 +3,8 @@ from django.conf import settings
 from django.utils.translation import gettext as _
 
 from modoboa.core.utils import generate_rsa_private_key
-
+from modoboa.parameters.tools import get_global_parameter
+from modoboa.core.password_hashers import get_password_hasher
 
 W001 = Warning(
     _(
@@ -14,6 +15,11 @@ W001 = Warning(
     hint=_("Set `USE_TZ = True` in settings.py"),
     id="modoboa.W001",
 )
+W002 = Warning(
+    _("The password scheme you are using is deprecated."),
+    hint=_("Upgrade your password scheme to a stronger one"),
+    id="modoboa.W002",
+)
 
 
 @register(deploy=True)
@@ -38,3 +44,13 @@ def check_rsa_private_key_exists(app_configs, **kwargs):
     if generate_rsa_private_key(settings.BASE_DIR):
         msgs.append(Info("An RSA private key has been generated for OIDC."))
     return msgs
+
+
+@register()
+def check_password_hasher(app_configs, **kwargs):
+    msgs = []
+    scheme_in_use = get_global_parameter("password_scheme", app="core")
+    hasher = get_password_hasher(scheme_in_use)
+    if hasher.deprecated:
+        msgs.append(W002)
+    return msgs

modoboa/core/models.py

@@ -2,6 +2,7 @@
 
 import re
 from email.header import Header
+from typing import Optional
 
 from django.core.serializers.json import DjangoJSONEncoder
 from django.conf import settings
@@ -96,7 +97,7 @@ class User(AbstractUser):
         super().__init__(*args, **kwargs)
         self.parameters = param_tools.Manager("user", self._parameters)
 
-    def _crypt_password(self, raw_value):
+    def _crypt_password(self, raw_value: str) -> str:
         """Crypt the local password using the appropriate scheme.
 
         In case we don't find the scheme (for example when the
@@ -117,7 +118,7 @@ class User(AbstractUser):
         raw_value = smart_bytes(raw_value)
         return get_password_hasher(scheme.upper())().encrypt(raw_value)
 
-    def set_password(self, raw_value, curvalue=None):
+    def set_password(self, raw_value: str, curvalue: Optional[str] = None) -> None:
         """Password update.
 
         Update the current mailbox's password with the given clear

modoboa/core/password_hashers/__init__.py

@@ -19,7 +19,7 @@ from modoboa.core.password_hashers.base import (  # NOQA:F401
 )
 
 
-def get_password_hasher(scheme):
+def get_password_hasher(scheme: str) -> PasswordHasher:
     """Retrieve the hasher corresponding to :keyword:`scheme`.
 
     If no class is found, `PLAINHasher` is returned.

modoboa/core/password_hashers/base.py

@@ -5,10 +5,12 @@ Contains weak hashers (the original ones) available with Modoboa.
 """
 
 import base64
+
 import crypt
 import hashlib
-import string
 from random import Random
+import string
+from typing import Optional
 
 from django.utils.crypto import constant_time_compare
 from django.utils.encoding import force_bytes, force_str
@@ -28,7 +30,15 @@ class MetaHasher(type):
     @property
     def label(cls):
         """Returns the label of the hasher"""
-        return cls.name if not cls._weak else "{} (weak)".format(cls.name)
+        result = cls.name
+        tags = []
+        if cls._weak:
+            tags.append("weak")
+        if cls.deprecated:
+            tags.append("deprecated")
+        if tags:
+            result += f" ({', '.join(tags)})"
+        return result
 
 
 class PasswordHasher(metaclass=MetaHasher):
@@ -36,12 +46,14 @@ class PasswordHasher(metaclass=MetaHasher):
     Base class of all hashers.
     """
 
-    _weak = False
+    _weak: bool = False
+    deprecated: bool = False
+    scheme: str
 
     def __init__(self, target="local"):
         self._target = target
 
-    def _encrypt(self, clearvalue, salt=None):
+    def _encrypt(self, clearvalue: str, salt: Optional[str] = None) -> str:
         raise NotImplementedError
 
     def _b64encode(self, pwhash):
@@ -54,7 +66,7 @@ class PasswordHasher(metaclass=MetaHasher):
             return base64.b64encode(pwhash)
         return pwhash
 
-    def encrypt(self, clearvalue):
+    def encrypt(self, clearvalue: str) -> str:
         """Encrypt a password.
 
         The format used to store passwords is the same than dovecot's one::
@@ -94,7 +106,7 @@ class PasswordHasher(metaclass=MetaHasher):
     @classmethod
     def get_password_hashers(cls):
         """Return all the PasswordHasher supported by Modoboa"""
-        return cls.__subclasses__()
+        return [hasher for hasher in cls.__subclasses__()]
 
 
 class PLAINHasher(PasswordHasher):
@@ -120,12 +132,13 @@ class CRYPTHasher(PasswordHasher):
     """
 
     _weak = True
+    deprecated = True
 
     @property
     def scheme(self):
         return "{CRYPT}"
 
-    def _encrypt(self, clearvalue, salt=None):
+    def _encrypt(self, clearvalue: str, salt: Optional[str] = None) -> str:
         if salt is None:
             salt = "".join(Random().sample(string.ascii_letters + string.digits, 2))
         return crypt.crypt(clearvalue, salt)

modoboa/core/tests/test_authentication.py

@@ -252,6 +252,10 @@ class AuthenticationTestCase(ModoTestCase):
         supported_schemes = get_dovecot_schemes()[0]
         self.assertEqual(supported_schemes, ["{MD5-CRYPT}", "{PLAIN}"])
 
+    def test_deprecated_password_scheme(self):
+        hasher = get_password_hasher("crypt")
+        self.assertEqual(hasher.label, "crypt (weak, deprecated)")
+
     def test_fido_auth_begin(self):
         url = reverse("core:fido_auth_begin")
         resp = self.client.post(url)

modoboa/core/tests/test_checks.py

@@ -2,13 +2,14 @@ import os
 import shutil
 import tempfile
 
-from django.test import SimpleTestCase
 from django.test.utils import override_settings
 
 from modoboa.core import checks
+from modoboa.lib.tests import SimpleModoTestCase
+from modoboa.core.password_hashers.base import CRYPTHasher
 
 
-class CheckSessionCookieSecureTest(SimpleTestCase):
+class CheckSessionCookieSecureTest(SimpleModoTestCase):
 
     def setUp(self):
         self.workdir = tempfile.mkdtemp()
@@ -35,3 +36,10 @@ class CheckSessionCookieSecureTest(SimpleTestCase):
             os.environ["OIDC_RSA_PRIVATE_KEY"] = "KEY"
             msgs = checks.check_rsa_private_key_exists(None)
             self.assertEqual(len(msgs), 0)
+
+    def test_password_hasher_checks(self):
+        msgs = checks.check_password_hasher(None)
+        self.assertEqual(msgs, [])
+        self.set_global_parameter("password_scheme", "crypt")
+        msgs = checks.check_password_hasher(None)
+        self.assertEqual(msgs, [checks.W002])

modoboa/lib/email_utils.py

@@ -9,11 +9,11 @@ from email.utils import formataddr, formatdate, getaddresses, make_msgid
 import chardet
 import lxml.html
 from lxml.html import defs
-from lxml.html.clean import Cleaner
+from lxml_html_clean import Cleaner
 
 from django.core.mail import EmailMessage
 from django.template.loader import render_to_string
-from django.utils.encoding import smart_str, smart_str
+from django.utils.encoding import smart_str
 from django.utils.html import conditional_escape, escape
 from django.utils.translation import gettext as _
 
@@ -26,7 +26,7 @@ _RE_REMOVE_EXTRA_WHITESPACE = re.compile(r"\n\s*\n")
 _RE_CID = re.compile(r".*[ ]*cid=\"([^\"]*)\".*", re.I)
 
 
-class EmailAddress(object):
+class EmailAddress:
 
     def __init__(self, address):
         self.name, self.address = u2u_decode.decode_address(address)
@@ -39,7 +39,7 @@ class EmailAddress(object):
         return self.fulladdress
 
 
-class Email(object):
+class Email:
 
     _basic_headers = (
         "From",

modoboa/lib/tests/__init__.py

@@ -30,7 +30,7 @@ except ImportError:
     NO_LDAP = True
 
 
-class ParametersMixin(object):
+class ParametersMixin:
     """Add tools to manage parameters."""
 
     @classmethod

modoboa/lib/tests/sample_messages/multipart-input.txt

@@ -57,7 +57,7 @@ This is an *html* /test /_message._
 :-)
 
 
--- 
+--
 Someone
 someone@example.net
 
@@ -83,12 +83,11 @@ Content-Transfer-Encoding: 7bit
     <img moz-do-not-send="false"
       src="cid:part1.D56823CE.7959DBD4@example.net" alt="" height="10"
       width="10"><br>
-    <pre class="moz-signature" cols="72">-- 
+    <pre class="moz-signature" cols="72">--
 someone
 <a class="moz-txt-link-abbreviated" href="mailto:someone@example.net">someone@example.net</a></pre>
   </body>
 </html>
-
 --------------D585B241EEF60F0F44EC20E5
 Content-Type: image/png;
  name="image.png"

modoboa/lib/tests/sample_messages/multipart-output-html_links.txt

@@ -8,13 +8,8 @@
     <span class="moz-smiley-s1"><span>:-)</span></span><br>
     <br>
     <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAF0lEQVQY02NgYGD4//8/ThKfHASMRN0AfaqVa6MQxIQAAAAASUVORK5CYII=" alt="" height="10" width="10" target="_blank"><br>
-    <pre class="moz-signature" cols="72">-- 
+    <pre class="moz-signature" cols="72">--
 someone
 <a class="moz-txt-link-abbreviated" href="mailto:someone@example.net" target="_blank" rel="nofollow">someone@example.net</a></pre>
   </body>
-<span><b>Some <i>HTML</i> text</b>, a <a href="http://example.net" target="_blank" rel="nofollow">link</a>, an
-embedded image and an external image.<br>
-<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAF0lEQVQY02NgYGD4//8/ThKfHASMRN0AfaqVa6MQxIQAAAAASUVORK5CYII=" target="_blank"><br>
-<img src="http://example.net/image.png" target="_blank"><br>
-Nifty!</span>
-</div>
+</div>

modoboa/lib/tests/sample_messages/multipart-output-html_nolinks.txt

@@ -8,13 +8,8 @@
     <span class="moz-smiley-s1"><span>:-)</span></span><br>
     <br>
     <img alt="" height="10" width="10"><br>
-    <pre class="moz-signature" cols="72">-- 
+    <pre class="moz-signature" cols="72">--
 someone
 <a class="moz-txt-link-abbreviated">someone@example.net</a></pre>
   </body>
-<span><b>Some <i>HTML</i> text</b>, a <a target="_blank" rel="nofollow">link</a>, an
-embedded image and an external image.<br>
-<img target="_blank"><br>
-<img target="_blank"><br>
-Nifty!</span>
-</div>
+</div>

modoboa/lib/tests/sample_messages/multipart-output-plain_nolinks.txt

@@ -2,6 +2,6 @@
 
 :-)
 
--- 
+--
 Someone
-someone@example.net</pre>
+someone@example.net</pre>

modoboa/lib/tests/test_email_utils.py

@@ -34,10 +34,10 @@ class EmailTests(SimpleTestCase):
     output: {message_id}-output-{dformat}_{no,}links.txt
     """
 
-    def _get_expected_output(self, message_id, **kwargs):
+    def _get_expected_output(self, message_id: str, **kwargs) -> str:
         ext = kwargs["dformat"] if "dformat" in kwargs else "plain"
         ext += "_links" if "links" in kwargs and kwargs["links"] else "_nolinks"
-        message_path = os.path.join(SAMPLES_DIR, "%s-output-%s.txt" % (message_id, ext))
+        message_path = os.path.join(SAMPLES_DIR, f"{message_id}-output-{ext}.txt")
         assert os.path.isfile(message_path), "%s does not exist." % message_path
 
         with open(message_path, "rb") as fp:
@@ -46,11 +46,11 @@ class EmailTests(SimpleTestCase):
 
         return mail_text
 
-    def _test_email(self, message_id, **kwargs):
+    def _test_email(self, message_id: str, **kwargs) -> None:
         """Boiler plate code for testing e-mails."""
         expected_output = self._get_expected_output(message_id, **kwargs)
         output = EmailTestImplementation(message_id, **kwargs).body
-        self.assertEqual(output, expected_output)
+        self.assertEqual(output.strip("\n"), expected_output.strip("\n"))
 
     def test_invalid_links_value(self):
         """modoboa-amavis set links = "0"; in python "0" == True, fixed in PR 79

requirements.txt

@@ -30,7 +30,7 @@ python-dateutil
 cryptography
 pytz
 requests
-lxml==4.9.4  # FIXME: issue with version 5+
+lxml[html_clean]
 chardet
 ovh
 oath