Merge branch 'master' into fix/1430

dev-requirements.txt

@@ -1,3 +1,2 @@
 django-bower
 django-debug-toolbar
-packaging

modoboa/admin/static/admin/js/admin.js

@@ -323,6 +323,7 @@ Domains.prototype = {
             },
             success_callback: $.proxy(this.reload_listing, this)
         });
+        $(document).trigger('domwizard_init');
     },
 
     domainform_cb: function() {

modoboa/admin/tests/test_api.py

@@ -8,7 +8,6 @@ import copy
 import json
 
 import dns.resolver
-from mock import patch
 
 from django.test import override_settings
 from django.urls import reverse
@@ -21,6 +20,13 @@ from modoboa.lib.tests import ModoAPITestCase
 from . import utils
 from .. import factories, models
 
+try:
+    # mock is part of the Python (>= 3.3) standard library
+    from unittest import mock
+except ImportError:
+    # fall back to the mock backport
+    import mock
+
 
 class DomainAPITestCase(ModoAPITestCase):
     """Check API."""
@@ -71,8 +77,8 @@ class DomainAPITestCase(ModoAPITestCase):
             url, {"name": "test4.com", "default_mailbox_quota": 10})
         self.assertEqual(response.status_code, 403)
 
-    @patch.object(dns.resolver.Resolver, "query")
-    @patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("socket.getaddrinfo")
     def test_create_domain_with_mx_check(self, mock_getaddrinfo, mock_query):
         """Check domain creation when MX check is activated."""
         self.set_global_parameter("enable_admin_limits", False, app="limits")

modoboa/admin/tests/test_domain.py

@@ -9,7 +9,6 @@ import shutil
 import tempfile
 
 import dns.resolver
-from mock import patch
 from testfixtures import compare
 
 from django.core.management import call_command
@@ -24,6 +23,13 @@ from . import utils
 from .. import factories
 from ..models import Alias, Domain
 
+try:
+    # mock is part of the Python (>= 3.3) standard library
+    from unittest import mock
+except ImportError:
+    # fall back to the mock backport
+    import mock
+
 
 class DomainTestCase(ModoTestCase):
     """Test case for Domain."""
@@ -205,8 +211,8 @@ class DomainTestCase(ModoTestCase):
         self.assertContains(response, "value=\"500\"")
         self.assertContains(response, "value=\"50\"")
 
-    @patch.object(dns.resolver.Resolver, "query")
-    @patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("socket.getaddrinfo")
     def test_create_and_check_mx(self, mock_getaddrinfo, mock_query):
         """Check for authorized MX record."""
         reseller = core_factories.UserFactory(

modoboa/admin/tests/test_import_.py

@@ -5,8 +5,6 @@ from __future__ import unicode_literals
 import os
 
 import dns.resolver
-import mock
-from mock import patch
 
 from django.core.files.base import ContentFile
 from django.core.management import call_command
@@ -20,6 +18,13 @@ from . import utils
 from .. import factories
 from ..models import Alias, Domain, DomainAlias
 
+try:
+    # mock is part of the Python (>= 3.3) standard library
+    from unittest import mock
+except ImportError:
+    # fall back to the mock backport
+    import mock
+
 
 class ImportTestCase(ModoTestCase):
 
@@ -83,8 +88,8 @@ domainalias; domalias1.com; domain1.com; True
             response,
             "Default mailbox quota cannot be greater than domain quota")
 
-    @patch.object(dns.resolver.Resolver, "query")
-    @patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("socket.getaddrinfo")
     def test_domain_import_with_mx_check(self, mock_getaddrinfo, mock_query):
         """Check domain import when MX check is enabled."""
         reseller = core_factories.UserFactory(

modoboa/admin/tests/test_mailbox_operations.py

@@ -8,8 +8,6 @@ import os
 import shutil
 import tempfile
 
-import mock
-
 from django.core.management import call_command
 from django.test import override_settings
 from django.urls import reverse
@@ -17,6 +15,13 @@ from django.urls import reverse
 from modoboa.lib.tests import ModoTestCase
 from .. import factories, models
 
+try:
+    # mock is part of the Python (>= 3.3) standard library
+    from unittest import mock
+except ImportError:
+    # fall back to the mock backport
+    import mock
+
 
 @override_settings(
     DOVECOT_LOOKUP_PATH=["{}/dovecot".format(os.path.dirname(__file__))])

modoboa/admin/tests/test_mx.py

@@ -5,7 +5,6 @@
 from __future__ import unicode_literals
 
 import dns.resolver
-from mock import patch
 from testfixtures import LogCapture
 
 from django.core import mail, management
@@ -19,6 +18,13 @@ from . import utils
 from .. import factories, models
 from ..lib import get_domain_mx_list
 
+try:
+    # mock is part of the Python (>= 3.3) standard library
+    from unittest import mock
+except ImportError:
+    # fall back to the mock backport
+    import mock
+
 
 class MXTestCase(ModoTestCase):
     """TestCase for DNSBL related features."""
@@ -49,9 +55,9 @@ class MXTestCase(ModoTestCase):
         cls.localconfig.save()
         models.MXRecord.objects.all().delete()
 
-    @patch("gevent.socket.gethostbyname")
-    @patch("socket.getaddrinfo")
-    @patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("gevent.socket.gethostbyname")
+    @mock.patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
     def test_management_command(
             self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
         """Check that command works fine."""
@@ -77,9 +83,9 @@ class MXTestCase(ModoTestCase):
         qs = models.MXRecord.objects.filter(domain=self.domain)
         self.assertEqual(id_, qs[0].id)
 
-    @patch("gevent.socket.gethostbyname")
-    @patch("socket.getaddrinfo")
-    @patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("gevent.socket.gethostbyname")
+    @mock.patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
     def test_single_domain_update(
             self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
         """Update only one domain."""
@@ -102,8 +108,8 @@ class MXTestCase(ModoTestCase):
         management.call_command(
             "modo", "check_mx", "--domain", "toto.com")
 
-    @patch("socket.getaddrinfo")
-    @patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
     def test_get_domain_mx_list_logging(self, mock_query, mock_getaddrinfo):
         mock_query.side_effect = utils.mock_dns_query_result
         mock_getaddrinfo.side_effect = utils.mock_ip_query_result
@@ -149,9 +155,9 @@ class DNSBLTestCase(ModoTestCase):
             name="modoboa.com", enable_dns_checks=False)
         models.DNSBLResult.objects.all().delete()
 
-    @patch("gevent.socket.gethostbyname")
-    @patch("socket.getaddrinfo")
-    @patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("gevent.socket.gethostbyname")
+    @mock.patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
     def test_management_command(
             self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
         """Check that command works fine."""
@@ -170,9 +176,9 @@ class DNSBLTestCase(ModoTestCase):
         self.assertFalse(self.domain.uses_a_reserved_tld)
         self.assertTrue(self.domain2.uses_a_reserved_tld)
 
-    @patch("gevent.socket.gethostbyname")
-    @patch("socket.getaddrinfo")
-    @patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("gevent.socket.gethostbyname")
+    @mock.patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
     def test_notifications(
             self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
         """Check notifications."""
@@ -186,9 +192,9 @@ class DNSBLTestCase(ModoTestCase):
                 print(message.subject)
         self.assertEqual(len(mail.outbox), 2)
 
-    @patch("gevent.socket.gethostbyname")
-    @patch("socket.getaddrinfo")
-    @patch.object(dns.resolver.Resolver, "query")
+    @mock.patch("gevent.socket.gethostbyname")
+    @mock.patch("socket.getaddrinfo")
+    @mock.patch.object(dns.resolver.Resolver, "query")
     def test_management_command_no_dnsbl(
             self, mock_query, mock_getaddrinfo, mock_g_gethostbyname):
         """Check that command works fine without dnsbl."""

modoboa/core/tests/test_authentication.py

@@ -7,8 +7,6 @@ from __future__ import unicode_literals
 import smtplib
 from unittest import skipIf
 
-from mock import patch
-
 from django.core import mail
 from django.test import override_settings
 from django.urls import reverse
@@ -16,6 +14,13 @@ from django.urls import reverse
 from modoboa.lib.tests import NO_SMTP, ModoTestCase
 from .. import factories, models
 
+try:
+    # mock is part of the Python (>= 3.3) standard library
+    from unittest import mock
+except ImportError:
+    # fall back to the mock backport
+    import mock
+
 
 class AuthenticationTestCase(ModoTestCase):
     """Validate authentication scenarios."""
@@ -123,24 +128,24 @@ class SMTPAuthenticationTestCase(ModoTestCase):
         self.assertTrue(
             models.User.objects.filter(username=username).exists())
 
-    @patch("smtplib.SMTP")
+    @mock.patch("smtplib.SMTP")
     def test_smtp_authentication(self, mock_smtp):
         """Check simple SMTP authentication."""
         self._test_smtp_authentication(mock_smtp)
 
-    @patch("smtplib.SMTP_SSL")
+    @mock.patch("smtplib.SMTP_SSL")
     @override_settings(AUTH_SMTP_SECURED_MODE="ssl")
     def test_smtp_authentication_over_ssl(self, mock_smtp):
         """Check SMTP authentication over SSL."""
         self._test_smtp_authentication(mock_smtp)
 
-    @patch("smtplib.SMTP")
+    @mock.patch("smtplib.SMTP")
     @override_settings(AUTH_SMTP_SECURED_MODE="starttls")
     def test_smtp_authentication_over_starttls(self, mock_smtp):
         """Check SMTP authentication over STARTTLS."""
         self._test_smtp_authentication(mock_smtp)
 
-    @patch("smtplib.SMTP")
+    @mock.patch("smtplib.SMTP")
     def test_smtp_authentication_failure(self, mock_smtp):
         """Check SMTP authentication failure."""
         instance = mock_smtp.return_value

modoboa/lib/ldap_utils.py

@@ -39,7 +39,7 @@ import ldap
 
 from django.conf import settings
 from django.utils import six
-from django.utils.encoding import force_bytes
+from django.utils.encoding import force_bytes, force_str
 from django.utils.translation import ugettext as _
 
 from modoboa.core.password_hashers import get_password_hasher
@@ -70,13 +70,12 @@ class LDAPAuthBackend(object):
 
     def _get_conn(self, dn, password):
         """Get a connection from the server."""
-        kwargs = {}
-        if six.PY3:
-            kwargs["bytes_mode"] = False
-        conn = ldap.initialize(self.server_uri, **kwargs)
+        conn = ldap.initialize(self.server_uri, bytes_mode=six.PY2)
         conn.set_option(ldap.OPT_X_TLS_DEMAND, True)
         conn.set_option(ldap.OPT_DEBUG_LEVEL, 255)
-        conn.simple_bind_s(dn, password)
+        conn.simple_bind_s(
+            force_str(dn), force_str(password)
+        )
         return conn
 
     def connect_to_server(self, user, password):
@@ -98,7 +97,9 @@ class LDAPAuthBackend(object):
         sbase = self.global_params["ldap_search_base"]
         sfilter = self.global_params["ldap_search_filter"]
         sfilter = sfilter % {"user": user}
-        res = self.conn.search_s(sbase, ldap.SCOPE_SUBTREE, sfilter)
+        res = self.conn.search_s(
+            force_str(sbase), ldap.SCOPE_SUBTREE, force_str(sfilter)
+        )
         try:
             dn = res[0][0]
         except IndexError:
@@ -124,10 +125,10 @@ class LDAPAuthBackend(object):
                 ('"%s"' % newpassword).encode("utf-16").lstrip("\377\376")
             )
         ldif = [(ldap.MOD_REPLACE,
-                 self.pwd_attr,
+                 force_str(self.pwd_attr),
                  force_bytes(self._crypt_password(newpassword)))]
         try:
-            self.conn.modify_s(user_dn, ldif)
+            self.conn.modify_s(force_str(user_dn), ldif)
         except ldap.LDAPError as e:
             raise InternalError(
                 _("Failed to update password: {}").format(e))

modoboa/test_settings.py

@@ -26,7 +26,15 @@ if DB == "MYSQL":
             # See:
             # https://docs.djangoproject.com/en/1.11/ref/databases/#mysql-sql-mode
             "OPTIONS": {
-                "init_command": 'SET sql_mode=\'STRICT_TRANS_TABLES\'',
+                "init_command": (
+                    "SET sql_mode = 'STRICT_TRANS_TABLES';"
+                    "SET innodb_strict_mode = ON;"
+                ),
+                "charset": "utf8",
+            },
+            "TEST": {
+                "CHARSET": "utf8",
+                "COLLATION": "utf8_unicode_ci",
             },
         },
     }
@@ -49,5 +57,11 @@ else:
             "HOST": "localhost",
             "PORT": "",
             "ATOMIC_REQUESTS": True,
+            "OPTIONS": {
+                    "client_encoding": "UTF8",
+            },
+            "TEST": {
+                "CHARSET": "UTF8",
+            },
         },
     }

modoboa/transport/static/transport/js/transport.js

@@ -21,5 +21,6 @@ function transportFormCallback () {
 }
 
 $(document).ready(function () {
+    $(this).bind('domwizard_init', transportFormCallback);
     $(this).bind('domform_init', transportFormCallback);
 });

setup.py

@@ -13,21 +13,26 @@ from __future__ import unicode_literals
 import io
 from os import path
 
-from packaging.requirements import Requirement
+try:
+    from pip.req import parse_requirements
+except ImportError:
+    # pip >= 10
+    from pip._internal.req import parse_requirements
+
 from setuptools import find_packages, setup
 
 
 def get_requirements(requirements_file):
-    """Parse requirements file."""
+    """Use pip to parse requirements file."""
     requirements = []
     if path.isfile(requirements_file):
-        with io.open(requirements_file, encoding="utf-8") as fp:
-            for line in fp:
-                if line.startswith("#") or line.strip() == "":
-                    continue
-                req = Requirement(line)
-                if req.marker is None or req.marker.evaluate():
-                    requirements.append('%s%s' % (req.name, req.specifier))
+        for req in parse_requirements(requirements_file, session="hack"):
+            # check markers, such as
+            #
+            #     rope_py3k    ; python_version >= '3.0'
+            #
+            if req.match_markers():
+                requirements.append(str(req.req))
     return requirements
 
 

test-requirements.txt

@@ -1,8 +1,7 @@
 factory-boy>=2.4
-mock==2.0.0
+mock==2.0.0; python_version < '3.3'
 httmock==1.2.5
 testfixtures==4.7.0
 tox
 psycopg2-binary>=2.7.4
 mysqlclient>=1.3.3
-packaging