From 846ad253236edd7e64e3744540dba1b9d88f4f44 Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Mon, 26 Mar 2018 18:31:42 +0100 Subject: [PATCH 1/7] when using LDAP use str (bytes) on PY2 and str on PY3 --- modoboa/lib/ldap_utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/modoboa/lib/ldap_utils.py b/modoboa/lib/ldap_utils.py index 600b31517..c7ec9235d 100644 --- a/modoboa/lib/ldap_utils.py +++ b/modoboa/lib/ldap_utils.py @@ -39,7 +39,7 @@ import ldap from django.conf import settings from django.utils import six -from django.utils.encoding import force_bytes +from django.utils.encoding import force_bytes, force_str from django.utils.translation import ugettext as _ from modoboa.core.password_hashers import get_password_hasher @@ -70,13 +70,12 @@ class LDAPAuthBackend(object): def _get_conn(self, dn, password): """Get a connection from the server.""" - kwargs = {} - if six.PY3: - kwargs["bytes_mode"] = False - conn = ldap.initialize(self.server_uri, **kwargs) + conn = ldap.initialize(self.server_uri, bytes_mode=six.PY2) conn.set_option(ldap.OPT_X_TLS_DEMAND, True) conn.set_option(ldap.OPT_DEBUG_LEVEL, 255) - conn.simple_bind_s(dn, password) + conn.simple_bind_s( + force_str(dn), force_str(password) + ) return conn def connect_to_server(self, user, password): @@ -98,7 +97,9 @@ class LDAPAuthBackend(object): sbase = self.global_params["ldap_search_base"] sfilter = self.global_params["ldap_search_filter"] sfilter = sfilter % {"user": user} - res = self.conn.search_s(sbase, ldap.SCOPE_SUBTREE, sfilter) + res = self.conn.search_s( + force_str(sbase), ldap.SCOPE_SUBTREE, force_str(sfilter) + ) try: dn = res[0][0] except IndexError: @@ -124,10 +125,10 @@ class LDAPAuthBackend(object): ('"%s"' % newpassword).encode("utf-16").lstrip("\377\376") ) ldif = [(ldap.MOD_REPLACE, - self.pwd_attr, + force_str(self.pwd_attr), force_bytes(self._crypt_password(newpassword)))] try: - self.conn.modify_s(user_dn, ldif) + self.conn.modify_s(force_str(user_dn), ldif) except ldap.LDAPError as e: raise InternalError( _("Failed to update password: {}").format(e)) From 86d758d841b24509d45dad5cb64056e5c9b26915 Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Sat, 31 Mar 2018 14:39:25 +0100 Subject: [PATCH 2/7] revert #1439 changes adding external dependencies to setup.py was a bad idea --- dev-requirements.txt | 1 - setup.py | 18 +++++++++--------- test-requirements.txt | 1 - 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index e2dd90121..06d73f33c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,2 @@ django-bower django-debug-toolbar -packaging diff --git a/setup.py b/setup.py index 3a7a3737d..43abc5c05 100644 --- a/setup.py +++ b/setup.py @@ -13,21 +13,21 @@ from __future__ import unicode_literals import io from os import path -from packaging.requirements import Requirement +from pip.req import parse_requirements from setuptools import find_packages, setup def get_requirements(requirements_file): - """Parse requirements file.""" + """Use pip to parse requirements file.""" requirements = [] if path.isfile(requirements_file): - with io.open(requirements_file, encoding="utf-8") as fp: - for line in fp: - if line.startswith("#") or line.strip() == "": - continue - req = Requirement(line) - if req.marker is None or req.marker.evaluate(): - requirements.append('%s%s' % (req.name, req.specifier)) + for req in parse_requirements(requirements_file, session="hack"): + # check markers, such as + # + # rope_py3k ; python_version >= '3.0' + # + if req.match_markers(): + requirements.append(str(req.req)) return requirements diff --git a/test-requirements.txt b/test-requirements.txt index 03529194c..45bb850f1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,4 +5,3 @@ testfixtures==4.7.0 tox psycopg2-binary>=2.7.4 mysqlclient>=1.3.3 -packaging From f64cdb1a93dc8cd706d47d63b288ba11e9a37a1c Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Sat, 31 Mar 2018 14:43:41 +0100 Subject: [PATCH 3/7] temporary solution for pip 10 internals moving to pip._internal --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43abc5c05..571a157da 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,12 @@ from __future__ import unicode_literals import io from os import path -from pip.req import parse_requirements +try: + from pip.req import parse_requirements +except ImportError: + # pip >= 10 + from pip._internal.req import parse_requirements + from setuptools import find_packages, setup From 1aff9b2a9868e63fa26e3ffbcf33ea706ead04f9 Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Sun, 25 Mar 2018 22:41:35 +0100 Subject: [PATCH 4/7] prefer standard library version of mock over back port --- modoboa/admin/tests/test_api.py | 12 ++++-- modoboa/admin/tests/test_domain.py | 12 ++++-- modoboa/admin/tests/test_import_.py | 12 ++++-- .../admin/tests/test_mailbox_operations.py | 9 +++- modoboa/admin/tests/test_mx.py | 42 +++++++++++-------- modoboa/core/tests/test_authentication.py | 17 +++++--- test-requirements.txt | 2 +- 7 files changed, 70 insertions(+), 36 deletions(-) diff --git a/modoboa/admin/tests/test_api.py b/modoboa/admin/tests/test_api.py index a5205742a..c32260ead 100644 --- a/modoboa/admin/tests/test_api.py +++ b/modoboa/admin/tests/test_api.py @@ -8,7 +8,6 @@ import copy import json import dns.resolver -from mock import patch from django.test import override_settings from django.urls import reverse @@ -21,6 +20,13 @@ from modoboa.lib.tests import ModoAPITestCase from . import utils from .. import factories, models +try: + # mock is part of the Python (>= 3.3) standard library + from unittest import mock +except ImportError: + # fall back to the mock backport + import mock + class DomainAPITestCase(ModoAPITestCase): """Check API.""" @@ -71,8 +77,8 @@ class DomainAPITestCase(ModoAPITestCase): url, {"name": "test4.com", "default_mailbox_quota": 10}) self.assertEqual(response.status_code, 403) - @patch.object(dns.resolver.Resolver, "query") - @patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") + @mock.patch("socket.getaddrinfo") def test_create_domain_with_mx_check(self, mock_getaddrinfo, mock_query): """Check domain creation when MX check is activated.""" self.set_global_parameter("enable_admin_limits", False, app="limits") diff --git a/modoboa/admin/tests/test_domain.py b/modoboa/admin/tests/test_domain.py index dc5529698..ad985f323 100644 --- a/modoboa/admin/tests/test_domain.py +++ b/modoboa/admin/tests/test_domain.py @@ -9,7 +9,6 @@ import shutil import tempfile import dns.resolver -from mock import patch from testfixtures import compare from django.core.management import call_command @@ -24,6 +23,13 @@ from . import utils from .. import factories from ..models import Alias, Domain +try: + # mock is part of the Python (>= 3.3) standard library + from unittest import mock +except ImportError: + # fall back to the mock backport + import mock + class DomainTestCase(ModoTestCase): """Test case for Domain.""" @@ -205,8 +211,8 @@ class DomainTestCase(ModoTestCase): self.assertContains(response, "value=\"500\"") self.assertContains(response, "value=\"50\"") - @patch.object(dns.resolver.Resolver, "query") - @patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") + @mock.patch("socket.getaddrinfo") def test_create_and_check_mx(self, mock_getaddrinfo, mock_query): """Check for authorized MX record.""" reseller = core_factories.UserFactory( diff --git a/modoboa/admin/tests/test_import_.py b/modoboa/admin/tests/test_import_.py index 540fede38..148b288a3 100644 --- a/modoboa/admin/tests/test_import_.py +++ b/modoboa/admin/tests/test_import_.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import dns.resolver -from mock import patch from django.core.files.base import ContentFile from django.urls import reverse @@ -15,6 +14,13 @@ from . import utils from .. import factories from ..models import Alias, Domain, DomainAlias +try: + # mock is part of the Python (>= 3.3) standard library + from unittest import mock +except ImportError: + # fall back to the mock backport + import mock + class ImportTestCase(ModoTestCase): @@ -78,8 +84,8 @@ domainalias; domalias1.com; domain1.com; True response, "Default mailbox quota cannot be greater than domain quota") - @patch.object(dns.resolver.Resolver, "query") - @patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") + @mock.patch("socket.getaddrinfo") def test_domain_import_with_mx_check(self, mock_getaddrinfo, mock_query): """Check domain import when MX check is enabled.""" reseller = core_factories.UserFactory( diff --git a/modoboa/admin/tests/test_mailbox_operations.py b/modoboa/admin/tests/test_mailbox_operations.py index d2ee501cc..6e598bb57 100644 --- a/modoboa/admin/tests/test_mailbox_operations.py +++ b/modoboa/admin/tests/test_mailbox_operations.py @@ -8,8 +8,6 @@ import os import shutil import tempfile -import mock - from django.core.management import call_command from django.test import override_settings from django.urls import reverse @@ -17,6 +15,13 @@ from django.urls import reverse from modoboa.lib.tests import ModoTestCase from .. import factories, models +try: + # mock is part of the Python (>= 3.3) standard library + from unittest import mock +except ImportError: + # fall back to the mock backport + import mock + @override_settings( DOVECOT_LOOKUP_PATH=["{}/dovecot".format(os.path.dirname(__file__))]) diff --git a/modoboa/admin/tests/test_mx.py b/modoboa/admin/tests/test_mx.py index 276afd990..fea42ee53 100644 --- a/modoboa/admin/tests/test_mx.py +++ b/modoboa/admin/tests/test_mx.py @@ -5,7 +5,6 @@ from __future__ import unicode_literals import dns.resolver -from mock import patch from testfixtures import LogCapture from django.core import mail, management @@ -19,6 +18,13 @@ from . import utils from .. import factories, models from ..lib import get_domain_mx_list +try: + # mock is part of the Python (>= 3.3) standard library + from unittest import mock +except ImportError: + # fall back to the mock backport + import mock + class MXTestCase(ModoTestCase): """TestCase for DNSBL related features.""" @@ -49,9 +55,9 @@ class MXTestCase(ModoTestCase): cls.localconfig.save() models.MXRecord.objects.all().delete() - @patch("gevent.socket.gethostbyname") - @patch("socket.getaddrinfo") - @patch.object(dns.resolver.Resolver, "query") + @mock.patch("gevent.socket.gethostbyname") + @mock.patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") def test_management_command( self, mock_query, mock_getaddrinfo, mock_g_gethostbyname): """Check that command works fine.""" @@ -77,9 +83,9 @@ class MXTestCase(ModoTestCase): qs = models.MXRecord.objects.filter(domain=self.domain) self.assertEqual(id_, qs[0].id) - @patch("gevent.socket.gethostbyname") - @patch("socket.getaddrinfo") - @patch.object(dns.resolver.Resolver, "query") + @mock.patch("gevent.socket.gethostbyname") + @mock.patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") def test_single_domain_update( self, mock_query, mock_getaddrinfo, mock_g_gethostbyname): """Update only one domain.""" @@ -102,8 +108,8 @@ class MXTestCase(ModoTestCase): management.call_command( "modo", "check_mx", "--domain", "toto.com") - @patch("socket.getaddrinfo") - @patch.object(dns.resolver.Resolver, "query") + @mock.patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") def test_get_domain_mx_list_logging(self, mock_query, mock_getaddrinfo): mock_query.side_effect = utils.mock_dns_query_result mock_getaddrinfo.side_effect = utils.mock_ip_query_result @@ -149,9 +155,9 @@ class DNSBLTestCase(ModoTestCase): name="modoboa.com", enable_dns_checks=False) models.DNSBLResult.objects.all().delete() - @patch("gevent.socket.gethostbyname") - @patch("socket.getaddrinfo") - @patch.object(dns.resolver.Resolver, "query") + @mock.patch("gevent.socket.gethostbyname") + @mock.patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") def test_management_command( self, mock_query, mock_getaddrinfo, mock_g_gethostbyname): """Check that command works fine.""" @@ -170,9 +176,9 @@ class DNSBLTestCase(ModoTestCase): self.assertFalse(self.domain.uses_a_reserved_tld) self.assertTrue(self.domain2.uses_a_reserved_tld) - @patch("gevent.socket.gethostbyname") - @patch("socket.getaddrinfo") - @patch.object(dns.resolver.Resolver, "query") + @mock.patch("gevent.socket.gethostbyname") + @mock.patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") def test_notifications( self, mock_query, mock_getaddrinfo, mock_g_gethostbyname): """Check notifications.""" @@ -186,9 +192,9 @@ class DNSBLTestCase(ModoTestCase): print(message.subject) self.assertEqual(len(mail.outbox), 2) - @patch("gevent.socket.gethostbyname") - @patch("socket.getaddrinfo") - @patch.object(dns.resolver.Resolver, "query") + @mock.patch("gevent.socket.gethostbyname") + @mock.patch("socket.getaddrinfo") + @mock.patch.object(dns.resolver.Resolver, "query") def test_management_command_no_dnsbl( self, mock_query, mock_getaddrinfo, mock_g_gethostbyname): """Check that command works fine without dnsbl.""" diff --git a/modoboa/core/tests/test_authentication.py b/modoboa/core/tests/test_authentication.py index 7d99c4e3f..020f3015b 100644 --- a/modoboa/core/tests/test_authentication.py +++ b/modoboa/core/tests/test_authentication.py @@ -7,8 +7,6 @@ from __future__ import unicode_literals import smtplib from unittest import skipIf -from mock import patch - from django.core import mail from django.test import override_settings from django.urls import reverse @@ -16,6 +14,13 @@ from django.urls import reverse from modoboa.lib.tests import NO_SMTP, ModoTestCase from .. import factories, models +try: + # mock is part of the Python (>= 3.3) standard library + from unittest import mock +except ImportError: + # fall back to the mock backport + import mock + class AuthenticationTestCase(ModoTestCase): """Validate authentication scenarios.""" @@ -123,24 +128,24 @@ class SMTPAuthenticationTestCase(ModoTestCase): self.assertTrue( models.User.objects.filter(username=username).exists()) - @patch("smtplib.SMTP") + @mock.patch("smtplib.SMTP") def test_smtp_authentication(self, mock_smtp): """Check simple SMTP authentication.""" self._test_smtp_authentication(mock_smtp) - @patch("smtplib.SMTP_SSL") + @mock.patch("smtplib.SMTP_SSL") @override_settings(AUTH_SMTP_SECURED_MODE="ssl") def test_smtp_authentication_over_ssl(self, mock_smtp): """Check SMTP authentication over SSL.""" self._test_smtp_authentication(mock_smtp) - @patch("smtplib.SMTP") + @mock.patch("smtplib.SMTP") @override_settings(AUTH_SMTP_SECURED_MODE="starttls") def test_smtp_authentication_over_starttls(self, mock_smtp): """Check SMTP authentication over STARTTLS.""" self._test_smtp_authentication(mock_smtp) - @patch("smtplib.SMTP") + @mock.patch("smtplib.SMTP") def test_smtp_authentication_failure(self, mock_smtp): """Check SMTP authentication failure.""" instance = mock_smtp.return_value diff --git a/test-requirements.txt b/test-requirements.txt index 03529194c..ed1d4da5e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ factory-boy>=2.4 -mock==2.0.0 +mock==2.0.0; python_version < '3.3' httmock==1.2.5 testfixtures==4.7.0 tox From 957591f9778d034fce00181cec36fbe0167108e3 Mon Sep 17 00:00:00 2001 From: Andrew Fyfe Date: Wed, 28 Mar 2018 00:24:13 +0100 Subject: [PATCH 5/7] ensure utf8 encoding is used when creating/accessing test database --- modoboa/test_settings.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/modoboa/test_settings.py b/modoboa/test_settings.py index 3e61f31c2..535b8c3c1 100644 --- a/modoboa/test_settings.py +++ b/modoboa/test_settings.py @@ -26,7 +26,15 @@ if DB == "MYSQL": # See: # https://docs.djangoproject.com/en/1.11/ref/databases/#mysql-sql-mode "OPTIONS": { - "init_command": 'SET sql_mode=\'STRICT_TRANS_TABLES\'', + "init_command": ( + "SET sql_mode = 'STRICT_TRANS_TABLES';" + "SET innodb_strict_mode = ON;" + ), + "charset": "utf8", + }, + "TEST": { + "CHARSET": "utf8", + "COLLATION": "utf8_unicode_ci", }, }, } @@ -49,5 +57,11 @@ else: "HOST": "localhost", "PORT": "", "ATOMIC_REQUESTS": True, + "OPTIONS": { + "client_encoding": "UTF8", + }, + "TEST": { + "CHARSET": "UTF8", + }, }, } From 5dbb5b1ea6236656e2ccb6d1c20aee41f18b84d2 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Thu, 5 Apr 2018 14:29:39 +0200 Subject: [PATCH 6/7] Trigger domform_init event for creation too. fix #1450 --- modoboa/admin/static/admin/js/admin.js | 1 + 1 file changed, 1 insertion(+) diff --git a/modoboa/admin/static/admin/js/admin.js b/modoboa/admin/static/admin/js/admin.js index e5aeb2144..04079ef70 100644 --- a/modoboa/admin/static/admin/js/admin.js +++ b/modoboa/admin/static/admin/js/admin.js @@ -323,6 +323,7 @@ Domains.prototype = { }, success_callback: $.proxy(this.reload_listing, this) }); + $(document).trigger('domform_init'); }, domainform_cb: function() { From ad91695d17e06d758b684cc0a6bc93c42fcc0e1d Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Thu, 5 Apr 2018 14:43:39 +0200 Subject: [PATCH 7/7] Better fix for #1450 --- modoboa/admin/static/admin/js/admin.js | 2 +- modoboa/transport/static/transport/js/transport.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modoboa/admin/static/admin/js/admin.js b/modoboa/admin/static/admin/js/admin.js index 04079ef70..3db83f02f 100644 --- a/modoboa/admin/static/admin/js/admin.js +++ b/modoboa/admin/static/admin/js/admin.js @@ -323,7 +323,7 @@ Domains.prototype = { }, success_callback: $.proxy(this.reload_listing, this) }); - $(document).trigger('domform_init'); + $(document).trigger('domwizard_init'); }, domainform_cb: function() { diff --git a/modoboa/transport/static/transport/js/transport.js b/modoboa/transport/static/transport/js/transport.js index c6b1a695a..e0097342e 100644 --- a/modoboa/transport/static/transport/js/transport.js +++ b/modoboa/transport/static/transport/js/transport.js @@ -21,5 +21,6 @@ function transportFormCallback () { } $(document).ready(function () { + $(this).bind('domwizard_init', transportFormCallback); $(this).bind('domform_init', transportFormCallback); });