diff --git a/modoboa/admin/handlers.py b/modoboa/admin/handlers.py index da360c6d8..a91a12667 100644 --- a/modoboa/admin/handlers.py +++ b/modoboa/admin/handlers.py @@ -18,6 +18,7 @@ from modoboa.lib import permissions from modoboa.lib import signals as lib_signals from modoboa.parameters import tools as param_tools +from . import signals as admin_signals from . import lib from . import models from . import postfix_maps @@ -339,3 +340,21 @@ def add_widgets_to_admin_dashboard(sender, user, **kwargs): "template": template, "context": context }] + + +@receiver(admin_signals.import_object) +def get_import_func(sender, objtype, **kwargs): + """Return function used to import objtype.""" + if objtype == "domain": + return lib.import_domain + elif objtype == "domainalias": + return lib.import_domainalias + elif objtype == "account": + return lib.import_account + elif objtype == "alias": + return lib.import_alias + elif objtype == "forward": + return lib.import_forward + elif objtype == "dlist": + return lib.import_dlist + return None diff --git a/modoboa/admin/management/commands/subcommands/_export.py b/modoboa/admin/management/commands/subcommands/_export.py index 6a5f0c121..2b7f7c206 100644 --- a/modoboa/admin/management/commands/subcommands/_export.py +++ b/modoboa/admin/management/commands/subcommands/_export.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals -import csv +from backports import csv import sys from django.core.management.base import BaseCommand diff --git a/modoboa/admin/management/commands/subcommands/_import.py b/modoboa/admin/management/commands/subcommands/_import.py index 8e199eabf..c9254febb 100644 --- a/modoboa/admin/management/commands/subcommands/_import.py +++ b/modoboa/admin/management/commands/subcommands/_import.py @@ -2,7 +2,8 @@ from __future__ import unicode_literals -import csv +from backports import csv +import io import os import progressbar @@ -13,7 +14,6 @@ from modoboa.core import models as core_models from modoboa.core.extensions import exts_pool from modoboa.lib.exceptions import Conflict -from .... import lib from .... import signals @@ -52,20 +52,18 @@ class ImportCommand(BaseCommand): progressbar.Percentage(), progressbar.Bar(), progressbar.ETA() ], maxval=num_lines ).start() - with open(filename) as f: + with io.open(filename, encoding="utf8") as f: reader = csv.reader(f, delimiter=options["sepchar"]) i = 0 for row in reader: if not row: continue - try: - fct = getattr(lib, "import_%s" % row[0].strip()) - except AttributeError: - fct = signals.import_object.send( - sender=self.__class__, objtype=row[0].strip()) - if not fct: - continue - fct = fct[0][1] + fct = signals.import_object.send( + sender=self.__class__, objtype=row[0].strip()) + fct = [func for x_, func in fct if func is not None] + if not fct: + continue + fct = fct[0] try: fct(superadmin, row, options) except Conflict: diff --git a/modoboa/admin/tests/test_import_.py b/modoboa/admin/tests/test_import_.py index 46a18d64c..cfe9f7317 100644 --- a/modoboa/admin/tests/test_import_.py +++ b/modoboa/admin/tests/test_import_.py @@ -20,7 +20,7 @@ from ..models import Domain, Alias, DomainAlias class ImportTestCase(ModoTestCase): @classmethod - def setUpTestData(cls): # noqa: N802 + def setUpTestData(cls): # noqa:N802 """Create test data.""" super(ImportTestCase, cls).setUpTestData() cls.localconfig.parameters.set_value( @@ -325,3 +325,23 @@ alias;user@test.com;True;admin@test.com self.assertTrue( Alias.objects.filter( address="user@test.com", internal=False).exists()) + + def test_domains_import_utf8(self): + response = self.client.get(reverse("admin:domain_import")) + self.assertEqual(response.status_code, 200) + self.assertIn("Provide a CSV", response.content.decode()) + + f = ContentFile("""domain; dómªin1.com; 1000; 100; True +dómain; dómªin2.com; 1000; 100; True +""".encode("utf8"), name="dómains.csv") + self.client.post( + reverse("admin:domain_import"), { + "sourcefile": f + } + ) + admin = User.objects.get(username="admin") + dom = Domain.objects.get(name="dómªin1.com") + self.assertEqual(dom.quota, 1000) + self.assertEqual(dom.default_mailbox_quota, 100) + self.assertTrue(dom.enabled) + self.assertTrue(admin.is_owner(dom)) diff --git a/modoboa/admin/views/export.py b/modoboa/admin/views/export.py index f916ea8c5..a2d82e79e 100644 --- a/modoboa/admin/views/export.py +++ b/modoboa/admin/views/export.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals -import csv +from backports import csv from rfc6266 import build_header @@ -13,7 +13,6 @@ from django.urls import reverse from django.http import HttpResponse from django.shortcuts import render from django.utils.translation import ugettext as _ -from django.utils.encoding import smart_str from django.utils import six from ..forms import ExportIdentitiesForm, ExportDomainsForm @@ -51,9 +50,7 @@ def export_identities(request): form = ExportIdentitiesForm(request.POST) form.is_valid() fp = six.StringIO() - csvwriter = csv.writer( - fp, delimiter=smart_str(form.cleaned_data["sepchar"]) - ) + csvwriter = csv.writer(fp, delimiter=form.cleaned_data["sepchar"]) identities = get_identities( request.user, **request.session['identities_filters']) for ident in identities: @@ -81,9 +78,7 @@ def export_domains(request): form = ExportDomainsForm(request.POST) form.is_valid() fp = six.StringIO() - csvwriter = csv.writer( - fp, delimiter=smart_str(form.cleaned_data["sepchar"]) - ) + csvwriter = csv.writer(fp, delimiter=form.cleaned_data["sepchar"]) for dom in get_domains(request.user, **request.session['domains_filters']): dom.to_csv(csvwriter) diff --git a/modoboa/admin/views/import_.py b/modoboa/admin/views/import_.py index 1291b41ee..b9b917807 100644 --- a/modoboa/admin/views/import_.py +++ b/modoboa/admin/views/import_.py @@ -2,9 +2,9 @@ from __future__ import unicode_literals -import csv +from backports import csv -from io import StringIO +import io from django.contrib.auth.decorators import ( login_required, permission_required, user_passes_test @@ -12,16 +12,14 @@ from django.contrib.auth.decorators import ( from django.urls import reverse from django.db import transaction from django.shortcuts import render +from django.utils.encoding import smart_text from django.utils.translation import ugettext as _ -from django.utils.encoding import smart_str, force_text -from django.utils import six from reversion import revisions as reversion from modoboa.lib.exceptions import ModoboaException, Conflict from ..forms import ImportIdentitiesForm, ImportDataForm -from .. import lib from .. import signals @@ -40,33 +38,23 @@ def importdata(request, formclass=ImportDataForm): form = formclass(request.POST, request.FILES) if form.is_valid(): try: - if six.PY2: - infile = request.FILES['sourcefile'] - else: - infile = StringIO( - force_text(request.FILES['sourcefile'].read()) - ) - reader = csv.reader( - infile, - delimiter=smart_str(form.cleaned_data['sepchar']) - ) + infile = io.TextIOWrapper( + request.FILES['sourcefile'].file, encoding="utf8") + reader = csv.reader(infile, delimiter=form.cleaned_data['sepchar']) except csv.Error as inst: - error = str(inst) - - if error is None: + error = smart_text(inst) + else: try: cpt = 0 for row in reader: if not row: continue - try: - fct = getattr(lib, "import_%s" % row[0].strip()) - except AttributeError: - fct = signals.import_object.send( - sender="importdata", objtype=row[0].strip()) - if not fct: - continue - fct = fct[0][1] + fct = signals.import_object.send( + sender="importdata", objtype=row[0].strip()) + fct = [func for x_, func in fct if func is not None] + if not fct: + continue + fct = fct[0] with transaction.atomic(): try: fct(request.user, row, form.cleaned_data) diff --git a/modoboa/core/models.py b/modoboa/core/models.py index 4750f645e..b7587d178 100644 --- a/modoboa/core/models.py +++ b/modoboa/core/models.py @@ -366,13 +366,13 @@ class User(AbstractUser): """ row = [ "account", - force_str(self.username), - force_str(self.password), - force_str(self.first_name), - force_str(self.last_name), - force_str(self.is_active), - force_str(self.role), - force_str(self.email) + smart_text(self.username), + smart_text(self.password), + smart_text(self.first_name), + smart_text(self.last_name), + smart_text(self.is_active), + smart_text(self.role), + smart_text(self.email) ] results = signals.account_exported.send( sender=self.__class__, user=self) diff --git a/requirements.txt b/requirements.txt index 74e0f4af7..96dfe8d44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ pytz requests rfc6266 lxml +backports.csv