Fix importing csv files containing utf8 characters

The csv module on Python 2 doesn't support utf8 (even encoded as bytes), the easiest fix is to use the csv module backported from Python 3.

Fixes #1312

See https://docs.python.org/2/library/csv.html
See https://github.com/ryanhiebert/backports.csv

modoboa/admin/handlers.py

@@ -18,6 +18,7 @@ from modoboa.lib import permissions
 from modoboa.lib import signals as lib_signals
 from modoboa.parameters import tools as param_tools
 
+from . import signals as admin_signals
 from . import lib
 from . import models
 from . import postfix_maps
@@ -339,3 +340,21 @@ def add_widgets_to_admin_dashboard(sender, user, **kwargs):
         "template": template,
         "context": context
     }]
+
+
+@receiver(admin_signals.import_object)
+def get_import_func(sender, objtype, **kwargs):
+    """Return function used to import objtype."""
+    if objtype == "domain":
+        return lib.import_domain
+    elif objtype == "domainalias":
+        return lib.import_domainalias
+    elif objtype == "account":
+        return lib.import_account
+    elif objtype == "alias":
+        return lib.import_alias
+    elif objtype == "forward":
+        return lib.import_forward
+    elif objtype == "dlist":
+        return lib.import_dlist
+    return None

modoboa/admin/management/commands/subcommands/_export.py

@@ -2,7 +2,7 @@
 
 from __future__ import unicode_literals
 
-import csv
+from backports import csv
 import sys
 
 from django.core.management.base import BaseCommand

modoboa/admin/management/commands/subcommands/_import.py

@@ -2,7 +2,8 @@
 
 from __future__ import unicode_literals
 
-import csv
+from backports import csv
+import io
 import os
 
 import progressbar
@@ -13,7 +14,6 @@ from modoboa.core import models as core_models
 from modoboa.core.extensions import exts_pool
 from modoboa.lib.exceptions import Conflict
 
-from .... import lib
 from .... import signals
 
 
@@ -52,20 +52,18 @@ class ImportCommand(BaseCommand):
                 progressbar.Percentage(), progressbar.Bar(), progressbar.ETA()
             ], maxval=num_lines
         ).start()
-        with open(filename) as f:
+        with io.open(filename, encoding="utf8") as f:
             reader = csv.reader(f, delimiter=options["sepchar"])
             i = 0
             for row in reader:
                 if not row:
                     continue
-                try:
-                    fct = getattr(lib, "import_%s" % row[0].strip())
-                except AttributeError:
-                    fct = signals.import_object.send(
-                        sender=self.__class__, objtype=row[0].strip())
-                    if not fct:
-                        continue
-                    fct = fct[0][1]
+                fct = signals.import_object.send(
+                    sender=self.__class__, objtype=row[0].strip())
+                fct = [func for x_, func in fct if func is not None]
+                if not fct:
+                    continue
+                fct = fct[0]
                 try:
                     fct(superadmin, row, options)
                 except Conflict:

modoboa/admin/tests/test_import_.py

@@ -20,7 +20,7 @@ from ..models import Domain, Alias, DomainAlias
 class ImportTestCase(ModoTestCase):
 
     @classmethod
-    def setUpTestData(cls):  # noqa: N802
+    def setUpTestData(cls):  # noqa:N802
         """Create test data."""
         super(ImportTestCase, cls).setUpTestData()
         cls.localconfig.parameters.set_value(
@@ -325,3 +325,23 @@ alias;user@test.com;True;admin@test.com
         self.assertTrue(
             Alias.objects.filter(
                 address="user@test.com", internal=False).exists())
+
+    def test_domains_import_utf8(self):
+        response = self.client.get(reverse("admin:domain_import"))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn("Provide a CSV", response.content.decode())
+
+        f = ContentFile("""domain; dómªin1.com; 1000; 100; True
+dómain; dómªin2.com; 1000; 100; True
+""".encode("utf8"), name="dómains.csv")
+        self.client.post(
+            reverse("admin:domain_import"), {
+                "sourcefile": f
+            }
+        )
+        admin = User.objects.get(username="admin")
+        dom = Domain.objects.get(name="dómªin1.com")
+        self.assertEqual(dom.quota, 1000)
+        self.assertEqual(dom.default_mailbox_quota, 100)
+        self.assertTrue(dom.enabled)
+        self.assertTrue(admin.is_owner(dom))

modoboa/admin/views/export.py

@@ -2,7 +2,7 @@
 
 from __future__ import unicode_literals
 
-import csv
+from backports import csv
 
 from rfc6266 import build_header
 
@@ -13,7 +13,6 @@ from django.urls import reverse
 from django.http import HttpResponse
 from django.shortcuts import render
 from django.utils.translation import ugettext as _
-from django.utils.encoding import smart_str
 from django.utils import six
 
 from ..forms import ExportIdentitiesForm, ExportDomainsForm
@@ -51,9 +50,7 @@ def export_identities(request):
         form = ExportIdentitiesForm(request.POST)
         form.is_valid()
         fp = six.StringIO()
-        csvwriter = csv.writer(
-            fp, delimiter=smart_str(form.cleaned_data["sepchar"])
-        )
+        csvwriter = csv.writer(fp, delimiter=form.cleaned_data["sepchar"])
         identities = get_identities(
             request.user, **request.session['identities_filters'])
         for ident in identities:
@@ -81,9 +78,7 @@ def export_domains(request):
         form = ExportDomainsForm(request.POST)
         form.is_valid()
         fp = six.StringIO()
-        csvwriter = csv.writer(
-            fp, delimiter=smart_str(form.cleaned_data["sepchar"])
-        )
+        csvwriter = csv.writer(fp, delimiter=form.cleaned_data["sepchar"])
         for dom in get_domains(request.user,
                                **request.session['domains_filters']):
             dom.to_csv(csvwriter)

modoboa/admin/views/import_.py

@@ -2,9 +2,9 @@
 
 from __future__ import unicode_literals
 
-import csv
+from backports import csv
 
-from io import StringIO
+import io
 
 from django.contrib.auth.decorators import (
     login_required, permission_required, user_passes_test
@@ -12,16 +12,14 @@ from django.contrib.auth.decorators import (
 from django.urls import reverse
 from django.db import transaction
 from django.shortcuts import render
+from django.utils.encoding import smart_text
 from django.utils.translation import ugettext as _
-from django.utils.encoding import smart_str, force_text
-from django.utils import six
 
 from reversion import revisions as reversion
 
 from modoboa.lib.exceptions import ModoboaException, Conflict
 
 from ..forms import ImportIdentitiesForm, ImportDataForm
-from .. import lib
 from .. import signals
 
 
@@ -40,33 +38,23 @@ def importdata(request, formclass=ImportDataForm):
     form = formclass(request.POST, request.FILES)
     if form.is_valid():
         try:
-            if six.PY2:
-                infile = request.FILES['sourcefile']
-            else:
-                infile = StringIO(
-                    force_text(request.FILES['sourcefile'].read())
-                )
-            reader = csv.reader(
-                infile,
-                delimiter=smart_str(form.cleaned_data['sepchar'])
-            )
+            infile = io.TextIOWrapper(
+                request.FILES['sourcefile'].file, encoding="utf8")
+            reader = csv.reader(infile, delimiter=form.cleaned_data['sepchar'])
         except csv.Error as inst:
-            error = str(inst)
-
-        if error is None:
+            error = smart_text(inst)
+        else:
             try:
                 cpt = 0
                 for row in reader:
                     if not row:
                         continue
-                    try:
-                        fct = getattr(lib, "import_%s" % row[0].strip())
-                    except AttributeError:
-                        fct = signals.import_object.send(
-                            sender="importdata", objtype=row[0].strip())
-                        if not fct:
-                            continue
-                        fct = fct[0][1]
+                    fct = signals.import_object.send(
+                        sender="importdata", objtype=row[0].strip())
+                    fct = [func for x_, func in fct if func is not None]
+                    if not fct:
+                        continue
+                    fct = fct[0]
                     with transaction.atomic():
                         try:
                             fct(request.user, row, form.cleaned_data)

modoboa/core/models.py

@@ -366,13 +366,13 @@ class User(AbstractUser):
         """
         row = [
             "account",
-            force_str(self.username),
-            force_str(self.password),
-            force_str(self.first_name),
-            force_str(self.last_name),
-            force_str(self.is_active),
-            force_str(self.role),
-            force_str(self.email)
+            smart_text(self.username),
+            smart_text(self.password),
+            smart_text(self.first_name),
+            smart_text(self.last_name),
+            smart_text(self.is_active),
+            smart_text(self.role),
+            smart_text(self.email)
         ]
         results = signals.account_exported.send(
             sender=self.__class__, user=self)

requirements.txt

@@ -28,3 +28,4 @@ pytz
 requests
 rfc6266
 lxml
+backports.csv