Merge pull request #795 from tonioo/utf8_addresses_790

Utf8 addresses

modoboa/admin/forms/account.py

@@ -13,10 +13,12 @@ from modoboa.core.models import User
 from modoboa.lib import events, parameters
 from modoboa.lib.email_utils import split_mailbox
 from modoboa.lib.exceptions import PermDeniedException, Conflict, NotFound
+from modoboa.lib.fields import DomainNameField, UTF8EmailField
 from modoboa.lib.form_utils import (
-    DomainNameField, DynamicForm, TabForms, WizardForm, WizardStep
+    DynamicForm, TabForms, WizardForm, WizardStep
 )
 from modoboa.lib.permissions import get_account_roles
+from modoboa.lib.validators import validate_utf8_email
 from modoboa.lib.web_utils import render_to_json_response
 
 from ..models import Domain, Mailbox, Alias
@@ -121,13 +123,12 @@ class AccountFormGeneral(forms.ModelForm):
         return self.cleaned_data["role"]
 
     def clean_username(self):
-        from django.core.validators import validate_email
         if "role" not in self.cleaned_data:
             return self.cleaned_data["username"]
         if self.cleaned_data["role"] != "SimpleUsers":
             return self.cleaned_data["username"]
         uname = self.cleaned_data["username"].lower()
-        validate_email(uname)
+        validate_utf8_email(uname)
         return uname
 
     def clean_master_user(self):
@@ -167,7 +168,7 @@ class AccountFormMail(forms.Form, DynamicForm):
 
     """Form to handle mail part."""
 
-    email = forms.EmailField(label=ugettext_lazy("E-mail"), required=False)
+    email = UTF8EmailField(label=ugettext_lazy("E-mail"), required=False)
     quota = forms.IntegerField(
         label=ugettext_lazy("Quota"),
         required=False,
@@ -178,7 +179,7 @@ class AccountFormMail(forms.Form, DynamicForm):
         widget=forms.widgets.TextInput(attrs={"class": "form-control"})
     )
     quota_act = forms.BooleanField(required=False)
-    aliases = forms.EmailField(
+    aliases = UTF8EmailField(
         label=ugettext_lazy("Alias(es)"),
         required=False,
         help_text=ugettext_lazy(
@@ -211,7 +212,8 @@ class AccountFormMail(forms.Form, DynamicForm):
                 if ralias.alias.recipients_count >= 2:
                     continue
                 name = "aliases_%d" % cpt
-                self._create_field(forms.EmailField, name, ralias.alias.address)
+                self._create_field(
+                    UTF8EmailField, name, ralias.alias.address)
                 cpt += 1
             self.fields["email"].initial = self.mb.full_address
             self.fields["quota_act"].initial = self.mb.use_domain_quota
@@ -221,7 +223,7 @@ class AccountFormMail(forms.Form, DynamicForm):
             self.fields["quota_act"].initial = True
 
         if len(args) and isinstance(args[0], QueryDict):
-            self._load_from_qdict(args[0], "aliases", forms.EmailField)
+            self._load_from_qdict(args[0], "aliases", UTF8EmailField)
 
     def clean_email(self):
         """Ensure lower case emails"""

modoboa/admin/forms/alias.py

@@ -7,19 +7,19 @@ from django.http import QueryDict
 from django.utils.translation import ugettext as _, ugettext_lazy
 
 from modoboa.lib.email_utils import split_mailbox
+from modoboa.lib import fields as lib_fields
 from modoboa.lib.form_utils import (
     DynamicForm
 )
 
 from ..models import Domain, Alias
-from .base import EmailField
 
 
 class AliasForm(forms.ModelForm, DynamicForm):
 
     """A form to create/modify an alias."""
 
-    address = EmailField(
+    address = lib_fields.UTF8AndEmptyUserEmailField(
         label=ugettext_lazy("Email address"),
         help_text=ugettext_lazy(
             "The alias address. To create a catchall alias, just enter the "
@@ -27,7 +27,7 @@ class AliasForm(forms.ModelForm, DynamicForm):
         ),
         widget=forms.TextInput(attrs={"class": "form-control"})
     )
-    recipients = EmailField(
+    recipients = lib_fields.UTF8AndEmptyUserEmailField(
         label=ugettext_lazy("Recipients"), required=False,
         help_text=ugettext_lazy(
             "Addresses this alias will point to. Indicate only one address "

modoboa/admin/forms/base.py

@@ -1,26 +0,0 @@
-"""Custom form tools."""
-
-import re
-
-from django.core.validators import EmailValidator
-from django.forms.fields import EmailField as OriginalEmailField
-
-
-class CustomEmailValidator(EmailValidator):
-
-    """Allow empty user_part."""
-
-    user_regex = re.compile(
-        r"(^$"
-        r"|^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*\Z"  # dot-atom
-        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)',  # quoted-string
-        re.IGNORECASE)
-
-validate_email = CustomEmailValidator()
-
-
-class EmailField(OriginalEmailField):
-
-    """A field that accepts email addresses."""
-
-    default_validators = [validate_email]

modoboa/admin/forms/domain.py

@@ -8,8 +8,9 @@ from django.utils.translation import ugettext as _, ugettext_lazy
 from modoboa.core.models import User
 from modoboa.lib import events, parameters
 from modoboa.lib.exceptions import ModoboaException, Conflict
+from modoboa.lib.fields import DomainNameField
 from modoboa.lib.form_utils import (
-    DomainNameField, YesNoField, WizardForm, WizardStep, DynamicForm, TabForms
+    YesNoField, WizardForm, WizardStep, DynamicForm, TabForms
 )
 from modoboa.lib.web_utils import render_to_json_response
 

modoboa/admin/handlers.py

@@ -12,16 +12,16 @@ def create_alias_for_domainalias(sender, instance, **kwargs):
     if not kwargs.get("created"):
         return
     alias = models.Alias.objects.create(
-        address="@{}".format(instance.name), enabled=True, internal=True)
+        address=u"@{}".format(instance.name), enabled=True, internal=True)
     models.AliasRecipient.objects.create(
-        address="@{}".format(instance.target.name), alias=alias)
+        address=u"@{}".format(instance.target.name), alias=alias)
 
 
 @receiver(signals.post_delete, sender=models.DomainAlias)
 def remove_alias_for_domainalias(sender, instance, **kwargs):
     """Remove the alias associated to domain alias."""
     models.Alias.objects.filter(
-        address="@{}".format(instance.name)).delete()
+        address=u"@{}".format(instance.name)).delete()
 
 
 @receiver(signals.post_save, sender=models.Mailbox)

modoboa/admin/models/domain.py

@@ -3,6 +3,7 @@
 from django.contrib.contenttypes import generic
 from django.db import models
 from django.db.models.manager import Manager
+from django.utils.encoding import python_2_unicode_compatible, smart_text
 from django.utils.translation import ugettext as _, ugettext_lazy
 
 import reversion
@@ -26,6 +27,7 @@ class DomainManager(Manager):
         return self.get_queryset().filter(owners__user=admin)
 
 
+@python_2_unicode_compatible
 class Domain(AdminObject):
 
     """Mail domain."""
@@ -135,7 +137,7 @@ class Domain(AdminObject):
             events.raiseEvent("MailboxAliasDeleted", self.alias_set.all())
             ungrant_access_to_objects(self.alias_set.all())
         if parameters.get_admin("AUTO_ACCOUNT_REMOVAL") == "yes":
-            for account in User.objects.filter(mailbox__domain__name=self.name):
+            for account in User.objects.filter(mailbox__domain=self):
                 account.delete(fromuser, keepdir)
         elif self.mailbox_set.count():
             Quota.objects.filter(username__contains='@%s' % self.name).delete()
@@ -144,7 +146,7 @@ class Domain(AdminObject):
         super(Domain, self).delete()
 
     def __str__(self):
-        return self.name
+        return smart_text(self.name)
 
     def from_csv(self, user, row):
         """Create a new domain from a CSV entry.

modoboa/admin/models/domain_alias.py

@@ -1,6 +1,7 @@
 """Models related to domain aliases management."""
 
 from django.db import models
+from django.utils.encoding import python_2_unicode_compatible, smart_text
 from django.utils.translation import ugettext as _, ugettext_lazy
 
 import reversion
@@ -10,6 +11,7 @@ from .domain import Domain
 from modoboa.lib.exceptions import BadRequest, Conflict
 
 
+@python_2_unicode_compatible
 class DomainAlias(AdminObject):
 
     """Domain aliases."""
@@ -32,8 +34,8 @@ class DomainAlias(AdminObject):
         )
         app_label = "admin"
 
-    def __unicode__(self):
-        return self.name
+    def __str__(self):
+        return smart_text(self.name)
 
     def from_csv(self, user, row):
         """Create a domain alias from a CSV row

modoboa/admin/models/mailbox.py

@@ -7,6 +7,7 @@ from django.contrib.contenttypes.models import ContentType
 from django.db import models
 from django.db.models import Q
 from django.db.models.manager import Manager
+from django.utils.encoding import python_2_unicode_compatible, smart_text
 from django.utils.translation import ugettext as _, ugettext_lazy
 
 import reversion
@@ -69,6 +70,7 @@ class MailboxManager(Manager):
         return self.get_queryset().select_related().filter(qf)
 
 
+@python_2_unicode_compatible
 class Mailbox(AdminObject):
 
     """User mailbox."""
@@ -96,7 +98,7 @@ class Mailbox(AdminObject):
         self.__mail_home = None
 
     def __str__(self):
-        return self.full_address
+        return smart_text(self.full_address)
 
     def __full_address(self, localpart):
         return "%s@%s" % (localpart, self.domain.name)

modoboa/admin/templatetags/admin_tags.py

@@ -147,7 +147,7 @@ def identities_menu(user, selection=None):
 def domain_actions(user, domain):
     actions = [
         {"name": "listidentities",
-         "url": "{0}#list/?searchquery=@{1}".format(
+         "url": u"{0}#list/?searchquery=@{1}".format(
              reverse("admin:identity_list"), domain.name),
          "title": _("View the domain's identities"),
          "img": "fa fa-user"}

modoboa/admin/tests/account.py

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 from django.core.urlresolvers import reverse
 
 from modoboa.core.models import User
@@ -65,6 +67,16 @@ class AccountTestCase(ModoTestCase):
                 alias__internal=True).exists()
         )
 
+    def test_utf8_username(self):
+        """Create an account with non-ASCII characters."""
+        values = dict(
+            username="téster@test.com", first_name="Tester", last_name="Toto",
+            password1="Toto1234", password2="Toto1234", role="SimpleUsers",
+            quota_act=True,
+            is_active=True, email="téster@test.com", stepid="step2"
+        )
+        self.ajax_post(reverse("admin:account_add"), values)
+
     def _set_quota(self, email, value, expected_status=200):
         account = User.objects.get(username=email)
         values = {

modoboa/admin/tests/alias.py

@@ -114,6 +114,17 @@ class AliasTestCase(ModoTestCase):
                 r_alias__isnull=True).exists()
         )
 
+    def test_utf8_alias(self):
+        """Test alias with non-ASCII characters."""
+        values = dict(address="testé@test.com",
+                      recipients="user@test.com",
+                      recipients_1="admin@test.com",
+                      recipients_2="ext@titi.com",
+                      enabled=True)
+        self.ajax_post(
+            reverse("admin:alias_add"), values
+        )
+
     def test_dlist(self):
         values = dict(address="all@test.com",
                       recipients="user@test.com",

modoboa/admin/tests/domain.py

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 """Domain related test cases."""
 
 import json
@@ -35,6 +37,19 @@ class DomainTestCase(ModoTestCase):
         self.assertEqual(dom.enabled, False)
         self.assertFalse(dom.admins)
 
+    def test_create_utf8(self):
+        """Test the creation of a domain with non-ASCII characters."""
+        values = {
+            "name": "pouét.com", "quota": 100, "create_dom_admin": "no",
+            "type": "domain", "stepid": "step3"
+        }
+        self.ajax_post(reverse("admin:domain_add"), values)
+        dom = Domain.objects.get(name=u"pouét.com")
+        self.assertEqual(dom.name, u"pouét.com")
+        self.assertEqual(dom.quota, 100)
+        self.assertEqual(dom.enabled, False)
+        self.assertFalse(dom.admins)
+
     def test_create_with_template(self):
         """Test the creation of a domain with a template
 

modoboa/admin/tests/domain_alias.py

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 from django.core.urlresolvers import reverse
 
 from modoboa.lib.tests import ModoTestCase
@@ -38,7 +40,7 @@ class DomainAliasTestCase(ModoTestCase):
         dom = Domain.objects.get(name="test.com")
         values = dict(
             name=dom.name, quota=dom.quota, enabled=dom.enabled,
-            aliases="domalias.net", aliases_1="domalias.com",
+            aliases="domalias.net", aliases_1="domaliasé.com",
             type="domain"
         )
         self.ajax_post(

modoboa/core/models.py

@@ -13,6 +13,7 @@ from django.db import models
 from django.db.models.signals import post_delete
 from django.dispatch import receiver
 from django.utils import timezone
+from django.utils.encoding import python_2_unicode_compatible, smart_text
 from django.utils.translation import ugettext as _, ugettext_lazy
 
 import reversion
@@ -33,6 +34,7 @@ except ImportError:
     ldap_available = False
 
 
+@python_2_unicode_compatible
 class User(PermissionsMixin):
 
     """Custom User model.
@@ -182,7 +184,7 @@ class User(PermissionsMixin):
         return getattr(self, self.USERNAME_FIELD)
 
     def __str__(self):
-        return self.get_username()
+        return smart_text(self.get_username())
 
     def natural_key(self):
         return (self.get_username(),)

modoboa/lib/fields.py

@@ -0,0 +1,38 @@
+"""Custom form fields."""
+
+from django import forms
+from django.utils.translation import ugettext_lazy
+
+from . import validators
+
+
+class DomainNameField(forms.fields.CharField):
+
+    """A subclass of CharField that only accepts a valid domain name."""
+
+    default_error_messages = {
+        "invalid": ugettext_lazy("Enter a valid domain name")
+    }
+
+    default_validators = [validators.validate_hostname]
+
+    def clean(self, value):
+        value = self.to_python(value).strip()
+        return super(DomainNameField, self).clean(value)
+
+
+class UTF8EmailField(forms.fields.EmailField):
+
+    """A subclass of EmailField to support UTF8 addresses."""
+
+    default_validators = [validators.validate_utf8_email]
+
+
+class UTF8AndEmptyUserEmailField(forms.fields.EmailField):
+
+    """
+    A subclass of EmailField to support UTF8 addresses (with or without
+    local part).
+    """
+
+    default_validators = [validators.validate_utf8_and_empty_user_email]

modoboa/lib/form_utils.py

@@ -6,9 +6,8 @@ import abc
 from collections import OrderedDict
 import re
 
-from django.core.exceptions import ValidationError
 from django.forms import ChoiceField
-from django.forms.fields import CharField, Field
+from django.forms.fields import Field
 from django.forms.widgets import RadioSelect
 from django.forms.widgets import RadioChoiceInput
 from django.shortcuts import render
@@ -371,37 +370,6 @@ class TabForms(object):
 #
 
 
-def is_valid_hostname(hostname):
-    """Domain name validaton."""
-    if len(hostname) > 255:
-        return False
-    if hostname[-1] == ".":
-        hostname = hostname[:-1]  # strip exactly one dot from the
-                                  # right, if present
-    allowed = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
-    return all(allowed.match(x) for x in hostname.split("."))
-
-
-def validate_domain_name(value):
-    if not is_valid_hostname(value):
-        raise ValidationError(_('Enter a valid domain name'), 'invalid')
-
-
-class DomainNameField(CharField):
-    """
-    A subclass of CharField that only accepts a valid domain name.
-    """
-    default_error_messages = {
-        'invalid': _('Enter a valid domain name')
-    }
-
-    default_validators = [validate_domain_name]
-
-    def clean(self, value):
-        value = self.to_python(value).strip()
-        return super(DomainNameField, self).clean(value)
-
-
 class CustomRadioInput(RadioChoiceInput):
     def __unicode__(self):
         if 'id' in self.attrs:

modoboa/lib/validators.py

@@ -0,0 +1,63 @@
+"""Field validators."""
+
+import re
+
+from django.core.exceptions import ValidationError
+from django.core.validators import EmailValidator, URLValidator
+from django.utils.translation import ugettext_lazy
+
+
+class HostnameValidator(object):
+
+    """Validator for fqdn."""
+
+    message = ugettext_lazy("Enter a valid domain name")
+    code = "invalid"
+    regex = re.compile(URLValidator.host_re)
+
+    def __init__(self, message=None, code=None):
+        """Constructor."""
+        if message is not None:
+            self.message = message
+        if code is not None:
+            self.code = code
+
+    def __call__(self, value):
+        """Check value."""
+        if len(value) > 255:
+            raise ValidationError(self.message, self.code)
+        if value[-1] == ".":
+            # strip exactly one dot from the right, if present.
+            value = value[:-1]
+        if not self.regex.match(value):
+            raise ValidationError(self.message, self.code)
+
+validate_hostname = HostnameValidator()
+
+
+class UTF8EmailValidator(EmailValidator):
+
+    """Validator for addresses using non-ASCII characters."""
+
+    # unicode letters range (must be a unicode string, not a raw string)
+    ul = u"\u00a1-\uffff"
+    ascii_set = u"-!#$%&'*+/=?^_`{}|~0-9A-Z"
+    user_regex_raw = (
+        # dot-atom
+        ur"^[" + ascii_set + ul + "]+(\.[" + ascii_set + ul + "]+)*\Z"
+        # quoted-string
+        ur'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"\Z)'
+    )
+    user_regex = re.compile("(" + user_regex_raw, re.IGNORECASE)
+
+validate_utf8_email = UTF8EmailValidator()
+
+
+class UT8AndEmptyUserEmailValidator(UTF8EmailValidator):
+
+    """Same as upper + allows empty local part."""
+
+    user_regex = re.compile(
+        ur"(^$|" + UTF8EmailValidator.user_regex_raw, re.IGNORECASE)
+
+validate_utf8_and_empty_user_email = UT8AndEmptyUserEmailValidator()

modoboa/relaydomains/forms.py

@@ -4,7 +4,8 @@ Postfix relay domains extension forms.
 from django import forms
 from django.utils.translation import ugettext_lazy
 
-from modoboa.lib.form_utils import DomainNameField, WizardStep
+from modoboa.lib.fields import DomainNameField
+from modoboa.lib.form_utils import WizardStep
 
 from .models import RelayDomain