From 2eeba77c476fe2f4c7f3206e3ed89180df83d3cd Mon Sep 17 00:00:00 2001 From: Carlos Ramos Date: Mon, 6 Oct 2014 00:39:34 -0600 Subject: [PATCH 1/3] Improve the postfixadmin migration code: - Assign right permissions to domain administrators and superadministrator. - Correctly prepend the password schema to all admins. - Update model classes with the correct varchar length and other useful relations. --- .../commands/migrate_from_postfixadmin.py | 82 ++++++++++--------- modoboa/tools/pfxadmin_migrate/models.py | 63 +++++++------- 2 files changed, 75 insertions(+), 70 deletions(-) diff --git a/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py b/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py index 757d1a9ba..bd453ed3c 100644 --- a/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py +++ b/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py @@ -55,6 +55,7 @@ from django.db.models import Q import modoboa.core.models as core_models import modoboa.extensions.admin.models as admin_models from modoboa.lib.emailutils import split_mailbox +from modoboa.extensions.admin import grant_access_to_all_objects from modoboa.tools.pfxadmin_migrate import models as pf_models @@ -208,74 +209,74 @@ class Command(BaseCommand): options["mboxes_path"], domain.name, old_mb.maildir) new_mb.rename_dir(oldpath) - def _migrate_domain(self, old_dom, options, creator): + def _migrate_domain(self, pf_domain, options, creator): """Single domain migration.""" - print "Migrating domain %s" % old_dom.domain + print "\nMigrating domain %s" % pf_domain.domain newdom = admin_models.Domain() - newdom.name = old_dom.domain - newdom.enabled = old_dom.active - newdom.quota = old_dom.maxquota - newdom.dates = self._migrate_dates(old_dom) + newdom.name = pf_domain.domain + newdom.enabled = pf_domain.active + newdom.quota = pf_domain.maxquota + newdom.dates = self._migrate_dates(pf_domain) newdom.save(creator=creator, using=options["_to"]) + self._migrate_domain_aliases(newdom, options, creator) self._migrate_mailboxes(newdom, options, creator) self._migrate_mailbox_aliases(newdom, options, creator) - self._migrate_domain_aliases(newdom, options, creator) + self._migrate_admins(options, creator, pf_domain, newdom) - def _migrate_admins(self, options, creator): - """Administrators migration.""" - print "Migrating administrators" + def _migrate_admins(self, options, creator, pf_domain, newdom=None): + """Administrators migration. + + :param pf_domain: The ``pf_models.Domain`` to get the admins to migrate. + :param newdom: If None the admins will not be linked to any domain and + they will be SuperAdmins. Otherwise link them to this domain and make them + DomainAdmins. + """ + + if (newdom is None): + print "\nMigrating super administrators" + else: + print "\tMigrating administrators" dagroup = Group.objects.using(options["_to"]).get(name="DomainAdmins") - for old_admin in pf_models.Admin.objects.using(options["_from"]).all(): - local_part, domname = split_mailbox(old_admin.username) - try: - query = Q(username=old_admin.username) & \ - (Q(domain="ALL") | Q(domain=domname)) - creds = pf_models.DomainAdmins.objects \ - .using(options["_from"]).get(query) - except pf_models.DomainAdmins.DoesNotExist: - print ( - "Warning: skipping useless admin %s" % (old_admin.username) - ) - continue - try: - domain = admin_models.Domain.objects \ - .using(options["_to"]).get(name=domname) - except admin_models.Domain.DoesNotExist: - print ( - "Warning: skipping domain admin %s, domain not found" - % old_admin.username - ) - continue + for old_admin in pf_domain.admins.all(): + if (newdom is None): + print "\tMigrating %s" % old_admin.username + + #In postfixadmin, it's possible to have an admin account and a user account with the same username but they are completely different entities, so they do not have a common password. In Modoboa this does not makes sense, so we actually merge these two entities into a single user if both are found. Question is, which password should we use?. The admin password is used to be on the secure side. try: user = core_models.User.objects \ .using(options["_to"]).get(username=old_admin.username) + + if ("SimpleUsers" == user.group): + print "Warning: Found an admin account with the same username as normal user '%s'. The existing user will be promoted to admin and it's password changed to the admin's account password. " % user.username + except core_models.User.DoesNotExist: user = core_models.User() user.username = old_admin.username user.email = old_admin.username - user.password = old_admin.password user.is_active = old_admin.active user.save(creator=creator, using=options["_to"]) user.date_joined = old_admin.modified - if creds.domain == "ALL": + user.password = "{%s}%s" % \ + (options["passwords_scheme"].upper(), old_admin.password) + + if (newdom is None): + grant_access_to_all_objects(user, "SuperAdmins") user.is_superuser = True else: user.groups.add(dagroup) - domain.add_admin(user) + newdom.add_admin(user) + user.save(using=options["_to"]) def _do_migration(self, options, creator_username='admin'): """Run the complete migration.""" - pf_domains = pf_models.Domain.objects.using(options["_from"]).all() + pf_domains = pf_models.Domain.objects.using(options["_from"]).exclude(domain = 'ALL') creator = core_models.User.objects.using(options["_to"]).get( username=creator_username) for pf_domain in pf_domains: - if pf_domain.domain == "ALL": - continue - try: # Skip this domain if it's an alias is_domain_alias = pf_models.AliasDomain.objects \ @@ -285,7 +286,10 @@ class Command(BaseCommand): continue except pf_models.AliasDomain.DoesNotExist: self._migrate_domain(pf_domain, options, creator) - self._migrate_admins(options, creator) + + #Handle the ALL domain + pf_domain = pf_models.Domain.objects.using(options["_from"]).get(domain = 'ALL') + self._migrate_admins(options, creator, pf_domain) def handle(self, *args, **options): """Command entry point.""" diff --git a/modoboa/tools/pfxadmin_migrate/models.py b/modoboa/tools/pfxadmin_migrate/models.py index cbb86ac92..94f143ce2 100755 --- a/modoboa/tools/pfxadmin_migrate/models.py +++ b/modoboa/tools/pfxadmin_migrate/models.py @@ -11,8 +11,8 @@ from django.db import models class Admin(models.Model): - username = models.CharField(max_length=765, primary_key=True) - password = models.CharField(max_length=765) + username = models.CharField(max_length=255, primary_key=True) + password = models.CharField(max_length=255) created = models.DateTimeField() modified = models.DateTimeField() active = models.IntegerField() @@ -24,9 +24,9 @@ class Admin(models.Model): class Alias(models.Model): - address = models.CharField(max_length=765, primary_key=True) + address = models.CharField(max_length=255, primary_key=True) goto = models.TextField() - domain = models.CharField(max_length=765) + domain = models.CharField(max_length=255) created = models.DateTimeField() modified = models.DateTimeField() active = models.IntegerField() @@ -38,8 +38,8 @@ class Alias(models.Model): class AliasDomain(models.Model): - alias_domain = models.CharField(max_length=765, primary_key=True) - target_domain = models.CharField(max_length=765) + alias_domain = models.CharField(max_length=255, primary_key=True) + target_domain = models.CharField(max_length=255) created = models.DateTimeField() modified = models.DateTimeField() active = models.IntegerField() @@ -62,17 +62,18 @@ class Config(models.Model): class Domain(models.Model): - domain = models.CharField(max_length=765, primary_key=True) - description = models.CharField(max_length=765) + domain = models.CharField(max_length=255, primary_key=True) + description = models.CharField(max_length=255) aliases = models.IntegerField() mailboxes = models.IntegerField() maxquota = models.BigIntegerField() quota = models.BigIntegerField() - transport = models.CharField(max_length=765) + transport = models.CharField(max_length=255) backupmx = models.IntegerField() created = models.DateTimeField() modified = models.DateTimeField() active = models.IntegerField() + admins = models.ManyToManyField(Admin, through='DomainAdmins') class Meta: db_table = u'domain' @@ -81,8 +82,8 @@ class Domain(models.Model): class DomainAdmins(models.Model): - username = models.CharField(max_length=765) - domain = models.CharField(max_length=765) + username = models.ForeignKey(Admin, db_column='username') + domain = models.ForeignKey(Domain, db_column='domain') created = models.DateTimeField(primary_key=True) active = models.IntegerField() @@ -94,12 +95,12 @@ class DomainAdmins(models.Model): class Fetchmail(models.Model): id = models.IntegerField(primary_key=True) - mailbox = models.CharField(max_length=765) - src_server = models.CharField(max_length=765) + mailbox = models.CharField(max_length=255) + src_server = models.CharField(max_length=255) src_auth = models.CharField(max_length=33, blank=True) - src_user = models.CharField(max_length=765) - src_password = models.CharField(max_length=765) - src_folder = models.CharField(max_length=765) + src_user = models.CharField(max_length=255) + src_password = models.CharField(max_length=255) + src_folder = models.CharField(max_length=255) poll_time = models.IntegerField() fetchall = models.IntegerField() keep = models.IntegerField() @@ -107,7 +108,7 @@ class Fetchmail(models.Model): usessl = models.IntegerField() extra_options = models.TextField(blank=True) returned_text = models.TextField(blank=True) - mda = models.CharField(max_length=765) + mda = models.CharField(max_length=255) date = models.DateTimeField() class Meta: @@ -118,9 +119,9 @@ class Fetchmail(models.Model): class Log(models.Model): timestamp = models.DateTimeField() - username = models.CharField(max_length=765) - domain = models.CharField(max_length=765) - action = models.CharField(max_length=765) + username = models.CharField(max_length=255) + domain = models.CharField(max_length=255) + action = models.CharField(max_length=255) data = models.TextField() class Meta: @@ -130,13 +131,13 @@ class Log(models.Model): class Mailbox(models.Model): - username = models.CharField(max_length=765, primary_key=True) - password = models.CharField(max_length=765) - name = models.CharField(max_length=765) - maildir = models.CharField(max_length=765) + username = models.CharField(max_length=255, primary_key=True) + password = models.CharField(max_length=255) + name = models.CharField(max_length=255) + maildir = models.CharField(max_length=255) quota = models.BigIntegerField() - local_part = models.CharField(max_length=765) - domain = models.CharField(max_length=765) + local_part = models.CharField(max_length=255) + domain = models.CharField(max_length=255) created = models.DateTimeField() modified = models.DateTimeField() active = models.IntegerField() @@ -148,7 +149,7 @@ class Mailbox(models.Model): class Quota(models.Model): - username = models.CharField(max_length=765, primary_key=True) + username = models.CharField(max_length=255, primary_key=True) path = models.CharField(max_length=300, primary_key=True) current = models.BigIntegerField(null=True, blank=True) @@ -170,11 +171,11 @@ class Quota2(models.Model): class Vacation(models.Model): - email = models.CharField(max_length=765) - subject = models.CharField(max_length=765) + email = models.CharField(max_length=255) + subject = models.CharField(max_length=255) body = models.TextField() cache = models.TextField() - domain = models.CharField(max_length=765) + domain = models.CharField(max_length=255) created = models.DateTimeField() active = models.IntegerField() @@ -186,7 +187,7 @@ class Vacation(models.Model): class VacationNotification(models.Model): on_vacation = models.ForeignKey(Vacation, db_column='on_vacation') - notified = models.CharField(max_length=765, primary_key=True) + notified = models.CharField(max_length=255, primary_key=True) notified_at = models.DateTimeField() class Meta: From 534b21c896ec4b772d25e487edc8d8bff78609a7 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Oct 2014 09:57:43 +0200 Subject: [PATCH 2/3] Last fixes. see #609 --- .../commands/migrate_from_postfixadmin.py | 61 +++++++++++++------ modoboa/tools/pfxadmin_migrate/models.py | 4 +- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py b/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py index bd453ed3c..aacab4111 100644 --- a/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py +++ b/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py @@ -45,12 +45,12 @@ used by PostfixAdmin. """ from optparse import make_option import os +import re import sys from django.contrib.auth.models import Group from django.core.management.base import BaseCommand from django.db import transaction -from django.db.models import Q import modoboa.core.models as core_models import modoboa.extensions.admin.models as admin_models @@ -60,6 +60,19 @@ from modoboa.extensions.admin import grant_access_to_all_objects from modoboa.tools.pfxadmin_migrate import models as pf_models +def set_account_password(account, password, scheme): + """Correclty format the password before storing it. + + It seems postfixadmin sometimes prepend the scheme... but not + always :p + + """ + if re.search(r"\{[\w-]+\}", password): + account.password = password + else: + account.password = "{%s}%s" % (scheme.upper(), password) + + class Command(BaseCommand): """Migration from postfixadmin.""" @@ -190,8 +203,8 @@ class Command(BaseCommand): new_user.email = old_mb.username new_user.is_active = old_mb.active new_user.date_joined = old_mb.created - new_user.password = "{%s}%s" % \ - (options["passwords_scheme"].upper(), old_mb.password) + set_account_password( + new_user, old_mb.password, options["passwords_scheme"]) new_user.dates = self._migrate_dates(old_mb) new_user.save(creator=creator, using=options["_to"]) new_user.set_role("SimpleUsers") @@ -226,29 +239,41 @@ class Command(BaseCommand): def _migrate_admins(self, options, creator, pf_domain, newdom=None): """Administrators migration. - :param pf_domain: The ``pf_models.Domain`` to get the admins to migrate. - :param newdom: If None the admins will not be linked to any domain and - they will be SuperAdmins. Otherwise link them to this domain and make them - DomainAdmins. + :param pf_domain: The ``pf_models.Domain`` to get the admins to migrate. + :param newdom: If None the admins will not be linked to any domain + and they will be SuperAdmins. Otherwise link them to + this domain and make them DomainAdmins. """ - if (newdom is None): + if newdom is None: print "\nMigrating super administrators" else: print "\tMigrating administrators" dagroup = Group.objects.using(options["_to"]).get(name="DomainAdmins") for old_admin in pf_domain.admins.all(): - if (newdom is None): + if newdom is None: print "\tMigrating %s" % old_admin.username - #In postfixadmin, it's possible to have an admin account and a user account with the same username but they are completely different entities, so they do not have a common password. In Modoboa this does not makes sense, so we actually merge these two entities into a single user if both are found. Question is, which password should we use?. The admin password is used to be on the secure side. + # In postfixadmin, it's possible to have an admin account + # and a user account with the same username but they are + # completely different entities, so they do not have a + # common password. In Modoboa this does not makes sense, so + # we actually merge these two entities into a single user + # if both are found. Question is, which password should we + # use?. The admin password is used to be on the secure + # side. try: user = core_models.User.objects \ .using(options["_to"]).get(username=old_admin.username) - if ("SimpleUsers" == user.group): - print "Warning: Found an admin account with the same username as normal user '%s'. The existing user will be promoted to admin and it's password changed to the admin's account password. " % user.username + if "SimpleUsers" == user.group: + print ( + "Warning: Found an admin account with the same " + "username as normal user '%s'. The existing user will " + "be promoted to admin and it's password changed to the " + "admin's account password. " % user.username + ) except core_models.User.DoesNotExist: user = core_models.User() @@ -258,10 +283,10 @@ class Command(BaseCommand): user.save(creator=creator, using=options["_to"]) user.date_joined = old_admin.modified - user.password = "{%s}%s" % \ - (options["passwords_scheme"].upper(), old_admin.password) + set_account_password( + user, old_admin.password, options["passwords_scheme"]) - if (newdom is None): + if newdom is None: grant_access_to_all_objects(user, "SuperAdmins") user.is_superuser = True else: @@ -272,7 +297,8 @@ class Command(BaseCommand): def _do_migration(self, options, creator_username='admin'): """Run the complete migration.""" - pf_domains = pf_models.Domain.objects.using(options["_from"]).exclude(domain = 'ALL') + pf_domains = pf_models.Domain.objects.using( + options["_from"]).exclude(domain='ALL') creator = core_models.User.objects.using(options["_to"]).get( username=creator_username) for pf_domain in pf_domains: @@ -288,7 +314,8 @@ class Command(BaseCommand): self._migrate_domain(pf_domain, options, creator) #Handle the ALL domain - pf_domain = pf_models.Domain.objects.using(options["_from"]).get(domain = 'ALL') + pf_domain = pf_models.Domain.objects.using( + options["_from"]).get(domain='ALL') self._migrate_admins(options, creator, pf_domain) def handle(self, *args, **options): diff --git a/modoboa/tools/pfxadmin_migrate/models.py b/modoboa/tools/pfxadmin_migrate/models.py index 94f143ce2..c95f46216 100755 --- a/modoboa/tools/pfxadmin_migrate/models.py +++ b/modoboa/tools/pfxadmin_migrate/models.py @@ -150,7 +150,7 @@ class Mailbox(models.Model): class Quota(models.Model): username = models.CharField(max_length=255, primary_key=True) - path = models.CharField(max_length=300, primary_key=True) + path = models.CharField(max_length=100, primary_key=True) current = models.BigIntegerField(null=True, blank=True) class Meta: @@ -160,7 +160,7 @@ class Quota(models.Model): class Quota2(models.Model): - username = models.CharField(max_length=300, primary_key=True) + username = models.CharField(max_length=100, primary_key=True) bytes = models.BigIntegerField() messages = models.IntegerField() From ce2b04723fe6249cbd8fbd0146f7949c8fb632e6 Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Wed, 8 Oct 2014 10:09:14 +0200 Subject: [PATCH 3/3] Documentation update. see #609 --- doc/additional/migrating.rst | 13 +++--- .../commands/migrate_from_postfixadmin.py | 41 +------------------ 2 files changed, 7 insertions(+), 47 deletions(-) diff --git a/doc/additional/migrating.rst b/doc/additional/migrating.rst index 4f1e8f71c..22acee28b 100644 --- a/doc/additional/migrating.rst +++ b/doc/additional/migrating.rst @@ -42,23 +42,20 @@ variable and save your changes. You are now ready to start the migration so run the following commands:: $ cd - $ python manage.py migrate_from_postfixadmin -r -p + $ python manage.py migrate_from_postfixadmin -s + +```` must be replaced by the scheme used within +postfixadmin (``crypt`` most of the time). Depending on how many domains/mailboxes your existing setup contains, the migration can be long. Just wait for the script's ending. -.. note:: - - The rename operation is not synchronous. If you choose this option, - you must enable **Handle mailboxes on filesystem** option. See - :ref:`fs_operations`. - The procedure is over, edit the :file:`settings.py` file and: * remove the ``pfxadmin`` database connection from the ``DATABASES`` variable * remove the ``'modoboa.tools.pfxadmin_migrate',`` from the - ``INSTALLED_APPS`` variable + ``MODOBOA_APPS`` variable You should be able to connect to Modoboa using the same credentials you were using to connect to PostfixAdmin. diff --git a/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py b/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py index aacab4111..7b40d5461 100644 --- a/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py +++ b/modoboa/tools/pfxadmin_migrate/management/commands/migrate_from_postfixadmin.py @@ -12,17 +12,6 @@ will be lost on the new database. Here is the list: * Logs * Fetchmail table -Mailboxes organisation on the filesystem will change. PostfixAdmin -uses the following layout:: - - /domain.tld/user@domain.tld/ - -Whereas Modoboa uses the following:: - - /domain.tld/user/ - -To rename directories, you'll need the appropriate permissions. - Domain administrators that manage more than one domain will not be completly restored. They will only be administrator of the domain that corresponds to their username. @@ -44,9 +33,7 @@ used by PostfixAdmin. """ from optparse import make_option -import os import re -import sys from django.contrib.auth.models import Group from django.core.management.base import BaseCommand @@ -89,15 +76,6 @@ class Command(BaseCommand): help="Name of the Modoboa db connection declared in settings.py" " (default is default)" ), - make_option( - "-r", "--rename-dirs", action="store_true", default=False, - help="Rename mailbox directories (default is no)" - ), - make_option( - "-p", "--mboxes-path", default=None, - help="Path where directories are stored on the filesystem" - "(used only if --rename-dirs)" - ), make_option( "-s", "--passwords-scheme", default="crypt", help="Scheme used to crypt user passwords" @@ -184,14 +162,7 @@ class Command(BaseCommand): ) def _migrate_mailboxes(self, domain, options, creator): - """Migrate mailboxes of a single domain. - - .. note:: - - Mailboxes rename will be done later by the - handle_mailboxes_operation script. - - """ + """Migrate mailboxes of a single domain.""" print "\tMigrating mailboxes" old_mboxes = pf_models.Mailbox.objects \ .using(options["_from"]).filter(domain=domain.name) @@ -217,11 +188,6 @@ class Command(BaseCommand): new_mb.set_quota(old_mb.quota / 1024000, override_rules=True) new_mb.save(creator=creator, using=options["_to"]) - if options["rename_dirs"]: - oldpath = os.path.join( - options["mboxes_path"], domain.name, old_mb.maildir) - new_mb.rename_dir(oldpath) - def _migrate_domain(self, pf_domain, options, creator): """Single domain migration.""" print "\nMigrating domain %s" % pf_domain.domain @@ -313,15 +279,12 @@ class Command(BaseCommand): except pf_models.AliasDomain.DoesNotExist: self._migrate_domain(pf_domain, options, creator) - #Handle the ALL domain + # Handle the ALL domain pf_domain = pf_models.Domain.objects.using( options["_from"]).get(domain='ALL') self._migrate_admins(options, creator, pf_domain) def handle(self, *args, **options): """Command entry point.""" - if options["rename_dirs"] and options["mboxes_path"] is None: - print "Error: you must provide the --mboxes-path option" - sys.exit(1) with transaction.commit_on_success(): self._do_migration(options)