Import / export features

This commit is contained in:
Antoine Nguyen
2022-04-21 18:13:08 +02:00
parent 32e70818b9
commit 74984d3420
22 changed files with 403 additions and 130 deletions
+2 -2
View File
@@ -1,4 +1,4 @@
FROM python:3.7-alpine as base
FROM python:3.9-alpine as base
MAINTAINER Antoine Nguyen <tonio@ngyn.org>
ENV VIRTUAL_ENV=/opt/venv
@@ -13,7 +13,7 @@ RUN python3 -m venv $VIRTUAL_ENV
WORKDIR /tmp
COPY requirements.txt /tmp
COPY test-requirements.txt /tmp
RUN python -m pip install -U pip
RUN python -m pip install -U pip
RUN pip install -r requirements.txt -r test-requirements.txt
FROM base as run
+1 -1
View File
@@ -13,6 +13,6 @@ services:
PYTHONPATH: ".."
DB: "SQLITE"
DEBUG: "True"
command: sh -c "python3 setup.py develop && cd test_project && python3 manage.py migrate && python3 manage.py load_initial_data && python3 manage.py runserver 0:8000"
command: sh -c "cd test_project && python3 manage.py migrate && python3 manage.py load_initial_data && python3 manage.py runserver 0:8000"
volumes:
- .:/code
+10
View File
@@ -46,5 +46,15 @@ export default {
},
deleteDomainAlias (aliasId) {
return repository.delete(`/${domainAliasResource}/${aliasId}/`)
},
exportAll () {
return repository.get(`/${domainResource}/export/`)
},
importFromCSV (data) {
return repository.post(`/${domainResource}/import/`, data, {
headers: {
'Content-Type': 'multipart/form-data'
}
})
}
}
+10
View File
@@ -5,5 +5,15 @@ const resource = 'identities'
export default {
getAll () {
return repository.get(`${resource}/`)
},
exportAll () {
return repository.get(`/${resource}/export/`)
},
importFromCSV (data) {
return repository.post(`/${resource}/import/`, data, {
headers: {
'Content-Type': 'multipart/form-data'
}
})
}
}
@@ -0,0 +1,95 @@
<template>
<v-card>
<v-card-title>
<span class="headline">{{ title }}</span>
</v-card-title>
<v-card-text>
<v-alert
text
type="info"
>
<translate tag="p">Provide a CSV file where lines respect one of the following formats:</translate>
<slot name="help" />
<translate tag="p">The first element of each line is mandatory and must be equal to one of the previous values.</translate>
<translate tag="p">You can use a different character as separator.</translate>
</v-alert>
<validation-observer ref="observer">
<validation-provider
v-slot="{ errors }"
name="name"
rules="required"
>
<label class="m-label"><translate>Select file</translate></label>
<v-file-input
v-model="form.sourcefile"
accept="text/csv"
:error-messages="errors"
truncate-length="15"
outlined
/>
</validation-provider>
<label class="m-label"><translate>Separator</translate></label>
<v-text-field
v-model="form.sepchar"
:error-messages="errors"
outlined
/>
<v-switch
v-model="form.continue_if_exists"
:label="'Continue on error'|translate"
color="primary"
/>
</validation-observer>
</v-card-text>
<v-card-actions>
<v-spacer></v-spacer>
<v-btn
@click="close"
>
<translate>Close</translate>
</v-btn>
<v-btn
color="primary"
@click="submit"
>
<translate>Import</translate>
</v-btn>
</v-card-actions>
</v-card>
</template>
<script>
export default {
props: {
helpText: String,
title: String
},
data () {
return {
form: {}
}
},
methods: {
close () {
this.form = {}
this.$refs.observer.reset()
this.$emit('close')
},
async submit () {
const valid = await this.$refs.observer.validate()
if (!valid) {
return
}
const data = new FormData()
data.append('sourcefile', this.form.sourcefile)
if (this.form.sepchar) {
data.append('sepchar', this.form.sepchar)
}
if (this.form.continue_if_exists) {
data.append('continue_if_exists', this.form.continue_if_exists)
}
this.$emit('submit', data)
}
}
}
</script>
+24
View File
@@ -0,0 +1,24 @@
import { bus } from '@/main'
export const importExportMixin = {
methods: {
exportContent (content, name) {
const blob = new Blob([content], { type: 'text/csv' })
const link = document.createElement('a')
link.href = URL.createObjectURL(blob)
link.download = `modoboa-${name}.csv`
link.click()
URL.revokeObjectURL(link.href)
},
importContent (api, data) {
api.importFromCSV(data).then(resp => {
if (resp.data.status) {
bus.$emit('notification', { msg: resp.data.message })
this.$refs.importForm.close()
} else {
bus.$emit('notification', { msg: resp.data.message, type: 'error' })
}
})
}
}
}
+45 -5
View File
@@ -3,10 +3,18 @@
<v-toolbar flat>
<v-toolbar-title><translate>Domains</translate></v-toolbar-title>
<v-spacer />
<v-btn class="mr-2">
<v-btn
class="mr-2"
:title="'Import domains and aliases from CSV file'|translate"
@click="showImportForm = true"
>
<v-icon>mdi-file-import-outline</v-icon>
</v-btn>
<v-btn class="mr-2">
<v-btn
class="mr-2"
:title="'Export domains and aliases to CSV'|translate"
@click="exportDomains"
>
<v-icon>mdi-file-export-outline</v-icon>
</v-btn>
<v-menu offset-y>
@@ -41,28 +49,60 @@
max-width="800px">
<domain-alias-form @close="showAliasForm = false" />
</v-dialog>
<v-dialog v-model="showImportForm"
persistent
max-width="800px">
<import-form
ref="importForm"
:title="'Import domains'|translate"
@submit="importDomains"
@close="showImportForm = false"
>
<template v-slot:help>
<ul>
<li><em>domain; name; quota; default mailbox quota; enabled</em></li>
<li><em>domainalias; name; targeted domain; enabled</em></li>
<li><em>relaydomain; name; target host; target port; service; enabled; verify recipients</em></li>
</ul>
</template>
</import-form>
</v-dialog>
</div>
</template>
<script>
import domains from '@/api/domains'
import DomainAliasForm from '@/components/domains/DomainAliasForm'
import DomainCreationForm from '@/components/domains/DomainCreationForm'
import DomainList from '@/components/domains/DomainList'
import { importExportMixin } from '@/mixins/importExport'
import ImportForm from '@/components/tools/ImportForm'
export default {
mixins: [importExportMixin],
components: {
DomainAliasForm,
DomainCreationForm,
DomainList
DomainList,
ImportForm
},
data () {
return {
showAliasForm: false,
showDomainWizard: false
showDomainWizard: false,
showImportForm: false
}
},
methods: {
exportDomains () {
domains.exportAll().then(resp => {
this.exportContent(resp.data, 'domains')
})
},
importDomains (data) {
this.importContent(domains, data)
}
}
}
</script>
+44 -5
View File
@@ -3,10 +3,18 @@
<v-toolbar flat>
<v-toolbar-title><translate>Identities</translate></v-toolbar-title>
<v-spacer />
<v-btn class="mr-2">
<v-btn
class="mr-2"
:title="'Import accounts and aliases from CSV file'|translate"
@click="showImportForm = true"
>
<v-icon>mdi-file-import-outline</v-icon>
</v-btn>
<v-btn class="mr-2">
<v-btn
class="mr-2"
:title="'Export accounts and aliases to CSV'|translate"
@click="exportIdentities"
>
<v-icon>mdi-file-export-outline</v-icon>
</v-btn>
<v-menu offset-y>
@@ -37,7 +45,7 @@
>
<account-creation-form @close="closeCreationWizard" @created="updateIdentities" />
</v-dialog>
<v-dialog
<v-dialog
v-model="showAliasCreationWizard"
fullscreen
scrollable
@@ -46,24 +54,47 @@
>
<alias-creation-form @close="closeAliasWizard" @created="updateIdentities" />
</v-dialog>
<v-dialog v-model="showImportForm"
persistent
max-width="800px">
<import-form
ref="importForm"
:title="'Import identities'|translate"
@submit="importIdentities"
@close="showImportForm = false"
>
<template v-slot:help>
<ul>
<li><em>account; loginname; password; first name; last name; enabled; group; address; quota; [, domain, ...]</em></li>
<li><em>alias; address; enabled; recipient; recipient; ...</em></li>
</ul>
</template>
</import-form>
</v-dialog>
</div>
</template>
<script>
import AccountCreationForm from '@/components/identities/AccountCreationForm'
import AliasCreationForm from '@/components/identities/AliasCreationForm'
import identities from '@/api/identities'
import IdentityList from '@/components/identities/IdentityList'
import { importExportMixin } from '@/mixins/importExport'
import ImportForm from '@/components/tools/ImportForm'
export default {
mixins: [importExportMixin],
components: {
AccountCreationForm,
AliasCreationForm,
IdentityList
IdentityList,
ImportForm
},
data () {
return {
showAliasCreationWizard: false,
showCreationWizard: false
showCreationWizard: false,
showImportForm: false
}
},
methods: {
@@ -75,6 +106,14 @@ export default {
},
updateIdentities () {
this.$refs.identities.fetchIdentities()
},
exportIdentities () {
identities.exportAll().then(resp => {
this.exportContent(resp.data, 'identities')
})
},
importIdentities (data) {
this.importContent(identities, data)
}
}
}
+1 -2
View File
@@ -9,7 +9,6 @@ from django.shortcuts import get_object_or_404
from django.utils.encoding import force_text
from django.utils.translation import ugettext as _, ugettext_lazy
from rest_flex_fields import FlexFieldsModelSerializer
from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import extend_schema_field
from rest_framework import serializers
@@ -111,7 +110,7 @@ class DomainSerializer(serializers.ModelSerializer):
return domain
class DomainAliasSerializer(FlexFieldsModelSerializer):
class DomainAliasSerializer(serializers.ModelSerializer):
"""Base DomainAlias serializer."""
class Meta:
+1 -2
View File
@@ -61,14 +61,13 @@ class DomainAliasFilterSet(dj_filters.FilterSet):
class DomainAliasViewSet(lib_viewsets.RevisionModelMixin,
lib_viewsets.ExpandableModelViewSet):
viewsets.ModelViewSet):
"""ViewSet for DomainAlias."""
filter_backends = (dj_filters.DjangoFilterBackend, )
filterset_class = DomainAliasFilterSet
permission_classes = [IsAuthenticated, DjangoModelPermissions, ]
renderer_classes = (renderers.JSONRenderer, lib_renderers.CSVRenderer)
serializer_expanded_fields = ["target"]
serializer_class = serializers.DomainAliasSerializer
def get_queryset(self):
+8
View File
@@ -548,3 +548,11 @@ class UserForwardSerializer(serializers.Serializer):
dj_validators.validate_email(recipient)
recipients.append(recipient)
return recipients
class CSVImportSerializer(serializers.Serializer):
"""Serializer for all CSV import endpoints."""
sourcefile = serializers.FileField()
sepchar = serializers.CharField(required=False, default=";")
continue_if_exists = serializers.BooleanField(default=False)
+56 -1
View File
@@ -7,13 +7,14 @@ from django.contrib.contenttypes.models import ContentType
from django_filters import rest_framework as dj_filters
from drf_spectacular.utils import extend_schema, extend_schema_view
from rest_framework import (
filters, mixins, permissions, response, status, viewsets
filters, mixins, parsers, permissions, response, status, viewsets
)
from rest_framework.decorators import action
from rest_framework.exceptions import PermissionDenied
from modoboa.admin.api.v1 import viewsets as v1_viewsets
from modoboa.core import models as core_models
from modoboa.lib import renderers as lib_renderers
from modoboa.lib import viewsets as lib_viewsets
from ... import lib
@@ -107,6 +108,33 @@ class DomainViewSet(lib_viewsets.RevisionModelMixin,
domain.remove_admin(serializer.validated_data["account"])
return response.Response()
@action(methods=["get"], detail=False,
renderer_classes=(lib_renderers.CSVRenderer,))
def export(self, request, **kwargs):
"""Export domains and aliases to CSV."""
result = []
for domain in self.get_queryset():
result += domain.to_csv_rows()
return response.Response(result)
@extend_schema(
request=serializers.CSVImportSerializer
)
@action(methods=["post"],
detail=False,
parser_classes=(parsers.MultiPartParser, parsers.FormParser),
url_path="import")
def import_from_csv(self, request, **kwargs):
"""Import domains and aliases from CSV file."""
serializer = serializers.CSVImportSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
status, msg = lib.import_data(
request.user,
request.FILES["sourcefile"],
serializer.validated_data
)
return response.Response({"status": status, "message": msg})
class AccountFilterSet(dj_filters.FilterSet):
"""Custom FilterSet for Account."""
@@ -192,6 +220,33 @@ class IdentityViewSet(viewsets.ViewSet):
lib.get_identities(request.user), many=True)
return response.Response(serializer.data)
@action(methods=["get"], detail=False,
renderer_classes=(lib_renderers.CSVRenderer,))
def export(self, request, **kwargs):
"""Export accounts and aliases to CSV."""
result = []
for idt in lib.get_identities(request.user):
result.append(idt.to_csv_row())
return response.Response(result)
@extend_schema(
request=serializers.CSVImportSerializer
)
@action(methods=["post"],
detail=False,
parser_classes=(parsers.MultiPartParser, parsers.FormParser),
url_path="import")
def import_from_csv(self, request, **kwargs):
"""Import accounts and aliases from CSV file."""
serializer = serializers.CSVImportSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
status, msg = lib.import_data(
request.user,
request.FILES["sourcefile"],
serializer.validated_data
)
return response.Response({"status": status, "message": msg})
class AliasViewSet(v1_viewsets.AliasViewSet):
"""Viewset for Alias."""
+50 -1
View File
@@ -1,5 +1,7 @@
"""Internal library for admin."""
import csv
import io
import ipaddress
import logging
import random
@@ -11,6 +13,7 @@ import dns.resolver
from dns.name import IDNA_2008_UTS_46
from django.core.exceptions import ValidationError
from django.db import transaction
from django.db.models import Q
from django.utils.encoding import smart_text
from django.utils.translation import ugettext as _
@@ -18,9 +21,13 @@ from django.utils.translation import ugettext as _
from django.contrib.auth import password_validation
from django.contrib.contenttypes.models import ContentType
from reversion import revisions as reversion
from modoboa.core import signals as core_signals
from modoboa.core.models import User
from modoboa.lib.exceptions import PermDeniedException
from modoboa.lib.exceptions import (
Conflict, ModoboaException, PermDeniedException
)
from modoboa.parameters import tools as param_tools
from . import signals
@@ -266,3 +273,45 @@ def make_password():
except ValidationError:
continue
return password
@reversion.create_revision()
def import_data(user, file_object, options: dict):
"""Generic import function
As the process of importing data from a CSV file is the same
whatever the type, we do a maximum of the work here.
"""
try:
infile = io.TextIOWrapper(file_object.file, encoding="utf8")
reader = csv.reader(infile, delimiter=options["sepchar"])
except csv.Error as inst:
error = str(inst)
else:
try:
cpt = 0
for row in reader:
if not row:
continue
fct = signals.import_object.send(
sender="importdata", objtype=row[0].strip())
fct = [func for x_, func in fct if func is not None]
if not fct:
continue
fct = fct[0]
with transaction.atomic():
try:
fct(user, row, options)
except Conflict:
if options["continue_if_exists"]:
continue
raise Conflict(
_("Object already exists: %s")
% options["sepchar"].join(row[:2])
)
cpt += 1
msg = _("%d objects imported successfully") % cpt
return True, msg
except (ModoboaException) as e:
error = str(e)
return False, error
+6 -2
View File
@@ -220,10 +220,14 @@ class Alias(AdminObject):
self.set_recipients([raddress.strip() for raddress in row[3:]])
self.post_create(user)
def to_csv(self, csvwriter):
def to_csv_row(self):
"""Return a row to include in a CSV file."""
row = ["alias", force_str(self.address), self.enabled]
row += self.recipients
csvwriter.writerow(row)
return row
def to_csv(self, csvwriter):
csvwriter.writerow(self.to_csv_row())
reversion.register(Alias)
+18 -10
View File
@@ -387,24 +387,32 @@ class Domain(mixins.MessageLimitMixin, AdminObject):
instance=self)
self.save(creator=user)
def to_csv_rows(self):
"""Return a row to include in a CSV file."""
result = [
[
"domain",
force_text(self.name),
self.quota,
self.default_mailbox_quota,
self.enabled
]
]
for dalias in self.domainalias_set.all():
result.append(dalias.to_csv_row())
return result
def to_csv(self, csvwriter):
"""Export domain and domain aliases to CSV format."""
csvwriter.writerow([
"domain",
force_text(self.name),
self.quota,
self.default_mailbox_quota,
self.enabled
])
for dalias in self.domainalias_set.all():
dalias.to_csv(csvwriter)
for row in self.to_csv_rows():
csvwriter.writerow(row)
def post_create(self, creator):
"""Post creation actions.
:param ``User`` creator: user whos created this domain
"""
super(Domain, self).post_create(creator)
super().post_create(creator)
for domalias in self.domainalias_set.all():
domalias.post_create(creator)
+6 -2
View File
@@ -78,13 +78,17 @@ class DomainAlias(AdminObject):
self.enabled = row[3].strip().lower() in ["true", "1", "yes", "y"]
self.save(creator=user)
def to_csv_row(self):
"""Export to row that can be included in a CSV file."""
return ["domainalias", self.name,
self.target.name, self.enabled]
def to_csv(self, csvwriter):
"""Export a domain alias using CSV format
:param csvwriter: a ``csv.writer`` object
"""
csvwriter.writerow(["domainalias", self.name,
self.target.name, self.enabled])
csvwriter.writerow(self.to_csv_row())
reversion.register(DomainAlias)
+9 -38
View File
@@ -15,6 +15,7 @@ from django.utils.encoding import smart_text
from django.utils.translation import ugettext as _
from modoboa.lib.exceptions import Conflict, ModoboaException
from .. import lib
from .. import signals
from ..forms import ImportDataForm, ImportIdentitiesForm
@@ -30,47 +31,17 @@ def importdata(request, formclass=ImportDataForm):
:param typ: a string indicating the object type being imported
:return: a ``Response`` instance
"""
error = None
msg = None
form = formclass(request.POST, request.FILES)
if form.is_valid():
try:
infile = io.TextIOWrapper(
request.FILES["sourcefile"].file, encoding="utf8")
reader = csv.reader(infile, delimiter=form.cleaned_data["sepchar"])
except csv.Error as inst:
error = smart_text(inst)
else:
try:
cpt = 0
for row in reader:
if not row:
continue
fct = signals.import_object.send(
sender="importdata", objtype=row[0].strip())
fct = [func for x_, func in fct if func is not None]
if not fct:
continue
fct = fct[0]
with transaction.atomic():
try:
fct(request.user, row, form.cleaned_data)
except Conflict:
if form.cleaned_data["continue_if_exists"]:
continue
raise Conflict(
_("Object already exists: %s"
% form.cleaned_data["sepchar"].join(row[:2]))
)
cpt += 1
msg = _("%d objects imported successfully" % cpt)
return render(request, "admin/import_done.html", {
"status": "ok", "msg": msg
})
except (ModoboaException) as e:
error = str(e)
status, msg = lib.import_data(
request.user, request.FILES["sourcefile"], form.cleaned_data)
if status:
return render(request, "admin/import_done.html", {
"status": "ok", "msg": msg
})
return render(request, "admin/import_done.html", {
"status": "ko", "msg": error
"status": "ko", "msg": msg
})
+12 -8
View File
@@ -365,13 +365,8 @@ class User(AbstractUser):
signals.account_imported.send(
sender=self.__class__, user=user, account=self, row=row[7:])
def to_csv(self, csvwriter):
"""Export this account.
The CSV format is used to export.
:param csvwriter: csv object
"""
def to_csv_row(self):
"""Return row that can be included in a CSV file."""
row = [
"account",
smart_text(self.username),
@@ -386,7 +381,16 @@ class User(AbstractUser):
sender=self.__class__, user=self)
for result in results:
row += result[1]
csvwriter.writerow(row)
return row
def to_csv(self, csvwriter):
"""Export this account.
The CSV format is used to export.
:param csvwriter: csv object
"""
csvwriter.writerow(self.to_csv_row())
reversion.register(User)
+1 -16
View File
@@ -13,24 +13,9 @@ class CSVRenderer(renderers.BaseRenderer):
format = "csv"
def render(self, data, media_type=None, renderer_context=None):
if renderer_context is None:
renderer_context = {}
headers = renderer_context.get("headers")
with StringIO() as fp:
csvwriter = csv.writer(fp)
for item in data:
if headers is None:
headers = item.keys()
row = ["domainalias"]
for header in headers:
if "__" in header:
parts = header.split("__")
value = item
for part in parts:
value = value.get(part)
else:
value = item.get(header)
row.append(value)
csvwriter.writerow(row)
csvwriter.writerow(item)
content = fp.getvalue()
return content
-32
View File
@@ -1,40 +1,8 @@
"""ViewSet related mixins and tools."""
from rest_framework import viewsets
from reversion.views import RevisionMixin
class ExpandableSerializerMixin:
"""
A mixin to allow serializers with expanded fields by default.
Useful for create/update operations that return a different output.
"""
serializer_expanded_fields = []
def get_serializer(self, *args, **kwargs):
"""
Return the serializer instance that should be used for validating and
deserializing input, and for serializing output.
"""
serializer_class = self.get_serializer_class()
kwargs['context'] = self.get_serializer_context()
if self.request.version == 'v2':
kwargs['expand'] = self.serializer_expanded_fields
return serializer_class(*args, **kwargs)
class ExpandableModelViewSet(ExpandableSerializerMixin,
viewsets.ModelViewSet):
"""
A ModelViewSet version which provides an expanded serializer
output for create/update actions.
"""
pass
class RevisionModelMixin(RevisionMixin):
"""
A mixin to wrap every request in a revision block.
-1
View File
@@ -12,7 +12,6 @@ djangorestframework>=3.11,<3.14
drf-spectacular
uritemplate
djangorestframework-simplejwt==5.1.0
drf-flex-fields
passlib~=1.7.4
bcrypt # Requires libffi-dev and python-dev
+4 -2
View File
@@ -79,9 +79,11 @@ if __name__ == "__main__":
zip_safe=False,
scripts=["bin/modoboa-admin.py"],
install_requires=INSTALL_REQUIRES,
use_scm_version={"local_scheme": local_scheme},
use_scm_version={
"local_scheme": local_scheme
},
python_requires=">=3.7",
setup_requires=["setuptools_scm"],
setup_requires=["setuptools_scm>6.4"],
extras_require={
"ldap": LDAP_REQUIRES,
"mysql": MYSQL_REQUIRES,