Merge pull request #3837 from modoboa/fix/contacts-reset-sync

Asynchronous sync from existing Address book
This commit is contained in:
Antoine Nguyen
2025-12-15 10:33:23 +01:00
committed by GitHub
15 changed files with 302 additions and 154 deletions
+1
View File
@@ -27,6 +27,7 @@ node_modules
# TOX stuff
test_project/modoboa.db
test_project/frontend
/.project
/.pydevproject
+1 -1
View File
@@ -17,7 +17,7 @@ services:
profiles: [ldap]
mariadb:
image: mariadb:10
image: mariadb:11
restart: unless-stopped
environment:
MARIADB_ROOT_PASSWORD: Password1000
+159 -103
View File
@@ -1,97 +1,120 @@
<template>
<v-card class="mt-6">
<v-data-table :headers="headers" :items="contacts">
<template #top>
<v-toolbar color="white" flat>
<v-text-field
v-model="search"
prepend-inner-icon="mdi-magnify"
:placeholder="$gettext('Search')"
variant="outlined"
single-line
flat
hide-details
density="compact"
class="flex-grow-0 w-33 mr-4"
></v-text-field>
<v-btn color="primary" variant="outlined" @click="showForm = true">
{{ $gettext('Add') }}
</v-btn>
<template v-if="preferences.enable_carddav_sync">
<v-btn
icon="mdi-information-outline"
:title="$gettext('Display address book information')"
class="ml-2"
@click="showAddressBookInfo = true"
/>
<v-btn
v-if="!addressBook.last_sync"
icon="mdi-sync"
color="success"
:title="$gettext('Synchronize contacts with address book')"
@click="syncToAddressBook"
/>
</template>
<v-chip
v-if="$route.params.category"
class="my-2 mx-4"
label
size="small"
color="green"
prepend-icon="mdi-label"
@click="editCategory"
>
{{ $route.params.category }}
</v-chip>
<v-spacer></v-spacer>
</v-toolbar>
</template>
<template #[`item.display_name`]="{ item }">
<template v-if="item.display_name">
{{ item.display_name }}
</template>
<template v-else> {{ item.first_name }} {{ item.last_name }} </template>
</template>
<template #[`item.emails`]="{ item }">
{{ displayEmails(item.emails) }}
</template>
<template #[`item.phone_numbers`]="{ item }">
{{ displayPhoneNumbers(item.phone_numbers) }}
</template>
<template #[`item.actions`]="{ item }">
<div class="text-right">
<v-menu offset-y>
<template #activator="{ props }">
<v-btn icon="mdi-dots-horizontal" variant="text" v-bind="props">
<div>
<v-card class="mt-6">
<v-data-table :headers="headers" :items="contacts">
<template #top>
<v-toolbar color="white" flat>
<v-text-field
v-model="search"
prepend-inner-icon="mdi-magnify"
:placeholder="$gettext('Search')"
variant="outlined"
single-line
flat
hide-details
density="compact"
class="flex-grow-0 w-33 mr-4"
></v-text-field>
<v-btn color="primary" variant="outlined" @click="showForm = true">
{{ $gettext('Add') }}
</v-btn>
<template v-if="preferences.enable_carddav_sync">
<v-btn class="ml-2" variant="tonal" icon size="small">
<v-icon icon="mdi-cog" />
<v-menu activator="parent">
<v-list density="compact">
<v-list-item
v-if="!addressBook.last_sync"
color="success"
:title="
$gettext('Synchronize contacts with address book')
"
prepend-icon="mdi-sync"
@click="syncToAddressBook"
/>
<v-list-item
:title="$gettext('Synchronize')"
prepend-icon="mdi-sync"
:disabled="addressBook.syncing"
@click="syncFromAddressBook(true)"
/>
</v-list>
</v-menu>
</v-btn>
<v-btn
icon="mdi-information-outline"
:title="$gettext('Display address book information')"
@click="showAddressBookInfo = true"
/>
</template>
<MenuItems :items="contactActions" :obj="item" />
</v-menu>
</div>
</template>
</v-data-table>
</v-card>
<v-dialog v-model="showForm" persistent max-width="800px">
<ContactForm
:contact="selectedContact"
@close="closeForm"
@added="fetchContacts"
@updated="fetchContacts"
/>
</v-dialog>
<v-dialog v-model="showCategoriesForm" persistent max-width="800px">
<ContactCategoriesForm
:contact="selectedContact"
@updated="fetchContacts"
@close="closeCategoriesForm"
/>
</v-dialog>
<v-dialog v-model="showAddressBookInfo" persistent max-width="800px">
<AddressBookInfo
:address-book-url="addressBook.url"
@close="showAddressBookInfo = false"
/>
</v-dialog>
<v-chip
v-if="$route.params.category"
class="my-2 mx-4"
label
size="small"
color="green"
prepend-icon="mdi-label"
@click="editCategory"
>
{{ $route.params.category }}
</v-chip>
<v-spacer></v-spacer>
<v-progress-circular
v-if="addressBook.syncing"
color="primary"
indeterminate
></v-progress-circular>
</v-toolbar>
</template>
<template #[`item.display_name`]="{ item }">
<template v-if="item.display_name">
{{ item.display_name }}
</template>
<template v-else>
{{ item.first_name }} {{ item.last_name }}
</template>
</template>
<template #[`item.emails`]="{ item }">
{{ displayEmails(item.emails) }}
</template>
<template #[`item.phone_numbers`]="{ item }">
{{ displayPhoneNumbers(item.phone_numbers) }}
</template>
<template #[`item.actions`]="{ item }">
<div class="text-right">
<v-menu offset-y>
<template #activator="{ props }">
<v-btn icon="mdi-dots-horizontal" variant="text" v-bind="props">
</v-btn>
</template>
<MenuItems :items="contactActions" :obj="item" />
</v-menu>
</div>
</template>
</v-data-table>
</v-card>
<v-dialog v-model="showForm" persistent max-width="800px">
<ContactForm
:contact="selectedContact"
@close="closeForm"
@added="fetchContacts"
@updated="fetchContacts"
/>
</v-dialog>
<v-dialog v-model="showCategoriesForm" persistent max-width="800px">
<ContactCategoriesForm
:contact="selectedContact"
@updated="fetchContacts"
@close="closeCategoriesForm"
/>
</v-dialog>
<v-dialog v-model="showAddressBookInfo" persistent max-width="800px">
<AddressBookInfo
:address-book-url="addressBook.url"
@close="showAddressBookInfo = false"
/>
</v-dialog>
</div>
</template>
<script setup>
@@ -191,8 +214,18 @@ function deleteContact(contact) {
})
}
async function fetchAddressBook() {
const resp = await contactsApi.getDefaultAddressBook()
addressBook.value = resp.data
}
async function fetchPreferences() {
const resp = await parametersApi.getUserApplication('contacts')
preferences.value = resp.data.params
}
function fetchContacts() {
contactsApi.getContacts(route.params).then((resp) => {
return contactsApi.getContacts(route.params).then((resp) => {
contacts.value = resp.data
})
}
@@ -203,20 +236,43 @@ function syncToAddressBook() {
})
}
contactsApi.getDefaultAddressBook().then((resp) => {
addressBook.value = resp.data
})
parametersApi.getUserApplication('contacts').then((resp) => {
preferences.value = resp.data.params
})
fetchContacts()
async function monitorAddressBook() {
await fetchAddressBook()
await fetchContacts()
if (!addressBook.value.syncing) {
clearInterval(intervalId)
intervalId = null
startAutomaticSync()
}
}
async function syncFromAddressBook(withNotification) {
if (intervalId) {
// Stop automatic sync for now.
clearInterval(intervalId)
}
await contactsApi.synchronizeFromAddressBook()
if (withNotification) {
displayNotification({ msg: $gettext('Synchronization started') })
}
addressBook.value.syncing = true
intervalId = setInterval(monitorAddressBook, 5000)
}
function startAutomaticSync() {
intervalId = setInterval(
syncFromAddressBook,
preferences.value.sync_frequency * 1000
)
}
await fetchAddressBook()
await fetchPreferences()
await fetchContacts()
watch(preferences, (value) => {
if (value) {
intervalId = setInterval(
contactsApi.synchronizeFromAddressBook,
value.sync_frequency * 1000
)
startAutomaticSync()
}
})
+13 -11
View File
@@ -1,16 +1,18 @@
<template>
<div class="text-h5 ml-4">
{{ $gettext('Contacts') }}
<div>
<div class="text-h5 ml-4">
{{ $gettext('Contacts') }}
</div>
<ContactList @edit-category="openCategoryForm" />
<v-dialog v-model="showCategoryForm" persistent max-width="800px">
<CategoryForm
:category="contactsStore.currentCategory"
@close="closeForm"
@added="fetchCategories"
@updated="fetchCategories"
/>
</v-dialog>
</div>
<ContactList @edit-category="openCategoryForm" />
<v-dialog v-model="showCategoryForm" persistent max-width="800px">
<CategoryForm
:category="contactsStore.currentCategory"
@close="closeForm"
@added="fetchCategories"
@updated="fetchCategories"
/>
</v-dialog>
</template>
<script setup>
+34 -18
View File
@@ -55,20 +55,23 @@ class MXTestCase(ModoTestCase):
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
self.assertEqual(models.MXRecord.objects.count(), 0)
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=0")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=0")
self.assertTrue(models.MXRecord.objects.filter(domain=self.domain).exists())
# we passed a ttl to 0. this will reset the cache this time
qs = models.MXRecord.objects.filter(domain=self.domain)
id_ = qs[0].id
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=7200")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=7200")
qs = models.MXRecord.objects.filter(domain=self.domain)
self.assertNotEqual(id_, qs[0].id)
id_ = qs[0].id
# assume that mxrecords ids are the same. means that we taking care of
# ttl
management.call_command("modo", "check_mx", "--no-dnsbl")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl")
qs = models.MXRecord.objects.filter(domain=self.domain)
self.assertEqual(id_, qs[0].id)
@@ -82,19 +85,24 @@ class MXTestCase(ModoTestCase):
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
management.call_command("modo", "check_mx", "--domain", self.domain.name)
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--domain", self.domain.name)
self.assertTrue(models.MXRecord.objects.filter(domain=self.domain).exists())
self.assertFalse(
models.MXRecord.objects.filter(domain=self.bad_domain).exists()
)
management.call_command("modo", "check_mx", "--domain", str(self.bad_domain.pk))
with LogCapture("modoboa.dns"):
management.call_command(
"modo", "check_mx", "--domain", str(self.bad_domain.pk)
)
self.assertFalse(
models.MXRecord.objects.filter(domain=self.bad_domain).exists()
)
self.assertEqual(len(mail.outbox), 1)
management.call_command("modo", "check_mx", "--domain", "toto.com")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--domain", "toto.com")
@mock.patch("gevent.socket.gethostbyname")
@mock.patch("socket.getaddrinfo")
@@ -113,13 +121,15 @@ class MXTestCase(ModoTestCase):
user__groups=("DomainAdmins",),
)
domain.add_admin(mb.user)
management.call_command(
"modo", "check_mx", "--domain", domain.name, "--no-dnsbl"
)
with LogCapture("modoboa.dns"):
management.call_command(
"modo", "check_mx", "--domain", domain.name, "--no-dnsbl"
)
self.assertEqual(domain.alarms.count(), 1)
management.call_command(
"modo", "check_mx", "--domain", domain.name, "--no-dnsbl"
)
with LogCapture("modoboa.dns"):
management.call_command(
"modo", "check_mx", "--domain", domain.name, "--no-dnsbl"
)
self.assertEqual(domain.alarms.count(), 1)
self.assertEqual(len(mail.outbox), 1)
@@ -130,7 +140,8 @@ class MXTestCase(ModoTestCase):
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
self.set_global_parameter("custom_dns_server", "123.45.67.89")
get_domain_mx_list("does-not-exist.example.com")
with LogCapture("modoboa.dns"):
get_domain_mx_list("does-not-exist.example.com")
@mock.patch("ipaddress.ip_address")
@mock.patch.object(dns.resolver.Resolver, "resolve")
@@ -235,7 +246,8 @@ class DNSBLTestCase(ModoTestCase):
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
self.assertEqual(models.DNSBLResult.objects.count(), 0)
management.call_command("modo", "check_mx")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx")
self.assertTrue(models.DNSBLResult.objects.filter(domain=self.domain).exists())
self.assertFalse(
models.DNSBLResult.objects.filter(domain=self.domain3).exists()
@@ -251,7 +263,8 @@ class DNSBLTestCase(ModoTestCase):
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "127.0.0.4"
management.call_command("modo", "check_mx", "--email", "user@example.test")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--email", "user@example.test")
self.assertEqual(len(mail.outbox), 2)
@mock.patch("gevent.socket.gethostbyname")
@@ -264,7 +277,8 @@ class DNSBLTestCase(ModoTestCase):
mock_query.side_effect = utils.mock_dns_query_result
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "127.255.255.254" # <--Spamhaus response when querying from an open resolver
management.call_command("modo", "check_mx", "--email", "user@example.test")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--email", "user@example.test")
self.assertEqual(len(mail.outbox), 1)
@mock.patch("gevent.socket.gethostbyname")
@@ -278,7 +292,8 @@ class DNSBLTestCase(ModoTestCase):
mock_getaddrinfo.side_effect = utils.mock_ip_query_result
mock_g_gethostbyname.return_value = "1.2.3.4"
self.assertEqual(models.DNSBLResult.objects.count(), 0)
management.call_command("modo", "check_mx", "--no-dnsbl")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl")
self.assertFalse(models.DNSBLResult.objects.filter(domain=self.domain).exists())
@@ -306,7 +321,8 @@ class DNSChecksTestCase(ModoTestCase):
self.domain.dkim_public_key = "XXXXX"
self.domain.save(update_fields=["enable_dkim", "dkim_public_key"])
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=0")
with LogCapture("modoboa.dns"):
management.call_command("modo", "check_mx", "--no-dnsbl", "--ttl=0")
self.assertIsNot(self.domain.spf_record, None)
self.assertIsNot(self.domain.dkim_record, None)
@@ -0,0 +1,18 @@
# Generated by Django 5.2.9 on 2025-12-12 14:42
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("contacts", "0007_alter_contact_address"),
]
operations = [
migrations.AddField(
model_name="addressbook",
name="syncing",
field=models.BooleanField(default=False),
),
]
+1
View File
@@ -22,6 +22,7 @@ class AddressBook(models.Model):
name = models.CharField(max_length=50)
sync_token = models.TextField(blank=True)
last_sync = models.DateTimeField(null=True)
syncing = models.BooleanField(default=False)
user = models.ForeignKey("core.User", on_delete=models.CASCADE)
_path = models.TextField()
+6 -3
View File
@@ -14,7 +14,7 @@ class AddressBookSerializer(serializers.ModelSerializer):
class Meta:
model = models.AddressBook
fields = ("pk", "name", "url", "last_sync")
fields = ("pk", "name", "url", "last_sync", "syncing")
class EmailAddressSerializer(serializers.ModelSerializer):
@@ -235,9 +235,12 @@ class UserPreferencesSerializer(serializers.Serializer):
def post_save(self, request):
"""Create remote cal if necessary."""
if not self.validated_data["enable_carddav_sync"]:
return
abook = request.user.addressbook_set.first()
if not self.validated_data["enable_carddav_sync"]:
abook.last_sync = None
abook.sync_token = ""
abook.save(update_fields=["last_sync", "sync_token"])
return
if abook.last_sync:
return
tasks.create_cdav_addressbook(abook, request.auth)
+9 -3
View File
@@ -49,12 +49,17 @@ def push_addressbook_to_carddav(request, addressbook):
addressbook.save(update_fields=["last_sync", "sync_token"])
def sync_addressbook_from_cdav(request, addressbook):
def sync_addressbook_from_cdav(addressbook_id: int, access_token: str):
"""Fetch changes from CardDAV server."""
clt = get_cdav_client_from_request(request, addressbook)
addressbook = models.AddressBook.objects.get(id=addressbook_id)
if addressbook.syncing:
return
clt = get_cdav_client(addressbook, addressbook.user.email, access_token)
changes = clt.sync_vcards(addressbook.sync_token)
if not len(changes["cards"]):
return
addressbook.syncing = True
addressbook.save()
for card in changes["cards"]:
# UID sometimes embded .vcf extension, sometimes not...
long_uid = card["href"].split("/")[-1]
@@ -73,7 +78,8 @@ def sync_addressbook_from_cdav(request, addressbook):
models.Contact.objects.filter(uid__in=[long_uid, short_uid]).delete()
addressbook.last_sync = timezone.now()
addressbook.sync_token = changes["token"]
addressbook.save(update_fields=["last_sync", "sync_token"])
addressbook.syncing = False
addressbook.save(update_fields=["last_sync", "sync_token", "syncing"])
def push_contact_to_cdav(request, contact):
+29 -3
View File
@@ -3,12 +3,14 @@
import os
import httmock
from rq import SimpleWorker
from django.core import management
from django.urls import reverse
from django.utils import timezone
import django_rq
from modoboa.admin import factories as admin_factories
from modoboa.core import models as core_models
from modoboa.lib.tests import ModoAPITestCase
@@ -65,13 +67,25 @@ class TestDataMixin:
)
self.assertEqual(response.status_code, 200)
def disable_cdav_sync(self):
url = reverse("v2:parameter-user-detail", args=["contacts"])
response = self.client.put(
url,
{
"enable_carddav_sync": False,
"sync_frequency": 300,
},
format="json",
)
self.assertEqual(response.status_code, 200)
class ViewsTestCase(TestDataMixin, ModoAPITestCase):
"""Check views."""
def test_user_settings(self):
"""Check that remote collection creation request is sent."""
# 1. Addressbook with contacts must be synced manually
# 1. Addressbook with contacts must be synced manuelle
self.client.force_authenticate(self.user)
self.enable_cdav_sync()
self.addressbook.refresh_from_db()
@@ -87,6 +101,10 @@ class ViewsTestCase(TestDataMixin, ModoAPITestCase):
self.enable_cdav_sync()
abook.refresh_from_db()
self.assertIsNot(abook.last_sync, None)
# And disable sync.
self.disable_cdav_sync()
abook.refresh_from_db()
self.assertIs(abook.last_sync, None)
class AddressBookViewSetTestCase(TestDataMixin, ModoAPITestCase):
@@ -118,10 +136,18 @@ class AddressBookViewSetTestCase(TestDataMixin, ModoAPITestCase):
data = {"username": self.user.username, "password": "toto"}
response = self.client.post(reverse("core:login"), data)
self.enable_cdav_sync()
self.user.addressbook_set.update(last_sync=timezone.now())
last_sync = timezone.now()
self.user.addressbook_set.update(last_sync=last_sync)
addressbook = self.user.addressbook_set.first()
self.assertEqual(addressbook.sync_token, "")
with httmock.HTTMock(mocks.options_mock, mocks.report_mock, mocks.get_mock):
response = self.client.get(reverse("api:addressbook-sync-from-cdav"))
queue = django_rq.get_queue("modoboa")
worker = SimpleWorker([queue], connection=queue.connection)
worker.work(burst=True)
self.assertEqual(response.status_code, 200)
addressbook.refresh_from_db()
self.assertNotEqual(addressbook.sync_token, "")
class CategoryViewSetTestCase(TestDataMixin, ModoAPITestCase):
+3 -1
View File
@@ -1,6 +1,7 @@
"""Contacts viewsets."""
import django_filters.rest_framework
import django_rq
from rest_framework.decorators import action
from rest_framework import filters, response, viewsets
from rest_framework.permissions import IsAuthenticated
@@ -40,7 +41,8 @@ class AddressBookViewSet(viewsets.GenericViewSet):
if not abook.last_sync:
return response.Response()
if request.user.parameters.get_value("enable_carddav_sync", app="contacts"):
tasks.sync_addressbook_from_cdav(request, abook)
queue = django_rq.get_queue("modoboa")
queue.enqueue(tasks.sync_addressbook_from_cdav, abook.pk, str(request.auth))
return response.Response({})
-9
View File
@@ -596,15 +596,6 @@ class FIDOViewSetTestCase(ModoAPITestCase):
self.assertIn("tokens", resp.json())
class CapabilitiesAPITestCase(ModoAPITestCase):
def test_get(self):
url = reverse("v2:capabilities")
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertIn("rspamd", response.json()["capabilities"])
class ThemeAPITestCase(ModoAPITestCase):
def test_get(self):
+1 -1
View File
@@ -18,7 +18,7 @@ class GlobalParametersAPITestCase(ModoAPITestCase):
resp = self.client.get(url)
self.assertEqual(resp.status_code, 200)
# core, admin, limits, pdf credentials, ...
self.assertEqual(len(resp.json()), 13)
self.assertEqual(len(resp.json()), 12)
def test_get_structure(self):
url = reverse("v2:parameter-global-structure")
+20
View File
@@ -2,12 +2,18 @@ import os
import shutil
from django.core import management
from django.test import modify_settings
from django.urls import reverse
from modoboa.admin import factories as admin_factories, models as admin_models
from modoboa.lib.tests import ModoAPITestCase
@modify_settings(
INSTALLED_APPS={
"append": "modoboa.rspamd",
}
)
class ManagementCommandTestCase(ModoAPITestCase):
@classmethod
@@ -76,3 +82,17 @@ class ParametersAPITestCase(ModoAPITestCase):
}
response = self.client.put(url, data, format="json")
self.assertEqual(response.status_code, 200)
@modify_settings(
MODOBOA_APPS={
"append": "modoboa.rspamd",
}
)
class CapabilitiesAPITestCase(ModoAPITestCase):
def test_get(self):
url = reverse("v2:capabilities")
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertIn("rspamd", response.json()["capabilities"])
+7 -1
View File
@@ -110,7 +110,7 @@ MODOBOA_APPS = (
"modoboa.imap_migration",
"modoboa.autoreply",
"modoboa.sievefilters",
"modoboa.rspamd",
# "modoboa.rspamd",
# Modoboa extensions here.
"modoboa.contacts",
"modoboa.calendars",
@@ -353,6 +353,11 @@ LOGGING = {
"class": "django.utils.log.AdminEmailHandler",
"include_html": True,
},
"syslog": {
"class": "logging.handlers.SysLogHandler",
"facility": SysLogHandler.LOG_SYSLOG,
"formatter": "syslog",
},
"syslog-auth": {
"class": "logging.handlers.SysLogHandler",
"facility": SysLogHandler.LOG_AUTH,
@@ -376,6 +381,7 @@ LOGGING = {
"propagate": False,
},
"modoboa.admin": {"handlers": ["modoboa"], "level": "INFO", "propagate": False},
"modoboa.dns": {"handlers": ["syslog"], "level": "INFO", "propagate": False},
"django.server": {
"handlers": ["django.server"],
"level": "INFO",