diff --git a/modoboa/core/__init__.py b/modoboa/core/__init__.py index fd3f335b2..95ccfde83 100644 --- a/modoboa/core/__init__.py +++ b/modoboa/core/__init__.py @@ -48,10 +48,8 @@ PERMISSIONS = { @events.observe("TopNotifications") -def check_for_new_version(request, include_all): - """ - Check if a new version of Modoboa is available. - """ +def check_for_new_versions(request, include_all): + """Check if new versions are available.""" from modoboa.core.utils import check_for_updates if not request.user.is_superuser: diff --git a/modoboa/core/app_settings.py b/modoboa/core/app_settings.py index c8ac1c080..28d1f1fe5 100644 --- a/modoboa/core/app_settings.py +++ b/modoboa/core/app_settings.py @@ -220,6 +220,30 @@ class GeneralParametersForm(parameters.AdminParametersForm): ) ) + api_sep = SeparatorField(label=ugettext_lazy("Public API")) + + enable_api_communication = YesNoField( + label=ugettext_lazy("Enable communication"), + initial="yes", + help_text=ugettext_lazy( + "Enable communication with Modoboa public API") + ) + + check_new_versions = YesNoField( + label=ugettext_lazy("Check new versions"), + initial="yes", + help_text=ugettext_lazy( + "Automatically checks if a newer version is available") + ) + + send_statistics = YesNoField( + label=ugettext_lazy("Send statistics"), + initial="yes", + help_text=ugettext_lazy( + "Send statistics to Modoboa public API " + "(counters and used extensions)") + ) + sep3 = SeparatorField(label=ugettext_lazy("Miscellaneous")) top_notifications_check_interval = forms.IntegerField( @@ -231,13 +255,6 @@ class GeneralParametersForm(parameters.AdminParametersForm): widget=forms.TextInput(attrs={"class": "form-control"}) ) - check_new_versions = YesNoField( - label=ugettext_lazy("Check new versions"), - initial="yes", - help_text=ugettext_lazy( - "Automatically checks if a newer version is available") - ) - log_maximum_age = forms.IntegerField( label=ugettext_lazy("Maximum log record age"), initial=365, @@ -279,6 +296,8 @@ class GeneralParametersForm(parameters.AdminParametersForm): "ldap_admin_groups": "authentication_type=ldap", "ldap_group_type": "authentication_type=ldap", "ldap_groups_search_base": "authentication_type=ldap", + "check_new_versions": "enable_api_communication=yes", + "send_statistics": "enable_api_communication=yes", } def __init__(self, *args, **kwargs): diff --git a/modoboa/core/apps.py b/modoboa/core/apps.py index 61435efbb..1b8e3cf17 100644 --- a/modoboa/core/apps.py +++ b/modoboa/core/apps.py @@ -1,6 +1,7 @@ """Core config for admin.""" from django.apps import AppConfig +from django.db.models import signals BASE_EVENTS = [ @@ -66,3 +67,5 @@ class CoreConfig(AppConfig): load_core_settings() from . import handlers + + signals.post_migrate.connect(handlers.create_local_config, sender=self) diff --git a/modoboa/core/extensions.py b/modoboa/core/extensions.py index 3abf569a6..6c29b2b27 100644 --- a/modoboa/core/extensions.py +++ b/modoboa/core/extensions.py @@ -3,11 +3,6 @@ from django.conf import settings from django.conf.urls import include -from versionfield.constants import DEFAULT_NUMBER_BITS -from versionfield.version import Version - -from modoboa.lib.api_client import ModoAPIClient - class ModoExtension(object): @@ -115,32 +110,14 @@ class ExtensionsPool(object): result += [ext_urls] return result - def list_all(self, check_new_versions=False): + def list_all(self): """List all defined extensions.""" result = [] - new_extensions = {} - if check_new_versions: - new_extensions = ModoAPIClient().list_extensions() - if new_extensions: - new_extensions = dict( - (ext["name"], ext["version"]) for ext in new_extensions - ) - else: - new_extensions = {} - for extname, extdef in self.extensions.iteritems(): if not extdef["show"]: continue infos = self.get_extension_infos(extname) infos["id"] = extname - local_version = Version(infos["version"], DEFAULT_NUMBER_BITS) - pkgname = infos["name"].replace("_", "-") - if pkgname in new_extensions: - infos["last_version"] = new_extensions[pkgname] - last_version = Version( - new_extensions[pkgname], DEFAULT_NUMBER_BITS) - if last_version > local_version: - infos["update"] = True result += [infos] return sorted(result, key=lambda i: i["name"]) diff --git a/modoboa/core/handlers.py b/modoboa/core/handlers.py index 2f77e81bd..576ed3798 100644 --- a/modoboa/core/handlers.py +++ b/modoboa/core/handlers.py @@ -6,8 +6,12 @@ from django.db.models.signals import post_delete from django.dispatch import receiver from django.utils.translation import ugettext as _ +from django.contrib.sites import models as sites_models + from reversion import revisions as reversion +from . import models + @receiver(reversion.post_revision_commit) def post_revision_commit(sender, **kwargs): @@ -63,3 +67,11 @@ def log_object_removal(sender, instance, **kwargs): "user": get_request().user.username } logger.critical(msg) + + +def create_local_config(sender, **kwargs): + """Create local config if needed.""" + if models.LocalConfig.objects.using(kwargs["using"]).exists(): + return + models.LocalConfig.objects.create( + site=sites_models.Site.objects.get_current()) diff --git a/modoboa/core/management/commands/communicate_with_public_api.py b/modoboa/core/management/commands/communicate_with_public_api.py new file mode 100644 index 000000000..e06a45956 --- /dev/null +++ b/modoboa/core/management/commands/communicate_with_public_api.py @@ -0,0 +1,52 @@ +"""Management command to communicate with public API.""" + +from django.core.management.base import BaseCommand, CommandError + +from modoboa.admin import models as admin_models +from modoboa.core.extensions import exts_pool +from modoboa.lib import api_client, parameters + +from ... import models + + +class Command(BaseCommand): + """Command class.""" + + help = "Communicate with Modoboa public API." + + def handle(self, *args, **options): + """Command entry point.""" + if parameters.get_admin("ENABLE_API_COMMUNICATION") == "no": + return + self.client = api_client.ModoAPIClient() + local_config = ( + models.LocalConfig.objects.select_related("site").first()) + if not local_config.api_pk: + pk = self.client.register_instance(local_config.site.domain) + if pk is None: + raise CommandError("Instance registration failed.") + local_config.api_pk = pk + + if parameters.get_admin("CHECK_NEW_VERSIONS") == "yes": + versions = self.client.versions() + if versions is None: + raise CommandError("Failed to retrieve versions from the API.") + local_config.api_versions = versions + + local_config.save() + + if parameters.get_admin("SEND_STATISTICS") == "no": + return + extensions = [ext["name"] for ext in exts_pool.list_all()] + data = { + "hostname": local_config.site.domain, + "known_version": self.client.local_core_version, + "domain_counter": admin_models.Domain.objects.count(), + "domain_alias_counter": admin_models.DomainAlias.objects.count(), + "mailbox_counter": admin_models.Mailbox.objects.count(), + "alias_counter": ( + admin_models.Alias.objects.filter(internal=False).count()), + "extensions": extensions + } + if not self.client.update_instance(local_config.api_pk, data): + raise CommandError("Failed to update instance.") diff --git a/modoboa/core/migrations/0008_localconfig.py b/modoboa/core/migrations/0008_localconfig.py new file mode 100644 index 000000000..be8040af8 --- /dev/null +++ b/modoboa/core/migrations/0008_localconfig.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-06-15 18:12 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('sites', '0002_alter_domain_unique'), + ('core', '0007_auto_20151116_1349'), + ] + + operations = [ + migrations.CreateModel( + name='LocalConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('api_pk', models.PositiveIntegerField(null=True)), + ('api_versions', jsonfield.fields.JSONField()), + ('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sites.Site')), + ], + ), + ] diff --git a/modoboa/core/mocks.py b/modoboa/core/mocks.py new file mode 100644 index 000000000..7c8f245f7 --- /dev/null +++ b/modoboa/core/mocks.py @@ -0,0 +1,41 @@ +"""Mocks used for testing.""" + +import httmock + + +# Modoboa API mocks + +@httmock.urlmatch( + netloc=r"api\.modoboa\.org$", path=r"^/1/instances/search/", method="post") +def modo_api_instance_search(url, request): + """Return empty response.""" + return {"status_code": 404} + + +@httmock.urlmatch( + netloc=r"api\.modoboa\.org$", path=r"^/1/instances/", method="post") +def modo_api_instance_create(url, request): + """Simulate successful creation.""" + return { + "status_code": 201, + "content": {"pk": 100} + } + + +@httmock.urlmatch( + netloc=r"api\.modoboa\.org$", path=r"^/1/instances/.+/", method="put") +def modo_api_instance_update(url, request): + """Simulate successful update.""" + return {"status_code": 200} + + +@httmock.urlmatch( + netloc=r"api\.modoboa\.org$", path=r"^/1/versions/", method="get") +def modo_api_versions(url, request): + """Simulate versions check.""" + return { + "status_code": 200, + "content": [ + {"name": "modoboa", "version": "9.0.0", "url": ""}, + ] + } diff --git a/modoboa/core/models.py b/modoboa/core/models.py index 87f6cb6d7..a7f782f02 100644 --- a/modoboa/core/models.py +++ b/modoboa/core/models.py @@ -13,6 +13,7 @@ from django.utils import timezone from django.utils.encoding import python_2_unicode_compatible, smart_text from django.utils.translation import ugettext as _, ugettext_lazy +import jsonfield from reversion import revisions as reversion from modoboa.core.password_hashers import get_password_hasher @@ -489,3 +490,13 @@ class Log(models.Model): message = models.CharField(max_length=255) level = models.CharField(max_length=15) logger = models.CharField(max_length=30) + + +class LocalConfig(models.Model): + """Store instance configuration here.""" + + api_pk = models.PositiveIntegerField(null=True) + site = models.ForeignKey("sites.Site") + + # API results cache + api_versions = jsonfield.JSONField() diff --git a/modoboa/core/tests.py b/modoboa/core/tests.py index cc27441e4..16871539f 100644 --- a/modoboa/core/tests.py +++ b/modoboa/core/tests.py @@ -1,5 +1,8 @@ """Tests for core application.""" +import httmock + +from django.core import management from django.core.urlresolvers import reverse from django.test import override_settings @@ -8,6 +11,7 @@ from modoboa.lib import parameters from modoboa.lib.tests import ModoTestCase from . import factories +from . import mocks from . import models @@ -240,3 +244,21 @@ class APIAccessFormTestCase(ModoTestCase): self.ajax_post(url, {"enable_api_access": False}) user = models.User.objects.get(username="admin") self.assertFalse(hasattr(user, "auth_token")) + + +class APICommunicationTestCase(ModoTestCase): + """Check communication with the API.""" + + def test_management_command(self): + """Check command.""" + with httmock.HTTMock( + mocks.modo_api_instance_search, + mocks.modo_api_instance_create, + mocks.modo_api_instance_update, + mocks.modo_api_versions): + management.call_command("communicate_with_public_api") + self.assertEqual(models.LocalConfig.objects.first().api_pk, 100) + + url = reverse("core:information") + response = self.ajax_request("get", url) + self.assertIn("9.0.0", response["content"]) diff --git a/modoboa/core/utils.py b/modoboa/core/utils.py index 683cb0c9d..6eb0afa36 100644 --- a/modoboa/core/utils.py +++ b/modoboa/core/utils.py @@ -2,30 +2,39 @@ from django.utils.translation import ugettext as _ -from django.contrib.sites.shortcuts import get_current_site +from versionfield.constants import DEFAULT_NUMBER_BITS +from versionfield.version import Version from modoboa.core.extensions import exts_pool -from modoboa.lib import parameters from modoboa.lib.api_client import ModoAPIClient +from . import models + def check_for_updates(request): """Check if a new version of Modoboa is available.""" - if parameters.get_admin("CHECK_NEW_VERSIONS") == "no": - return False, [] + local_config = models.LocalConfig.objects.first() client = ModoAPIClient() - extensions = exts_pool.list_all(True) + extensions = exts_pool.list_all() extensions = [{ "label": "Modoboa", + "name": "modoboa", "description": _("The core part of Modoboa"), - "update": client.new_core_version(get_current_site(request)), - "last_version": client.latest_core_version, - "changelog_url": client.changelog_url, "version": client.local_core_version }] + extensions update_avail = False for extension in extensions: - if extension.get("update"): - update_avail = True - break + local_version = Version(extension["version"], DEFAULT_NUMBER_BITS) + pkgname = extension["name"].replace("_", "-") + for api_extension in local_config.api_versions: + if api_extension["name"] != pkgname: + continue + last_version = Version( + api_extension["version"], DEFAULT_NUMBER_BITS) + extension["last_version"] = api_extension["version"] + if last_version > local_version: + extension["update"] = True + extension["changelog_url"] = api_extension["url"] + update_avail = True + break return update_avail, extensions diff --git a/modoboa/lib/api_client.py b/modoboa/lib/api_client.py index 945cd759d..ea44bcf59 100644 --- a/modoboa/lib/api_client.py +++ b/modoboa/lib/api_client.py @@ -19,8 +19,6 @@ class ModoAPIClient(object): else: self._api_url = api_url self._local_core_version = None - self._latest_core_version = None - self.changelog_url = None def __send_request(self, url, params=None): """Send a request to the API.""" @@ -42,33 +40,35 @@ class ModoAPIClient(object): "modoboa").version return self._local_core_version - @property - def latest_core_version(self): - """Return the latest core version.""" - if self._latest_core_version is None: - return self._local_core_version - return self._latest_core_version - - def new_core_version(self, site): - """Check the latest version.""" - from versionfield.constants import DEFAULT_NUMBER_BITS - from versionfield.version import Version - - url = os.path.join(self._api_url, "current_version/") - params = { - "client_version": self.local_core_version, "client_site": site - } - resp = self.__send_request(url, params) - if resp is None: - return None - version = Version(resp["version"], DEFAULT_NUMBER_BITS) - if version <= Version(self.local_core_version, DEFAULT_NUMBER_BITS): - return False - self._latest_core_version = resp["version"] - self.changelog_url = resp["changelog_url"] - return True - def list_extensions(self): """List all official extensions.""" url = os.path.join(self._api_url, "extensions/") return self.__send_request(url) + + def register_instance(self, hostname): + """Register this instance.""" + url = "{}instances/search/?hostname={}".format( + self._api_url, hostname) + instance = self.__send_request(url) + if instance is None: + url = "{}instances/".format(self._api_url) + data = { + "hostname": hostname, "known_version": self.local_core_version} + response = requests.post(url, data=data) + if response.status_code != 201: + return None + instance = response.json() + return int(instance["pk"]) + + def update_instance(self, pk, data): + """Update instance and send stats.""" + url = "{}instances/{}/".format(self._api_url, pk) + response = requests.put(url, data=data) + if response.status_code != 200: + return False + return True + + def versions(self): + """Fetch core and extension versions.""" + url = "{}versions/".format(self._api_url) + return self.__send_request(url) diff --git a/requirements.txt b/requirements.txt index 9f02704e1..bb39671ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ django-subcommand2==0.1.1 gevent==1.1.1 ipaddress dnspython==1.12.0 +jsonfield==1.0.3 diff --git a/test-requirements.txt b/test-requirements.txt index e37c21246..a1b4ef1c5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,3 @@ factory-boy>=2.4 testfixtures==4.7.0 - +httmock==1.2.5