Merge pull request #893 from tonioo/feature/anonymous_stats_890

Feature: Anonymous stats

modoboa/core/__init__.py

@@ -48,10 +48,8 @@ PERMISSIONS = {
 
 
 @events.observe("TopNotifications")
-def check_for_new_version(request, include_all):
-    """
-    Check if a new version of Modoboa is available.
-    """
+def check_for_new_versions(request, include_all):
+    """Check if new versions are available."""
     from modoboa.core.utils import check_for_updates
 
     if not request.user.is_superuser:

modoboa/core/app_settings.py

@@ -220,6 +220,30 @@ class GeneralParametersForm(parameters.AdminParametersForm):
         )
     )
 
+    api_sep = SeparatorField(label=ugettext_lazy("Public API"))
+
+    enable_api_communication = YesNoField(
+        label=ugettext_lazy("Enable communication"),
+        initial="yes",
+        help_text=ugettext_lazy(
+            "Enable communication with Modoboa public API")
+    )
+
+    check_new_versions = YesNoField(
+        label=ugettext_lazy("Check new versions"),
+        initial="yes",
+        help_text=ugettext_lazy(
+            "Automatically checks if a newer version is available")
+    )
+
+    send_statistics = YesNoField(
+        label=ugettext_lazy("Send statistics"),
+        initial="yes",
+        help_text=ugettext_lazy(
+            "Send statistics to Modoboa public API "
+            "(counters and used extensions)")
+    )
+
     sep3 = SeparatorField(label=ugettext_lazy("Miscellaneous"))
 
     top_notifications_check_interval = forms.IntegerField(
@@ -231,13 +255,6 @@ class GeneralParametersForm(parameters.AdminParametersForm):
         widget=forms.TextInput(attrs={"class": "form-control"})
     )
 
-    check_new_versions = YesNoField(
-        label=ugettext_lazy("Check new versions"),
-        initial="yes",
-        help_text=ugettext_lazy(
-            "Automatically checks if a newer version is available")
-    )
-
     log_maximum_age = forms.IntegerField(
         label=ugettext_lazy("Maximum log record age"),
         initial=365,
@@ -279,6 +296,8 @@ class GeneralParametersForm(parameters.AdminParametersForm):
         "ldap_admin_groups": "authentication_type=ldap",
         "ldap_group_type": "authentication_type=ldap",
         "ldap_groups_search_base": "authentication_type=ldap",
+        "check_new_versions": "enable_api_communication=yes",
+        "send_statistics": "enable_api_communication=yes",
     }
 
     def __init__(self, *args, **kwargs):

modoboa/core/apps.py

@@ -1,6 +1,7 @@
 """Core config for admin."""
 
 from django.apps import AppConfig
+from django.db.models import signals
 
 
 BASE_EVENTS = [
@@ -66,3 +67,5 @@ class CoreConfig(AppConfig):
         load_core_settings()
 
         from . import handlers
+
+        signals.post_migrate.connect(handlers.create_local_config, sender=self)

modoboa/core/extensions.py

@@ -3,11 +3,6 @@
 from django.conf import settings
 from django.conf.urls import include
 
-from versionfield.constants import DEFAULT_NUMBER_BITS
-from versionfield.version import Version
-
-from modoboa.lib.api_client import ModoAPIClient
-
 
 class ModoExtension(object):
 
@@ -115,32 +110,14 @@ class ExtensionsPool(object):
                 result += [ext_urls]
         return result
 
-    def list_all(self, check_new_versions=False):
+    def list_all(self):
         """List all defined extensions."""
         result = []
-        new_extensions = {}
-        if check_new_versions:
-            new_extensions = ModoAPIClient().list_extensions()
-            if new_extensions:
-                new_extensions = dict(
-                    (ext["name"], ext["version"]) for ext in new_extensions
-                )
-            else:
-                new_extensions = {}
-
         for extname, extdef in self.extensions.iteritems():
             if not extdef["show"]:
                 continue
             infos = self.get_extension_infos(extname)
             infos["id"] = extname
-            local_version = Version(infos["version"], DEFAULT_NUMBER_BITS)
-            pkgname = infos["name"].replace("_", "-")
-            if pkgname in new_extensions:
-                infos["last_version"] = new_extensions[pkgname]
-                last_version = Version(
-                    new_extensions[pkgname], DEFAULT_NUMBER_BITS)
-                if last_version > local_version:
-                    infos["update"] = True
             result += [infos]
         return sorted(result, key=lambda i: i["name"])
 

modoboa/core/handlers.py

@@ -6,8 +6,12 @@ from django.db.models.signals import post_delete
 from django.dispatch import receiver
 from django.utils.translation import ugettext as _
 
+from django.contrib.sites import models as sites_models
+
 from reversion import revisions as reversion
 
+from . import models
+
 
 @receiver(reversion.post_revision_commit)
 def post_revision_commit(sender, **kwargs):
@@ -63,3 +67,11 @@ def log_object_removal(sender, instance, **kwargs):
         "user": get_request().user.username
     }
     logger.critical(msg)
+
+
+def create_local_config(sender, **kwargs):
+    """Create local config if needed."""
+    if models.LocalConfig.objects.using(kwargs["using"]).exists():
+        return
+    models.LocalConfig.objects.create(
+        site=sites_models.Site.objects.get_current())

modoboa/core/management/commands/communicate_with_public_api.py

@@ -0,0 +1,52 @@
+"""Management command to communicate with public API."""
+
+from django.core.management.base import BaseCommand, CommandError
+
+from modoboa.admin import models as admin_models
+from modoboa.core.extensions import exts_pool
+from modoboa.lib import api_client, parameters
+
+from ... import models
+
+
+class Command(BaseCommand):
+    """Command class."""
+
+    help = "Communicate with Modoboa public API."
+
+    def handle(self, *args, **options):
+        """Command entry point."""
+        if parameters.get_admin("ENABLE_API_COMMUNICATION") == "no":
+            return
+        self.client = api_client.ModoAPIClient()
+        local_config = (
+            models.LocalConfig.objects.select_related("site").first())
+        if not local_config.api_pk:
+            pk = self.client.register_instance(local_config.site.domain)
+            if pk is None:
+                raise CommandError("Instance registration failed.")
+            local_config.api_pk = pk
+
+        if parameters.get_admin("CHECK_NEW_VERSIONS") == "yes":
+            versions = self.client.versions()
+            if versions is None:
+                raise CommandError("Failed to retrieve versions from the API.")
+            local_config.api_versions = versions
+
+        local_config.save()
+
+        if parameters.get_admin("SEND_STATISTICS") == "no":
+            return
+        extensions = [ext["name"] for ext in exts_pool.list_all()]
+        data = {
+            "hostname": local_config.site.domain,
+            "known_version": self.client.local_core_version,
+            "domain_counter": admin_models.Domain.objects.count(),
+            "domain_alias_counter": admin_models.DomainAlias.objects.count(),
+            "mailbox_counter": admin_models.Mailbox.objects.count(),
+            "alias_counter": (
+                admin_models.Alias.objects.filter(internal=False).count()),
+            "extensions": extensions
+        }
+        if not self.client.update_instance(local_config.api_pk, data):
+            raise CommandError("Failed to update instance.")

modoboa/core/migrations/0008_localconfig.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.5 on 2016-06-15 18:12
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sites', '0002_alter_domain_unique'),
+        ('core', '0007_auto_20151116_1349'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LocalConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('api_pk', models.PositiveIntegerField(null=True)),
+                ('api_versions', jsonfield.fields.JSONField()),
+                ('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sites.Site')),
+            ],
+        ),
+    ]

modoboa/core/mocks.py

@@ -0,0 +1,41 @@
+"""Mocks used for testing."""
+
+import httmock
+
+
+# Modoboa API mocks
+
+@httmock.urlmatch(
+    netloc=r"api\.modoboa\.org$", path=r"^/1/instances/search/", method="post")
+def modo_api_instance_search(url, request):
+    """Return empty response."""
+    return {"status_code": 404}
+
+
+@httmock.urlmatch(
+    netloc=r"api\.modoboa\.org$", path=r"^/1/instances/", method="post")
+def modo_api_instance_create(url, request):
+    """Simulate successful creation."""
+    return {
+        "status_code": 201,
+        "content": {"pk": 100}
+    }
+
+
+@httmock.urlmatch(
+    netloc=r"api\.modoboa\.org$", path=r"^/1/instances/.+/", method="put")
+def modo_api_instance_update(url, request):
+    """Simulate successful update."""
+    return {"status_code": 200}
+
+
+@httmock.urlmatch(
+    netloc=r"api\.modoboa\.org$", path=r"^/1/versions/", method="get")
+def modo_api_versions(url, request):
+    """Simulate versions check."""
+    return {
+        "status_code": 200,
+        "content": [
+            {"name": "modoboa", "version": "9.0.0", "url": ""},
+        ]
+    }

modoboa/core/models.py

@@ -13,6 +13,7 @@ from django.utils import timezone
 from django.utils.encoding import python_2_unicode_compatible, smart_text
 from django.utils.translation import ugettext as _, ugettext_lazy
 
+import jsonfield
 from reversion import revisions as reversion
 
 from modoboa.core.password_hashers import get_password_hasher
@@ -489,3 +490,13 @@ class Log(models.Model):
     message = models.CharField(max_length=255)
     level = models.CharField(max_length=15)
     logger = models.CharField(max_length=30)
+
+
+class LocalConfig(models.Model):
+    """Store instance configuration here."""
+
+    api_pk = models.PositiveIntegerField(null=True)
+    site = models.ForeignKey("sites.Site")
+
+    # API results cache
+    api_versions = jsonfield.JSONField()

modoboa/core/tests.py

@@ -1,5 +1,8 @@
 """Tests for core application."""
 
+import httmock
+
+from django.core import management
 from django.core.urlresolvers import reverse
 from django.test import override_settings
 
@@ -8,6 +11,7 @@ from modoboa.lib import parameters
 from modoboa.lib.tests import ModoTestCase
 
 from . import factories
+from . import mocks
 from . import models
 
 
@@ -240,3 +244,21 @@ class APIAccessFormTestCase(ModoTestCase):
         self.ajax_post(url, {"enable_api_access": False})
         user = models.User.objects.get(username="admin")
         self.assertFalse(hasattr(user, "auth_token"))
+
+
+class APICommunicationTestCase(ModoTestCase):
+    """Check communication with the API."""
+
+    def test_management_command(self):
+        """Check command."""
+        with httmock.HTTMock(
+                mocks.modo_api_instance_search,
+                mocks.modo_api_instance_create,
+                mocks.modo_api_instance_update,
+                mocks.modo_api_versions):
+            management.call_command("communicate_with_public_api")
+        self.assertEqual(models.LocalConfig.objects.first().api_pk, 100)
+
+        url = reverse("core:information")
+        response = self.ajax_request("get", url)
+        self.assertIn("9.0.0", response["content"])

modoboa/core/utils.py

@@ -2,30 +2,39 @@
 
 from django.utils.translation import ugettext as _
 
-from django.contrib.sites.shortcuts import get_current_site
+from versionfield.constants import DEFAULT_NUMBER_BITS
+from versionfield.version import Version
 
 from modoboa.core.extensions import exts_pool
-from modoboa.lib import parameters
 from modoboa.lib.api_client import ModoAPIClient
 
+from . import models
+
 
 def check_for_updates(request):
     """Check if a new version of Modoboa is available."""
-    if parameters.get_admin("CHECK_NEW_VERSIONS") == "no":
-        return False, []
+    local_config = models.LocalConfig.objects.first()
     client = ModoAPIClient()
-    extensions = exts_pool.list_all(True)
+    extensions = exts_pool.list_all()
     extensions = [{
         "label": "Modoboa",
+        "name": "modoboa",
         "description": _("The core part of Modoboa"),
-        "update": client.new_core_version(get_current_site(request)),
-        "last_version": client.latest_core_version,
-        "changelog_url": client.changelog_url,
         "version": client.local_core_version
     }] + extensions
     update_avail = False
     for extension in extensions:
-        if extension.get("update"):
-            update_avail = True
-            break
+        local_version = Version(extension["version"], DEFAULT_NUMBER_BITS)
+        pkgname = extension["name"].replace("_", "-")
+        for api_extension in local_config.api_versions:
+            if api_extension["name"] != pkgname:
+                continue
+            last_version = Version(
+                api_extension["version"], DEFAULT_NUMBER_BITS)
+            extension["last_version"] = api_extension["version"]
+            if last_version > local_version:
+                extension["update"] = True
+                extension["changelog_url"] = api_extension["url"]
+                update_avail = True
+                break
     return update_avail, extensions

modoboa/lib/api_client.py

@@ -19,8 +19,6 @@ class ModoAPIClient(object):
         else:
             self._api_url = api_url
         self._local_core_version = None
-        self._latest_core_version = None
-        self.changelog_url = None
 
     def __send_request(self, url, params=None):
         """Send a request to the API."""
@@ -42,33 +40,35 @@ class ModoAPIClient(object):
                 "modoboa").version
         return self._local_core_version
 
-    @property
-    def latest_core_version(self):
-        """Return the latest core version."""
-        if self._latest_core_version is None:
-            return self._local_core_version
-        return self._latest_core_version
-
-    def new_core_version(self, site):
-        """Check the latest version."""
-        from versionfield.constants import DEFAULT_NUMBER_BITS
-        from versionfield.version import Version
-
-        url = os.path.join(self._api_url, "current_version/")
-        params = {
-            "client_version": self.local_core_version, "client_site": site
-        }
-        resp = self.__send_request(url, params)
-        if resp is None:
-            return None
-        version = Version(resp["version"], DEFAULT_NUMBER_BITS)
-        if version <= Version(self.local_core_version, DEFAULT_NUMBER_BITS):
-            return False
-        self._latest_core_version = resp["version"]
-        self.changelog_url = resp["changelog_url"]
-        return True
-
     def list_extensions(self):
         """List all official extensions."""
         url = os.path.join(self._api_url, "extensions/")
         return self.__send_request(url)
+
+    def register_instance(self, hostname):
+        """Register this instance."""
+        url = "{}instances/search/?hostname={}".format(
+            self._api_url, hostname)
+        instance = self.__send_request(url)
+        if instance is None:
+            url = "{}instances/".format(self._api_url)
+            data = {
+                "hostname": hostname, "known_version": self.local_core_version}
+            response = requests.post(url, data=data)
+            if response.status_code != 201:
+                return None
+            instance = response.json()
+        return int(instance["pk"])
+
+    def update_instance(self, pk, data):
+        """Update instance and send stats."""
+        url = "{}instances/{}/".format(self._api_url, pk)
+        response = requests.put(url, data=data)
+        if response.status_code != 200:
+            return False
+        return True
+
+    def versions(self):
+        """Fetch core and extension versions."""
+        url = "{}versions/".format(self._api_url)
+        return self.__send_request(url)

requirements.txt

@@ -18,3 +18,4 @@ django-subcommand2==0.1.1
 gevent==1.1.1
 ipaddress
 dnspython==1.12.0
+jsonfield==1.0.3

test-requirements.txt

@@ -1,3 +1,3 @@
 factory-boy>=2.4
 testfixtures==4.7.0
-
+httmock==1.2.5