Use REDIS cache instead, updated frontend locals

Removed old code

.gitignore

@@ -33,3 +33,4 @@ test_project/modoboa.db
 # Misc
 .ash_history
 modoboa.kdev4
+.kateproject

doc/upgrade.rst

@@ -181,6 +181,18 @@ Then restart supervisor. ``#> supervisorctl reread && supervisorctl update`` on
       },
    }
 
+3. Add the CACHE section bellow the ``# RQ`` one:
+
+.. sourcecode:: python
+
+   # CACHE
+   CACHES = {
+      "default": {
+         "BACKEND": 'django.core.cache.backends.redis.RedisCache',
+         "LOCATION": REDIS_URL
+      }
+   }
+
 
 2.2.0
 =====

modoboa/core/api/v2/serializers.py

@@ -65,7 +65,7 @@ class CoreGlobalParametersSerializer(serializers.Serializer):
     )
     password_scheme = serializers.ChoiceField(
         choices=[("sha512crypt", "sha512crypt")],
-        default="sha512crypt"
+        required=False
     )
     rounds_number = serializers.IntegerField(default=70000)
     update_scheme = serializers.BooleanField(default=True)
@@ -169,27 +169,10 @@ class CoreGlobalParametersSerializer(serializers.Serializer):
         for field, definition in sms_backend_fields.items():
             self.fields[field] = definition["type"](
                 **definition["attrs"])
-
-        try:
-            self.saved_password_scheme = args[0].get("password_scheme")
-            self.saved_default_top_redirection = args[0].get("default_top_redirection")
-        except IndexError:
-            self.saved_password_scheme = None
-            self.saved_default_top_redirection = None
         # Choices serializer for default_top_redirection field
         self.fields["default_top_redirection"].choices = app_settings.enabled_applications()
         # Populate choices of password_scheme
-        localconfig = models.LocalConfig.objects.first()
-        self.fields["password_scheme"].choices = app_settings.get_password_scheme(localconfig)
-
-    @property
-    def data(self):
-        ret = super().data
-        if self.saved_password_scheme is not None:
-            ret["password_scheme"] = self.saved_password_scheme
-        if self.saved_default_top_redirection is not None:
-            ret["default_top_redirection"] = self.saved_default_top_redirection
-        return ret
+        self.fields["password_scheme"].choices = app_settings.get_password_scheme()
 
     def validate_ldap_user_dn_template(self, value):
         try:

modoboa/core/api/v2/tests.py

@@ -6,6 +6,7 @@ import getpass
 import oath
 from unittest import mock
 
+from django.core.cache import cache
 from django.test import override_settings
 from django.urls import reverse
 from django.utils.encoding import force_bytes
@@ -81,11 +82,20 @@ class ParametersAPITestCase(ModoAPITestCase):
     @override_settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH],
                        DOVECOT_USER=DOVECOT_USER)
     def test_update(self):
+        cache.delete("password_scheme_choice")
+        queue = get_queue("modoboa", is_async=False)
+        queue.empty()
+
         url = reverse("v2:parameter-detail", args=["core"])
         data = copy.copy(CORE_SETTINGS)
         resp = self.client.put(url, data, format="json")
         self.assertEqual(resp.status_code, 200)
 
+        get_worker("modoboa").work(burst=True)
+        self.assertIsNotNone(cache.get("password_scheme_choice"))
+        self.assertIn(('plain', 'plain (weak)'),
+                      cache.get("password_scheme_choice")
+        )
         # Modify SMS related settings
         data["sms_password_recovery"] = True
         resp = self.client.put(url, data, format="json")
@@ -125,49 +135,6 @@ class ParametersAPITestCase(ModoAPITestCase):
         resp = self.client.put(url, data, format="json")
         self.assertEqual(resp.status_code, 200)
 
-    @override_settings(DOVEADM_LOOKUP_PATH=[DOVEADM_TEST_PATH],
-                       DOVECOT_USER=DOVECOT_USER)
-    def test_cache(self):
-        """Launch the password scheme job manually and check the result."""
-        queue = get_queue("modoboa", is_async=False)
-        queue.empty()
-        job = queue.enqueue(jobs.job_retrieve_available_hashers,
-                            bypass_condition=True)
-        get_worker("modoboa").work(burst=True)
-        job_result = job.latest_result()
-        self.assertEqual(job_result.type, job_result.Type.SUCCESSFUL)
-        localconfig = models.LocalConfig.objects.first()
-        cache = localconfig.cache
-        self.assertIn(['plain', 'plain (weak)'],
-                      cache.get_value("password_scheme_choice", "core"))
-        entry = cache.get_cache_entry("password_scheme_choice", "core")
-        self.assertEqual(entry["last_update"],
-                         cache.get_last_mod("password_scheme_choice", "core"))
-        entry_guess_extension = cache.get_cache_entry("password_scheme_choice")
-        self.assertEqual(entry, entry_guess_extension)
-        cache.set_cache_entry("test_cache", "test")
-        self.assertEqual(
-            cache.get_value("test_cache", "core"),
-            "test")
-
-        # Check that the cache will be updated once a month
-        localconfig._cache["core"]["password_scheme_choice"]["value"] = "outofdate!"
-        localconfig.save()
-        job = queue.enqueue(jobs.job_retrieve_available_hashers)
-        get_worker("modoboa").work(burst=True)
-        localconfig = models.LocalConfig.objects.first()
-        value = localconfig.cache.get_value("password_scheme_choice")
-        self.assertEqual(value, "outofdate!")
-
-        old_datetime = datetime.datetime.utcnow() - datetime.timedelta(weeks=5)
-        localconfig._cache["core"]["password_scheme_choice"]["last_update"] = old_datetime
-        localconfig.save()
-        job = queue.enqueue(jobs.job_retrieve_available_hashers)
-        get_worker("modoboa").work(burst=True)
-        localconfig = models.LocalConfig.objects.first()
-        value = localconfig.cache.get_value("password_scheme_choice")
-        self.assertFalse(value == "outofdate!")
-
 
 class AccountViewSetTestCase(ModoAPITestCase):
 

modoboa/core/app_settings.py

@@ -3,9 +3,11 @@
 import collections
 
 from django import forms
+from django.core.cache import cache
 from django.conf import settings
 from django.contrib.auth import password_validation
 from django.utils.translation import gettext as _, gettext_lazy
+
 from django_rq import get_queue
 
 from modoboa.core.jobs import job_retrieve_available_hashers
@@ -42,14 +44,10 @@ def get_default_password_scheme():
     ]
 
 
-def get_password_scheme(localconfig):
-    if localconfig is None:
-        return get_default_password_scheme()
-    available_schemes = localconfig.cache.get_value(
-        "password_scheme_choice", "core", False)
-    get_queue("modoboa").enqueue(
-        job_retrieve_available_hashers)
+def get_password_scheme():
+    available_schemes = cache.get("password_scheme_choice")
     if available_schemes is None:
+        get_queue("modoboa").enqueue(job_retrieve_available_hashers)
         return get_default_password_scheme()
     return available_schemes
 
@@ -558,7 +556,7 @@ class GeneralParametersForm(param_forms.AdminParametersForm):
         self._add_visibilty_rules(
             sms_backends.get_all_backend_visibility_rules()
         )
-        self.fields["password_scheme"].choices = get_password_scheme(self.localconfig)
+        self.fields["password_scheme"].choices = get_password_scheme()
 
     def _add_dynamic_fields(self):
         new_fields = collections.OrderedDict()

modoboa/core/commands/templates/settings.py.tpl

@@ -269,6 +269,16 @@ RQ_QUEUES = {
     },
 }
 
+# CACHE
+
+CACHES = {
+    "default": {
+        "BACKEND": 'django.core.cache.backends.redis.RedisCache',
+        "LOCATION": REDIS_URL
+    }
+}
+
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 

modoboa/core/jobs.py

@@ -1,19 +1,9 @@
-from datetime import timedelta, datetime
+from rq import get_current_job
 
-from modoboa.core import models
 from modoboa.core.password_hashers import cache_available_passowrd_hasher
 
 
-def job_retrieve_available_hashers(bypass_condition=False):
-    localconfig = models.LocalConfig.objects.first()
-    cache = localconfig.cache
-    password_scheme_last_mod = cache.get_last_mod(
-        "password_scheme_choice", "core", False)
-    cur_dt = datetime.utcnow()
-    condition = (bypass_condition or
-                 password_scheme_last_mod is None or
-                 password_scheme_last_mod < cur_dt - timedelta(weeks=4)
-                 )
-    if condition:
-        cache_available_passowrd_hasher()
-
+def job_retrieve_available_hashers():
+    if get_current_job() is None:
+        return
+    cache_available_passowrd_hasher()

modoboa/core/migrations/0026_alter_localconfig__parameters_and_more.py

@@ -2,7 +2,6 @@
 
 import django.core.serializers.json
 from django.db import migrations, models
-import modoboa.lib.json_field_utils
 
 
 class Migration(migrations.Migration):
@@ -32,11 +31,6 @@ class Migration(migrations.Migration):
             name='api_versions',
             field=models.JSONField(default=dict),
         ),
-        migrations.AddField(
-            model_name='localconfig',
-            name='_cache',
-            field=models.JSONField(default=dict, encoder=modoboa.lib.json_field_utils.JSONDatetimeEncoder),
-        ),
         migrations.AlterField(
             model_name='localconfig',
             name='_parameters',

modoboa/core/models.py

@@ -22,8 +22,7 @@ from modoboa.core.password_hashers import get_password_hasher
 from modoboa.lib.exceptions import (
     BadRequest, Conflict, InternalError, PermDeniedException
 )
-from modoboa.lib.json_field_utils import JSONDatetimeEncoder
-from modoboa.parameters import tools as param_tools, cache_tools
+from modoboa.parameters import tools as param_tools
 from . import constants, signals
 
 try:
@@ -451,7 +450,6 @@ class LocalConfig(models.Model):
     api_versions = models.JSONField(default=dict, encoder=DjangoJSONEncoder)
 
     _parameters = models.JSONField(default=dict, encoder=DjangoJSONEncoder)
-    _cache = models.JSONField(default=dict, encoder=JSONDatetimeEncoder)
 
     # Dovecot LDAP update
     need_dovecot_update = models.BooleanField(default=False)
@@ -460,7 +458,6 @@ class LocalConfig(models.Model):
         """Load parameter manager."""
         super(LocalConfig, self).__init__(*args, **kwargs)
         self.parameters = param_tools.Manager("global", self._parameters)
-        self.cache = cache_tools.CacheManager(self._cache)
 
 
 class ExtensionUpdateHistory(models.Model):

modoboa/core/password_hashers/__init__.py

@@ -2,6 +2,7 @@
 Password hashers for Modoboa.
 """
 
+from django.core.cache import cache
 from django.conf import settings
 from django.utils.encoding import smart_str
 
@@ -12,8 +13,6 @@ from modoboa.core.password_hashers.advanced import (  # NOQA:F401
 from modoboa.core.password_hashers.base import (  # NOQA:F401
     CRYPTHasher, MD5Hasher, PLAINHasher, SHA256Hasher, PasswordHasher
 )
-from modoboa.core import models
-
 from modoboa.lib.sysutils import doveadm_cmd
 
 
@@ -66,8 +65,6 @@ def cache_available_passowrd_hasher():
         for hasher in PasswordHasher.get_password_hashers()
         if hasher().scheme in available_schemes
         ]
-    localconfig = models.LocalConfig.objects.first()
-    localconfig.cache.set_cache_entry("password_scheme_choice",
-                                      password_scheme_choice,
-                                      "core")
-    localconfig.save(update_fields=["_cache"])
+    cache.set("password_scheme_choice",
+              password_scheme_choice,
+              2592000)

modoboa/lib/json_field_utils.py

@@ -1,32 +0,0 @@
-"""Utils regarding time management."""
-
-import json
-
-from django.core.serializers.json import DjangoJSONEncoder
-
-from datetime import datetime, timezone
-
-
-def datetime_to_json_tz_unaware(datetime_object=None):
-    """Take a datetime object and convert it to YYYY-MM-DDT-HH:MM:SS."""
-    if datetime_object is None:
-        datetime_object = datetime.utcnow()
-    datetime_object = datetime_object.astimezone(timezone.utc)
-    return datetime_object.strftime("%Y-%m-%dT%H:%M:%S")
-
-
-def json_to_datetime(datetime_string):
-    """Take a string in the YYYY-MM-DDT-HH:MM:SS form
-    and convert it to a datetime object."""
-    if isinstance(datetime_string, datetime):
-        return datetime_string
-    return datetime.strptime(datetime_string,
-                             "%Y-%m-%dT%H:%M:%S")
-
-
-class JSONDatetimeEncoder(DjangoJSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, datetime):
-            return datetime_to_json_tz_unaware(obj)
-        else:
-            return super().default(obj)

modoboa/parameters/cache_tools.py

@@ -1,61 +0,0 @@
-"""Cache management."""
-
-from datetime import datetime
-
-from modoboa.lib.json_field_utils import json_to_datetime
-from modoboa.lib.sysutils import guess_extension_name
-
-
-class CacheManager:
-    """A manager for cache."""
-
-    def __init__(self, cache):
-        """Constructor."""
-        self._cache = cache
-
-    def get_value(self, cache_label, app=None, raise_exception=True):
-        """Return the value associated to the specified parameter."""
-        if app is None:
-            app = guess_extension_name()
-        cache_entry = self.get_cache_entry(
-            cache_label, app, raise_exception)
-        if cache_entry is None:
-            return None
-        return cache_entry.get("value")
-
-    def get_last_mod(self, cache_label, app=None, raise_exception=True):
-        """Returns the last modification date for the specified parameter."""
-        if app is None:
-            app = guess_extension_name()
-        cache_entry = self.get_cache_entry(
-            cache_label, app, raise_exception)
-        if cache_entry is None:
-            return None
-        return cache_entry.get("last_update")
-
-    def get_cache_entry(self, cache_label, app=None, raise_exception=True):
-        """Return the entry associated to the specified parameter."""
-        if app is None:
-            app = guess_extension_name()
-        # Compat. with the old way...
-        parameter = cache_label.lower()
-        try:
-            entry = self._cache[app][parameter]
-            entry["last_update"] = json_to_datetime(entry["last_update"])
-        except KeyError:
-            if raise_exception:
-                raise
-            return None
-
-        return entry
-
-    def set_cache_entry(self, parameter, value, app=None):
-        """Set parameter for the given app."""
-        if app is None:
-            app = guess_extension_name()
-        if self._cache.get(app) is None:
-            self._cache[app] = {}
-        self._cache[app][parameter] = {
-            "value": value,
-            "last_update": datetime.utcnow()
-            }

test_project/test_project/settings.py

@@ -250,6 +250,15 @@ RQ_QUEUES = {
     },
 }
 
+# CACHE
+
+CACHES = {
+    "default": {
+        "BACKEND": 'django.core.cache.backends.redis.RedisCache',
+        "LOCATION": REDIS_URL
+    }
+}
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators