Merge pull request #1976 from modoboa/feature/policy_daemon

New policy daemon.

.travis.yml

@@ -27,10 +27,10 @@ cache: pip
 services:
   - postgres
   - mysql
+  - redis-server
 
 before_install:
   - pip install codecov
-  - pip install codacy-coverage
 
 install:
   - pip install -r requirements.txt
@@ -49,12 +49,11 @@ before_script:
 script:
   - python ./tests.py
   - cd test_project
-  - coverage run --source ../modoboa manage.py test modoboa.core modoboa.lib modoboa.admin modoboa.limits modoboa.transport modoboa.relaydomains modoboa.dnstools modoboa.ldapsync
+  - coverage run manage.py test modoboa.core modoboa.lib modoboa.admin modoboa.limits modoboa.transport modoboa.relaydomains modoboa.dnstools modoboa.ldapsync modoboa.policyd
 
 after_success:
-  - coverage xml
+  - coverage combine
   - codecov
-  - python-codacy-coverage -r coverage.xml
 
 before_deploy:
   - cd ../modoboa

doc/manual_installation/modoboa.rst

@@ -204,5 +204,38 @@ following content inside::
   # Generate DKIM keys (they will belong to the user running this job)
   *       *       *       *       *       root    umask 077 && $PYTHON $INSTANCE/manage.py modo manage_dkim_keys
 
+.. _policy_daemon:
+
+Policy daemon
+-------------
+
+Modoboa comes with a built-in `Policy Daemon for Postfix <http://www.postfix.org/SMTPD_POLICY_README.html>`_. Current features are:
+
+* Define daily sending limits for domains and/or accounts
+
+A `redis server <https://redis.io/>`_ is required to run this new daemon.
+
+You can launch it manually using the following command:
+
+.. sourcecode:: bash
+
+   (env)> python manage.py policy_daemon
+
+But we recommend an automatic start using ``systemd`` or
+``supervisor``. Here is a configuration example for ``supervisor``:
+
+.. sourcecode:: ini
+
+   [program:policyd]
+   autostart=true
+   autorestart=true
+   command=/srv/modoboa/env/bin/python /srv/modoboa/instance/manage.py policy_daemon
+   directory=/srv/modoboa
+   redirect_stderr=true
+   user=modoboa
+   numprocs=1
+
+It will listen by default on ``127.0.0.1`` and port ``9999``. The
+policy daemon won't do anything unless you tell :ref:`postfix <policyd_config>` to use it.
 
 Now you can continue to the :ref:`webserver` section.

doc/manual_installation/postfix.rst

@@ -65,3 +65,23 @@ Use the following configuration in the :file:`/etc/postfix/main.cf` file
 Replace ``<driver>`` by the name of the database you use.
 
 Restart Postfix.
+
+.. _policyd_config:
+
+Policy daemon
+-------------
+
+If you want to enable the built-in policy daemon, add the following
+content to the :file:`/etc/postfix/main.cf` file::
+
+    smtpd_recipient_restrictions =
+        # ...
+        check_policy_service inet:127.0.0.1:9999
+        # ...
+
+And reload postfix.
+
+.. note::
+
+   The ``check_policy_service`` line must be placed before the
+   ``permit_mynetworks`` one, otherwise the daemon won't be called.

doc/upgrade.rst

@@ -91,6 +91,20 @@ Rebuild Virtual Environment
 Specific instructions
 *********************
 
+1.16.0
+======
+
+A new :ref:`policy daemon <policy_daemon>` has been added. Make sure
+to have a Redis instance running on your server, then add the following
+settings to your ``settings.py`` file:
+
+.. sourcecode:: python
+
+REDIS_HOST = '<IP or hostname here>'
+REDIS_PORT = 6379
+REDIS_QUOTA_DB = 0
+REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
+
 1.15.0
 ======
 

modoboa/admin/forms/account.py

@@ -238,6 +238,13 @@ class AccountFormMail(forms.Form, DynamicForm):
         widget=forms.widgets.TextInput(attrs={"class": "form-control"})
     )
     quota_act = forms.BooleanField(required=False)
+    message_limit = forms.IntegerField(
+        label=ugettext_lazy("Message sending limit"),
+        required=False,
+        min_value=0,
+        help_text=ugettext_lazy(
+            "Number of messages this mailbox can send per day")
+    )
     aliases = lib_fields.UTF8AndEmptyUserEmailField(
         label=ugettext_lazy("Alias(es)"),
         required=False,
@@ -261,7 +268,7 @@ class AccountFormMail(forms.Form, DynamicForm):
     def __init__(self, user, *args, **kwargs):
         self.mb = kwargs.pop("instance", None)
         self.user = user
-        super(AccountFormMail, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.field_widths = {
             "quota": 3
         }
@@ -282,6 +289,8 @@ class AccountFormMail(forms.Form, DynamicForm):
             self.fields["quota_act"].initial = self.mb.use_domain_quota
             if not self.mb.use_domain_quota and self.mb.quota:
                 self.fields["quota"].initial = self.mb.quota
+            if self.mb.message_limit:
+                self.fields["message_limit"].initial = self.mb.message_limit
             self.fields["create_alias_with_old_address"].initial = (
                 param_tools.get_global_parameter(
                     "create_alias_on_mbox_rename")
@@ -367,7 +376,9 @@ class AccountFormMail(forms.Form, DynamicForm):
             self.__class__, context=user, klass=models.Mailbox)
         self.mb = models.Mailbox(
             address=self.locpart, domain=self.domain, user=account,
-            use_domain_quota=self.cleaned_data["quota_act"])
+            use_domain_quota=self.cleaned_data["quota_act"],
+            message_limit=self.cleaned_data.get("message_limit")
+        )
         override_rules = (
             user.is_superuser or
             user.has_perm("admin.add_domain") and

modoboa/admin/forms/domain.py

@@ -57,8 +57,8 @@ class DomainFormGeneral(forms.ModelForm, DynamicForm):
     class Meta:
         model = Domain
         fields = (
-            "name", "type", "quota", "default_mailbox_quota", "aliases",
-            "enabled", "enable_dns_checks", "enable_dkim",
+            "name", "type", "quota", "default_mailbox_quota", "message_limit",
+            "aliases", "enabled", "enable_dns_checks", "enable_dkim",
             "dkim_key_selector", "dkim_key_length"
         )
 

modoboa/admin/migrations/0016_auto_20200602_1201.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.12 on 2020-06-02 10:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('admin', '0015_rename_view_permissions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='domain',
+            name='message_limit',
+            field=models.PositiveIntegerField(blank=True, help_text='Number of messages this domain can send per day', null=True, verbose_name='Message sending limit'),
+        ),
+        migrations.AddField(
+            model_name='mailbox',
+            name='message_limit',
+            field=models.PositiveIntegerField(blank=True, help_text='Number of messages this mailbox can send per day', null=True, verbose_name='Message sending limit'),
+        ),
+    ]

modoboa/admin/models/base.py

@@ -43,13 +43,13 @@ class AdminObject(models.Model):
 
     def __init__(self, *args, **kwargs):
         """Custom constructor."""
-        super(AdminObject, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._loaded_values = {}
 
     @classmethod
     def from_db(cls, db, field_names, values):
         """Store loaded values."""
-        instance = super(AdminObject, cls).from_db(db, field_names, values)
+        instance = super().from_db(db, field_names, values)
         instance._loaded_values = dict(zip(field_names, values))
         return instance
 

modoboa/admin/models/domain.py

@@ -6,9 +6,7 @@ from reversion import revisions as reversion
 
 from django.db import models
 from django.utils import timezone
-from django.utils.encoding import (
-    force_text, python_2_unicode_compatible, smart_text
-)
+from django.utils.encoding import force_text, smart_text
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext as _, ugettext_lazy
 
@@ -16,12 +14,13 @@ from modoboa.core import signals as core_signals
 from modoboa.core.models import User
 from modoboa.lib.exceptions import BadRequest, Conflict
 from modoboa.parameters import tools as param_tools
+
 from .. import constants
 from .base import AdminObject
+from . import mixins
 
 
-@python_2_unicode_compatible
-class Domain(AdminObject):
+class Domain(mixins.MessageLimitMixin, AdminObject):
     """Mail domain."""
 
     name = models.CharField(ugettext_lazy("name"), max_length=100, unique=True,
@@ -33,6 +32,13 @@ class Domain(AdminObject):
         verbose_name=ugettext_lazy("Default mailbox quota"),
         default=0
     )
+    message_limit = models.PositiveIntegerField(
+        ugettext_lazy("Message sending limit"),
+        null=True, blank=True,
+        help_text=ugettext_lazy(
+            "Number of messages this domain can send per day"
+        )
+    )
     enabled = models.BooleanField(
         ugettext_lazy("enabled"),
         help_text=ugettext_lazy("Check to activate this domain"),
@@ -213,6 +219,11 @@ class Domain(AdminObject):
             return 0
         return int(self.used_quota / float(self.quota) * 100)
 
+    @property
+    def message_counter_key(self):
+        """Return the key used to store messages count."""
+        return self.name
+
     @property
     def bind_format_dkim_public_key(self):
         """Return DKIM public key using Bind format."""

modoboa/admin/models/mailbox.py

@@ -20,6 +20,7 @@ from modoboa.parameters import tools as param_tools
 
 from .base import AdminObject
 from .domain import Domain
+from . import mixins
 
 
 class QuotaManager(models.Manager):
@@ -87,8 +88,7 @@ class MailboxManager(Manager):
         return self.get_queryset().select_related().filter(qf)
 
 
-@python_2_unicode_compatible
-class Mailbox(AdminObject):
+class Mailbox(mixins.MessageLimitMixin, AdminObject):
     """User mailbox."""
 
     address = models.CharField(
@@ -98,6 +98,11 @@ class Mailbox(AdminObject):
     )
     quota = models.PositiveIntegerField(default=0)
     use_domain_quota = models.BooleanField(default=False)
+    message_limit = models.PositiveIntegerField(
+        ugettext_lazy("Message sending limit"), null=True, blank=True,
+        help_text=ugettext_lazy(
+            "Number of messages this mailbox can send per day")
+    )
     domain = models.ForeignKey(Domain, on_delete=models.CASCADE)
     user = models.OneToOneField(User, on_delete=models.CASCADE)
 
@@ -107,9 +112,10 @@ class Mailbox(AdminObject):
         app_label = "admin"
 
     def __init__(self, *args, **kwargs):
-        super(Mailbox, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.__mail_home = None
         self.old_full_address = self.full_address
+        self.old_message_limit = self.message_limit
 
     def __str__(self):
         return smart_text(self.full_address)
@@ -190,6 +196,11 @@ class Mailbox(AdminObject):
         """Set the ``Quota`` for this mailbox."""
         self._quota_value = instance
 
+    @property
+    def message_counter_key(self):
+        """Return the key used to store messages count."""
+        return self.full_address
+
     def rename_dir(self, old_mail_home):
         """Rename local directory if needed."""
         hm = param_tools.get_global_parameter(
@@ -319,6 +330,7 @@ class Mailbox(AdminObject):
                 not user.userobjectlimit_set.get(name="quota").max_value
             )
             self.set_quota(values["quota"], override_rules)
+        self.message_limit = values.get("message_limit")
         if newaddress:
             self.rename(local_part, domain)
         self.save()

modoboa/admin/models/mixins.py

@@ -0,0 +1,27 @@
+"""Admin model mixins."""
+
+from modoboa.policyd.utils import get_message_counter
+
+
+class MessageLimitMixin:
+    """A mixin to add message limit support."""
+
+    @property
+    def message_counter_key(self):
+        raise NotImplementedError
+
+    @property
+    def sent_messages(self):
+        """Return number of sent messages for the current day."""
+        if self.message_limit is None:
+            return None
+        return (
+            self.message_limit - get_message_counter(self.message_counter_key)
+        )
+
+    @property
+    def sent_messages_in_percent(self):
+        """Return number of sent messages as a percentage."""
+        if self.message_limit is None:
+            return None
+        return int(self.sent_messages / float(self.message_limit) * 100)

modoboa/admin/serializers.py

@@ -45,7 +45,8 @@ class DomainSerializer(serializers.ModelSerializer):
             "pk", "name", "quota", "default_mailbox_quota", "enabled", "type",
             "enable_dkim", "dkim_key_selector", "dkim_key_length",
             "dkim_public_key", "dkim_private_key_path",
-            "mailbox_count", "mbalias_count", "domainalias_count"
+            "mailbox_count", "mbalias_count", "domainalias_count",
+            "message_limit"
         )
         read_only_fields = (
             "pk", "mailbox_count", "mbalias_count", "domainalias_count"
@@ -137,7 +138,9 @@ class MailboxSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = models.Mailbox
-        fields = ("pk", "full_address", "use_domain_quota", "quota", )
+        fields = (
+            "pk", "full_address", "use_domain_quota", "quota", "message_limit"
+        )
 
     def validate_full_address(self, value):
         """Lower case address."""

modoboa/admin/templates/admin/account_detail.html

@@ -45,6 +45,12 @@
               <th>{% trans "Quota" %}</th>
               <td>{% include "common/progressbar.html" with id='quotabar' value=object.mailbox.get_quota_in_percent %}</td>
             </tr>
+            {% if object.mailbox.message_limit is not None %}
+              <tr>
+                <th>{% trans "Message sending limit" %}</a></th>
+                <td>{% include "common/progressbar.html" with id='messagelimitbar' value=object.mailbox.sent_messages_in_percent %}</td>
+              </tr>
+            {% endif %}
           {% endif %}
         </table>
       </div>
@@ -59,7 +65,7 @@
             <tr><td>{{ domain }}</td></tr>
           {% empty %}
             <tr><td>{% trans "No domain defined." %}</td></tr>
-          {% endfor %}          
+          {% endfor %}
         </table>
       </div>
       {% endif %}

modoboa/admin/templates/admin/domain_detail.html

@@ -64,6 +64,20 @@
           <th>{% trans "Default mailbox quota" %}</a></th>
           <td>{{ domain.default_mailbox_quota }} {% trans "MB" %}</td>
         </tr>
+        {% if domain.message_limit != None %}
+          <tr>
+            <th>{% trans "Message sending limit" %}</a></th>
+            <td>{{ domain.message_limit }} {% trans "per day" %}</td>
+          </tr>
+          <tr>
+            <th>{% trans "Usage" %}</a></th>
+            <td>
+              <div class="progress">
+                <div class="{% progress_color domain.sent_messages_in_percent %}" style="width: {{ domain.sent_messages_in_percent }}%" title="{{ domain.sent_messages_in_percent }}% ({{ domain.sent_messages }}) {% trans 'used' context 'message count' %}"></div>
+              </div>
+            </td>
+          </tr>
+        {% endif %}
       </table>
     </div>
 
@@ -141,7 +155,7 @@
         </tbody>
       </table>
     </div>
-    
+
     {% for template in templates.right %}
       {% include template %}
     {% endfor %}

modoboa/admin/templates/admin/domain_general_form.html

@@ -3,6 +3,7 @@
 {% render_field form.type %}
 {% render_field form.quota %}
 {% render_field form.default_mailbox_quota %}
+{% render_field form.message_limit %}
 {% render_fields_group form "aliases" %}
 {% render_field form.enabled %}
 {% render_field form.enable_dns_checks %}

modoboa/admin/templates/admin/mailform.html

@@ -5,5 +5,6 @@
   {% render_field form.create_alias_with_old_address %}
 {% endif %}
 {% include "common/generic_field.html" with field=form.quota help_display_mode="tooltip" label_width="col-sm-4" activator=True activator_value=form.quota_act.value deactivate_if_empty=False activator_label=_("Use domain default value") activator_size="col-sm-5" %}
+{% render_field form.message_limit %}
 {% render_fields_group form "aliases" %}
 {% render_fields_group form "senderaddress" %}

modoboa/core/commands/templates/settings.py.tpl

@@ -194,6 +194,13 @@ MODOBOA_API_URL = 'https://api.modoboa.org/1/'
 
 DISABLE_DASHBOARD_EXTERNAL_QUERIES = False
 
+# REDIS
+
+REDIS_HOST = 'localhost'
+REDIS_PORT = 6379
+REDIS_QUOTA_DB = 0
+REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 

modoboa/policyd/__init__.py

@@ -0,0 +1 @@
+default_app_config = "modoboa.policyd.apps.PolicydConfig"

modoboa/policyd/apps.py

@@ -0,0 +1,13 @@
+"""AppConfig for policyd."""
+
+from django.apps import AppConfig
+
+
+class PolicydConfig(AppConfig):
+    """App configuration."""
+
+    name = "modoboa.policyd"
+    verbose_name = "Modoboa policy daemon"
+
+    def ready(self):
+        from . import handlers  # NOQA:F401

modoboa/policyd/constants.py

@@ -0,0 +1,3 @@
+"""App. related constants."""
+
+REDIS_HASHNAME = "messages_count"

modoboa/policyd/core.py

@@ -0,0 +1,201 @@
+"""Core components of the policy daemon."""
+
+import asyncio
+import concurrent.futures
+from email.message import EmailMessage
+import logging
+
+import aiosmtplib
+from dateutil.relativedelta import relativedelta
+import aioredis
+
+from django.conf import settings
+from django.template.loader import render_to_string
+from django.utils import timezone
+from django.utils import translation
+from django.utils.translation import ugettext as _, ugettext_lazy
+
+from modoboa.admin import models as admin_models
+from modoboa.core import models as core_models
+from modoboa.lib.email_utils import split_mailbox
+
+from . import constants
+
+logger = logging.getLogger("modoboa.policyd")
+
+
+SUCCESS_ACTION = b"dunno"
+FAILURE_ACTION = b"defer_if_permit Daily limit reached, retry later"
+
+
+async def wait_for(dt):
+    """sleep until the specified datetime."""
+    one_day = 86400
+    while True:
+        now = timezone.now()
+        remaining = (dt - now).total_seconds()
+        if remaining < one_day:
+            break
+        # asyncio.sleep doesn't like long sleeps, so don't sleep more
+        # than a day at a time
+        await asyncio.sleep(one_day)
+    await asyncio.sleep(remaining)
+
+
+async def run_at(dt, coro, *args):
+    """Run coroutine at given datetime."""
+    await wait_for(dt)
+    return await coro(*args)
+
+
+def get_local_config():
+    """Return local configuration."""
+    return core_models.LocalConfig.objects.first()
+
+
+def get_notification_recipients():
+    """Return superadmins with a mailbox."""
+    return (
+        core_models.User.objects
+        .filter(is_superuser=True, mailbox__isnull=False)
+    )
+
+
+async def notify_limit_reached(ltype, name):
+    """Send a notification to super admins about item."""
+    ltype_translations = {
+        "account": ugettext_lazy("account"),
+        "domain": ugettext_lazy("domain")
+    }
+    # We're going to execute sync code so we need an executor
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=3)
+    loop = asyncio.get_event_loop()
+    futures = [
+        loop.run_in_executor(executor, get_local_config),
+        loop.run_in_executor(executor, get_notification_recipients)
+    ]
+    lc, recipients = await asyncio.gather(*futures)
+    sender = lc.parameters.get_value("sender_address", app="core")
+    for recipient in recipients:
+        with translation.override(recipient.language):
+            content = render_to_string(
+                "policyd/notifications/limit_reached.html", {
+                    "ltype": ltype_translations[ltype], "name": name
+                })
+            subject = _("[modoboa] Sending limit reached")
+        msg = EmailMessage()
+        msg["From"] = sender
+        msg["To"] = recipient.email
+        msg["Subject"] = subject
+        msg.set_content(content)
+        await aiosmtplib.send(msg)
+
+
+async def decrement_limit(rclient, ltype, name):
+    """Decrement the given limit by one."""
+    new_counter = await rclient.hincrby(constants.REDIS_HASHNAME, name, -1)
+    if new_counter <= 0:
+        logger.info("Limit reached for {} {}".format(ltype, name))
+        asyncio.ensure_future(notify_limit_reached(ltype, name))
+
+
+async def apply_policies(attributes):
+    """Apply defined policies to received request."""
+    sasl_username = attributes.get("sasl_username")
+    if not sasl_username:
+        return SUCCESS_ACTION
+    rclient = await aioredis.create_redis_pool(settings.REDIS_URL)
+    decr_domain = False
+    decr_user = False
+    localpart, domain = split_mailbox(sasl_username)
+    if await rclient.hexists(constants.REDIS_HASHNAME, domain):
+        counter = await rclient.hget(constants.REDIS_HASHNAME, domain)
+        logger.info("Domain {} current counter: {}".format(domain, counter))
+        if int(counter) <= 0:
+            return FAILURE_ACTION
+        decr_domain = True
+    if await rclient.hexists(constants.REDIS_HASHNAME, sasl_username):
+        counter = await rclient.hget(constants.REDIS_HASHNAME, sasl_username)
+        logger.info("Account {} current counter: {}".format(
+            sasl_username, counter))
+        if int(counter) <= 0:
+            return FAILURE_ACTION
+        decr_user = True
+    if decr_domain:
+        await decrement_limit(rclient, "domain", domain)
+    if decr_user:
+        await decrement_limit(rclient, "account", sasl_username)
+    return SUCCESS_ACTION
+
+
+async def handle_connection(reader, writer):
+    """Coroutine to handle a new connection to the server."""
+    action = SUCCESS_ACTION
+    try:
+        data = await reader.readuntil(b"\n\n")
+    except asyncio.IncompleteReadError:
+        pass
+    else:
+        attributes = {}
+        for line in data.decode().split("\n"):
+            if not line:
+                continue
+            try:
+                name, value = line.split("=")
+            except ValueError:
+                continue
+            attributes[name] = value
+        state = attributes.get("protocol_state")
+        if state == "RCPT":
+            action = await apply_policies(attributes)
+    writer.write(b"action=" + action + b"\n\n")
+    await writer.drain()
+    writer.close()
+
+
+def get_next_execution_dt():
+    """Return next execution date and time."""
+    return (timezone.now() + relativedelta(days=1)).replace(
+        hour=0, minute=0, second=0)
+
+
+def get_domains_to_reset():
+    """Return a list of domain to reset."""
+    return admin_models.Domain.objects.filter(message_limit__isnull=False)
+
+
+def get_mailboxes_to_reset():
+    """Return a list of mailboxes to reset."""
+    return (
+        admin_models.Mailbox.objects.filter(message_limit__isnull=False)
+        .select_related("domain")
+    )
+
+
+async def reset_counters():
+    """Reset all counters."""
+    rclient = await aioredis.create_redis_pool(settings.REDIS_URL)
+    logger.info("Resetting all counters")
+    # We're going to execute sync code so we need an executor
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=3)
+    loop = asyncio.get_event_loop()
+    futures = [
+        loop.run_in_executor(executor, get_domains_to_reset),
+        loop.run_in_executor(executor, get_mailboxes_to_reset)
+    ]
+    domains, mboxes = await asyncio.gather(*futures)
+    for domain in domains:
+        rclient.hset(
+            constants.REDIS_HASHNAME, domain.name, domain.message_limit)
+    for mb in mboxes:
+        rclient.hset(
+            constants.REDIS_HASHNAME, mb.full_address, mb.message_limit)
+    # reschedule
+    asyncio.ensure_future(run_at(get_next_execution_dt(), reset_counters))
+
+
+def start_reset_counters_coro():
+    """Start coroutine."""
+    first_time = (timezone.now() + relativedelta(days=1)).replace(
+        hour=0, minute=0, second=0)
+    asyncio.ensure_future(run_at(first_time, reset_counters))

modoboa/policyd/handlers.py

@@ -0,0 +1,45 @@
+"""App related signal handlers."""
+
+import redis
+
+from django.conf import settings
+from django.db.models import signals
+from django.dispatch import receiver
+
+from modoboa.admin import models as admin_models
+
+from . import constants
+
+
+def set_message_limit(instance, key):
+    """Store message limit in Redis."""
+    old_message_limit = instance._loaded_values.get("message_limit")
+    if old_message_limit == instance.message_limit:
+        return
+    rclient = redis.Redis(
+        host=settings.REDIS_HOST,
+        port=settings.REDIS_PORT,
+        db=settings.REDIS_QUOTA_DB
+    )
+    if instance.message_limit is None:
+        # delete existing key
+        if rclient.hexists(constants.REDIS_HASHNAME, key):
+            rclient.hdel(constants.REDIS_HASHNAME, key)
+        return
+    if old_message_limit is not None:
+        diff = instance.message_limit - old_message_limit
+    else:
+        diff = instance.message_limit
+    rclient.hincrby(constants.REDIS_HASHNAME, key, diff)
+
+
+@receiver(signals.post_save, sender=admin_models.Domain)
+def set_domain_message_limit(sender, instance, created, **kwargs):
+    """Store domain message limit in Redis."""
+    set_message_limit(instance, instance.name)
+
+
+@receiver(signals.post_save, sender=admin_models.Mailbox)
+def set_mailbox_message_limit(sender, instance, created, **kwargs):
+    """Store mailbox message limit in Redis."""
+    set_message_limit(instance, instance.full_address)

modoboa/policyd/locale/en/LC_MESSAGES/django.po

@@ -0,0 +1,36 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-06-11 20:26+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: core.py:74
+msgid "account"
+msgstr ""
+
+#: core.py:75
+msgid "domain"
+msgstr ""
+
+#: core.py:92
+msgid "[modoboa] Sending limit reached"
+msgstr ""
+
+#: templates/policyd/notifications/limit_reached.html:3
+#, python-format
+msgid "Daily limit of the following %(ltype)s has been reached: %(name)s."
+msgstr ""

modoboa/policyd/locale/fr/LC_MESSAGES/django.po

@@ -0,0 +1,35 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2020-06-11 20:26+0200\n"
+"PO-Revision-Date: 2020-06-11 20:27+0200\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=(n > 1);\n"
+
+#: core.py:74
+msgid "account"
+msgstr "compte"
+
+#: core.py:75
+msgid "domain"
+msgstr "domaine"
+
+#: core.py:92
+msgid "[modoboa] Sending limit reached"
+msgstr "[modoboa] Limite d'envoi atteinte"
+
+#: templates/policyd/notifications/limit_reached.html:3
+#, python-format
+msgid "Daily limit of the following %(ltype)s has been reached: %(name)s."
+msgstr "La limite journalière du %(ltype)s suivant a été atteinte : %(name)s."

modoboa/policyd/management/commands/policy_daemon.py

@@ -0,0 +1,63 @@
+"""Policy daemon management command."""
+
+import asyncio
+from contextlib import suppress
+import functools
+import logging
+import signal
+
+from django.core.management.base import BaseCommand
+
+from ... import core
+
+logger = logging.getLogger("modoboa.policyd")
+
+
+def ask_exit(signame, loop):
+    """Stop event loop."""
+    loop.stop()
+
+
+class Command(BaseCommand):
+    """Management command for policy daemon."""
+
+    help = "Launch Modoboa policy daemon"
+
+    def add_arguments(self, parser):
+        """Add command line arguments."""
+        parser.add_argument("--host", type=str, default="127.0.0.1")
+        parser.add_argument("--port", type=int, default=9999)
+
+    def handle(self, *args, **options):
+        """Entry point."""
+        loop = asyncio.get_event_loop()
+        coro = asyncio.start_server(
+            core.handle_connection, options["host"], options["port"], loop=loop
+        )
+        server = loop.run_until_complete(coro)
+
+        # Schedule reset task
+        core.start_reset_counters_coro()
+
+        for signame in {'SIGINT', 'SIGTERM'}:
+            loop.add_signal_handler(
+                getattr(signal, signame),
+                functools.partial(ask_exit, signame, loop)
+            )
+
+        logger.info("Serving on {}".format(server.sockets[0].getsockname()))
+
+        loop.run_forever()
+
+        logger.info("Stopping policy daemon...")
+        # Close the server
+        server.close()
+        loop.run_until_complete(server.wait_closed())
+        # Cancel pending tasks
+        for task in asyncio.Task.all_tasks():
+            task.cancel()
+            # Await task to execute it's cancellation. Cancelled task
+            # raises asyncio.CancelledError that we can suppress
+            with suppress(asyncio.CancelledError):
+                loop.run_until_complete(task)
+        loop.close()

modoboa/policyd/templates/policyd/notifications/limit_reached.html

@@ -0,0 +1,5 @@
+{% load i18n %}
+
+{% blocktrans trimmed with ltype=ltype name=name %}
+  Daily limit of the following {{ ltype }} has been reached: {{ name }}.
+{% endblocktrans %}

modoboa/policyd/tests.py

@@ -0,0 +1,264 @@
+"""Policy daemon related tests."""
+
+import asyncio
+# from mock import patch
+from asynctest import patch
+from multiprocessing import Process
+import socket
+import time
+
+import redis
+
+from django.conf import settings
+from django.core.management import call_command
+from django.urls import reverse
+
+from modoboa.admin import factories as admin_factories
+from modoboa.admin import models as admin_models
+from modoboa.core import models as core_models
+from modoboa.lib.tests import ModoTestCase
+from modoboa.policyd import core as policyd_core
+
+from . import constants
+
+
+def start_policy_daemon():
+    call_command("policy_daemon")
+
+
+class RedisTestCaseMixin:
+    """Mixin to provide a redis client."""
+
+    def setUp(self):
+        super().setUp()
+        self.rclient = redis.Redis(
+            host=settings.REDIS_HOST,
+            port=settings.REDIS_PORT,
+            db=settings.REDIS_QUOTA_DB
+        )
+        self.rclient.set_response_callback("HGET", int)
+        self.rclient.delete(constants.REDIS_HASHNAME)
+
+
+class PolicyDaemonTestCase(RedisTestCaseMixin, ModoTestCase):
+    """Test cases for policy daemon.
+
+    A redis instance is required to run those tests.
+    """
+
+    @classmethod
+    def setUpTestData(cls):  # NOQA:N802
+        """Create test data."""
+        super().setUpTestData()
+        admin_factories.populate_database()
+        # FIXME: all database modifications must be made before the
+        # daemon is started otherwise they won't be seen by the other
+        # process. I think it's just a sqlite issue...
+        cls.admin = core_models.User.objects.get(username="user@test2.com")
+        cls.admin.role = "SuperAdmins"
+
+    def setUp(self):
+        super().setUp()
+        patcher = patch("aiosmtplib.send")
+        self.send_mock = patcher.start()
+        self.process = Process(target=start_policy_daemon)
+        self.process.daemon = True
+        self.process.start()
+        # Wait a bit for the daemon to start
+        self.process.join(0.1)
+
+    def set_domain_limit(self, name, value):
+        """Set daily limit for domain."""
+        domain = admin_models.Domain.objects.get(name=name)
+        domain.message_limit = value
+        domain.save()
+        self.assertEqual(
+            self.rclient.hget(constants.REDIS_HASHNAME, domain.name),
+            domain.message_limit
+        )
+        return domain
+
+    def set_account_limit(self, name, value):
+        """Set daily limit for account."""
+        account = core_models.User.objects.get(username=name)
+        mb = account.mailbox
+        mb.message_limit = value
+        mb.save()
+        self.assertEqual(
+            self.rclient.hget(constants.REDIS_HASHNAME, account.email),
+            mb.message_limit
+        )
+        return account
+
+    def tearDown(self):
+        self.process.terminate()
+        self.process.join()
+
+    def connect_to_daemon(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.connect(("127.0.0.1", 9999))
+        return s
+
+    def test_daemon_starts(self):
+        s = self.connect_to_daemon()
+        s.send(b"protocol_state=RCPT\n\n")
+        res = s.recv(1024)
+        self.assertEqual(res, b"action=dunno\n\n")
+        s.close()
+
+    def test_domain_limit(self):
+        domain = self.set_domain_limit("test.com", 2)
+        s = self.connect_to_daemon()
+
+        # This one will be accepted
+        s.send(b"""protocol_state=RCPT
+sasl_username=user@test.com
+
+""")
+        res = s.recv(1024)
+        self.assertEqual(res, b"action=dunno\n\n")
+        self.assertEqual(
+            self.rclient.hget(constants.REDIS_HASHNAME, domain.name),
+            domain.message_limit - 1
+        )
+        s.close()
+
+        # This one will be accepted and will trigger a notification
+        # but I can't check it yet
+        s = self.connect_to_daemon()
+        s.send(b"""protocol_state=RCPT
+sasl_username=user@test.com
+
+""")
+        time.sleep(0.1)
+        res = s.recv(1024)
+        self.assertEqual(res, b"action=dunno\n\n")
+        s.close()
+
+        # This one will be denied
+        s = self.connect_to_daemon()
+        s.send(b"""protocol_state=RCPT
+sasl_username=user@test.com
+
+""")
+        res = s.recv(1024)
+        self.assertEqual(
+            res, b"action=defer_if_permit Daily limit reached, retry later\n\n"
+        )
+
+        s.close()
+
+    def test_account_limit(self):
+        account = self.set_account_limit("user@test.com", 2)
+        s = self.connect_to_daemon()
+
+        # This one will be accepted
+        s.send(b"""protocol_state=RCPT
+sasl_username=user@test.com
+
+""")
+        res = s.recv(1024)
+        self.assertEqual(res, b"action=dunno\n\n")
+        self.assertEqual(
+            self.rclient.hget(constants.REDIS_HASHNAME, account.email),
+            account.mailbox.message_limit - 1
+        )
+        s.close()
+
+        # This one will be accepted and will trigger a notification
+        # but I can't check it yet
+        s = self.connect_to_daemon()
+        s.send(b"""protocol_state=RCPT
+sasl_username=user@test.com
+
+""")
+        time.sleep(0.1)
+        res = s.recv(1024)
+        self.assertEqual(res, b"action=dunno\n\n")
+        s.close()
+
+        # This one will be denied
+        s = self.connect_to_daemon()
+        s.send(b"""protocol_state=RCPT
+sasl_username=user@test.com
+
+""")
+        res = s.recv(1024)
+        self.assertEqual(
+            res, b"action=defer_if_permit Daily limit reached, retry later\n\n"
+        )
+
+        s.close()
+
+    def test_reset_counters(self):
+        domain = self.set_domain_limit("test.com", 20)
+        account = self.set_account_limit("user@test.com", 10)
+        self.rclient.hset(constants.REDIS_HASHNAME, domain.name, 10)
+        self.rclient.hset(constants.REDIS_HASHNAME, account.email, 0)
+        event_loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(event_loop)
+
+        async def run_test():
+            await policyd_core.reset_counters()
+
+        # Run the async test
+        coro = asyncio.coroutine(run_test)
+        event_loop.run_until_complete(coro())
+        event_loop.close()
+
+        self.assertEqual(
+            self.rclient.hget(constants.REDIS_HASHNAME, domain.name), 20)
+        self.assertEqual(
+            self.rclient.hget(constants.REDIS_HASHNAME, account.email), 10)
+
+
+class ModelsTestCase(RedisTestCaseMixin, ModoTestCase):
+    """Admin models test cases."""
+
+    @classmethod
+    def setUpTestData(cls):  # NOQA:N802
+        """Create test data."""
+        super().setUpTestData()
+        admin_factories.populate_database()
+
+    def test_domain_detail_view(self):
+        domain = admin_models.Domain.objects.get(name="test.com")
+        domain.message_limit = 10
+        domain.save()
+        url = reverse("admin:domain_detail", args=[domain.pk])
+        response = self.client.get(url)
+        self.assertContains(response, "Message sending limit")
+
+    def test_account_detail_view(self):
+        account = core_models.User.objects.get(username="user@test.com")
+        mb = account.mailbox
+        mb.message_limit = 10
+        mb.save()
+
+        url = reverse("admin:account_detail", args=[account.pk])
+        response = self.client.get(url)
+        self.assertContains(response, "Message sending limit")
+
+    def test_domain_signal_handler(self):
+        domain = admin_models.Domain.objects.get(name="test.com")
+        domain.message_limit = 10
+        domain.save()
+        self.assertEqual(
+            self.rclient.hget(constants.REDIS_HASHNAME, domain.name),
+            domain.message_limit
+        )
+
+        # Force constructor call to fill _loaded_values
+        domain = admin_models.Domain.objects.get(name="test.com")
+        domain.message_limit = 50
+        domain.save()
+        self.assertEqual(
+            self.rclient.hget(constants.REDIS_HASHNAME, domain.name),
+            domain.message_limit
+        )
+
+        domain.message_limit = None
+        domain.save()
+        self.assertFalse(
+            self.rclient.hexists(constants.REDIS_HASHNAME, domain.name)
+        )

modoboa/policyd/utils.py

@@ -0,0 +1,24 @@
+"""Tooling."""
+
+import redis
+
+from django.conf import settings
+
+from . import constants
+
+
+def get_redis_connection():
+    """Return a client connection to Redis server."""
+    rclient = redis.Redis(
+        host=settings.REDIS_HOST,
+        port=settings.REDIS_PORT,
+        db=settings.REDIS_QUOTA_DB
+    )
+    rclient.set_response_callback("HGET", int)
+    return rclient
+
+
+def get_message_counter(key):
+    """Return current counter for given key."""
+    rclient = get_redis_connection()
+    return rclient.hget(constants.REDIS_HASHNAME, key)

modoboa/test_settings.py

@@ -54,7 +54,8 @@ else:
             "PORT": "",
             "ATOMIC_REQUESTS": True,
             "OPTIONS": {
-                    "client_encoding": "UTF8",
+                "client_encoding": "UTF8",
+                "sslmode": "disable"
             },
             "TEST": {
                 "CHARSET": "UTF8",

requirements.txt

@@ -27,3 +27,7 @@ lxml
 chardet
 ovh
 oath
+redis
+
+aiosmtplib
+aioredis

test-requirements.txt

@@ -3,3 +3,4 @@ factory-boy>=2.4
 httmock==1.3.0
 testfixtures==6.10.3
 tox
+asynctest

test_project/.coveragerc

@@ -0,0 +1,5 @@
+[run]
+branch = True
+concurrency = multiprocessing, greenlet
+parallel = True
+source = ../modoboa

test_project/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '3'
+
+services:
+  redis:
+    image: redis:5-alpine
+    ports:
+      - 6379:6379

test_project/test_project/settings.py

@@ -78,6 +78,7 @@ MODOBOA_APPS = (
     'modoboa.limits',
     'modoboa.parameters',
     'modoboa.dnstools',
+    'modoboa.policyd',
     # Modoboa extensions here.
 )
 
@@ -186,6 +187,13 @@ REST_FRAMEWORK = {
 
 MODOBOA_API_URL = 'https://api.modoboa.org/1/'
 
+# REDIS
+
+REDIS_HOST = 'localhost'
+REDIS_PORT = 6379
+REDIS_QUOTA_DB = 0
+REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators