From ac00c8e1959098cb1cc960c7ace8426881d5b82f Mon Sep 17 00:00:00 2001 From: Antoine Nguyen Date: Tue, 21 Oct 2014 21:40:52 +0200 Subject: [PATCH] Initial work about SA training from quarantine content. see #622 --- modoboa/extensions/amavis/app_settings.py | 42 +++++++++- .../extensions/amavis/general_callbacks.py | 27 ++++-- modoboa/extensions/amavis/lib.py | 84 +++++++++++++++---- .../amavis/static/amavis/js/quarantine.js | 54 ++++++++++-- .../templates/amavis/main_action_bar.html | 9 ++ .../amavis/templatetags/amavis_tags.py | 20 ++++- modoboa/extensions/amavis/urls.py | 4 +- modoboa/extensions/amavis/views.py | 69 ++++++++++++++- modoboa/lib/sysutils.py | 16 ++-- modoboa/static/js/global.js | 4 +- 10 files changed, 288 insertions(+), 41 deletions(-) diff --git a/modoboa/extensions/amavis/app_settings.py b/modoboa/extensions/amavis/app_settings.py index 9d933476c..de5c43b77 100644 --- a/modoboa/extensions/amavis/app_settings.py +++ b/modoboa/extensions/amavis/app_settings.py @@ -17,7 +17,7 @@ class ParametersForm(AdminParametersForm): ) sep1 = SeparatorField(label=_("Messages releasing")) - + released_msgs_cleanup = YesNoField( label=_("Remove released messages"), initial="no", @@ -49,7 +49,7 @@ class ParametersForm(AdminParametersForm): initial="/var/amavis/amavisd.sock", help_text=_("Path to the PDP server socket (if unix mode)") ) - + check_requests_interval = forms.IntegerField( label=_("Check requests interval"), initial=30, @@ -74,10 +74,46 @@ class ParametersForm(AdminParametersForm): help_text=_("The e-mail address used to send notitications") ) + lsep = SeparatorField(label=_("Learning from messages")) + + amavis_is_local = YesNoField( + label=_("Is amavis local?"), + initial="yes", + help_text=_("Tell if amavis is running on the same server than modoboa") + ) + + default_user = forms.CharField( + label=_("Default user"), + initial="amavis", + help_text=_( + "Name of the user used to store default bayesian filter data" + ) + ) + + per_user_bayesian_filter = YesNoField( + label=_("Enable per-user bayesian filters?"), + initial="no", + help_text=_("Allow users to personally train spamassassin") + ) + + spamd_address = forms.CharField( + label=_("Spamd address"), + initial="127.0.0.1", + help_text=_("The IP address where spamd can be reached") + ) + + spamd_port = forms.IntegerField( + label=_("Spamd port"), + initial=783, + help_text=_("The TCP port spamd is listening on") + ) + visibility_rules = { "am_pdp_host": "am_pdp_mode=inet", "am_pdp_port": "am_pdp_mode=inet", - "am_pdp_socket": "am_pdp_mode=unix" + "am_pdp_socket": "am_pdp_mode=unix", + "spamd_address": "amavis_is_local=no", + "spamd_port": "amavis_is_local=no", } diff --git a/modoboa/extensions/amavis/general_callbacks.py b/modoboa/extensions/amavis/general_callbacks.py index ce2f76e91..1ef740e64 100644 --- a/modoboa/extensions/amavis/general_callbacks.py +++ b/modoboa/extensions/amavis/general_callbacks.py @@ -23,22 +23,27 @@ def menu(target, user): @events.observe("DomainCreated") def on_domain_created(user, domain): - create_user_and_policy(domain.name) + create_user_and_policy("@{0}".format(domain.name)) @events.observe("DomainModified") def on_domain_modified(domain): - update_user_and_policy(domain.oldname, domain.name) + update_user_and_policy( + "@{0}".format(domain.oldname), + "@{0}".format(domain.name) + ) @events.observe("DomainDeleted") def on_domain_deleted(domain): - delete_user_and_policy(domain.name) + delete_user_and_policy("@{0}".format(domain.name)) @events.observe("DomainAliasCreated") def on_domain_alias_created(user, domainalias): - create_user_and_use_policy(domainalias.name, domainalias.target.name) + create_user_and_use_policy( + "@{0}".format(domainalias.name), domainalias.target.name + ) @events.observe("DomainAliasDeleted") @@ -46,7 +51,19 @@ def on_domain_alias_deleted(domainaliases): if isinstance(domainaliases, DomainAlias): domainaliases = [domainaliases] for domainalias in domainaliases: - delete_user(domainalias.name) + delete_user("@{0}".format(domainalias.name)) + + +@events.observe("MailboxCreated") +def on_mailbox_created(user, mailbox): + """Create amavis records for the new mailbox.""" + create_user_and_policy(mailbox.full_address) + + +@events.observe("MailboxDeleted") +def on_mailbox_deleted(mailbox): + """Delete amavis records related to this mailbox.""" + delete_user_and_policy(mailbox.full_address) @events.observe("GetStaticContent") diff --git a/modoboa/extensions/amavis/lib.py b/modoboa/extensions/amavis/lib.py index 20f02a6c3..118687f4f 100644 --- a/modoboa/extensions/amavis/lib.py +++ b/modoboa/extensions/amavis/lib.py @@ -1,13 +1,17 @@ # coding: utf-8 -import socket -import re -import struct -import string + from functools import wraps -from django.core.urlresolvers import reverse +import re +import socket +import string +import struct + from django.utils.translation import ugettext as _ +from django.core.urlresolvers import reverse + from modoboa.lib import parameters from modoboa.lib.exceptions import InternalError +from modoboa.lib.sysutils import exec_cmd from modoboa.lib.webutils import NavigationParameters from modoboa.extensions.amavis.models import Users, Policy @@ -80,6 +84,58 @@ recipient=%s return False +class SpamassassinClient(object): + + """A stupid spamassassin client.""" + + def __init__(self, username=None): + """Constructor.""" + self._amavis_is_local = parameters.get_admin("AMAVIS_IS_LOCAL") + if username is None: + self._username = parameters.get_admin("DEFAULT_USER") + else: + self._username = username + self.error = None + if self._amavis_is_local == "yes": + self._learn_cmd = "sa-learn --{0} --no-sync -u {1}" + self._learn_cmd_kwargs = {} + # self._learn_cmd_kwargs = { + # "sudo_user": parameters.get_admin("AMAVIS_SYSTEM_USER") + # } + self._expected_exit_codes = [0] + else: + self._learn_cmd = "spamc -d {0} -p {1}".format( + parameters.get_admin("SPAMD_ADDRESS"), + parameters.get_admin("SPAMD_PORT") + ) + self._learn_cmd += " -L {0} -u {1}" + self._learn_cmd_kwargs = {} + self._expected_exit_codes = [5, 6] + + def _learn(self, msg, mtype): + """Internal method to call the learning command.""" + cmd = self._learn_cmd.format(mtype, self._username) + code, output = exec_cmd(cmd, pinput=msg, **self._learn_cmd_kwargs) + if code in self._expected_exit_codes: + return True + self.error = output + return False + + def learn_spam(self, msg): + """Learn new spam.""" + return self._learn(msg, "spam") + + def learn_ham(self, msg): + """Learn new ham.""" + return self._learn(msg, "ham") + + def done(self): + """Call this method at the end of the processing.""" + if self._amavis_is_local: + exec_cmd("sa-learn -u {0} --sync".format(self._username), + **self._learn_cmd_kwargs) + + class QuarantineNavigationParameters(NavigationParameters): """ Specific NavigationParameters subclass for the quarantine. @@ -114,7 +170,7 @@ class QuarantineNavigationParameters(NavigationParameters): return url -def create_user_and_policy(name): +def create_user_and_policy(name, priority=7): """Create records. Create two records (a user and a policy) using :keyword:`name` as @@ -124,12 +180,11 @@ def create_user_and_policy(name): """ policy = Policy.objects.create(policy_name=name[:32]) Users.objects.create( - email="@%s" % name, fullname=name, - priority=7, policy=policy + email=name, fullname=name, priority=priority, policy=policy ) -def create_user_and_use_policy(name, policy_name): +def create_user_and_use_policy(name, policy_name, priority=7): """Create a *users* record and use an existing policy. :param str name: user record name @@ -137,8 +192,7 @@ def create_user_and_use_policy(name, policy_name): """ policy = Policy.objects.get(policy_name=policy_name[:32]) Users.objects.create( - email="@%s" % name, fullname=name, - priority=7, policy=policy + email=name, fullname=name, priority=priority, policy=policy ) @@ -150,8 +204,8 @@ def update_user_and_policy(oldname, newname): """ if oldname == newname: return - u = Users.objects.get(email="@%s" % oldname) - u.email = "@%s" % newname + u = Users.objects.get(email=oldname) + u.email = newname u.fullname = newname u.policy.policy_name = newname[:32] u.policy.save() @@ -164,7 +218,7 @@ def delete_user_and_policy(name): :param str name: identifier """ try: - u = Users.objects.get(email="@%s" % name) + u = Users.objects.get(email=name) except Users.DoesNotExist: return u.policy.delete() @@ -177,6 +231,6 @@ def delete_user(name): :param str name: user record name """ try: - Users.objects.get(email="@%s" % name).delete() + Users.objects.get(email=name).delete() except Users.DoesNotExist: pass diff --git a/modoboa/extensions/amavis/static/amavis/js/quarantine.js b/modoboa/extensions/amavis/static/amavis/js/quarantine.js index cb8c88db2..905016fc9 100644 --- a/modoboa/extensions/amavis/static/amavis/js/quarantine.js +++ b/modoboa/extensions/amavis/static/amavis/js/quarantine.js @@ -62,10 +62,18 @@ Quarantine.prototype = { $.proxy(this.release_selection, this)); $(document).on("click", "a[name=delete-multi]", $.proxy(this.delete_selection, this)); + $(document).on("click", "a[name=mark-as-spam-multi]", + $.proxy(this.mark_selection_as_spam, this)); + $(document).on("click", "a[name=mark-as-ham-multi]", + $.proxy(this.mark_selection_as_ham, this)); $(document).on("click", "a[name=viewrequests]", $.proxy(this.view_requests, this)); $(document).on("click", "a[name=release]", $.proxy(this.release, this)); $(document).on("click", "a[name=delete]", $.proxy(this.delete, this)); + $(document).on("click", "a[name=mark-as-spam]", + $.proxy(this.mark_as_spam, this)); + $(document).on("click", "a[name=mark-as-ham]", + $.proxy(this.mark_as_ham, this)); $(document).on("click", "a[name=headers]", $.proxy(this.headers, this)); $(document).on("click", "td[name=type] span", $.proxy(this.filter_by_type, this)); $(document).on("click", "#filters button", $.proxy(this.filter_by_type, this)); @@ -191,6 +199,26 @@ Quarantine.prototype = { this._send_selection(e, "delete", gettext("Delete this selection?")); }, + /** + * Mark the current selection as spam. + * + * @param {Object} e - event object + */ + mark_selection_as_spam: function(e) { + this._send_selection( + e, "mark_as_spam", gettext("Mark this selection as spam?")); + }, + + /** + * Mark the current selection as ham. + * + * @param {Object} e - event object + */ + mark_selection_as_ham: function(e) { + this._send_selection( + e, "mark_as_ham", gettext("Mark this selection as non-spam?")); + }, + _send_action: function(e, message) { var $link = get_target(e, "a"); @@ -206,7 +234,7 @@ Quarantine.prototype = { }).done($.proxy(this.action_cb, this)); }, - release: function(e) { + release : function(e) { this._send_action(e, gettext("Release this message?")); }, @@ -214,6 +242,24 @@ Quarantine.prototype = { this._send_action(e, gettext("Delete this message?")); }, + /** + * Mark the current message as spam. + * + * @param {Object} e - event object + */ + mark_as_spam: function(e) { + this._send_action(e, gettext("Mark as spam?")); + }, + + /** + * Mark the current message as ham. + * + * @param {Object} e - event object + */ + mark_as_ham: function(e) { + this._send_action(e, gettext("Mark as non-spam?")); + }, + show_rawheaders: function($mailcontent) { $mailcontent.find("#rawheaders").removeClass("hidden"); $mailcontent.find("#emailheaders").addClass("hidden"); @@ -261,15 +307,13 @@ Quarantine.prototype = { }, activate_buttons: function($tr) { - $("a[name=release-multi]").removeClass('disabled'); - $("a[name=delete-multi]").removeClass('disabled'); + $("a[name*=-multi]").removeClass('disabled'); $("#selectall").prop('checked', true); }, deactivate_buttons: function($tr) { if (!this.htmltable || !this.htmltable.current_selection().length) { - $("a[name=release-multi]").addClass('disabled'); - $("a[name=delete-multi]").addClass('disabled'); + $("a[name*=-multi]").addClass('disabled'); $("#selectall").prop('checked', false); } }, diff --git a/modoboa/extensions/amavis/templates/amavis/main_action_bar.html b/modoboa/extensions/amavis/templates/amavis/main_action_bar.html index 709460220..3b3684f03 100644 --- a/modoboa/extensions/amavis/templates/amavis/main_action_bar.html +++ b/modoboa/extensions/amavis/templates/amavis/main_action_bar.html @@ -23,6 +23,15 @@ {% trans "Delete" %} +
+ + +
{% include "common/email_searchbar.html" with extraopts=extraopts %}
diff --git a/modoboa/extensions/amavis/templatetags/amavis_tags.py b/modoboa/extensions/amavis/templatetags/amavis_tags.py index d5b0ec620..187cbe997 100644 --- a/modoboa/extensions/amavis/templatetags/amavis_tags.py +++ b/modoboa/extensions/amavis/templatetags/amavis_tags.py @@ -2,8 +2,6 @@ from django import template from django.template.loader import render_to_string from django.utils.translation import ugettext as _ from django.core.urlresolvers import reverse -from django.conf import settings -from modoboa.lib.webutils import _render_to_string register = template.Library() @@ -23,8 +21,24 @@ def viewm_menu(mail_id, rcpt): "url": reverse('modoboa.extensions.amavis.views.delete', args=[mail_id]) + ("?rcpt=%s" % rcpt if rcpt else ""), "label": _("Delete")}, + {"name": "process", + "img": "icon-cog", + "menu": [ + {"name": "mark-as-spam", + "label": _("Mark as spam"), + "url": reverse( + "modoboa.extensions.amavis.views.mark_as_spam", args=[mail_id] + ) + ("?rcpt=%s" % rcpt if rcpt else "")}, + {"name": "mark-as-ham", + "label": _("Mark as non-spam"), + "url": reverse( + "modoboa.extensions.amavis.views.mark_as_ham", args=[mail_id] + ) + ("?rcpt=%s" % rcpt if rcpt else "")} + ]}, {"name": "headers", - "url": reverse('modoboa.extensions.amavis.views.viewheaders', args=[mail_id]), + "url": reverse( + 'modoboa.extensions.amavis.views.viewheaders', args=[mail_id] + ), "label": _("View full headers")}, ] diff --git a/modoboa/extensions/amavis/urls.py b/modoboa/extensions/amavis/urls.py index 6c79062ee..806f62f38 100644 --- a/modoboa/extensions/amavis/urls.py +++ b/modoboa/extensions/amavis/urls.py @@ -9,6 +9,8 @@ urlpatterns = patterns( (r'^nbrequests/$', 'nbrequests'), (r'^delete/(?P[\w\-\+]+)/$', 'delete'), (r'^release/(?P[\w\-\+]+)/$', 'release'), + (r'^markspam/(?P[\w\-\+]+)/$', 'mark_as_spam'), + (r'^markham/(?P[\w\-\+]+)/$', 'mark_as_ham'), (r'^(?P[\w\-\+]+)/$', 'viewmail'), (r'^(?P[\w\-\+]+)/headers/$', 'viewheaders'), - ) +) diff --git a/modoboa/extensions/amavis/views.py b/modoboa/extensions/amavis/views.py index 76fc990ec..5cd084222 100644 --- a/modoboa/extensions/amavis/views.py +++ b/modoboa/extensions/amavis/views.py @@ -17,7 +17,10 @@ from modoboa.extensions.admin.models import Mailbox, Domain from modoboa.extensions.amavis.templatetags.amavis_tags import ( quar_menu, viewm_menu ) -from .lib import selfservice, AMrelease, QuarantineNavigationParameters +from .lib import ( + selfservice, AMrelease, QuarantineNavigationParameters, + SpamassassinClient +) from .models import Msgrcpt from .sql_connector import get_connector from .sql_listing import SQLlisting, SQLemail @@ -254,11 +257,67 @@ def release(request, mail_id): ) +def mark_messages(request, selection, mtype): + """Mark a selection of messages as spam. + + :param str selection: message unique identifier + :param str mtype: type of marking (spam or ham) + """ + selection = check_mail_id(request, selection) + connector = get_connector() + per_user_bayesian_filter = parameters.get_admin("PER_USER_BAYESIAN_FILTER") + if per_user_bayesian_filter == "yes" \ + and request.user.group == "SimpleUsers": + username = request.user.email + else: + username = None + saclient = SpamassassinClient(username) + for item in selection: + rcpt, mail_id = item.split() + content = "".join( + [msg.mail_text for msg in connector.get_mail_content(mail_id)] + ) + result = saclient.learn_spam(content) if mtype == "spam" \ + else saclient.learn_ham(content) + if not result: + break + if saclient.error is None: + saclient.done() + message = ungettext("%(count)d message processed successfully", + "%(count)d messages processed successfully", + len(selection)) % {"count": len(selection)} + else: + message = saclient.error + return ajax_response( + request, "ko" if saclient.error else "ok", respmsg=message, + url=QuarantineNavigationParameters(request).back_to_listing() + ) + + +@login_required +def mark_as_spam(request, mail_id): + """Mark a single message as spam.""" + return mark_messages(request, mail_id, "spam") + + +@login_required +def mark_as_ham(request, mail_id): + """Mark a single message as ham.""" + return mark_messages(request, mail_id, "ham") + + @login_required def process(request): + """Process a selection of messages. + + The request must specify an action to execute against the + selection. + + """ + action = request.POST.get("action", None) ids = request.POST.get("selection", "") ids = ids.split(",") - if not len(ids): + if not ids or action is None: return HttpResponseRedirect(reverse(index)) if request.POST["action"] == "release": @@ -267,6 +326,12 @@ def process(request): if request.POST["action"] == "delete": return delete(request, ids) + if request.POST["action"] == "mark_as_spam": + return mark_messages(request, ids, "spam") + + if request.POST["action"] == "mark_as_ham": + return mark_messages(request, ids, "ham") + @login_required @user_passes_test(lambda u: u.group != 'SimpleUsers') diff --git a/modoboa/lib/sysutils.py b/modoboa/lib/sysutils.py index 02af77148..b30444277 100644 --- a/modoboa/lib/sysutils.py +++ b/modoboa/lib/sysutils.py @@ -9,7 +9,7 @@ import inspect import re -def exec_cmd(cmd, sudo_user=None, **kwargs): +def exec_cmd(cmd, sudo_user=None, pinput=None, **kwargs): """Execute a shell command. Run a command using the current user. Set :keyword:`sudo_user` if @@ -17,15 +17,21 @@ def exec_cmd(cmd, sudo_user=None, **kwargs): :param str cmd: the command to execute :param str sudo_user: a valid system username + :param str pinput: data to send to process's stdin :rtype: tuple :return: return code, command output """ if sudo_user is not None: cmd = "sudo -u %s %s" % (sudo_user, cmd) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, **kwargs) - output = p.communicate()[0] - return p.returncode, output + p_args = {"shell": True, "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT} + if pinput is not None: + p_args["stdin"] = subprocess.PIPE + p_args.update(**kwargs) + process = subprocess.Popen(cmd, **p_args) + c_args = [pinput] if pinput is not None else [] + output = process.communicate(*c_args)[0] + return process.returncode, output def guess_extension_name(): diff --git a/modoboa/static/js/global.js b/modoboa/static/js/global.js index ae8478b2d..3b54ed91c 100644 --- a/modoboa/static/js/global.js +++ b/modoboa/static/js/global.js @@ -239,11 +239,11 @@ function parse_qs(raw) { * Ref: http://stackoverflow.com/questions/901115/get-query-string-values-in-javascript */ function get_parameter_by_name(url, name) { - var name = name.replace(/[\[]/, "\\\[").replace(/[\]]/, "\\\]"); + name = name.replace(/[\[]/, "\\\[").replace(/[\]]/, "\\\]"); var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"), results = regex.exec(url); - return results == null ? "" : decodeURIComponent(results[1].replace(/\+/g, " ")); + return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " ")); } /*