Initial work about SA training from quarantine content.

see #622
This commit is contained in:
Antoine Nguyen
2014-10-21 21:40:52 +02:00
parent 43d9b22679
commit ac00c8e195
10 changed files with 288 additions and 41 deletions
+39 -3
View File
@@ -17,7 +17,7 @@ class ParametersForm(AdminParametersForm):
)
sep1 = SeparatorField(label=_("Messages releasing"))
released_msgs_cleanup = YesNoField(
label=_("Remove released messages"),
initial="no",
@@ -49,7 +49,7 @@ class ParametersForm(AdminParametersForm):
initial="/var/amavis/amavisd.sock",
help_text=_("Path to the PDP server socket (if unix mode)")
)
check_requests_interval = forms.IntegerField(
label=_("Check requests interval"),
initial=30,
@@ -74,10 +74,46 @@ class ParametersForm(AdminParametersForm):
help_text=_("The e-mail address used to send notitications")
)
lsep = SeparatorField(label=_("Learning from messages"))
amavis_is_local = YesNoField(
label=_("Is amavis local?"),
initial="yes",
help_text=_("Tell if amavis is running on the same server than modoboa")
)
default_user = forms.CharField(
label=_("Default user"),
initial="amavis",
help_text=_(
"Name of the user used to store default bayesian filter data"
)
)
per_user_bayesian_filter = YesNoField(
label=_("Enable per-user bayesian filters?"),
initial="no",
help_text=_("Allow users to personally train spamassassin")
)
spamd_address = forms.CharField(
label=_("Spamd address"),
initial="127.0.0.1",
help_text=_("The IP address where spamd can be reached")
)
spamd_port = forms.IntegerField(
label=_("Spamd port"),
initial=783,
help_text=_("The TCP port spamd is listening on")
)
visibility_rules = {
"am_pdp_host": "am_pdp_mode=inet",
"am_pdp_port": "am_pdp_mode=inet",
"am_pdp_socket": "am_pdp_mode=unix"
"am_pdp_socket": "am_pdp_mode=unix",
"spamd_address": "amavis_is_local=no",
"spamd_port": "amavis_is_local=no",
}
+22 -5
View File
@@ -23,22 +23,27 @@ def menu(target, user):
@events.observe("DomainCreated")
def on_domain_created(user, domain):
create_user_and_policy(domain.name)
create_user_and_policy("@{0}".format(domain.name))
@events.observe("DomainModified")
def on_domain_modified(domain):
update_user_and_policy(domain.oldname, domain.name)
update_user_and_policy(
"@{0}".format(domain.oldname),
"@{0}".format(domain.name)
)
@events.observe("DomainDeleted")
def on_domain_deleted(domain):
delete_user_and_policy(domain.name)
delete_user_and_policy("@{0}".format(domain.name))
@events.observe("DomainAliasCreated")
def on_domain_alias_created(user, domainalias):
create_user_and_use_policy(domainalias.name, domainalias.target.name)
create_user_and_use_policy(
"@{0}".format(domainalias.name), domainalias.target.name
)
@events.observe("DomainAliasDeleted")
@@ -46,7 +51,19 @@ def on_domain_alias_deleted(domainaliases):
if isinstance(domainaliases, DomainAlias):
domainaliases = [domainaliases]
for domainalias in domainaliases:
delete_user(domainalias.name)
delete_user("@{0}".format(domainalias.name))
@events.observe("MailboxCreated")
def on_mailbox_created(user, mailbox):
"""Create amavis records for the new mailbox."""
create_user_and_policy(mailbox.full_address)
@events.observe("MailboxDeleted")
def on_mailbox_deleted(mailbox):
"""Delete amavis records related to this mailbox."""
delete_user_and_policy(mailbox.full_address)
@events.observe("GetStaticContent")
+69 -15
View File
@@ -1,13 +1,17 @@
# coding: utf-8
import socket
import re
import struct
import string
from functools import wraps
from django.core.urlresolvers import reverse
import re
import socket
import string
import struct
from django.utils.translation import ugettext as _
from django.core.urlresolvers import reverse
from modoboa.lib import parameters
from modoboa.lib.exceptions import InternalError
from modoboa.lib.sysutils import exec_cmd
from modoboa.lib.webutils import NavigationParameters
from modoboa.extensions.amavis.models import Users, Policy
@@ -80,6 +84,58 @@ recipient=%s
return False
class SpamassassinClient(object):
"""A stupid spamassassin client."""
def __init__(self, username=None):
"""Constructor."""
self._amavis_is_local = parameters.get_admin("AMAVIS_IS_LOCAL")
if username is None:
self._username = parameters.get_admin("DEFAULT_USER")
else:
self._username = username
self.error = None
if self._amavis_is_local == "yes":
self._learn_cmd = "sa-learn --{0} --no-sync -u {1}"
self._learn_cmd_kwargs = {}
# self._learn_cmd_kwargs = {
# "sudo_user": parameters.get_admin("AMAVIS_SYSTEM_USER")
# }
self._expected_exit_codes = [0]
else:
self._learn_cmd = "spamc -d {0} -p {1}".format(
parameters.get_admin("SPAMD_ADDRESS"),
parameters.get_admin("SPAMD_PORT")
)
self._learn_cmd += " -L {0} -u {1}"
self._learn_cmd_kwargs = {}
self._expected_exit_codes = [5, 6]
def _learn(self, msg, mtype):
"""Internal method to call the learning command."""
cmd = self._learn_cmd.format(mtype, self._username)
code, output = exec_cmd(cmd, pinput=msg, **self._learn_cmd_kwargs)
if code in self._expected_exit_codes:
return True
self.error = output
return False
def learn_spam(self, msg):
"""Learn new spam."""
return self._learn(msg, "spam")
def learn_ham(self, msg):
"""Learn new ham."""
return self._learn(msg, "ham")
def done(self):
"""Call this method at the end of the processing."""
if self._amavis_is_local:
exec_cmd("sa-learn -u {0} --sync".format(self._username),
**self._learn_cmd_kwargs)
class QuarantineNavigationParameters(NavigationParameters):
"""
Specific NavigationParameters subclass for the quarantine.
@@ -114,7 +170,7 @@ class QuarantineNavigationParameters(NavigationParameters):
return url
def create_user_and_policy(name):
def create_user_and_policy(name, priority=7):
"""Create records.
Create two records (a user and a policy) using :keyword:`name` as
@@ -124,12 +180,11 @@ def create_user_and_policy(name):
"""
policy = Policy.objects.create(policy_name=name[:32])
Users.objects.create(
email="@%s" % name, fullname=name,
priority=7, policy=policy
email=name, fullname=name, priority=priority, policy=policy
)
def create_user_and_use_policy(name, policy_name):
def create_user_and_use_policy(name, policy_name, priority=7):
"""Create a *users* record and use an existing policy.
:param str name: user record name
@@ -137,8 +192,7 @@ def create_user_and_use_policy(name, policy_name):
"""
policy = Policy.objects.get(policy_name=policy_name[:32])
Users.objects.create(
email="@%s" % name, fullname=name,
priority=7, policy=policy
email=name, fullname=name, priority=priority, policy=policy
)
@@ -150,8 +204,8 @@ def update_user_and_policy(oldname, newname):
"""
if oldname == newname:
return
u = Users.objects.get(email="@%s" % oldname)
u.email = "@%s" % newname
u = Users.objects.get(email=oldname)
u.email = newname
u.fullname = newname
u.policy.policy_name = newname[:32]
u.policy.save()
@@ -164,7 +218,7 @@ def delete_user_and_policy(name):
:param str name: identifier
"""
try:
u = Users.objects.get(email="@%s" % name)
u = Users.objects.get(email=name)
except Users.DoesNotExist:
return
u.policy.delete()
@@ -177,6 +231,6 @@ def delete_user(name):
:param str name: user record name
"""
try:
Users.objects.get(email="@%s" % name).delete()
Users.objects.get(email=name).delete()
except Users.DoesNotExist:
pass
@@ -62,10 +62,18 @@ Quarantine.prototype = {
$.proxy(this.release_selection, this));
$(document).on("click", "a[name=delete-multi]",
$.proxy(this.delete_selection, this));
$(document).on("click", "a[name=mark-as-spam-multi]",
$.proxy(this.mark_selection_as_spam, this));
$(document).on("click", "a[name=mark-as-ham-multi]",
$.proxy(this.mark_selection_as_ham, this));
$(document).on("click", "a[name=viewrequests]",
$.proxy(this.view_requests, this));
$(document).on("click", "a[name=release]", $.proxy(this.release, this));
$(document).on("click", "a[name=delete]", $.proxy(this.delete, this));
$(document).on("click", "a[name=mark-as-spam]",
$.proxy(this.mark_as_spam, this));
$(document).on("click", "a[name=mark-as-ham]",
$.proxy(this.mark_as_ham, this));
$(document).on("click", "a[name=headers]", $.proxy(this.headers, this));
$(document).on("click", "td[name=type] span", $.proxy(this.filter_by_type, this));
$(document).on("click", "#filters button", $.proxy(this.filter_by_type, this));
@@ -191,6 +199,26 @@ Quarantine.prototype = {
this._send_selection(e, "delete", gettext("Delete this selection?"));
},
/**
* Mark the current selection as spam.
*
* @param {Object} e - event object
*/
mark_selection_as_spam: function(e) {
this._send_selection(
e, "mark_as_spam", gettext("Mark this selection as spam?"));
},
/**
* Mark the current selection as ham.
*
* @param {Object} e - event object
*/
mark_selection_as_ham: function(e) {
this._send_selection(
e, "mark_as_ham", gettext("Mark this selection as non-spam?"));
},
_send_action: function(e, message) {
var $link = get_target(e, "a");
@@ -206,7 +234,7 @@ Quarantine.prototype = {
}).done($.proxy(this.action_cb, this));
},
release: function(e) {
release : function(e) {
this._send_action(e, gettext("Release this message?"));
},
@@ -214,6 +242,24 @@ Quarantine.prototype = {
this._send_action(e, gettext("Delete this message?"));
},
/**
* Mark the current message as spam.
*
* @param {Object} e - event object
*/
mark_as_spam: function(e) {
this._send_action(e, gettext("Mark as spam?"));
},
/**
* Mark the current message as ham.
*
* @param {Object} e - event object
*/
mark_as_ham: function(e) {
this._send_action(e, gettext("Mark as non-spam?"));
},
show_rawheaders: function($mailcontent) {
$mailcontent.find("#rawheaders").removeClass("hidden");
$mailcontent.find("#emailheaders").addClass("hidden");
@@ -261,15 +307,13 @@ Quarantine.prototype = {
},
activate_buttons: function($tr) {
$("a[name=release-multi]").removeClass('disabled');
$("a[name=delete-multi]").removeClass('disabled');
$("a[name*=-multi]").removeClass('disabled');
$("#selectall").prop('checked', true);
},
deactivate_buttons: function($tr) {
if (!this.htmltable || !this.htmltable.current_selection().length) {
$("a[name=release-multi]").addClass('disabled');
$("a[name=delete-multi]").addClass('disabled');
$("a[name*=-multi]").addClass('disabled');
$("#selectall").prop('checked', false);
}
},
@@ -23,6 +23,15 @@
<i class="icon-white icon-trash"></i> {% trans "Delete" %}
</a>
</div>
<div class="btn-group" title="{% trans 'More actions' %}">
<button class="btn dropdown-toggle" data-toggle="dropdown">
<i class="icon-cog"></i> <span class="caret"></span>
</button>
<ul class="dropdown-menu">
<li><a name="mark-as-spam-multi" href="{% url 'modoboa.extensions.amavis.views.process' %}">{% trans "Mark as spam" %}</a></li>
<li><a name="mark-as-ham-multi" href="{% url 'modoboa.extensions.amavis.views.process' %}">{% trans "Mark as non-spam" %}</a></li>
</ul>
</div>
{% include "common/email_searchbar.html" with extraopts=extraopts %}
</div>
<div id="filters_bar" class="btn-toolbar pull-right">
@@ -2,8 +2,6 @@ from django import template
from django.template.loader import render_to_string
from django.utils.translation import ugettext as _
from django.core.urlresolvers import reverse
from django.conf import settings
from modoboa.lib.webutils import _render_to_string
register = template.Library()
@@ -23,8 +21,24 @@ def viewm_menu(mail_id, rcpt):
"url": reverse('modoboa.extensions.amavis.views.delete', args=[mail_id])
+ ("?rcpt=%s" % rcpt if rcpt else ""),
"label": _("Delete")},
{"name": "process",
"img": "icon-cog",
"menu": [
{"name": "mark-as-spam",
"label": _("Mark as spam"),
"url": reverse(
"modoboa.extensions.amavis.views.mark_as_spam", args=[mail_id]
) + ("?rcpt=%s" % rcpt if rcpt else "")},
{"name": "mark-as-ham",
"label": _("Mark as non-spam"),
"url": reverse(
"modoboa.extensions.amavis.views.mark_as_ham", args=[mail_id]
) + ("?rcpt=%s" % rcpt if rcpt else "")}
]},
{"name": "headers",
"url": reverse('modoboa.extensions.amavis.views.viewheaders', args=[mail_id]),
"url": reverse(
'modoboa.extensions.amavis.views.viewheaders', args=[mail_id]
),
"label": _("View full headers")},
]
+3 -1
View File
@@ -9,6 +9,8 @@ urlpatterns = patterns(
(r'^nbrequests/$', 'nbrequests'),
(r'^delete/(?P<mail_id>[\w\-\+]+)/$', 'delete'),
(r'^release/(?P<mail_id>[\w\-\+]+)/$', 'release'),
(r'^markspam/(?P<mail_id>[\w\-\+]+)/$', 'mark_as_spam'),
(r'^markham/(?P<mail_id>[\w\-\+]+)/$', 'mark_as_ham'),
(r'^(?P<mail_id>[\w\-\+]+)/$', 'viewmail'),
(r'^(?P<mail_id>[\w\-\+]+)/headers/$', 'viewheaders'),
)
)
+67 -2
View File
@@ -17,7 +17,10 @@ from modoboa.extensions.admin.models import Mailbox, Domain
from modoboa.extensions.amavis.templatetags.amavis_tags import (
quar_menu, viewm_menu
)
from .lib import selfservice, AMrelease, QuarantineNavigationParameters
from .lib import (
selfservice, AMrelease, QuarantineNavigationParameters,
SpamassassinClient
)
from .models import Msgrcpt
from .sql_connector import get_connector
from .sql_listing import SQLlisting, SQLemail
@@ -254,11 +257,67 @@ def release(request, mail_id):
)
def mark_messages(request, selection, mtype):
"""Mark a selection of messages as spam.
:param str selection: message unique identifier
:param str mtype: type of marking (spam or ham)
"""
selection = check_mail_id(request, selection)
connector = get_connector()
per_user_bayesian_filter = parameters.get_admin("PER_USER_BAYESIAN_FILTER")
if per_user_bayesian_filter == "yes" \
and request.user.group == "SimpleUsers":
username = request.user.email
else:
username = None
saclient = SpamassassinClient(username)
for item in selection:
rcpt, mail_id = item.split()
content = "".join(
[msg.mail_text for msg in connector.get_mail_content(mail_id)]
)
result = saclient.learn_spam(content) if mtype == "spam" \
else saclient.learn_ham(content)
if not result:
break
if saclient.error is None:
saclient.done()
message = ungettext("%(count)d message processed successfully",
"%(count)d messages processed successfully",
len(selection)) % {"count": len(selection)}
else:
message = saclient.error
return ajax_response(
request, "ko" if saclient.error else "ok", respmsg=message,
url=QuarantineNavigationParameters(request).back_to_listing()
)
@login_required
def mark_as_spam(request, mail_id):
"""Mark a single message as spam."""
return mark_messages(request, mail_id, "spam")
@login_required
def mark_as_ham(request, mail_id):
"""Mark a single message as ham."""
return mark_messages(request, mail_id, "ham")
@login_required
def process(request):
"""Process a selection of messages.
The request must specify an action to execute against the
selection.
"""
action = request.POST.get("action", None)
ids = request.POST.get("selection", "")
ids = ids.split(",")
if not len(ids):
if not ids or action is None:
return HttpResponseRedirect(reverse(index))
if request.POST["action"] == "release":
@@ -267,6 +326,12 @@ def process(request):
if request.POST["action"] == "delete":
return delete(request, ids)
if request.POST["action"] == "mark_as_spam":
return mark_messages(request, ids, "spam")
if request.POST["action"] == "mark_as_ham":
return mark_messages(request, ids, "ham")
@login_required
@user_passes_test(lambda u: u.group != 'SimpleUsers')
+11 -5
View File
@@ -9,7 +9,7 @@ import inspect
import re
def exec_cmd(cmd, sudo_user=None, **kwargs):
def exec_cmd(cmd, sudo_user=None, pinput=None, **kwargs):
"""Execute a shell command.
Run a command using the current user. Set :keyword:`sudo_user` if
@@ -17,15 +17,21 @@ def exec_cmd(cmd, sudo_user=None, **kwargs):
:param str cmd: the command to execute
:param str sudo_user: a valid system username
:param str pinput: data to send to process's stdin
:rtype: tuple
:return: return code, command output
"""
if sudo_user is not None:
cmd = "sudo -u %s %s" % (sudo_user, cmd)
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, **kwargs)
output = p.communicate()[0]
return p.returncode, output
p_args = {"shell": True, "stdout": subprocess.PIPE,
"stderr": subprocess.STDOUT}
if pinput is not None:
p_args["stdin"] = subprocess.PIPE
p_args.update(**kwargs)
process = subprocess.Popen(cmd, **p_args)
c_args = [pinput] if pinput is not None else []
output = process.communicate(*c_args)[0]
return process.returncode, output
def guess_extension_name():
+2 -2
View File
@@ -239,11 +239,11 @@ function parse_qs(raw) {
* Ref: http://stackoverflow.com/questions/901115/get-query-string-values-in-javascript
*/
function get_parameter_by_name(url, name) {
var name = name.replace(/[\[]/, "\\\[").replace(/[\]]/, "\\\]");
name = name.replace(/[\[]/, "\\\[").replace(/[\]]/, "\\\]");
var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"),
results = regex.exec(url);
return results == null ? "" : decodeURIComponent(results[1].replace(/\+/g, " "));
return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " "));
}
/*