mirror of
https://github.com/jwinterm/modoboa.git
synced 2026-10-17 05:41:25 +02:00
POC replace travis by Github Actions (#2066)
* POC replace travis by Github Actions * add libldap2-dev * add libsasl2-dev libssl-dev * Delete .travis.yml * Add specific configuration for PostgreSQL * Retrieve PostgreSQL or MySQL port from env var * Print affected port for the docker service * Add Database type and dburl * Fix dbtype * Use environment variable to define the port in use for tests. * Configured password for postgres connection. * Added redis service. * Enabled OpenLDAP service. * Use a container for OpenLDAP too. * Removed wrong option. * Fixed volume syntax. * Try using the option directly. * Try using an absolute path. * Fixed wrong syntax. * Try different format for env variable. * Try a different way to load data. * Added missing method call. * Fix envar * Try to wait a bit before starting tests. * Display openldap service logs. * Fixed ldif scripts path. * Try to use a custom docker image. * Try to provide credentials. * Try with github token. * use main reference * Changed LDAP Admin password. * Added debug. * Added more debug * Removed debug. * dd test for MySQL * Fix name * Test is independant * Link build * add envar DB for MySQL * remove depends * fix fast failed * fix mysql user * Use 127.0.0.1 to force for IPv4 * For parameters for MySQL * Database name test_modoboa * change mysql user to launch test * Move to the last mysql version lose connection on 5.7 * TEst some timout parameters * remove bad parameters * Add CONN_MAX_AGE parameters * Add docker logs * Try to reset db connections before we spawn processes. * POC replace travis by Github Actions * add libldap2-dev * add libsasl2-dev libssl-dev * Delete .travis.yml * Add specific configuration for PostgreSQL * Retrieve PostgreSQL or MySQL port from env var * Print affected port for the docker service * Add Database type and dburl * Fix dbtype * Use environment variable to define the port in use for tests. * Configured password for postgres connection. * Added redis service. * Enabled OpenLDAP service. * Use a container for OpenLDAP too. * Removed wrong option. * Fixed volume syntax. * Try using the option directly. * Try using an absolute path. * Fixed wrong syntax. * Try different format for env variable. * Try a different way to load data. * Added missing method call. * Fix envar * Try to wait a bit before starting tests. * Display openldap service logs. * Fixed ldif scripts path. * Try to use a custom docker image. * Try to provide credentials. * Try with github token. * use main reference * Changed LDAP Admin password. * Added debug. * Added more debug * Removed debug. * dd test for MySQL * Fix name * Test is independant * Link build * add envar DB for MySQL * remove depends * fix fast failed * fix mysql user * Use 127.0.0.1 to force for IPv4 * For parameters for MySQL * Database name test_modoboa * change mysql user to launch test * Move to the last mysql version lose connection on 5.7 * TEst some timout parameters * remove bad parameters * Add CONN_MAX_AGE parameters * Add docker logs * Try to reset db connections before we spawn processes. * Add relase on pypi (test server first) * Fix setuptools scm to fix upload on Pypi test * Retrieve all repos when publish on pypi * if file exists, skip it * Deploy only on master * Added python 3.5 in matrix. * Removed comments. * Add translation and codecov * Missing argument for upload in production * fix bad indentation * move codecov * Codecov only in one build * fix command * Try to run codecov after PG tests. * Fixed expressions syntax. * Fixed path issue. * Fixed path issue. * Try workflow_run statement. * Removed useless triggers. * Test trigger on one workflow * Another test * Exclude master and main branch * Regroup all steps in one workflow. * Fixed typo * Fixed bad syntax * Added missing requirement. * Push on Pypi Test only on master branch Co-authored-by: Antoine Nguyen <tonio@ngyn.org>
This commit is contained in:
Christophe CHAUVET
co-authored by
Antoine Nguyen
Antoine Nguyen
parent
b6b07f2d15
commit
02efd7f0e3
@@ -0,0 +1,180 @@
|
||||
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
|
||||
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
|
||||
|
||||
name: Modoboa App
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
env:
|
||||
POSTGRES_HOST: localhost
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:11
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: postgres
|
||||
ports:
|
||||
# will assign a random free host port
|
||||
- 5432/tcp
|
||||
# needed because the postgres container does not provide a healthcheck
|
||||
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
MYSQL_USER: modoboa
|
||||
MYSQL_PASSWORD: modoboa
|
||||
MYSQL_DATABASE: modoboa
|
||||
ports:
|
||||
- 3306/tcp
|
||||
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
|
||||
redis:
|
||||
image: redis
|
||||
ports:
|
||||
- 6379/tcp
|
||||
openldap:
|
||||
image: docker.pkg.github.com/modoboa/docker-openldap/docker-openldap
|
||||
ports:
|
||||
- 389/tcp
|
||||
env:
|
||||
LDAP_DOMAIN: example.com
|
||||
LDAP_ADMIN_PASSWORD: test
|
||||
credentials:
|
||||
username: tonioo
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
database: ['postgres', 'mysql']
|
||||
python-version: [3.6, 3.7, 3.8]
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get install postfix librrd-dev rrdtool libldap2-dev libsasl2-dev libssl-dev
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install -r ldap-requirements.txt
|
||||
pip install -r test-requirements.txt
|
||||
python setup.py develop
|
||||
- name: Install postgres requirements
|
||||
if: ${{ matrix.database == 'postgres' }}
|
||||
run: |
|
||||
pip install -r postgresql-requirements.txt
|
||||
pip install coverage
|
||||
echo "DB=postgres" >> $GITHUB_ENV
|
||||
- name: Install mysql requirements
|
||||
if: ${{ matrix.database == 'mysql' }}
|
||||
run: |
|
||||
pip install -r mysql-requirements.txt
|
||||
echo "DB=mysql" >> $GITHUB_ENV
|
||||
- name: Test with pytest
|
||||
if: ${{ matrix.python-version != '3.8' || matrix.database != 'postgres' }}
|
||||
run: |
|
||||
python ./tests.py
|
||||
cd test_project
|
||||
python3 manage.py test modoboa.core modoboa.lib modoboa.admin modoboa.limits modoboa.transport modoboa.relaydomains modoboa.dnstools modoboa.ldapsync modoboa.policyd modoboa.maillog
|
||||
env:
|
||||
# use localhost for the host here because we are running the job on the VM.
|
||||
# If we were running the job on in a container this would be postgres
|
||||
POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} # get randomly assigned published port
|
||||
MYSQL_HOST: 127.0.0.1
|
||||
MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port
|
||||
MYSQL_USER: root
|
||||
REDIS_PORT: ${{ job.services.redis.ports[6379] }}
|
||||
LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }}
|
||||
|
||||
- name: Test with pytest and coverage
|
||||
if: ${{ matrix.python-version == '3.8' && matrix.database == 'postgres' }}
|
||||
run: |
|
||||
python ./tests.py
|
||||
cd test_project
|
||||
coverage run manage.py test modoboa.core modoboa.lib modoboa.admin modoboa.limits modoboa.transport modoboa.relaydomains modoboa.dnstools modoboa.ldapsync modoboa.policyd modoboa.maillog
|
||||
coverage combine
|
||||
env:
|
||||
# use localhost for the host here because we are running the job on the VM.
|
||||
# If we were running the job on in a container this would be postgres
|
||||
POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} # get randomly assigned published port
|
||||
MYSQL_HOST: 127.0.0.1
|
||||
MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port
|
||||
MYSQL_USER: root
|
||||
REDIS_PORT: ${{ job.services.redis.ports[6379] }}
|
||||
LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }}
|
||||
- name: Upload coverage result
|
||||
if: ${{ matrix.python-version == '3.8' }}
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: coverage-results
|
||||
path: test_project/.coverage
|
||||
|
||||
coverage:
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.8
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.8'
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install codecov
|
||||
- name: Download coverage results
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: coverage-results
|
||||
- name: Report coverage
|
||||
run: |
|
||||
coverage report
|
||||
codecov
|
||||
|
||||
release:
|
||||
needs: coverage
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Python 3.8
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.8'
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get install librrd-dev rrdtool libssl-dev gettext
|
||||
python -m pip install --upgrade pip setuptools wheel
|
||||
pip install -r requirements.txt
|
||||
cd modoboa
|
||||
django-admin compilemessages
|
||||
cd ..
|
||||
python setup.py sdist bdist_wheel
|
||||
- name: Publish to Test PyPI
|
||||
if: endsWith(github.event.ref, '/master')
|
||||
uses: pypa/gh-action-pypi-publish@master
|
||||
with:
|
||||
user: __token__
|
||||
password: ${{ secrets.test_pypi_password }}
|
||||
repository_url: https://test.pypi.org/legacy/
|
||||
skip_existing: true
|
||||
- name: Publish distribution to PyPI
|
||||
if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
|
||||
uses: pypa/gh-action-pypi-publish@master
|
||||
with:
|
||||
user: __token__
|
||||
password: ${{ secrets.pypi_password }}
|
||||
skip_existing: true
|
||||
-77
@@ -1,77 +0,0 @@
|
||||
language: python
|
||||
python:
|
||||
- '3.5'
|
||||
- '3.6'
|
||||
- '3.7'
|
||||
- '3.8'
|
||||
|
||||
env:
|
||||
global:
|
||||
- PYTHONWARNINGS="default,ignore::PendingDeprecationWarning,ignore::ResourceWarning"
|
||||
matrix:
|
||||
- DB="POSTGRESQL"
|
||||
- DB="MYSQL"
|
||||
|
||||
sudo: false
|
||||
|
||||
addons:
|
||||
postgresql: '9.6'
|
||||
mysql: '5.5'
|
||||
apt:
|
||||
packages:
|
||||
- ldap-utils
|
||||
- postfix
|
||||
- slapd
|
||||
- librrd-dev
|
||||
- rrdtool
|
||||
|
||||
cache: pip
|
||||
|
||||
services:
|
||||
- postgres
|
||||
- mysql
|
||||
- redis-server
|
||||
|
||||
before_install:
|
||||
- pip install codecov
|
||||
|
||||
install:
|
||||
- pip install -r requirements.txt
|
||||
- pip install -r ldap-requirements.txt
|
||||
- pip install -r test-requirements.txt
|
||||
- if [ $DB = POSTGRESQL ]; then pip install -r postgresql-requirements.txt ; fi
|
||||
- if [ $DB = MYSQL ]; then pip install -r mysql-requirements.txt ; fi
|
||||
- python setup.py develop
|
||||
|
||||
before_script:
|
||||
- mkdir /tmp/slapd
|
||||
- slapd -f test_data/slapd.conf -h ldap://localhost:3389 &
|
||||
- sleep 3
|
||||
- ldapadd -h localhost:3389 -D cn=admin,dc=example,dc=com -w test -f test_data/base.ldif
|
||||
|
||||
script:
|
||||
- python ./tests.py
|
||||
- cd test_project
|
||||
- coverage run manage.py test modoboa.core modoboa.lib modoboa.admin modoboa.limits modoboa.transport modoboa.relaydomains modoboa.dnstools modoboa.ldapsync modoboa.policyd modoboa.maillog
|
||||
|
||||
after_success:
|
||||
- coverage combine
|
||||
- coverage report
|
||||
- codecov
|
||||
|
||||
before_deploy:
|
||||
- cd ../modoboa
|
||||
- django-admin compilemessages
|
||||
- cd ..
|
||||
|
||||
deploy:
|
||||
provider: pypi
|
||||
user: tonio
|
||||
password:
|
||||
secure: SKFrH4cri56YQziqMZ4uvbh5Vto9YrQuBlYRnO9BwcjXbcp6jMpZ+9j+aytMdvcN+xG5of6DmgSIiZsLCOl4LDoPn2oPwWMzZA9T8knSVYt9v1uiBTwDmssIt9QH3wD/o3edpTsSoKw+O2cTT2GaE75+3aTK01+CX4ZsUtkGbTk=
|
||||
skip_cleanup: true
|
||||
distributions: sdist bdist_wheel
|
||||
on:
|
||||
tags: true
|
||||
python: '3.8'
|
||||
condition: "$DB = POSTGRESQL"
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from unittest import skipIf
|
||||
|
||||
from django.conf import settings
|
||||
from django.test import override_settings
|
||||
from django.urls import reverse
|
||||
from django.utils.functional import cached_property
|
||||
@@ -25,7 +26,7 @@ class LDAPTestCaseMixin(object):
|
||||
"""Modify settings."""
|
||||
self.set_global_parameters({
|
||||
"authentication_type": "ldap",
|
||||
"ldap_server_port": 3389
|
||||
"ldap_server_port": settings.LDAP_SERVER_PORT
|
||||
})
|
||||
|
||||
def restore_user_password(self, username, new_password):
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from unittest import skipIf
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management import call_command
|
||||
from django.utils.encoding import force_bytes, force_str
|
||||
|
||||
@@ -23,7 +24,7 @@ class LDAPExportTestCase(ModoTestCase):
|
||||
super().setUp()
|
||||
self.set_global_parameters({
|
||||
"ldap_enable_sync": True,
|
||||
"ldap_server_port": 3389,
|
||||
"ldap_server_port": settings.LDAP_SERVER_PORT,
|
||||
"ldap_sync_bind_dn": "cn=admin,dc=example,dc=com",
|
||||
"ldap_sync_bind_password": "test",
|
||||
"ldap_sync_account_dn_template": (
|
||||
@@ -111,7 +112,7 @@ class LDAPImportTestCase(ModoTestCase):
|
||||
super().setUp()
|
||||
self.set_global_parameters({
|
||||
"ldap_enable_import": True,
|
||||
"ldap_server_port": 3389,
|
||||
"ldap_server_port": settings.LDAP_SERVER_PORT,
|
||||
"ldap_sync_bind_dn": "cn=admin,dc=example,dc=com",
|
||||
"ldap_sync_bind_password": "test",
|
||||
"ldap_import_search_base": "ou=users,dc=example,dc=com",
|
||||
|
||||
@@ -9,6 +9,7 @@ import time
|
||||
|
||||
import redis
|
||||
|
||||
from django import db
|
||||
from django.conf import settings
|
||||
from django.core.management import call_command
|
||||
from django.test import TransactionTestCase
|
||||
@@ -55,6 +56,7 @@ class PolicyDaemonTestCase(
|
||||
self.admin = core_models.User.objects.get(username="user@test2.com")
|
||||
self.admin.role = "SuperAdmins"
|
||||
|
||||
db.connections.close_all()
|
||||
patcher = patch("aiosmtplib.send")
|
||||
self.send_mock = patcher.start()
|
||||
self.process = Process(target=start_policy_daemon)
|
||||
|
||||
@@ -3,17 +3,17 @@ import os
|
||||
# Database
|
||||
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
|
||||
|
||||
DB = os.environ.get("DB", "POSTGRESQL")
|
||||
DB = os.environ.get("DB", "postgres").lower()
|
||||
|
||||
if DB == "MYSQL":
|
||||
if DB == "mysql":
|
||||
DATABASES = {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.mysql",
|
||||
"NAME": "modoboa",
|
||||
"USER": "root",
|
||||
"PASSWORD": "",
|
||||
"HOST": "localhost",
|
||||
"PORT": "",
|
||||
"USER": os.environ.get("MYSQL_USER", "root"),
|
||||
"PASSWORD": os.environ.get("MYSQL_PASSWORD", "root"),
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": os.environ.get("MYSQL_PORT", "3306"),
|
||||
"ATOMIC_REQUESTS": True,
|
||||
# MySQL's Strict Mode fixes many data integrity problems in MySQL,
|
||||
# such as data truncation upon insertion, by escalating warnings
|
||||
@@ -27,6 +27,7 @@ if DB == "MYSQL":
|
||||
"SET innodb_strict_mode = ON;"
|
||||
),
|
||||
"charset": "utf8",
|
||||
"connect_timeout": 10,
|
||||
},
|
||||
"TEST": {
|
||||
"CHARSET": "utf8",
|
||||
@@ -34,7 +35,7 @@ if DB == "MYSQL":
|
||||
},
|
||||
},
|
||||
}
|
||||
elif DB == "SQLITE":
|
||||
elif DB == "sqlite":
|
||||
DATABASES = {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.sqlite3",
|
||||
@@ -49,9 +50,9 @@ else:
|
||||
"ENGINE": "django.db.backends.postgresql",
|
||||
"NAME": "modoboa",
|
||||
"USER": "postgres",
|
||||
"PASSWORD": "",
|
||||
"PASSWORD": "postgres",
|
||||
"HOST": "localhost",
|
||||
"PORT": "",
|
||||
"PORT": os.environ.get("POSTGRES_PORT", "5432"),
|
||||
"ATOMIC_REQUESTS": True,
|
||||
"OPTIONS": {
|
||||
"client_encoding": "UTF8",
|
||||
|
||||
@@ -45,6 +45,11 @@ if __name__ == "__main__":
|
||||
with open(path.join(HERE, "README.rst")) as readme:
|
||||
LONG_DESCRIPTION = readme.read()
|
||||
|
||||
def local_scheme(version):
|
||||
"""Skip the local version (eg. +xyz of 0.6.1.dev4+gdf99fe2)
|
||||
to be able to upload to Test PyPI"""
|
||||
return ""
|
||||
|
||||
setup(
|
||||
name="modoboa",
|
||||
description="Mail hosting made simple",
|
||||
@@ -74,7 +79,7 @@ if __name__ == "__main__":
|
||||
zip_safe=False,
|
||||
scripts=["bin/modoboa-admin.py"],
|
||||
install_requires=INSTALL_REQUIRES,
|
||||
use_scm_version=True,
|
||||
use_scm_version={"local_scheme": local_scheme},
|
||||
python_requires=">=3.4",
|
||||
setup_requires=["setuptools_scm"],
|
||||
extras_require={
|
||||
|
||||
@@ -196,7 +196,7 @@ MODOBOA_API_URL = 'https://api.modoboa.org/1/'
|
||||
# REDIS
|
||||
|
||||
REDIS_HOST = 'localhost'
|
||||
REDIS_PORT = 6379
|
||||
REDIS_PORT = os.environ.get('REDIS_PORT', 6379)
|
||||
REDIS_QUOTA_DB = 0
|
||||
REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB)
|
||||
|
||||
@@ -281,6 +281,9 @@ LOGGING = {
|
||||
},
|
||||
'modoboa': {
|
||||
'class': 'modoboa.core.loggers.SQLHandler',
|
||||
},
|
||||
'console': {
|
||||
'class': 'logging.StreamHandler'
|
||||
}
|
||||
},
|
||||
'loggers': {
|
||||
@@ -298,7 +301,11 @@ LOGGING = {
|
||||
'handlers': ['modoboa'],
|
||||
'level': 'INFO',
|
||||
'propagate': False
|
||||
}
|
||||
},
|
||||
# 'django_auth_ldap': {
|
||||
# 'level': 'DEBUG',
|
||||
# 'handlers': ['console']
|
||||
# },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,3 +315,5 @@ SILENCED_SYSTEM_CHECKS = [
|
||||
|
||||
DISABLE_DASHBOARD_EXTERNAL_QUERIES = False
|
||||
# Load settings from extensions
|
||||
|
||||
LDAP_SERVER_PORT = os.environ.get('LDAP_SERVER_PORT', 3389)
|
||||
|
||||
@@ -5,22 +5,29 @@ import unittest
|
||||
from modoboa.lib.sysutils import exec_cmd
|
||||
|
||||
DB = os.environ.get("DB", "postgres")
|
||||
if DB.lower() == "postgres":
|
||||
PORT = os.environ.get("POSTGRES_PORT", "5432")
|
||||
DB = "postgres"
|
||||
else:
|
||||
PORT = os.environ.get("MYSQL_PORT", "3306")
|
||||
DB = "mysql"
|
||||
|
||||
|
||||
class DeployTest(unittest.TestCase):
|
||||
dbtype = DB.lower()
|
||||
dbtype = DB
|
||||
dbhost = "localhost"
|
||||
dbport = PORT
|
||||
projname = "modoboa_test"
|
||||
dbuser = DB == "MYSQL" and "modoboa" or "postgres"
|
||||
dbpassword = DB == "MYSQL" and "modoboa" or ""
|
||||
dbuser = DB == "mysql" and "modoboa" or "postgres"
|
||||
dbpassword = DB == "mysql" and "modoboa" or ""
|
||||
|
||||
def setUp(self):
|
||||
self.workdir = tempfile.mkdtemp()
|
||||
|
||||
def test_silent(self):
|
||||
dburl = "default:%s://%s:%s@%s/%s" \
|
||||
dburl = "default:%s://%s:%s@%s:%s/%s" \
|
||||
% (self.dbtype, self.dbuser, self.dbpassword,
|
||||
self.dbhost, self.projname)
|
||||
self.dbhost, self.dbport, self.projname)
|
||||
cmd = (
|
||||
"modoboa-admin.py deploy --collectstatic "
|
||||
"--dburl %s --domain %s --admin-username admin %s"
|
||||
|
||||
Reference in New Issue
Block a user