From 02efd7f0e3ca8bd4abd135ce2ae1b8eb37352d80 Mon Sep 17 00:00:00 2001 From: Christophe CHAUVET Date: Mon, 9 Nov 2020 20:44:48 +0100 Subject: [PATCH] POC replace travis by Github Actions (#2066) * POC replace travis by Github Actions * add libldap2-dev * add libsasl2-dev libssl-dev * Delete .travis.yml * Add specific configuration for PostgreSQL * Retrieve PostgreSQL or MySQL port from env var * Print affected port for the docker service * Add Database type and dburl * Fix dbtype * Use environment variable to define the port in use for tests. * Configured password for postgres connection. * Added redis service. * Enabled OpenLDAP service. * Use a container for OpenLDAP too. * Removed wrong option. * Fixed volume syntax. * Try using the option directly. * Try using an absolute path. * Fixed wrong syntax. * Try different format for env variable. * Try a different way to load data. * Added missing method call. * Fix envar * Try to wait a bit before starting tests. * Display openldap service logs. * Fixed ldif scripts path. * Try to use a custom docker image. * Try to provide credentials. * Try with github token. * use main reference * Changed LDAP Admin password. * Added debug. * Added more debug * Removed debug. * dd test for MySQL * Fix name * Test is independant * Link build * add envar DB for MySQL * remove depends * fix fast failed * fix mysql user * Use 127.0.0.1 to force for IPv4 * For parameters for MySQL * Database name test_modoboa * change mysql user to launch test * Move to the last mysql version lose connection on 5.7 * TEst some timout parameters * remove bad parameters * Add CONN_MAX_AGE parameters * Add docker logs * Try to reset db connections before we spawn processes. * POC replace travis by Github Actions * add libldap2-dev * add libsasl2-dev libssl-dev * Delete .travis.yml * Add specific configuration for PostgreSQL * Retrieve PostgreSQL or MySQL port from env var * Print affected port for the docker service * Add Database type and dburl * Fix dbtype * Use environment variable to define the port in use for tests. * Configured password for postgres connection. * Added redis service. * Enabled OpenLDAP service. * Use a container for OpenLDAP too. * Removed wrong option. * Fixed volume syntax. * Try using the option directly. * Try using an absolute path. * Fixed wrong syntax. * Try different format for env variable. * Try a different way to load data. * Added missing method call. * Fix envar * Try to wait a bit before starting tests. * Display openldap service logs. * Fixed ldif scripts path. * Try to use a custom docker image. * Try to provide credentials. * Try with github token. * use main reference * Changed LDAP Admin password. * Added debug. * Added more debug * Removed debug. * dd test for MySQL * Fix name * Test is independant * Link build * add envar DB for MySQL * remove depends * fix fast failed * fix mysql user * Use 127.0.0.1 to force for IPv4 * For parameters for MySQL * Database name test_modoboa * change mysql user to launch test * Move to the last mysql version lose connection on 5.7 * TEst some timout parameters * remove bad parameters * Add CONN_MAX_AGE parameters * Add docker logs * Try to reset db connections before we spawn processes. * Add relase on pypi (test server first) * Fix setuptools scm to fix upload on Pypi test * Retrieve all repos when publish on pypi * if file exists, skip it * Deploy only on master * Added python 3.5 in matrix. * Removed comments. * Add translation and codecov * Missing argument for upload in production * fix bad indentation * move codecov * Codecov only in one build * fix command * Try to run codecov after PG tests. * Fixed expressions syntax. * Fixed path issue. * Fixed path issue. * Try workflow_run statement. * Removed useless triggers. * Test trigger on one workflow * Another test * Exclude master and main branch * Regroup all steps in one workflow. * Fixed typo * Fixed bad syntax * Added missing requirement. * Push on Pypi Test only on master branch Co-authored-by: Antoine Nguyen --- .github/workflows/modoboa.yml | 180 ++++++++++++++++++++++++++ .travis.yml | 77 ----------- modoboa/core/tests/test_ldap.py | 3 +- modoboa/ldapsync/tests.py | 5 +- modoboa/policyd/tests.py | 2 + modoboa/test_settings.py | 19 +-- setup.py | 7 +- test_project/test_project/settings.py | 13 +- tests.py | 17 ++- 9 files changed, 226 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/modoboa.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/modoboa.yml b/.github/workflows/modoboa.yml new file mode 100644 index 000000000..cfd765302 --- /dev/null +++ b/.github/workflows/modoboa.yml @@ -0,0 +1,180 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Modoboa App + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + POSTGRES_HOST: localhost + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:11 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + # will assign a random free host port + - 5432/tcp + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_USER: modoboa + MYSQL_PASSWORD: modoboa + MYSQL_DATABASE: modoboa + ports: + - 3306/tcp + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + redis: + image: redis + ports: + - 6379/tcp + openldap: + image: docker.pkg.github.com/modoboa/docker-openldap/docker-openldap + ports: + - 389/tcp + env: + LDAP_DOMAIN: example.com + LDAP_ADMIN_PASSWORD: test + credentials: + username: tonioo + password: ${{ secrets.GITHUB_TOKEN }} + + strategy: + matrix: + database: ['postgres', 'mysql'] + python-version: [3.6, 3.7, 3.8] + fail-fast: false + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get install postfix librrd-dev rrdtool libldap2-dev libsasl2-dev libssl-dev + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r ldap-requirements.txt + pip install -r test-requirements.txt + python setup.py develop + - name: Install postgres requirements + if: ${{ matrix.database == 'postgres' }} + run: | + pip install -r postgresql-requirements.txt + pip install coverage + echo "DB=postgres" >> $GITHUB_ENV + - name: Install mysql requirements + if: ${{ matrix.database == 'mysql' }} + run: | + pip install -r mysql-requirements.txt + echo "DB=mysql" >> $GITHUB_ENV + - name: Test with pytest + if: ${{ matrix.python-version != '3.8' || matrix.database != 'postgres' }} + run: | + python ./tests.py + cd test_project + python3 manage.py test modoboa.core modoboa.lib modoboa.admin modoboa.limits modoboa.transport modoboa.relaydomains modoboa.dnstools modoboa.ldapsync modoboa.policyd modoboa.maillog + env: + # use localhost for the host here because we are running the job on the VM. + # If we were running the job on in a container this would be postgres + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} # get randomly assigned published port + MYSQL_HOST: 127.0.0.1 + MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port + MYSQL_USER: root + REDIS_PORT: ${{ job.services.redis.ports[6379] }} + LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }} + + - name: Test with pytest and coverage + if: ${{ matrix.python-version == '3.8' && matrix.database == 'postgres' }} + run: | + python ./tests.py + cd test_project + coverage run manage.py test modoboa.core modoboa.lib modoboa.admin modoboa.limits modoboa.transport modoboa.relaydomains modoboa.dnstools modoboa.ldapsync modoboa.policyd modoboa.maillog + coverage combine + env: + # use localhost for the host here because we are running the job on the VM. + # If we were running the job on in a container this would be postgres + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} # get randomly assigned published port + MYSQL_HOST: 127.0.0.1 + MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} # get randomly assigned published port + MYSQL_USER: root + REDIS_PORT: ${{ job.services.redis.ports[6379] }} + LDAP_SERVER_PORT: ${{ job.services.openldap.ports[389] }} + - name: Upload coverage result + if: ${{ matrix.python-version == '3.8' }} + uses: actions/upload-artifact@v2 + with: + name: coverage-results + path: test_project/.coverage + + coverage: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + pip install codecov + - name: Download coverage results + uses: actions/download-artifact@v2 + with: + name: coverage-results + - name: Report coverage + run: | + coverage report + codecov + + release: + needs: coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + sudo apt-get install librrd-dev rrdtool libssl-dev gettext + python -m pip install --upgrade pip setuptools wheel + pip install -r requirements.txt + cd modoboa + django-admin compilemessages + cd .. + python setup.py sdist bdist_wheel + - name: Publish to Test PyPI + if: endsWith(github.event.ref, '/master') + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.test_pypi_password }} + repository_url: https://test.pypi.org/legacy/ + skip_existing: true + - name: Publish distribution to PyPI + if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_password }} + skip_existing: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 41ae74815..000000000 --- a/.travis.yml +++ /dev/null @@ -1,77 +0,0 @@ -language: python -python: - - '3.5' - - '3.6' - - '3.7' - - '3.8' - -env: - global: - - PYTHONWARNINGS="default,ignore::PendingDeprecationWarning,ignore::ResourceWarning" - matrix: - - DB="POSTGRESQL" - - DB="MYSQL" - -sudo: false - -addons: - postgresql: '9.6' - mysql: '5.5' - apt: - packages: - - ldap-utils - - postfix - - slapd - - librrd-dev - - rrdtool - -cache: pip - -services: - - postgres - - mysql - - redis-server - -before_install: - - pip install codecov - -install: - - pip install -r requirements.txt - - pip install -r ldap-requirements.txt - - pip install -r test-requirements.txt - - if [ $DB = POSTGRESQL ]; then pip install -r postgresql-requirements.txt ; fi - - if [ $DB = MYSQL ]; then pip install -r mysql-requirements.txt ; fi - - python setup.py develop - -before_script: - - mkdir /tmp/slapd - - slapd -f test_data/slapd.conf -h ldap://localhost:3389 & - - sleep 3 - - ldapadd -h localhost:3389 -D cn=admin,dc=example,dc=com -w test -f test_data/base.ldif - -script: - - python ./tests.py - - cd test_project - - coverage run manage.py test modoboa.core modoboa.lib modoboa.admin modoboa.limits modoboa.transport modoboa.relaydomains modoboa.dnstools modoboa.ldapsync modoboa.policyd modoboa.maillog - -after_success: - - coverage combine - - coverage report - - codecov - -before_deploy: - - cd ../modoboa - - django-admin compilemessages - - cd .. - -deploy: - provider: pypi - user: tonio - password: - secure: SKFrH4cri56YQziqMZ4uvbh5Vto9YrQuBlYRnO9BwcjXbcp6jMpZ+9j+aytMdvcN+xG5of6DmgSIiZsLCOl4LDoPn2oPwWMzZA9T8knSVYt9v1uiBTwDmssIt9QH3wD/o3edpTsSoKw+O2cTT2GaE75+3aTK01+CX4ZsUtkGbTk= - skip_cleanup: true - distributions: sdist bdist_wheel - on: - tags: true - python: '3.8' - condition: "$DB = POSTGRESQL" diff --git a/modoboa/core/tests/test_ldap.py b/modoboa/core/tests/test_ldap.py index a1b843319..bc666ccca 100644 --- a/modoboa/core/tests/test_ldap.py +++ b/modoboa/core/tests/test_ldap.py @@ -2,6 +2,7 @@ from unittest import skipIf +from django.conf import settings from django.test import override_settings from django.urls import reverse from django.utils.functional import cached_property @@ -25,7 +26,7 @@ class LDAPTestCaseMixin(object): """Modify settings.""" self.set_global_parameters({ "authentication_type": "ldap", - "ldap_server_port": 3389 + "ldap_server_port": settings.LDAP_SERVER_PORT }) def restore_user_password(self, username, new_password): diff --git a/modoboa/ldapsync/tests.py b/modoboa/ldapsync/tests.py index 398de8603..0429cebdb 100644 --- a/modoboa/ldapsync/tests.py +++ b/modoboa/ldapsync/tests.py @@ -2,6 +2,7 @@ from unittest import skipIf +from django.conf import settings from django.core.management import call_command from django.utils.encoding import force_bytes, force_str @@ -23,7 +24,7 @@ class LDAPExportTestCase(ModoTestCase): super().setUp() self.set_global_parameters({ "ldap_enable_sync": True, - "ldap_server_port": 3389, + "ldap_server_port": settings.LDAP_SERVER_PORT, "ldap_sync_bind_dn": "cn=admin,dc=example,dc=com", "ldap_sync_bind_password": "test", "ldap_sync_account_dn_template": ( @@ -111,7 +112,7 @@ class LDAPImportTestCase(ModoTestCase): super().setUp() self.set_global_parameters({ "ldap_enable_import": True, - "ldap_server_port": 3389, + "ldap_server_port": settings.LDAP_SERVER_PORT, "ldap_sync_bind_dn": "cn=admin,dc=example,dc=com", "ldap_sync_bind_password": "test", "ldap_import_search_base": "ou=users,dc=example,dc=com", diff --git a/modoboa/policyd/tests.py b/modoboa/policyd/tests.py index a53b5b790..1461b7e47 100644 --- a/modoboa/policyd/tests.py +++ b/modoboa/policyd/tests.py @@ -9,6 +9,7 @@ import time import redis +from django import db from django.conf import settings from django.core.management import call_command from django.test import TransactionTestCase @@ -55,6 +56,7 @@ class PolicyDaemonTestCase( self.admin = core_models.User.objects.get(username="user@test2.com") self.admin.role = "SuperAdmins" + db.connections.close_all() patcher = patch("aiosmtplib.send") self.send_mock = patcher.start() self.process = Process(target=start_policy_daemon) diff --git a/modoboa/test_settings.py b/modoboa/test_settings.py index cee94b28b..669959437 100644 --- a/modoboa/test_settings.py +++ b/modoboa/test_settings.py @@ -3,17 +3,17 @@ import os # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases -DB = os.environ.get("DB", "POSTGRESQL") +DB = os.environ.get("DB", "postgres").lower() -if DB == "MYSQL": +if DB == "mysql": DATABASES = { "default": { "ENGINE": "django.db.backends.mysql", "NAME": "modoboa", - "USER": "root", - "PASSWORD": "", - "HOST": "localhost", - "PORT": "", + "USER": os.environ.get("MYSQL_USER", "root"), + "PASSWORD": os.environ.get("MYSQL_PASSWORD", "root"), + "HOST": "127.0.0.1", + "PORT": os.environ.get("MYSQL_PORT", "3306"), "ATOMIC_REQUESTS": True, # MySQL's Strict Mode fixes many data integrity problems in MySQL, # such as data truncation upon insertion, by escalating warnings @@ -27,6 +27,7 @@ if DB == "MYSQL": "SET innodb_strict_mode = ON;" ), "charset": "utf8", + "connect_timeout": 10, }, "TEST": { "CHARSET": "utf8", @@ -34,7 +35,7 @@ if DB == "MYSQL": }, }, } -elif DB == "SQLITE": +elif DB == "sqlite": DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", @@ -49,9 +50,9 @@ else: "ENGINE": "django.db.backends.postgresql", "NAME": "modoboa", "USER": "postgres", - "PASSWORD": "", + "PASSWORD": "postgres", "HOST": "localhost", - "PORT": "", + "PORT": os.environ.get("POSTGRES_PORT", "5432"), "ATOMIC_REQUESTS": True, "OPTIONS": { "client_encoding": "UTF8", diff --git a/setup.py b/setup.py index 19e19ac07..654824d25 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,11 @@ if __name__ == "__main__": with open(path.join(HERE, "README.rst")) as readme: LONG_DESCRIPTION = readme.read() + def local_scheme(version): + """Skip the local version (eg. +xyz of 0.6.1.dev4+gdf99fe2) + to be able to upload to Test PyPI""" + return "" + setup( name="modoboa", description="Mail hosting made simple", @@ -74,7 +79,7 @@ if __name__ == "__main__": zip_safe=False, scripts=["bin/modoboa-admin.py"], install_requires=INSTALL_REQUIRES, - use_scm_version=True, + use_scm_version={"local_scheme": local_scheme}, python_requires=">=3.4", setup_requires=["setuptools_scm"], extras_require={ diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py index c626b04de..bd94c34a7 100644 --- a/test_project/test_project/settings.py +++ b/test_project/test_project/settings.py @@ -196,7 +196,7 @@ MODOBOA_API_URL = 'https://api.modoboa.org/1/' # REDIS REDIS_HOST = 'localhost' -REDIS_PORT = 6379 +REDIS_PORT = os.environ.get('REDIS_PORT', 6379) REDIS_QUOTA_DB = 0 REDIS_URL = 'redis://{}:{}/{}'.format(REDIS_HOST, REDIS_PORT, REDIS_QUOTA_DB) @@ -281,6 +281,9 @@ LOGGING = { }, 'modoboa': { 'class': 'modoboa.core.loggers.SQLHandler', + }, + 'console': { + 'class': 'logging.StreamHandler' } }, 'loggers': { @@ -298,7 +301,11 @@ LOGGING = { 'handlers': ['modoboa'], 'level': 'INFO', 'propagate': False - } + }, + # 'django_auth_ldap': { + # 'level': 'DEBUG', + # 'handlers': ['console'] + # }, } } @@ -308,3 +315,5 @@ SILENCED_SYSTEM_CHECKS = [ DISABLE_DASHBOARD_EXTERNAL_QUERIES = False # Load settings from extensions + +LDAP_SERVER_PORT = os.environ.get('LDAP_SERVER_PORT', 3389) diff --git a/tests.py b/tests.py index d34d331d3..df463c12e 100644 --- a/tests.py +++ b/tests.py @@ -5,22 +5,29 @@ import unittest from modoboa.lib.sysutils import exec_cmd DB = os.environ.get("DB", "postgres") +if DB.lower() == "postgres": + PORT = os.environ.get("POSTGRES_PORT", "5432") + DB = "postgres" +else: + PORT = os.environ.get("MYSQL_PORT", "3306") + DB = "mysql" class DeployTest(unittest.TestCase): - dbtype = DB.lower() + dbtype = DB dbhost = "localhost" + dbport = PORT projname = "modoboa_test" - dbuser = DB == "MYSQL" and "modoboa" or "postgres" - dbpassword = DB == "MYSQL" and "modoboa" or "" + dbuser = DB == "mysql" and "modoboa" or "postgres" + dbpassword = DB == "mysql" and "modoboa" or "" def setUp(self): self.workdir = tempfile.mkdtemp() def test_silent(self): - dburl = "default:%s://%s:%s@%s/%s" \ + dburl = "default:%s://%s:%s@%s:%s/%s" \ % (self.dbtype, self.dbuser, self.dbpassword, - self.dbhost, self.projname) + self.dbhost, self.dbport, self.projname) cmd = ( "modoboa-admin.py deploy --collectstatic " "--dburl %s --domain %s --admin-username admin %s"