From 80216c5fdc9fbb73c60ff24429aab73dfeaf6556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sara=20Aim=C3=A9e=20Smiseth?= <51710585+SaraSmiseth@users.noreply.github.com> Date: Fri, 7 May 2021 14:06:29 +0200 Subject: [PATCH] Add support for postgres, mariadb/mysql databases (#23) * Add environment variables for Database configuration. * Install postgres and mysql dependencies. * Test refactoring * Run tests for prosody with postgres and also for prosody with sqlite. * Add tests that check if postgres or sqlite is used. --- CHANGELOG.md | 12 ++++ Dockerfile | 10 ++- conf.d/02-storage.cfg.lua | 9 ++- data/.gitkeep | 0 ...er-entrypoint.sh => docker-entrypoint.bash | 2 + readme.md | 6 ++ tests/bats/bats-assert | 2 +- tests/bats/bats-core | 2 +- tests/docker-compose.yml | 39 +++++++++-- tests/requirements.txt | 6 +- tests/test.bash | 64 +++++++++++++------ tests/tests-prosody.bats | 10 +++ tests/tests-prosody_postgres.bats | 10 +++ tests/tests.bats | 30 ++++----- 14 files changed, 153 insertions(+), 49 deletions(-) delete mode 100644 data/.gitkeep rename docker-entrypoint.sh => docker-entrypoint.bash (94%) create mode 100644 tests/tests-prosody.bats create mode 100644 tests/tests-prosody_postgres.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 77049b1..09f0fb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,22 @@ ## Unreleased +* Nothing + +## v1.2.0 + +### New features + +* New environment variables for database settings. It is now possible to use MariaDB or Postgres instead of SQLite. SQLite is the default. See [README](https://github.com/SaraSmiseth/prosody#environment-variables). + +### Updates + * Updated luarocks to version 3.7.0. ## v1.1.4 +### Updates + * Updated to Prosody version [0.11.8](https://blog.prosody.im/prosody-0.11.8-released/). * Updated luarocks to version 3.5.0. diff --git a/Dockerfile b/Dockerfile index 6c7c2ce..54d01e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,13 +27,15 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y \ libevent-dev `# this is no build dependency, but needed for luaevent` \ libidn11 \ + libpq-dev \ + libsqlite3-0 \ lua5.2 \ lua-bitop \ + lua-dbi-mysql \ lua-expat \ lua-filesystem \ lua-socket \ lua-sec \ - sqlite3 \ wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -65,6 +67,8 @@ RUN buildDeps='gcc git libc6-dev libidn11-dev liblua5.2-dev libsqlite3-dev libss \ && luarocks install luaevent \ && luarocks install luadbi \ + `#&& luarocks install luadbi-mysql MYSQL_INCDIR=/usr/include/mariadb/` \ + && luarocks install luadbi-postgresql POSTGRES_INCDIR=/usr/include/postgresql/ \ && luarocks install luadbi-sqlite3 \ && luarocks install stringy \ \ @@ -85,7 +89,7 @@ ENV __FLUSH_LOG yes VOLUME ["/usr/local/var/lib/prosody"] COPY prosody.cfg.lua /usr/local/etc/prosody/prosody.cfg.lua -COPY docker-entrypoint.sh /entrypoint.sh +COPY docker-entrypoint.bash /entrypoint.bash COPY conf.d/*.cfg.lua /usr/local/etc/prosody/conf.d/ COPY *.bash /usr/local/bin/ @@ -106,6 +110,6 @@ RUN download-prosody-modules.bash \ USER prosody -ENTRYPOINT ["/entrypoint.sh"] +ENTRYPOINT ["/entrypoint.bash"] CMD ["prosody", "-F"] diff --git a/conf.d/02-storage.cfg.lua b/conf.d/02-storage.cfg.lua index b8cb6d7..549573a 100644 --- a/conf.d/02-storage.cfg.lua +++ b/conf.d/02-storage.cfg.lua @@ -1,7 +1,12 @@ default_storage = "sql" + sql = { - driver = "SQLite3"; - database = "prosody.sqlite"; + driver = os.getenv("DB_DRIVER"); + database = os.getenv("DB_DATABASE"); + host = os.getenv("DB_HOST"); + port = os.getenv("DB_PORT"); + username = os.getenv("DB_USERNAME"); + password = os.getenv("DB_PASSWORD"); } -- make 0.10-distributed mod_mam use sql store diff --git a/data/.gitkeep b/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docker-entrypoint.sh b/docker-entrypoint.bash similarity index 94% rename from docker-entrypoint.sh rename to docker-entrypoint.bash index a7090d3..dd317e0 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.bash @@ -6,6 +6,8 @@ export DOMAIN_HTTP_UPLOAD=${DOMAIN_HTTP_UPLOAD:-"upload.$DOMAIN"} export DOMAIN_MUC=${DOMAIN_MUC:-"conference.$DOMAIN"} export DOMAIN_PROXY=${DOMAIN_PROXY:-"proxy.$DOMAIN"} export DOMAIN_PUBSUB=${DOMAIN_PUBSUB:-"pubsub.$DOMAIN"} +export DB_DRIVER=${DB_DRIVER:-"SQLite3"} +export DB_DATABASE=${DB_DATABASE:-"prosody.sqlite"} export E2E_POLICY_CHAT=${E2E_POLICY_CHAT:-"required"} export E2E_POLICY_MUC=${E2E_POLICY_MUC:-"required"} export E2E_POLICY_WHITELIST=${E2E_POLICY_WHITELIST:-""} diff --git a/readme.md b/readme.md index cc4d04d..55c93ad 100644 --- a/readme.md +++ b/readme.md @@ -194,6 +194,12 @@ Inspect logs: ```docker-compose logs -f```. | **DOMAIN_MUC** | Domain for Multi-user chat (MUC) for allowing you to create hosted chatrooms/conferences for XMPP users | *optional* | conference.**DOMAIN** | | **DOMAIN_PROXY** | Domain for SOCKS5 bytestream proxy for server-proxied file transfers | *optional* | proxy.**DOMAIN** | | **DOMAIN_PUBSUB** | Domain for a XEP-0060 pubsub service | *optional* | pubsub.**DOMAIN** | +| **DB_DRIVER** | May also be "PostgreSQL" or "MySQL" or "SQLite3" (case sensitive!) | *optional* | SQLite3 | +| **DB_DATABASE** | The database name to use. For SQLite3 this the database filename (relative to the data storage directory). | *optional* | prosody.sqlite | +| **DB_HOST** | The address of the database server | *optional* | | +| **DB_PORT** | Port on which the database is listening | *optional* | | +| **DB_USERNAME** | The username to authenticate to the database | *optional* | | +| **DB_PASSWORD** | The password to authenticate to the database | *optional* | | | **E2E_POLICY_CHAT** | Policy for chat messages. Possible values: "none", "optional" and "required". | *optional* | "required" | | **E2E_POLICY_MUC** | Policy for MUC messages. Possible values: "none", "optional" and "required". | *optional* | "required" | | **E2E_POLICY_WHITELIST** | Make this module ignore messages sent to and from this JIDs or MUCs. | *optional* | "" | diff --git a/tests/bats/bats-assert b/tests/bats/bats-assert index 0a8dd57..e0de84e 160000 --- a/tests/bats/bats-assert +++ b/tests/bats/bats-assert @@ -1 +1 @@ -Subproject commit 0a8dd57e2cc6d4cc064b1ed6b4e79b9f7fee096f +Subproject commit e0de84e9c011223e7f88b7ccf1c929f4327097ba diff --git a/tests/bats/bats-core b/tests/bats/bats-core index 8fb853a..49b377a 160000 --- a/tests/bats/bats-core +++ b/tests/bats/bats-core @@ -1 +1 @@ -Subproject commit 8fb853a6cbc0169958707381985f3cd59789ccb1 +Subproject commit 49b377a751e6f9379abfdfb3dfa3aafabd8495a1 diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index c04c41b..8c59053 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -15,10 +15,39 @@ services: E2E_POLICY_WHITELIST: "admin@localhost, user1@localhost" LOG_LEVEL: debug PROSODY_ADMINS: "admin@localhost, admin2@localhost" - extra_hosts: - - "conference.localhost:127.0.0.1" - - "pubsub.localhost:127.0.0.1" - - "proxy.localhost:127.0.0.1" - - "upload.localhost:127.0.0.1" volumes: - ./certs:/usr/local/etc/prosody/certs + + prosody_postgres: + image: prosody + restart: unless-stopped + ports: + - "5000:5000" + - "5222:5222" + - "5223:5223" + - "5269:5269" + - "5281:5281" + environment: + DOMAIN: localhost + E2E_POLICY_WHITELIST: "admin@localhost, user1@localhost" + LOG_LEVEL: debug + PROSODY_ADMINS: "admin@localhost, admin2@localhost" + #DB_DRIVER: "MySQL" + DB_DRIVER: "PostgreSQL" + DB_DATABASE: "prosody" + DB_HOST: "postgres" + DB_PORT: "5432" + DB_USERNAME: "prosody" + DB_PASSWORD: "prosody" + volumes: + - ./certs:/usr/local/etc/prosody/certs + depends_on: + - postgres + + postgres: + image: postgres:13-alpine + restart: unless-stopped + environment: + POSTGRES_DB: prosody + POSTGRES_USER: prosody + POSTGRES_PASSWORD: prosody diff --git a/tests/requirements.txt b/tests/requirements.txt index bcd35ba..7bb2363 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,3 @@ -aioxmpp==0.11.0 -pip-chill==1.0.0 -pytest-asyncio==0.14.0 +aioxmpp==0.12.2 +pip-chill==1.0.1 +pytest-asyncio==0.15.1 diff --git a/tests/test.bash b/tests/test.bash index 75048cd..7b1e436 100755 --- a/tests/test.bash +++ b/tests/test.bash @@ -15,31 +15,57 @@ generateCert() { fi } +registerTestUser() { + local userName="$1" + local containerName="$2" + sudo docker exec "$containerName" /bin/bash -c "/entrypoint.bash register $userName localhost 12345678" +} + +registerTestUsers() { + local containerName="$1" + registerTestUser admin "$containerName" + registerTestUser user1 "$containerName" + registerTestUser user2 "$containerName" + registerTestUser user3 "$containerName" +} + +runTests() { + local containerName="$1" + python --version \ + && python3 --version \ + && python3 -m venv venv \ + && source venv/bin/activate \ + && python --version \ + && pip --version \ + && pip install -r requirements.txt \ + && pytest \ + && deactivate \ + && sleep 5 \ + && sudo docker-compose logs "$containerName" \ + && export batsContainerName="$containerName" \ + && ./bats/bats-core/bin/bats tests.bats \ + && ./bats/bats-core/bin/bats tests-"$containerName".bats +} + generateCert "localhost" generateCert "conference.localhost" generateCert "proxy.localhost" generateCert "pubsub.localhost" generateCert "upload.localhost" +# Run tests for first container with postgres +# Start postgres first and wait for 10 seconds before starting prosody. sudo docker-compose down \ -&& sudo docker-compose up -d \ -\ -&& sudo docker exec tests_prosody_1 /bin/bash -c "/entrypoint.sh register admin localhost 12345678" \ -&& sudo docker exec tests_prosody_1 /bin/bash -c "/entrypoint.sh register user1 localhost 12345678" \ -&& sudo docker exec tests_prosody_1 /bin/bash -c "/entrypoint.sh register user2 localhost 12345678" \ -&& sudo docker exec tests_prosody_1 /bin/bash -c "/entrypoint.sh register user3 localhost 12345678" \ -\ -&& python --version \ -&& python3 --version \ -&& python3 -m venv venv \ -&& source venv/bin/activate \ -&& python --version \ -&& pip --version \ -&& pip install -r requirements.txt \ -&& pytest \ -&& deactivate \ -&& sleep 5 \ -&& sudo docker-compose logs \ -&& ./bats/bats-core/bin/bats tests.bats +&& sudo docker-compose up -d postgres \ +&& sleep 10 \ +&& sudo docker-compose up -d prosody_postgres +registerTestUsers tests_prosody_postgres_1 +runTests prosody_postgres +sudo docker-compose down + +# Run tests for second container with SQLite +sudo docker-compose up -d prosody +registerTestUsers tests_prosody_1 +runTests prosody sudo docker-compose down diff --git a/tests/tests-prosody.bats b/tests/tests-prosody.bats new file mode 100644 index 0000000..74ddc22 --- /dev/null +++ b/tests/tests-prosody.bats @@ -0,0 +1,10 @@ +# For tests with pipes see: https://github.com/sstephenson/bats/issues/10 + +load 'bats/bats-support/load' +load 'bats/bats-assert/load' + +@test "Should use sqlite" { + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Connecting to \[SQLite3\] \/usr\/local\/var\/lib\/prosody\/prosody\.sqlite\.\.\.\"" + assert_success + assert_output +} diff --git a/tests/tests-prosody_postgres.bats b/tests/tests-prosody_postgres.bats new file mode 100644 index 0000000..2a1d1d0 --- /dev/null +++ b/tests/tests-prosody_postgres.bats @@ -0,0 +1,10 @@ +# For tests with pipes see: https://github.com/sstephenson/bats/issues/10 + +load 'bats/bats-support/load' +load 'bats/bats-assert/load' + +@test "Should use postgres" { + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Connecting to \[PostgreSQL\] prosody\.\.\.\"" + assert_success + assert_output +} diff --git a/tests/tests.bats b/tests/tests.bats index f9f8df2..31dac1e 100644 --- a/tests/tests.bats +++ b/tests/tests.bats @@ -5,91 +5,91 @@ load 'bats/bats-assert/load' # group alternation in regex because the xml properties switch around. sometimes 'type=...' comes after 'to=...' and sometimes before @test "Should send 5 messages" { - run bash -c "sudo docker-compose logs | grep -E \"Received\[c2s\]: \" | wc -l" + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Received\[c2s\]: \" | wc -l" assert_success assert_output "5" } @test "Should select certificate for localhost" { - run bash -c "sudo docker-compose logs | grep \"Selecting certificate /usr/local/etc/prosody/certs/localhost/fullchain.pem with key /usr/local/etc/prosody/certs/localhost/privkey.pem for localhost\" | wc -l" + run bash -c "sudo docker-compose logs $batsContainerName | grep \"Selecting certificate /usr/local/etc/prosody/certs/localhost/fullchain.pem with key /usr/local/etc/prosody/certs/localhost/privkey.pem for localhost\" | wc -l" assert_success assert_output "3" } @test "Should select certificate for conference.localhost" { - run bash -c "sudo docker-compose logs | grep \"Selecting certificate /usr/local/etc/prosody/certs/conference.localhost/fullchain.pem with key /usr/local/etc/prosody/certs/conference.localhost/privkey.pem for conference.localhost\" | wc -l" + run bash -c "sudo docker-compose logs $batsContainerName | grep \"Selecting certificate /usr/local/etc/prosody/certs/conference.localhost/fullchain.pem with key /usr/local/etc/prosody/certs/conference.localhost/privkey.pem for conference.localhost\" | wc -l" assert_success assert_output "3" } @test "Should select certificate for proxy.localhost" { - run bash -c "sudo docker-compose logs | grep \"Selecting certificate /usr/local/etc/prosody/certs/proxy.localhost/fullchain.pem with key /usr/local/etc/prosody/certs/proxy.localhost/privkey.pem for proxy.localhost\" | wc -l" + run bash -c "sudo docker-compose logs $batsContainerName | grep \"Selecting certificate /usr/local/etc/prosody/certs/proxy.localhost/fullchain.pem with key /usr/local/etc/prosody/certs/proxy.localhost/privkey.pem for proxy.localhost\" | wc -l" assert_success assert_output "3" } @test "Should select certificate for pubsub.localhost" { - run bash -c "sudo docker-compose logs | grep \"Selecting certificate /usr/local/etc/prosody/certs/pubsub.localhost/fullchain.pem with key /usr/local/etc/prosody/certs/pubsub.localhost/privkey.pem for pubsub.localhost\" | wc -l" + run bash -c "sudo docker-compose logs $batsContainerName | grep \"Selecting certificate /usr/local/etc/prosody/certs/pubsub.localhost/fullchain.pem with key /usr/local/etc/prosody/certs/pubsub.localhost/privkey.pem for pubsub.localhost\" | wc -l" assert_success assert_output "3" } @test "Should select certificate for upload.localhost" { - run bash -c "sudo docker-compose logs | grep \"Selecting certificate /usr/local/etc/prosody/certs/upload.localhost/fullchain.pem with key /usr/local/etc/prosody/certs/upload.localhost/privkey.pem for upload.localhost\" | wc -l" + run bash -c "sudo docker-compose logs $batsContainerName | grep \"Selecting certificate /usr/local/etc/prosody/certs/upload.localhost/fullchain.pem with key /usr/local/etc/prosody/certs/upload.localhost/privkey.pem for upload.localhost\" | wc -l" assert_success assert_output "3" } @test "Should log error for user with wrong password" { - run bash -c "sudo docker-compose logs | grep \"Session closed by remote with error: undefined-condition (user intervention: authentication failed: authentication aborted by user)\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep \"Session closed by remote with error: undefined-condition (user intervention: authentication failed: authentication aborted by user)\"" assert_success assert_output } @test "Should activate s2s" { - run bash -c "sudo docker-compose logs | grep -E \"Activated service 's2s' on (\[::\]:5269|\[\*\]:5269), (\[::\]:5269|\[\*\]:5269)\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Activated service 's2s' on (\[::\]:5269|\[\*\]:5269), (\[::\]:5269|\[\*\]:5269)\"" assert_success assert_output } @test "Should activate c2s" { - run bash -c "sudo docker-compose logs | grep -E \"Activated service 'c2s' on (\[::\]:5222|\[\*\]:5222), (\[::\]:5222|\[\*\]:5222)\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Activated service 'c2s' on (\[::\]:5222|\[\*\]:5222), (\[::\]:5222|\[\*\]:5222)\"" assert_success assert_output } @test "Should activate legacy_ssl" { - run bash -c "sudo docker-compose logs | grep -E \"Activated service 'legacy_ssl' on (\[::\]:5223|\[\*\]:5223), (\[::\]:5223|\[\*\]:5223)\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Activated service 'legacy_ssl' on (\[::\]:5223|\[\*\]:5223), (\[::\]:5223|\[\*\]:5223)\"" assert_success assert_output } @test "Should activate proxy65" { - run bash -c "sudo docker-compose logs | grep -E \"Activated service 'proxy65' on (\[::\]:5000|\[\*\]:5000), (\[::\]:5000|\[\*\]:5000)\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Activated service 'proxy65' on (\[::\]:5000|\[\*\]:5000), (\[::\]:5000|\[\*\]:5000)\"" assert_success assert_output } @test "Should activate http" { - run bash -c "sudo docker-compose logs | grep -E \"Activated service 'http' on (\[::\]:5280|\[\*\]:5280), (\[::\]:5280|\[\*\]:5280)\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Activated service 'http' on (\[::\]:5280|\[\*\]:5280), (\[::\]:5280|\[\*\]:5280)\"" assert_success assert_output } @test "Should activate https" { - run bash -c "sudo docker-compose logs | grep -E \"Activated service 'https' on (\[::\]:5281|\[\*\]:5281), (\[::\]:5281|\[\*\]:5281)\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep -E \"Activated service 'https' on (\[::\]:5281|\[\*\]:5281), (\[::\]:5281|\[\*\]:5281)\"" assert_success assert_output } @test "Should load module cloud_notify" { - run bash -c "sudo docker-compose logs | grep \"localhost:cloud_notify.*info.*Module loaded\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep \"localhost:cloud_notify.*info.*Module loaded\"" assert_success assert_output } @test "Should show upload URL" { - run bash -c "sudo docker-compose logs | grep \"URL: - Ensure this can be reached by users\"" + run bash -c "sudo docker-compose logs $batsContainerName | grep \"URL: - Ensure this can be reached by users\"" assert_success assert_output }