From abde31148fde51858e728e566a25912a5740547e Mon Sep 17 00:00:00 2001 From: Elias Elwyn Date: Tue, 18 Jun 2024 12:31:15 +0000 Subject: [PATCH] Integration test for redis alternatives - garnet, redict (#4138) For #4082. ~~Per the discussion in the issue, the current plan will likely involve duplicating the redis library calling code once for each cacher, as neither garnet nor redict guarantee continued compatibility with redis.~~ See discussion below for details. ## Tasklist - [x] Write workflow to run cache-specific unit test(s) only (cache, session, queue, nosql) for each cacher - [x] Check whether garnet and redict pass unit tests with no code modification (gauge required work) - both passed, but that is because there were very few tests that test the remote cache store ### Out of scope for this PR - Improve test coverage - `modules/cache` against a server - `modules/session` against a server (also needs tests in general) - _(?) Duplicate implementation for each cacher_ - _Restructure redis usage in `modules/cache` and `modules/settings/cache`_ - _Restructure `modules/session` and its settings_ - _Restructure `modules/queue` and its settings_ - _Restructure `modules/nosql` and its settings_ Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4138 Reviewed-by: Earl Warren Co-authored-by: Elias Elwyn Co-committed-by: Elias Elwyn --- .forgejo/workflows/testing.yml | 55 ++++++++++++++++++++++++++++++++-- Makefile | 8 +++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 2157175005..a9edacad87 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -57,8 +57,6 @@ jobs: MINIO_DOMAIN: minio MINIO_ROOT_USER: 123456 MINIO_ROOT_PASSWORD: 12345678 - redis: - image: redis:7.2 steps: - uses: https://code.forgejo.org/actions/checkout@v3 - uses: https://code.forgejo.org/actions/setup-go@v4 @@ -88,8 +86,59 @@ jobs: env: RACE_ENABLED: 'true' TAGS: bindata - TEST_REDIS_SERVER: redis:6379 TEST_ELASTICSEARCH_URL: http://elasticsearch:9200 + test-remote-cacher: + if: ${{ !startsWith(vars.ROLE, 'forgejo-') }} + runs-on: docker + needs: [backend-checks, frontend-checks] + container: + image: 'docker.io/node:20-bookworm' + strategy: + matrix: + cacher: + # redis + - image: redis:7.2 + port: 6379 + # redict + - image: registry.redict.io/redict:7.3.0-scratch + port: 6379 + # garnet + - image: ghcr.io/microsoft/garnet-alpine:1.0.14 + port: 6379 + services: + cacher: + image: ${{ matrix.cacher.image }} + options: ${{ matrix.cacher.options }} + steps: + - uses: https://code.forgejo.org/actions/checkout@v3 + - uses: https://code.forgejo.org/actions/setup-go@v4 + with: + go-version: "1.22" + - run: | + git config --add safe.directory '*' + adduser --quiet --comment forgejo --disabled-password forgejo + chown -R forgejo:forgejo . + - name: install git >= 2.42 + run: | + export DEBIAN_FRONTEND=noninteractive + echo deb http://deb.debian.org/debian/ testing main > /etc/apt/sources.list.d/testing.list + apt-get update -qq + apt-get -q install -qq -y git + rm /etc/apt/sources.list.d/testing.list + apt-get update -qq + - run: | + su forgejo -c 'make deps-backend' + - run: | + su forgejo -c 'make backend' + env: + TAGS: bindata + - run: | + su forgejo -c 'make test-remote-cacher test-check' + timeout-minutes: 50 + env: + RACE_ENABLED: 'true' + TAGS: bindata + TEST_REDIS_SERVER: cacher:${{ matrix.cacher.port }} test-mysql: if: ${{ !startsWith(vars.ROLE, 'forgejo-') }} runs-on: docker diff --git a/Makefile b/Makefile index 41b051cf9b..b5bca83cdb 100644 --- a/Makefile +++ b/Makefile @@ -124,6 +124,8 @@ LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64 ifeq ($(HAS_GO), yes) GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) $(shell $(GO) list code.gitea.io/gitea/models/forgejo_migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/)) endif +REMOTE_CACHER_MODULES ?= cache nosql session queue +GO_TEST_REMOTE_CACHER_PACKAGES ?= $(addprefix code.gitea.io/gitea/modules/,$(REMOTE_CACHER_MODULES)) FOMANTIC_WORK_DIR := web_src/fomantic @@ -248,6 +250,7 @@ help: @echo " - show-version-major show major release number only" @echo " - test-frontend test frontend files" @echo " - test-backend test backend files" + @echo " - test-remote-cacher test backend files that use a remote cache" @echo " - test-e2e-sqlite[\#name.test.e2e] test end to end using playwright and sqlite" @echo " - webpack build webpack files" @echo " - svg build svg files" @@ -509,6 +512,11 @@ test-backend: @echo "Running go test with $(GOTESTFLAGS) -tags '$(TEST_TAGS)'..." @$(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' $(GO_TEST_PACKAGES) +.PHONY: test-remote-cacher +test-remote-cacher: + @echo "Running go test with $(GOTESTFLAGS) -tags '$(TEST_TAGS)'..." + @$(GO) test $(GOTESTFLAGS) -tags='$(TEST_TAGS)' $(GO_TEST_REMOTE_CACHER_PACKAGES) + .PHONY: test-frontend test-frontend: node_modules npx vitest