diff --git a/.github/workflows/dockerfile.yaml b/.github/workflows/dockerfile.yaml index 8f694c3882..9a51d75cc5 100644 --- a/.github/workflows/dockerfile.yaml +++ b/.github/workflows/dockerfile.yaml @@ -5,6 +5,7 @@ on: pull_request: branches: - main + - next jobs: lint: @@ -155,13 +156,29 @@ jobs: kubectl -n "$namespace" logs "$pod" kubectl -n "$namespace" exec "$pod" -c zulip -- cat /var/log/zulip/errors.log + docker-compose-collect: + runs-on: ubuntu-latest + outputs: + dirs: ${{ steps.dirs.outputs.dirs }} + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - id: dirs + run: echo "dirs=$(ls -d ci/*/ | jq -Rnc '[inputs]')" >> ${GITHUB_OUTPUT} + docker-compose-test: runs-on: ubuntu-latest timeout-minutes: 10 needs: - build + - docker-compose-collect env: GITHUB_CI_IMAGE: ghcr.io/${{ github.repository }}:pr-${{ github.event.pull_request.number }} + strategy: + fail-fast: false + matrix: + dir: ${{ fromJson(needs.docker-compose-collect.outputs.dirs) }} steps: - name: Checkout code uses: actions/checkout@v5 @@ -169,12 +186,14 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Verify Docker Compose config validation + - name: Verify Docker Compose config run: | docker compose \ -f compose.yaml \ - -f ci/compose.override.yaml \ - --env-file ci/env \ + -f ci/base.yaml \ + --env-file ci/base.env \ + -f ${{ matrix.dir }}/compose.yaml \ + --env-file ${{ matrix.dir }}/env \ config - name: Log in to GHCR @@ -188,8 +207,10 @@ jobs: run: | docker compose \ -f compose.yaml \ - -f ci/compose.override.yaml \ - --env-file ci/env \ + -f ci/base.yaml \ + --env-file ci/base.env \ + -f ${{ matrix.dir }}/compose.yaml \ + --env-file ${{ matrix.dir }}/env \ up -d --no-build - name: Wait for services to be healthy @@ -206,6 +227,19 @@ jobs: exit 1 fi + - name: Run tests + run: | + docker=("docker" "compose" \ + "-f" "compose.yaml" \ + "-f" "ci/base.yaml" \ + "--env-file" "ci/base.env" \ + "-f" "${{ matrix.dir }}/compose.yaml" \ + "--env-file" "${{ matrix.dir }}/env") + manage=("${docker[@]}" "exec" "-u" "zulip" "zulip" + "/home/zulip/deployments/current/manage.py") + hostname="localhost" + source "${{ matrix.dir }}/test.sh" + - name: Check service logs for critical errors if: success() || failure() continue-on-error: true diff --git a/ci/env b/ci/base.env similarity index 100% rename from ci/env rename to ci/base.env diff --git a/ci/compose.override.yaml b/ci/base.yaml similarity index 96% rename from ci/compose.override.yaml rename to ci/base.yaml index 90aff181d0..b42c34144a 100644 --- a/ci/compose.override.yaml +++ b/ci/base.yaml @@ -1,4 +1,6 @@ --- +name: docker-zulip + secrets: zulip__postgres_password: environment: "ZULIP__POSTGRES_PASSWORD" diff --git a/ci/basic/compose.yaml b/ci/basic/compose.yaml new file mode 100644 index 0000000000..b9d3f609f7 --- /dev/null +++ b/ci/basic/compose.yaml @@ -0,0 +1,2 @@ +--- +name: docker-zulip diff --git a/ci/basic/env b/ci/basic/env new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci/basic/test.sh b/ci/basic/test.sh new file mode 100755 index 0000000000..2cabdba1e0 --- /dev/null +++ b/ci/basic/test.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -eux +set -o pipefail + +url="https://${hostname:?}" + +# Starts off as a 404 until a realm exists +curl --insecure -si "$url" | grep -Ei '^HTTP/\S+ 404' + +"${manage[@]:?}" create_realm 'Testing Realm' admin@example.com 'Test Admin' --password very-secret + +# Realm exists after creation +curl --insecure -sfi "$url" + +# HTTP redirects to HTTPS +curl -si "http://$hostname" 2>&1 | grep -i "location: $url" + +# / redirects to /login/ +curl --insecure -sfi "$url" 2>&1 | grep -i "location: /login/" + +# Login page has the name of the realm +curl --insecure -sfL "$url" | grep "Testing Realm" + +# Authenticate +api_key=$(curl --insecure -sSX POST "$url/api/v1/fetch_api_key" \ + --data-urlencode username=admin@example.com \ + --data-urlencode password=very-secret | jq -r .api_key) + +# Make a queue +registered=$(curl --insecure -sfSX POST "$url/api/v1/register" \ + -u "admin@example.com:$api_key" \ + --data-urlencode 'event_types=["message"]') + +queue_id=$(echo "$registered" | jq -r .queue_id) +last_event_id=$(echo "$registered" | jq -r .last_event_id) + +# Post a message +curl --insecure -sfSX POST "$url/api/v1/messages" \ + -u "admin@example.com:$api_key" \ + --data-urlencode type=stream \ + --data-urlencode 'to="general"' \ + --data-urlencode topic=end-to-end \ + --data-urlencode 'content=This is a piping hot test message.' + +# See the message come back over the event queue +queue=$(curl --insecure -sfSX GET -G "$url/api/v1/events" \ + -u "admin@example.com:$api_key" \ + --data-urlencode "queue_id=$queue_id" \ + --data-urlencode "last_event_id=$last_event_id") + +echo "$queue" | jq -r '.events[] | .message.content' | grep "This is a piping hot test message." + +exit 0 diff --git a/ci/certbot/compose.yaml b/ci/certbot/compose.yaml new file mode 100644 index 0000000000..366ea3b173 --- /dev/null +++ b/ci/certbot/compose.yaml @@ -0,0 +1,44 @@ +--- +services: + zulip: + environment: + SSL_CERTIFICATE_GENERATION: certbot + networks: + zulip-backend: + ipv4_address: 172.28.5.100 + depends_on: + - pebble + - challtestsrv + + database: + networks: [zulip-backend] + memcached: + networks: [zulip-backend] + rabbitmq: + networks: [zulip-backend] + redis: + networks: [zulip-backend] + + pebble: + image: ghcr.io/letsencrypt/pebble:latest + volumes: + - ./ci/certbot/pebble-config/:/config/ + command: -config /config/pebble-config.json -strict -dnsserver challtestsrv:8053 + ports: + - 14000:14000 # HTTPS ACME API + - 15000:15000 # HTTPS Management API + networks: [zulip-backend] + challtestsrv: + image: ghcr.io/letsencrypt/pebble-challtestsrv:latest + command: -defaultIPv6 "" -defaultIPv4 172.28.5.100 + networks: [zulip-backend] + +networks: + zulip-backend: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.28.0.0/16 + ip_range: 172.28.5.0/24 + gateway: 172.28.5.254 diff --git a/ci/certbot/env b/ci/certbot/env new file mode 120000 index 0000000000..348b25de74 --- /dev/null +++ b/ci/certbot/env @@ -0,0 +1 @@ +../basic/env \ No newline at end of file diff --git a/ci/certbot/pebble-config/pebble-config.json b/ci/certbot/pebble-config/pebble-config.json new file mode 100644 index 0000000000..b2c0e56a1d --- /dev/null +++ b/ci/certbot/pebble-config/pebble-config.json @@ -0,0 +1,28 @@ +{ + "pebble": { + "listenAddress": "0.0.0.0:14000", + "managementListenAddress": "0.0.0.0:15000", + "certificate": "test/certs/localhost/cert.pem", + "privateKey": "test/certs/localhost/key.pem", + "httpPort": 80, + "tlsPort": 443, + "ocspResponderURL": "", + "externalAccountBindingRequired": false, + "domainBlocklist": ["blocked-domain.example"], + "retryAfter": { + "authz": 3, + "order": 5 + }, + "keyAlgorithm": "ecdsa", + "profiles": { + "default": { + "description": "The profile you know and love", + "validityPeriod": 7776000 + }, + "shortlived": { + "description": "A short-lived cert profile, without actual enforcement", + "validityPeriod": 518400 + } + } + } +} diff --git a/ci/certbot/test.sh b/ci/certbot/test.sh new file mode 100644 index 0000000000..2015c11b57 --- /dev/null +++ b/ci/certbot/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eux +set -o pipefail + +url="https://${hostname:?}" + +curl --verbose --insecure "${url}" diff --git a/ci/http-only/compose.yaml b/ci/http-only/compose.yaml new file mode 100644 index 0000000000..31e20bfa51 --- /dev/null +++ b/ci/http-only/compose.yaml @@ -0,0 +1,25 @@ +--- +services: + zulip: + environment: + DISABLE_HTTPS: True + networks: [zulip-backend] + + database: + networks: [zulip-backend] + memcached: + networks: [zulip-backend] + rabbitmq: + networks: [zulip-backend] + redis: + networks: [zulip-backend] + +networks: + zulip-backend: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.28.0.0/16 + ip_range: 172.28.5.0/24 + gateway: 172.28.5.254 diff --git a/ci/http-only/env b/ci/http-only/env new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci/http-only/test.sh b/ci/http-only/test.sh new file mode 100755 index 0000000000..df0d223b24 --- /dev/null +++ b/ci/http-only/test.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -eux +set -o pipefail + +url="http://${hostname:?}" + +# This is a server error, which describes the need to set LOADBALANCER_IPS +error_page=$(curl -sSi "$url") +echo "$error_page" | grep -Ei "HTTP/\S+ 500" +echo "$error_page" | grep "You have not configured any reverse proxies" +echo "$error_page" | grep "LOADBALANCER_IPS" + +# This is a server error, which notes the reverse proxy exists +error_page=$(curl -H "X-Forwarded-For: 1.2.3.4" -sSi "$url") +echo "$error_page" | grep -Ei "HTTP/\S+ 500" +echo "$error_page" | grep "You have not configured any reverse proxies" +echo "$error_page" | grep "reverse proxy headers were detected" +echo "$error_page" | grep "LOADBALANCER_IPS" + +# Restart with LOADBALANCER_IPS set +"${docker[@]:?}" -f ci/http-only/with-loadbalancer-ips.yaml up -d --no-build --force-recreate zulip + +# Wait for it to come back up +instance=$("${docker[@]}" ps -q zulip) +timeout 300 bash -c \ + "until docker inspect --format '{{.State.Health.Status}}' '$instance' | grep -q healthy; do sleep 5; done" + +# This is a server error, which notes the lack of X-Forwarded-Proto +error_page=$(curl -H "X-Forwarded-For: 1.2.3.4" -sSi "$url") +echo "$error_page" | grep -Ei "HTTP/\S+ 500" +echo "$error_page" | grep "You have configured reverse proxies" +echo "$error_page" | grep "X-Forwarded-Proto" + +# This is a 404 due to no realm existing +error_page=$(curl -H "X-Forwarded-For: 1.2.3.4" -H "X-Forwarded-Proto: https" -sSi "$url") +echo "$error_page" | grep -Ei "HTTP/\S+ 404" + +"${manage[@]:?}" create_realm 'Testing Realm' admin@example.com 'Test Admin' --password very-secret + +success=$(curl -H "X-Forwarded-For: 1.2.3.4" -H "X-Forwarded-Proto: https" -sfLSi "$url") +echo "$success" | grep "Testing Realm" + +exit 0 diff --git a/ci/http-only/with-loadbalancer-ips.yaml b/ci/http-only/with-loadbalancer-ips.yaml new file mode 100644 index 0000000000..59110ae873 --- /dev/null +++ b/ci/http-only/with-loadbalancer-ips.yaml @@ -0,0 +1,7 @@ +--- +services: + zulip: + environment: + # This is the network's gateway address, set in compose.yaml, and the + # Docker Desktop IP VM's address (for local testing). + LOADBALANCER_IPS: 172.28.5.254,192.168.65.1