Use 3-node cluster in test suite

.github/workflows/test-rabbitmq-alphas.yml

@@ -43,19 +43,19 @@ jobs:
         run: make deps
       - name: Test with NIO
         run: |
-          ./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
-            -Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
+          ./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
+            -Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
+            -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
+            -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
             -Dmaven.javadoc.skip=true \
-            -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
             --no-transfer-progress
       - name: Test with blocking IO
         run: |
-          ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
-          -Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
-          -Dmaven.javadoc.skip=true \
-          -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
-          --no-transfer-progress
-      - name: Stop broker A
-        run: docker stop rabbitmq && docker rm rabbitmq
-      - name: Stop broker B
-        run: docker stop hare && docker rm hare
+          ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
+            -Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
+            -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
+            -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
+            -Dmaven.javadoc.skip=true \
+            --no-transfer-progress
+      - name: Stop cluster
+        run: docker compose --file ci/cluster/docker-compose.yml down

.github/workflows/test-supported-java-versions-5.x.yml

@@ -45,17 +45,19 @@ jobs:
         run: |
           ./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
             -Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
-            -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
+            -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
+            -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
             -Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
             --no-transfer-progress \
             -Dnet.bytebuddy.experimental=true
       - name: Test with blocking IO
         run: |
           ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
-          -Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
-          -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
-          -Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
-          --no-transfer-progress \
-          -Dnet.bytebuddy.experimental=true
+            -Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
+            -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
+            -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
+            -Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
+            --no-transfer-progress \
+            -Dnet.bytebuddy.experimental=true
       - name: Stop broker
         run: docker stop rabbitmq && docker rm rabbitmq

.github/workflows/test-supported-java-versions-main.yml

@@ -43,17 +43,19 @@ jobs:
         run: |
           ./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
             -Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
-            -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
+            -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
+            -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
             -Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
             --no-transfer-progress \
             -Dnet.bytebuddy.experimental=true
       - name: Test with blocking IO
         run: |
           ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
-          -Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
-          -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
-          -Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
-          --no-transfer-progress \
-          -Dnet.bytebuddy.experimental=true
+            -Dtest-broker.A.nodename=rabbit@$(hostname) -Dmaven.javadoc.skip=true \
+            -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
+            -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
+            -Dit.test=ClientTestSuite,FunctionalTestSuite,ServerTestSuite,SslTestSuite \
+            --no-transfer-progress \
+            -Dnet.bytebuddy.experimental=true
       - name: Stop broker
         run: docker stop rabbitmq && docker rm rabbitmq

.github/workflows/test.yml

@@ -41,22 +41,23 @@ jobs:
         run: make deps
       - name: Test with NIO
         run: |
-          ./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq \
-            -Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
+          ./mvnw verify -P use-nio -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
+            -Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
+            -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
+            -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
             -Dmaven.javadoc.skip=true \
-            -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
             --no-transfer-progress
       - name: Test with blocking IO
         run: |
-          ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \
-          -Dtest-broker.A.nodename=rabbit@$(hostname) -Dtest-broker.B.nodename=hare@$(hostname) \
-          -Dmaven.javadoc.skip=true \
-          -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
-          --no-transfer-progress
-      - name: Stop broker A
-        run: docker stop rabbitmq && docker rm rabbitmq
-      - name: Stop broker B
-        run: docker stop hare && docker rm hare
+          ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq0 \
+            -Dtest-broker.A.nodename=rabbit@node0 -Dtest-broker.B.nodename=rabbit@node1 \
+            -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \
+            -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \
+            -Dmaven.javadoc.skip=true \
+            -Dtest-client-cert.password= -Dtest-tls-certs.dir=rabbitmq-configuration/tls \
+            --no-transfer-progress
+      - name: Stop cluster
+        run: docker compose --file ci/cluster/docker-compose.yml down
       - name: Publish snapshot
         if: ${{ github.event_name != 'pull_request' }}
         run: ./mvnw clean deploy -Psnapshots -DskipITs -DskipTests --no-transfer-progress

ci/_start-cluster.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+LOCAL_SCRIPT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.0}
+
+wait_for_message() {
+  while ! docker logs "$1" | grep -q "$2";
+  do
+      sleep 5
+      echo "Waiting 5 seconds for $1 to start..."
+  done
+}
+
+make -C "${PWD}"/tls-gen/basic
+
+mv tls-gen/basic/result/server_$(hostname -s)_certificate.pem tls-gen/basic/result/server_certificate.pem
+mv tls-gen/basic/result/server_$(hostname -s)_key.pem tls-gen/basic/result/server_key.pem
+mv tls-gen/basic/server_$(hostname -s) tls-gen/basic/server
+mv tls-gen/basic/client_$(hostname -s) tls-gen/basic/client
+
+rm -rf rabbitmq-configuration
+mkdir -p rabbitmq-configuration/tls
+
+cp -R "${PWD}"/tls-gen/basic/* rabbitmq-configuration/tls
+chmod -R o+r rabbitmq-configuration/tls/*
+chmod -R g+r rabbitmq-configuration/tls/*
+./mvnw -q clean resources:testResources -Dtest-tls-certs.dir=/etc/rabbitmq/tls
+cp target/test-classes/[email protected] rabbitmq-configuration/[email protected]
+cp target/test-classes/[email protected] rabbitmq-configuration/[email protected]
+
+echo "Running RabbitMQ ${RABBITMQ_IMAGE}"
+
+docker rm -f rabbitmq 2>/dev/null || echo "rabbitmq was not running"
+docker run -d --name rabbitmq \
+    --network host \
+    -v "${PWD}"/rabbitmq-configuration:/etc/rabbitmq \
+    --env RABBITMQ_CONFIG_FILE=/etc/rabbitmq/[email protected] \
+    --env RABBITMQ_NODENAME=rabbit@$(hostname) \
+    --env RABBITMQ_NODE_PORT=5672 \
+    --env RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-setcookie do-not-do-this-in-production" \
+    "${RABBITMQ_IMAGE}"
+
+# for CLI commands to share the same cookie
+docker exec rabbitmq bash -c "echo 'do-not-do-this-in-production' > /var/lib/rabbitmq/.erlang.cookie"
+docker exec rabbitmq chmod 0600 /var/lib/rabbitmq/.erlang.cookie
+
+wait_for_message rabbitmq "completed with"
+
+docker run -d --name hare \
+    --network host \
+    -v "${PWD}"/rabbitmq-configuration:/etc/rabbitmq \
+    --env RABBITMQ_CONFIG_FILE=/etc/rabbitmq/[email protected] \
+    --env RABBITMQ_NODENAME=hare@$(hostname) \
+    --env RABBITMQ_NODE_PORT=5673 \
+    --env RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-setcookie do-not-do-this-in-production" \
+    "${RABBITMQ_IMAGE}"
+
+# for CLI commands to share the same cookie
+docker exec hare bash -c "echo 'do-not-do-this-in-production' > /var/lib/rabbitmq/.erlang.cookie"
+docker exec hare chmod 0600 /var/lib/rabbitmq/.erlang.cookie
+
+wait_for_message hare "completed with"
+
+docker exec hare rabbitmqctl --node hare@$(hostname) status
+
+docker exec rabbitmq rabbitmq-diagnostics --node rabbit@$(hostname) is_running
+docker exec hare rabbitmq-diagnostics --node hare@$(hostname) is_running
+
+docker exec hare rabbitmqctl --node hare@$(hostname) stop_app
+docker exec hare rabbitmqctl --node hare@$(hostname) join_cluster rabbit@$(hostname)
+docker exec hare rabbitmqctl --node hare@$(hostname) start_app
+
+sleep 10
+
+docker exec hare rabbitmqctl --node hare@$(hostname) await_startup
+
+docker exec hare rabbitmqctl --node hare@$(hostname) enable_feature_flag --opt-in khepri_db
+docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) enable_feature_flag --opt-in khepri_db
+
+docker exec rabbitmq rabbitmq-diagnostics --node rabbit@$(hostname) erlang_version
+docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) version
+docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) status
+docker exec rabbitmq rabbitmqctl --node rabbit@$(hostname) cluster_status

ci/cluster/configuration/rabbitmq.conf

@@ -0,0 +1,20 @@
+cluster_formation.peer_discovery_backend  = rabbit_peer_discovery_classic_config
+cluster_formation.classic_config.nodes.1 = rabbit@node0
+cluster_formation.classic_config.nodes.2 = rabbit@node1
+cluster_formation.classic_config.nodes.3 = rabbit@node2
+loopback_users = none
+
+listeners.ssl.default = 5671
+
+ssl_options.cacertfile = /etc/rabbitmq/tls/ca_certificate.pem
+ssl_options.certfile   = /etc/rabbitmq/tls/server_certificate.pem
+ssl_options.keyfile    = /etc/rabbitmq/tls/server_key.pem
+ssl_options.verify     = verify_peer
+ssl_options.fail_if_no_peer_cert = false
+ssl_options.honor_cipher_order   = true
+
+auth_mechanisms.1 = PLAIN
+auth_mechanisms.2 = ANONYMOUS
+auth_mechanisms.3 = AMQPLAIN
+auth_mechanisms.4 = EXTERNAL
+auth_mechanisms.5 = RABBIT-CR-DEMO

ci/cluster/docker-compose.yml

@@ -0,0 +1,49 @@
+services:
+  node0:
+    environment:
+      - RABBITMQ_ERLANG_COOKIE='secret_cookie'
+    networks:
+      - rabbitmq-cluster
+    hostname: node0
+    container_name: rabbitmq0
+    image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
+    pull_policy: always
+    ports:
+      - "5672:5672"
+      - "5671:5671"
+    tty: true
+    volumes:
+      - ./configuration/:/etc/rabbitmq/
+      - ../../rabbitmq-configuration/tls:/etc/rabbitmq/tls/
+  node1:
+    environment:
+      - RABBITMQ_ERLANG_COOKIE='secret_cookie'
+    networks:
+      - rabbitmq-cluster
+    hostname: node1
+    container_name: rabbitmq1
+    image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
+    pull_policy: always
+    ports:
+      - "5673:5672"
+    tty: true
+    volumes:
+      - ./configuration/:/etc/rabbitmq/
+      - ../../rabbitmq-configuration/tls:/etc/rabbitmq/tls/
+  node2:
+    environment:
+      - RABBITMQ_ERLANG_COOKIE='secret_cookie'
+    networks:
+      - rabbitmq-cluster
+    hostname: node2
+    container_name: rabbitmq2
+    image: ${RABBITMQ_IMAGE:-rabbitmq:4.0}
+    pull_policy: always
+    ports:
+      - "5674:5672"
+    tty: true
+    volumes:
+      - ./configuration/:/etc/rabbitmq/
+      - ../../rabbitmq-configuration/tls:/etc/rabbitmq/tls/
+networks:
+  rabbitmq-cluster:

ci/start-broker.sh

@@ -12,21 +12,33 @@ wait_for_message() {
   done
 }
 
-make -C "${PWD}"/tls-gen/basic
+rm -rf rabbitmq-configuration
+mkdir -p rabbitmq-configuration/tls
 
-mv tls-gen/basic/result/server_$(hostname -s)_certificate.pem tls-gen/basic/result/server_certificate.pem
-mv tls-gen/basic/result/server_$(hostname -s)_key.pem tls-gen/basic/result/server_key.pem
-mv tls-gen/basic/server_$(hostname -s) tls-gen/basic/server
-mv tls-gen/basic/client_$(hostname -s) tls-gen/basic/client
+make -C "${PWD}"/tls-gen/basic
 
 rm -rf rabbitmq-configuration
 mkdir -p rabbitmq-configuration/tls
-
-cp -R "${PWD}"/tls-gen/basic/* rabbitmq-configuration/tls
-chmod -R o+r rabbitmq-configuration/tls/*
-chmod -R g+r rabbitmq-configuration/tls/*
-./mvnw -q clean resources:testResources -Dtest-tls-certs.dir=/etc/rabbitmq/tls
-cp target/test-classes/[email protected] rabbitmq-configuration/rabbitmq.config
+cp -R "${PWD}"/tls-gen/basic/result/* rabbitmq-configuration/tls
+chmod o+r rabbitmq-configuration/tls/*
+chmod g+r rabbitmq-configuration/tls/*
+
+echo "loopback_users = none
+
+listeners.ssl.default = 5671
+
+ssl_options.cacertfile = /etc/rabbitmq/tls/ca_certificate.pem
+ssl_options.certfile   = /etc/rabbitmq/tls/server_$(hostname)_certificate.pem
+ssl_options.keyfile    = /etc/rabbitmq/tls/server_$(hostname)_key.pem
+ssl_options.verify     = verify_peer
+ssl_options.fail_if_no_peer_cert = false
+ssl_options.honor_cipher_order   = true
+
+auth_mechanisms.1 = PLAIN
+auth_mechanisms.2 = ANONYMOUS
+auth_mechanisms.3 = AMQPLAIN
+auth_mechanisms.4 = EXTERNAL
+auth_mechanisms.5 = RABBIT-CR-DEMO" >> rabbitmq-configuration/rabbitmq.conf
 
 echo "Running RabbitMQ ${RABBITMQ_IMAGE}"