From 551da2da9e2328043de207fabe416be0c35ab96b Mon Sep 17 00:00:00 2001 From: zhou zhuohan <843520313@qq.com> Date: Tue, 8 Oct 2024 17:20:39 +0800 Subject: [PATCH] [improve] use pulsar-client-go-test docker to run blue-green cluster test (#1287) --- Makefile | 12 +-- .../blue-green/docker-compose.yml | 94 ++++++++++--------- pulsar/blue_green_migration_test.go | 14 +-- scripts/run-ci-blue-green-cluster.sh | 22 +++++ 4 files changed, 82 insertions(+), 60 deletions(-) create mode 100755 scripts/run-ci-blue-green-cluster.sh diff --git a/Makefile b/Makefile index 1fd66be6a3..23844c41c9 100644 --- a/Makefile +++ b/Makefile @@ -53,27 +53,25 @@ container: test: container test_standalone test_clustered test_extensible_load_manager test_standalone: container - docker run -v /var/run/docker.sock:/var/run/docker.sock -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci.sh" + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci.sh" test_clustered: container PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/clustered/docker-compose.yml up -d until curl http://localhost:8080/metrics > /dev/null 2>&1; do sleep 1; done - docker run --network "clustered_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-clustered.sh" + docker run --rm --network "clustered_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-clustered.sh" PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/clustered/docker-compose.yml down test_extensible_load_manager: container PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/extensible-load-manager/docker-compose.yml up -d until curl http://localhost:8080/metrics > /dev/null 2>&1; do sleep 1; done - docker run --network "extensible-load-manager_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-extensible-load-manager.sh" + docker run --rm --network "extensible-load-manager_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-extensible-load-manager.sh" PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/blue-green/docker-compose.yml up -d until curl http://localhost:8081/metrics > /dev/null 2>&1 ; do sleep 1; done - # run blue-green migration test (run this test from this env to access both clusters) - go test -race -coverprofile=/tmp/coverage-blue_green_topic_migration -timeout=5m -tags extensible_load_manager -v -run TestBlueGreenMigrationTestSuite ./pulsar - go tool cover -html=/tmp/coverage-blue_green_topic_migration -o coverage-blue_green_topic_migration.html - PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/extensible-load-manager/docker-compose.yml down + docker run --rm --network="extensible-load-manager_pulsar" -i ${IMAGE_NAME} bash -c "cd /pulsar/pulsar-client-go && ./scripts/run-ci-blue-green-cluster.sh" PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/blue-green/docker-compose.yml down + PULSAR_VERSION=${PULSAR_VERSION} docker compose -f integration-tests/extensible-load-manager/docker-compose.yml down clean: docker rmi --force $(IMAGE_NAME) || true diff --git a/integration-tests/blue-green/docker-compose.yml b/integration-tests/blue-green/docker-compose.yml index eea58520cb..b026ce8824 100644 --- a/integration-tests/blue-green/docker-compose.yml +++ b/integration-tests/blue-green/docker-compose.yml @@ -17,18 +17,20 @@ version: '3' networks: - green-pulsar: - driver: bridge + # We use external extensible-load-manager_pulsar docker network + # to make sure blue cluster and green cluster are in the same network + extensible-load-manager_pulsar: + external: true services: # Start ZooKeeper - zookeeper: + green-zookeeper: image: apachepulsar/pulsar:${PULSAR_VERSION} container_name: green-zookeeper restart: on-failure networks: - - green-pulsar + - extensible-load-manager_pulsar environment: - - metadataStoreUrl=zk:zookeeper:2181 + - metadataStoreUrl=zk:green-zookeeper:2181 - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m command: > bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ @@ -41,79 +43,79 @@ services: retries: 30 # Initialize cluster metadata - pulsar-init: + green-pulsar-init: container_name: green-pulsar-init - hostname: pulsar-init + hostname: green-pulsar-init image: apachepulsar/pulsar:${PULSAR_VERSION} networks: - - green-pulsar + - extensible-load-manager_pulsar environment: - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m command: > bin/pulsar initialize-cluster-metadata \ - --cluster cluster-a \ - --zookeeper zookeeper:2181 \ - --configuration-store zookeeper:2181 \ - --web-service-url http://broker-1:8080 \ - --broker-service-url pulsar://broker-1:6650 + --cluster cluster-green \ + --zookeeper green-zookeeper:2181 \ + --configuration-store green-zookeeper:2181 \ + --web-service-url http://green-broker-1:8080 \ + --broker-service-url pulsar://green-broker-1:6650 depends_on: - zookeeper: + green-zookeeper: condition: service_healthy # Start bookie - bookie: + green-bookie: image: apachepulsar/pulsar:${PULSAR_VERSION} container_name: green-bookie restart: on-failure networks: - - green-pulsar + - extensible-load-manager_pulsar environment: - - clusterName=cluster-a - - zkServers=zookeeper:2181 - - metadataServiceUri=metadata-store:zk:zookeeper:2181 - - advertisedAddress=bookie + - clusterName=cluster-green + - zkServers=green-zookeeper:2181 + - metadataServiceUri=metadata-store:zk:green-zookeeper:2181 + - advertisedAddress=green-bookie - BOOKIE_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m depends_on: - zookeeper: + green-zookeeper: condition: service_healthy - pulsar-init: + green-pulsar-init: condition: service_completed_successfully command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" - proxy: + green-proxy: image: apachepulsar/pulsar:${PULSAR_VERSION} container_name: green-proxy - hostname: proxy + hostname: green-proxy restart: on-failure networks: - - green-pulsar + - extensible-load-manager_pulsar environment: - - metadataStoreUrl=zk:zookeeper:2181 - - zookeeperServers=zookeeper:2181 - - clusterName=cluster-a + - metadataStoreUrl=zk:green-zookeeper:2181 + - zookeeperServers=green-zookeeper:2181 + - clusterName=cluster-green - PULSAR_MEM=-Xms128m -Xmx128m -XX:MaxDirectMemorySize=56m ports: - "8081:8080" - "6651:6650" depends_on: - broker-1: + green-broker-1: condition: service_started - broker-2: + green-broker-2: condition: service_started command: bash -c "bin/apply-config-from-env.py conf/proxy.conf && exec bin/pulsar proxy" # Start broker 1 - broker-1: + green-broker-1: image: apachepulsar/pulsar:${PULSAR_VERSION} container_name: green-broker-1 - hostname: broker-1 + hostname: green-broker-1 restart: on-failure networks: - - green-pulsar + - extensible-load-manager_pulsar environment: - - metadataStoreUrl=zk:zookeeper:2181 - - zookeeperServers=zookeeper:2181 - - clusterName=cluster-a + - metadataStoreUrl=zk:green-zookeeper:2181 + - zookeeperServers=green-zookeeper:2181 + - clusterName=cluster-green - managedLedgerDefaultEnsembleSize=1 - managedLedgerDefaultWriteQuorum=1 - managedLedgerDefaultAckQuorum=1 @@ -129,24 +131,24 @@ services: - brokerServiceCompactionThresholdInBytes=1000000 - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1 depends_on: - zookeeper: + green-zookeeper: condition: service_healthy - bookie: + green-bookie: condition: service_started command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" # Start broker 2 - broker-2: + green-broker-2: image: apachepulsar/pulsar:${PULSAR_VERSION} container_name: green-broker-2 - hostname: broker-2 + hostname: green-broker-2 restart: on-failure networks: - - green-pulsar + - extensible-load-manager_pulsar environment: - - metadataStoreUrl=zk:zookeeper:2181 - - zookeeperServers=zookeeper:2181 - - clusterName=cluster-a + - metadataStoreUrl=zk:green-zookeeper:2181 + - zookeeperServers=green-zookeeper:2181 + - clusterName=cluster-green - managedLedgerDefaultEnsembleSize=1 - managedLedgerDefaultWriteQuorum=1 - managedLedgerDefaultAckQuorum=1 @@ -162,8 +164,8 @@ services: - brokerServiceCompactionThresholdInBytes=1000000 - PULSAR_PREFIX_defaultNumberOfNamespaceBundles=1 depends_on: - zookeeper: + green-zookeeper: condition: service_healthy - bookie: + green-bookie: condition: service_started command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" \ No newline at end of file diff --git a/pulsar/blue_green_migration_test.go b/pulsar/blue_green_migration_test.go index 672ef343f8..4cd2e4765c 100644 --- a/pulsar/blue_green_migration_test.go +++ b/pulsar/blue_green_migration_test.go @@ -55,15 +55,15 @@ func (suite *BlueGreenMigrationTestSuite) TestTopicMigration() { { testCaseName: "proxyConnection", - blueAdminURL: "http://localhost:8080", - blueClientUrl: "pulsar://localhost:6650", - greenAdminURL: "http://localhost:8081", + blueAdminURL: "http://proxy:8080", + blueClientUrl: "pulsar://proxy:6650", + greenAdminURL: "http://green-proxy:8080", migrationBody: ` { - "serviceUrl": "http://localhost:8081", - "serviceUrlTls":"https://localhost:8085", - "brokerServiceUrl": "pulsar://localhost:6651", - "brokerServiceUrlTls": "pulsar+ssl://localhost:6655" + "serviceUrl": "http://green-proxy:8080", + "serviceUrlTls":"https://green-proxy:8081", + "brokerServiceUrl": "pulsar://green-proxy:6650", + "brokerServiceUrlTls": "pulsar+ssl://green-proxy:6651" } `, }, diff --git a/scripts/run-ci-blue-green-cluster.sh b/scripts/run-ci-blue-green-cluster.sh new file mode 100755 index 0000000000..a8528cb980 --- /dev/null +++ b/scripts/run-ci-blue-green-cluster.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e -x + +go test -race -coverprofile=/tmp/coverage-blue_green_topic_migration -timeout=5m -tags extensible_load_manager -v -run TestBlueGreenMigrationTestSuite ./pulsar +go tool cover -html=/tmp/coverage-blue_green_topic_migration -o coverage-blue_green_topic_migration.html \ No newline at end of file