Skip to content

Commit 78a6bd5

Browse files
authored
CI: retry docker containers startup to work around CircleCI port mapping errors (#39)
1 parent 1b4e792 commit 78a6bd5

File tree

6 files changed

+32
-14
lines changed

6 files changed

+32
-14
lines changed

.circleci/config.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ jobs:
8585
- load_cache
8686
- run:
8787
name: Create Kafka Environment
88-
command: ./docker/startup.sh
88+
command: ./docker/startup_retry.sh
8989
- run:
9090
name: Integration Tests
9191
command: mvn integration-test
@@ -122,7 +122,7 @@ jobs:
122122
command: mvn package -Ddistributed
123123
- run:
124124
name: Create Kafka Environment
125-
command: ./docker/startup.sh
125+
command: ./docker/startup_retry.sh
126126
- run:
127127
name: Integration Tests
128128
command: mvn integration-test -Ddistributed -Darango.topology=$STARTER_MODE
@@ -149,7 +149,7 @@ jobs:
149149
command: mvn package -Ddistributed
150150
- run:
151151
name: Create Kafka Environment
152-
command: ./docker/startup.sh
152+
command: ./docker/startup_retry.sh
153153
- run:
154154
name: Integration Tests
155155
command: mvn integration-test -Ddistributed -DSslTest=true -Dit.test=com.arangodb.kafka.SslIT
@@ -173,7 +173,7 @@ jobs:
173173
- load_cache
174174
- run:
175175
name: Create Kafka Environment
176-
command: ./docker/startup.sh
176+
command: ./docker/startup_retry.sh
177177
- run:
178178
name: Start Toxiproxy
179179
command: ./bin/startProxy.sh

docker/start_db.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,14 +55,14 @@ if [ "$EXTENDED_NAMES" == "true" ]; then
5555
fi
5656

5757
# data volume
58-
docker create -v /data --name arangodb-data alpine:3 /bin/true
59-
docker cp "$LOCATION"/jwtSecret arangodb-data:/data
60-
docker cp "$LOCATION"/server.pem arangodb-data:/data
58+
docker create -v /data --name adb-data alpine:3 /bin/true
59+
docker cp "$LOCATION"/jwtSecret adb-data:/data
60+
docker cp "$LOCATION"/server.pem adb-data:/data
6161

6262
docker run -d \
6363
--name=adb \
6464
-p 8528:8528 \
65-
--volumes-from arangodb-data \
65+
--volumes-from adb-data \
6666
-v /var/run/docker.sock:/var/run/docker.sock \
6767
-e ARANGO_LICENSE_KEY="$ARANGO_LICENSE_KEY" \
6868
$STARTER_DOCKER_IMAGE \

docker/start_kafka_zk.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ ZK_DOCKER_IMAGE=docker.io/bitnami/zookeeper:3.8
1111
docker pull $ZK_DOCKER_IMAGE
1212

1313
docker run -d \
14-
--name zk-1 -h zk-1 \
14+
--name kafka-zk-1 -h kafka-zk-1 \
1515
--network arangodb \
1616
-e BITNAMI_DEBUG=true \
1717
-e ALLOW_ANONYMOUS_LOGIN="yes" \
@@ -22,7 +22,7 @@ docker run -d \
2222
--network arangodb \
2323
-p 19092:19092 \
2424
-e BITNAMI_DEBUG=true \
25-
-e KAFKA_CFG_ZOOKEEPER_CONNECT=zk-1 \
25+
-e KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zk-1 \
2626
-e ALLOW_PLAINTEXT_LISTENER="yes" \
2727
-e KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,PLAINTEXT_HOST://0.0.0.0:19092 \
2828
-e KAFKA_CFG_ADVERTISED_LISTENERS="PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://172.28.0.1:19092" \
@@ -35,7 +35,7 @@ docker run -d \
3535
--network arangodb \
3636
-p 29092:29092 \
3737
-e BITNAMI_DEBUG=true \
38-
-e KAFKA_CFG_ZOOKEEPER_CONNECT=zk-1 \
38+
-e KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zk-1 \
3939
-e ALLOW_PLAINTEXT_LISTENER="yes" \
4040
-e KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,PLAINTEXT_HOST://0.0.0.0:29092 \
4141
-e KAFKA_CFG_ADVERTISED_LISTENERS="PLAINTEXT://kafka-2:9092,PLAINTEXT_HOST://172.28.0.1:29092" \
@@ -48,7 +48,7 @@ docker run -d \
4848
--network arangodb \
4949
-p 39092:39092 \
5050
-e BITNAMI_DEBUG=true \
51-
-e KAFKA_CFG_ZOOKEEPER_CONNECT=zk-1 \
51+
-e KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zk-1 \
5252
-e ALLOW_PLAINTEXT_LISTENER="yes" \
5353
-e KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,PLAINTEXT_HOST://0.0.0.0:39092 \
5454
-e KAFKA_CFG_ADVERTISED_LISTENERS="PLAINTEXT://kafka-3:9092,PLAINTEXT_HOST://172.28.0.1:39092" \

docker/start_schema_registry.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ docker pull $DOCKER_IMAGE
66
KAFKA_BOOTSTRAP_SERVERS=PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092
77

88
docker run -d \
9-
--name schema-registry -h schema-registry \
9+
--name kafka-schema-registry -h kafka-schema-registry \
1010
--network arangodb \
1111
-p 8081:8081 \
1212
-e SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS="$KAFKA_BOOTSTRAP_SERVERS" \

docker/startup_retry.sh

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
# exit when any command fails
4+
set -e
5+
6+
until ./docker/startup.sh
7+
do
8+
# rm all containers, in case of retries
9+
# retries might be needed in case of port-mapping errors in CircleCI, e.g. for errors like:
10+
# docker: Error response from daemon: driver failed programming external connectivity on endpoint kafka-3:
11+
# Error starting userland proxy: listen tcp4 0.0.0.0:39092: bind: address already in use.
12+
13+
docker ps -a -f name=kafka-.* -q | while read x ; do docker rm -f $x ; done
14+
docker rm -f adb
15+
docker ps -a -f name=adb-.* -q | while read x ; do docker rm -f $x ; done
16+
echo "Startup failed, retrying in 10 seconds..."
17+
sleep 10
18+
done

src/connect-distributed/java/spi/ClusterKafkaConnectDeployment.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public KafkaConnectOperations client() {
5757

5858
@Override
5959
public String getSchemaRegistryUrlConnect() {
60-
return "http://schema-registry:8081";
60+
return "http://kafka-schema-registry:8081";
6161
}
6262

6363
@Override

0 commit comments

Comments
 (0)