01/02/2021 - CASSANDRA, DOCKER, GO
Assume that you have two docker services/containers. One must be ready to accept connections from the other one. As you have just noticed, I didn't say "one must be running" because there is different between being "ready to accept connections" and "running". The depends_on key of docker compose doesn't always solve the problem because it just checks if the container is running. This is often the case if a service depends on Cassandra, MySQL, RabbitMQ, Elasticsearch like services. They take time to accept connections. To solve such issue, we can use an additional script. This script would periodically ping these services until they are ready to accept connections before actually running the main service.
In this example we have a Go and Cassandra service. Go depends on Cassandra. Cassandra is slow in accepting connections. We will use our script in Go to force it to wait for Cassandra to accept connections first. Once Cassandra says it is ready then we will bring up the Go service.
├── cassandra
│ └── cassandra.go
├── docker
│ ├── cassandra
│ │ ├── Dockerfile
│ │ ├── init.sh
│ │ └── keyspace.cql
│ ├── docker-compose.yaml
│ └── go
│ ├── Dockerfile
│ └── init.sh
└── main.go
package cassandra
import (
"time"
"github.com/gocql/gocql"
)
// The `gocql: no response received from cassandra within timeout period` error
// will be prevented by increasing the default timeout value. e.g. 5 sec
type Config struct {
Hosts []string
Port int
ProtoVersion int
Consistency string
Keyspace string
Timeout time.Duration
}
func New(config Config) (*gocql.Session, error) {
cluster := gocql.NewCluster(config.Hosts...)
cluster.Port = config.Port
cluster.ProtoVersion = config.ProtoVersion
cluster.Keyspace = config.Keyspace
cluster.Consistency = gocql.ParseConsistency(config.Consistency)
cluster.Timeout = config.Timeout
return cluster.CreateSession()
}
package main
import (
"log"
"time"
"github.com/you/auth/cassandra"
)
func main() {
cass, err := cassandra.New(cassandra.Config{
// From docker go to docker cassandra: "172.17.0.1"
// From local go to docker cassandra: "127.0.0.1"
Hosts: []string{"172.17.0.1"},
Port: 9042,
ProtoVersion: 4,
Consistency: "Quorum",
Keyspace: "tetris",
Timeout: time.Second * 5,
})
if err != nil {
log.Fatalln(err)
}
defer cass.Close()
log.Printf("%+v\n", cass)
}
version: "3.7"
services:
tetris-cassandra:
build: "cassandra"
container_name: "tetris-cassandra"
ports:
- "9042:9042"
environment:
- "MAX_HEAP_SIZE=256M"
- "HEAP_NEWSIZE=128M"
tetris-go:
build:
context: ".."
dockerfile: "docker/go/Dockerfile"
container_name: "tetris-go"
environment:
DB_HOST: "tetris-cassandra"
DB_PORT: "9042"
depends_on:
- "tetris-cassandra"
FROM cassandra:3.11.9
COPY ./keyspace.cql /keyspace.cql
COPY ./init.sh /init.sh
RUN chmod +x /init.sh
ENTRYPOINT ["./init.sh"]
#!/bin/bash
set -eu
echo "Creating keyspace ..."
sleep=1
index=0
limit=60
while :
do
cqlsh < /keyspace.cql && echo "$(date) Created keyspace ..." && break
index=$(( index + 1 ))
echo "$(date) Waiting for keyspace $sleep seconds ($index/$limit) ..."
sleep $sleep
if [ $index -eq $limit ]
then
echo "$(date) Failed to create keyspace, terminating ..."
exit 1
fi
done &
./docker-entrypoint.sh "$@"
DROP KEYSPACE IF EXISTS tetris;
CREATE KEYSPACE IF NOT EXISTS tetris
WITH replication = {
'class': 'NetworkTopologyStrategy',
'datacenter1': 1
};
FROM golang:1.15-alpine3.12 as build
WORKDIR /source
COPY . .
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o bin/tetris main.go
FROM alpine:3.12
COPY --from=build /source/bin/tetris /bin/tetris
COPY --from=build /source/docker/go/init.sh /init.sh
RUN chmod +x /init.sh
ENTRYPOINT ["./init.sh"]
#!/bin/sh
set -eu
echo "$(date) Connecting to database ..."
sleep=1
index=0
limit=60
until [ $index -ge $limit ]
do
nc -z "${DB_HOST}" "${DB_PORT}" && break
index=$(( index + 1 ))
echo "$(date) Waiting for database $sleep seconds ($index/$limit) ..."
sleep $sleep
done
if [ $index -eq $limit ]
then
echo "$(date) Failed to connect to database, terminating ..."
exit 1
fi
echo "$(date) Connected to database ..."
echo "$(date) Allowing database 3 seconds to complete migrations ..."
sleep 3
./bin/tetris
$ docker-compose -f docker/docker-compose.yaml up
tetris-cassandra | Mon 01 Feb 2021 10:02:52 PM UTC Waiting for keyspace 1 seconds (14/60) ...
tetris-go | Mon Feb 1 22:02:52 UTC 2021 Waiting for database 1 seconds (19/60) ...
tetris-go | Mon Feb 1 22:02:53 UTC 2021 Waiting for database 1 seconds (20/60) ...
tetris-cassandra | Connection error: ('Unable to connect to any servers', {'127.0.0.1': error(111, "Tried connecting to [('127.0.0.1', 9042)]. Last error: Connection refused")})
tetris-cassandra | Mon 01 Feb 2021 10:02:53 PM UTC Waiting for keyspace 1 seconds (15/60) ...
tetris-go | Mon Feb 1 22:02:54 UTC 2021 Waiting for database 1 seconds (21/60) ...
tetris-cassandra | INFO [main] 2021-02-01 22:02:54,643 CassandraDaemon.java:650 - Startup complete
tetris-go | Mon Feb 1 22:02:55 UTC 2021 Connected to database ...
tetris-go | Mon Feb 1 22:02:55 UTC 2021 Allowing database 3 seconds to complete migrations ...
tetris-cassandra | INFO 2021-02-01 22:02:55,354 MigrationManager.java:338 - Create new Keyspace: KeyspaceMetadata{name=tetris, replication=ReplicationParams{class=NetworkTopologyStrategy, datacenter1=1}}}
tetris-cassandra | Mon 01 Feb 2021 10:02:55 PM UTC Created keyspace ...
tetris-cassandra | INFO 2021-02-01 22:02:55,940 CassandraRoleManager.java:372 - Created default superuser role 'cassandra'
tetris-go | 2021/02/01 22:02:58 &{cons:4 pageSize:5000 ... cfg:{Hosts:[172.17.0.1] CQLVersion:3.0.0 ProtoVersion:4 Port:9042 Keyspace:tetris Consistency:4 ... isClosed:false}