前書き
重要なデータを扱う場合は、遅かれ早かれ、フェイルオーバークラスターを立ち上げるのに何が良いかを考えるでしょう。ベースのあるメインサーバーが完全にノックアウトして飛び去ったとしても、ショーは続けなければなりませんよね？そうすることで、私たちは2つのことを意味します：
- ;
, .
Patroni . Github ( ) , . . - .
Patroni , . DCS , «split brain».
, «out of the box solution. :
“Patroni is a template for you to create your own customized, high-availability solution using Python...”
«template». . - — , , .
, , DevOps . . , , , .
, .
, , .
, Patroni?
?
, , Patroni. docker swarm Zookeeper DCS.
Zookeeper?
, , , . , Patroni . — DCS (Dynamic Configuration Storage).
- Patroni, , , , — DCS Etcd .
Etcd , :
Since etcd writes data to disk, its performance strongly depends on disk performance. For this reason, SSD is highly recommended.
( Etcd)
, SSD , Etcd, . , , , , , ( ), Etcd . IO . ? . .
Zookeeper, . , SSD, RAM .
Docker Swarm?
, , Swarm’. , , !
, , Docker Swarm’ . , ( Docker’, ) . , , Docker , , , .
, Docker’, Patroni 3 , Kubernetes .
- Docker’ , .
( 3), .
, . .
Docker Swarm
, , , Swarm . , Swarm’, , .
, Docker Engine. , :
docker swarm init
//now check your single-node cluster
docker node ls
ID HOSTNAME STATUS AVAILABILITY
a9ej2flnv11ka1hencoc1mer2 * floitet Ready Active
Swarm’ , Docker , . — . , — , . yml-.
hostname — ‘constraint’ .
, Docker Swarm’, . . .
Zookeeper
Patroni, DCS ( , , Zookeeper). 3.4, . docker-compose , , , .
docker-compose-zookeeper.yml
docker-compose-zookeeper.yml
version: '3.7'
services:
zoo1:
image: zookeeper:3.4
hostname: zoo1
ports:
- 2191:2181
networks:
- patroni
environment:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
deploy:
replicas: 1
placement:
constraints:
- node.hostname == floitet
restart_policy:
condition: any
zoo2:
image: zookeeper:3.4
hostname: zoo2
networks:
- patroni
ports:
- 2192:2181
environment:
ZOO_MY_ID: 2
ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zoo3:2888:3888
deploy:
replicas: 1
placement:
constraints:
- node.hostname == floitet
restart_policy:
condition: any
zoo3:
image: zookeeper:3.4
hostname: zoo3
networks:
- patroni
ports:
- 2193:2181
environment:
ZOO_MY_ID: 3
ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=0.0.0.0:2888:3888
deploy:
replicas: 1
placement:
constraints:
- node.hostname == floitet
restart_policy:
condition: any
networks:
patroni:
driver: overlay
attachable: true
Details
, . Hostname .
zoo1:
image: zookeeper:3.4
hostname: zoo1
ports:
- 2191:2181
, host’ : server.1 0.0.0.0, , , zoo2 server.2 ..
ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
. , , , node.hostname .
placement:
constraints:
- node.hostname == floitet
, , network. Zookeeper’ Patroni overlay, , IP ( , ).
networks:
patroni:
driver: overlay
// attachable
//
attachable: true
, Zookeeper:
sudo docker stack deploy --compose-file docker-compose-zookeeper.yml patroni
, . :
sudo docker service ls
gxfj9rs3po7z patroni_zoo1 replicated 1/1 zookeeper:3.4 *:2191->2181/tcp
ibp0mevmiflw patroni_zoo2 replicated 1/1 zookeeper:3.4 *:2192->2181/tcp
srucfm8jrt57 patroni_zoo3 replicated 1/1 zookeeper:3.4 *:2193->2181/tcp
mntr:
echo mntr | nc localhost 2191
// with the output being smth like this
zk_version 3.4.14-4c25d480e66aadd371de8bd2fd8da255ac140bcf, built on 03/06/2019 16:18 GMT
zk_avg_latency 6
zk_max_latency 205
zk_min_latency 0
zk_packets_received 1745
zk_packets_sent 1755
zk_num_alive_connections 3
zk_outstanding_requests 0
zk_server_state follower
zk_znode_count 16
zk_watch_count 9
zk_ephemerals_count 4
zk_approximate_data_size 1370
zk_open_file_descriptor_count 34
zk_max_file_descriptor_count 1048576
zk_fsync_threshold_exceed_count 0
, :
docker service logs $zookeeper-service-id
// service-id comes from 'docker service ls' command.
// in my case it could be
docker service logs gxfj9rs3po7z
, Zookeeper’. Patroni.
Patroni
, Patroni. — Patroni, . , , .
’patroni-test’ . , .
patroni.yml
. Patroni, patroni.yml — . , , , .
, , , . . , - , , Posgtres’ (, max_connections ..). .
patroni.yml
scope: patroni
namespace: /service/
bootstrap:
dcs:
ttl: 30
loop_wait: 10
retry_timeout: 10
maximum_lag_on_failover: 1048576
postgresql:
use_pg_rewind: true
postgresql:
use_pg_rewind: true
initdb:
- encoding: UTF8
- data-checksums
pg_hba:
- host replication all all md5
- host all all all md5
zookeeper:
hosts:
- zoo1:2181
- zoo2:2181
- zoo3:2181
postgresql:
data_dir: /data/patroni
bin_dir: /usr/lib/postgresql/11/bin
pgpass: /tmp/pgpass
parameters:
unix_socket_directories: '.'
tags:
nofailover: false
noloadbalance: false
clonefrom: false
nosync: false
Details
Patroni Postgres’. , Postgres 11, : ’/usr/lib/postgresql/11/bin’.
, , Patroni Postgres’. ( ). ’data_dir’ — , . mount , , . , , -, .
postgresql:
data_dir: /data/patroni
bin_dir: /usr/lib/postgresql/11/bin
Zookeeper’ , patronictl. , patroni.yml, patronictl. , , IP, . Docker Swarm’ .
zookeeper:
hosts:
- zoo1:2181
- zoo2:2181
- zoo3:2181
patroni-entrypoint.sh
. , , .
patroni-entrypoint.sh
#!/bin/sh
readonly CONTAINER_IP=$(hostname --ip-address)
readonly CONTAINER_API_ADDR="${CONTAINER_IP}:${PATRONI_API_CONNECT_PORT}"
readonly CONTAINER_POSTGRE_ADDR="${CONTAINER_IP}:5432"
export PATRONI_NAME="${PATRONI_NAME:-$(hostname)}"
export PATRONI_RESTAPI_CONNECT_ADDRESS="$CONTAINER_API_ADDR"
export PATRONI_RESTAPI_LISTEN="$CONTAINER_API_ADDR"
export PATRONI_POSTGRESQL_CONNECT_ADDRESS="$CONTAINER_POSTGRE_ADDR"
export PATRONI_POSTGRESQL_LISTEN="$CONTAINER_POSTGRE_ADDR"
export PATRONI_REPLICATION_USERNAME="$REPLICATION_NAME"
export PATRONI_REPLICATION_PASSWORD="$REPLICATION_PASS"
export PATRONI_SUPERUSER_USERNAME="$SU_NAME"
export PATRONI_SUPERUSER_PASSWORD="$SU_PASS"
export PATRONI_approle_PASSWORD="$POSTGRES_APP_ROLE_PASS"
export PATRONI_approle_OPTIONS="${PATRONI_admin_OPTIONS:-createdb, createrole}"
exec /usr/local/bin/patroni /etc/patroni.yml
Details. !
, , Patroni, IP host’. , host’ Docker-, - IP , Patroni. :
readonly CONTAINER_IP=$(hostname --ip-address)
readonly CONTAINER_API_ADDR="${CONTAINER_IP}:${PATRONI_API_CONNECT_PORT}"
readonly CONTAINER_POSTGRE_ADDR="${CONTAINER_IP}:5432"
...
export PATRONI_RESTAPI_CONNECT_ADDRESS="$CONTAINER_API_ADDR"
export PATRONI_RESTAPI_LISTEN="$CONTAINER_API_ADDR"
export PATRONI_POSTGRESQL_CONNECT_ADDRESS="$CONTAINER_POSTGRE_ADDR"
, Patroni . , — ’Environment configuration’. ’PATRONIRESTAPICONNECTADDRESS’, ’PATRONIRESTAPILISTEN’, ’PATRONIPOSTGRESQLCONNECTADDRESS’ — , Patroni . , patroni.yml, be aware!
. Patroni superuser’ . .. , , superuser’ replicator’ . . , - ‘approle’, ‘approle’ - .
export PATRONI_approle_PASSWORD="$POSTGRES_APP_ROLE_PASS"
export PATRONI_approle_OPTIONS="${PATRONI_admin_OPTIONS:-createdb, createrole}"
, , Patroni :
exec /usr/local/bin/patroni /etc/patroni.yml
Dockerfile
Dockerfile , . , Docker-. , - .
Dockerfile
FROM postgres:11
RUN apt-get update -y\
&& apt-get install python3 python3-pip -y\
&& pip3 install --upgrade setuptools\
&& pip3 install psycopg2-binary \
&& pip3 install patroni[zookeeper] \
&& mkdir /data/patroni -p \
&& chown postgres:postgres /data/patroni \
&& chmod 700 /data/patroni
COPY patroni.yml /etc/patroni.yml
COPY patroni-entrypoint.sh ./entrypoint.sh
USER postgres
ENTRYPOINT ["bin/sh", "/entrypoint.sh"]
Details
, , . , Patroni, , mount .
// 'postgres', mode 700
mkdir /data/patroni -p \
chown postgres:postgres /data/patroni \
chmod 700 /data/patroni
...
// -
// postgres
USER postgres
, , :
COPY patroni.yml /etc/patroni.yml
COPY patroni-entrypoint.sh ./entrypoint.sh
, , :
ENTRYPOINT ["bin/sh", "/entrypoint.sh"]
, , . Patroni .
docker build -t patroni-test .
, Patroni — compose yml.
docker-compose-patroni.yml
compose — . , , .
docker-compose-patroni.yml
version: "3.4"
networks:
patroni_patroni:
external: true
services:
patroni1:
image: patroni-test
networks: [ patroni_patroni ]
ports:
- 5441:5432
- 8091:8091
hostname: patroni1
volumes:
- /patroni1:/data/patroni
environment:
PATRONI_API_CONNECT_PORT: 8091
REPLICATION_NAME: replicator
REPLICATION_PASS: replpass
SU_NAME: postgres
SU_PASS: supass
POSTGRES_APP_ROLE_PASS: appass
deploy:
replicas: 1
placement:
constraints: [node.hostname == floitet]
patroni2:
image: patroni-test
networks: [ patroni_patroni ]
ports:
- 5442:5432
- 8092:8091
hostname: patroni2
volumes:
- /patroni2:/data/patroni
environment:
PATRONI_API_CONNECT_PORT: 8091
REPLICATION_NAME: replicator
REPLICATION_PASS: replpass
SU_NAME: postgres
SU_PASS: supass
POSTGRES_APP_ROLE_PASS: appass
deploy:
replicas: 1
placement:
constraints: [node.hostname == floitet]
patroni3:
image: patroni-test
networks: [ patroni_patroni ]
ports:
- 5443:5432
- 8093:8091
hostname: patroni3
volumes:
- /patroni3:/data/patroni
environment:
PATRONI_API_CONNECT_PORT: 8091
REPLICATION_NAME: replicator
REPLICATION_PASS: replpass
SU_NAME: postgres
SU_PASS: supass
POSTGRES_APP_ROLE_PASS: appass
deploy:
replicas: 1
placement:
constraints: [node.hostname == floitet]
Details
, , external network, . Patroni , Zookeeper. , : ’zoo1′, ’zoo2′, ’zoo3′, — patroni.yml, Zookeeper’, , .
networks:
patroni_patroni:
external: true
, end point’: API. :
ports:
- 5441:5432
- 8091:8091
...
environment:
PATRONI_API_CONNECT_PORT: 8091
// , PATRONI_API_CONNECT_PORT
// ,
, , , , entrypoint . . mount’, :
volumes:
- /patroni3:/data/patroni
, ’/data/patroni’, Dockerfile, . . , , :
sudo mkdir /patroni3
sudo chown 999:999 /patroni3
sudo chmod 700 /patroni3
// 999 uid postgres
// Patroni
Patroni :
sudo docker stack deploy --compose-file docker-compose-patroni.yml patroni
- :
INFO: Lock owner: patroni3; I am patroni1
INFO: does not have lock
INFO: no action. i am a secondary and i am following a leader
, . — patronictl. id Patroni:
sudo docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
a0090ce33a05 patroni-test:latest "bin/sh /entrypoint.…" 3 hours ago Up 3 hours 5432/tcp patroni_patroni1.1.tgjzpjyuip6ge8szz5lsf8kcq
...
exec :
sudo docker exec -ti a0090ce33a05 /bin/bash
//
// 'scope' patroni.yml ('patroni' )
patronictl list patroni
// ...
Error: 'Can not find suitable configuration of distributed configuration store\nAvailable implementations: exhibitor, kubernetes, zookeeper'
patronictl patroni.yml, Zookeeper’. , . :
patronictl -c /etc/patroni.yml list patroni
// and here is the nice output with the current states
+ Cluster: patroni (6893104757524385823) --+----+-----------+
| Member | Host | Role | State | TL | Lag in MB |
+----------+-----------+---------+---------+----+-----------+
| patroni1 | 10.0.1.93 | Replica | running | 8 | 0 |
| patroni2 | 10.0.1.91 | Replica | running | 8 | 0 |
| patroni3 | 10.0.1.92 | Leader | running | 8 | |
+----------+-----------+---------+---------+----+-----------+
PostgreSQL Connection
! Postgres - . «patroni_patroni». , :
docker run --rm -ti --network=patroni_patroni postgres:11 /bin/bash
//
psql --host patroni3 --port 5432 -U approle -d postgres
// haproxy
// - '-d'
Patroni. , . , , .