重建损坏的etcd集群

系统: Ubuntu Server 22.04 LTS

• node1: 192.168.1.120
• node1: 192.168.1.121
• node1: 192.168.1.122

step1: 备份数据

# 在一个可用的节点上执行此命令, 导出数据
ETCDCTL_API=3 etcdctl \
    --endpoints=https://192.168.1.120:2379 \
    --cacert=/etc/ssl/etcd/ca.pem \
    --cert=/etc/ssl/etcd/etcd.pem \
    --key=/etc/ssl/etcd/etcd-key.pem \
    snapshot save /root/etcd.db

step2: 重建第一个节点

# 每个节点执行一次
systemctl stop etcd && rm -rf /var/lib/etcd

# 回到node1, 导入备份好的数据
ETCDCTL_API=3 etcdctl \
    --endpoints=https://192.168.1.120:2379 \
    --cacert=/etc/ssl/etcd/ca.pem \
    --cert=/etc/ssl/etcd/etcd.pem \
    --data-dir=/var/lib/etcd \
    snapshot restore /root/etcd.db 
    
# 在node1启动etcd
ETCD_DATA_DIR=/var/lib/etcd \
ETCD_ADVERTISE_CLIENT_URLS=https://192.168.1.120:2379 \
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.1.120:2380 \
ETCD_INITIAL_CLUSTER_STATE=existing \
ETCD_METRICS=basic \
ETCD_LISTEN_CLIENT_URLS=https://192.168.1.120:2379 \
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd \
ETCD_LISTEN_PEER_URLS=https://192.168.1.120:2380 \
ETCD_NAME=etcd01 \
ETCD_PROXY=off \
ETCD_ENABLE_V2=true \
ETCD_INITIAL_CLUSTER=etcd01=https://192.168.1.120:2380 \
ETCD_ELECTION_TIMEOUT=5000 \
ETCD_HEARTBEAT_INTERVAL=250 \
ETCD_AUTO_COMPACTION_RETENTION=8 \
ETCD_SNAPSHOT_COUNT=10000 \
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ca.pem \
ETCD_CERT_FILE=/etc/ssl/etcd/etcd.pem \
ETCD_KEY_FILE=/etc/ssl/etcd/etcd-key.pem \
ETCD_CLIENT_CERT_AUTH=true \
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ca.pem \
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/etcd.pem \
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/etcd-key.pem \
ETCD_PEER_CLIENT_CERT_AUTH=true \
ETCDCTL_ENDPOINTS=https://192.168.1.120:2379 \
ETCDCTL_CACERT=/etc/ssl/etcd/ca.pem \
ETCDCTL_KEY=/etc/ssl/etcd/etcd-key.pem \
ETCDCTL_CERT=/etc/ssl/etcd/etcd.pem \
etcd

step3: 加入第二个节点

# 查看集群成员
export ETCDCTL_API=2 && \
export ETCDCTL_CERT_FILE='/etc/ssl/etcd/etcd.pem' && \
export ETCDCTL_KEY_FILE='/etc/ssl/etcd/etcd-key.pem' && \
export ETCDCTL_CA_FILE='/etc/ssl/etcd/ca.pem' && \
etcdctl --endpoints=https://192.168.1.120:2379 member list

# 如果看到了localhost, 需要将其修改为对应的内网IP, 否则会导致新成员无法加入
etcdctl --endpoints=https://192.168.1.120:2379 member update 8e9e05c52164694d https://192.168.1.120:2380

# 加入成员
etcdctl --endpoints=https://192.168.1.120:2379 member add etcd02 https://192.168.1.121:2380

# 在node2启动etcd
ETCD_DATA_DIR=/var/lib/etcd \
ETCD_ADVERTISE_CLIENT_URLS=https://192.168.1.121:2379 \
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.1.121:2380 \
ETCD_INITIAL_CLUSTER_STATE=existing \
ETCD_METRICS=basic \
ETCD_LISTEN_CLIENT_URLS=https://192.168.1.121:2379 \
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd \
ETCD_LISTEN_PEER_URLS=https://192.168.1.121:2380 \
ETCD_NAME=etcd02 \
ETCD_PROXY=off \
ETCD_ENABLE_V2=true \
ETCD_INITIAL_CLUSTER=etcd01=https://192.168.1.120:2380,etcd02=https://192.168.1.121:2380 \
ETCD_ELECTION_TIMEOUT=5000 \
ETCD_HEARTBEAT_INTERVAL=250 \
ETCD_AUTO_COMPACTION_RETENTION=8 \
ETCD_SNAPSHOT_COUNT=10000 \
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ca.pem \
ETCD_CERT_FILE=/etc/ssl/etcd/etcd.pem \
ETCD_KEY_FILE=/etc/ssl/etcd/etcd-key.pem \
ETCD_CLIENT_CERT_AUTH=true \
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ca.pem \
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/etcd.pem \
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/etcd-key.pem \
ETCD_PEER_CLIENT_CERT_AUTH=true \
ETCDCTL_ENDPOINTS=https://192.168.1.121:2379 \
ETCDCTL_CACERT=/etc/ssl/etcd/ca.pem \
ETCDCTL_KEY=/etc/ssl/etcd/etcd-key.pem \
ETCDCTL_CERT=/etc/ssl/etcd/etcd.pem \
etcd

step4: 加入第三个节点

# 重复上一步, 加入最后一个节点
etcdctl --endpoints=https://192.168.1.120:2379 member add etcd03 https://192.168.1.122:2380

ETCD_DATA_DIR=/var/lib/etcd \
ETCD_ADVERTISE_CLIENT_URLS=https://192.168.1.122:2379 \
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.1.122:2380 \
ETCD_INITIAL_CLUSTER_STATE=existing \
ETCD_METRICS=basic \
ETCD_LISTEN_CLIENT_URLS=https://192.168.1.122:2379 \
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd \
ETCD_LISTEN_PEER_URLS=https://192.168.1.122:2380 \
ETCD_NAME=etcd03 \
ETCD_PROXY=off \
ETCD_ENABLE_V2=true \
ETCD_INITIAL_CLUSTER=etcd01=https://192.168.1.120:2380,etcd02=https://192.168.1.121:2380,etcd03=https://192.168.1.122:2380 \
ETCD_ELECTION_TIMEOUT=5000 \
ETCD_HEARTBEAT_INTERVAL=250 \
ETCD_AUTO_COMPACTION_RETENTION=8 \
ETCD_SNAPSHOT_COUNT=10000 \
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ca.pem \
ETCD_CERT_FILE=/etc/ssl/etcd/etcd.pem \
ETCD_KEY_FILE=/etc/ssl/etcd/etcd-key.pem \
ETCD_CLIENT_CERT_AUTH=true \
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ca.pem \
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/etcd.pem \
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/etcd-key.pem \
ETCD_PEER_CLIENT_CERT_AUTH=true \
ETCDCTL_ENDPOINTS=https://192.168.1.122:2379 \
ETCDCTL_CACERT=/etc/ssl/etcd/ca.pem \
ETCDCTL_KEY=/etc/ssl/etcd/etcd-key.pem \
ETCDCTL_CERT=/etc/ssl/etcd/etcd.pem \
etcd

step5: 保存修改

CTRL C杀死所有etcd进程, 将变更写入systemd环境变量文件, ETCD_INITIAL_CLUSTER需要加入所有节点.
然后执行:

systemctl start etcd
systemctl enable etcd

etcd.service

[Unit]
Description=etcd - highly-available key value store
Documentation=https://etcd.io/docs
Documentation=man:etcd
After=network.target
Wants=network-online.target

[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
User=root
ExecStart=/usr/bin/etcd
Restart=on-abnormal
RestartSec=10s
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd2.service