kubernetes上部署Memcached高可用记录

部署Memcached

memcached实现高可用常用的两个工具：
1）magent：连接多个memcached，请求转发
2）repcached：单master单slave，互为主辅

magent+memcached结构

repcache+memcached结构

主要部署步骤(repcache+memcached)

使用statefulset启动2个memcached

md-5c107eeae9fb1-0                  1/1       Running     0          59s
md-5c107eeae9fb1-1                  1/1       Running     0          49s

每个容器初始化脚本

#得到另一个节点的HOSTNAME(因为就两个节点，不是0就是1)
PARTNER=${HOSTNAME##*-}
if [ "${PARTNER}" = "0" ]; then
        PARTNER=${MY_POD_NAME}-1
fi
if [ "${PARTNER}" = "1" ]; then
        PARTNER=${MY_POD_NAME}-0
fi

#判断另一个节点的监听状态(如果在监听说明是主节点需要复制它，否则反之)
</dev/tcp/${PARTNER}.${MY_POD_NAME}.${NAMESPACE}.svc.cluster.local/${COPY_PORT}
if [[ $? == 0 ]];then
        # 复制master节点
        memcached -p 11211 -X ${COPY_PORT} -x ${PARTNER}.${MY_POD_NAME}.${NAMESPACE}.svc.cluster.local -v -d -u root -m ${MEMCACHED_CACHE_SIZE}
else
        # 启动master节点
        memcached -p 11211 -X ${COPY_PORT} -v -d -u root -m ${MEMCACHED_CACHE_SIZE}
fi

测试

主备切换

//启动主节点后状态为Listen，设置同步端口为22222
root@md-5c107eeae9fb1-0:/# memcached -p 11211 -X 22222 -v -d -u root -m 512
root@md-5c107eeae9fb1-0:/# replication: listen
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:11211           0.0.0.0:*               LISTEN      260/memcached
tcp        0      0 0.0.0.0:22222           0.0.0.0:*               LISTEN      260/memcached
tcp6       0      0 :::11211                :::*                    LISTEN      260/memcached

//启动复制节点，同样设置同步端口为22222
root@md-5c107eeae9fb1-1:/# memcached -p 11211 -X 22222 -x md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local -v -d -u root -m 512
root@md-5c107eeae9fb1-1:/# replication: connect (peer=10.244.5.50:22222)
replication: marugoto copying
replication: start

//从节点加入后，主节点状态为accept并取消对22222的监听
root@md-5c107eeae9fb1-0:/# replication: listen
replication: accept
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:11211           0.0.0.0:*               LISTEN      263/memcached
tcp6       0      0 :::11211                :::*                    LISTEN      263/memcached

//kill掉主节点
root@md-5c107eeae9fb1-0:/# kill 263
replication: cleanup start
replication: close
root@md-5c107eeae9fb1-0:/# replication: cleanup complete

//此时从节点成Listen状态，监听复制端口22222，等待加入
root@md-5c107eeae9fb1-1:/# replication: connect (peer=10.244.5.50:22222)
replication: marugoto copying
replication: start
replication: close
replication: listen
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:11211           0.0.0.0:*               LISTEN      258/memcached
tcp        0      0 0.0.0.0:22222           0.0.0.0:*               LISTEN      258/memcached
tcp6       0      0 :::11211                :::*                    LISTEN      258/memcached

高可用

//启动2个节点（一主一备）
md-5c107eeae9fb1-0                  1/1       Running     0          59s
md-5c107eeae9fb1-1                  1/1       Running     0          49s

//在主节点中添加item
$ telnet md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local 11211
Trying 10.244.4.162...
Connected to md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local.
Escape character is '^]'.
set name 0 0 8
zhangsan
STORED
quit
Connection closed by foreign host.

//从节点查看item
$ telnet md-5c107eeae9fb1-1.md-5c107eeae9fb1.itfarm3.svc.cluster.local 11211
Trying 10.244.3.176...
Connected to md-5c107eeae9fb1-1.md-5c107eeae9fb1.itfarm3.svc.cluster.local.
Escape character is '^]'.
get name
VALUE name 0 8
zhangsan
END

//模拟故障
$ kubectl.exe delete po md-5c107eeae9fb1-0
pod "md-5c107eeae9fb1-0" deleted

//重启后数据恢复
$ telnet md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local 11211
Trying 10.244.4.162...
Connected to md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local.
Escape character is '^]'.
get name
VALUE name 0 8
zhangsan
END

部署sasl安全认证的memcached

测试

//使用statefulset启动N个memcached
md-5c11b67a6ae9b-0                  1/1       Running            0          1h
md-5c11b67a6ae9b-1                  1/1       Running            0          1h
md-5c11b67a6ae9b-2                  1/1       Running            0          1h

使用python连接memcached

$ python3
Python 3.5.2 (default, Nov 23 2017, 16:37:01)
[GCC 5.4.0 20160609] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import bmemcached
>>> client = bmemcached.Client(('md-5c11b67a6ae9b-ha:11211'), 'md-5c11b67a6ae9b', 'ZcUu1AVGbB3PnDzhxfOjRLJhatugwZBe')
>>> client.set('name', 'zhangsan')
True
>>> client.get('name')
'zhangsan'
>>>

分布式存储

使用无头服务来区分多个memcache

#!/usr/bin/env python3
import bmemcached
client = bmemcached.Client(('md-5c11b67a6ae9b-0.md-5c11b67a6ae9b.itfarm3.svc.cluster.local:11211',
    'md-5c11b67a6ae9b-1.md-5c11b67a6ae9b.itfarm3.svc.cluster.local:11211',
    'md-5c11b67a6ae9b-1.md-5c11b67a6ae9b.itfarm3.svc.cluster.local:11211'), 'md-5c11b67a6ae9b', 'ZcUu1AVGbB3PnDzhxfOjRLJhatugwZBe')
print(client.set('name', 'zhangsan'))
print(client.get('name'))

运行

$ python3 test.py
True
zhangsan

各客户端连接开启了sasl安全认证的memcached方案