前提：基础镜像的构建请参考分层镜像构建并部署业务到Kubernetes集群生产案例
PV/PVC及Redis单机业务容器化
架构及部署
- 架构图
- 单机redis迁移至kubernetes步骤
- 构建redis镜像
#准备镜像构建文件
[root@K8s-ansible redis]#ls
Dockerfile build-command.sh redis-4.0.14.tar.gz redis.conf run_redis.sh
#redis配置 - 更多优化方式参考redis专题博客
[root@K8s-ansible redis]#grep -Ev "^ *#|^$" redis.conf
bind 0.0.0.0
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis_6379.pid
loglevel notice
logfile ""
databases 16
always-show-logo yes
save 900 1 #定义触发快照规则 - save m秒 n次变化 可配置多个
save 5 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error no #后台出现错误时不停止写入
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /data/redis-data #快照存放
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
requirepass 123456
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble no
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
#redis服务启动脚本
[root@K8s-ansible redis]#cat run_redis.sh
#!/bin/bash
/usr/sbin/redis-server /usr/local/redis/redis.conf
tail -f /etc/hosts
#镜像构建
[root@K8s-ansible redis]#vim Dockerfile
[root@K8s-ansible redis]#cat Dockerfile
#Redis Image
FROM K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009 #以构建的基础镜像
MAINTAINER mooreyxia "mooreyxia@gmail.com"
ADD redis-4.0.14.tar.gz /usr/local/src #源码
RUN ln -sv /usr/local/src/redis-4.0.14 /usr/local/redis && cd /usr/local/redis && make && cp src/redis-cli /usr/sbin/ && cp src/redis-server /usr/sbin/ && mkdir -pv /data/redis-data
ADD redis.conf /usr/local/redis/redis.conf #配置文件
ADD run_redis.sh /usr/local/redis/run_redis.sh
EXPOSE 6379
CMD ["/usr/local/redis/run_redis.sh"] #拉起redis服务脚本
#生成镜像并上传到harbor
[root@K8s-ansible redis]#cat build-command.sh
#!/bin/bash
TAG=$1
docker build -t K8s-harbor01.mooreyxia.com/demo/redis:${TAG} .
sleep 3
docker push K8s-harbor01.mooreyxia.com/demo/redis:${TAG}
[root@K8s-ansible redis]#bash build-command.sh 4.0.14
...
Successfully built f4c0b8e14f7e
Successfully tagged K8s-harbor01.mooreyxia.com/demo/redis:4.0.14
The push refers to repository [K8s-harbor01.mooreyxia.com/demo/redis]
ef00240fbc60: Pushed
ecfb952b7eeb: Pushed
deca2c414cae: Pushed
7a54ec7cbc12: Pushed
3e6fa9b75f89: Mounted from demo/nginx-web1
cf71274b159a: Mounted from demo/nginx-web1
174f56854903: Mounted from demo/nginx-web1
4.0.14: digest: sha256:b41df008ed3ae9b274999e52a277d6216997a929d5632f5e0230d67fd9ac83f7 size: 1793- 测试redis 镜像
#运行容器
[root@K8s-ansible redis]#docker run -it -p 6379:6379 K8s-harbor01.mooreyxia.com/demo/redis:4.0.14
7:C 11 Apr 15:26:33.304 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
7:C 11 Apr 15:26:33.305 # Redis versinotallow=4.0.14, bits=64, commit=00000000, modified=0, pid=7, just started
7:C 11 Apr 15:26:33.306 # Configuration loaded
127.0.0.1 localhost
::1 localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
172.17.0.4 ec744d12b8d4
#测试访问
[root@K8s-harbor01 ~]#telnet 192.168.11.205 6379
Trying 192.168.11.205...
Connected to 192.168.11.205.
Escape character is '^]'.
auth 123456
+OK
info
$2726
# Server
redis_version:4.0.14
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:598238f247b3743a
redis_mode:standalone
os:Linux 5.15.0-69-generic x86_64
arch_bits:64
...- 创建PV与PVC
[root@K8s-ansible redis]#tree .
.
├── pv
│ ├── redis-persistentvolume.yaml
│ └── redis-persistentvolumeclaim.yaml
└── redis.yaml
1 directory, 3 files
#创建NFS存储
[root@K8s-haproxy01 ~]#mkdir -p /data/k8sdata/mooreyxia/redis-datadir-1
[root@K8s-haproxy01 ~]#vim /etc/exports
[root@K8s-haproxy01 ~]#cat /etc/exports
# /etc/exports: the access control list for filesystems which may be exported
# to NFS clients. See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4 gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes gss/krb5i(rw,sync,no_subtree_check)
#
/data/k8sdata *(rw,no_root_squash)
/data/volumes *(rw,no_root_squash)
[root@K8s-haproxy01 ~]#exportfs -avs
exportfs: /etc/exports [2]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/k8sdata".
Assuming default behaviour ('no_subtree_check').
NOTE: this default has changed since nfs-utils version 1.0.x
exportfs: /etc/exports [3]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/volumes".
Assuming default behaviour ('no_subtree_check').
NOTE: this default has changed since nfs-utils version 1.0.x
exporting *:/data/volumes
exporting *:/data/k8sdata
#showmount
[root@K8s-ansible redis]#showmount -e 192.168.11.203
Export list for 192.168.11.203:
/data/volumes *
/data/k8sdata *
#创建pv
[root@K8s-ansible redis]#cat pv/redis-persistentvolume.yaml
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-datadir-pv-1
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
nfs:
path: /data/k8sdata/mooreyxia/redis-datadir-1
server: 192.168.11.203
[root@K8s-ansible redis]#kubectl apply -f pv/redis-persistentvolume.yaml
persistentvolume/redis-datadir-pv-1 created
[root@K8s-ansible redis]#kubectl get pv|grep redis-datadir-pv-1
redis-datadir-pv-1 10Gi RWO Retain Available 23s
#创建pvc
[root@K8s-ansible redis]#cat pv/redis-persistentvolumeclaim.yaml
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: redis-datadir-pvc-1
namespace: mooreyxia
spec:
volumeName: redis-datadir-pv-1
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
[root@K8s-ansible redis]#kubectl apply -f pv/redis-persistentvolumeclaim.yaml
persistentvolumeclaim/redis-datadir-pvc-1 created
[root@K8s-ansible redis]#kubectl get pvc -n mooreyxia|grep redis-datadir-pvc-1
redis-datadir-pvc-1 Bound redis-datadir-pv-1 10Gi RWO 90s- 运行Redis服务
#创建控制器并运行
[root@K8s-ansible redis]#cat redis.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
app: devops-redis
name: deploy-devops-redis
namespace: mooreyxia
spec:
replicas: 1
selector:
matchLabels:
app: devops-redis
template:
metadata:
labels:
app: devops-redis
spec:
containers:
- name: redis-container
image: K8s-harbor01.mooreyxia.com/demo/redis:4.0.14
imagePullPolicy: Always
volumeMounts:
- mountPath: "/data/redis-data/"
name: redis-datadir
volumes:
- name: redis-datadir
persistentVolumeClaim:
claimName: redis-datadir-pvc-1
---
kind: Service
apiVersion: v1
metadata:
labels:
app: devops-redis
name: srv-devops-redis
namespace: mooreyxia
spec:
type: NodePort
ports:
- name: http
port: 6379
targetPort: 6379
nodePort: 30379 #这里要在kubernetes设定的端口范围内
selector:
app: devops-redis
sessionAffinity: ClientIP #亲和,pod内部访问
sessionAffinityConfig:
clientIP:
timeoutSeconds: 10800
[root@K8s-ansible redis]#kubectl apply -f redis.yaml
deployment.apps/deploy-devops-redis unchanged
service/srv-devops-redis created
#确认pod运行没有报错
[root@K8s-ansible redis]#kubectl get svc -n mooreyxia|grep srv-devops-redis
srv-devops-redis NodePort 10.100.185.130 <none> 6379:30379/TCP 47s
[root@K8s-ansible redis]#kubectl get pod -n mooreyxia|grep devops-redis
deploy-devops-redis-54c95d8765-rzl85 1/1 Running 0 3m41s
[root@K8s-ansible redis]#kubectl describe pod deploy-devops-redis-54c95d8765-rzl85 -n mooreyxia
Name: deploy-devops-redis-54c95d8765-rzl85
Namespace: mooreyxia
Priority: 0
Service Account: default
Node: 192.168.11.215/192.168.11.215
Start Time: Tue, 11 Apr 2023 07:53:46 +0000
Labels: app=devops-redis
pod-template-hash=54c95d8765
Annotations: <none>
Status: Running
IP: 10.200.67.39
IPs:
IP: 10.200.67.39
Controlled By: ReplicaSet/deploy-devops-redis-54c95d8765
Containers:
redis-container:
Container ID: containerd://47ecdeb9e8b7a8639c21c829d572d392d70c694e1a10e63cc451f22bf297aa67
Image: K8s-harbor01.mooreyxia.com/demo/redis:4.0.14
Image ID: K8s-harbor01.mooreyxia.com/demo/redis@sha256:b41df008ed3ae9b274999e52a277d6216997a929d5632f5e0230d67fd9ac83f7
Port: <none>
Host Port: <none>
State: Running
Started: Tue, 11 Apr 2023 07:53:54 +0000
Ready: True
Restart Count: 0
Environment: <none>
Mounts:
/data/redis-data/ from redis-datadir (rw)
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6cmz4 (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
redis-datadir:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: redis-datadir-pvc-1
ReadOnly: false
kube-api-access-6cmz4:
Type: Projected (a volume that contains injected data from multiple sources)
TokenExpirationSeconds: 3607
ConfigMapName: kube-root-ca.crt
ConfigMapOptional: <nil>
DownwardAPI: true
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 4m1s default-scheduler Successfully assigned mooreyxia/deploy-devops-redis-54c95d8765-rzl85 to 192.168.11.215
Normal Pulling 3m59s kubelet Pulling image "K8s-harbor01.mooreyxia.com/demo/redis:4.0.14"
Normal Pulled 3m54s kubelet Successfully pulled image "K8s-harbor01.mooreyxia.com/demo/redis:4.0.14" in 5.810379411s (5.810427887s including waiting)
Normal Created 3m53s kubelet Created container redis-container
Normal Started 3m53s kubelet Started container redis-container
[root@K8s-ansible redis]#kubectl logs deploy-devops-redis-54c95d8765-rzl85 -n mooreyxia
8:C 11 Apr 15:53:54.232 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
8:C 11 Apr 15:53:54.232 # Redis versinotallow=4.0.14, bits=64, commit=00000000, modified=0, pid=8, just started
8:C 11 Apr 15:53:54.232 # Configuration loaded
# Kubernetes-managed hosts file.
127.0.0.1 localhost
::1 localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
fe00::0 ip6-mcastprefix
fe00::1 ip6-allnodes
fe00::2 ip6-allrouters
10.200.67.39 deploy-devops-redis-54c95d8765-rzl85- 验证Redis数据读写
#远程客户端连接,注意:Redis没有用户的概念
redis-cli -h <Redis服务器IP> -p <PORT> -a <PASSWORD> --no-auth-warning
--------------------------------------------------------------------------
#进入pod验证
[root@K8s-ansible redis]#kubectl exec -it deploy-devops-redis-54c95d8765-rzl85 -n mooreyxia bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@deploy-devops-redis-54c95d8765-rzl85 /]# redis-cli
127.0.0.1:6379> auth 123456
OK
127.0.0.1:6379> set name mooreyxia
OK
127.0.0.1:6379> get name
"mooreyxia"
127.0.0.1:6379> set name MOOREYXIA
OK
127.0.0.1:6379> get name
"MOOREYXIA"
127.0.0.1:6379> DEL name
(integer) 1
127.0.0.1:6379> get name
(nil)
--------------------------------------------------------------------------
[root@K8s-haproxy01 ~]#cat redis_test.sh
#!/bin/bash
NUM=500
PASS=123456
for i in `seq $NUM`;do
redis-cli -h 192.168.11.211 -p 30379 -a "$PASS" --no-auth-warning set key${i} value${i}
echo "key${i} value${i} 写入完成"
done
echo "$NUM个key写入到Redis完成"
#批量写入
[root@K8s-haproxy01 ~]#apt install redis
[root@K8s-haproxy01 ~]#bash redis_test.sh
...
key499 value499 写入完成
OK
key500 value500 写入完成
500个key写入到Redis完成
#进入pod验证
[root@K8s-ansible redis]#kubectl exec -it deploy-devops-redis-54c95d8765-rzl85 -n mooreyxia bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@deploy-devops-redis-54c95d8765-rzl85 /]# redis-cli
127.0.0.1:6379> auth 123456
OK
127.0.0.1:6379> get key499
"value499"
#根据快照生成策略，验证快照是否生成
[root@K8s-ansible redis]#grep -Ev "^ *#|^$" redis.conf |grep save
save 900 1
save 5 1
save 300 10
save 60 10000
...
[root@K8s-haproxy01 ~]#ll /data/k8sdata/mooreyxia/redis-datadir-1/ -h
total 20K
drwxr-xr-x 2 root root 4.0K Apr 11 08:19 ./
drwxr-xr-x 8 root root 4.0K Apr 11 07:36 ../
-rw-r--r-- 1 root root 8.2K Apr 11 08:19 dump.rdb
[root@K8s-haproxy01 ~]#file /data/k8sdata/mooreyxia/redis-datadir-1/dump.rdb
/data/k8sdata/mooreyxia/redis-datadir-1/dump.rdb: Redis RDB file, version 0008
--------------------------------------------------------------------------------
#尝试删除pod，根据控制器策略，kubernetes会重建pod，我们希望数据未丢失
[root@K8s-ansible redis]#kubectl get pod -n mooreyxia|grep devops-redis
deploy-devops-redis-54c95d8765-rzl85 1/1 Running 0 35m
[root@K8s-ansible redis]#kubectl delete pod deploy-devops-redis-54c95d8765-rzl85 -n mooreyxia
pod "deploy-devops-redis-54c95d8765-rzl85" deleted
[root@K8s-ansible redis]#kubectl get pod -n mooreyxia|grep devops-redis
deploy-devops-redis-54c95d8765-zmxfw 1/1 Running 0 45s
[root@K8s-ansible redis]#kubectl exec -it deploy-devops-redis-54c95d8765-zmxfw -n mooreyxia bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@deploy-devops-redis-54c95d8765-zmxfw /]# redis-cli
127.0.0.1:6379> auth
(error) ERR wrong number of arguments for 'auth' command
127.0.0.1:6379> auth 123456
OK
127.0.0.1:6379> get key499 #数据未丢失
"value499"
127.0.0.1:6379> get key500
"value500"PV/PVC及RedisCluster集群业务容器化
架构及部署
- 架构图
- 集群redis迁移至kubernetes步骤
- 创建存储
#这里用NFS，生产根据需要调整
[root@K8s-haproxy01 ~]#apt install nfs-server
[root@K8s-haproxy01 ~]#mkdir -pv /data/k8sdata/mooreyxia/{redis0,redis1,redis2,redis3,redis4,redis5}
mkdir: created directory '/data/k8sdata/mooreyxia/redis0'
mkdir: created directory '/data/k8sdata/mooreyxia/redis1'
mkdir: created directory '/data/k8sdata/mooreyxia/redis2'
mkdir: created directory '/data/k8sdata/mooreyxia/redis3'
mkdir: created directory '/data/k8sdata/mooreyxia/redis4'
mkdir: created directory '/data/k8sdata/mooreyxia/redis5'
[root@K8s-haproxy01 ~]#vim /etc/exports
[root@K8s-haproxy01 ~]#cat /etc/exports
# /etc/exports: the access control list for filesystems which may be exported
# to NFS clients. See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4 gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes gss/krb5i(rw,sync,no_subtree_check)
#
/data/k8sdata *(rw,no_root_squash)
/data/volumes *(rw,no_root_squash)
[root@K8s-haproxy01 ~]#exportfs -avs
exportfs: /etc/exports [2]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/k8sdata".
Assuming default behaviour ('no_subtree_check').
NOTE: this default has changed since nfs-utils version 1.0.x
exportfs: /etc/exports [3]: Neither 'subtree_check' or 'no_subtree_check' specified for export "*:/data/volumes".
Assuming default behaviour ('no_subtree_check').
NOTE: this default has changed since nfs-utils version 1.0.x
exporting *:/data/volumes
exporting *:/data/k8sdata
#showmount
[root@K8s-ansible redis]#showmount -e 192.168.11.203
Export list for 192.168.11.203:
/data/volumes *
/data/k8sdata *- 创建PV与PVC
[root@K8s-ansible redis-cluster]#vim pv/redis-cluster-pv.yaml
[root@K8s-ansible redis-cluster]#cat pv/redis-cluster-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv0
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.11.203
path: /data/k8sdata/mooreyxia/redis0
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv1
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.11.203
path: /data/k8sdata/mooreyxia/redis1
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv2
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.11.203
path: /data/k8sdata/mooreyxia/redis2
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv3
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.11.203
path: /data/k8sdata/mooreyxia/redis3
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv4
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.11.203
path: /data/k8sdata/mooreyxia/redis4
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: redis-cluster-pv5
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.11.203
path: /data/k8sdata/mooreyxia/redis5
[root@K8s-ansible redis-cluster]#kubectl apply -f pv/redis-cluster-pv.yaml
persistentvolume/redis-cluster-pv0 created
persistentvolume/redis-cluster-pv1 created
persistentvolume/redis-cluster-pv2 created
persistentvolume/redis-cluster-pv3 created
persistentvolume/redis-cluster-pv4 created
persistentvolume/redis-cluster-pv5 created
#确认pv可用
[root@K8s-ansible redis-cluster]#kubectl get pv |grep redis-cluster
redis-cluster-pv0 5Gi RWO Retain Available 44s
redis-cluster-pv1 5Gi RWO Retain Available 44s
redis-cluster-pv2 5Gi RWO Retain Available 44s
redis-cluster-pv3 5Gi RWO Retain Available 44s
redis-cluster-pv4 5Gi RWO Retain Available 44s
redis-cluster-pv5 5Gi RWO Retain Available 44s- 部署redis cluster
-----------------------------------------------------------------------------------
#基于配置文件创建configmap(或者提前将配置文件加入到基础镜像中)
[root@K8s-ansible redis-cluster]#tree .
.
├── pv
│ └── redis-cluster-pv.yaml
├── redis.conf
└── redis.yaml
1 directory, 3 files
#配置文件 - 简化，生产根据实际需要进行优化
[root@K8s-ansible redis-cluster]#vim redis.conf
[root@K8s-ansible redis-cluster]#cat redis.conf
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379
#基于配置文件创建configmap
[root@K8s-ansible redis-cluster]#kubectl create configmap redis-conf --from-file=redis.conf -n mooreyxia
configmap/redis-conf created
#验证configmap：
[root@K8s-ansible redis-cluster]#kubectl get configmaps -n mooreyxia |grep redis
redis-conf 1 86s
#想看的更详细加参数-o json 或者 -o yaml，此处省略
[root@K8s-ansible redis-cluster]#kubectl describe configmaps redis-conf -n mooreyxia
Name: redis-conf
Namespace: mooreyxia
Labels: <none>
Annotations: <none>
Data
====
redis.conf:
----
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379
BinaryData
====
Events: <none>
-----------------------------------------------------------------------------------
#创建redis-cluster需要使用的所有pod节点,创建后为单机模式
[root@K8s-ansible redis-cluster]#vim redis.yaml
[root@K8s-ansible redis-cluster]#cat redis.yaml
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: mooreyxia
labels:
app: redis
spec:
selector:
app: redis
appCluster: redis-cluster
ports:
- name: redis
port: 6379
clusterIP: None #用于内部服务基于service name的访问
---
apiVersion: v1
kind: Service
metadata:
name: redis-access
namespace: mooreyxia
labels:
app: redis
spec:
selector:
app: redis
appCluster: redis-cluster
ports:
- name: redis-access
protocol: TCP
port: 6379
targetPort: 6379 #用于集群外访问
---
apiVersion: apps/v1
kind: StatefulSet #创建有状态控制器
metadata:
name: redis #StatefulSet控制器名称
namespace: mooreyxia
spec:
serviceName: redis
replicas: 6 #副本数
selector:
matchLabels:
app: redis
appCluster: redis-cluster
template:
metadata:
labels:
app: redis
appCluster: redis-cluster
spec:
terminationGracePeriodSeconds: 20
affinity: #以主机hostname做pod亲和
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- redis
topologyKey: kubernetes.io/hostname
containers:
- name: redis
image: redis:4.0.14 #镜像
command: #启动服务并传参
- "redis-server"
args:
- "/etc/redis/redis.conf"
- "--protected-mode"
- "no"
resources:
requests: #资源限制
cpu: "500m"
memory: "500Mi"
ports: #集群情况下端口设定
- containerPort: 6379 #客户端端口
name: redis
protocol: TCP
- containerPort: 16379 #集群端口 - 默认在客户端端口前+1
name: cluster
protocol: TCP
volumeMounts: #挂载
- name: conf
mountPath: /etc/redis
- name: data
mountPath: /var/lib/redis
volumes:
- name: conf
configMap:
name: redis-conf
items:
- key: redis.conf
path: redis.conf
volumeClaimTemplates: #以模板的形式批量创建pvc - 模板名称+StatefulSet控制器名称+id(data-redis-number)
- metadata:
name: data #模板名称
namespace: mooreyxia
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 5Gi
#确认pod运行 - 注意这里的redis-pod都是单机，集群配置需要单独创建并初始化，涉及到槽位划分16384
[root@K8s-ansible redis-cluster]#kubectl get pod -n mooreyxia
NAME READY STATUS RESTARTS AGE
redis-0 1/1 Running 0 140m
redis-1 1/1 Running 0 139m
redis-2 1/1 Running 0 139m
redis-3 1/1 Running 0 138m
redis-4 1/1 Running 0 138m
redis-5 1/1 Running 0 138m
#确认pvc创建 - 模板名称+StatefulSet控制器名称+id
[root@K8s-ansible redis-cluster]#kubectl get pvc -n mooreyxia
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
data-redis-0 Bound redis-cluster-pv1 5Gi RWO 149m
data-redis-1 Bound redis-cluster-pv4 5Gi RWO 148m
data-redis-2 Bound redis-cluster-pv2 5Gi RWO 148m
data-redis-3 Bound redis-cluster-pv0 5Gi RWO 148m
data-redis-4 Bound redis-cluster-pv3 5Gi RWO 147m
data-redis-5 Bound redis-cluster-pv5 5Gi RWO 147m- 初始化redis cluster
#初始化只需要初始化一次，redis 4 及之前的版本需要使用redis-tribe 工具进行初始化，redis5 开始使用redis-cli。
#创建初始化pod - 这里使用redis-tribe进行初始化，命令使用方式和redis-cli基本相同 restart=Never 退出后不启用
[root@K8s-ansible redis-cluster]#kubectl run -it ubuntu1804 --image=ubuntu:18.04 --restart=Never -n mooreyxia bash
If you don't see a command prompt, try pressing enter.
#环境初始化-安装redis-tribe
root@ubuntu1804:/# apt install python2.7 python-pip redis-tools dnsutils iputils-ping net-tools
root@ubuntu1804:/# pip install --upgrade pip
root@ubuntu1804:/# pip install redis-trib==0.5.1
#确认redis-pod服务可以被解析
root@ubuntu1804:/# dig redis-0.redis.mooreyxia.svc.mooreyxia.local
; <<>> DiG 9.11.3-1ubuntu1.18-Ubuntu <<>> redis-0.redis.mooreyxia.svc.mooreyxia.local
;; global options: +cmd
;; Got answer:
;; WARNING: .local is reserved for Multicast DNS
;; You are currently testing what happens when an mDNS query is leaked to DNS
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 63752
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1
;; WARNING: recursion requested but not available
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
; COOKIE: 6b3ba2281c9cf58a (echoed)
;; QUESTION SECTION:
;redis-0.redis.mooreyxia.svc.mooreyxia.local. IN A
;; ANSWER SECTION:
redis-0.redis.mooreyxia.svc.mooreyxia.local. 30 IN A 10.200.67.40
;; Query time: 15 msec
;; SERVER: 10.100.0.2#53(10.100.0.2)
;; WHEN: Tue Apr 11 13:22:37 UTC 2023
;; MSG SIZE rcvd: 143
#创建集群
root@ubuntu1804:/# redis-trib.py create \
> `dig +short redis-0.redis.mooreyxia.svc.mooreyxia.local`:6379 \
> `dig +short redis-1.redis.mooreyxia.svc.mooreyxia.local`:6379 \
> `dig +short redis-2.redis.mooreyxia.svc.mooreyxia.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.200.209.43:6379 checked
INFO:root:Instance at 10.200.67.40:6379 checked
INFO:root:Instance at 10.200.128.181:6379 checked
INFO:root:Add 5462 slots to 10.200.209.43:6379 #自动分配槽位，合计16384
INFO:root:Add 5461 slots to 10.200.67.40:6379
INFO:root:Add 5461 slots to 10.200.128.181:6379
#接下来为集群中每个pod增加一个副本节点，做高可用。
#虽然k8s会在pod节点宕机时自动重建，但是存在延迟，建议利用服务本身的高可用方案
#将redis-3 加入redis-0：
root@ubuntu1804:/# redis-trib.py replicate \
> --master-addr `dig +short redis-0.redis.magedu.svc.magedu.local`:6379 \
> ^Cslave-addr `dig +short redis-3.redis.magedu.svc.magedu.local`:6379
root@ubuntu1804:/# redis-trib.py replicate \
> --master-addr `dig +short redis-0.redis.mooreyxia.svc.mooreyxia.local`:6379 \
> --slave-addr `dig +short redis-3.redis.mooreyxia.svc.mooreyxia.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.200.67.41:6379 has joined 10.200.67.40:6379; now set replica
INFO:root:Instance at 10.200.67.41:6379 set as replica to 98d89d30afda50c347f3420e6d97842214bb885f
#将redis-4 加入redis-1：
root@ubuntu1804:/# redis-trib.py replicate \
> --master-addr `dig +short redis-1.redis.mooreyxia.svc.mooreyxia.local`:6379 \
> --slave-addr `dig +short redis-4.redis.mooreyxia.svc.mooreyxia.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.200.128.182:6379 has joined 10.200.128.181:6379; now set replica
INFO:root:Instance at 10.200.128.182:6379 set as replica to b54dc4d56255058339435d86aaab978140617caf
#将redis-5 加入redis-2：
root@ubuntu1804:/# redis-trib.py replicate \
> --master-addr `dig +short redis-2.redis.mooreyxia.svc.mooreyxia.local`:6379 \
> --slave-addr `dig +short redis-5.redis.mooreyxia.svc.mooreyxia.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.200.209.44:6379 has joined 10.200.209.43:6379; now set replica
INFO:root:Instance at 10.200.209.44:6379 set as replica to ed496e09c658f6d09a8ce664e41f0e4dec580fa7
#完成集群搭建后退出初始化pod，下一步验证redis cluster状态
root@ubuntu1804:/# exit
exit- 验证redis cluster状态
[root@K8s-ansible redis-cluster]#kubectl get pod -n mooreyxia|grep redis-
deploy-devops-redis-54c95d8765-zmxfw 1/1 Running 0 5h6m
redis-0 1/1 Running 0 3h42m
redis-1 1/1 Running 0 3h42m
redis-2 1/1 Running 0 3h41m
redis-3 1/1 Running 0 3h41m
redis-4 1/1 Running 0 3h40m
redis-5 1/1 Running 0 3h40m
#进入任意集群节点内确认redis状态
[root@K8s-ansible redis-cluster]#kubectl exec -it redis-0 bash -n mooreyxia
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@redis-0:/data# redis-cli
127.0.0.1:6379> CLUSTER INFO
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:4
cluster_my_epoch:2
cluster_stats_messages_ping_sent:1116
cluster_stats_messages_pong_sent:1172
cluster_stats_messages_meet_sent:3
cluster_stats_messages_sent:2291
cluster_stats_messages_ping_received:1170
cluster_stats_messages_pong_received:1119
cluster_stats_messages_meet_received:2
cluster_stats_messages_received:2291
127.0.0.1:6379> CLUSTER NODES
301d3e34d8a0f2951bafca67f3ee05bcab49e5fa 10.200.209.44:6379@16379 slave ed496e09c658f6d09a8ce664e41f0e4dec580fa7 0 1681220382000 4 connected
ed496e09c658f6d09a8ce664e41f0e4dec580fa7 10.200.209.43:6379@16379 master - 0 1681220382161 4 connected 0-5461
98d89d30afda50c347f3420e6d97842214bb885f 10.200.67.40:6379@16379 myself,master - 0 1681220381000 2 connected 5462-10922
b54dc4d56255058339435d86aaab978140617caf 10.200.128.181:6379@16379 master - 0 1681220382000 1 connected 10923-16383
2dbe646d7bac54811f46cb57305970cc45623e0a 10.200.67.41:6379@16379 slave 98d89d30afda50c347f3420e6d97842214bb885f 0 1681220382566 3 connected
f00b2ceb87eaf4a3d62153692726ead615420162 10.200.128.182:6379@16379 slave b54dc4d56255058339435d86aaab978140617caf 0 1681220381000 1 connected
127.0.0.1:6379> info
# Server
redis_version:4.0.14
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:165c932261a105d7
redis_mode:cluster
os:Linux 5.15.0-69-generic x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:8.3.0
process_id:1
run_id:acb6353ce676c7cf63a92436a73972fbcb8c6c2c
tcp_port:6379
uptime_in_seconds:14544
uptime_in_days:0
hz:10
lru_clock:3499826
executable:/data/redis-server
config_file:/etc/redis/redis.conf
# Clients
connected_clients:1
client_longest_output_list:0
client_biggest_input_buf:0
blocked_clients:0
# Memory
used_memory:2643360
used_memory_human:2.52M
used_memory_rss:5111808
used_memory_rss_human:4.88M
used_memory_peak:2683264
used_memory_peak_human:2.56M
used_memory_peak_perc:98.51%
used_memory_overhead:2559936
used_memory_startup:1444856
used_memory_dataset:83424
used_memory_dataset_perc:6.96%
total_system_memory:2071683072
total_system_memory_human:1.93G
used_memory_lua:37888
used_memory_lua_human:37.00K
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
mem_fragmentation_ratio:1.93
mem_allocator:jemalloc-4.0.3
active_defrag_running:0
lazyfree_pending_objects:0
# Persistence
loading:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1681219901
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:0
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:315392
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:0
aof_current_size:0
aof_base_size:0
aof_pending_rewrite:0
aof_buffer_length:0
aof_rewrite_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:0
# Stats
total_connections_received:4
total_commands_processed:1545
instantaneous_ops_per_sec:0
total_net_input_bytes:111791
total_net_output_bytes:25844
instantaneous_input_kbps:0.02
instantaneous_output_kbps:0.00
rejected_connections:0
sync_full:1
sync_partial_ok:0
sync_partial_err:1
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
evicted_keys:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:769
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
# Replication
role:master
connected_slaves:1 #有一个slave节点连接，下面是备份策略
slave0:ip=10.200.67.41,port=6379,state=online,offset=2128,lag=1
master_replid:e038a1203874d441a2611089bd7bfa22cd9a1a4a
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:2128
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:2128
# CPU
used_cpu_sys:14.78
used_cpu_user:6.83
used_cpu_sys_children:0.00
used_cpu_user_children:0.00
# Cluster
cluster_enabled:1
#测试写入数据:需要根据槽位分配的位置写入
127.0.0.1:6379> set key1 valye1
OK
127.0.0.1:6379> set key2 valye2
(error) MOVED 4998 10.200.209.43:6379 #这里告诉用户需要到指定的节点写入
127.0.0.1:6379> exit
root@redis-0:~# exit
exit
[root@K8s-ansible redis-cluster]#kubectl get pod -n mooreyxia -o wide|grep redis-
deploy-devops-redis-54c95d8765-zmxfw 1/1 Running 0 5h38m 10.200.128.180 192.168.11.216 <none> <none>
redis-0 1/1 Running 0 4h14m 10.200.67.40 192.168.11.215 <none> <none>
redis-1 1/1 Running 0 4h14m 10.200.128.181 192.168.11.216 <none> <none>
redis-2 1/1 Running 0 4h13m 10.200.209.43 192.168.11.214 <none> <none>
redis-3 1/1 Running 0 4h13m 10.200.67.41 192.168.11.215 <none> <none>
redis-4 1/1 Running 0 4h13m 10.200.128.182 192.168.11.216 <none> <none>
redis-5 1/1 Running 0 4h13m 10.200.209.44 192.168.11.214 <none> <none>
[root@K8s-ansible redis-cluster]#kubectl exec -it redis-2 bash -n mooreyxia
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@redis-2:/data# redis-cli
127.0.0.1:6379> set key2 valye2
OK
127.0.0.1:6379> KEYS *
1) "key2"- 验证redis cluster高可用
#删除任意pod主节点，检查对应的副本是否提升为master，此处省略
#确认集群配置开启的数据持久化是否生效-到数据目录中确认
[root@K8s-haproxy01 ~]#tree /data/k8sdata/mooreyxia/redis*
/data/k8sdata/mooreyxia/redis-datadir-1
└── dump.rdb
/data/k8sdata/mooreyxia/redis0
├── appendonly.aof #aof
├── dump.rdb #快照
└── nodes.conf #集群配置
/data/k8sdata/mooreyxia/redis1
├── appendonly.aof
├── dump.rdb
└── nodes.conf
/data/k8sdata/mooreyxia/redis2
├── appendonly.aof
├── dump.rdb
└── nodes.conf
/data/k8sdata/mooreyxia/redis3
├── appendonly.aof
├── dump.rdb
└── nodes.conf
/data/k8sdata/mooreyxia/redis4
├── appendonly.aof
├── dump.rdb
└── nodes.conf
/data/k8sdata/mooreyxia/redis5
├── appendonly.aof
├── dump.rdb
└── nodes.conf
0 directories, 3 files
#集群配置信息自动生成，可以查看
[root@K8s-haproxy01 ~]#cat /data/k8sdata/mooreyxia/redis0/nodes.conf
301d3e34d8a0f2951bafca67f3ee05bcab49e5fa 10.200.209.44:6379@16379 slave ed496e09c658f6d09a8ce664e41f0e4dec580fa7 0 1681220041599 4 connected
ed496e09c658f6d09a8ce664e41f0e4dec580fa7 10.200.209.43:6379@16379 master - 0 1681220041197 4 connected 0-5461
f00b2ceb87eaf4a3d62153692726ead615420162 10.200.128.182:6379@16379 slave b54dc4d56255058339435d86aaab978140617caf 0 1681220040195 1 connected
98d89d30afda50c347f3420e6d97842214bb885f 10.200.67.40:6379@16379 master - 0 1681220040000 2 connected 5462-10922
b54dc4d56255058339435d86aaab978140617caf 10.200.128.181:6379@16379 master - 0 1681220041000 1 connected 10923-16383
2dbe646d7bac54811f46cb57305970cc45623e0a 10.200.67.41:6379@16379 myself,slave 98d89d30afda50c347f3420e6d97842214bb885f 0 1681220039000 3 connected
vars currentEpoch 4 lastVoteEpoch 0我是moore,大家一起加油！！！
