keeplived安装及高可用实现redis集群

2024年5月30日 182次阅读 0人点赞

keeplived安装及高可用实现redis集群

环境

ip vip 服务
192.168.125.20 192.168.125.50 keeplived/nginx负载均衡/redis从
192.168.125.31 192.168.125.50 keeplived/nginx负载均衡/redis主
192.168.125.21 无 nginx
192.168.125.22 无 nginx

安装 keepalived

yum install keepalived -y

nginx负载均衡配置

两台代理服务器配置nginx的负载均衡
192.168.125.20
192.168.125.31

nginx.conf

upstream webserverx {
        server 192.168.125.21:80;
        server 192.168.125.22:80;
}
server {
    listen       80;
   # server_name  localhost;

    #access_log  /var/log/nginx/host.access.log  main;
    location / {
        proxy_pass http://webserverx;
       # root   /usr/share/nginx/html;
       # index  index.html index.htm;
    }

}

两台nginx 服务器
192.168.125.21
192.168.125.22

keepalived配置

首先查看网卡名称

root@eab5aec2df6c test]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
13: eth0@if14: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.31/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
注意我使用的时wsl2容器中的centos7虚拟机，网卡为eth0

备份配置
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

virtual_server 全部删除到文件末尾

cat >/etc/keepalived/keepalived.conf <<'END'
global_defs {
   router_id LVS_DEVEL
   script_user root #root用户时必须配置
}
##添加自定义脚本
vrrp_script chk_nginx {
    script "/etc/keepalived/scripts/ch_nginx.sh"
    interval 1 #间隔一秒执行一次
    weight 10 #权重。脚本结果的执行导致权重的改变
}
vrrp_instance VI_1 {
    state MASTER  #定义主备
    #修改为实际网卡
    interface eth0
    virtual_router_id 51 #整个集群的调度器一致
    priority 100 #bake改为50
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        chk_nginx
    }
    virtual_ipaddress {
    #这是要增加的虚拟ip
        192.168.125.50
    }
}

END

同一网段中同一组virtual_router_id值相同,不同组virtual_router_id值唯一.
如server-1、server-2为一组，virtual_router_id=51
server-3、server-4为一组，则virtual_router_id不能为51

创建目录
mkdir /etc/keepalived/scripts

新建脚本
cat > /etc/keepalived/scripts/ch_nginx.sh <<'END'
#!/bin/bash
counter=$(ps -C nginx  | grep -v PID  | wc -l)
if [ $counter == 0 ];then
        systemctl restart nginx
        sleep 1s
        counter=$(ps -C nginx  | grep -v PID  | wc -l)
        if [ $counter == 0 ];then
                systemctl stop keepalived
        fi
fi
END
赋予权限
chmod +x /etc/keepalived/scripts/ch_nginx.sh

[root@eab5aec2df6c ~]# scp -r  /etc/keepalived/scripts root@192.168.125.20:/etc/keepalived/

备用服务器

[root@centos70 ~]# mv  /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

主服务器
[root@eab5aec2df6c ~]# scp   /etc/keepalived/keepalived.conf root@192.168.125.20:/etc/keepalived/keepalived.conf

修改配置文件state和priority即可

! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   script_user root #root用户时必须配置
}
##添加自定义脚本
vrrp_script chk_nginx {
    script "/etc/keepalived/scripts/ch_nginx.sh"
    interval 1 #间隔一秒执行一次
    weight 10 #权重。脚本结果的执行导致权重的改变
}
vrrp_instance VI_1 {
    state BACKUP  #备用
    #修改为实际网卡
    interface eth0
    virtual_router_id 51 #整个集群的调度器一致
    priority 50 #bake改为50
    advert_int 1 #检测问题1s
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        chk_nginx
    }
    virtual_ipaddress {
    #这是要增加的虚拟ip
        192.168.125.50
    }
}

vip漂移测试

开始时vip在master
[root@eab5aec2df6c ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
45: eth0@if46: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.31/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 192.168.125.50/32 scope global eth0
       valid_lft forever preferred_lft forever

[root@centos70 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
17: eth0@if18: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:14 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.20/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever


模拟master停机，vip一会就飘移过去了
[root@eab5aec2df6c ~]# systemctl stop keepalived.service
[root@eab5aec2df6c ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
45: eth0@if46: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.31/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
[root@centos70 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
17: eth0@if18: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:14 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.20/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
[root@centos70 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
17: eth0@if18: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:14 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.20/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 192.168.125.50/32 scope global eth0
       valid_lft forever preferred_lft forever


master恢复，一会就拿回vip

[root@eab5aec2df6c ~]# systemctl start keepalived.service
[root@eab5aec2df6c ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
45: eth0@if46: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.31/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 192.168.125.50/32 scope global eth0
       valid_lft forever preferred_lft forever

可以看到主负载服务器停掉之后虚拟ip由主服务器转交给了备用服务器，无缝衔接对两个nginx服务器进行负载

nginx负载均衡测试

《keeplived安装及高可用实现redis集群》
《keeplived安装及高可用实现redis集群》

keepalived日志管理

查看日志
journalctl -u keepalived.service

 /var/log/messages不存在
 安装日志服务
 yum install -y rsyslog
systemctl enable --now rsyslog
安装之后可以通过以下命令查看日志
tail -f /var/log/messages

我们知道，当前系统下几乎所有应用日志基本上都会输出到该文件中，通过我们在验证keepalived的过程中，我们也查看到了该文件中不仅仅输出了keepalived日志，还输出了其他无关的日志。

我们在维护web项目的过程中，一般都会将应用日志单独管理，所以接下来，我们需要将keepalived日志从messages中剥离开，单独存放。

vim /etc/sysconfig/keepalived
KEEPALIVED_OPTIONS="-D -d -S 4"

vim /etc/rsyslog.conf
local4.*                                                /var/log/keepalived.log


systemctl restart rsyslog
systemctl restart keepalived

我在rocky上配置成功了centos7上配置不成功，原因未知

keepalived+redis实现高可用

两台主机分别安装redis和keepalived，当redis主机发生问题后实现自动主从切换，恢复重新恢复主从

192.168.125.31 主机 redis主
192.168.125.20 备机 Redis从
192.168.125.50 VIP

实现原理是用keepalied对redis进行状态检查，redis不可用时切换ip所在机器，由于ip切换到备用机，此时需要对redis的主从也要进行切换，这就用到notify_master和notify_backup这两个脚本了

当redis所在服务器成为主机时先让自己成为从机，等待数据同步后在切换自己为主机
当redis所在服务器成为从机时，等待主机同步完数据后再将主机设置为从机

定义通知脚本：
notify_master <STRING>|<QUOTED-STRING>：当前节点成为主节点时触发的脚本
notify_backup <STRING>|<QUOTED-STRING>：当前节点转为备节点时触发的脚本
notify_fault <STRING>|<QUOTED-STRING>：当前节点转为“失败”状态时触发的脚本
notify <STRING>|<QUOTED-STRING>：通用格式的通知触发机制，一个脚本可完成以上三种状态的转换时

reids配置中添加主机的配置，实现主从

slaveof 192.168.125.31 6379

keepalive主要配置

1、主机keepalive配置


global_defs {
   router_id LVS_DEVEL
   script_user root #root用户时必须配置
}
##添加自定义脚本
vrrp_script chk_redis {
    script "/etc/keepalived/scripts/ch_redis.sh"
    interval 1 #间隔一秒执行一次
}
vrrp_instance VI_1 {
    state MASTER  #定义主备
    #修改为实际网卡
    interface eth0
    virtual_router_id 51 #整个集群的调度器一致
    priority 100 #bake改为50
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        chk_redis
    }
    virtual_ipaddress {
    #这是要增加的虚拟ip
        192.168.125.50
    }
    notify_master /etc/keepalived/scripts/redis_master.sh
    notify_backup /etc/keepalived/scripts/redis_slave.sh
    notify_fault /etc/keepalived/scripts/redis_fault.sh
    notify_stop /etc/keepalived/scripts/redis_stop.sh

}

2、从机keepalive配置


! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   script_user root #root用户时必须配置
}
##添加自定义脚本
vrrp_script chk_redis {
    script "/etc/keepalived/scripts/ch_redis.sh"
    interval 1 #间隔一秒执行一次
}
vrrp_instance VI_1 {
    state BACKUP  #备用
    #修改为实际网卡
    interface eth0
    virtual_router_id 51 #整个集群的调度器一致
    priority 50 #bake改为50
    advert_int 1 #检测问题1s
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        chk_redis
    }
    virtual_ipaddress {
    #这是要增加的虚拟ip
        192.168.125.50
    }
    notify_master /etc/keepalived/scripts/redis_master.sh
    notify_backup /etc/keepalived/scripts/redis_slave.sh
    notify_fault /etc/keepalived/scripts/redis_fault.sh
    notify_stop /etc/keepalived/scripts/redis_stop.sh
}

keepalived脚本配置

主机脚本配置

1、ch_redis.sh

#!/bin/bash
ALIVE=`/usr/local/redis/bin/redis-cli PING`
if [ "$ALIVE" == "PONG" ];then
        echo $ALIVE
        exit 0
else
        echo $ALIVE
        exit 1
fi

2、redis_master.sh

#!/bin/bash
REDISCLI="/usr/local/redis/bin/redis-cli"
LOGFILE="/var/log/keepalived-redis-state.log"
echo "[master]">>$LOGFILE
date '+%F %T'  >> $LOGFILE
echo "Being master....">>$LOGFILE 2>&1
echo "Run SLAVEOF cmd...">>$LOGFILE
$REDISCLI SLAVEOF 192.168.125.20 6379 >>$LOGFILE 2>&1
sleep 10 #延迟10秒以后待数据同步完成后再取消同步状态
echo "Run SLAVEOF NO ONE cmd ...">> $LOGFILE
$REDISCLI SLAVEOF NO ONE >> $LOGFILE  2>&1

3、redis_slave.sh

REDISCLI="/usr/local/redis/bin/redis-cli"
LOGFILE="/var/log/keepalived-redis-state.log"
echo "[backup]">>$LOGFILE
date '+%F %T'  >> $LOGFILE
echo "Being slave....">>$LOGFILE 2>&1
sleep 10 #延迟10秒以后待数据同步完成后再取消同步状态
echo "Run SLAVEOF  cmd ...">> $LOGFILE
$REDISCLI SLAVEOF 192.168.125.20 6379  >> $LOGFILE  2>&1

4、redis_fault.sh

#!/bin/bash
LOGFILE="/var/log/keepalived-redis-state.log"
echo "[fault]">>$LOGFILE
date '+%F %T'  >> $LOGFILE

5、redis_stop.sh

#!/bin/bash
LOGFILE="/var/log/keepalived-redis-state.log"
echo "[stop]">>$LOGFILE
date '+%F %T'  >> $LOGFILE

chmod +x /etc/keepalived/scripts/*
systemctl restart keepalived.service

将脚本复制到从机上
scp -r /etc/keepalived/scripts root@192.168.125.20:/etc/keepalived/

从机脚本配置

修改从机脚本
1、redis_master.sh

#!/bin/bash
REDISCLI="/usr/local/redis/bin/redis-cli"
LOGFILE="/var/log/keepalived-redis-state.log"
echo "[master]">>$LOGFILE
date '+%F %T'  >> $LOGFILE
echo "Being master....">>$LOGFILE 2>&1
echo "Run SLAVEOF cmd...">>$LOGFILE
$REDISCLI SLAVEOF 192.168.125.31 6379 >>$LOGFILE 2>&1
sleep 10 #延迟10秒以后待数据同步完成后再取消同步状态
echo "Run SLAVEOF NO ONE cmd ...">> $LOGFILE
$REDISCLI SLAVEOF NO ONE >> $LOGFILE  2>&1

2、redis_slave.sh

REDISCLI="/usr/local/redis/bin/redis-cli"
LOGFILE="/var/log/keepalived-redis-state.log"
echo "[backup]">>$LOGFILE
date '+%F %T'  >> $LOGFILE
echo "Being slave....">>$LOGFILE 2>&1
sleep 10 #延迟10秒以后待数据同步完成后再取消同步状态
echo "Run SLAVEOF  cmd ...">> $LOGFILE
$REDISCLI SLAVEOF 192.168.125.31 6379  >> $LOGFILE  2>&1

systemctl restart keepalived

测试redis高可用

主机获取ip，查看状态

[root@eab5aec2df6c keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
45: eth0@if46: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.31/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 192.168.125.50/32 scope global eth0
       valid_lft forever preferred_lft forever

[root@eab5aec2df6c keepalived]# redis-cli -h 192.168.125.50
192.168.125.50:6379> info replication
#Replication
role:master
connected_slaves:1
slave0:ip=192.168.125.20,port=6379,state=online,offset=48266049,lag=1
master_failover_state:no-failover
master_replid:7f10983f5c47d0cbf5dee1ddd0f0e4c97c316323
master_replid2:a097b976486e3cf69431a3c20ba6df4f47ac6748
192.168.125.50:6379> shutdown
not connected>

[root@centos70 scripts]# ip a
47: eth0@if48: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:14 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.20/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 192.168.125.50/32 scope global eth0
       valid_lft forever preferred_lft forever

停止主机redis，备用机获取ip

[root@eab5aec2df6c keepalived]# redis-cli -h 192.168.125.50
192.168.125.50:6379> shutdown
not connected>

[root@centos70 scripts]# ip a
47: eth0@if48: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:14 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.20/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 192.168.125.50/32 scope global eth0
       valid_lft forever preferred_lft forever

主机启动redis，重新获取ip

[root@eab5aec2df6c keepalived]# systemctl start redis.service
[root@eab5aec2df6c keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
45: eth0@if46: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:1f brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.31/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 192.168.125.50/32 scope global eth0
       valid_lft forever preferred_lft forever
[root@centos70 scripts]# systemctl start redis.service
[root@centos70 scripts]# ip a
47: eth0@if48: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 02:42:c0:a8:7d:14 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.125.20/24 brd 192.168.125.255 scope global eth0
       valid_lft forever preferred_lft forever

主从日志信息如下keepalived-redis-state.log

[root@eab5aec2df6c keepalived]# cat /var/log/keepalived-redis-state.log
[fault]
2024-05-06 14:51:53
[master]
2024-05-06 14:55:05
Being master....
Run SLAVEOF cmd...
OK
Run SLAVEOF NO ONE cmd ...
OK

[root@centos70 scripts]# cat /var/log/keepalived-redis-state.log
[stop]
2024-05-06 14:50:15
[backup]
2024-05-06 14:50:15
Being slave....
Run SLAVEOF  cmd ...
OK
[master]
2024-05-06 14:51:55
Being master....
Run SLAVEOF cmd...
OK Already connected to specified master
Run SLAVEOF NO ONE cmd ...
OK
[fault]
2024-05-06 14:54:02
[backup]
2024-05-06 14:55:36
Being slave....
Run SLAVEOF  cmd ...
OK

报错

keepalived.service: Failed to send unit change signal for keepalived.service: Connection reset
Can’t open PID file /var/run/keepalived.pid (yet?) after running: No such

网卡没有设置好导致的，eth0。而不是eth0@if14

参考

Keepalived简介 https://blog.csdn.net/tangsiqi130/article/details/133565546
自动化运维（keepalived） https://blog.csdn.net/weixin_44954554/article/details/109034772

作品采用知识共享署名-非商业性使用-相同方式共享 4.0 国际许可协议进行许可
点赞
标签：

发表回复

您的电子邮箱地址不会被公开。 必填项已用*标注