搭建keepalive+nginx高可用集群
1.安装nginx1.18
sudo apt install curl gnupg2 ca-certificates lsb-release
echo "deb http://nginx.org/packages/ubuntu `lsb_release -cs` nginx" \
| sudo tee /etc/apt/sources.list.d/nginx.list
curl -fsSL https://nginx.org/keys/nginx_signing.key | sudo apt-key add -
sudo apt-key fingerprint ABF5BD827BD9BF62
sudo apt update
sudo apt install nginx
查看版本
suixiaofeng@nginx01-devopstack.cn:~/gtj$ nginx -v
nginx version: nginx/1.18.0
2.安装keepalive2.0
方法一：
wget https://www.keepalived.org/software/keepalived-2.0.0.tar.gz
tar -xf keepalived-2.0.0.tar.gz
cd keepalived-2.0.0/
sudo apt-get install libssl-dev
sudo ./configure
sudo make &&sudo make install
sudo mkdir /etc/keepalived
sudo cp keepalived/etc/keepalived/keepalived.conf /etc/keepalived/
cat /lib/systemd/system/keepalived.service
[Unit]
Description=Keepalive Daemon (LVS and VRRP)
After=syslog.target network-online.target
Wants=network-online.target
# Only start if there is a configuration file
ConditionFileNotEmpty=/etc/keepalived/keepalived.conf
[Service]
Type=forking
KillMode=process
# Read configuration variable file if it is present
EnvironmentFile=-/etc/default/keepalived
ExecStart=/usr/sbin/keepalived DAEMON_ARGS
ExecReload=/bin/kill -HUPMAINPID
[Install]
WantedBy=multi-user.target
sudo cp /lib/systemd/system/keepalived.service /var/lib/systemd/deb-systemd-helper-enabled/multi-user.target.wants/keepalived.service
sudo cp bin/keepalived /usr/sbin/keepalived
这个方式keepalive只启动两个进程，导致vip 漂移失败，后续查看原因
方法二：
sudo apt-get install -y keepalived
启动
sudo systemctl start keepalived
sudo systemctl status keepalived
sudo systemctl enable keepalived
这个方式没问题。
3 配置keepalived
sudo cp /etc/keepalived/keepalived.conf{,.bak}
cat /etc/keepalived/keepalived.conf
(主的配置)
! Configuration File for keepalived
global_defs {
router_id nginx01-devopstack.cn
}
vrrp_script chk_nginx {
script "/srv/check_nginx.sh"
interval 2
weight 10
fall 3
rise 2
}
vrrp_instance VI_1 {
state MASTER
interface bond0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
172.40.10.10
}
track_script {
chk_nginx
}
}
备的配置
! Configuration File for keepalived
global_defs {
router_id nginx02-ad-gds02
}
vrrp_script chk_nginx {
script "/srv/check_nginx.sh"
interval 2
weight 10
fall 3
rise 2
}
vrrp_instance VI_1 {
state BACKUP
interface bond0
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
172.40.10.10
}
track_script {
chk_nginx
}
}
nginx检查脚本
cat /srv/check_nginx.sh
#!/bin/bash
# check nginx server status
counter=(ps -C nginx --no-heading|wc -l)
if [ "{counter}" = "0" ]; then
systemctl start nginx
sleep 2
counter=(ps -C nginx --no-heading|wc -l)
if [ "{counter}" = "0" ]; then
systemctl stop keepalived
fi
fi
4 nginx相关配置
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
worker_rlimit_nofile 51200;
events {
use epoll;
worker_connections 51200;
multi_accept on;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
server_tokens off;
log_format main 'remote_addr -remote_user [time_local] "request" '
'statusbody_bytes_sent "http_referer" '
'"http_user_agent" "http_x_forwarded_for" "request_time"';
log_format suixiaofeng_log 'remote_addr -remote_user [time_local] "request" '
'statusbody_bytes_sent "http_referer" '
'"http_user_agent" "http_x_forwarded_for" "request_time"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 6;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript;
gzip_vary on;
gzip_disable "msie6";
proxy_cache_path proxy_cache levels=1:2 keys_zone=one:100m inactive=3d;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 10m;
ssl_protocols TLSv1.1 TLSv1.2;
include /etc/nginx/conf.d/*.conf;
}
5 日志切割优化
cat /etc/logrotate.d/nginx/etc/logrotate.d/nginx
/var/log/nginx/*.log {
daily
missingok
rotate 52
compress
dateext #加日期标志
delaycompress
notifempty
create 640 nginx adm
sharedscripts
postrotate
if [ -f /var/run/nginx.pid ]; then
kill -USR1 `cat /var/run/nginx.pid`
fi
endscript
}
执行测试
sudo /usr/sbin/logrotate -f /etc/logrotate.d/nginx
sudo ls -ltr /var/log/nginx/
-rw-r--r-- 1 nginx root 0 9月 24 11:12 access.log
-rw-r--r-- 1 root root 65 9月 24 11:12 error.log-20200924
-rw-r----- 1 nginx adm 0 9月 24 11:13 error.log
6 内核优化
# 避免放大攻击
net.ipv4.icmp_echo_ignore_broadcasts = 1
# 开启恶意icmp错误消息保护
net.ipv4.icmp_ignore_bogus_error_responses = 1
# 开启SYN洪水攻击保护
net.ipv4.tcp_syncookies = 1
# 开启并记录欺骗，源路由和重定向包
net.ipv4.conf.all.log_martians = 1
net.ipv4.conf.default.log_martians = 1
# 处理无源路由的包
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0
# 开启反向路径过滤
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
# 确保无人能修改路由表
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0
# 不充当路由器
net.ipv4.ip_forward = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
# 增加系统文件描述符限制
fs.file-max = 65535
# 允许更多的PIDs (减少滚动翻转问题); may break some programs 32768
kernel.pid_max = 65536
# 增加系统IP端口限制
net.ipv4.ip_local_port_range = 2000 65000
# 增加TCP最大缓冲区大小
net.ipv4.tcp_rmem = 4096 87380 8388608
net.ipv4.tcp_wmem = 4096 87380 8388608
# 增加Linux自动调整TCP缓冲区限制
#启用 RFC 1323 定义的 window scaling；要支持超过 64KB 的窗口，必须启用该值。
net.ipv4.tcp_window_scaling = 1
# 最小，默认和最大可使用的字节数
# 最大值不低于4MB，如果你使用非常高的BDP路径可以设置得更高
# Tcp窗口等
net.core.rmem_max = 8388608
net.core.wmem_max = 8388608
net.core.netdev_max_backlog = 5000
#add timewait conf
net.ipv4.tcp_tw_reuse=1
net.ipv4.neigh.default.gc_stale_time=120
net.ipv4.conf.all.arp_announce=2
net.ipv4.conf.lo.arp_announce=2
kernel.sysrq=1
net.ipv4.tcp_syncookies=1
net.core.somaxconn=1024
net.ipv4.tcp_fin_timeout=30
net.ipv4.tcp_tw_reuse=1
vm.swappiness=0
7 高可用测试
suixiaofeng@nginx01-devopstack.cn:~ip a |grep bond0
3: enp130s0f0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
7: enp130s0f1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
8: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
inet 172.40.10.11/24 brd 172.40.10.255 scope global bond0
inet 172.40.10.10/32 scope global bond0
如上 vip绑定在nginx01-devopstack.cn上的
手动关闭nginx服务，测试vip是否漂移
suixiaofeng@nginx01-devopstack.cn:~/gtj sudo systemctl stop nginx
这样停nginx不行 ，keepalive会把nginx起来的
我是改了nginx配置
假设nginx起不来了
suixiaofeng@nginx01-devopstack.cn:~sudo nginx -t
nginx: [emerg] unknown directive "tets" in /etc/nginx/nginx.conf:54
nginx: configuration file /etc/nginx/nginx.conf test failed
suixiaofeng@nginx01-devopstack.cn:~ sudo killall nginx
suixiaofeng@nginx01-devopstack.cn:~ps -ef|grep nginx
suixiaofeng 63712 60239 0 18:35 pts/0 00:00:00 grep --color=auto nginx
suixiaofeng@nginx01-devopstack.cn:~ ps -ef|grep keep
suixiaofeng 63728 60239 0 18:36 pts/0 00:00:00 grep --color=auto keep
查看vip
suixiaofeng@nginx01-devopstack.cn:~ip a |grep bond0
3: enp130s0f0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
7: enp130s0f1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
8: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
inet 172.40.10.11/24 brd 172.40.10.255 scope global bond0
已经不在nginx01上了
已经漂移到nginx02上了
suixiaofeng@nginx02-ad-gds02:~ ip a |grep bond0
3: eno1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
5: eno2: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
6: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
inet 172.40.10.12/24 brd 172.40.10.255 scope global bond0
inet 172.40.10.10/32 scope global bond0
恢复nginx01的nginx和keepalive之后 vip会漂移到nginx01上的
suixiaofeng@nginx01-devopstack.cn:~sudo nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
suixiaofeng@nginx01-devopstack.cn:~ sudo systemctl start keepalived.service
suixiaofeng@nginx01-devopstack.cn:~ip a |grep bond0
3: enp130s0f0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
7: enp130s0f1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
8: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
inet 172.40.10.11/24 brd 172.40.10.255 scope global bond0
inet 172.40.10.10/32 scope global bond0
查看nginx02上 vip已经不在了
suixiaofeng@nginx02-ad-gds02:/var/log ip a |grep bond0
3: eno1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
5: eno2: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
6: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
inet 172.40.10.12/24 brd 172.40.10.255 scope global bond0
