nginx反向代理与负载均衡

2022-10-17 20:29 由 linux-ada 发表于 #操作系统

nginx反向代理与负载均衡

nginx反向代理与负载均衡
- nginx负载均衡调度器高可用配置

nginx通常被用作后端服务器的反向代理，这样就可以很方便的实现动静分离以及负载均衡，从而大大提高服务器的处理能力。

nginx实现动静分离，其实就是在反向代理的时候，如果是静态资源，就直接从nginx发布的路径去读取，而不需要从后台服务器获取了。

nginx通过upstream模块来实现简单的负载均衡，upstream需要定义在http段内

在upstream段内，定义一个服务器列表，默认的方式是轮询，如果要确定同一个访问者发出的请求总是由同一个后端服务器来处理，可以设置ip_hash，如：

upstream idfsoft.com {
  ip_hash;
  server 127.0.0.1:9080 weight=5;
  server 127.0.0.1:8080 weight=5;
  server 127.0.0.1:1111;
}

这个方法本质还是轮询，而且由于客户端的ip可能是不断变化的，比如动态ip，代理，FQ等，因此ip_hash并不能完全保证同一个客户端总是由同一个服务器来处理。

定义好upstream后，需要在server段内添加如下内容：

server {
  location / {
    proxy_pass http://idfsoft.com;
  }
}

nginx负载均衡调度器高可用配置

环境说明

主机名	ip地址	服务	系统
129	192.168.118.129	nginx keepalived	centos8
130	192.168.118.130	nginx keepalived	centos8
131	192.168.118.131	nginx	centos8
132	192.168.118.132	httpd	centos8

安装服务部分

129端

源码安装nginx

#修改主机名
[root@localhost ~]# hostnamectl set-hostname 129
[root@localhost ~]# bash

#关闭防火墙和selinux
[root@129 ~]# setenforce 0
[root@129 ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@129 ~]# systemctl disable --now firewalld.service

#创建用户
[root@129 ~]# useradd -rMs /sbin/nologin nginx

#源码安装nginx
#安装所需要的依赖包
[root@129 ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim

#下载nginx源码包，并解压配置
[root@129 ~]# wget https://nginx.org/download/nginx-1.22.0.tar.gz
[root@129 ~]# tar -xf nginx-1.22.0.tar.gz
[root@129 ~]# cd nginx-1.22.0/
[root@129 nginx-1.22.0]# ./configure \
--prefix=/usr/local/nginx \
--user=nginx \
--group=nginx \
--with-debug \
--with-http_ssl_module \
--with-http_realip_module \
--with-http_image_filter_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_stub_status_module

#编译安装
[root@129 nginx-1.22.0]# make -j $(grep 'processor' /proc/cpuinfo | wc -l)
[root@129 nginx-1.22.0]# make install

#配置环境变量
[root@129 nginx-1.22.0]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh
[root@129 nginx-1.22.0]# source /etc/profile.d/nginx.sh

#编写servic文件
[root@129 nginx-1.22.0]# cd /usr/lib/systemd/system/
[root@129 system]# vim nginx.service
[Unit]
Description=nginx server daemon
After=network.target

[Service]
Type=forking
ExecStart=/usr/local/nginx/sbin/nginx
ExecStop=/usr/local/nginx/sbin/nginx -s stop
ExecReload=/bin/kill -HUP \$MAINPID

[Install]
WantedBy=multi-user.target
[root@131 system]# systemctl daemon-reload

#启动服务并开机自启
[root@131 ~]# systemctl enable --now nginx.service
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@131 ~]# ss -antl
State    Recv-Q   Send-Q     Local Address:Port       Peer Address:Port   Process   
LISTEN   0        128              0.0.0.0:80              0.0.0.0:*                
LISTEN   0        128              0.0.0.0:22              0.0.0.0:*                
LISTEN   0        128                 [::]:22                 [::]:*

130端

源码按nginx

#修改主机名
[root@localhost ~]# hostnamectl set-hostname 130
[root@localhost ~]# bash

#关闭防火墙和selinux
[root@130 ~]# setenforce 0
[root@130 ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@130 ~]# systemctl disable --now firewalld.service

#创建用户
[root@130 ~]# useradd -rMs /sbin/nologin nginx

#源码安装nginx
#安装所需要的依赖包
[root@130 ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim

#下载nginx源码包，并解压配置
[root@130 ~]# wget https://nginx.org/download/nginx-1.22.0.tar.gz
[root@130 ~]# tar -xf nginx-1.22.0.tar.gz
[root@130 ~]# cd nginx-1.22.0/
[root@130 nginx-1.22.0]# ./configure \
--prefix=/usr/local/nginx \
--user=nginx \
--group=nginx \
--with-debug \
--with-http_ssl_module \
--with-http_realip_module \
--with-http_image_filter_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_stub_status_module

#编译安装
[root@130 nginx-1.22.0]# make -j $(grep 'processor' /proc/cpuinfo | wc -l)
[root@130 nginx-1.22.0]# make install

#配置环境变量
[root@130 nginx-1.22.0]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh
[root@130 nginx-1.22.0]# source /etc/profile.d/nginx.sh

#编写servic文件
[root@130 nginx-1.22.0]# cd /usr/lib/systemd/system/
[root@130 system]# vim nginx.service
[Unit]
Description=nginx server daemon
After=network.target

[Service]
Type=forking
ExecStart=/usr/local/nginx/sbin/nginx
ExecStop=/usr/local/nginx/sbin/nginx -s stop
ExecReload=/bin/kill -HUP \$MAINPID

[Install]
WantedBy=multi-user.target
[root@130 system]# systemctl daemon-reload

#启动服务并开机自启
[root@130 ~]# systemctl enable --now nginx.service
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@130 ~]# ss -antl
State    Recv-Q   Send-Q     Local Address:Port       Peer Address:Port   Process   
LISTEN   0        128              0.0.0.0:80              0.0.0.0:*                
LISTEN   0        128              0.0.0.0:22              0.0.0.0:*                
LISTEN   0        128                 [::]:22                 [::]:*

部署web页面

131端

#修改主机名
[root@localhost ~]# hostnamectl set-hostname 131
[root@localhost ~]# bash

#关闭防火墙和selinux
[root@131 ~]# setenforce 0
[root@131 ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@131 ~]# systemctl disable --now firewalld.service

#yum安装nginx
[root@131 ~]# dnf -y install nginx
[root@131 ~]# echo "web111" > /usr/share/nginx/html/index.html
[root@131 ~]# systemctl enable --now nginx.service
[root@131 ~]# ss -antl
State    Recv-Q   Send-Q     Local Address:Port       Peer Address:Port   Process   
LISTEN   0        128              0.0.0.0:80              0.0.0.0:*                
LISTEN   0        128              0.0.0.0:22              0.0.0.0:*                
LISTEN   0        128                 [::]:80                 [::]:*                
LISTEN   0        128                 [::]:22                 [::]:* 
[root@131 ~]# curl 192.168.118.131
web111

132端

#修改主机名
[root@localhost ~]# hostnamectl set-hostname 132
[root@localhost ~]# bash

#关闭防火墙和selinux
[root@132 ~]# setenforce 0
[root@132 ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@132 ~]# systemctl disable --now firewalld.service

#yum安装apache
[root@132 ~]# dnf -y install httpd
[root@132 ~]# echo "web222" > /var/www/html/index.html
[root@132 ~]# systemctl enable --now httpd.service
[root@132 ~]# ss -antl
State    Recv-Q   Send-Q     Local Address:Port       Peer Address:Port   Process   
LISTEN   0        128              0.0.0.0:22              0.0.0.0:*                
LISTEN   0        128                    *:80                    *:*                
LISTEN   0        128                 [::]:22                 [::]:*                
[root@132 ~]# curl 192.168.118.132
web222

实现nginx复制均衡

129端

[root@129 ~]# cd /usr/local/nginx/conf/
[root@129 conf]# vim nginx.conf
...
http {
...
    upstream web {                        #添加此处内容
    server 192.168.118.131;
    server 192.168.118.132;
    }

    server {
        listen       80;
        server_name  localhost;

        location / {
            root   html;
            proxy_pass http://web;         #修改为proxy_pass http://web;
        }
...
}
...
#重启服务开始访问
[root@129 conf]# systemctl restart nginx.service

[root@129 ~]# curl 192.168.118.129
web111
[root@129 ~]# curl 192.168.118.129
web222
[root@129 ~]# curl 192.168.118.129
web111
[root@129 ~]# curl 192.168.118.129
web222

130端

[root@130 ~]# cd /usr/local/nginx/conf/
[root@130 conf]# vim nginx.conf
...
http {
...
    upstream web {                        #添加此处内容
    server 192.168.118.131;
    server 192.168.118.132;
    }

    server {
        listen       80;
        server_name  localhost;

        location / {
            root   html;
            proxy_pass http://web;         #修改为proxy_pass http://web;
        }
...
}
...

#重启服务开始访问
[root@130 conf]# systemctl restart nginx.service

[root@130 conf]# curl 192.168.118.130
web111
[root@130 conf]# curl 192.168.118.130
web222
[root@130 conf]# curl 192.168.118.130
web111
[root@130 conf]# curl 192.168.118.130
web222

#先关闭，后面高可用需要将130端的nginx关掉
[root@130 ~]# systemctl stop nginx.service

部署keepalived高可用

129端

#安装keepalived
[root@129 ~]# dnf -y install keepalived

#编辑配置文件
[root@129 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
   router_id lb01
}
 
vrrp_instance VI_1 {
    state 129
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.118.250
    }
}
 
virtual_server 192.168.118.250 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.118.129 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.118.130 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}

[root@129 ~]# systemctl enable --now keepalived.service
[root@129 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:b5:9a:13 brd ff:ff:ff:ff:ff:ff
    inet 192.168.118.129/24 brd 192.168.118.255 scope global dynamic noprefixroute ens33
       valid_lft 1539sec preferred_lft 1539sec
    inet 192.168.118.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feb5:9a13/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

#通过vip访问，130端的nginx需要关闭

[root@129 ~]# curl 192.168.118.250
web111
[root@129 ~]# curl 192.168.118.250
web222
[root@129 ~]# curl 192.168.118.250
web111
[root@129 ~]# curl 192.168.118.250
web222

130端

#安装keepalived
[root@130 ~]# dnf -y install keepalived

#编辑配置文件
[root@130 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
   router_id lb02
}
 
vrrp_instance VI_1 {
    state 130
    interface ens33
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.118.250
    }
}
 
virtual_server 192.168.118.250 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.118.129 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.118.130 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@130 ~]# systemctl enable --now keepalived.service

模拟129端服务出现问题

129端

[root@129 ~]# systemctl stop keepalived.service

130端

[root@130 ~]# systemctl start nginx.service
[root@130 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:e7:e8:44 brd ff:ff:ff:ff:ff:ff
    inet 192.168.118.130/24 brd 192.168.118.255 scope global dynamic noprefixroute ens33
       valid_lft 1117sec preferred_lft 1117sec
    inet 192.168.118.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fee7:e844/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
       
#访问测试
[root@130 ~]# curl 192.168.118.250
web111
[root@130 ~]# curl 192.168.118.250
web222
[root@130 ~]# curl 192.168.118.250
web111
[root@130 ~]# curl 192.168.118.250
web222

编写脚本

129端

[root@129 ~]# mkdir /scripts
[root@129 ~]# cd /scripts/
[root@129 scripts]# vim check_nginx.sh
#!/bin/bash
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
if [ $nginx_status -lt 1 ];then
            systemctl stop keepalived
fi
[root@129 scripts]# vim notify.sh
#!/bin/bash
VIP=$2
case "$1" in
  master)
        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
        if [ $nginx_status -lt 1 ];then
            systemctl start nginx
        fi
  ;;
  backup)
        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
        if [ $nginx_status -gt 0 ];then
            systemctl stop nginx
        fi
  ;;
  *)
        echo "Usage:$0 master|backup VIP"
  ;;
esac
[root@129 scripts]# chmod +x check_nginx.sh
[root@129 scripts]# chmod +x notify.sh

130端

[root@130 ~]# mkdir /scripts
[root@130 ~]# cd /scripts/
[root@130 scripts]# scp [email protected]:/scripts/notify.sh .
[root@130 scripts]# ll
total 4
-rwxr-xr-x. 1 root root 451 Oct  8 19:17 notify.sh

配置keepalived加入监控脚本

129端

[root@130 ~]# vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
 
global_defs {
   router_id lb01
}

vrrp_script nginx_check {		          #增加这一块
    script "/scripts/check_nginx.sh"
    interval 1
    weight -20
}

vrrp_instance VI_1 {
    state 129
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.118.250
    }
    track_script {		                  #还有这一部分
        nginx_check
    }
    notify_129 "/scripts/notify.sh 129 192.168.118.250"	
}
 
virtual_server 192.168.118.250 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.118.129 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.118.130 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@130 ~]# systemctl restart keepalived.service

130端
130端无需检测nginx是否正常，当升级为129时启动nginx，当降级为BACKUP时关闭

[root@backup ~]# vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
 
global_defs {
   router_id lb02
}

vrrp_script nginx_check {		          #增加这一块
    script "/scripts/check_nginx.sh"
    interval 1
    weight -20
}

vrrp_instance VI_1 {
    state 130
    interface ens33
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.118.250
    }
    track_script {		                  #还有这一部分
        nginx_check
    }
    notify_129 "/scripts/notify.sh 129 192.168.118.250"	
}
 
virtual_server 192.168.118.250 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.118.129 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.118.130 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@backup ~]# systemctl restart keepalived.service

测试

#129端，nginx服务出现异常
[root@129 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:b5:9a:13 brd ff:ff:ff:ff:ff:ff
    inet 192.168.118.129/24 brd 192.168.118.255 scope global dynamic noprefixroute ens33
       valid_lft 1539sec preferred_lft 1539sec
    inet 192.168.118.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feb5:9a13/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@129 ~]# curl 192.168.118.250
web111
[root@129 ~]# curl 192.168.118.250
web222
[root@129 ~]# systemctl stop nginx.service 
 
#130端
[root@130 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:b5:9a:13 brd ff:ff:ff:ff:ff:ff
    inet 192.168.118.130/24 brd 192.168.118.255 scope global dynamic noprefixroute ens33
       valid_lft 1539sec preferred_lft 1539sec
    inet 192.168.118.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feb5:9a13/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@130 ~]# curl 192.168.202.160
web111
[root@130 ~]# curl 192.168.202.160
web222