Linux: 企业级调度器LVS

[root@centos8 ~]#grep -i -C 10 ipvs /boot/config-4.18.0-147.el8.x86_64
...（省略部分内容）...
CONFIG_NETFILTER_XT_MATCH_IPVS=m
CONFIG_NETFILTER_XT_MATCH_POLICY=m
...（省略部分内容）...
#
# IPVS transport protocol load balancing support
#
CONFIG_IP_VS_PROTO_TCP=y
CONFIG_IP_VS_PROTO_UDP=y
CONFIG_IP_VS_PROTO_AH_ESP=y
CONFIG_IP_VS_PROTO_ESP=y
CONFIG_IP_VS_PROTO_AH=y
CONFIG_IP_VS_PROTO_SCTP=y
#
# IPVS scheduler
#
CONFIG_IP_VS_RR=m
CONFIG_IP_VS_WRR=m
CONFIG_IP_VS_LC=m
CONFIG_IP_VS_WLC=m
CONFIG_IP_VS_FO=m     #新增
CONFIG_IP_VS_OVF=m     #新增
CONFIG_IP_VS_LBLC=m
CONFIG_IP_VS_LBLCR=m
CONFIG_IP_VS_DH=m
CONFIG_IP_VS_SH=m
# CONFIG_IP_VS_MH is not set
CONFIG_IP_VS_SED=m
CONFIG_IP_VS_NQ=m
...（省略部分内容）...

--set-mark value # 其中：value 可为0xffff格式，表示十六进制数字

iptables -t mangle -A PREROUTING -d $vip -p $proto -m multiport --dports
$port1,$port2,… -j MARK --set-mark NUMBER

ipvsadm -A -f NUMBER [options]

[root@lvs ~]#iptables -t mangle -A PREROUTING -d 172.16.0.100 -p tcp -m
multiport --dports 80,443 -j MARK --set-mark 10
[root@lvs ~]#ipvsadm -C
[root@lvs ~]#ipvsadm -A -f 10 -s rr
[root@lvs ~]#ipvsadm -a -f 10 -r 10.0.0.7 -g
[root@lvs ~]#ipvsadm -a -f 10 -r 10.0.0.17 -g
[root@lvs ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
FWM     10 rr
    -> 10.0.0.7:0                                     Route     1             0                     0             
    -> 10.0.0.17:0                                 Route     1             0                     0             
[root@lvs ~]#cat /proc/net/ip_vs
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port Forward Weight ActiveConn InActConn
FWM 0000000A rr
    -> 0A000011:0000         Route     1             0                     9             
    -> 0A000007:0000         Route     1             0                     9

[root@lvs ~]#ipvsadm -A -f 10
[root@lvs ~]#ipvsadm -a -f 10 -r 10.0.0.7 -g
[root@lvs ~]#ipvsadm -a -f 10 -r 10.0.0.17 -g
[root@lvs ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
FWM     10 wlc
    -> 10.0.0.7:0                             Route     1             0                     0             
    -> 10.0.0.17:0                         Route     1             0                     0
[root@LVS ~]#cat /proc/net/ip_vs
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP AC14C8C8:0050 rr 
    -> 0A000011:0050         Masq         1             0                     0             
    -> 0A000007:0050         Masq         1             0                     0 

ipvsadm -A|E -t|u|f service-address [-s scheduler] [-p [timeout]]

[root@lvs ~]#ipvsadm -E -f 10 -p
[root@lvs ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
FWM     10 wlc persistent 360
    -> 10.0.0.7:0                             Route     1             0                     15             
    -> 10.0.0.17:0                         Route     1             0                     7             
[root@lvs ~]#ipvsadm -E -f 10 -p 3600
[root@lvs ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
FWM     10 wlc persistent 3600
    -> 10.0.0.7:0                             Route     1             0                     79             
    -> 10.0.0.17:0                         Route     1             0                     7

[root@lvs ~]#cat /proc/net/ip_vs_conn
Pro FromIP     FPrt ToIP         TPrt DestIP     DPrt State             Expires PEName PEData
TCP C0A80006 C816 AC100064 01BB 0A000011 01BB FIN_WAIT                 67
TCP C0A80006 C812 AC100064 01BB 0A000011 01BB FIN_WAIT                 67
TCP C0A80006 9A36 AC100064 0050 0A000011 0050 FIN_WAIT                 65
TCP C0A80006 C806 AC100064 01BB 0A000011 01BB FIN_WAIT                 65
TCP C0A80006 9A3E AC100064 0050 0A000011 0050 FIN_WAIT                 66
TCP C0A80006 C81A AC100064 01BB 0A000011 01BB FIN_WAIT                 67
TCP C0A80006 C80A AC100064 01BB 0A000011 01BB FIN_WAIT                 66
TCP C0A80006 9A3A AC100064 0050 0A000011 0050 FIN_WAIT                 66
TCP C0A80006 9A4E AC100064 0050 0A000011 0050 FIN_WAIT                 68
TCP C0A80006 9A42 AC100064 0050 0A000011 0050 FIN_WAIT                 67
TCP C0A80006 9A46 AC100064 0050 0A000011 0050 FIN_WAIT                 67
TCP C0A80006 C81E AC100064 01BB 0A000011 01BB FIN_WAIT                 68
IP C0A80006 0000 0000000A 0000 0A000011 0000 NONE                         948
TCP C0A80006 C80E AC100064 01BB 0A000011 01BB FIN_WAIT                 66
TCP C0A80006 9A4A AC100064 0050 0A000011 0050 FIN_WAIT                 67

[root@lvs ~]#ipvsadm -Lnc
IPVS connection entries
pro expire state             source                         virtual                     destination
TCP 00:46 FIN_WAIT         192.168.10.6:51222     172.16.0.100:443     10.0.0.17:443
TCP 00:46 FIN_WAIT         192.168.10.6:51218     172.16.0.100:443     10.0.0.17:443
TCP 00:45 FIN_WAIT         192.168.10.6:39478     172.16.0.100:80         10.0.0.17:80
TCP 00:45 FIN_WAIT         192.168.10.6:51206     172.16.0.100:443     10.0.0.17:443
TCP 00:46 FIN_WAIT         192.168.10.6:39486     172.16.0.100:80         10.0.0.17:80
TCP 00:47 FIN_WAIT         192.168.10.6:51226     172.16.0.100:443     10.0.0.17:443
TCP 00:45 FIN_WAIT         192.168.10.6:51210     172.16.0.100:443     10.0.0.17:443
TCP 00:45 FIN_WAIT         192.168.10.6:39482     172.16.0.100:80         10.0.0.17:80
TCP 00:47 FIN_WAIT         192.168.10.6:39502     172.16.0.100:80         10.0.0.17:80
TCP 00:46 FIN_WAIT         192.168.10.6:39490     172.16.0.100:80         10.0.0.17:80
TCP 00:46 FIN_WAIT         192.168.10.6:39494     172.16.0.100:80         10.0.0.17:80
TCP 00:47 FIN_WAIT         192.168.10.6:51230     172.16.0.100:443     10.0.0.17:443
IP     15:27 NONE                 192.168.10.6:0             0.0.0.10:0                 10.0.0.17:0
TCP 00:46 FIN_WAIT         192.168.10.6:51214     172.16.0.100:443     10.0.0.17:443
TCP 00:47 FIN_WAIT         192.168.10.6:39498     172.16.0.100:80         10.0.0.17:80

共四台主机
一台： internet client：192.168.10.6/24     GW:无 仅主机

一台：lvs 
eth1 仅主机 192.168.10.100/16
eth0 NAT 10.0.0.8/24

两台RS：
RS1: 10.0.0.7/24 GW：10.0.0.8 NAT
RS2: 10.0.0.17/24 GW：10.0.0.8 NAT

[root@internet ~]#cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
NAME=eth0
BOOTPROTO=static
IPADDR=192.168.10.6
PREFIX=24
ONBOOT=yes
[root@lvs network-scripts]#cat ifcfg-eth0
DEVICE=eth0
NAME=eth0
BOOTPROTO=static
IPADDR=10.0.0.8
PREFIX=24
ONBOOT=yes
[root@lvs network-scripts]#cat ifcfg-eth1
DEVICE=eth1
NAME=eth1
BOOTPROTO=static
IPADDR=192.168.10.100
PREFIX=24
ONBOOT=yes
[root@rs1 ~]#cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
NAME=eth0
BOOTPROTO=static
IPADDR=10.0.0.7
PREFIX=24
GATEWAY=10.0.0.8
ONBOOT=yes

[root@rs2 ~]#cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
NAME=eth0
BOOTPROTO=static
IPADDR=10.0.0.17
PREFIX=24
GATEWAY=10.0.0.8
ONBOOT=yes

[root@rs1 ~]#curl 10.0.0.7
10.0.0.7 RS1
[root@rs2 ~]#curl 10.0.0.17
10.0.0.17 RS2

[root@lvs-server ~]#vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
[root@lvs-server ~]#sysctl -p
net.ipv4.ip_forward = 1

[root@lvs-server ~]#ipvsadm -A -t 192.168.10.100:80 -s wrr
[root@lvs-server ~]#ipvsadm -a -t 192.168.10.100:80 -r 10.0.0.7:80 -m
[root@lvs-server ~]#ipvsadm -a -t 192.168.10.100:80 -r 10.0.0.17:80 -m

[root@lvs-server ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
TCP     192.168.10.100:80 wrr
    -> 10.0.0.7:80                                 Masq         1             1                     0             
    -> 10.0.0.17:80                                 Masq         1             0                     0 
[root@internet ~]#while :;do curl 192.168.10.100;sleep 0.5;done
rs1.me.org
rs2.me.org
rs1.me.org
rs2.me.org
rs1.me.org
rs2.me.org
[root@lvs-server ~]#ipvsadm -Ln --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port                             Conns     InPkts OutPkts InBytes OutBytes
    -> RemoteAddress:Port
TCP     192.168.10.100:80                                     67             405             255         32436         30092
    -> 10.0.0.7:80                                                 34             203             128         16244         15072
    -> 10.0.0.17:80                                             33             202             127         16192         15020
[root@lvs-server ~]#cat /proc/net/ip_vs
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port Forward Weight ActiveConn InActConn

TCP C0A80A64:0050 wrr 
    -> 0A000011:0050         Masq         1             0                     98             
    -> 0A000007:0050         Masq         1             0                     97 

[root@lvs-server ~]#ipvsadm -Lnc
IPVS connection entries
pro expire state             source                         virtual                     destination
TCP 01:55 TIME_WAIT     192.168.10.6:43486 192.168.10.100:80     10.0.0.17:80
TCP 00:19 TIME_WAIT     192.168.10.6:43476 192.168.10.100:80     10.0.0.7:80
TCP 01:58 TIME_WAIT     192.168.10.6:43500 192.168.10.100:80     10.0.0.7:80
TCP 01:58 TIME_WAIT     192.168.10.6:43498 192.168.10.100:80     10.0.0.17:80
TCP 01:59 TIME_WAIT     192.168.10.6:43502 192.168.10.100:80     10.0.0.17:80
TCP 01:57 TIME_WAIT     192.168.10.6:43494 192.168.10.100:80     10.0.0.17:80
TCP 01:57 TIME_WAIT     192.168.10.6:43496 192.168.10.100:80     10.0.0.7:80
TCP 01:56 TIME_WAIT     192.168.10.6:43490 192.168.10.100:80     10.0.0.17:80
TCP 00:20 TIME_WAIT     192.168.10.6:43480 192.168.10.100:80     10.0.0.7:80
TCP 01:56 TIME_WAIT     192.168.10.6:43492 192.168.10.100:80     10.0.0.7:80
TCP 01:55 TIME_WAIT     192.168.10.6:43488 192.168.10.100:80     10.0.0.7:80
TCP 00:20 TIME_WAIT     192.168.10.6:43478 192.168.10.100:80     10.0.0.17:80
TCP 01:59 TIME_WAIT     192.168.10.6:43504 192.168.10.100:80     10.0.0.7:80
TCP 01:54 TIME_WAIT     192.168.10.6:43484 192.168.10.100:80     10.0.0.7:80
TCP 01:54 TIME_WAIT     192.168.10.6:43482 192.168.10.100:80     10.0.0.17:80
[root@lvs-server ~]#cat /proc/net/ip_vs_conn
Pro FromIP     FPrt ToIP         TPrt DestIP     DPrt State             Expires PEName PEData
TCP C0A80A06 A9DE C0A80A64 0050 0A000011 0050 TIME_WAIT                 72
TCP C0A80A06 A9EC C0A80A64 0050 0A000007 0050 TIME_WAIT                 76
TCP C0A80A06 AA64 C0A80A64 0050 0A000007 0050 TIME_WAIT             106
TCP C0A80A06 AA0C C0A80A64 0050 0A000007 0050 TIME_WAIT                 84
TCP C0A80A06 AA3A C0A80A64 0050 0A000011 0050 TIME_WAIT                 95
TCP C0A80A06 AA86 C0A80A64 0050 0A000011 0050 TIME_WAIT             115
TCP C0A80A06 AA78 C0A80A64 0050 0A000007 0050 TIME_WAIT             111
TCP C0A80A06 AA06 C0A80A64 0050 0A000011 0050 TIME_WAIT                 82
TCP C0A80A06 AA44 C0A80A64 0050 0A000007 0050 TIME_WAIT                 98
TCP C0A80A06 AA2C C0A80A64 0050 0A000007 0050 TIME_WAIT                 92

#保存规则
[root@lvs-server ~]#ipvsadm -Sn > /etc/sysconfig/ipvsadm
[root@lvs-server ~]#systemctl enable --now ipvsadm.service

共四台主机
一台： internet client ：172.20.200.6/16     GW:无

一台：lvs 
eth1 桥接 172.20.200.200/16
eth0 NAT 10.0.0.200/24

两台RS：
RS1: 10.0.0.7/24 GW： 10.0.0.200
RS2: 10.0.0.17/24 GW： 10.0.0.200

#LVS启用IP_FORWORD功能
[root@lvs ~]#vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
[root@LVS ~]#sysctl -p

[root@lvs ~]#ipvsadm -A -t 172.20.200.200:80 -s rr
[root@lvs ~]#ipvsadm -a -t 172.20.200.200:80 -r 10.0.0.7 -m
[root@lvs ~]#ipvsadm -a -t 172.20.200.200:80 -r 10.0.0.17 -m

[root@LVS ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
TCP     172.20.200.200:80 rr
    -> 10.0.0.7:80                                 Masq         1             0                     0             
    -> 10.0.0.17:80                                 Masq         1             0                     0 

#测试
[root@client ~]#curl 172.20.200.200
RS2 Server on 10.0.0.17
[root@client ~]#curl 172.20.200.200
RS1 Server on 10.0.0.7
[root@client ~]#curl 172.20.200.200
RS2 Server on 10.0.0.17
[root@client ~]#curl 172.20.200.200
RS1 Server on 10.0.0.7

[root@LVS ~]#cat /proc/net/ip_vs_conn
Pro FromIP     FPrt ToIP         TPrt DestIP     DPrt State             Expires PEName PEData
TCP AC14C806 BD6A AC14C8C8 0050 0A000011 0050 TIME_WAIT                 97
TCP AC14C806 BD6C AC14C8C8 0050 0A000007 0050 TIME_WAIT                 97
TCP AC14C806 BD66 AC14C8C8 0050 0A000011 0050 TIME_WAIT                 90
TCP AC14C806 BD68 AC14C8C8 0050 0A000007 0050 TIME_WAIT                 92

#保存规则
[root@LVS ~]#ipvsadm -Sn > /etc/sysconfig/ipvsadm
[root@LVS ~]#cat /etc/sysconfig/ipvsadm
-A -t 172.20.200.200:80 -s rr
-a -t 172.20.200.200:80 -r 10.0.0.7:80 -m -w 1
-a -t 172.20.200.200:80 -r 10.0.0.17:80 -m -w 1

#清除规则
[root@LVS ~]#ipvsadm -C
[root@LVS ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn

#重新加载规则
[root@LVS ~]#ipvsadm -R < /etc/sysconfig/ipvsadm
[root@LVS ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
TCP     172.20.200.200:80 rr
    -> 10.0.0.7:80                                 Masq         1             0                     0             
    -> 10.0.0.17:80

#开机加载ipvs规则
[root@LVS ~]#ipvsadm -C
[root@LVS ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
[root@LVS ~]#systemctl enable --now ipvsadm.service
[root@LVS ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
TCP     172.20.200.200:80 rr
    -> 10.0.0.7:80                                 Masq         1             0                     0             
    -> 10.0.0.17:80                                 Masq         1             0                     0
[root@rs1 ~]#tail /var/log/httpd/access_log
172.20.200.6 - - [24/Mar/2020:16:38:29 +0800] "GET / HTTP/1.1" 200 23 "-"
"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.27.1 zlib/1.2.3
libidn/1.18 libssh2/1.4.2"
172.20.200.6 - - [24/Mar/2020:16:38:35 +0800] "GET / HTTP/1.1" 200 23 "-"
"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.27.1 zlib/1.2.3
libidn/1.18 libssh2/1.4.2"
172.20.200.6 - - [24/Mar/2020:16:52:16 +0800] "GET / HTTP/1.1" 200 23 "-"
"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.27.1 zlib/1.2.3
libidn/1.18 libssh2/1.4.2"
172.20.200.6 - - [24/Mar/2020:16:52:17 +0800] "GET / HTTP/1.1" 200 23 "-"
"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.27.1 zlib/1.2.3
libidn/1.18 libssh2/1.4.2"
172.20.200.6 - - [24/Mar/2020:16:53:36 +0800] "GET / HTTP/1.1" 200 23 "-"
"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.27.1 zlib/1.2.3
libidn/1.18 libssh2/1.4.2"
172.20.200.6 - - [24/Mar/2020:16:53:37 +0800] "GET / HTTP/1.1" 200 23 "-"
"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.27.1 zlib/1.2.3
libidn/1.18 libssh2/1.4.2"

#修改调度算法为 WRR 和后端服务器的端口
[root@LVS ~]#ipvsadm -E -t 172.20.200.200:80 -s wrr
[root@LVS ~]#ipvsadm -d -t 172.20.200.200:80 -r 10.0.0.7
[root@LVS ~]#ipvsadm -a -t 172.20.200.200:80 -r 10.0.0.7:8080 -m -w 3
[root@LVS ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
TCP     172.20.200.200:80 wrr
    -> 10.0.0.7:8080                             Masq         3             0                     0             
    -> 10.0.0.17:80                                 Masq         1             0                     1 

[root@rs1 ~]#vim /etc/httpd/conf/httpd.conf
Listen 8080
[root@rs1 ~]#systemctl restart httpd

[root@client ~]#curl 172.20.200.200
RS1 Server on 10.0.0.7
[root@client ~]#curl 172.20.200.200
RS1 Server on 10.0.0.7
[root@client ~]#curl 172.20.200.200
RS1 Server on 10.0.0.7
[root@client ~]#curl 172.20.200.200
RS2 Server on 10.0.0.17

环境：五台主机
一台：客户端 eth0:仅主机 192.168.10.6/24 GW:192.168.10.200

一台：ROUTER
eth0 :NAT     10.0.0.200/24
eth1: 仅主机 192.168.10.200/24
启用 IP_FORWARD

一台：LVS
eth0:NAT:DIP:10.0.0.8/24 GW:10.0.0.200

两台RS：
RS1：eth0:NAT:10.0.0.7/24     GW：10.0.0.200
RS2：eth0:NAT:10.0.0.17/24 GW：10.0.0.200

#internet主机的网络配置和4.2一样
[root@internet ~]#hostname -I
192.168.10.6

#router的网络配置在4.2基础上添加172.16.0.200/24的地址
[root@router ~]#ip addr add 172.16.0.200/24 dev eth0
[root@router ~]#ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group
default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
            valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
            valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP
group default qlen 1000
    link/ether 00:0c:29:ab:8f:2b brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.200/24 brd 10.0.0.255 scope global noprefixroute eth0
            valid_lft forever preferred_lft forever
    inet 172.16.0.200/24 brd 172.16.0.255 scope global noprefixroute eth0
            valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feab:8f2b/64 scope link tentative
            valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP
group default qlen 1000
    link/ether 00:0c:29:ab:8f:35 brd ff:ff:ff:ff:ff:ff
    inet 192.168.10200/24 brd 192.168.10255 scope global noprefixroute eth1
             valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feab:8f35/64 scope link
            valid_lft forever preferred_lft forever
[root@router ~]#hostname -I
10.0.0.200 172.16.0.200 192.168.10200

#LVS主机的网络配置和4.2一样
[root@lvs ~]#hostname -I
10.0.0.8
[root@lvs ~]#route -n
Kernel IP routing table
Destination         Gateway                 Genmask                 Flags Metric Ref     Use Iface
0.0.0.0                 10.0.0.200             0.0.0.0                 UG         0             0                 0 eth0
10.0.0.0                 0.0.0.0                 255.255.255.0     U         100         0                 0 eth0
[root@rs1 ~]#hostname -I
10.0.0.7
[root@rs1 ~]#route -n
Kernel IP routing table
Destination         Gateway                 Genmask                 Flags Metric Ref     Use Iface
0.0.0.0                 10.0.0.200             0.0.0.0                 UG         100         0                 0 eth0
10.0.0.0                 0.0.0.0                 255.255.255.0     U         100         0                 0 eth0

#RS主机的网络配置和4.2一样
[root@rs2 ~]#hostname -I
10.0.0.17
[root@rs2 ~]#route -n
Kernel IP routing table
Destination         Gateway                 Genmask                 Flags Metric Ref     Use Iface
0.0.0.0                 10.0.0.200             0.0.0.0                 UG         100         0                 0 eth0
10.0.0.0                 0.0.0.0                 255.255.255.0     U         100         0                 0 eth0

#在LVS主机运行的脚本
[root@lvs ~]#cat lvs_dr_vs.sh
#!/bin/bash
#Author:wang
#Date:2017-08-13
vip='172.16.0.100'
iface='lo:1'
mask='255.255.255.255'
port='80'
rs1='10.0.0.7'
rs2='10.0.0.17'
scheduler='wrr'
type='-g'
rpm -q ipvsadm &> /dev/null || yum -y install ipvsadm &> /dev/null
case $1 in
start)
    ifconfig $iface $vip netmask $mask #broadcast $vip up
    iptables -F
    ipvsadm -A -t ${vip}:${port} -s $scheduler
    ipvsadm -a -t ${vip}:${port} -r ${rs1} $type -w 1
    ipvsadm -a -t ${vip}:${port} -r ${rs2} $type -w 1
        echo "The VS Server is Ready!"
    ;;
stop)
    ipvsadm -C
    ifconfig $iface down
        echo "The VS Server is Canceled!"
    ;;
*)
        echo "Usage: $(basename $0) start|stop"
        exit 1
    ;;
esac

[root@lvs ~]#bash lvs_dr_vs.sh start
The VS Server is Ready!

#在RS后端服务器运行的脚本
[root@rs1 ~]#cat lvs_dr_rs.sh
#!/bin/bash
#Author:wang
#Date:2017-08-13
vip=172.16.0.100
mask='255.255.255.255'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "`hostname -I`" > /var/www/html/index.html
case $1 in
start)
        echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ifconfig $dev $vip netmask $mask #broadcast $vip up
        echo "The RS Server is Ready!"
    ;;
stop)
    ifconfig $dev down
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
        echo "The RS Server is Canceled!"
    ;;
*)
        echo "Usage: $(basename $0) start|stop"
        exit 1
    ;;
esac
[root@rs1 ~]#bash lvs_dr_rs.sh start
The RS Server is Ready!

#在RS后端服务器运行的脚本和RS1是一样的
[root@rs2 ~]#bash lvs_dr_rs.sh start
The RS Server is Ready!

#测试访问
[root@internet ~]#curl 172.16.0.100
10.0.0.7
[root@internet ~]#curl 172.16.0.100
10.0.0.17

#!/bin/bash
vip=10.0.0.100
mask='255.255.255.255'
dev=lo:1
case $1 in
start)
        echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
        ifconfig $dev $vip netmask $You can't use 'macro parameter character #' in
math modemask #broadcast $vip up
        #route add -host $vip dev $dev
        ;;
stop)
        ifconfig $dev down
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
        ;;
*)
        echo "Usage: $(basename $0) start|stop"
        exit 1
        ;;
esac

#!/bin/bash
vip='10.0.0.100'
iface='lo:1'
mask='255.255.255.255'
port='80'
rs1='192.168.8.101'
rs2='192.168.8.102'
scheduler='wrr'
type='-g'
case $1 in
start)
ifconfig $iface $vip netmask $mask #broadcast $vip up
iptables -F
ipvsadm -A -t ${vip}:${port} -s $scheduler
ipvsadm -a -t ${vip}:${port} -r ${rs1} $type -w 1
ipvsadm -a -t ${vip}:${port} -r ${rs2} $type -w 1
;;
stop)
ipvsadm -C
ifconfig $iface down
;;
*)
echo "Usage $(basename $0) start|stop"
exit 1
esac

[root@rs1 ~]#cat lvs_dr_rs.sh
#!/bin/bash
#Author:wang
#Date:2017-08-13
vip=192.168.10100
mask='255.255.255.255'
dev=lo:1
#rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
#service httpd start &> /dev/null && echo "The httpd Server is Ready!"
#echo "<h1>`hostname`</h1>" > /var/www/html/index.html
case $1 in
start)
        echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ifconfig $dev $vip netmask $mask #broadcast $vip up
        #route add -host $vip dev $dev
        echo "The RS Server is Ready!"
    ;;
stop)
    ifconfig $dev down
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
        echo "The RS Server is Canceled!"
    ;;
*)
        echo "Usage: $(basename $0) start|stop"
        exit 1
    ;;
esac

[root@rs1 ~]#bash lvs_dr_rs.sh start
[root@rs2 ~]#bash lvs_dr_rs.sh start

[root@LVS ~]#cat lvs_dr_vs.sh
#!/bin/bash
#Author:wang
#Date:2017-08-13
vip='192.168.10100'
iface='lo:1'
mask='255.255.255.255'
port='80'
rs1='10.0.0.7'
rs2='10.0.0.17'
scheduler='wrr'
type='-g'
rpm -q ipvsadm &> /dev/null || yum -y install ipvsadm &> /dev/null
case $1 in
start)
    ifconfig $iface $vip netmask $mask #broadcast $vip up
    iptables -F
    ipvsadm -A -t ${vip}:${port} -s $scheduler
    ipvsadm -a -t ${vip}:${port} -r ${rs1} $type -w 1
    ipvsadm -a -t ${vip}:${port} -r ${rs2} $type -w 1
        echo "The VS Server is Ready!"
    ;;
stop)
    ipvsadm -C
    ifconfig $iface down
        echo "The VS Server is Canceled!"
    ;;
*)
        echo "Usage: $(basename $0) start|stop"
        exit 1
    ;;
esac
[root@LVS ~]#bash lvs_dr_vs.sh start

[root@Router ~]#nmcli connection modify eth0 +ipv4.addresses 192.168.10200/24
[root@Router ~]#nmcli connection reload
[root@Router ~]#nmcli connection up eth0
[root@Router ~]#ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group
default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
            valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
            valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP
group default qlen 1000
    link/ether 00:0c:29:4d:ef:3e brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.200/24 brd 10.0.0.255 scope global noprefixroute eth0
            valid_lft forever preferred_lft forever
    inet 192.168.10200/24 brd 192.168.10255 scope global noprefixroute eth0
            valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe4d:ef3e/64 scope link
            valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP
group default qlen 1000
    link/ether 00:0c:29:4d:ef:48 brd ff:ff:ff:ff:ff:ff
    inet 172.20.200.200/16 brd 172.20.255.255 scope global noprefixroute eth1
            valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe4d:ef48/64 scope link
            valid_lft forever preferred_lft forever

/etc/ha.d/ldirectord.cf #主配置文件
/usr/share/doc/ldirectord-3.9.6/ldirectord.cf # 配置模版
/usr/lib/systemd/system/ldirectord.service # 服务
/usr/sbin/ldirectord #主程序,Perl实现
/var/log/ldirectord.log #日志
/var/run/ldirectord.ldirectord.pid #pid文件

[root@centos7 ~]#vim /etc/ha.d/ldirectord.cf
checktimeout=3
checkinterval=1
autoreload=yes
logfile="/var/log/ldirectord.log"
quiescent=no         #当RS down时 yes将修改权重为0,此配置有bug ，no为从调度列表中删除RS
virtual=10.0.0.100:80
real=192.168.39.17 gate 1     #gate 表示DR模式，1 表示weight
real=192.168.39.18 gate 2
fallback=127.0.0.1:80 gate
service=http
scheduler=wrr
#persistent=600
#netmask=255.255.255.255
protocol=tcp
checktype=negotiate
checkport=80
[root@lvs ~]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
    -> RemoteAddress:Port                     Forward Weight ActiveConn InActConn
TCP     10.0.0.100:80 wrr
    -> 192.168.39.17:80                         Route     1             0                     31             
    -> 192.168.39.18:80                         Route     2             0                     62

[root@centos7 ~]# /etc/ha.d/ldirectord.cf
checktimeout=3
checkinterval=1
autoreload=yes
logfile="/var/log/ldirectord.log"         #日志文件
quiescent=no                         #当RS down时 yes将修改权重为0,此配置有bug ，no为从调度列表中删
除RS
virtual=66                             #指定VS的FWM 或 IP:PORT
real=172.16.0.7:80 gate 2         #DR模型，权重为 2
real=172.16.0.8:80 gate 1
fallback=127.0.0.1:80 gate     #sorry server
service=http
scheduler=wrr
#protocol=tcp         #如果FWM模式，此行必须注释掉
checktype=negotiate
checkport=80
request="index.html"
receive="Test Ldirectord"