环境就不多做介绍了，还是上一篇中用到的四台机器，这里只是之前Heartbeat+Haproxy实现负载均衡高可用的补充罢了，废话少说，进入正题。
本文的目的将实现heartbeat绑定多个VIP，多个VIP又将分别代理多个不同的web服务，这些web服务之间做负载均衡，而VIP是高可用，进而实现haproxy的高可用。
| 主机名 | 角色 | IP地址 | 说明 |
| mylinux1.contoso.com | Heartbeat+Haproxy | eth0:192.168.100.121 eth1:172.16.100.121 |
VIP:192.168.100.120 |
| mylinux2.contoso.com | Heartbeat+Haproxy | eth0:192.168.100.122 eth1:172.16.100.122 |
VIP:192.168.100.110 |
| mylinux3.contoso.com | apache | eth0:192.168.100.181 | Web:80,8001,8002 |
| mylinux4.contoso.com | apache | eth0:192.168.100.182 | Web:80,8001,8002 |
这里heartbeat服务将产生两个VIP，mylinux1上默认启动VIP 192.168.100.120，而mylinux2上默认启动VIP 192.168.100.110，当某一台发生故障时，另一台将接管故障服务器的VIP。Haproxy两个服务器的配置相同，都将绑定192.168.100.110和192.168.100.120两个IP地址，从而达到高可用的目的。
注意：大家应该注意到，如果将两个VIP都绑定到同一台服务器上，然后让heartbeat控制haproxy服务，也可以达到上面的目的，但是这样的话，无论何时必定有一台主机获得两个VIP，且提供代理服务，而另外一个主机可能什么服务都没有，完全处于备用状态，为了充分利用服务器资源，所以不采用这种方式，因此才有了本文的介绍。
一、配置heartbeat
heartbeat的配置就不多介绍了，这里主要是修改haresources文件。
|
1
2
3
4
5
6
7
|
[root@mylinux1 conf]
# vi /etc/ha.d/haresources
[root@mylinux1 conf]
# tail -2 /etc/ha.d/haresources
mylinux1.contoso.com IPaddr::192.168.100.120
/24/eth0
mylinux2.contoso.com IPaddr::192.168.100.110
/24/eth0
[root@mylinux2 ~]
# tail -2 /etc/ha.d/haresources
mylinux1.contoso.com IPaddr::192.168.100.120
/24/eth0
mylinux2.contoso.com IPaddr::192.168.100.110
/24/eth0
|
二、修改haproxy配置文件
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
|
[root@mylinux1 conf]
# cat haproxy.cfg
# this config needs haproxy-1.1.28 or haproxy-1.2.1
global
#log 127.0.0.1 local0
log 127.0.0.1:514 local0 warning
pidfile
/usr/local/haproxy/var/run/haproxy
.pid
daemon
maxconn 4096
chroot
/usr/local/haproxy/var/chroot
user haproxy
group haproxy
nbproc 1
defaults
logglobal
mode http
retries3
option httplog
option httpclose
option dontlognull
option forwardfor
option redispatch
maxconn2000
balance roundrobin
timeout connect 5000
timeout client 50000
timeoutserver 50000
listen haproxy_stats
bind *:8000
mode http
option httplog
maxconn 20
stats
enable
stats refresh 30s
stats uri
/haproxy_status
stats auth admin:123456
stats hide-version
listenwebsites_01
bind 192.168.100.120:80
option forwardfor
#option httpchk GET /info.txt
#option httpchk HEAD /check.html HTTP/1.0
timeout server 15s
timeout connect 30s
server web1 192.168.100.181:8001 check port 8001 inter 2000 fall 3
server web2 192.168.100.182:8001 check port 8001 inter 2000 fall 3
listenwebsites_02
bind 192.168.100.110:80
option forwardfor
#option httpchk GET /info.txt
#option httpchk HEAD /check.html HTTP/1.0
timeout server 15s
timeout connect 30s
server web1 192.168.100.181:8002 check port 8002 inter 2000 fall 3
server web2 192.168.100.182:8002 check port 8002 inter 2000 fall 3
[root@mylinux1 conf]
# scp haproxy.cfg mylinux2:/usr/local/haproxy/conf/
root@mylinux2's password:
haproxy.cfg 100% 1608 1.6KB
/s
00:00
|
注意，要保证mylinux1和mylinux2上的配置文件一模一样。
三、同时启动heartbeat服务
|
1
2
3
4
5
6
7
8
|
[root@mylinux1 conf]
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO: Resource is stopped
INFO: Resource is stopped
Done.
[root@mylinux2 conf]
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO: Resource is stopped
INFO: Resource is stopped
Done.
|
最后，要确保VIP成功绑定：
|
1
2
3
4
|
[root@mylinux1 conf]
# ip a |grep 120
inet 192.168.100.120
/24
brd 192.168.100.255 scope global secondary eth0
[root@mylinux2 conf]
# ip a |grep 110
inet 192.168.100.110
/24
brd 192.168.100.255 scope global secondary eth0
|
四、启动haproxy服务
在mylinux1上启动haproxy服务：
|
1
2
3
4
5
|
[root@mylinux1 conf]
# service haproxy start
[ALERT] 275
/163638
(2078) : Starting proxy websites_02: cannot bind socket [192.168.100.110:80]
Start haproxy failed.
[root@mylinux1 conf]
# ps -ef|grep haproxy
root 2080 1035 0 16:36 pts
/0
00:00:00
grep
haproxy
|
发现无法启动，错误是因为无法绑定IP地址192.168.100.110，所以启动不成功。同样的，在mylinux2上也因为无法绑定IP地址192.168.100.120而无法启动。
解决方法：
在/etc/sysctl.conf中添加如下配置：
net.ipv4.ip_nonlocal_bind = 1
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
|
[root@mylinux1 conf]
# echo "net.ipv4.ip_nonlocal_bind = 1" >>/etc/sysctl.conf
[root@mylinux1 conf]
# tail -1 /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind = 1
[root@mylinux1 conf]
# sysctl -p
net.ipv4.ip_forward = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
fs.
file
-max = 2097152
fs.nr_open = 2097152
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_time = 1200
net.ipv4.ip_local_port_range = 1024 65000
net.ipv4.tcp_max_syn_backlog = 81920
net.ipv4.ip_nonlocal_bind = 1
|
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
|
[root@mylinux2 conf]
# echo "net.ipv4.ip_nonlocal_bind = 1" >>/etc/sysctl.conf
[root@mylinux2 conf]
# sysctl -p
net.ipv4.ip_forward = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
error:
"net.bridge.bridge-nf-call-ip6tables"
is an unknown key
error:
"net.bridge.bridge-nf-call-iptables"
is an unknown key
error:
"net.bridge.bridge-nf-call-arptables"
is an unknown key
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.ip_nonlocal_bind = 1
|
然后再次尝试启动haproxy服务：
|
1
2
3
4
5
|
[root@mylinux1 conf]
# service haproxy start
Start haproxy successful.
[root@mylinux1 conf]
# ps -ef|grep haproxy
haproxy 2102 1 0 16:43 ? 00:00:00
/usr/local/haproxy/sbin/haproxy
-f
/usr/local/haproxy/conf/haproxy
.cfg
root 2104 1035 0 16:43 pts
/0
00:00:00
grep
haproxy
|
|
1
2
3
4
5
|
[root@mylinux2 conf]
# service haproxy start
Start haproxy successful.
[root@mylinux2 conf]
# ps -ef|grep haproxy
haproxy 3225 1 0 16:44 ? 00:00:00
/usr/local/haproxy/sbin/haproxy
-f
/usr/local/haproxy/conf/haproxy
.cfg
root 3227 2036 0 16:44 pts
/0
00:00:00
grep
haproxy
|
五、测试代理访问
访问192.168.100.120，是转发给http://192.168.100.181:8001/和http://192.168.100.182:8001/，没有问题。
访问192.168.100.110，是转发给http://192.168.100.181:8002/和http://192.168.100.182:8002/，这里也显示正常。
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
[root@mylinux1 conf]
# for i in {1..10};do curl http://192.168.100.120/;done
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
[root@mylinux1 conf]
# for i in {1..10};do curl http://192.168.100.110/;done
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
|
在linux上进行测试，代理访问也正常。
六、模拟故障切换
这里将mylinux1上的heartbeat服务关闭，然后再进行代理访问测试。
|
1
2
3
4
5
|
[root@mylinux1 conf]
# /etc/init.d/heartbeat stop
Stopping High-Availability services: Done.
[root@mylinux1 conf]
# ip a |grep 192.168.100.120
[root@mylinux1 conf]
# service haproxy status
Haproxy (pid 2102) is running...
|
|
1
2
3
4
5
6
|
[root@mylinux2 conf]
# ip a |grep 192.168.100
inet 192.168.100.122
/24
brd 192.168.100.255 scope global eth0
inet 192.168.100.110
/24
brd 192.168.100.255 scope global secondary eth0
inet 192.168.100.120
/24
brd 192.168.100.255 scope global secondary eth0
[root@mylinux2 conf]
# service haproxy status
Haproxy (pid 3225) is running...
|
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
[root@mylinux1 conf]
# for i in {1..10};do curl http://192.168.100.120/;done
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
[root@mylinux1 conf]
# for i in {1..10};do curl http://192.168.100.110/;done
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
|
然后将mylinux1的heartbeat服务开启，同时将mylinux2的heartbeat服务关闭，再次进行代理访问测试。
|
1
2
3
4
5
6
|
[root@mylinux1 conf]
# /etc/init.d/heartbeat start
Starting High-Availability services: INFO: Resource is stopped
INFO: Resource is stopped
Done.
[root@mylinux2 conf]
# /etc/init.d/heartbeat stop
Stopping High-Availability services: Done.
|
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
[root@mylinux3 conf]
# for i in {1..1000};do curl http://192.168.100.120/;sleep 1;done
web3
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
web1
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
web1
web3
[root@mylinux4 conf]
# for i in {1..1000};do curl http://192.168.100.110/;sleep 1;done
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
web2
web4
|
在客户端上的测试发现，VIP的转移基本没有造成服务的中断，说明haproxy代理服务高可用设置成功。
七、建议配置
因为heartbeat的停止或者服务器宕机都会影响VIP的切换，但是haproxy服务需要自动启动，而不能由heartbeat控制，所以建议在开机启动项中把haproxy设置为开机启动，至于heartbeat服务，不建议设置开机启动，以防止出现裂脑现象。
|
1
2
3
4
5
6
7
8
|
[root@mylinux1 conf]
# chkconfig --add haproxy
[root@mylinux1 conf]
# chkconfig haproxy on
[root@mylinux1 conf]
# chkconfig --list haproxy
haproxy 0:off1:off2:on3:on4:on5:on6:off
[root@mylinux2 conf]
# chkconfig --add haproxy
[root@mylinux2 conf]
# chkconfig haproxy on
[root@mylinux2 conf]
# chkconfig --list haproxy
haproxy 0:off1:off2:on3:on4:on5:on6:off
|
注意：要让haproxy脚本能添加到chkconfig列表中去，需要添加如下内容：
#!/bin/bash
#
#chkconfig: 2345 20 70
#description: Start and stop haproxy service.
#
...
这样，以后只需要在重启服务器后手动开启heartbeat服务即可，即使出现脑裂，也只需要人为的处理一下，从而避免了haproxy服务的维护，提高了工作效率。
本文转自 jerry1111111 51CTO博客，原文链接：http://blog.51cto.com/jerry12356/1858448，如需转载请自行联系原作者
