Three-node HAProxy with Keepalived

Aug. 12, 2017
Category: Linux
Tags: HAProxy Keepalived

1. Configure servers
2. Configure HAProxy on all three nodes
3. Configure keepalived
4. Bind to non-local interfaces
5. Syslog config
6. Configure postfix on all three to just send email

I am using these three-node configuration with HAProxy as LB and reverse proxy for SMTP, IMAP, MariaDB Galera Cluster.

srv10 is master for SMTP
srv11 is master for mariadb galera cluster02
srv27 is master for IMAP

All servers - CentOS 7

1. Configure servers

yum install epel-release
yum update

/etc/sysconfig/selinux

permissive

systemctl disable NetworkManager
systemctl disable firewalld
systemctl enable network.service
yum install iptables-services
systemctl enable iptables.service

yum install haproxy
yum install keepalived
systemctl enable haproxy
systemctl enable keepalived

/etc/hosts
192.168.122.144  srv20.domain.dom srv20
192.168.122.145  srv21.domain.dom srv21
192.168.122.146  srv10.domain.dom srv10
192.168.122.147  srv11.domain.dom srv11
192.168.122.128  srv27.domain.dom srv27

2. Configure HAProxy on all three nodes

On node srv10:

/etc/haproxy/haproxy.cfg

global
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats

defaults
    log                     global
    option                  dontlognull
    option http-server-close
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# HAProxy statistics backend
#---------------------------------------------------------------------
listen srv10-stats 192.168.122.142:80
   mode http
   stats enable
   stats show-legends
   stats refresh 5s
   stats uri /
   stats realm Haproxy\ Statistics
   stats auth username:password
   stats admin if TRUE

#---------------------------------------------------------------------
# Frontend for mariadb galera cluster 
#---------------------------------------------------------------------
frontend srv10-cluster02
   bind 192.168.122.143:3306
   mode tcp
   log global
   option tcplog
   default_backend srv10-galera-cluster02

#---------------------------------------------------------------------
# Backend for mariadb galera cluster
#---------------------------------------------------------------------
backend srv10-galera-cluster02
   balance leastconn
   option httpchk
   mode tcp
   option log-health-checks
   option tcplog
   server srv20 192.168.122.144:3306 check port 9200 inter 300s weight 1 maxconn 5000 rise 2 fall 2
   server srv21 192.168.122.145:3306 check port 9200 inter 300s weight 1 maxconn 5000 rise 2 fall 2

#---------------------------------------------------------------------
# SMTP postfix
#---------------------------------------------------------------------
listen srv10-smtp 192.168.122.139:25
    mode tcp
    log global
    option tcplog
    option log-health-checks
    no option http-server-close
    balance roundrobin
    option smtpchk HELO srv10.domain.dom
    server srv14 192.168.122.140:25 send-proxy check inter 900s
    server srv15 192.168.122.141:25 send-proxy check inter 900s

#---------------------------------------------------------------------
# Dovecot IMAP and IPMAS
#---------------------------------------------------------------------
listen srv10-imaps 192.168.122.122:993
   timeout client 1m
   no option http-server-close
   option log-health-checks
   stick store-request src
   stick-table type ip size 200k expire 30m
   mode tcp
   option tcplog
   server srv31 192.168.122.123:10993 check send-proxy-v2 inter 1900s

On node srv11:

/etc/haproxy/haproxy.cfg

global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats

defaults
    log                     global
    option                  dontlognull
    option http-server-close
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000


#---------------------------------------------------------------------
# HAProxy statistics backend
#---------------------------------------------------------------------
listen srv11-stats 192.168.122.142:80
   mode http
   stats enable
   stats show-legends
   stats refresh 5s
   stats uri /
   stats realm Haproxy\ Statistics
   stats auth username:password
   stats admin if TRUE

#---------------------------------------------------------------------
# Frontend for mariadb galera cluster 
#---------------------------------------------------------------------
frontend srv11-cluster02 
   bind 192.168.122.143:3306
   mode tcp
   log global
   option tcplog
   default_backend srv11-galera-cluster02

#---------------------------------------------------------------------
# Backend for mariadb galera cluster
#---------------------------------------------------------------------
backend srv11-galera-cluster02
   balance leastconn
   option httpchk
   mode tcp
   option log-health-checks
   option tcplog
   server srv20 192.168.122.144:3306 check port 9200 inter 300s weight 1 maxconn 5000 rise 2 fall 2
   server srv21 192.168.122.145:3306 check port 9200 inter 300s weight 1 maxconn 5000 rise 2 fall 2

#---------------------------------------------------------------------
# SMTP postfix
#---------------------------------------------------------------------
listen srv11-smtp 192.168.122.139:25
    mode tcp
    log global
    option tcplog
    option  log-health-checks
    no option http-server-close
    balance roundrobin
    option smtpchk HELO srv11.domain.dom
    server srv14 192.168.122.140:25 send-proxy check inter 900s
    server srv15 192.168.122.141:25 send-proxy check inter 900s

#---------------------------------------------------------------------
# Dovecot IMAP and IPMAS
#---------------------------------------------------------------------
listen srv11-imaps 192.168.122.122:993
   timeout client 1m
   no option http-server-close
   option log-health-checks
   stick store-request src
   stick-table type ip size 200k expire 30m
   mode tcp
   option tcplog
   server srv31 192.168.122.123:10993 check send-proxy-v2 inter 1900s

On node srv27:

/etc/haproxy/haproxy.cfg

global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats

defaults
    log                     global
    option                  dontlognull
    option http-server-close
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000


#---------------------------------------------------------------------
# HAProxy statistics backend
#---------------------------------------------------------------------
listen srv27-stats 192.168.122.142:80
   mode http
   stats enable
   stats show-legends
   stats refresh 5s
   stats uri /
   stats realm Haproxy\ Statistics
   stats auth username:password
   stats admin if TRUE

#---------------------------------------------------------------------
# Frontend for mariadb galera cluster 
#---------------------------------------------------------------------
frontend srv27-cluster02 # change on 2nd HAProxy
   bind 192.168.122.143:3306
   mode tcp
   log global
   option tcplog
   default_backend srv27-galera-cluster02

#---------------------------------------------------------------------
# Backend for mariadb galera cluster
#---------------------------------------------------------------------
backend srv27-galera-cluster02
   balance leastconn
   option httpchk
   mode tcp
   option log-health-checks
   option tcplog
   server srv20 192.168.122.144:3306 check port 9200 inter 300s weight 1 maxconn 5000 rise 2 fall 2
   server srv21 192.168.122.145:3306 check port 9200 inter 300s weight 1 maxconn 5000 rise 2 fall 2

#---------------------------------------------------------------------
# SMTP postfix
#---------------------------------------------------------------------
listen srv27-smtp 192.168.122.139:25
    mode tcp
    log global
    option tcplog
    option log-health-checks
    no option http-server-close
    balance roundrobin
    option smtpchk HELO srv10.domain.dom
    server srv14 192.168.122.140:25 send-proxy check inter 900s
    server srv15 192.168.122.141:25 send-proxy check inter 900s

#---------------------------------------------------------------------
# Dovecot IMAP and IPMAS
#---------------------------------------------------------------------
listen srv27-imaps 192.168.122.122:993
   timeout client 1m
   no option http-server-close
   option log-health-checks
   stick store-request src
   stick-table type ip size 200k expire 30m
   mode tcp
   option tcplog
   server srv31 192.168.122.123:10993 check send-proxy-v2 inter 1900s

3. Configure keepalived

On node srv10:

/etc/keepalived/keepalived.conf

! Configuration File for keepalived - srv10

global_defs {
   notification_email {
     admin@domain.dom
   }
   notification_email_from srv10@domain.dom
   smtp_server 192.168.122.139
   smtp_connect_timeout 30
   router_id srv10
}

vrrp_script check_haproxy {
   script "killall -0 haproxy"
   interval 2 # every 2 seconds
   weight 2 # add 2 points if OK
}

vrrp_instance cluster02 {
    state BACKUP
    interface ens160
    virtual_router_id 51
    priority 99
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.143
    }

    track_script {
       check_haproxy
   }
}

vrrp_instance haproxy-stats {
    state BACKUP
    interface ens160
    virtual_router_id 52
    priority 99
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.142
    }

    track_script {
       check_haproxy
   }

}

vrrp_instance mail {
    state MASTER
    interface ens160
    virtual_router_id 53
    priority 101
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.139
    }

    track_script {
       check_haproxy
   }
}

vrrp_instance imap {
    state BACKUP
    interface ens160
    virtual_router_id 54
    priority 100
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.122
    }

    track_script {
       check_haproxy
   }
}

On node srv11:

/etc/keepalived/keepalived.conf

! Configuration File for keepalived - srv11

global_defs {
   notification_email {
     admin@domain.dom
   }
   notification_email_from srv11@domain.dom
   smtp_server 192.168.122.139
   smtp_connect_timeout 30
   router_id srv11
}

vrrp_script check_haproxy {
   script "killall -0 haproxy"
   interval 2 # every 2 seconds
   weight 2 # add 2 points if OK
}

vrrp_instance cluster02 {
    state MASTER
    interface ens160
    virtual_router_id 51
    priority 101
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.143
    }

    track_script {
       check_haproxy
   }
}

vrrp_instance haproxy-stats {
    state BACKUP
    interface ens160
    virtual_router_id 52
    priority 100
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.142
    }

    track_script {
       check_haproxy
   }
}

vrrp_instance mail {
    state BACKUP
    interface ens160
    virtual_router_id 53
    priority 100
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.139
    }

    track_script {
       check_haproxy
   }
}


vrrp_instance imap {
    state BACKUP
    interface ens160
    virtual_router_id 54
    priority 99
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.122
    }

    track_script {
       check_haproxy
   }
}

On node srv27:

/etc/keepalived/keepalived.conf

! Configuration File for keepalived - srv27

global_defs {
   notification_email {
     admin@domain.dom
   }
   notification_email_from srv27@domain.dom
   smtp_server 192.168.122.139
   smtp_connect_timeout 30
   router_id srv27
}

vrrp_script check_haproxy {
   script "killall -0 haproxy"
   interval 2 # every 2 seconds
   weight 2 # add 2 points if OK
}

vrrp_instance cluster02 {
    state BACKUP
    interface ens160
    virtual_router_id 51
    priority 100
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.143
    }

    track_script {
       check_haproxy
   }
}

vrrp_instance haproxy-stats {
    state BACKUP
    interface ens160
    virtual_router_id 52
    priority 101
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.142
    }

    track_script {
       check_haproxy
   }
}

vrrp_instance mail {
    state BACKUP
    interface ens160
    virtual_router_id 53
    priority 99
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.139
    }

    track_script {
       check_haproxy
   }
}

vrrp_instance imap {
    state MASTER
    interface ens160
    virtual_router_id 54
    priority 101
    smtp_alert
    advert_int 1

    virtual_ipaddress {
        192.168.122.122
    }

    track_script {
       check_haproxy
   }
}

4. Bind to non-local interfaces

echo 1 > /proc/sys/net/ipv4/ip_nonlocal_bind

/etc/sysctl.conf

net.ipv4.ip_nonlocal_bind = 1

5. Syslog config

/etc/rsyslog.d/haproxy.conf

local2.=info     /var/log/haproxy-access.log
local2.notice    /var/log/haproxy-info.log

/etc/rsyslog.conf

# Provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514

### added
$ModLoad imudp
$UDPServerRun 514

6. Configure postfix on all three to just send email

on srv10:

mynetworks_style = host
# Setting postfix to only send email:
myhostname = srv10.domain.dom
myorigin = $mydomain
relayhost = $mydomain
inet_interfaces = loopback-only
mydestination =

on srv11:

mynetworks_style = host
# Setting postfix to only send email:
myhostname = srv11.domain.dom
myorigin = $mydomain
relayhost = $mydomain
inet_interfaces = loopback-only
mydestination =

on srv27:

mynetworks_style = host
# Setting postfix to only send email:
myhostname = srv27.domain.dom
myorigin = $mydomain
relayhost = $mydomain
inet_interfaces = loopback-only
mydestination =