High Availability Mysql DRBD with Heartbeat

What is DRBD ?

DRBD (Distributed Replicated Block Device) provides an alternative high availability solution for MySQL. DRBD mirrors a partition between two machines allowing only one of them to mount it at a time. DRBD uses a block file to synchronize a number of tasks, including the two independent HDD partitions in the active and passive servers for read and write operations. When the hot standby takes over, there is zero downtime because it already contains a copy of all data.

What is Heartbeat ?

Heartbeat then monitors the machines, and if it detects that one of the machines has died, it takes control by mounting the mirrored disk and starting all the services the other machine is running.

Server :- drbd1.shashi.com 192.168.0.101 /dev/sdb1
Server :- drbd2.shashi.com 192.168.0.102 /dev/sdb1

Both Server :-
# vi /etc/hosts
192.168.0.101  drbd1.shashi.com drbd1
192.168.0.102  drbd2.shashi.com drbd2

Install both server :-
# rpm -ivh    kmod-drbd82-8.2.6-2.i686.rpm
# rpm  -ivh   heartbeat-pils-2.1.3-3.el5.centos.i386.rpm
# rpm -ivh    heartbeat-stonith-2.1.3-3.el5.centos.i386.rpm
# rpm -ivh    heartbeat-devel-2.1.3-3.el5.centos.i386.rpm
# rpm -ivh    heartbeat-2.1.3-3.el5.centos.i386.rpm
 
# yum install mysql* mysql-server
# cat /proc/modules | grep drbd
drbd 199424 3 - Live 0xdcda7000 (U)
# more /etc/drbd.conf
# cp -rvf  /usr/share/doc/drbd82-8.2.6/drbd.conf  /etc/drbd.conf

Both Server Edit /etc/drbd.conf
##################################################################
# please have a look at the example configuration file in
# /usr/share/doc/drbd83/drbd.conf
#
global {
usage-count yes;
}
common {
syncer { rate 10M; }
}
resource r0 {
protocol C;
handlers {
pri-on-incon-degr "echo o > /proc/sysrq-trigger ; halt -f";
pri-lost-after-sb "echo o > /proc/sysrq-trigger ; halt -f";
local-io-error "echo o > /proc/sysrq-trigger ; halt -f";
outdate-peer "/usr/lib/heartbeat/drbd-peer-outdater -t 5";
}
startup {
degr-wfc-timeout 120;
}
 
  disk {
on-io-error   detach;
}
  net {
after-sb-0pri disconnect;
after-sb-1pri disconnect;
after-sb-2pri disconnect;
rr-conflict disconnect;
}
syncer {
rate 10M;
al-extents 257;
}
on drbd1.shashi.com {
device /dev/drbd0;
disk /dev/sdb1;
address 192.168.0.101:7789;
flexible-meta-disk internal;
}
on drbd2.shashi.com {
device /dev/drbd0;
disk /dev/sdb1;
address 192.168.0.102:7789;
meta-disk internal;
}

# rpm -ivh   drbd82-8.2.6-1.el5.centos.i386.rpm

#####################################################################

Create device metadata {any name r0} :-
Server-1.
# modprobe drbd
# drbdadm create-md r0
# service drbd start
# more /proc/drbd {check drbd}
#echo "modprobe drbd" >> /etc/rc.local
 Server-2.
# modprobe drbd
# drbdadm create-md r0
# service drbd start
# more /proc/drbd {check drbd}
#echo "modprobe drbd" >> /etc/rc.local
 
Run the following command to start the initial synchronization. Run this command only in the  server-1 :-
# drbdadm  - -   - -overwrite-data-of-peer primary r0
Both server Check status by :-
# cat /proc/drbd  OR more /proc/drbd
Run this command on server-1
# mkfs.ext3 /dev/drbd0
Create directory Both server :-
mkdir /data
Server1:-
# mount /dev/drbd0  /data
# cd /data
# touch file{1,2,3,4,5}
# cat /proc/drbd
#  umount /data
# drbdadm secondary  r0
 
Server 2:-
 
#drbdadm primary  r0
 
# mount /dev/drbd0  /data
 
# cd /data {showing all data}
 
#cat /proc/drbd
 
# drbdadm  state  r0
Primary/Secondary
 
# drbdadm  cstate  r0
Connected
 
# drbdadm  dstate  r0
UpToDate/UpToDate
 
Other command :-
# drbdadm  disconnect  r0
# drbdadm  detach  r0
# drbdadm  down  r0
# drbdadm  dstate  r0
# drbdadm  state  r0
unconfigured
 
As we are using dopd-peer-outdater as fence-peer program. If you use that mechanism the dopd heartbeat plug-in program needs to be able to call drbdsetup and drbdmeta with root privileges.
U need to fix this with the following commands.
Chgrp haclient /sbin/drbdsetup
Chmod o-x /sbin/drbdsetup
Chmod u+s /sbin/drbdsetup
Chgrp haclient /sbin/drbdmeta
Chmod o-x /sbin/drbdmeta
Chmod u+s /sbin/drbdmeta
 
 
heartbeat
 
rpm -ivh heartbeat-2.1.3-3.el5.centos.x86_64.rpm
rpm -ivh heartbeat-pils-2.1.3-3.el5.centos.x86_64.rpm
rpm -ivh heartbeat-stonith-2.1.3-3.el5.centos.x86_64.rpm
 
OR
 
yum install heartbeat
 
3. Create configuration file :-
# vim /etc/ha.d/ha.cf
 
logfacility local0
keepalive 2
deadtime 10
bcast eth0
baud 19200
auto_failback off
node drbd1.shashi.com
node drbd2.shashi.com
 
# scp -rv ha.cf  drbd2:/etc/ha.d/
 
4.Create directory both Server :-
# mkdir  /data/mysql
# chown -R mysql:mysql /data/mysql
 
5.Create haresources file both Server :-
{IP virtual :- 192.168.0.50}
Server-1
# vim /etc/ha.d/haresources
drbd1.shashi.com IPaddr::192.168.0.50/24/eth0 drbddisk::r0 Filesystem::/dev/drbd0::/data::ext3 mysqld
 
Server-2
# vim /etc/ha.d/haresources
drbd2.shashi.com IPaddr::192.168.0.50/24/eth0 drbddisk::r0 Filesystem::/dev/drbd0::/data::ext3 mysqld
 
6.Create a auth file both Server :- {redhat password}
vi /etc/ha.d/authkeys
auth 3
3 md5 redhat
 
# chmod 600 /etc/ha.d/authkeys
 
7.Change some line in my.cnf, Both Server :-
 
 # vim /etc/my.cnf
datadir=/data/mysql
 
mysqladmin -u root password 'shashi'
 
mysql> create database jk;
mysql>GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY 'shashi'
mysql> FLUSH PRIVILEGES;
mysql>quit 
 
# /etc/init.d/heartbeat start
#chkconfig  heartbeat on
 
check command
 
# ifconfig
 
eth0      Link encap:Ethernet  HWaddr 08:00:27:8C:31:FD 
          inet addr:192.168.0.10  Bcast:192.168.0.255  Mask:255.255.255.0
          inet6 addr: fe80::a00:27ff:fe8c:31fd/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:9317 errors:0 dropped:0 overruns:0 frame:0
          TX packets:11407 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:2023598 (1.9 MiB)  TX bytes:2060303 (1.9 MiB)
          Base address:0xd010 Memory:f0000000-f0020000
 
eth0:0    Link encap:Ethernet  HWaddr 08:00:27:8C:31:FD 
          inet addr:192.168.0.50  Bcast:192.168.0.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          Base address:0xd010 Memory:f0000000-f0020000
 
lo        Link encap:Local Loopback 
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:3290 errors:0 dropped:0 overruns:0 frame:0
          TX packets:3290 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:3739088 (3.5 MiB)  TX bytes:3739088 (3.5 MiB)
 
 
# moun t  {auth mount}
/dev/drbd0 on /data type ext3 (rw)

You can find the log of Heartbeat in /var/log/messages
Restart the machine . Check the mysql is running and partition is mounted in active machine only.