Fork me on GitHub
Suzf  Blog

how-to create a high-availability mysql setup with corosync pacemaker and drbd on ubuntu

前言 坑无处不有。对各个组件大家都是仁者见仁智者见智。 各个组件的工作原理适用场景就不在一 一阐述。 ...... 待续 ...... 环境准备 Corosync 安装与配置 Pacemaker 安装与配置 DRBD 安装与配置 MySQL 安装与配置 Crm 资源管理 系统结构 mysql-drbd-ha 环境准备
$cat /etc/issue
Ubuntu 14.04.4 LTS \n \l

$uname  -r
4.2.0-27-generic

[Both]
$grep '10\|20' /etc/hosts
172.16.9.10 eva.suzf.net  eva
172.16.9.20 cali.suzf.net cali
ssh互信
eva
  ssh-keygen -t rsa -P ''
  ssh-copy-id -i .ssh/id_rsa.pub cali

cali
  ssh-keygen -t rsa -P ''
  ssh-copy-id -i .ssh/id_rsa.pub eva
Selinux Ubuntu 默认没有 安装,跳过 sudo service iptables stop Corosync 安装与配置
[Both]
sudo apt-get update
sudo apt-get install corosync -y
修改配置文件和生成认证文件
$grep -v "^.*#\|^$" /etc/corosync/corosync.conf
totem {
  version: 2
  token: 3000
  token_retransmits_before_loss_const: 10
  join: 60
  consensus: 3600
  vsftype: none
  max_messages: 20
  clear_node_high_bit: yes
  secauth: off
  threads: 0
  rrp_mode: none
  interface {
        member{
            memberaddr: 172.16.9.10
        }
        member{
            memberaddr: 172.16.9.20
        }
    ringnumber: 0
    bindnetaddr: 172.16.9.0
    mcastaddr: 226.94.1.1
    mcastport: 5405
  }
}
amf {
  mode: disabled
}
quorum {
  provider: corosync_votequorum
  expected_votes: 1
}
aisexec {
        user:   root
        group:  root
}
logging {
        fileline: off
        to_stderr: yes
        to_logfile: no
        to_syslog: yes
  syslog_facility: daemon
        debug: off
        timestamp: on
        logger_subsys {
                subsys: AMF
                debug: off
                tags: enter|leave|trace1|trace2|trace3|trace4|trace6
        }
}
同步文件到 cali 节点
rsync -av corosync.conf cali:/etc/corosync/
认证文件
corosync-keygen
Corosync Cluster Engine Authentication key generator.
Gathering 1024 bits for key from /dev/random.
Press keys on your keyboard to generate entropy (bits = 152).
#遇到这个情况,表示电脑的随机数不够,各位朋友可以不停的随便敲键盘,或者安装软件也可以生成随机数
把认证文件也复制到cali主机上
rsync -av authkey cali:/etc/corosync/
开启服务
cat /etc/default/corosync
# start corosync at boot [yes|no]
START=yes

[both]
sudo service corosync start
Pacemaker 安装配置
[Both]
sudo apt-get-install pacemaker -y
sudo service pacemaker start
查看集群中的节点信息
$sudo crm status
Last updated: Wed Jun 29 14:35:21 2016
Last change: Mon Jun 27 16:18:55 2016 via crmd on eva
Stack: corosync
Current DC: cali (739248404) - partition with quorum
Version: 1.1.10-42f2063
2 Nodes configured
0 Resources configured

Online: [ cali eva ]
DRBD 安装与配置 创建一个新的磁盘分区
[Only on node]
@eva ~]$sudo fdisk /dev/sdb
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel with disk identifier 0x56c8d4fc.
Changes will remain in memory only, until you decide to write them.
After that, of course, the previous content won't be recoverable.

Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)

Command (m for help): n
Partition type:
   p   primary (0 primary, 0 extended, 4 free)
   e   extended
Select (default p): p
Partition number (1-4, default 1): 1
First sector (2048-81788927, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-81788927, default 81788927):
Using default value 81788927


Command (m for help): p

Disk /dev/sdb: 41.9 GB, 41875931136 bytes
255 heads, 63 sectors/track, 5091 cylinders, total 81788928 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x56c8d4fc

   Device Boot      Start         End      Blocks   Id  System
/dev/sdb1            2048    81788927    40893440   83  Linux

Command (m for help): w
The partition table has been altered!

Calling ioctl() to re-read partition table.
Syncing disks.



sudo pvcreate  /dev/sdb1
sudo vgcreate  vol2 /dev/sdb1

sudo lvcreate  -L 3G -n mysql-drbd vol2
# sudo mkfs.ext4 /dev/vol2/mysql-drbd #这里不需要格式化
安装drbd和修改配置文件
[both]
sudo apt-get install drbd8-utils -y
ubuntu14.4.04 kernel中编译的 drbd module 版本是 8.4.5, 而官方trusty所提供的drbd8-utils的版本只到8.4.4, 只能手动编译了 详见 http://suzf.net/thread-0628-918.html
DRBD有如下3种模式: 协议A:异步复制协议。本地写成功后立即返回,数据放在发送buffer中,可能丢失。 协议B:内存同步(半同步)复制协议。本地写成功并将数据发送到对方后立即返回,如果双机掉电,数据可能丢失。 协议C:同步复制协议。本地和对方写成功确认后返回。如果双机掉电或磁盘同时损坏,则数据可能丢失。 在使用时,一般用协议C。由于协议C是本地和对方写成功时再认为写入成功,因此会有一定时延。
修改 drbd 配置文件
@eva ~]$uname  -n
eva

$pwd
/etc/drbd.d

$cat global_common.conf
global {
  usage-count no;
}
common {
  protocol C;
  startup {
    wfc-timeout 10;
    degr-wfc-timeout 1;
    outdated-wfc-timeout 1;
  }
}


$cat mysql-drbd.res
resource mysql-drbd {

  protocol  C;
  device    /dev/drbd0;
  disk      /dev/vol2/mysql-drbd;
  meta-disk internal;

  net {
    cram-hmac-alg sha1;
    shared-secret "mysql-drbd";
    #allow-two-primaries;
  }

  syncer {
    verify-alg crc32c;
    rate 1000M;
  }

  on eva {
    address 172.16.9.10:7789;
  }

  on cali {
    address 172.16.9.20:7789;
  }

}
同步brbd配置文件到 cali 节点
rsync  -av ./* cali:/etc/drbd.d/
初始化drbd的资源并启动 drbd 创建资源之前不需要讲分区格式化 如果已经格式化请执行下面操作
dd if=/dev/zero of=/dev/vol2/mysql-drbd bs=1M count=1
创建DRBD 资源
@eva ~]$sudo drbdadm create-md mysql-drbd
initializing activity log
NOT initializing bitmap
Writing meta data...
New drbd meta data block successfully created.

@eva ~]$sudo service drbd start
* Starting DRBD resources                                                                                                                      [
create res: mysql-drbd
prepare disk: mysql-drbd
adjust disk: mysql-drbd
adjust net: mysql-drbd
]
[ OK ]
Cali 节点操作
sudo drbdadm create-md mysql-drbd
sudo service drbd start
格式化分区
@eva ~]$sudo drbdadm primary --force mysql-drbd
@eva ~]$sudo cat /proc/drbd
version: 8.4.5 (api:1/proto:86-101)
srcversion: 5A4F43804B37BB28FCB1F47
0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate B r-----
ns:0 nr:0 dw:0 dr:728 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:3145596

@eva ~]$sudo mke2fs -t ext4 /dev/drbd0
mke2fs 1.42.9 (4-Feb-2014)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
196608 inodes, 786399 blocks
39319 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=805306368
24 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912

Allocating group tables: done
Writing inode tables: done
Creating journal (16384 blocks): done
Writing superblocks and filesystem accounting information: done

@eva ~]$sudo mkdir -p       /data/mysql-drbd/
@eva ~]$sudo chown -R mysql /data/mysql-drbd/
@eva ~]$sudo chgrp -R mysql /data/mysql-drbd/
@eva ~]$sudo chmod -R 755   /data/mysql-drbd/
    Mysql 安装与配置 安装Mysql 修改数据目录
[Both]
sudo apt-get install mysql-server -y

#vim /etc/my.cnf
datadir = /data/mysql-drbd

# grep data -B 2 /etc/apparmor.d/usr.sbin.mysqld
/var/lib/mysql/ r,
/var/lib/mysql/** rwk,
/data/mysql-drbd/ r,
/data/mysql-drbd/** rwk,
sudo service apparmor restart
在其中一个node 初始化数据
@eva ~]$sudo mysql_install_db --user=mysql --datadir=/data/mysql-drbd/
@eva ~]$sudo service mysql start
在eva上创建一个测数据库
mysql> show databases;
+---------------------+
| Database            |
+---------------------+
| information_schema  |
| eva                 |
| #mysql50#lost+found |
| mysql               |
| performance_schema  |
+---------------------+
5 rows in set (0.00 sec)

mysql> system hostname
eva
停掉mysql服务,卸载drbd挂载的目录
# Eva
sudo service mysql stop
sudo umount /dev/drbd0
sudo drbdadm  secondary mysql-drbd  #把此节点改为drbd的备用节点
挂载到 Cali, 验证数据是否同步
# Cali
sudo drbdadm  primary mysql-drbd   #把此节点改为drbd的主节点
sudo chown -p       /data/mysql-drbd/
sudo chown -R mysql /data/mysql-drbd/
sudo chgrp -R mysql /data/mysql-drbd/
sudo chmod -R 755   /data/mysql-drbd/
sudo mount /dev/drbd0 /data/mysql-drbd

sudo service mysql start   #此节点上不用初始化数据库,直接开启服务即可
查看数据是否同步
@cali ~]$sudo mysql -e "show databases;"
+---------------------+
| Database            |
+---------------------+
| information_schema  |
| eva                 |
| #mysql50#lost+found |
| mysql               |
| performance_schema  |
+---------------------+

@eva ~]$sudo cat /proc/drbd
version: 8.4.5 (api:1/proto:86-101)
srcversion: 5A4F43804B37BB28FCB1F47
0: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate B r-----
ns:159000 nr:2400 dw:161400 dr:8738 al:38 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0

@cali ~]$sudo cat /proc/drbd
version: 8.4.5 (api:1/proto:86-101)
srcversion: 5A4F43804B37BB28FCB1F47
0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate B r-----
ns:2388 nr:159000 dw:161388 dr:7602 al:6 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0

@cali ~]$sudo service drbd  status
drbd driver loaded OK; device status:
version: 8.4.5 (api:1/proto:86-101)
srcversion: 5A4F43804B37BB28FCB1F47
m:res         cs         ro                 ds                 p  mounted           fstype
0:mysql-drbd  Connected  Primary/Secondary  UpToDate/UpToDate  B  /data/mysql-drbd  ext4
其中第一行为DRBD的版本信息,第二行为DRBD开发都最后提交源码的时间点。 第三行就是DRBD的状态信息了: cs: connection status 连接状态 ro: role 角色,本机角色/远程主机角色(注:旧版本DRBD为st) ds: data status 数据同步状态,本机状态/远程主机状态(由于示例没有设置主备,因此两边的状态都显示Incosistent) C 使用的同步协议
Crm  资源管理 需要定义集群资源而mysql、drbd都是集群的资源,由集群管理的资源开机是一定不能够自行启动的。 关闭mysql的服务 卸载drbd分区 和 drbd的服务, 设置mysql 开机禁止启动
=== CRM CMD START ===
# verify 验证配置是否正确
# commit 提交配置

sudo crm configure
# 定义drbd的资源(提供drbd的资源代理RA由OCF类别中的linbit提供)
property stonith-enabled=false
property no-quorum-policy=ignore
primitive p_mysql_drbd ocf:linbit:drbd params drbd_resource=mysql-drbd op monitor role=Master interval=10 timeout=20  op monitor  role=Slave interval=20 timeout=20 op start timeout=240 op stop timeout=100

# 定义drbd的主从资源
ms m_mysql_drbd p_mysql_drbd meta master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true

# 定义文件系统资源和约束关系
primitive p_fs_mysql_data ocf:heartbeat:Filesystem params device="/dev/drbd0" directory="/data/mysql-drbd" fstype="ext4" op monitor interval=40 timeout=40 op start timeout=60 op stop timeout=60
colocation c_mysql_drbd inf: p_fs_mysql_data m_mysql_drbd:Master
order o_mysql_drbd mandatory: m_mysql_drbd:promote p_fs_mysql_data:start

# 定义vip资源、mysql服务的资源约束关系
primitive p_mysql_vip ocf:heartbeat:IPaddr2 \
params ip="172.16.9.33" cidr_netmask="24" \
op monitor interval="30s"

primitive p_mysqld ocf:heartbeat:mysql \
params binary="/usr/sbin/mysqld" config="/etc/mysql/my.cnf" datadir="/data/mysql-drbd" \
pid="/var/run/mysqld/mysqld.pid" socket="/var/run/mysqld/mysqld.sock" log="/var/log/mysql/error.log" \
op start interval="0" timeout="120" \
op stop interval="0" timeout="120" \
op monitor interval="10" timeout="30" depth="0"

colocation c_mysql_with_data inf:  p_mysqld p_fs_mysql_data
order o_drbd_before_mysql mandatory: p_fs_mysql_data:start  p_mysqld:start

colocation p_mysql_vip_with_mysql inf: p_mysql_vip  p_mysqld
order o_mysql_vip_before_mysql mandatory: p_mysql_vip  p_mysqld
commit
=== CRM CMD END ===
在线查看当前配置
crm(live)configure# show
node $id="739248394" eva \
attributes standby="off" maintenance="off"
node $id="739248404" cali \
attributes standby="off" maintenance="off"
primitive p_fs_mysql_data ocf:heartbeat:Filesystem \
params device="/dev/drbd0" directory="/data/mysql-drbd" fstype="ext4" \
op monitor interval="40" timeout="40" \
op start timeout="60" interval="0" \
op stop timeout="60" interval="0"
primitive p_mysql_drbd ocf:linbit:drbd \
params drbd_resource="mysql-drbd" \
op monitor role="Master" interval="10" timeout="20" \
op monitor role="Slave" interval="20" timeout="20" \
op start timeout="240" interval="0" \
op stop timeout="100" interval="0"
primitive p_mysql_vip ocf:heartbeat:IPaddr2 \
params ip="172.16.9.33" cidr_netmask="24" \
op monitor interval="30s"
primitive p_mysqld ocf:heartbeat:mysql \
params binary="/usr/sbin/mysqld" config="/etc/mysql/my.cnf" datadir="/data/mysql-drbd" pid="/var/run/mysqld/mysqld.pid" socket="/var/run/mysqld/mysqld.sock" log="/var/log/mysql/error.log" \
op start interval="0" timeout="120" \
op stop interval="0" timeout="120" \
op monitor interval="10" timeout="30" depth="0"
ms m_mysql_drbd p_mysql_drbd \
meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true"
location cli-prefer-p_mysql_vip p_mysql_vip inf: eva
colocation c_mysql_drbd inf: p_fs_mysql_data m_mysql_drbd:Master
colocation c_mysql_with_data inf: p_mysqld p_fs_mysql_data
colocation p_mysql_vip_with_mysql inf: p_mysql_vip p_mysqld
order o_drbd_before_mysql inf: p_fs_mysql_data:start p_mysqld:start
order o_mysql_drbd inf: m_mysql_drbd:promote p_fs_mysql_data:start
order o_mysql_vip_before_mysql inf: p_mysql_vip p_mysqld
property $id="cib-bootstrap-options" \
stonith-enabled="false" \
no-quorum-policy="ignore" \
dc-version="1.1.10-42f2063" \
cluster-infrastructure="corosync"
20160630174437 验证 可以看到第一次 资源全部在 eva 这个节点上 当 eva 节点设为 standby 可以看到资源全部迁移到 cali 节点 20160630174211 ~~~ END made in suzf.net ~~~