Linux · 2013-04-01

MogileFS安装及相关测试

#检查系统时间
date

#安装环境所需工具
yum install make gcc unzip perl-DBD-MySQL perl perl-CPAN perl-YAML perl-Time-HiRes -y

#使用cpanm安装所需模块
wget https://raw.github.com/miyagawa/cpanminus/master/cpanm -O /usr/local/bin/cpanm –no-check-certificate && chmod +x /usr/local/bin/cpanm

#使用cpanm安装mogileFS及所需模块
cpanm App::cpanminus MogileFS::Server MogileFS::Utils IO::AIO IO::WrapTie Danga::Socket

#安装Mysql服务器,将存放MogileFS所有元信息,建议主从模式
yum install mysql-server.i386 -y

# mysql
mysql> CREATE DATABASE mogilefs;
mysql> GRANT ALL ON mogilefs.* TO ‘mogilefs’@’%’ identified by ‘mogilefs’;
mysql> FLUSH PRIVILEGES;
mysql> quit

#使用moglieFS自带工具创建MogileFS表等数据
mogdbsetup –dbhost=$Mysql_Server –dbname=mogilefs –dbuser=mogilefs –dbpassword=mogilefs

#创建mogilefs文件夹
mkdir /etc/mogilefs
#定义Trackers配置文件
vim /etc/mogilefs/mogilefsd.conf
#===cut===begin===#
db_dsn = DBI:mysql:mogilefs:host=$Mysql_Server;port=3306;mysql_connect_timeout=5
db_user = $MogileFS_DB_User
db_pass = $MogileFS_DB_PassWord
conf_port = 7001
listener_jobs = 5
node_timeout = 5
rebalance_ignore_missing = 1
#===cut====end====#

#添加用户-Trackers无法用root身份运行
useradd mogilefs
su mogilefs
mogilefsd -c /etc/mogilefs/mogilefsd.conf –daemon && exit
#Trackers完成

#添加Storage存储节点
mkdir /data
vim /etc/mogilefs/mogstored.conf
#===cut===begin===#
maxconns = 10000
httplisten = 0.0.0.0:7500
mgmtlisten = 0.0.0.0:7501
docroot = /data
#===cut====end====#

#以root身份运行mogstored
mogstored -c /etc/mogilefs/mogstored.conf –daemon

#通知Trackers有什么host需要添加 有什么Storage需要添加
mogadm host add 192.168.0.203 –ip=192.168.0.203 –port=7500 –status=alive
mogadm host add 192.168.0.114 –ip=192.168.0.114 –port=7500 –status=alive
mogadm host add 192.168.0.97 –ip=192.168.0.97 –port=7500 –status=alive

#Storage节点上依次添加
#/data/目录下文件名以dev开头+数字表示
#–192.168.0.203
mkdir /data/dev1 && chown mogilefs:mogilefs /data/* -R
mogadm device add 192.168.0.203 1 –status={down,alive} #1为唯一值,如果可以的话 尽量别用root权限跑存储节点

#–192.168.0.114
mkdir /data/dev2 && chown mogilefs:mogilefs /data/* -R
mogadm device add 192.168.0.114 2 –status={down,alive} #2为唯一值,如果可以的话 尽量别用root权限跑存储节点

#–192.168.0.97
mkdir /data/dev3 && chown mogilefs:mogilefs /data/* -R
mogadm device add 192.168.0.114 3 –status={down,alive} #3为唯一值,如果可以的话 尽量别用root权限跑存储节点

#===cut===begin===#
[root@Centos ~]# mogadm check
Checking trackers…
127.0.0.1:7001 … OK

Checking hosts…
[ 1] 192.168.0.203 … OK
[ 2] 192.168.0.114 … OK

Checking devices…
host device size(G) used(G) free(G) use% ob state I/O%
—- ———— ———- ———- ———- —— ———- —–
[ 1] dev1 16.421 1.291 15.130 7.86% writeable N/A
[ 2] dev2 16.421 1.293 15.128 7.87% writeable N/A
—- ———— ———- ———- ———- ——
total: 32.843 2.584 30.259 7.87%

[root@Centos ~]# mogadm host list
192.168.0.203 [1]: alive
IP: 192.168.0.203:7500

192.168.0.114 [2]: alive
IP: 192.168.0.114:7500

[root@Centos ~]# mogadm device list
192.168.0.203 [1]: alive
used(G) free(G) total(G) weight(%)
dev1: alive 1.291 15.130 16.421 100

192.168.0.114 [2]: alive
used(G) free(G) total(G) weight(%)
dev2: alive 1.293 15.128 16.421 100
#===cut====end====#

#MogileFS存储由[1]Doamin;[2]Class;[3]Key;这3个条件完成
#
#添加Domain [#添加Domain默认已经生成Default的Calss了?]
mogadm domain add test
#===cut===begin===#
[root@clone ~]# mogadm domain list
domain class mindevcount replpolicy hashtype
——————– ——————– ————- ———— ——-
test default 2 MultipleHosts() NONE
#===cut====end====#

#添加Class
mogadm class add test test_class –mindevcount=2
#===cut===begin===#
[root@Centos ~]# mogadm class list
domain class mindevcount replpolicy hashtype
——————– ——————– ————- ———— ——-
test default 2 MultipleHosts() NONE
test test_class 2 MultipleHosts() NONE
#===cut====end====#

#文件写入MogileFS
#===cut===begin===#
[root@clone ~]# ls -lh
total 257M
-rw-r–r– 1 root root 256M Mar 25 15:19 testfile.big

[root@clone ~]# time mogupload –tracker=192.168.0.97 –domain=test –key=’/testfile.big’ –file=’./testfile.big’

real 0m16.592s
user 0m0.301s
sys 0m5.038s
#===cut====end====#

#查询文件存储位置
[root@clone ~]# mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/testfile.big’
– file: /testfile.big
class: default
devcount: 2
domain: test
fid: 8
key: /testfile.big
length: 268435456
– http://192.168.0.203:7500/dev1/0/000/000/0000000008.fid
– http://192.168.0.114:7500/dev2/0/000/000/0000000008.fid

#删除本地文件,并从MogileFS下载文件回来
[root@clone ~]# rm -f testfile.big
[root@clone ~]# mogfetch –tracker=192.168.0.97 –domain=test –key=’/testfile.big’ –file=’./testfile.big’
[root@clone ~]# ls -lh
total 257M
-rw-r–r– 1 root root 256M Mar 25 17:02 testfile.big

#模拟故障
[root@clone ~]# mogadm device list
192.168.0.203 [1]: alive
used(G) free(G) total(G) weight(%)
dev1: alive 1.556 14.865 16.421 100

192.168.0.114 [2]: alive
used(G) free(G) total(G) weight(%)
dev2: down 1.545 14.876 16.421 100

192.168.0.97 [3]: alive
used(G) free(G) total(G) weight(%)
dev3: alive 1.527 14.894 16.421 100

#标记192.168.0.114的硬盘故障
[root@clone ~]# mogadm device mark 192.168.0.114 2 dead

[root@clone ~]# mogadm device list
192.168.0.203 [1]: alive
used(G) free(G) total(G) weight(%)
dev1: alive 1.556 14.865 16.421 100

192.168.0.114 [2]: alive
used(G) free(G) total(G) weight(%)

192.168.0.97 [3]: alive
used(G) free(G) total(G) weight(%)
dev3: alive 1.777 14.644 16.421 100

#MogileFS会将数据自动复制到192.168.0.97
[root@clone ~]# mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/testfile.big’
– file: /testfile.big
class: default
devcount: 2
domain: test
fid: 8
key: /testfile.big
length: 268435456
– http://192.168.0.203:7500/dev1/0/000/000/0000000008.fid
– http://192.168.0.97:7500/dev3/0/000/000/0000000008.fid

#再增加已修复的device [已经dead的device,ID号具有唯一性,需要另外定一个device ID号]
[root@clone ~]# mogadm device add 192.168.0.114 4 –status=alive

#已标记dead的192.168.0.114的服务器上,新增/data/dev4/
[root@192.168.0.114 ~]# mkdir /data/dev4
[root@192.168.0.114 ~]# chown mogilefs:mogilefs /data/dev4/* -R
[root@192.168.0.114 ~]# mogadm check
Checking trackers…
127.0.0.1:7001 … OK

Checking hosts…
[ 1] 192.168.0.203 … OK
[ 2] 192.168.0.114 … OK
[ 3] 192.168.0.97 … OK

Checking devices…
host device size(G) used(G) free(G) use% ob state I/O%
—- ———— ———- ———- ———- —— ———- —–
[ 1] dev1 16.421 1.556 14.866 9.47% writeable N/A
[ 2] dev4 16.421 1.545 14.876 9.41% writeable N/A
[ 3] dev3 16.421 1.778 14.643 10.83% writeable N/A
—- ———— ———- ———- ———- ——
total: 49.264 4.879 44.385 9.90%

#新增10个文件,测试是否从3台服务器随机写入
[root@clone ~]# touch {1..10}
[root@clone ~]# ls
1 10 2 3 4 5 6 7 8 9 testfile.big
[root@clone ~]# for c in {1..10} ; do echo “mogupload –tracker=192.168.0.97 –domain=test –key=’/$c’ –file=’./$c'”|bash ;done
[root@clone ~]# for f in {1..10};do echo “mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/$f'”|bash;done
– file: /1
class: default
devcount: 2
domain: test
fid: 9
key: /1
length: 0
– http://192.168.0.97:7500/dev3/0/000/000/0000000009.fid
– http://192.168.0.114:7500/dev4/0/000/000/0000000009.fid
– file: /2
class: default
devcount: 2
domain: test
fid: 10
key: /2
length: 0
– http://192.168.0.114:7500/dev4/0/000/000/0000000010.fid
– http://192.168.0.203:7500/dev1/0/000/000/0000000010.fid
– file: /3
class: default
devcount: 2
domain: test
fid: 11
key: /3
length: 0
– http://192.168.0.97:7500/dev3/0/000/000/0000000011.fid
– http://192.168.0.203:7500/dev1/0/000/000/0000000011.fid
– file: /4
class: default
devcount: 2
domain: test
fid: 12
key: /4
length: 0
– http://192.168.0.97:7500/dev3/0/000/000/0000000012.fid
– http://192.168.0.114:7500/dev4/0/000/000/0000000012.fid
– file: /5
class: default
devcount: 2
domain: test
fid: 13
key: /5
length: 0
– http://192.168.0.114:7500/dev4/0/000/000/0000000013.fid
– http://192.168.0.97:7500/dev3/0/000/000/0000000013.fid
– file: /6
class: default
devcount: 2
domain: test
fid: 14
key: /6
length: 0
– http://192.168.0.114:7500/dev4/0/000/000/0000000014.fid
– http://192.168.0.97:7500/dev3/0/000/000/0000000014.fid
– file: /7
class: default
devcount: 2
domain: test
fid: 15
key: /7
length: 0
– http://192.168.0.114:7500/dev4/0/000/000/0000000015.fid
– http://192.168.0.97:7500/dev3/0/000/000/0000000015.fid
– file: /8
class: default
devcount: 2
domain: test
fid: 16
key: /8
length: 0
– http://192.168.0.203:7500/dev1/0/000/000/0000000016.fid
– http://192.168.0.114:7500/dev4/0/000/000/0000000016.fid
– file: /9
class: default
devcount: 2
domain: test
fid: 17
key: /9
length: 0
– http://192.168.0.114:7500/dev4/0/000/000/0000000017.fid
– http://192.168.0.203:7500/dev1/0/000/000/0000000017.fid
– file: /10
class: default
devcount: 2
domain: test
fid: 18
key: /10
length: 0
– http://192.168.0.114:7500/dev4/0/000/000/0000000018.fid
– http://192.168.0.97:7500/dev3/0/000/000/0000000018.fid

#连续写入MogileFS 十万个空文件
[root@clone ~]# time for c in {1..99999} ; do echo “mogupload –tracker=192.168.0.97 –domain=test –key=’/$c’ –file=’./$c'”|bash ;done

real 185m14.814s
user 70m31.902s
sys 23m1.011s

#连续写入MogileFS 十个大文件,单个文件大小256M
[root@clone ~]# time for t in $(ls -1 *);do echo “mogupload –trackers=192.168.0.97 –domain=test –key=’/$t’ –file=’./$t'”|bash;done

real 8m52.081s
user 0m4.389s
sys 0m31.443s

[root@clone ~]# for t in $(ls -1 *);do echo “mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/$t'”|bash;done|more
– file: /testfile.big
class: default
devcount: 2
domain: test
fid: 100018
key: /testfile.big
length: 268435456
– http://192.168.0.203:7500/dev1/0/000/100/0000100018.fid
– http://192.168.0.114:7500/dev4/0/000/100/0000100018.fid
– file: /testfile.big.1
class: default
devcount: 2
domain: test
fid: 100019
key: /testfile.big.1
length: 268435456
– http://192.168.0.203:7500/dev1/0/000/100/0000100019.fid
– http://192.168.0.114:7500/dev4/0/000/100/0000100019.fid
– file: /testfile.big.10
class: default
devcount: 2
domain: test
fid: 100020
key: /testfile.big.10
length: 268435456
– http://192.168.0.114:7500/dev4/0/000/100/0000100020.fid
– http://192.168.0.97:7500/dev3/0/000/100/0000100020.fid
– file: /testfile.big.2
class: default
devcount: 2
domain: test
fid: 100021
key: /testfile.big.2
length: 268435456
– http://192.168.0.114:7500/dev4/0/000/100/0000100021.fid
– http://192.168.0.203:7500/dev1/0/000/100/0000100021.fid
– file: /testfile.big.3
class: default
devcount: 2
domain: test
fid: 100022
key: /testfile.big.3
length: 268435456
– http://192.168.0.203:7500/dev1/0/000/100/0000100022.fid
– http://192.168.0.97:7500/dev3/0/000/100/0000100022.fid
– file: /testfile.big.4
class: default
devcount: 2
domain: test
fid: 100023
key: /testfile.big.4
length: 268435456
– http://192.168.0.97:7500/dev3/0/000/100/0000100023.fid
– http://192.168.0.114:7500/dev4/0/000/100/0000100023.fid
– file: /testfile.big.5
class: default
devcount: 2
domain: test
fid: 100024
key: /testfile.big.5
length: 268435456
– http://192.168.0.114:7500/dev4/0/000/100/0000100024.fid
– http://192.168.0.203:7500/dev1/0/000/100/0000100024.fid
– file: /testfile.big.6
class: default
devcount: 2
domain: test
fid: 100025
key: /testfile.big.6
length: 268435456
– http://192.168.0.114:7500/dev4/0/000/100/0000100025.fid
– http://192.168.0.203:7500/dev1/0/000/100/0000100025.fid
– file: /testfile.big.7
class: default
devcount: 2
domain: test
fid: 100026
key: /testfile.big.7
length: 268435456
– http://192.168.0.203:7500/dev1/0/000/100/0000100026.fid
– http://192.168.0.97:7500/dev3/0/000/100/0000100026.fid
– file: /testfile.big.8
class: default
devcount: 2
domain: test
fid: 100027
key: /testfile.big.8
length: 268435456
– http://192.168.0.114:7500/dev4/0/000/100/0000100027.fid
– http://192.168.0.203:7500/dev1/0/000/100/0000100027.fid
– file: /testfile.big.9
class: default
devcount: 2
domain: test
fid: 100028
key: /testfile.big.9
length: 268435456
– http://192.168.0.114:7500/dev4/0/000/100/0000100028.fid
– http://192.168.0.97:7500/dev3/0/000/100/0000100028.fid

#单个大文件写入,文件大小为2G
[root@clone ~]# dd if=/dev/zero of=/root/local.iso.2g bs=1M count=2048
[root@clone ~]# mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/local.iso.2g’
– file: /local.iso.2g
class: test_class
devcount: 1
domain: test
fid: 100033
key: /local.iso.2g
length: 2147483648
– http://192.168.0.114:7500/dev4/0/000/100/0000100033.fid #{同步会有时间差,先写入本地,trackers再传入其他可用的机器}
[root@clone ~]# mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/local.iso’
– file: /local.iso
class: test_class
devcount: 2
domain: test
fid: 100032
key: /local.iso
length: 1073741824
– http://192.168.0.97:7500/dev3/0/000/100/0000100032.fid #{大文件似乎优先写入本地storage?}
– http://192.168.0.114:7500/dev4/0/000/100/0000100032.fid

#如果大文件优先写入本地,模拟本地硬盘已满的情况下,MogileFS单个大文件写入过程
[root@clone ~]# dd if=/dev/zero of=/root/local.iso.2g bs=1M count=10240
[root@clone ~]# df -h #{Mogilefs的chroot已满}
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/VolGroup00-LogVol00
18G 15G 1.9G 89% /
/dev/hdd1 99M 13M 82M 13% /boot
tmpfs 252M 0 252M 0% /dev/shm

#写入
[root@clone ~]# mogupload –trackers=192.168.0.97 –domain=test –class=test_class –key=’/local.iso.2g’ –file=’./local.iso.2g’

#通过抓包发现MogileFS在本地chroot已满的情况下,随机向其中一台可用的机器写入数据,
#写入状态未完成
[root@clone ~]# mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/local.iso.2g’
– file: /local.iso.2g
class: test_class
devcount: 2
domain: test
fid: 100033
key: /local.iso.2g
length: 2147483648
– http://192.168.0.203:7500/dev1/0/000/100/0000100033.fid
– http://192.168.0.114:7500/dev4/0/000/100/0000100033.fid

#写入状态完成
[root@clone ~]# date
Tue Mar 26 11:49:34 CST 2013
[root@clone ~]# mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/local.iso.2g’
– file: /local.iso.2g
class: test_class
devcount: 1
domain: test
fid: 100034
key: /local.iso.2g
length: 10737418240
– http://192.168.0.203:7500/dev1/0/000/100/0000100034.fid #{此时,另外两台在做大量的数据交互,各自的网卡都跑满了}
[root@clone ~]# date
Tue Mar 26 11:49:34 CST 2013
[root@clone ~]# mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/local.iso.2g’
– file: /local.iso.2g
class: test_class
devcount: 1
domain: test
fid: 100034
key: /local.iso.2g
length: 10737418240
– http://192.168.0.203:7500/dev1/0/000/100/0000100034.fid

[root@clone ~]# date
Tue Mar 26 12:06:29 CST 2013
[root@clone ~]# mogfileinfo –trackers=192.168.0.97 –domain=test –key=’/local.iso.2g’
– file: /local.iso.2g
class: test_class
devcount: 2
domain: test
fid: 100034
key: /local.iso.2g
length: 10737418240
– http://192.168.0.203:7500/dev1/0/000/100/0000100034.fid
– http://192.168.0.114:7500/dev4/0/000/100/0000100034.fid #{storage node已经同步}

#模拟极端情况下,storage无损坏,单台free spaces已经没了,但是storage node还剩下4.5G
[root@Centos ~]# mogadm device summary
Hostname HostID Status used(G) free(G) total(G) %Used
192.168.0.203 [ 1]: alive 15.362 1.059 16.421 93.55
192.168.0.114 [ 2]: alive 14.812 1.608 16.421 90.21
192.168.0.97 [ 3]: alive 14.547 1.874 16.421 88.59
[root@Centos ~]# mogadm check
Checking trackers…
127.0.0.1:7001 … OK

Checking hosts…
[ 1] 192.168.0.203 … OK
[ 2] 192.168.0.114 … OK
[ 3] 192.168.0.97 … OK

Checking devices…
host device size(G) used(G) free(G) use% ob state I/O%
—- ———— ———- ———- ———- —— ———- —–
[ 1] dev1 16.421 15.363 1.058 93.55% writeable N/A
[ 2] dev4 16.421 14.813 1.608 90.21% writeable N/A
[ 3] dev3 16.421 14.547 1.874 88.59% writeable N/A
—- ———— ———- ———- ———- ——
total: 49.264 44.723 4.541 90.78%
#上传2G的文件,测试是否能成功写入
[root@Centos ~]# dd if=/dev/zero of=/root/iso.2g bs=1M count=2048
2048+0 records in
2048+0 records out
2147483648 bytes (2.1 GB) copied, 42.5198 seconds, 50.5 MB/s
[root@Centos ~]# mogupload –trackers=192.168.0.203 –domain=test –class=test_class –key=’/iso.2g’ –file=’./iso.2g’
[root@Centos ~]# echo $?
0
[root@Centos ~]# mogfileinfo –trackers=192.168.0.203 –domain=test –key=’/iso.2g’
– file: /iso.2g
class: test_class
devcount: 2
domain: test
fid: 100035
key: /iso.2g
length: 2147483648
– http://192.168.0.114:7500/dev4/0/000/100/0000100035.fid
– http://192.168.0.203:7500/dev1/0/000/100/0000100035.fid

#有写入,但源文件删除后再下载,两个文件的md5不同[待确认]
[root@kvm ~]# mogadm device summary
Hostname HostID Status used(G) free(G) total(G) %Used
192.168.0.203 [ 1]: alive 17.310 0.000 17.310 100.00
192.168.0.114 [ 2]: alive 16.814 0.000 16.814 100.00
192.168.0.97 [ 3]: alive 14.547 1.874 16.421 88.59
[root@kvm ~]# mogadm check
Checking trackers…
127.0.0.1:7001 … OK

Checking hosts…
[ 1] 192.168.0.203 … OK
[ 2] 192.168.0.114 … OK
[ 3] 192.168.0.97 … OK

Checking devices…
host device size(G) used(G) free(G) use% ob state I/O%
—- ———— ———- ———- ———- —— ———- —–
[ 1] dev1 MISSING FIELD (available) FROM USAGE FILE: http://192.168.0.203:7500/dev1/usage
[ 2] dev4 MISSING FIELD (available) FROM USAGE FILE: http://192.168.0.114:7500/dev4/usage
[ 3] dev3 16.421 14.547 1.874 88.59% writeable N/A
—- ———— ———- ———- ———- ——
total: 16.421 14.547 1.874 88.59%

#mount挂载

#改变mindevcount个数

#测试多点失败