nginx代理mogilefs集群实现
一、实验拓扑
二、实验环境
三、实验步骤
1.节点部署
192.168.0.3 node1 [Nginx,Tracker,Storage,Mariadb]
192.168.0.4 node2 [Tracker,Storage]
192.168.0.5 node3 [Tracker,Storage]
2.初始化工作
配置好三台服务器IP，hosts文件等网络环境，时间同步以及以下的rpm包安装
注：以下操作在三个节点都执行
#下载rpm包（附件有提供）
MogileFS-Server-2.46-2.el7.centos.noarch.rpm
MogileFS-Server-mogilefsd-2.46-2.el7.centos.noarch.rpm
MogileFS-Server-mogstored-2.46-2.el7.centos.noarch.rpm
MogileFS-Utils-2.19-1.el7.centos.noarch.rpm
perl-Danga-Socket-1.61-1.el6.rf.noarch.rpm
perl-MogileFS-Client-1.14-1.el7.centos.noarch.rpm
perl-Perlbal-1.78-1.el6.noarch.rpm
#安装依赖环境
yum install perl-Net-Netmask perl-IO-stringy perl-Sys-Syslog perl-IO-AIO -y
#安装rpm包
yum install *.rpm -y
3.node1节点部署mariadb，mogilefsd，mogstored
3.1.node1部署Mariadb
yum install mariadb-server -y
vim /etc/my.cnf.d/server.cnf
[mysqld]
skip_name_resolve=1
innodb_file_per_table = 1
#启动mariadb
systemctl start mariadb
#连接mysql，进行以下操作
mysql
GRANT ALL ON *.* TO 'root'@'192.168.0.%' IDENTIFIED BY 'rootpass';
GRANT ALL ON mogilefs.* TO 'moguser'@'192.168.0.%' IDENTIFIED BY 'moguserpass';
CREATE DATABASE mogilefs;
FLUSH PRIVILEGES;
quit
#设定mogilefs数据库，生成数据表
mogdbsetup --dbhost=192.168.0.3 --dbuser=moguser --dbpass=moguserpass --dbname=mogilefs --dbrootpass=rootpass
#连接mysql，查看数据库是否生成
mysql -u root -h192.168.0.3 -prootpass
MariaDB [(none)]> use mogilefs;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
MariaDB [mogilefs]> SHOW TABLES;
+----------------------+
| Tables_in_mogilefs |
+----------------------+
| checksum |
| class |
| device |
| domain |
| file |
| file_on |
| file_on_corrupt |
| file_to_delete |
| file_to_delete2 |
| file_to_delete_later |
| file_to_queue |
| file_to_replicate |
| fsck_log |
| host |
| server_settings |
| tempfile |
| unreachable_fids |
+----------------------+
17 rows in set (0.00 sec)
3.2node1配置mogilefsd
vim /etc/mogilefs/mogilefsd.conf
db_dsn = DBI:mysql:mogilefs:host=192.168.0.3
db_user = moguser
db_pass = moguserpass
listen = 0.0.0.0:7001
3.3启动mogilefsd
[root@master ~]# service mogilefsd start
Starting mogilefsd (via systemctl): [ OK ]
[root@master ~]# ss -tln
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 50 *:3306 *:*
LISTEN 0 128 *:22 *:*
LISTEN 0 128 *:7001 *:*
LISTEN 0 100 127.0.0.1:25 *:*
LISTEN 0 128 :::22 :::*
LISTEN 0 100 ::1:25 :::*
3.4配置storage（推荐所有节点都配置）
#创建存放文件的目录
mkdir /data/mogilefs/dev1 -pv
mkdir: created directory ‘/data’
mkdir: created directory ‘/data/mogilefs’
mkdir: created directory ‘/data/mogilefs/dev1’
vim /etc/mogilefs/mogstored.conf
maxconns = 10000
httplisten = 0.0.0.0:7500
mgmtlisten = 0.0.0.0:7501
docroot = /data/mogilefs/
#修改目录的权限
chown mogilefs.mogilefs /data/mogilefs/ -R
#启动mogstored
[root@master ~]# service mogstored start
Starting mogstored (via systemctl): [ OK ]
4.node2、node3节点部署mogstored操作
4.1node2操作
[root@slave ~]# mkdir /data/mogilefs/dev2 -pv
mkdir: created directory ‘/data’
mkdir: created directory ‘/data/mogilefs’
mkdir: created directory ‘/data/mogilefs/dev2’
vim /etc/mogilefs/mogstored.conf
maxconns = 10000
httplisten = 0.0.0.0:7500
mgmtlisten = 0.0.0.0:7501
docroot = /data/mogilefs/
#修改目录的权限
chown mogilefs.mogilefs /data/mogilefs/ -R
#启动mogstored
[root@slave ~]# service mogstored start
Starting mogstored (via systemctl): [ OK ]
4.2node3操作
[root@localhost ~]# mkdir /data/mogilefs/dev3 -pv
mkdir: created directory ‘/data’
mkdir: created directory ‘/data/mogilefs’
mkdir: created directory ‘/data/mogilefs/dev3’
vim /etc/mogilefs/mogstored.conf
maxconns = 10000
httplisten = 0.0.0.0:7500
mgmtlisten = 0.0.0.0:7501
docroot = /data/mogilefs
#修改目录的权限
chown mogilefs.mogilefs /data/mogilefs/ -R
#启动mogstored
[root@localhost ~]# service mogstored start
Starting mogstored (via systemctl): [ OK ]
5.node1配置操作
5.1.配置mogilefsd
#添加主机
mogadm --trackers=192.168.0.3:7001 host add node1 --ip=192.168.0.3 --status=alive
mogadm --trackers=192.168.0.3:7001 host add node2 --ip=192.168.0.4 --status=alive
mogadm --trackers=192.168.0.3:7001 host add node3 --ip=192.168.0.5 --status=alive
#查看主机状态
[root@master ~]# mogadm host list
node1 [1]: alive
IP: 192.168.0.3:7500
node2 [2]: alive
IP: 192.168.0.4:7500
node3 [3]: alive
IP: 192.168.0.5:7500
#查看设备（设备为空）
mogadm device list
node1 [1]: alive
used(G) free(G) total(G) weight(%)
node2 [2]: alive
used(G) free(G) total(G) weight(%)
node3 [3]: alive
used(G) free(G) total(G) weight(%)
#添加设备
mogadm device add node1 1
mogadm device add node2 2
mogadm device add node3 3
#再次查看设备
[root@master ~]# mogadm device list
node1 [1]: alive
used(G) free(G) total(G) weight(%)
dev1: alive 1.389 8.602 9.990 100
node2 [2]: alive
used(G) free(G) total(G) weight(%)
dev2: alive 1.254 8.736 9.990 100
node3 [3]: alive
used(G) free(G) total(G) weight(%)
dev3: alive 1.150 8.840 9.990 100
#查看设备的状态
[root@master ~]# mogadm check
Checking trackers...
127.0.0.1:7001 ... OK
Checking hosts...
[ 1] node1 ... OK
[ 2] node2 ... OK
[ 3] node3 ... OK
Checking devices...
host device size(G) used(G) free(G) use% ob state I/O%
---- ------------ ---------- ---------- ---------- ------ ---------- -----
[ 1] dev1 9.990 1.389 8.601 13.91% writeable N/A
[ 2] dev2 9.990 1.254 8.736 12.56% writeable N/A
[ 3] dev3 9.990 1.151 8.840 11.52% writeable N/A
---- ------------ ---------- ---------- ---------- ------
total: 29.971 3.795 26.176 12.66%
#查看域（目前为空）
[root@master ~]# mogadm domain list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
#创建域
[root@master ~]# mogadm domain add p_w_picpaths
[root@master ~]# mogadm domain add files
[root@master ~]# mogadm domain list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
p_w_picpaths default 2 MultipleHosts() NONE
#查看类
[root@master ~]# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
p_w_picpaths default 2 MultipleHosts() NONE
#在p_w_picpaths域中添加一个类jpeg
[root@master ~]# mogadm class add p_w_picpaths jpeg --mindevcount=2 --replpolicy="MultipleHosts(2)"
#再次查看类
[root@master ~]# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
p_w_picpaths default 2 MultipleHosts() NONE
p_w_picpaths jpeg 2 MultipleHosts(2) NONE
6.上传文件/图片测试
#在linu主机中找几张图片
find / -name "*.jpg" -exec cp {} /tmp/ \;
#上传图片（图片格式写错了，囧）
mogupload --tracker=192.168.0.3:7001 --domain=p_w_picpaths --key='1.gpg' --file='/tmp/morning.jpg' --class=jpeg
#上传图片
mogupload --tracker=192.168.0.3:7001 --domain=p_w_picpaths --key='1.jpg' --file='/tmp/day.jpg' --class=jpeg
#查看图片的信息
[root@master tmp]# mogfileinfo --tracker=192.168.0.3:7001 --domain=p_w_picpaths --key='1.jpg'
- file: 1.jpg
class: jpeg
devcount: 1
domain: p_w_picpaths
fid: 3
key: 1.jpg
length: 961243
- http://192.168.0.3:7500/dev1/0/000/000/0000000003.fid
[root@master tmp]# mogfileinfo --tracker=192.168.0.3:7001 --domain=p_w_picpaths --key='1.gpg'
- file: 1.gpg
class: jpeg
devcount: 1
domain: p_w_picpaths
fid: 2
key: 1.gpg
length: 980265
-
#上传文件
mogupload --trackers=192.168.0.3 --domain=files --key='/fstab.txt' --file=/etc/fstab
#查看文件
[root@master tmp]# mogfileinfo --trackers=192.168.0.3 --domain=files --key='/fstab.txt'
- file: /fstab.txt
class: default
devcount: 1
domain: files
fid: 4
key: /fstab.txt
length: 595
-
#访问测试(在浏览器中输入)
http://192.168.0.3:7500/dev1/0/000/000/0000000003.fid
http://192.168.0.5:7500/dev3/0/000/000/0000000002.fid 
(ps：可以看出，访问的地址实在反人类，其实我们可以通过加入其他的模块解决这种办法，在下面会介绍这个模块）
#删除文件
mogdelete --tracker=192.168.0.3:7001 --domain=p_w_picpaths --key='1.jpg'
mogdelete --tracker=192.168.0.3:7001 --domain=p_w_picpaths --key='1.gpg'
mogdelete --tracker=192.168.0.3:7001 --domain=files --key='/fstab.txt'
7.配置nginx
nginx 的mogilefs模块的官方文档地址：http://www.grid.net.ru/nginx/mogilefs.en.html
#下载nginx_mogilefs模块
wget http://www.grid.net.ru/nginx/download/nginx_mogilefs_module-1.0.4.tar.gz
tar xvf nginx_mogilefs_module-1.0.4.tar.gz
mv nginx_mogilefs_module-1.0.4 /usr/local/ cd /usr/local/src
wget
#安装nginx依赖环境包
yum install -y gcc pcre pcre-devel zlib zlib-devel openssl openssl-devel
useradd -r nginx
tar xvf nginx-1.10.3.tar.gz
cd nginx-1.10.3
#编译安装
./configure --with-pcre --user=nginx --group=nginx --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module --with-http_realip_module --add-module=/usr/local/nginx_mogilefs_module-1.0.4
make && make install
#编译之后出现报错
make[1]: *** [objs/addon/nginx_mogilefs_module-1.0.4/ngx_http_mogilefs_module.o] Error 1
make[1]: Leaving directory `/usr/local/src/nginx-1.10.3'
make: *** [build] Error 2
解决办法:
vim objs/Makefile
#把-Werror删除掉
#再次安装，成功
make && make install
#配置nginx环境变量
vi /etc/profile.d/nginx.sh
export PATH="$PATH:/usr/local/nginx/sbin"
#重新加载
. /etc/profile.d/nginx.sh
#查看环境变量
[root@master ~]# echo $PATH
/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/nginx/sbin
#接下来，编辑nginx配置文件
cd /usr/local/nginx/conf/
vi nginx.conf
#加入的配置段
upstream trackers {
server 192.168.0.3:7001 weight=1;
server 192.168.0.4:7001 weight=1;
server 192.168.0.5:7001 weight=1;
check interval=1000 rise=2 fall=5 timeout=1000;
}
location /p_w_picpaths/ {
mogilefs_tracker trackers;
mogilefs_domain p_w_picpaths;
mogilefs_methods GET;
mogilefs_noverify on;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-type;
proxy_buffering off;
}
}
#检查并启动nginx
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
[root@master conf]# nginx
#上传图片
mogupload --trackers=192.168.0.3 --domain=p_w_picpaths --key='1.jpg' --file='/tmp/day.jpg' --class=jpeg
#浏览器访问测试
http://192.168.0.3/p_w_picpaths/1.jpg
8.将三个节点都配置成tracker #将mogilefsd.conf文件拷贝到其他的节点 [root@master conf]# scp /etc/mogilefs/mogilefsd.conf node2:/etc/mogilefs/ mogilefsd.conf 100% 1465 1.4KB/s 00:00 [root@master conf]# scp /etc/mogilefs/mogilefsd.conf node3:/etc/mogilefs/ root@node3's password: mogilefsd.conf 100% 1465 1.4KB/s 00:00 #启动node2、node3的mogilefsd [root@slave ~]# service mogilefsd start Starting mogilefsd (via systemctl): [ OK ] [root@localhost ~]# service mogilefsd start Starting mogilefsd (via systemctl): [ OK ] #测试是否成为tracker [root@master conf]# moglistkeys --trackers=192.168.0.3:7001 --domain=p_w_picpaths 1.jpg [root@master conf]# moglistkeys --trackers=192.168.0.4:7001 --domain=p_w_picpaths 1.jpg [root@master conf]# moglistkeys --trackers=192.168.0.5:7001 --domain=p_w_picpaths 1.jpg 三个节点都可以查看内容列表，由此证明三个节点都是tracker。
