7-Kylin麒麟系统MongoDB分片集群3.0部署_mongodb

一、部署

1.1 环境准备

IP

角色

系统

192.168.1.81

Mongos:27017、configserver:27020、shard1:27021、shard2:27022

Kylin V10 x86

192.168.1.82

Mongos:27017、configserver:27020、shard1:27021、shard2:27022

Kylin V10 x86

192.168.1.83

Mongos:27017、configserver:27020、shard1:27021、shard2:27022

Kylin V10 x86

三台机器同样操作

[root@mongo81 data]# tar xf mongodb-linux-x86_64-rhel62-3.0.12.tar.gz 
[root@mongo81 data]# mv mongodb-linux-x86_64-rhel62-3.0.12 mongodb
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/mongos/conf
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/mongos/log
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/configsvr/db
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/configsvr/conf
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/configsvr/log
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/shard1/db
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/shard1/conf
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/shard1/log
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/shard2/db
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/shard2/conf
[root@mongo81 data]# mkdir -p /data/cluster-mongodb/shard2/log

1.2 分片集群搭建步骤

分片集群各部分组件搭建顺序（程序启动顺序也是如此）：

  1. “config server” -> 2. “shard” -> 3. “mongos”

1.2.1 安装前配置

三台机器同样操作

#修改系统参数

[root@mongo81 data]# sudo vi /etc/security/limits.conf

*    soft   nproc    65536
*    hard   nproc    65536
*    soft   nofile   100000
*    hard   nofile   100000
*    soft   memlock  2048
*    hard   memlock  2048
#关闭透明大页
[root@mongo81 data]# echo 'never' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
[root@mongo81 data]# sudo vi /etc/systemd/system/disable-thp.service
[Unit] 
Description=Disable Transparent Huge Pages (THP)

[Service] 
Type=simple 
ExecStart=/bin/sh -c "echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled && echo never > /sys/kernel/mm/transparent_hugepage/defrag"

[Install] 
WantedBy=multi-user.target

[root@mongo81 data]# sudo yum install numactl -y
[root@mongo81 data]# sudo systemctl daemon-reload
[root@mongo81 data]# sudo systemctl start disable-thp
[root@mongo81 data]# sudo systemctl enable disable-thp

[root@mongo81 data]# echo "PATH=$PATH:/data/mongodb/bin/" >> /etc/profile
[root@mdb1 data]# source /etc/profile

1.2.2 配置shard1配置文件

192.168.1.81:27021

[root@mongo81 data]# vim /data/cluster-mongodb/shard1/conf/mongod-shard1.conf
##shard
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/shard1/log/shard1.log # 日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/shard1/db # 数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
  # pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile, default no pid file

# network interfaces
net:
  port: 27021  # 端口，分片内的端口相同
  bindIp: 192.168.1.81  # 对外暴露的ip，定义成192.168.1.81以后不能用localhost访问 # Listen to local interface only, comment to listen on all interfaces.

replication:
  # oplogSizeMB: 10000  # Once the mongod has created the oplog, changing this option will not affect the size of the oplog.
  replSetName: shard1 # 分片名，分片内必须一致

sharding:
  clusterRole: shardsvr  # 同样是mongod启动，shard和configsvr的区别

#security:
#  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

192.168.1.82:27021

[root@mongo82 data]# vim /data/cluster-mongodb/shard1/conf/mongod-shard1.conf
##shard
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/shard1/log/shard1.log # 日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/shard1/db # 数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
  # pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile, default no pid file

# network interfaces
net:
  port: 27021  # 端口，分片内的端口相同
  bindIp: 192.168.1.82  # 对外暴露的ip，定义成192.168.1.81以后不能用localhost访问 # Listen to local interface only, comment to listen on all interfaces.

replication:
  # oplogSizeMB: 10000  # Once the mongod has created the oplog, changing this option will not affect the size of the oplog.
  replSetName: shard1 # 分片名，分片内必须一致

sharding:
  clusterRole: shardsvr  # 同样是mongod启动，shard和configsvr的区别

#security:
#  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

192.168.1.83:27021

[root@mongo83 data]# vim /data/cluster-mongodb/shard1/conf/mongod-shard1.conf
##shard
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/shard1/log/shard1.log # 日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/shard1/db # 数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
  # pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile, default no pid file

# network interfaces
net:
  port: 27021  # 端口，分片内的端口相同
  bindIp: 192.168.1.83  # 对外暴露的ip，定义成192.168.1.83以后不能用localhost访问 # Listen to local interface only, comment to listen on all interfaces.

replication:
  # oplogSizeMB: 10000  # Once the mongod has created the oplog, changing this option will not affect the size of the oplog.
  replSetName: shard1 # 分片名，分片内必须一致

sharding:
  clusterRole: shardsvr  # 同样是mongod启动，shard和configsvr的区别

#security:
#  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

启动shard1

[root@mdb1 bin]# numactl --interleave=all /data/mongodb/bin/mongod -f /data/cluster-mongodb/shard1/conf/mongod-shard1.conf

1.2.3 配置shard2

192.168.1.81:27022

[root@mongo81 data]# vim /data/cluster-mongodb/shard2/conf/mongod-shard2.conf
##shard
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/shard2/log/shard2.log # 日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/shard2/db # 数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
  # pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile, default no pid file

# network interfaces
net:
  port: 27022  # 端口，分片内的端口相同
  bindIp: 192.168.1.81  # 对外暴露的ip，定义成192.168.1.81以后不能用localhost访问 # Listen to local interface only, comment to listen on all interfaces.

replication:
  # oplogSizeMB: 10000  # Once the mongod has created the oplog, changing this option will not affect the size of the oplog.
  replSetName: shard2 # 分片名，分片内必须一致

sharding:
  clusterRole: shardsvr  # 同样是mongod启动，shard和configsvr的区别

#security:
#  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

192.168.1.82:27022

[root@mongo82 data]# vim /data/cluster-mongodb/shard2/conf/mongod-shard2.conf
##shard
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/shard2/log/shard2.log # 日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/shard2/db # 数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
  # pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile, default no pid file

# network interfaces
net:
  port: 27022  # 端口，分片内的端口相同
  bindIp: 192.168.1.82  # 对外暴露的ip，定义成192.168.1.82以后不能用localhost访问 # Listen to local interface only, comment to listen on all interfaces.

replication:
  # oplogSizeMB: 10000  # Once the mongod has created the oplog, changing this option will not affect the size of the oplog.
  replSetName: shard2 # 分片名，分片内必须一致

sharding:
  clusterRole: shardsvr  # 同样是mongod启动，shard和configsvr的区别

#security:
#  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

192.168.1.83:27022

[root@mongo83 data]# vim /data/cluster-mongodb/shard2/conf/mongod-shard2.conf
##shard
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/shard2/log/shard2.log # 日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/shard2/db # 数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
  # pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile, default no pid file

# network interfaces
net:
  port: 27022  # 端口，分片内的端口相同
  bindIp: 192.168.1.83  # 对外暴露的ip，定义成192.168.1.83以后不能用localhost访问 # Listen to local interface only, comment to listen on all interfaces.

replication:
  # oplogSizeMB: 10000  # Once the mongod has created the oplog, changing this option will not affect the size of the oplog.
  replSetName: shard2 # 分片名，分片内必须一致

sharding:
  clusterRole: shardsvr  # 同样是mongod启动，shard和configsvr的区别

#security:
#  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

启动shard2

三台机器执行同样操作

[root@mongo83 bin]# numactl --interleave=all /data/mongodb/bin/mongod -f /data/cluster-mongodb/shard2/conf/mongod-shard2.conf

1.2.4 配置config server 配置文件

192.168.1.81:27020

[root@mongo81 data]# vim /data/cluster-mongodb/configsvr/conf/configsvr.conf
##configsvr
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/configsvr/log/config.log #日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/configsvr/db #数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
#  pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile,default no pid file

# network interfaces
net:
  port: 27020 #端口
  bindIp: 192.168.1.81 #对外暴露的ip # Listen to local interface only, comment to listen on all interfaces.

sharding:
  clusterRole: configsvr  #同样是mongod启动，configsvr和shard的区别


#security:
#  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

192.168.1.82:27020

[root@mongo81 data]# vim /data/cluster-mongodb/configsvr/conf/configsvr.conf
##configsvr
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/configsvr/log/config.log #日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/configsvr/db #数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
#  pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile,default no pid file

# network interfaces
net:
  port: 27020 #端口
  bindIp: 192.168.1.82 #对外暴露的ip # Listen to local interface only, comment to listen on all interfaces.

sharding:
  clusterRole: configsvr  #同样是mongod启动，configsvr和shard的区别


#security:
  #keyFile: /data/mongodb/key/mongodb-keyfile

192.168.1.83:27020

[root@mongo81 data]# vim /data/cluster-mongodb/configsvr/conf/configsvr.conf
##configsvr
# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /data/cluster-mongodb/configsvr/log/config.log #日志文件的位置

# Where and how to store data.
storage:
  dbPath: /data/cluster-mongodb/configsvr/db #数据目录的位置
  journal:
    enabled: true

# how the process runs
processManagement:
  fork: true  # fork and run in background
#  pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile,default no pid file

# network interfaces
net:
  port: 27020 #端口
  bindIp: 192.168.1.83 #对外暴露的ip # Listen to local interface only, comment to listen on all interfaces.

sharding:
  clusterRole: configsvr  #同样是mongod启动，configsvr和shard的区别


#security:
#  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

启动configserver

[root@mongo83 bin]# numactl --interleave=all /data/mongodb/bin/mongod -f /data/cluster-mongodb/configsvr/conf/configsvr.conf

1.2.5 配置mongos配置文件

192.168.1.81:27017

[root@mongo81 data]# sudo vi /data/cluster-mongodb/mongos/conf/mongos.conf
##mongos
#where to write logging data.
systemLog:
  destination: file
  path: /data/cluster-mongodb/mongos/log/mongos.log #日志文件的位置
  logAppend: true
  logRotate: reopen

# how the process runs
processManagement:
  fork: true  # fork and run in background
  #  pidFilePath: /data/cluster-mongodb/mongos/log/mongos.pid  # location of pidfile, default no pid file

net:
  port: 27017 #端口
  bindIp: 192.168.1.81 #对外暴露的ip # Listen to local interface only, comment to listen on all interfaces.

sharding:
  configDB: 192.168.1.81:27020,192.168.1.82:27020,192.168.1.83:27020 #mongos启动时，使用configsvr集群存取路由信息


#security:
  #keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

192.168.1.82:27017

[root@mongo81 data]# sudo vi /data/cluster-mongodb/mongos/conf/mongos.conf
##mongos
#where to write logging data.
systemLog:
  destination: file
  path: /data/cluster-mongodb/mongos/log/mongos.log #日志文件的位置
  logAppend: true
  logRotate: reopen

# how the process runs
processManagement:
  fork: true  # fork and run in background
  #  pidFilePath: /data/cluster-mongodb/mongos/log/mongos.pid  # location of pidfile, default no pid file

net:
  port: 27017 #端口
  bindIp: 192.168.1.82 #对外暴露的ip # Listen to local interface only, comment to listen on all interfaces.

sharding:
  configDB: 192.168.1.81:27020,192.168.1.82:27020,192.168.1.83:27020 #mongos启动时，使用configsvr集群存取路由信息


#security:
  #keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

192.168.1.83:27017

[root@mongo81 data]# sudo vi /data/cluster-mongodb/mongos/conf/mongos.conf
##mongos
#where to write logging data.
systemLog:
  destination: file
  path: /data/cluster-mongodb/mongos/log/mongos.log #日志文件的位置
  logAppend: true
  logRotate: reopen

# how the process runs
processManagement:
  fork: true  # fork and run in background
  #  pidFilePath: /data/cluster-mongodb/mongos/log/mongos.pid  # location of pidfile, default no pid file

net:
  port: 27017 #端口
  bindIp: 192.168.1.83 #对外暴露的ip # Listen to local interface only, comment to listen on all interfaces.

sharding:
  configDB: 192.168.1.81:27020,192.168.1.82:27020,192.168.1.83:27020 #mongos启动时，使用configsvr集群存取路由信息


#security:
  #keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

启动mongos

三台机器执行同样操作

[root@mdb1 bin]# numactl --interleave=all /data/mongodb/bin/mongos -f /data/cluster-mongodb/mongos/conf/mongos.conf
mongos -f /data/cluster-mongodb/mongos/conf/mongos.conf

1.1.6 构建分片集群

#shard1创建
mongo 192.168.1.81:27021
#切换到shard1 登录

use admin
config ={_id:'shard1',members:[
			{_id: 0, host:'192.168.1.81:27021'},
			{_id: 1, host:'192.168.1.82:27021'},
			{_id: 2, host:'192.168.1.83:27021'}
]}

rs.initiate(config)

#shard2创建
mongo 192.168.1.81:27022
#切换到shard2 登录

use admin
config ={_id:'shard2',members:[
			{_id: 0, host:'192.168.1.81:27022'},
			{_id: 1, host:'192.168.1.82:27022'},
			{_id: 2, host:'192.168.1.83:27022'}
]}

rs.initiate(config)

#configserver 3.0的mongo不需要对configserver进行配置 启动就行了


#mongos创建
mongo 192.168.1.81:27017
#切换到mongos登录

use admin
#添加分片
db.runCommand({addshard:"shard1/192.168.1.81:27021,192.168.1.82:27021,192.168.1.83:27021",name:"shard1"})
db.runCommand({addshard:"shard2/192.168.1.81:27022,192.168.1.82:27022,192.168.1.83:27022",name:"shard2"})


#列出分片
db.runCommand({listshards:1})

#整体状态查看
sh.status();

1.2.7 创建用户

#shard1和shard2创建用户
#  mongo 192.168.1.81:27021 &  mongo 192.168.1.81:27022

use admin
db.createUser({user:'docshard', pwd:'123456', roles:['root']})

#登录方式
use admin
db.auth('docshard','123456')

#mongos建用户
mongo 192.168.1.81:27017
#mongos的用户可以在configserver上登录
use admin
db.createUser({user:'document', pwd:'123456', roles:['root']})

#MongoDB 分片集群中的 mongos 路由节点和 config server 是相互关联的，因此在 mongos 上创建的用户，实际上也可以在 config server 上使用

#建库用户  创建库要去mongos上创建  创建库前先登录管理员账号才可以有权限
mongo 192.168.1.81:27017
use admin
db.auth('document','123456')
use oldDoc;
db.createUser({user:'docold', pwd:'123456', roles: [{ role: "dbOwner", db: "oldDoc" }]})

2.2.8 添加集群内部认证 keyfile

kiall掉所有mong进程 所有机器执行

[root@mdb3 ~]# ps -ef | grep mongodb | grep -v grep | awk '{print $2}' | xargs kill
#多执行几次 直到返回没有进程号为止

创建keyfile存放目录

[root@mdb1 conf]# mkdir /data/cluster-mongodb/keys/
#所有节点创建目录

创建keyfile

[root@mdb1 keys]# cd /data/cluster-mongodb/keys/
[root@mdb1 keys]# openssl rand -base64  741 > mongodb-keyfile
[root@mdb1 keys]# chmod 600 mongodb-keyfile
[root@mdb1 keys]# scp /data/cluster-mongodb/keys/mongodb-keyfile root@192.168.1.82:/data/cluster-mongodb/keys/
[root@mdb1 keys]# scp /data/cluster-mongodb/keys/mongodb-keyfile root@192.168.1.83:/data/cluster-mongodb/keys/
#复制到其他节点

将keyfile应用到配置文件内

所有机器做同样操作

[root@mdb1 keys]# vim /data/cluster-mongodb/mongos/conf/mongos.conf
[root@mdb1 keys]# vim /data/cluster-mongodb/shard1/conf/mongod-shard1.conf
[root@mdb1 keys]# vim /data/cluster-mongodb/shard2/conf/mongod-shard2.conf
[root@mdb1 keys]# vim /data/cluster-mongodb/configsvr/conf/configsvr.conf
#在上面这些配置文件内 添加下面两行
security:
  keyFile: /data/cluster-mongodb/keys/mongodb-keyfile

2.2.9 启动

numactl --interleave=all mongod -f /data/cluster-mongodb/configsvr/conf/configsvr.conf
numactl --interleave=all mongod -f /data/cluster-mongodb/shard1/conf/mongod-shard1.conf
numactl --interleave=all mongod -f /data/cluster-mongodb/shard2/conf/mongod-shard2.conf
mongos -f /data/cluster-mongodb/mongos/conf/mongos.conf

#### 

mongo 192.168.1.31:27017

use admin 
db.auth('document','123456')
admin:document:123456

use oldDoc
db.auth('docold', '123456')
oldDoc:docold:123456

> use database_name
db.dropUser('username')

测试速度
 rs.remove('192.168.1.31:27021')
 sudo cp mongodb-keyfile /data
 sudo chown document:document mongodb-keyfile
 chmod 400 mongodb-keyfile
 
 mongod -f /data/cluster-mongodb/shard1/conf/mongod-shard1.conf

备份

备份全部库