
MongoShake实时同步迁移
https:///alibaba/MongoShake
一、开始部署
1.下载
wget https:///alibaba/MongoShake/releases/download/release-v2.4.7-20200630/mongo-shake-v2.4.7_2.tar.gz
2 . 解包
[root@mdb1 data]# tar xf mongo-shake-v2.4.7_2.tar.gz
3 . 修改配置文件
[root@mdb1 data]# cd mongo-shake-v2.4.7/
[root@mdb1 mongo-shake-v2.4.7]# cp collector.conf  collector.conf.bak
#修改collector.conf内容:
mongo_urls = mongodb://docshard:123456@192.168.1.31:27021,192.168.1.32:27021,192.168.1.33:27021
#原mongodb分片的  shard地址 同一个shard地址用,分割,不同的用;分隔
#比如:mongo_urls = mongodb://docshard:123456@192.168.1.81:27021,192.168.1.82:27021,192.168.1.83:27021;mongodb://docshard:123456@192.168.1.81:27022,192.168.1.82:27022,192.168.1.83:27022  #这样是两个shard的
mongo_cs_url = mongodb://document:123456@192.168.1.31:27020,192.168.1.32:27020,192.168.1.33:27020
#原mongodb分片的 config server地址
mongo_s_url = mongodb://document:123456@192.168.1.31:27017,192.168.1.32:27017,192.168.1.33:27017
#原mongodb分片的 mongos地址
sync_mode = incr
# 同步模式,all表示全量+增量同步,full表示全量同步,incr表示增量同步。
tunnel.address = mongodb://document:123456@192.168.1.51:27017
# 写被同步的mongodb分片集群的mongos地址
checkpoint.storage.url = mongodb://document:123456@192.168.1.51:27017
#这里写被同步的mongodb分片集群的mongos地址
#checkpoint的具体写入的MongoDB地址,如果不配置,对于副本集将写入源库(db=mongoshake),对于分片集
完整显示
[root@mdb1 mongo-shake-v2.4.7]# egrep -v "^$|^#" collector.conf
conf.version = 5
id = mongoshake
master_quorum = true
full_sync.http_port = 9101
incr_sync.http_port = 9100
system_profile_port = 9200
log.level = info
log.dir =
log.file = collector.log
log.flush = false
sync_mode = incr
mongo_urls = mongodb://docshard:123456@192.168.1.31:27021,192.168.1.32:27021,192.168.1.33:27021
mongo_cs_url = mongodb://document:123456@192.168.1.31:27020,192.168.1.32:27020,192.168.1.33:27020
mongo_s_url = mongodb://document:123456@192.168.1.31:27017,192.168.1.32:27017,192.168.1.33:27017
tunnel = direct
tunnel.address = mongodb://document:123456@192.168.1.51:27017
tunnel.message = raw
mongo_connect_mode = secondaryPreferred
filter.namespace.black =
filter.namespace.white =
filter.pass.special.db =
filter.ddl_enable = false
checkpoint.storage.url = mongodb://document:123456@192.168.1.51:27017
checkpoint.storage.db = mongoshake
checkpoint.storage.collection = ckpt_default
checkpoint.start_position = 1970-01-01T00:00:00Z
transform.namespace =
full_sync.reader.collection_parallel = 6
full_sync.reader.write_document_parallel = 8
full_sync.reader.document_batch_size = 128
full_sync.reader.read_document_count = 0
full_sync.collection_exist_drop = true
full_sync.create_index = none
full_sync.executor.insert_on_dup_update = false
full_sync.executor.filter.orphan_document = false
full_sync.executor.majority_enable = false
incr_sync.mongo_fetch_method = oplog
incr_sync.oplog.gids =
incr_sync.shard_key = collection
incr_sync.worker = 8
incr_sync.worker.oplog_compressor = none
incr_sync.target_delay = 0
incr_sync.worker.batch_queue_size = 64
incr_sync.adaptive.batching_max_size = 1024
incr_sync.fetcher.buffer_capacity = 256
incr_sync.executor.upsert = false
incr_sync.executor.insert_on_dup_update = false
incr_sync.conflict_write_to = none
incr_sync.executor.majority_enable = false
incr_sync.change_stream.watch_full_document = false
二、启动同步
1. 查看两个mongo集群
源集群
[root@mongo1 ~]# mongo 192.168.1.31:27017
MongoDB shell version v3.6.3
connecting to: mongodb://192.168.1.31:27017/test
MongoDB server version: 3.6.3
mongos> use admin
switched to db admin
mongos> db.auth('document','123456')
1
mongos> show dbs
admin   0.000GB
config  0.079GB
data    0.078GB
dir     0.078GB
test    0.078GB
#库有5个
新集群
[root@mdb1 mongo-shake-v2.4.20]# mongo 192.168.1.51:27017
MongoDB shell version v3.6.3
connecting to: mongodb://192.168.1.51:27017/test
MongoDB server version: 3.6.3
mongos> use admin
switched to db admin
mongos> db.auth('document','123456')
1
mongos> show dbs
admin   0.000GB
config  0.079GB
#库只有2个
2.启动
[root@mdb1 mongo-shake-v2.4.7]# ./collector.linux -verbose -conf=collector.conf
#后台启动
nohup ./collector.linux -verbose -conf=collector.conf > collector.log 2>&1 &
正在同步数据
[2024/12/06 16:10:04 CST] [WARN] CheckpointOperation updated is not suitable. lowest [0]. current [1[0, 1]]. inputTs [0]. reason : smallest candidates is zero
[2024/12/06 16:10:04 CST] [INFO] New session to mongodb://document:***@192.168.1.51:27017 successfully
[2024/12/06 16:10:04 CST] [INFO] New session to mongodb://document:***@192.168.1.51:27017 successfully
[2024/12/06 16:10:04 CST] [INFO] New session to mongodb://document:***@192.168.1.51:27017 successfully
[2024/12/06 16:10:04 CST] [INFO] New session to mongodb://document:***@192.168.1.51:27017 successfully
[2024/12/06 16:10:04 CST] [INFO] Replayer-6 Executor-6 doSync oplogRecords received[1] merged[1]. merge to 100.00% chunks
[2024/12/06 16:10:04 CST] [INFO] Collector-worker-6 transfer retransmit:false send [1] logs. reply_acked [7444934952707162114[1733408997, 2]], list_unack [0]
[2024/12/06 16:10:04 CST] [INFO] Replayer-6 Executor-6 doSync oplogRecords received[1] merged[1]. merge to 100.00% chunks
[2024/12/06 16:10:04 CST] [INFO] Collector-worker-6 transfer retransmit:false send [1] logs. reply_acked [7445115959808884737[1733451141, 1]], list_unack [0]
[2024/12/06 16:10:05 CST] [INFO] Replayer-2 Executor-2 doSync oplogRecords received[70] merged[1]. merge to 1.43% chunks
[2024/12/06 16:10:05 CST] [INFO] Replayer-4 Executor-4 doSync oplogRecords received[6] merged[1]. merge to 16.67% chunks
[2024/12/06 16:10:05 CST] [INFO] Collector-worker-4 transfer retransmit:false send [6] logs. reply_acked [7444916952499224589[1733404806, 13]], list_unack [0]
[2024/12/06 16:10:05 CST] [INFO] Collector-worker-2 transfer retransmit:false send [70] logs. reply_acked [7445178361388728331[1733465670, 11]], list_unack [0]
[2024/12/06 16:10:05 CST] [INFO] Replayer-4 Executor-4 doSync oplogRecords received[1] merged[1]. merge to 100.00% chunks
[2024/12/06 16:10:05 CST] [INFO] Collector-worker-4 transfer retransmit:false send [1] logs. reply_acked [7445115981283721217[1733451146, 1]], list_unack [0]
[2024/12/06 16:10:05 CST] [INFO] Replayer-2 Executor-2 doSync oplogRecords received[38] merged[1]. merge to 2.63% chunks
[2024/12/06 16:10:05 CST] [INFO] Collector-worker-2 transfer retransmit:false send [38] logs. reply_acked [7445190224088399874[1733468432, 2]], list_unack [0]
[2024/12/06 16:10:05 CST] [INFO] Replayer-7 Executor-7 doSync oplogRecords received[3] merged[1]. merge to 33.33% chunks
[2024/12/06 16:10:05 CST] [INFO] Collector-worker-7 transfer retransmit:false send [3] logs. reply_acked [7445190666470031369[1733468535, 9]], list_unack [0]
[2024/12/06 16:10:09 CST] [INFO] become the master and notify waiter
[2024/12/06 16:10:09 CST] [INFO] [name=shard1, stage=incr, get=19197, filter=19077, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:10:14 CST] [INFO] [name=shard1, stage=incr, get=19198, filter=19077, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:10:19 CST] [INFO] [name=shard1, stage=incr, get=19198, filter=19078, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:10:24 CST] [INFO] [name=shard1, stage=incr, get=19199, filter=19078, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:10:29 CST] [INFO] [name=shard1, stage=incr, get=19199, filter=19079, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:10:34 CST] [INFO] [name=shard1, stage=incr, get=19200, filter=19079, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
查看一下新集群
mongos> show dbs
admin   0.000GB
config  0.079GB
mongos> show dbs
admin       0.000GB
config      0.079GB
data        0.078GB
dir         0.078GB
mongoshake  0.078GB
test        0.078GB
#已经同步了很多库了
mongos> use data
switched to db data
#查看内容
mongos> db.doc.find()
{ "_id" : ObjectId("6751a8867ef1e20c8d4eb82c"), "title" : "MongoDB入门教程", "author" : "张三", "category" : "数据库", "content" : "这是一个关于MongoDB基础知识的中文教程。", "tags" : [ "教程", "数据库", "MongoDB" ], "views" : 1200, "likes" : 230, "published_date" : ISODate("2024-01-15T00:00:00Z"), "updated_date" : ISODate("2024-03-01T00:00:00Z"), "comments" : [ { "user" : "李四", "message" : "很有帮助,谢谢!", "date" : ISODate("2024-02-01T00:00:00Z") }, { "user" : "王五", "message" : "内容详细,推荐阅读!", "date" : ISODate("2024-02-10T00:00:00Z") } ] }
{ "_id" : ObjectId("6751a8867ef1e20c8d4eb82d"), "title" : "高并发系统设计", "author" : "李四", "category" : "技术架构", "content" : "如何设计支持高并发的系统。", "tags" : [ "技术", "架构", "高并发" ], "views" : 500, "likes" : 85, "published_date" : ISODate("2023-12-25T00:00:00Z"), "updated_date" : ISODate("2024-02-15T00:00:00Z"), "comments" : [ { "user" : "张三", "message" : "学习了,写得不错。", "date" : ISODate("2024-01-05T00:00:00Z") } ] }
{ "_id" : ObjectId("6751a8867ef1e20c8d4eb82e"), "title" : "前端框架对比", "author" : "王五", "category" : "前端开发", "content" : "对比React、Vue和Angular的优缺点。", "tags" : [ "前端", "框架", "对比" ], "views" : 920, "likes" : 300, "published_date" : ISODate("2024-03-01T00:00:00Z"), "updated_date" : ISODate("2024-03-10T00:00:00Z"), "comments" : [ { "user" : "李四", "message" : "Vue很适合初学者。", "date" : ISODate("2024-03-05T00:00:00Z") }, { "user" : "张三", "message" : "React生态强大。", "date" : ISODate("2024-03-06T00:00:00Z") } ] }
{ "_id" : ObjectId("6751a8867ef1e20c8d4eb82f"), "title" : "机器学习与AI", "author" : "赵六", "category" : "人工智能", "content" : "机器学习的基本概念与应用。", "tags" : [ "AI", "机器学习", "技术" ], "views" : 1500, "likes" : 400, "published_date" : ISODate("2024-02-01T00:00:00Z"), "updated_date" : ISODate("2024-02-20T00:00:00Z"), "comments" : [ { "user" : "王五", "message" : "期待更多案例讲解!", "date" : ISODate("2024-02-15T00:00:00Z") }, { "user" : "李四", "message" : "能否讲一下深度学习?", "date" : ISODate("2024-02-18T00:00:00Z") } ] }
{ "_id" : ObjectId("6751a8867ef1e20c8d4eb830"), "title" : "旅行攻略:国内篇", "author" : "张三", "category" : "旅行", "content" : "介绍中国境内的热门旅行目的地。", "tags" : [ "旅行", "国内", "攻略" ], "views" : 2100, "likes" : 560, "published_date" : ISODate("2024-03-15T00:00:00Z"), "updated_date" : ISODate("2024-03-20T00:00:00Z"), "comments" : [ { "user" : "赵六", "message" : "很适合假期出行!", "date" : ISODate("2024-03-16T00:00:00Z") }, { "user" : "李四", "message" : "推荐加上西藏的行程。", "date" : ISODate("2024-03-18T00:00:00Z") } ] }
{ "_id" : ObjectId("6751a8867ef1e20c8d4eb831"), "title" : "家庭理财:从零开始", "author" : "李四", "category" : "理财", "content" : "普通家庭如何进行合理的理财规划。", "tags" : [ "理财", "家庭", "规划" ], "views" : 800, "likes" : 150, "published_date" : ISODate("2023-12-01T00:00:00Z"), "updated_date" : ISODate("2024-01-10T00:00:00Z"), "comments" : [ { "user" : "王五", "message" : "简单易懂,受益匪浅!", "date" : ISODate("2023-12-05T00:00:00Z") } ] }
{ "_id" : ObjectId("67525d8af1edacedef355eb4"), "title" : "MongoDB入门教程", "author" : "张三" }
mongos>
3.查看同步情况
#它自带一个监控
[root@mdb1 mongo-shake-v2.4.7]# ./mongoshake-stat --port=9100
|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|
|        logs_get/sec |       logs_repl/sec |    logs_success/sec |            lsn.time |        lsn_ack.time |       lsn_ckpt.time |            now.time |             replset |             tps/sec |
|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|
|                none |                none |                none | 2024-12-06 15:02:15 | 2024-12-06 15:02:15 | 1970-01-01 08:00:00 | 2024-12-06 16:12:22 |              shard1 |                none |
|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|
|                   0 |                   0 |                   0 | 2024-12-06 15:02:15 | 2024-12-06 15:02:15 | 1970-01-01 08:00:00 | 2024-12-06 16:12:23 |              shard1 |                   0 |
|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|
|                   0 |                   0 |                   0 | 2024-12-06 15:02:15 | 2024-12-06 15:02:15 | 1970-01-01 08:00:00 | 2024-12-06 16:12:24 |              shard1 |                   0 |
|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|
|                   1 |                   0 |                   0 | 2024-12-06 15:02:15 | 2024-12-06 15:02:15 | 1970-01-01 08:00:00 | 2024-12-06 16:12:25 |              shard1 |                   0 |
|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|
|                   0 |                   0 |                   0 | 2024-12-06 15:02:15 | 2024-12-06 15:02:15 | 1970-01-01 08:00:00 | 2024-12-06 16:12:26 |              shard1 |                   0 |
|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|
#也可以查看输出的前台日志
0031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:12:24 CST] [INFO] [name=shard1, stage=incr, get=19211, filter=19090, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:12:29 CST] [INFO] [name=shard1, stage=incr, get=19211, filter=19091, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:12:34 CST] [INFO] [name=shard1, stage=incr, get=19212, filter=19091, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:12:39 CST] [INFO] [name=shard1, stage=incr, get=19212, filter=19092, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:12:44 CST] [INFO] [name=shard1, stage=incr, get=19213, filter=19092, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:12:49 CST] [INFO] [name=shard1, stage=incr, get=19213, filter=19093, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
[2024/12/06 16:12:54 CST] [INFO] [name=shard1, stage=incr, get=19214, filter=19093, success=120, tps=0, ckpt_times=0, lsn_ckpt={0[0, 0], 1970-01-01 08:00:00}, lsn_ack={7445190666470031369[1733468535, 9], 2024-12-06 15:02:15}]]
4.测试是否增量同步
在源集群新建一个库,然后在新集合增加点数据,查看新集群是否同步正常
#源集群
//创建个库
use hello
//造点数据
db.file.insertMany([
  { 
    title: "我的第一篇文档", 
    description: "这是一篇关于MongoDB的介绍文章,帮助你入门使用MongoDB", 
    author: "张三", 
    created_at: new Date("2024-12-06T15:00:00"), 
    size: 1024, 
    language: "中文" 
  },
  { 
    title: "MongoDB配置教程", 
    description: "本文将详细介绍如何配置MongoDB分片集群", 
    author: "李四", 
    created_at: new Date("2024-12-06T15:05:00"), 
    size: 2048, 
    language: "中文" 
  },
  { 
    title: "数据科学与MongoDB", 
    description: "结合MongoDB分析大数据的应用案例", 
    author: "王五", 
    created_at: new Date("2024-12-06T15:10:00"), 
    size: 3072, 
    language: "中文" 
  },
  { 
    title: "MongoDB高级用法", 
    description: "深入MongoDB的高阶特性与性能优化", 
    author: "赵六", 
    created_at: new Date("2024-12-06T15:15:00"), 
    size: 5120, 
    language: "中文" 
  },
  { 
    title: "如何使用MongoDB实现数据分片", 
    description: "本文讲解如何使用MongoDB进行数据分片部署", 
    author: "钱七", 
    created_at: new Date("2024-12-06T15:20:00"), 
    size: 4096, 
    language: "中文" 
  }
]);
// 查询写入的数据
db.file.find()
{ "_id" : ObjectId("6752b308622338e615eb435c"), "title" : "我的第一篇文档", "description" : "这是一篇关于MongoDB的介绍文章,帮助你入门使用MongoDB", "author" : "张三", "created_at" : ISODate("2024-12-06T07:00:00Z"), "size" : 1024, "language" : "中文" }
{ "_id" : ObjectId("6752b308622338e615eb435d"), "title" : "MongoDB配置教程", "description" : "本文将详细介绍如何配置MongoDB分片集群", "author" : "李四", "created_at" : ISODate("2024-12-06T07:05:00Z"), "size" : 2048, "language" : "中文" }
{ "_id" : ObjectId("6752b308622338e615eb435e"), "title" : "数据科学与MongoDB", "description" : "结合MongoDB分析大数据的应用案例", "author" : "王五", "created_at" : ISODate("2024-12-06T07:10:00Z"), "size" : 3072, "language" : "中文" }
{ "_id" : ObjectId("6752b308622338e615eb435f"), "title" : "MongoDB高级用法", "description" : "深入MongoDB的高阶特性与性能优化", "author" : "赵六", "created_at" : ISODate("2024-12-06T07:15:00Z"), "size" : 5120, "language" : "中文" }
{ "_id" : ObjectId("6752b308622338e615eb4360"), "title" : "如何使用MongoDB实现数据分片", "description" : "本文讲解如何使用MongoDB进行数据分片部署", "author" : "钱七", "created_at" : ISODate("2024-12-06T07:20:00Z"), "size" : 4096, "language" : "中文" }
去新集群上查看是否同步正常
mongos> show dbs
admin       0.000GB
config      0.079GB
data        0.078GB
dir         0.078GB
hello       0.078GB
mongoshake  0.078GB
test        0.078GB
mongos> use hello
switched to db hello
mongos> db.file.find()
{ "_id" : ObjectId("6752b308622338e615eb435c"), "title" : "我的第一篇文档", "description" : "这是一篇关于MongoDB的介绍文章,帮助你入门使用MongoDB", "author" : "张三", "created_at" : ISODate("2024-12-06T07:00:00Z"), "size" : 1024, "language" : "中文" }
{ "_id" : ObjectId("6752b308622338e615eb435d"), "title" : "MongoDB配置教程", "description" : "本文将详细介绍如何配置MongoDB分片集群", "author" : "李四", "created_at" : ISODate("2024-12-06T07:05:00Z"), "size" : 2048, "language" : "中文" }
{ "_id" : ObjectId("6752b308622338e615eb435e"), "title" : "数据科学与MongoDB", "description" : "结合MongoDB分析大数据的应用案例", "author" : "王五", "created_at" : ISODate("2024-12-06T07:10:00Z"), "size" : 3072, "language" : "中文" }
{ "_id" : ObjectId("6752b308622338e615eb435f"), "title" : "MongoDB高级用法", "description" : "深入MongoDB的高阶特性与性能优化", "author" : "赵六", "created_at" : ISODate("2024-12-06T07:15:00Z"), "size" : 5120, "language" : "中文" }
{ "_id" : ObjectId("6752b308622338e615eb4360"), "title" : "如何使用MongoDB实现数据分片", "description" : "本文讲解如何使用MongoDB进行数据分片部署", "author" : "钱七", "created_at" : ISODate("2024-12-06T07:20:00Z"), "size" : 4096, "language" : "中文" }
成功