mongoDB分页查询优化设计实践
- 背景
- 设计原理及开发过程
- 方式1
- 方式2
- 关键代码
- 具体可详见项目
背景
这个是针对上一篇博客的具体实现,主要是为了解决大数据量下(比如:物联设备上报数据的查询),mongoDB采用skip() + limit()越往后翻页查询越慢的问题。
补一句题外话: mongoDB有利于灵活存储不确定的数据结构(设备上报的不确定属性、灵活多变的数据结构),不像mysql等关系数据库需要事先确定并设计好数据表相关字段,这也是mongoDB优势之一。
设计原理及开发过程
可以根据某个字段(值不重复)进行排序查询,边查询边返回下一次的查询起点,边查询边缩小查询范围,向后查询查询效率就可以越高。
个人觉得:mongoDB文档设计过程,不要涉及太多不必要的业务字段。业务字段可以在mysql等数据库的业务表存储,按业务查询时,先根据业务表过滤筛选合适的设备ID,做到按设备ID、上报时间分页查询接口。底层存储查询通用接口编写,尽可能涉及少按业务字段查询处理,否则,还有各种业务查询组合,可想而知查询效率也快不了。因此,能业务层提前筛选过滤掉的,就不要放到mongoDB来查询处理了。
方式1
主键设计规则: _id = 设备ID + “.” + 上报时间(时间戳整数,秒为单位)。
注意事项:根据实际场景,如果时间跨度很大,秒的位数不一样,则注意前面补0,毕竟查询的时候是按字符串比较的方式;否则会出现查询时间比较不对(前面的时间大于后面的 例如: “12” < “5”)。
方式2
除了主键设计规则外,项目也可以改造成通过额外创建索引的方式进行设计,即通过 nodeId(设备ID)、上报时间字段 创建组合索引的方式。 代码中的具体逻辑只有小调整(将根据主键,改为根据设备ID、上报时间字段为主,按上报时间排序,不断缩小上报时间的查询范围)即可,依然可以根据上报时间对某个设备ID上报的数据进行上述分页查询。
关键代码
通用接口编写(可实现向前翻,不过向前翻页比向后翻页需要注意下细节处理)
package com.ws.app.controller;
import com.ws.app.core.constants.MongoDBCollectionNameConstant;
import com.ws.app.core.exception.BaseBusinessException;
import com.ws.app.core.web.WebResponse;
import com.ws.app.dto.mongo.NodeReportInfoPageDTO;
import com.ws.app.request.mongo.NodeLatestReportInfoRequest;
import com.ws.app.request.mongo.NodeReportInfoAggregateRequest;
import com.ws.app.request.mongo.NodeReportInfoPageRequest;
import com.ws.app.service.MongoDBService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* @desc 节点数据存储、获取控制类
*/
@RestController
@RequestMapping("/dcp/reportQuery")
public class DevicePutAndQueryController {
private final static Logger log = LoggerFactory.getLogger(DevicePutAndQueryController.class);
@Autowired
MongoDBService mongoDBService;
/**
* 测试get
* @return
*/
@RequestMapping(value = "/dcpNodeReportedInfo/{id}", method = RequestMethod.GET)
WebResponse testGet(@PathVariable("id") String id) {
Map<String, Object> map = mongoDBService.findById(id, MongoDBCollectionNameConstant.dcpNodeReportedInfoTable);
return new WebResponse(map);
}
/**
* 通用接口:根据dcp节点、集合获取最新上报记录
* @return
*/
@RequestMapping(value = "/dcpNodeReportedInfo/latest", method = RequestMethod.POST)
WebResponse getLatestDcpNodeReportedInfoByNodeId(@RequestBody NodeLatestReportInfoRequest nodeLatestReportInfoRequest) {
if (StringUtils.isEmpty(nodeLatestReportInfoRequest.getNodeId())) {
throw new BaseBusinessException("nodeId不能为空!");
}
if (StringUtils.isEmpty(nodeLatestReportInfoRequest.getCollectionName())) {
throw new BaseBusinessException("collectionName不能为空!");
}
Map<String, Object> map = mongoDBService.getLatestDcpNodeReportedInfoByNodeId(nodeLatestReportInfoRequest);
return new WebResponse(map);
}
/**
* 通用接口:根据dcp节点、集合分页获取节点上报
* @return
*/
@RequestMapping(value = "/dcpNodeReportedInfo/page", method = RequestMethod.POST)
WebResponse getDcpNodeReportedInfoPage(@RequestBody NodeReportInfoPageRequest reportInfoPageRequest) {
log.info(reportInfoPageRequest.toString());
log.info("StartId={}, EndId={}", reportInfoPageRequest.getCurrentStartId(), reportInfoPageRequest.getCurrentEndId());
if (StringUtils.isEmpty(reportInfoPageRequest.getNodeId())) {
throw new BaseBusinessException("nodeId不能为空!");
}
if (StringUtils.isEmpty(reportInfoPageRequest.getCollectionName())) {
throw new BaseBusinessException("collectionName不能为空!");
}
if (StringUtils.isEmpty(reportInfoPageRequest.getStartId()) && reportInfoPageRequest.isPageUpStatus() ) {
// startId为空, 不能向前翻页
throw new BaseBusinessException("startId为空,不能向前翻页!");
}
NodeReportInfoPageDTO nodeReportInfoPageDTO = mongoDBService.getDcpNodeReportedInfoPage(reportInfoPageRequest);
return new WebResponse(nodeReportInfoPageDTO);
}
/**
* 聚合接口:
* 通用接口:根据dcp节点、集合获取属性最高、最低、均值
* @return
*/
@RequestMapping(value = "/dcpNodeReportedInfo/aggregateData", method = RequestMethod.POST)
WebResponse getDcpNodeReportedInfoAggregationData(@RequestBody NodeReportInfoAggregateRequest aggregateRequest) {
if (StringUtils.isEmpty(aggregateRequest.getNodeId())) {
throw new BaseBusinessException("nodeId不能为空!");
}
if (StringUtils.isEmpty(aggregateRequest.getCollectionName())) {
throw new BaseBusinessException("collectionName不能为空!");
}
if (Objects.isNull(aggregateRequest.getKeyList()) || aggregateRequest.getKeyList().size() <= 0) {
throw new BaseBusinessException("keyList数组不能为空!");
}
if (Objects.isNull(aggregateRequest.getAggregateWayList()) || aggregateRequest.getAggregateWayList().size() <= 0) {
throw new BaseBusinessException("aggregateWayList数组不能为空!");
}
List<String> aggregateWayList = Arrays.asList(NodeReportInfoAggregateRequest.AGGREGATE_WAY_ARR);
if (!aggregateWayList.containsAll(aggregateRequest.getAggregateWayList())) {
throw new BaseBusinessException("存在未知的聚合方式!");
}
Map<String, Object> map = mongoDBService.getDcpNodeReportedInfoAggregationData(aggregateRequest);
return new WebResponse(map);
}
}
服务层
package com.ws.app.service;
import com.ws.app.dto.mongo.NodeReportInfoPageDTO;
import com.ws.app.request.mongo.NodeLatestReportInfoRequest;
import com.ws.app.request.mongo.NodeReportInfoAggregateRequest;
import com.ws.app.request.mongo.NodeReportInfoPageRequest;
import java.util.Map;
/**
* @desc mongoDB 存储、查询,统一操作服务类
* @author ws
* @since 2021/1/13
*/
public interface MongoDBService {
/**
* 存储Map对象到mongoDB集合中: _id = nodeId + "." + 时间(秒)
* @param saveMap
* @param collectionName
*/
void putMapObj(Map<String, Object> saveMap, String collectionName);
/**
* 根据主键获取集合记录
* @param id
* @param collectionName
* @return
*/
Map<String,Object> findById(String id, String collectionName);
/**
* 根据dcp节点,集合名称获取最新上报记录
* @param nodeLatestReportInfoRequest
* @return
*/
Map<String,Object> getLatestDcpNodeReportedInfoByNodeId(NodeLatestReportInfoRequest nodeLatestReportInfoRequest);
/**
* 根据dcp节点,集合名称分页获取上报信息
* @param reportInfoPageRequest
* @return
*/
NodeReportInfoPageDTO getDcpNodeReportedInfoPage(NodeReportInfoPageRequest reportInfoPageRequest);
/**
* 根据dcp节点、集合获取属性最高、最低、均值
* @param aggregateRequest
* @return
*/
Map<String,Object> getDcpNodeReportedInfoAggregationData(NodeReportInfoAggregateRequest aggregateRequest);
}
服务层实现类
package com.ws.app.service.impl;
import cn.hutool.core.date.DateUtil;
import com.ws.app.dto.mongo.NodeReportInfoPageDTO;
import com.ws.app.request.mongo.NodeLatestReportInfoRequest;
import com.ws.app.request.mongo.NodeReportInfoAggregateRequest;
import com.ws.app.request.mongo.NodeReportInfoPageRequest;
import com.ws.app.service.MongoDBService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
import org.springframework.data.mongodb.core.aggregation.GroupOperation;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import java.util.*;
/**
* @desc mongoDB 存储、查询,统一操作服务实现类
* @author ws
* @since 2021/1/13
*/
@Service
public class MongoDBServiceImpl implements MongoDBService {
private final static Logger log = LoggerFactory.getLogger(MongoDBServiceImpl.class);
// 聚合方式
private final String aggregateAliasFormat = "%s_%s";
public final static String AGGREGATE_WAY_MAX = "max";
public final static String AGGREGATE_WAY_MIN = "min";
public final static String AGGREGATE_WAY_AVG = "avg";
@Autowired
private MongoTemplate mongoTemplate;
/**
* 存储Map对象到mongoDB中: _id = nodeId + "." + 时间(秒)
* @param saveMap
* @param collectionName
*/
@Override
public void putMapObj(Map<String, Object> saveMap, String collectionName) {
// 统一设置入库时间
saveMap.put("createTime", new Date());
// 插入记录
//mongoTemplate.save(data);
// 一次性插入一整个列表,而不用进行遍历操作,效率相对较高
mongoTemplate.insert(saveMap, collectionName);
}
@Override
public Map<String, Object> findById(String id, String collectionName) {
return mongoTemplate.findById(id, Map.class, collectionName);
}
/**
* 根据dcp节点获取最新上报记录
* @param nodeLatestReportInfoRequest
* @return
*/
@Override
public Map<String, Object> getLatestDcpNodeReportedInfoByNodeId(NodeLatestReportInfoRequest nodeLatestReportInfoRequest) {
// 拼接完整ID: _id = nodeId + "." + 时间(秒)
// String _id = nodeId + "." + (new Date()).getTime() / 1000;
Criteria criteria = Criteria.where("_id").lte(nodeLatestReportInfoRequest.getId());
// 取最新的记录
Query query = new Query().addCriteria(criteria)
.with(Sort.by(
Sort.Order.desc("_id")
)).limit(1);
Map<String, Object> returnMap = mongoTemplate.findOne(query, Map.class, nodeLatestReportInfoRequest.getCollectionName());
// 填充测量时间:时间转换
if (returnMap != null) {
String _id = (String) returnMap.get("_id");
returnMap.put("measureDate", getMeasureDateById(_id));
}
return returnMap;
}
/**
* 根据主键,集合名称分页获取上报信息
* @param reportInfoPageRequest
* @return
*/
@Override
public NodeReportInfoPageDTO getDcpNodeReportedInfoPage(NodeReportInfoPageRequest reportInfoPageRequest) {
Query query = new Query();
Criteria criteria = Criteria.where("_id")
.gte(reportInfoPageRequest.getCurrentStartId())
.lte(reportInfoPageRequest.getCurrentEndId());
query.addCriteria(criteria);
if (reportInfoPageRequest.getOrderBy().equals(NodeReportInfoPageRequest.ASC_ORDER)) {
if (!reportInfoPageRequest.isPageUpStatus()) {
// 正常向后翻,不用考虑过多
// 升序
query.with(Sort.by(Sort.Order.asc("_id")));
} else {
// 向前翻特殊处理:需要跟正常后翻的排序方式相反, 结果返回(倒序)时需要还得手动排序(正序)
query.with(Sort.by(Sort.Order.desc("_id")));
}
} else {
if (!reportInfoPageRequest.isPageUpStatus()) {
// 正常向后翻,不用考虑过多
// 降序
query.with(Sort.by(Sort.Order.desc("_id")));
} else {
// 向前翻特殊处理:需要跟正常后翻的排序方式相反, 结果返回(正序)时需要还得手动排序(倒序)
query.with(Sort.by(Sort.Order.asc("_id")));
}
}
// 分页大小
query.limit(reportInfoPageRequest.getPageSize());
List<Map> mapList = mongoTemplate.find(query, Map.class, reportInfoPageRequest.getCollectionName());
NodeReportInfoPageDTO nodeReportInfoPageDTO = new NodeReportInfoPageDTO();
nodeReportInfoPageDTO.setDataList(mapList);
int listSize = mapList.size();
nodeReportInfoPageDTO.setRecordTotal(listSize);
if (listSize > 0) {
if (reportInfoPageRequest.isPageUpStatus()) {
// 向前翻特殊处理: 返回结果需要反转
Collections.reverse(mapList);
}
Map nextMap = mapList.get(listSize - 1);
// 返回下次查询起点 (优化设计:用主键方式查询分页,可以避免数据量过大、查询过慢的问题)
String nextKey = nextMap.get("_id").toString();
Map preMap = mapList.get(0);
String preKey = preMap.get("_id").toString();
// 需要根据 排序方式: 设置前一个点、后一个点
if (NodeReportInfoPageRequest.ASC_ORDER.equals(reportInfoPageRequest.getOrderBy())) {
// 向后翻:下次查询点:时间戳加一秒
nodeReportInfoPageDTO.setNextStartId(getQueryNextStartId(nextKey, 1L));
// 向前翻:下次查询点:时间戳减一秒
nodeReportInfoPageDTO.setPrevStartId(getQueryNextStartId(preKey, -1L));
} else {
// 降序,NextStartId 跟 PrevStartId反转
nodeReportInfoPageDTO.setNextStartId(getQueryNextStartId(nextKey, -1L));
nodeReportInfoPageDTO.setPrevStartId(getQueryNextStartId(preKey, 1L));
}
//填充测量时间:时间转换
mapList.parallelStream().forEach(map -> {
String _id = (String) map.get("_id");
map.put("measureDate", getMeasureDateById(_id));
});
//没数据返回即结束翻页
}
return nodeReportInfoPageDTO;
}
/**
* 根据dcp节点、集合获取属性最高、最低、均值
* @param aggregateRequest
* @return
*/
@Override
public Map<String, Object> getDcpNodeReportedInfoAggregationData(NodeReportInfoAggregateRequest aggregateRequest) {
String currentStartId = null;
String currentEndId = null;
if (StringUtils.isEmpty(aggregateRequest.getStartTime())) {
//为空,设置开始值:时间取最小即可
currentStartId = aggregateRequest.getNodeId() + ".0";
} else {
currentStartId = aggregateRequest.getNodeId() + "." + aggregateRequest.getStartTime().getTime() / 1000;
}
if (StringUtils.isEmpty(aggregateRequest.getEndTime())) {
//为空,设置结束值:取当前时间
currentEndId = aggregateRequest.getNodeId() + "." + (new Date()).getTime() / 1000;
} else {
currentEndId = aggregateRequest.getNodeId() + "." + aggregateRequest.getEndTime().getTime() / 1000;
}
// 设置返回属性列表
List<String> outputAttrNameList = new ArrayList<>();
GroupOperation groupOperation = Aggregation.group("nodeId"); //分组, 设置一个不存在的字段即可统计该查询记录下的全部数据
// 根据属性名、集合函数,返回别名 -> 动态拼接需要分组聚合统计的属性
// 例如下面:注释部分
/*groupOperation = groupOperation.max("temp").as("max_temp");
groupOperation = groupOperation.min("temp").as("min_temp");
groupOperation = groupOperation.avg("temp").as("avg_temp");
String[] names = {"max_temp", "min_temp", "avg_temp"};*/
for (String item : aggregateRequest.getKeyList()) {
// 该属性需要进行何种聚合统计
for (int i = 0; i< NodeReportInfoAggregateRequest.AGGREGATE_WAY_ARR.length; i++) {
String aggregateWay = NodeReportInfoAggregateRequest.AGGREGATE_WAY_ARR[i];
if (aggregateRequest.getAggregateWayList().contains(aggregateWay)) {
// 该属性按设置的所有聚合方式进行聚合统计
String attrAlias = String.format(aggregateAliasFormat, aggregateWay, item);
outputAttrNameList.add(attrAlias);
switch (aggregateWay) {
case MongoDBServiceImpl.AGGREGATE_WAY_AVG :
groupOperation = groupOperation.avg(item).as(attrAlias);
break;
case MongoDBServiceImpl.AGGREGATE_WAY_MAX :
groupOperation = groupOperation.max(item).as(attrAlias);
break;
case MongoDBServiceImpl.AGGREGATE_WAY_MIN :
groupOperation = groupOperation.min(item).as(attrAlias);
break;
default:
}
}
}
}
Aggregation aggregation = Aggregation.newAggregation(Aggregation.match(Criteria.where("_id")
.gt(currentStartId)
.lte(currentEndId)),
groupOperation,
Aggregation.project(outputAttrNameList.toArray(new String[outputAttrNameList.size()])).and("nodeId").previousOperation() // 用nodeId字段 替换调返回的 _id字段
);
AggregationResults<Map> results = mongoTemplate.aggregate(aggregation, aggregateRequest.getCollectionName(), Map.class);
int returnSize = results.getMappedResults().size();
log.info("results size= {}", returnSize);
Map returnMap = null;
if (returnSize > 0) {
returnMap = results.getMappedResults().get(0);
} else {
returnMap = new HashMap();
}
returnMap.put("nodeId", aggregateRequest.getNodeId());
return returnMap;
}
/**
* 返回翻页下一个查询起点
* @param idKey _id主键
* @param addNum 加秒数
* @return
*/
private String getQueryNextStartId(String idKey, Long addNum) {
String[] keyArr = idKey.split("\\.");
String nodeId = keyArr[0];
String timeStampStr = keyArr[1];
String nextStartId = nodeId + "." + (Long.valueOf(timeStampStr) + addNum);
log.info("nodeId={},timeStampStr={}, nextStartId={}", nodeId, timeStampStr, nextStartId);
return nextStartId;
}
/**
* 根据_id返回上报时间(测量时间)
* @param idKey
* @return String
*/
private String getMeasureDateStrById(String idKey) {
Date measureDate = getMeasureDateById(idKey);
return DateUtil.format(measureDate, "yyyy-MM-dd HH:mm:ss");
}
/**
* 根据_id返回上报时间(测量时间)
* @param idKey
* @return Date
*/
private Date getMeasureDateById(String idKey) {
String[] idArr = idKey.split("\\.");
Date measureDate = new Date(Long.valueOf(idArr[1]) * 1000L);
return measureDate;
}
}
分页查询请求实体类
package com.ws.app.request.mongo;
import org.springframework.util.StringUtils;
import java.io.Serializable;
import java.util.Date;
/**
* @desc mongo节点上报分页查询
* @author ws
* @since 2021/1/14
*/
public class NodeReportInfoPageRequest implements Serializable {
private static final long serialVersionUID = 2266336842598690895L;
/**
* 节点ID/设备ID
*/
private String nodeId;
/**
* 开始时间
*/
private Date startTime;
/**
* 结束时间
*/
private Date endTime;
/**
* 当前页查询点,翻页需要传(include)
*/
private String startId;
/**
* true 向前翻页:false向后翻页
*/
private boolean pageUpStatus;
/**
* 集合名称
*/
private String collectionName;
/**
* 分页大小
*/
private int pageSize = 1000;
/**
* 排序方式 asc or desc, 默认升序
*/
private String orderBy = NodeReportInfoPageRequest.ASC_ORDER;
public final static String ASC_ORDER = "asc";
public final static String DESC_ORDER = "desc";
public String getNodeId() {
return nodeId;
}
public void setNodeId(String nodeId) {
this.nodeId = nodeId;
}
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
/**
* 根据查询条件,推导出当前查询起点
* 注意:翻页,此方法挺重要!!!!
* @return
*/
public String getCurrentStartId() {
String currentStartId = null;
if (StringUtils.isEmpty(startTime)) {
//为空,设置开始值:时间取最小即可
currentStartId = nodeId + ".0";
} else {
currentStartId = nodeId + "." + startTime.getTime() / 1000;
}
// 根据查询点startId和翻页方式调整当前查询起点
if (!StringUtils.isEmpty(startId)) {
if (pageUpStatus) {
// 向前翻页 (跟向后翻页刚好相反)
if (NodeReportInfoPageRequest.ASC_ORDER.equals(orderBy)) {
} else {
currentStartId = startId;
}
} else {
// 向后翻页
if (NodeReportInfoPageRequest.ASC_ORDER.equals(orderBy)) {
// 升序; 下一页调整起始查询点
currentStartId = startId;
} else {
// 降序,不需要调整起始查询点;要调整结束查询点
}
}
}
return currentStartId;
}
public void setStartId(String startId) {
this.startId = startId;
}
/**
* 根据查询条件,推导出当前查询结束点
* 注意:翻页,此方法挺重要!!!!
* @return
*/
public String getCurrentEndId() {
String currentEndId = null;
if (StringUtils.isEmpty(endTime)) {
//为空,设置结束值:取当前时间
endTime = new Date();
}
currentEndId = nodeId + "." + endTime.getTime() / 1000;
// 根据查询点startId和翻页方式调整当前查询结束点
if (!StringUtils.isEmpty(startId)) {
if (pageUpStatus) {
// 向前翻页 (跟向后翻页刚好相反)
if (NodeReportInfoPageRequest.ASC_ORDER.equals(orderBy)) {
currentEndId = startId;
} else {
}
} else {
// 向后翻页
if (NodeReportInfoPageRequest.ASC_ORDER.equals(orderBy)) {
// 升序;不需要调整结束查询点
} else {
// 降序,不需要调整起始查询点;要调整结束查询点
currentEndId = startId;
}
}
}
return currentEndId;
}
public boolean isPageUpStatus() {
return pageUpStatus;
}
public void setPageUpStatus(boolean pageUpStatus) {
this.pageUpStatus = pageUpStatus;
}
public String getOrderBy() {
return orderBy;
}
public void setOrderBy(String orderBy) {
this.orderBy = orderBy;
}
public String getCollectionName() {
return collectionName;
}
public void setCollectionName(String collectionName) {
this.collectionName = collectionName;
}
public Date getStartTime() {
return startTime;
}
public void setStartTime(Date startTime) {
this.startTime = startTime;
}
public Date getEndTime() {
return endTime;
}
public void setEndTime(Date endTime) {
this.endTime = endTime;
}
public String getStartId() {
return startId;
}
@Override
public String toString() {
return "NodeReportInfoPageRequest{" +
"nodeId='" + nodeId + '\'' +
", startTime=" + startTime +
", endTime=" + endTime +
", pageSize=" + pageSize +
", startId='" + startId + '\'' +
", pageUpStatus=" + pageUpStatus +
", collectionName='" + collectionName + '\'' +
", orderBy='" + orderBy + '\'' +
'}';
}
}
具体可详见项目
链接: 项目 github地址: https://github.com/wusong1994/springBoot-mongoDB-queryOptimized-demo/tree/main/queryOptimized-demo1