说明
maven依赖
官方客户端 https://www.elastic.co/guide/en/elasticsearch/client/java-rest/6.4/index.html
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>6.5.0</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.5.0</version>
<exclusions>
<exclusion>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
</exclusion>
<!--此处要排除掉自带的，这个自带的版本低，会报错-->
<exclusion>
<artifactId>elasticsearch</artifactId>
<groupId>org.elasticsearch</groupId>
</exclusion>
</exclusions>
</dependency>
Util类
配置类
package com.crb.ocms.product.domain.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* @Project crb-product-domain
* @PackageName com.crb.ocms.product.admin.config
* @ClassName ESConfiguration
* @Author liqiang
* @Date 2019/3/6 5:52 PM
* @Description 用于加载es的相关配置
*/
@ConfigurationProperties(prefix = "esconfig")
@Data
public class ESConfiguration {
/**
* index别名
*/
private String aliasName;
/**
* 索引名字
*/
private String indexName;
/**
* 全量索引的最大处理线程大小
*/
private int threadSize;
/**
* 全量索引并行执行每个线程每次执行数据大小
*/
private int treadDataSize;
/**
* typename
*/
private String typeName;
/**
* ESURL
*/
private String esUrl;
/**
* es ip
*/
private String host;
/**
* es端口
*/
private Integer port;
/***
* 用于标识是否正在处理的rediskey
*/
private String redissKey;
/**
*mappingJson
*/
private String mappingJson;
}
util工具类
package com.crb.ocms.product.service.impl;
import com.crb.ocms.product.domain.config.ESConfiguration;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.replication.ReplicationResponse;
import org.elasticsearch.client.*;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestBuilders;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;
/**
* @Project crb-product-service
* @PackageName com.crb.ocms.demo.service.test.impl
* @ClassName ESHLRestUtil
* @Author liqiang
* @Date 2019/2/25 13:45
* @Description es工具类
*/
@Slf4j
@Component
@EnableConfigurationProperties(ESConfiguration.class)
public class ESHLRestUtil {
private RestHighLevelClient client = null;
public RestHighLevelClient getClient() {
return client;
}
public void setClient(RestHighLevelClient client) {
this.client = client;
}
ESConfiguration esConfiguration;
@Autowired
public ESHLRestUtil(ESConfiguration configuration) {
client = new SimpleRestHighLevelClient(RestClient.builder(new HttpHost(configuration.getHost(), configuration.getPort(), "http")));
this.esConfiguration=configuration;
}
public SimpleRestHighLevelClient getSimpleClient(){
return (SimpleRestHighLevelClient)client;
}
/**
*根据indexname获得index名字 支持通配符匹配
* @param indexName
* @return
*/
public String [] getIndexNames(String indexName) {
GetIndexResponse getIndexResponse= null;
try {
getIndexResponse = getIndexInfo(indexName);
} catch (IOException e) {
e.printStackTrace();
}
return getIndexResponse!=null?getIndexResponse.getIndices():null;
}
/**
* 根据index名字获得index信息
* @param indexName
* @return
*/
public GetIndexResponse getIndexInfo(String indexName) throws IOException {
//GetIndexRequest request = new GetIndexRequest().indices(indexName);
// try {
// GetIndexResponse getIndexResponse =client.indices().get(request, RequestOptions.DEFAULT);
// return getIndexResponse;
// } catch (IOException e) {
// e.printStackTrace();
// }
//master_timeout
/**
* 因为现在是api使用6.5 线上是6.24 master_timeout 使用以下方式 替换掉
*/
GetIndexRequest request = new GetIndexRequest().indices(indexName);
String[] indices = request.indices() == null ? Strings.EMPTY_ARRAY : request.indices();
String endpoint =indexName;
Request httpequest = new Request("GET", endpoint);
httpequest.addParameter("ignore_unavailable", "false");
httpequest.addParameter("expand_wildcards", "open");
httpequest.addParameter("allow_no_indices", "true");
org.apache.http.HttpEntity entity = new NStringEntity("", ContentType.APPLICATION_JSON);
httpequest.setEntity(entity);
Response httpResponse= getClient().getLowLevelClient().performRequest(httpequest);
GetIndexResponse getIndexResponse=getSimpleClient().parseGetIndexRespons(httpResponse);
return getIndexResponse;
}
/**
* 获得最大index名字
* @param names indexName_number 格式
* @return
*/
public Integer getMaxIndexNumber(String []names) {
if(names==null||names.length<=0){
return null;
}
List<Integer> indexNumber=new ArrayList<Integer>();
for (String name:
names) {
String[] arrs=name.split("_");
if(arrs.length<=1){
continue;
}
indexNumber.add(Integer.valueOf(arrs[1]));
}
indexNumber.sort(new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
return o2 - o1;
}
});
return CollectionUtils.isEmpty(indexNumber)?null:indexNumber.get(0);
}
/**
* 验证索引是否存在
*
* @param index 索引名称
* @return
* @throws Exception
*/
public boolean indexExists(String index) throws Exception {
GetIndexRequest request = new GetIndexRequest();
request.indices(index);
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
return exists;
}
/**
* 创建index
*
* @param index
* @param indexType
* @param properties 结构: {name:{type:text}} {age:{type:integer}}
* @return
* @throws Exception
*/
public boolean indexCreate(String index, String indexType,
Map properties) throws Exception {
if (indexExists(index)) {
return true;
}
CreateIndexRequest request = new CreateIndexRequest(index);
request.settings(Settings.builder().put("index.number_of_shards", 3)
.put("index.number_of_replicas", 2));
Map jsonMap = new HashMap<>();
Map mapping = new HashMap<>();
mapping.put("properties", properties);
jsonMap.put(indexType, mapping);
request.mapping(indexType, jsonMap);
CreateIndexResponse createIndexResponse = client.indices().create(
request,RequestOptions.DEFAULT);
boolean acknowledged = createIndexResponse.isAcknowledged();
return acknowledged;
}
/**
* 删除指定索引
* @param indexName
* @return
* @throws IOException
*/
public boolean deleteIndex(String indexName) throws IOException {
DeleteIndexRequest request = new DeleteIndexRequest(indexName);
AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
return deleteIndexResponse.isAcknowledged();
}
/**
* 创建索引
* @param index 索引名字
* @param settiongs settiongs
* @return
* @throws Exception
* @author lqiang
*/
public boolean indexCreate(String index, String settiongs
) throws Exception {
if (indexExists(index)) {
return true;
}
CreateIndexRequest request = new CreateIndexRequest(index);
// request.settings(Settings.builder().put("index.number_of_shards", 3)
// .put("index.number_of_replicas", 2));
request.source(settiongs, XContentType.JSON);
CreateIndexResponse createIndexResponse = client.indices().create(
request,RequestOptions.DEFAULT);
boolean acknowledged = createIndexResponse.isAcknowledged();
return acknowledged;
}
/**
* 创建更新文档
*
* @param index
* @param indexType
* @param documentId
* @param josonStr
* @return
* @throws Exception
*/
public boolean documentCreate(String index, String indexType,
String documentId, String josonStr) throws Exception {
IndexRequest request = new IndexRequest(index, indexType, documentId);
request.source(josonStr, XContentType.JSON);
IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT);
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
|| indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return true;
}
ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return true;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return false;
}
/**
* 创建更新文档
*
* @param index
* @param indexType
* @param documentId
* @param map
* @return
* @throws Exception
*/
public boolean documentCreate(String index, String indexType,
String documentId, Map map) throws Exception {
IndexRequest request = new IndexRequest(index, indexType, documentId);
request.source(map);
IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
|| indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return true;
}
ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return true;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return false;
}
/**
* 批量创建更新文档
*
* @param index
* @param indexType
* @param list 建索引时传入_id作为docuemntId
* @return
* @throws Exception
*/
public boolean documentCreateBulk(String index, String indexType,
List<Map<String, Object>> list) throws Exception {
if (list.size() > 0) {
BulkRequest bulkRequest = new BulkRequest();
for (Map map : list) {
IndexRequest indexRequest;
Object idObj = map.get("_id");
if (idObj != null && StringUtils.isNotBlank(idObj.toString())) {
map.remove("_id");
String documentId = idObj.toString();
indexRequest = new IndexRequest(index, indexType, documentId);
} else {
indexRequest = new IndexRequest(index, indexType);
}
indexRequest.source(map);
bulkRequest.add(indexRequest);
}
BulkResponse bulkResponse = client.bulk(bulkRequest,RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
System.out.println("索引异常信息：" + bulkResponse.buildFailureMessage());
return false;
}
}
return true;
}
/**
* 创建更新文档
*
* @param index
* @param indexType
* @param documentId
* @param routing
* @param map
* @return
* @throws Exception
*/
public boolean documentCreate(String index, String indexType,
String documentId, String routing, Map map) throws Exception {
IndexRequest request = new IndexRequest(index, indexType, documentId);
request.routing(routing);
request.source(map);
IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
|| indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return true;
}
ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return true;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return false;
}
/**
* 创建索引
*
* @param index
* @param indexType
* @param josonStr
* @return
* @throws Exception
*/
public String documentCreate(String index, String indexType, String josonStr)
throws Exception {
IndexRequest request = new IndexRequest(index, indexType);
request.source(josonStr, XContentType.JSON);
IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);
String id = indexResponse.getId();
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
|| indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return id;
}
ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return id;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return null;
}
/**
* 创建索引
*
* @param index
* @param indexType
* @param map
* @return
* @throws Exception
*/
public String documentCreate(String index, String indexType,
Map map) throws Exception {
IndexRequest request = new IndexRequest(index, indexType);
request.source(map);
IndexResponse indexResponse = client.index(request,RequestOptions.DEFAULT);
String id = indexResponse.getId();
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
|| indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return id;
}
ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return id;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return null;
}
public boolean documentDelete(String index, String indexType,
String documentId) throws Exception {
DeleteRequest request = new DeleteRequest(index, indexType, documentId);
DeleteResponse deleteResponse = client.delete(request,RequestOptions.DEFAULT);
if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
return true;
}
ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo();
if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
return true;
}
if (shardInfo.getFailed() > 0) {
for (ReplicationResponse.ShardInfo.Failure failure : shardInfo
.getFailures()) {
throw new Exception(failure.reason());
}
}
return false;
}
/**
* 为指定index指定别名
* @param indexName index名字
* @param aliasesName 别名名字
* @return
*/
public boolean aliases(String indexName,String aliasesName){
IndicesAliasesRequest request=new IndicesAliasesRequest();
IndicesAliasesRequest.AliasActions aliasAction =
new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
.index(indexName)
.alias(aliasesName);
request.addAliasAction(aliasAction);
try {
AcknowledgedResponse indicesAliasesResponse =
getClient().indices().updateAliases(request, RequestOptions.DEFAULT);
return indicesAliasesResponse.isAcknowledged();
} catch (IOException e) {
log.info("指定别名失败");
return false;
}
}
/**
* 指定别名 并删除oldIndex的别名
* @param oldindex
* @param newIndex
* @param aliasesName
* @return
*/
public boolean aliases(String oldindex, String newIndex,String aliasesName) {
IndicesAliasesRequest request=new IndicesAliasesRequest();
//旧的删除
IndicesAliasesRequest.AliasActions aliasAction =
new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.REMOVE)
.index(oldindex)
.alias(aliasesName);
request.addAliasAction(aliasAction);
//新的绑定
aliasAction =
new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
.index(newIndex)
.alias(aliasesName);
request.addAliasAction(aliasAction);
try {
AcknowledgedResponse indicesAliasesResponse =
getClient().indices().updateAliases(request, RequestOptions.DEFAULT);
return indicesAliasesResponse.isAcknowledged();
} catch (IOException e) {
return false;
}
}
/**
* 根据id获得文档信息
* @param index indexName
* @param type typeName
* @param id id
* @return
* @throws IOException
*/
public GetResponse getDoucmentById(String index, String type, String id) throws IOException {
GetRequest getRequest = new GetRequest(
index,//索引
type,//类型
id);//文档ID
GetResponse getResponse = client.get(getRequest,RequestOptions.DEFAULT);
return getResponse;
}
/**
* 简单的单表根据条件进行查询
*
* @param index index
* @param type type
* @param parameters 参数
* @return
*/
public List<String> queryByMatch(String index, String type, Map<String, String> parameters) {
List<String> jsons=new ArrayList<String>();
try {
SearchRequest searchRequest=new SearchRequest();
BoolQueryBuilder booleanQueryBuilder= QueryBuilders.boolQuery();
SearchSourceBuilder searchSourceBuilder=new SearchSourceBuilder();
for (String key :
parameters.keySet()) {
booleanQueryBuilder.must().add(QueryBuilders.termQuery(key,parameters.get(key)));
}
searchSourceBuilder.query(booleanQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse response = client.search(searchRequest,RequestOptions.DEFAULT);
if(response.getHits().getHits()!=null){
for (SearchHit searchHit:
response.getHits().getHits() ) {
jsons.add(searchHit.getSourceAsString());
}
}
} catch (IOException e) {
e.printStackTrace();
}
return jsons;
}
/**
* 只包含字母
*
* @return 验证成功返回true，验证失败返回false
*/
public static boolean checkLetter(String cardNum) {
String regex = "^[A-Za-z]+$";
return Pattern.matches(regex, cardNum);
}
/**
* 验证中文
*
* @param chinese 中文字符
* @return 验证成功返回true，验证失败返回false
*/
public static boolean checkChinese(String chinese) {
String regex = "^[\u4E00-\u9FA5]+$";
return Pattern.matches(regex, chinese);
}
/**
* Description:提示词，支持中文、拼音、首字母等
* <p>
* 1、检测搜索词是中文还是拼音
* 2、若是中文，直接按照name字段提示
* 3、若是拼音（拼音+汉字），先按照name.keyword_pinyin获取，若是无结果按照首字母name.keyword_first_py获取
* @param index
* @param type
* @param field 提示字段名字
* @param text 文本
* @return
* @author liqiang
*/
public Set<String> getSuggestWord(String index, String type, String field, String text) throws IOException {
String postField=field;
if (checkLetter(text)) {
postField = field + ".keyword_pinyin";
} else if (checkChinese(text)) {
postField = field;
} else {
postField = field + ".keyword_pinyin";
}
Set<String> suggestTexts= postSuggestWord(index,type,postField,text);
if(org.springframework.util.CollectionUtils.isEmpty(suggestTexts)){
return postSuggestWord(index,type,field+".keyword_first_py",text);
}
return suggestTexts;
}
/**
* Description:提示词，支持中文、拼音、首字母等
* @param index
* @param type
* @param field 提示字段名字
* @param text 文本
* @return
* @author liqiang
*/
private Set<String> postSuggestWord(String index, String type, String field, String text) throws IOException {
// 1、创建search请求
SearchRequest searchRequest = new SearchRequest(index);
searchRequest.types(type);
// 2、用SearchSourceBuilder来构造查询请求体 ,请仔细查看它的方法，构造各种查询的方法都在这。
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.fetchSource(false);//不返回_source数据
sourceBuilder.size(0);//忽略hits
//做查询建议
//词项建议
SuggestionBuilder termSuggestionBuilder =
SuggestBuilders.completionSuggestion(field).text(text);
SuggestBuilder suggestBuilder = new SuggestBuilder();
suggestBuilder.addSuggestion("suggest_productName", termSuggestionBuilder);
sourceBuilder.suggest(suggestBuilder);
searchRequest.source(sourceBuilder);
Set<String> suggestTextList = new HashSet<String>();
//3、发送请求
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
//4、处理响应
//搜索结果状态信息
if (RestStatus.OK.equals(searchResponse.status())) {
// 获取建议结果
Suggest suggest = searchResponse.getSuggest();
CompletionSuggestion termSuggestion = suggest.getSuggestion("suggest_productName");
for (CompletionSuggestion.Entry entry : termSuggestion.getEntries()) {
for (CompletionSuggestion.Entry.Option option : entry) {
suggestTextList.add(option.getText().string());
}
}
}
return suggestTextList;
}
/**
* @Project demo
* @PackageName com.crb.ocms.product.service.impl
* @ClassName SimpleRestHighLevelClient
* @Author qiang.li
* @Date 2019/4/2 10:45 AM
* @Description es版本降级导致的高级api会生成部分属性 低版本不能识别 将解析结果的方法暴露在外面
*/
public class SimpleRestHighLevelClient extends RestHighLevelClient {
public SimpleRestHighLevelClient(RestClientBuilder restClientBuilder) {
super(restClientBuilder);
}
/**
* 父类解析的解析响应内容是受保护的 所以定义一个继承
* 解决版本不一致的 自定义请求 解析
* @param response
* @param entityParser
* @param <Req>
* @param <Resp>
* @return
* @throws IOException
*/
public <Req extends Validatable, Resp> Resp parseEntity(Response response,CheckedFunction<XContentParser, Resp, IOException> entityParser)throws IOException {
return this.parseEntity(response.getEntity(),entityParser);
}
/**
* 解析获得idnex的响应
* @param response
* @return
* @throws IOException
*/
public GetIndexResponse parseGetIndexRespons(Response response) throws IOException {
return parseEntity(response,GetIndexResponse::fromXContent);
}
/**
* 解析查询响应
* @param response
* @return
* @throws IOException
*/
public SearchResponse parseSearchResponse(Response response) throws IOException {
return parseEntity(response,SearchResponse::fromXContent);
}
/**
* 解析索引迁移响应
* @param response
* @return
* @throws IOException
*/
public BulkByScrollResponse parseBulkByScrollResponse(Response response) throws IOException {
return parseEntity(response,BulkByScrollResponse::fromXContent);
}
}
}
封装的处理器
工作原理
将指定条件的数据迁移到新索引,然后再新索引上面进行导入（多线程并行导入）,导入完毕删除老索引和别名 然后为新索引绑定别名 实现不停机更新
抽象接口
package com.crb.ocms.product.serviceTool;
import org.elasticsearch.index.reindex.ReindexRequest;
import java.io.IOException;
import java.util.concurrent.Future;
/**
* @Project crb-product-service
* @PackageName com.crb.ocms.product.serviceTool
* @ClassName EsImportService
* @Author liqiang
* @Date 2019/3/29 1:28 PM
* @Description es导入抽象接口
*/
public interface ESImportService {
/**
* 获得索引迁移条件
* @param reindexRequest
* @return
* @throws IOException
*/
public boolean reindex(ReindexRequest reindexRequest) throws IOException;
/**
* 异步导入
* @return
*/
public Future<Boolean> importAllAsyn();
}
抽象的处理器
使用模板模式将通用代码抽出来
package com.crb.ocms.product.serviceTool;
import com.crb.ocms.product.domain.config.ESConfiguration;
import com.crb.ocms.product.domain.entity.MdProduct;
import com.crb.ocms.product.domain.redisskey.MdEsProductRedisKeyEnum;
import com.crb.ocms.product.domain.util.exceptions.OCmsExceptions;
import com.crb.ocms.product.service.impl.ESHLRestUtil;
import com.hazelcast.util.StringUtil;
import lombok.extern.log4j.Log4j2;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.ReindexRequest;
import org.redisson.api.RedissonClient;
import org.springframework.data.domain.PageRequest;
import org.springframework.web.client.RestTemplate;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
/**
* @Project crb-product-service
* @PackageName com.crb.ocms.product.serviceTool
* @ClassName ESAbstrctImpor
* @Author liqiang
* @Date 2019/3/29 1:35 PM
* @Description 抽象的es导入处理器
*/
@Log4j2
public abstract class ESAbstractImport implements ESImportService {
private RestTemplate restTemplate;
private ESConfiguration esConfiguration;
private ESHLRestUtil eshlRestUtil;
private RedissonClient redissonClient;
/**
* 当前index名字
*/
private String templateIndexName;
/**
* 之前的index名字
*/
private String beforeIndexName;
public ESAbstractImport(RestTemplate restTemplate, ESConfiguration esConfiguration,RedissonClient redissonClient){
esConfiguration.setIndexName("cmsproudct");
esConfiguration.setAliasName("cmsproductAliasName");
this.restTemplate=restTemplate;
this.esConfiguration=esConfiguration;
this.eshlRestUtil=new ESHLRestUtil(esConfiguration);
this.redissonClient=redissonClient;
}
/**
* 获得创建索引的mapping抽象方法
* @return
*/
public abstract String getMapping();
/**
* 获得所有迁移requestBody抽象方法
* @return
*/
public abstract ReindexRequest getReindex();
/**
* es索引秦阿姨
* @return
*/
@Override
public boolean reindex(ReindexRequest reindexRequest) throws IOException {
// BulkByScrollResponse bulkByScrollResponse= eshlRestUtil.getClient().reindex(reindexRequest,RequestOptions.DEFAULT);
/**
* 因为现在是api使用6.5 线上是6.24 查询语句多生成zero_terms_query 使用以下方式 替换掉
*/
String endpoint ="/_reindex";
Request httpequest = new Request("POST", endpoint);
BytesRef source = XContentHelper.toXContent(reindexRequest, XContentType.JSON, false).toBytesRef();
org.apache.http.HttpEntity entity = new NStringEntity(new String(source.bytes).replaceAll("\"zero_terms_query\":\"NONE\","," "), ContentType.APPLICATION_JSON);
httpequest.setEntity(entity);
Response httpResponse=eshlRestUtil.getClient().getLowLevelClient().performRequest(httpequest);
BulkByScrollResponse getIndexResponse=eshlRestUtil.getSimpleClient().parseBulkByScrollResponse(httpResponse);
return true;
}
/**
* 导入数据 根据设置的线程大小根据数据大小算出线程数量 通过多线程并行处理
* @return
*/
public boolean importAll() throws IOException {
log.info("正在产品全量导入2:"+esConfiguration.getEsUrl());
if (redissonClient.getMap(esConfiguration.getRedissKey()).isExists()) {
throw new OCmsExceptions("全量索引正在导入中....");
}
String []indexNames=eshlRestUtil.getIndexNames(esConfiguration.getIndexName()+"*");
if (indexNames!=null&&indexNames.length>0) {
Integer number= eshlRestUtil.getMaxIndexNumber(indexNames);
beforeIndexName=number==null?esConfiguration.getIndexName():esConfiguration.getIndexName()+"_"+number;
templateIndexName=esConfiguration.getIndexName()+"_"+(number==null?1:(number+1));
} else {
templateIndexName = esConfiguration.getIndexName();
}
boolean isSuccess = true;
long count = getCount();
//根据线程处理大小获得页码
int index = (new Double(Math.ceil(count*1.0 / esConfiguration.getTreadDataSize()))).intValue();
if (index <= 0 && count > 0) {
index = 1;
}
//获得最大处理线程大小
int threadSize = index > esConfiguration.getThreadSize() ? esConfiguration.getThreadSize() : index;
//导入处理器
ESAbstractImport.ProcessImportIndex processImportIndex = new ESAbstractImport.ProcessImportIndex(esConfiguration.getTreadDataSize(), index, templateIndexName);
ThreadPoolExecutor executorService=null;
try {
//第一次导入直接创建索引
if (beforeIndexName == null) {
log.info("开始创建索引");
eshlRestUtil.indexCreate(templateIndexName,getMapping());
log.info("创建索引成功");
} else {
log.info("开始创建索引1");
eshlRestUtil.indexCreate(templateIndexName,getMapping());
log.info("创建索引成功1");
log.info("开始创建索引2");
//非第一次导入将子集数据迁移进来
reindex(getReindex());
log.info("开始创建索引2");
}
/** 线程池的自定义配置，IO密集型任务. */
executorService = new ThreadPoolExecutor(
// 核心线程数
threadSize,
// 最大线程数
100,
// 存活时间，30s
30,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(150),
Executors.defaultThreadFactory(),
new ThreadPoolExecutor.CallerRunsPolicy());
redissonClient.getMap(esConfiguration.getRedissKey()).put("templateIndexName", templateIndexName);
//统一保存异步处理结果
List<Future<Boolean>> processResult = new ArrayList<Future<Boolean>>();
for (int i = 0; i < threadSize; i++) {
processResult.add(executorService.submit(processImportIndex));
}
for (Future<Boolean> future :
processResult) {
if (!future.get()) {
isSuccess = future.get();
break;
}
}
//表示重建成功
if (isSuccess) {
//表示第一次初始化 指定别名
if (StringUtil.isNullOrEmpty(beforeIndexName)) {
boolean isFail = false;
int i = 0;
do {
isFail = !eshlRestUtil.aliases(templateIndexName,esConfiguration.getAliasName());
if(i>0) {
//休眠
Thread.sleep(i * 1000);
}
i++;
} while (isFail && i < 10);
if (isFail) {
log.info("开始删除索引2");
eshlRestUtil.deleteIndex(templateIndexName);
log.info("开始删除成功");
}
} else {
boolean isFail = false;
int i = 0;
do {
//重新绑定别名 失败重试十次
isFail = !eshlRestUtil.aliases(beforeIndexName,templateIndexName,esConfiguration.getAliasName());
if(i>0) {
//休眠
Thread.sleep(i * 1000);
}
i++;
} while (isFail && i < 10);
if (isFail) {
//删除备份索引
eshlRestUtil.deleteIndex(templateIndexName);
throw new OCmsExceptions("索引重做失败,请重试");
} else {
//删除备份索引
eshlRestUtil.deleteIndex(beforeIndexName);
}
}
} else {
for (Future<Boolean> future :
processResult) {
if (!future.get()) {
//没有重做成功删除创建的新索引
eshlRestUtil.deleteIndex(templateIndexName);
future.isCancelled();//取消其他正在执行的线程
}
}
}
} catch (Exception e) {
String uuid = UUID.randomUUID().toString();
log.error(uuid + "\n" + e.getMessage());
eshlRestUtil.deleteIndex(templateIndexName);
throw new OCmsExceptions("<ES文档导入失败>异常编码" + uuid + "\n" + e.getMessage());
} finally {
if (executorService != null) {
executorService.shutdown();
}
redissonClient.getBucket(esConfiguration.getRedissKey()).delete();
}
return isSuccess;
}
/**
* 异步的导入方法
* @return
*/
@Override
public Future<Boolean> importAllAsyn() {
ExecutorService executorService=new ThreadPoolExecutor(1, 1,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>());
return executorService.submit(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return importAll();
}
});
}
/**
* 抽象的处理导入方法
* @param pageRequest
* @return
* @throws IOException
*/
public abstract boolean processImport(PageRequest pageRequest) throws IOException;
public abstract Long getCount();
/**
* 用于处理批量索引导入
*/
class ProcessImportIndex implements Callable<Boolean> {
//数据总页数
private int pageCount;
//总条数
private int pageSize;
private String indexName;
private AtomicLong currentIndex = new AtomicLong();
//当前处理页数
AtomicInteger currentPage = new AtomicInteger();
public ProcessImportIndex(int pageSize, int pageCount, String indexName) {
this.pageSize = pageSize;
this.pageCount = pageCount;
this.indexName = indexName;
}
/**
* Computes a result, or throws an exception if unable to do so.
*
* @return computed result
* @throws Exception if unable to compute a result
*/
@Override
public Boolean call() throws Exception {
try {
return process();
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
public boolean process() throws IOException {
int page = currentPage.getAndIncrement();
redissonClient.getMap(MdEsProductRedisKeyEnum.IMPORT_INDEX).put("pageCount", pageCount);
while (page < pageCount) {
redissonClient.getMap(MdEsProductRedisKeyEnum.IMPORT_INDEX).put("currentPage", page+1);
processImport(new PageRequest(page, pageSize));
page = currentPage.getAndIncrement();
}
return true;
}
public AtomicInteger getCurrentPage() {
return currentPage;
}
public void setCurrentPage(AtomicInteger currentPage) {
this.currentPage = currentPage;
}
}
public String getTemplateIndexName() {
return templateIndexName;
}
public void setTemplateIndexName(String templateIndexName) {
this.templateIndexName = templateIndexName;
}
public interface BulkCallback {
public void process(MdProduct mdProduct);
}
public RestTemplate getRestTemplate() {
return restTemplate;
}
public void setRestTemplate(RestTemplate restTemplate) {
this.restTemplate = restTemplate;
}
public ESConfiguration getEsConfiguration() {
return esConfiguration;
}
public void setEsConfiguration(ESConfiguration esConfiguration) {
this.esConfiguration = esConfiguration;
}
public ESHLRestUtil getEshlRestUtil() {
return eshlRestUtil;
}
public void setEshlRestUtil(ESHLRestUtil eshlRestUtil) {
this.eshlRestUtil = eshlRestUtil;
}
public RedissonClient getRedissonClient() {
return redissonClient;
}
public void setRedissonClient(RedissonClient redissonClient) {
this.redissonClient = redissonClient;
}
public String getBeforeIndexName() {
return beforeIndexName;
}
public void setBeforeIndexName(String beforeIndexName) {
this.beforeIndexName = beforeIndexName;
}
}
使用例子
package com.crb.ocms.product.serviceTool;
import com.crb.ocms.product.domain.config.ESConfiguration;
import com.crb.ocms.product.domain.entity.MdProduct;
import com.crb.ocms.product.domain.repository.MdProductRepository;
import com.crb.ocms.product.domain.vo.req.FindProductCharacterStocksVo;
import com.crb.ocms.product.domain.vo.resp.ProductCharacterStocksVo;
import com.crb.ocms.product.service.MdProductService;
import com.crb.ocms.product.service.feign.IcProductStoreAccountService;
import com.crb.ocms.product.service.impl.MdESProductServiceImpl;
import lombok.extern.log4j.Log4j2;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.ReindexRequest;
import org.redisson.api.RedissonClient;
import org.springframework.data.domain.Example;
import org.springframework.data.domain.ExampleMatcher;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.web.client.RestTemplate;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
/**
* @Project crb-product-service
* @PackageName com.crb.ocms.product.serviceTool
* @ClassName ProductStockImport
* @Author liqiang
* @Date 2019/3/29 2:15 PM
* @Description 处理库存全量索引导入
*/
@Log4j2
@Deprecated
public class ProductStockImport extends ProductIndexImport {
MdProductRepository mdProductRepository;
IcProductStoreAccountService icProductStoreAccountService;
String regionCode;
public ProductStockImport(RestTemplate restTemplate, ESConfiguration esConfiguration, RedissonClient redissonClient, MdProductRepository mdProductRepository, IcProductStoreAccountService icProductStoreAccountService, String regionCode) {
super(restTemplate, esConfiguration, redissonClient);
this.mdProductRepository=mdProductRepository;
this.regionCode=regionCode;
this.icProductStoreAccountService=icProductStoreAccountService;
}
/**
* 索引迁移条件。将指定条件数据迁移到新的索引
* @return
*/
@Override
public ReindexRequest getReindex() {
ReindexRequest reindexRequest=new ReindexRequest();
reindexRequest.setSourceIndices(getBeforeIndexName());
reindexRequest.setSourceQuery(QueryBuilders.boolQuery().mustNot(QueryBuilders.matchPhraseQuery("info","ic_product_store_account")));
reindexRequest.setDestIndex(getTemplateIndexName());
return reindexRequest;
}
/**
* 已经迁移到经销权 先禁用
* @param pageRequest
* @return
* @throws IOException
*/
@Deprecated
@Override
public boolean processImport(PageRequest pageRequest) throws IOException {
MdProduct mdProduct = new MdProduct();
mdProduct.setRegionCode(regionCode);
ExampleMatcher exampleMatcher = ExampleMatcher.matching().withMatcher("regionCode", ExampleMatcher.GenericPropertyMatchers.exact()).withIgnorePaths("optCounter","createdDate","updatedDate","id");;
Example example=Example.of(mdProduct,exampleMatcher);
Page<MdProduct> result = mdProductRepository.findAll(example,pageRequest);
for (MdProduct md:result){
if (md.getFullPalletSaleFlag()==null){
md.setFullPalletSaleFlag(0);
}
}
List<Long> productIds=result.getContent().stream().map(MdProduct::getMdProductId).collect(Collectors.toList());
FindProductCharacterStocksVo findProductCharacterStocksVo=new FindProductCharacterStocksVo();
findProductCharacterStocksVo.setProductIds(productIds);
findProductCharacterStocksVo.setRegionCode(regionCode);
List<ProductCharacterStocksVo> productCharacterStocksVoList= icProductStoreAccountService.findProductCharacterStocks(findProductCharacterStocksVo);
BulkRequest bulkRequest=new BulkRequest();
for (ProductCharacterStocksVo productCharacterStocksVo:
productCharacterStocksVoList) {
if(productCharacterStocksVo.getCharacterId().longValue()==-1L){
continue;
}
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
builder.field("productId",productCharacterStocksVo.getProductId());
builder.field("factoryId",productCharacterStocksVo.getFactoryId());
builder.field("characterId",productCharacterStocksVo.getCharacterId());
builder.field("sumCount",productCharacterStocksVo.getSumCount());
builder.field("regionCode",regionCode);
builder.field("info").startObject()
.field("name","ic_product_store_account")
.field("parent",productCharacterStocksVo.getProductId()+"_"+regionCode)
.endObject();
builder.endObject();
String indexId="ic_product_store_account_"+productCharacterStocksVo.getProductId()+"_"+productCharacterStocksVo.getCharacterId()+"_"+productCharacterStocksVo.getFactoryId();
IndexRequest indexRequest=new IndexRequest(getTemplateIndexName(),getEsConfiguration().getTypeName(), indexId);
indexRequest.source(builder);
indexRequest.routing(productCharacterStocksVo.getProductId()+"_"+regionCode);
bulkRequest.add(indexRequest);
}
log.info(bulkRequest.getDescription());
getEshlRestUtil().getClient().bulk(bulkRequest, RequestOptions.DEFAULT);
return true;
}
/**
* 数据总条数
* @return
*/
@Override
public Long getCount() {
MdProduct mdProduct = new MdProduct();
mdProduct.setRegionCode(regionCode);
ExampleMatcher exampleMatcher = ExampleMatcher.matching().
withMatcher("regionCode", ExampleMatcher.GenericPropertyMatchers.exact()).withIgnorePaths("optCounter","createdDate","updatedDate","id");
Example example=Example.of(mdProduct,exampleMatcher);
return mdProductRepository.count(example);
}
}
processImport就只用关系查询出指定分页的数据 然后导入到es
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
