目录
一:ES存储设计
(1)新建索引设置映射
(2)开启索引(否则会报index_closed_exception异常)
(3)查看所有索引
(4)查看索引映射
(5)测试插入数据
二:依赖
三:ES通用工具类
一:ES存储设计
(1)新建索引设置映射
接口:127.0.0.1:9200/question_answer(6.6版本es)
127.0.0.1:9200/question_answer?include_type_name=true(7.13版本es)
说明:question_answer 为自定义的索引index(index相当于mysql中的数据库);
question_answer_type 为自定义的索引下的type(type相当于mysql中的数据库的表);
properties中为存储字段(相当于mysql表中的字段)。
{
"settings":{
"index":{
"number_of_shards":"5",
"number_of_replicas":"1"
}
},
"mappings":{
"question_answer_type":{
"properties":{
"id":{
"type":"text",
"fields":{
"keyword":{
"type":"keyword",
"ignore_above":256
}
}
},
"title":{
"type":"text",
"fields":{
"keyword":{
"type":"keyword",
"ignore_above":256
}
}
},
"content":{
"type":"text",
"fields":{
"keyword":{
"type":"keyword",
"ignore_above":256
}
}
},
"createTime":{
"type":"date",
"format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
},
"updateTime":{
"type":"date",
"format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
},
"createUser":{
"type":"long"
},
"deleteFlag":{
"type":"boolean"
}
}
}
}
}
(2)开启索引(否则会报index_closed_exception异常)
POST /question_answer/_open
(3)查看所有索引
GET _cat/indices?v&pretty
(4)查看索引映射
GET /question_answer/_mapping?pretty
(5)测试插入数据
接口:127.0.0.1:9200/question_answer/dms_question_answer_type/
请求方式:P0ST
{
"id":"1111111",
"title":"我是一个标题",
"content":"我是一个内容",
"createTime":"2020-10-01T10:00:00.000+0800",
"updateTime":"2020-10-01T10:00:00.000+0800",
"createUser":10,
"deleteFlag":false
}
二:依赖
<dependency>
<groupId>com.bbossgroups.plugins</groupId>
<artifactId>bboss-elasticsearch-rest-jdbc</artifactId>
<version>6.3.0</version>
</dependency>
<dependency>
<groupId>com.bbossgroups.plugins</groupId>
<artifactId>bboss-elasticsearch-spring-boot-starter</artifactId>
<version>6.3.0</version>
</dependency>
三:ES通用工具类
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.http.HttpHost;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.*;
@Slf4j
@Component
public class ESUtil {
@Value("${spring.elasticsearch.bboss.elasticsearch.rest.hostNames}")
private String ip;
private HttpHost[] getHttpHosts(String ip) {
log.info("ip 地址为{}",ip);
String[] ips = {ip};
HttpHost[] httpHosts = new HttpHost[ips.length];
for (int i = 0; i < ips.length; i++) {
httpHosts[i] = HttpHost.create(ips[i]);
}
return httpHosts;
}
/**
* 实时保存数据
* @param jsonObject 要保存的数据
* @param index 索引,类似数据库
* @param type type,类似于数据库的表
* @param id id 为null时es会随机生成
* @return
* @throws IOException
*/
public String addDataByRealTime(JSONObject jsonObject, String index, String type, String id) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
//创建请求
IndexRequest request = new IndexRequest(index, type);
//规则 put /index/type/id
request.id(id);
//保证数据实时更新
request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
request.timeout(TimeValue.timeValueSeconds(1));
//将数据放入请求 json
IndexRequest source = request.source(jsonObject, XContentType.JSON);
//客户端发送请求
IndexResponse response = restHighLevelClient.index(request, RequestOptions.DEFAULT);
log.info("添加数据成功 索引为: {},,type为: {},response 状态: {}, id为: {},结果为{}", index, type, response.status().getStatus(), response.getId(),response);
return response.getId();
}
/**
* 实时自动添加 uuid,并保存数据
*
* @param jsonObject 保存的数据
* @param index 索引(数据库)
* @param type type(数据库的表)
* @return
* @throws IOException
*/
public String addData(JSONObject jsonObject, String index, String type) throws IOException {
return addDataByRealTime(jsonObject, index, type, UUID.randomUUID().toString().replaceAll("-", "").toUpperCase());
}
/**
* 通过ID删除数据
*
* @param index 索引,类似数据库
* @param id 数据ID
*/
public void deleteDataById(String index, String type, String id) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
//删除请求
DeleteRequest request = new DeleteRequest(index, type, id);
//执行客户端请求
DeleteResponse result = restHighLevelClient.delete(request, RequestOptions.DEFAULT);
log.info("索引为: {},type为: {}, id为: {},结果为{},删除数据成功", index, type, id,result);
}
/**
* @param object 要增加的数据
* @param index 索引,类似数据库
* @param type type,类似于数据库的表
* @param id 数据id
* @return
* @throws IOException
*/
public String updateDataById(Object object, String index, String type, String id) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
//更新请求
UpdateRequest update = new UpdateRequest(index, type, id);
update.timeout("1s");
update.doc(JSON.toJSONString(object), XContentType.JSON);
//执行更新请求
UpdateResponse result = restHighLevelClient.update(update, RequestOptions.DEFAULT);
log.info("索引为: {},type 为{}, id为: {},结果为{}, 更新数据成功", index, type,id,result);
return result.getId();
}
/**
* 实时通过ID 更新数据
*
* @param object 要增加的数据
* @param index 索引,类似数据库
* @param type type,类似于数据库的表
* @param id 数据ID
* @throws IOException
*/
public String updateDataByIdByRealTime(Object object, String index, String type, String id) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
//更新请求
UpdateRequest update = new UpdateRequest(index, type, id);
//保证数据实时更新
update.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
update.timeout("1s");
update.doc(JSON.toJSONString(object), XContentType.JSON);
//执行更新请求
UpdateResponse result = restHighLevelClient.update(update, RequestOptions.DEFAULT);
log.info("索引为: {},type 为{} ,id为: {},结果为{}, 更新数据成功", index, type,id,result);
return result.getId();
}
/**
* 通过ID获取数据
*
* @param index 索引,类似数据库
* @param type type,类似于数据库的表
* @param id 数据ID
* @return
* @throws IOException
*/
public Map<String, Object> searchDataById(String index, String type, String id) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
GetRequest request = new GetRequest(index, type, id);
GetResponse result = restHighLevelClient.get(request, RequestOptions.DEFAULT);
log.info("索引为: {},type 为{} ,id为: {},结果为{}, 更新数据成功", index, type,id,result);
Map<String, Object> map = result.getSource();
//为返回的数据添加id
map.put("id", result.getId());
return map;
}
/**
* 通过ID判断内容是否存在
*
* @param index 索引,类似数据库
* @param type type,类似于数据库的表
* @param id 数据ID
* @return
* @throws IOException
*/
public boolean existsById(String index, String type, String id) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
GetRequest request = new GetRequest(index, type, id);
//不获取返回的_source的上下文
request.fetchSourceContext(new FetchSourceContext(false));
request.storedFields("_none_");
return restHighLevelClient.exists(request, RequestOptions.DEFAULT);
}
/**
* 获取低水平客户端
*
* @return
*/
public RestClient getLowLevelClient() {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
return restHighLevelClient.getLowLevelClient();
}
/**
* 将返回的数据进行处理
*
* @param searchResponse
*/
public List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse) {
//解析结果
List<Map<String, Object>> list = new ArrayList<>();
for (SearchHit hit : searchResponse.getHits().getHits()) {
list.add(hit.getSourceAsMap());
}
return list;
}
/**
* 列表查询
*
* @param index
* @param builder
* @return
* @throws IOException
*/
public List<Map<String, Object>> searchListData(String index, SearchSourceBuilder builder) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
SearchRequest request = new SearchRequest(index);
request.source(builder);
log.info("searchListData es query index 为{},查询语句为{}", index,builder);
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
if (response.status().getStatus() == 200) {
// 解析对象
return setSearchResponse(response);
}
return null;
}
/**
* 列表查询,并处理高亮查询结果
*
* @param searchResponse
*/
public List<Map<String, Object>> setSearchResponseByHighLevel(SearchResponse searchResponse, List<String> fields) {
//解析结果
List<Map<String, Object>> list = new ArrayList<>();
for (SearchHit hit : searchResponse.getHits().getHits()) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();//原来的结果
//解析高亮的字段并处理
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
fields.stream().forEach(field -> {
HighlightField content = highlightFields.get(field);
//将原来的字段替换为高亮字段即可
if (content != null) {
Text[] fragments = content.fragments();
String newTitle = "";
for (Text text : fragments) {
newTitle += text;
}
sourceAsMap.put(field, newTitle);//替换掉原来的内容
}
});
list.add(sourceAsMap);
}
return list;
}
/**
* 列表查询,返回部分高亮查询结果
*
* @param index
* @param builder
* @param fields 需要进行高亮处理的字段集合
* @return
* @throws IOException
*/
public List<Map<String, Object>> searchListDataByHighLevel(String index, SearchSourceBuilder builder, List<String> fields) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
SearchRequest request = new SearchRequest(index);
if (CollectionUtils.isNotEmpty(fields)) {
HighlightBuilder highlightBuilder = new HighlightBuilder();
fields.stream().forEach(field -> {
//高亮的字段显示处理,fragmentSize,代表高亮后显示的长度,numOfFragments表示高亮片段,一篇文章内容也许有很多处被高亮,此处代表显示适配度最高的内容。
//noMatchSize是搜索出来字段已显示出高亮情况,可是其他字段并没有任何显示,设置这个属性就可以显示出来
highlightBuilder.field(field).fragmentSize(50).numOfFragments(2).noMatchSize(100);
});
highlightBuilder.requireFieldMatch(false);//是否多个字段都高亮
highlightBuilder.preTags("<span style='color:red'>");//前缀后缀
highlightBuilder.postTags("</span>");
builder.highlighter(highlightBuilder);
}
request.source(builder);
log.info("searchListDataByHighLevel es 查询 index为{},查询语句为{},高亮字段为{}", index,builder,fields);
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
if (response.status().getStatus() == 200) {
// 解析对象
return setSearchResponseByHighLevel(response, fields);
}
return null;
}
/**
* 列表查询,返回高亮查询结果的完整内容
*
* @param index
* @param builder
* @param fields 需要进行高亮处理的字段集合
* @return
* @throws IOException
*/
public List<Map<String, Object>> searchListDataByHighLevelForAll(String index, SearchSourceBuilder builder, List<String> fields) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
SearchRequest request = new SearchRequest(index);
if (CollectionUtils.isNotEmpty(fields)) {
HighlightBuilder highlightBuilder = new HighlightBuilder();
fields.stream().forEach(field -> {
highlightBuilder.field(field);//高亮的字段显示处理
});
//为了避免因为高亮部分导致后期部分字段丢失,所以需要设置number_of_fragments=0,即可返回完整内容
highlightBuilder.numOfFragments(0);
highlightBuilder.requireFieldMatch(false);//是否多个字段都高亮
highlightBuilder.preTags("<span style='color:red'>");//前缀后缀
highlightBuilder.postTags("</span>");
builder.highlighter(highlightBuilder);
}
request.source(builder);
log.info("searchListDataByHighLevel es 查询 index为{},查询语句为{},高亮字段为{}", index,builder,fields);
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
if (response.status().getStatus() == 200) {
// 解析对象
return setSearchResponseByHighLevel(response, fields);
}
return null;
}
/**
* 获取总条数
*
* @param index
* @param builder
* @return
* @throws IOException
*/
public Long searchCounte(String index, SearchSourceBuilder builder) throws IOException {
//建立连接
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(getHttpHosts(ip)));
CountRequest countRequest = new CountRequest(index);
countRequest.source(builder);
log.info("searchCounte es 查询,index为{},语句为 {}",index, builder);
CountResponse response = restHighLevelClient.count(countRequest, RequestOptions.DEFAULT);
if (response.status().getStatus() == 200) {
// 解析对象
return response.getCount();
}
return null;
}
}