目录
1、elasticsearch安装及中文分词配置
2、springboot整合elasticsearch配置
3、elasticsearch公共配置及代码编写
4、保存、同步数据至elasticsearch中
5、elasticsearch相关度查询、排序、高亮显示
6、elasticsearch搜索自动补全
1、elasticsearch安装及中文分词配置
可以在 Past Releases of Elastic Stack Software | Elastic 官网选择对应的版本进行下载。
下载对应版本的中文分词。在Releases · medcl/elasticsearch-analysis-ik · GitHub官网找到对应版本的中文分词器进行下载,本文中采用的是v7.6.1。
在下载解压后的elasticsearch-7.6.1\plugins 文件夹中,新建ik文件夹。并将下载的中文分词解压到此目录下。
2、springboot整合elasticsearch配置
创建springboot项目,引用相应pom文件。
<dependencies>
<!-- springboot项目创建成功后,引入elasticsearch -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.4.4</version>
</dependency>
</dependencies>
在yaml文件中进行配置elasticsearch相关配置
spring:
elasticsearch:
uris: 127.0.0.1:9200
3、elasticsearch公共配置及代码编写
建立公共的elasticsearch服务,目录如下:
BaseDoc代码如下:
/**
* 基础实体类
*/
@Data
public abstract class BaseDoc implements Serializable {
/**
* 主键id
*/
@Id
@Field(type = FieldType.Keyword, store = true)
private String id;
/**
* 创建时间
*/
@Field(type = FieldType.Keyword, store = true)
private String createTime;
/**
* 更新时间
*/
@Field(type = FieldType.Keyword, store = true)
private String updateTime;
@Transient
private String highlightTitle;
@Transient
private Float score;
protected abstract String getUnicode();
}
EscBaseRepository代码如下:
/**
* 基础业务接口
*
* @param <T>
*/
public interface EscBaseRepository<T> extends ElasticsearchRepository<T, String> {
}
EscBaseService代码如下:
/**
* 基础业务接口
*/
public interface EscBaseService<T> {
EscBaseRepository<T> getBaseRepository();
boolean save(T doc);
boolean update(T doc);
boolean saveOrUpdate(T doc);
T getById(String id);
boolean saveOrUpdateBatch(Collection<T> docList);
boolean deleteById(String id);
boolean delete(T doc);
}
EscBaseServiceImpl代码如下:
/**
* 业务封装基础类
*/
public abstract class EscBaseServiceImpl<M extends EscBaseRepository<T>, T extends BaseDoc> implements EscBaseService<T> {
private static String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
@Override
public boolean save(T doc) {
String now = DateFormatUtils.format(new Date(), DATE_FORMAT);
boolean exists = getBaseRepository().existsById(doc.getUnicode());
doc.setId(doc.getUnicode());
if (!exists) {
doc.setCreateTime(now);
doc.setUpdateTime(now);
getBaseRepository().save(doc);
return true;
}
return false;
}
@Override
public boolean update(T doc) {
String now = DateFormatUtils.format(new Date(), DATE_FORMAT);
boolean exists = getBaseRepository().existsById(doc.getUnicode());
doc.setId(doc.getUnicode());
if (exists) {
doc.setUpdateTime(now);
getBaseRepository().save(doc);
return true;
}
return false;
}
@Override
public boolean saveOrUpdate(T doc) {
this.resolveDoc(doc);
getBaseRepository().save(doc);
return true;
}
@Override
public T getById(String id) {
Optional<T> op = getBaseRepository().findById(id);
// if (op.isPresent()) {
// return op.get();
// }
// return null;
return op.orElse(null);
}
@Override
public boolean saveOrUpdateBatch(Collection<T> docList) {
docList.forEach(this::resolveDoc);
getBaseRepository().saveAll(docList);
return true;
}
@Override
public boolean deleteById(String id) {
getBaseRepository().deleteById(id);
return true;
}
@Override
public boolean delete(T doc) {
getBaseRepository().delete(doc);
return true;
}
@SneakyThrows
private void resolveDoc(T doc) {
String now = DateFormatUtils.format(new Date(), DATE_FORMAT);
boolean exist = getBaseRepository().existsById(doc.getUnicode());
if (!exist) {
doc.setCreateTime(now);
}
doc.setId(doc.getUnicode());
doc.setUpdateTime(now);
}
}
ElasticSearchConfig代码如下:
@Configuration
public class ElasticSearchConfig {
@Value("${spring.elasticsearch.uris}")
private String uris;
@Bean
public RestHighLevelClient restHighLevelClient() {
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(
RestClient.builder(
new HttpHost(splitHost()[0].trim(), Integer.valueOf(splitHost()[1].trim()), "http")
)
);
return restHighLevelClient;
}
private String[] splitHost() {
return uris.split(":");
}
}
4、保存、同步数据至elasticsearch中
在具体的业务模块下,创建es所需创建的索引及文档实体类,目录如下:
TestGoodsDoc代码如下:
@Data
@Accessors(chain = true)
@Document(indexName = "index_test_goods")
public class TestGoodsDoc extends BaseDoc implements Serializable {
/**
* spu 编码
*/
@Field(type = FieldType.Keyword, store = true)
private String spuCode;
/**
* sku编码
*/
@Field(type = FieldType.Keyword, store = true)
private String skuCode;
/**
* 商品检索名称 analyzer = "ik_max_word" 做细致的拆分,也就是分词分的更细致,更具体
*/
@Field(type = FieldType.Text, store = true, searchAnalyzer = "ik_smart", analyzer = "ik_max_word")
private String showName;
/**
* 商品主图
*/
@Field(type = FieldType.Keyword, index = false, store = true)
private String mainImageUrl;
/**
* 真正售价
*/
@Field(type = FieldType.Double, store = true)
private BigDecimal sellPrice;
/**
* 划线价格
*/
@Field(type = FieldType.Double, store = true)
private BigDecimal linePrice;
/**
* 上架时间
*/
// @Field(type = FieldType.Integer, store = true)
// private Integer goodsUpTime;
@Field(type = FieldType.Long, store = true)
private Long goodsUpTime;
/**
* 商品样式、颜色 analyzer = "ik_smart" 做粗粒度的分词,不需要分词细化
*/
@Field(type = FieldType.Text, store = true, searchAnalyzer = "ik_smart", analyzer = "ik_smart")
private String tags;
/**
* 手动排序
*/
@Field(type = FieldType.Integer, store = true)
private Integer goodsOrderNum;
/**
* 商品样式个数
*/
@Field(type = FieldType.Integer, store = true)
private Integer totalStyle;
/**
* 是否有库存 1 有 0 没有
*/
@Field(type = FieldType.Integer, store = true)
private Integer goodsStock;
/**
* 活动开始时间 存储时间戳形式
*/
@Field(type = FieldType.Long, store = true)
private Long activityStartTime;
/**
* 活动结束时间 存储时间戳形式
*/
@Field(type = FieldType.Long, store = true)
private Long activityEndTime;
//活动tag标签 通过|;进行分割
@Field(type = FieldType.Keyword, store = true)
private String goodsActTag;
/**
* 一级商品分类,多个用;分隔 存储的是数字字符串,不需要进行中文分词
*/
@Field(type = FieldType.Text, store = true)
private String firstLevel;
/**
* 二级商品分类,多个用;分隔 存储的是数字字符串,不需要进行中文分词
*/
@Field(type = FieldType.Text, store = true)
private String secondLevel;
/**
* 商品分类名称
*/
@Field(type = FieldType.Text, store = true, searchAnalyzer = "ik_smart", analyzer = "ik_smart")
private String categoryName;
/**
* 自动补全
*/
@CompletionField(searchAnalyzer = "ik_smart", analyzer = "ik_max_word")
private Completion suggest;
@Override
protected String getUnicode() {
return this.skuCode;
}
}
TestGoodsRepository代码如下:
public interface TestGoodsRepository extends EscBaseRepository<TestGoodsDoc> {
}
编写相应业务代码,目录结构如下
SearchController代码如下:
/**
* 全量数据同步至ES中
*/
@GetMapping("/save")
public ResultMsg<Void> save() {
List<TestGoodsDoc> list = searchService.dataList();
if (!CollectionUtils.isEmpty(list)) {
searchService.saveOrUpdateBatch(list);
return ResultMsg.success();
}
return ResultMsg.error("全量同步数据至ES异常");
}
SearchService代码如下:
public interface SearchService extends EscBaseService<TestGoodsDoc> {
/**
* 全量同步数据至es业务数据处理
*/
List<TestGoodsDoc> dataList();
}
SearchService代码如下:
@Service
public class SearchServiceImpl extends EscBaseServiceImpl<EscBaseRepository<TestGoodsDoc>, TestGoodsDoc> implements SearchService {
@Resource
private TestGoodsRepository testGoodsRepository;
@Override
public EscBaseRepository<TestGoodsDoc> getBaseRepository() {
return this.testGoodsRepository;
}
@Override
public List<TestGoodsDoc> dataList() {
// 根据自己的业务进行数据库操作查询,将数据库中所需要的信息存储至elasticsearch中
List<TestGoodsDoc> list = new LinkedList<>();
TestGoodsDoc testGoodsDoc1 = new TestGoodsDoc();
testGoodsDoc1.setSkuCode("test001").setSpuCode("spu001").setGoodsStock(1).setGoodsOrderNum(1);
list.add(testGoodsDoc1);
TestGoodsDoc testGoodsDoc2 = new TestGoodsDoc();
testGoodsDoc1.setSkuCode("test002").setSpuCode("spu002").setGoodsStock(1).setGoodsOrderNum(1);
list.add(testGoodsDoc2);
TestGoodsDoc testGoodsDoc3 = new TestGoodsDoc();
testGoodsDoc1.setSkuCode("test003").setSpuCode("spu003").setGoodsStock(1).setGoodsOrderNum(1);
list.add(testGoodsDoc3);
return list;
}
}
5、elasticsearch相关度查询、排序、高亮显示
在elasticsearch包中,新建support包,并编写相应公共查询方法。
代码如下:
@Component
public class EsSearchSupport {
@Resource
private RestHighLevelClient client;
@SneakyThrows
public SearchHits search(HighlightBuilder highlightBuilder, FunctionScoreQueryBuilder functionScoreQueryBuilder, List<SortBuilder<?>> sortBuilder,
Integer page, Integer size, String... indices) {
// 构造SearchSourceBuilder
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(functionScoreQueryBuilder);
searchSourceBuilder.sort(sortBuilder);
searchSourceBuilder.highlighter(highlightBuilder);
searchSourceBuilder.from((page - 1) * size);
searchSourceBuilder.size(size);
// 构造SearchRequest
SearchRequest searchRequest = new SearchRequest(indices);
searchRequest.source(searchSourceBuilder);
// 执行请求
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHits hits = searchResponse.getHits();
// SearchHit[] searchHits = hits.getHits();
// List<T> list = new ArrayList<>();
// for (SearchHit hit : searchHits) {
// T t = JSONUtil.toBean(hit.getSourceAsString(), clazz);
// Map<String, HighlightField> highlightFields = hit.getHighlightFields();
// if (!highlightFields.isEmpty()) {
// HighlightField showName = highlightFields.get("showName");
// if (showName != null) {
// t.setHighlightTitle(Arrays.stream(showName.getFragments()).map(Text::string).collect(Collectors.joining(" ")));
// }
// }
// t.setId(hit.getId()); // 数据的唯一标识
// t.setScore(hit.getScore());
// list.add(t);
// }
// return list;
return hits;
}
/**
* es 自动补全
*
* @param indices 索引名称
* @param fileName 搜索字段
* @param keyword 搜索关键词
* @param size 搜索显示条数
* @param skip 是否跳过重复值
* @param name 自动补全搜索名称
*/
public Suggest getSearchSuggest(String fileName, String keyword, Integer size, Boolean skip, String name, String... indices) {
if (size == null || size <= 0) {
size = 5;
}
SearchRequest searchRequest = new SearchRequest(indices);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
SuggestionBuilder termSuggestionBuilder = SuggestBuilders.completionSuggestion(fileName).prefix(keyword).skipDuplicates(skip).size(size);
SuggestBuilder suggestBuilder = new SuggestBuilder();
suggestBuilder.addSuggestion(name, termSuggestionBuilder);
searchSourceBuilder.suggest(suggestBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse response = null;
try {
response = client.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
Suggest suggest = response.getSuggest();
return suggest;
}
@SneakyThrows
public long count(QueryBuilder queryBuilder, String... indices) {
// 构造请求
CountRequest countRequest = new CountRequest(indices);
countRequest.query(queryBuilder);
// 执行请求
CountResponse countResponse = client.count(countRequest, RequestOptions.DEFAULT);
long count = countResponse.getCount();
return count;
}
}
在SearchServiceImpl中编写自己的业务查询,代码如下:
@Override
public PageInfo<TestGoodsDoc> pageListTest(Page<TestGoodsReq> req) {
// 查询条件
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
if (StringUtils.isNotBlank(req.getData().getKeyword())) {
queryBuilder.should(QueryBuilders.matchQuery("showName", req.getData().getKeyword()).boost(5.0f))
.should(QueryBuilders.matchQuery("categoryName", req.getData().getKeyword()).boost(0.5f))
.should(QueryBuilders.termQuery("skuCode", req.getData().getKeyword()));
}
if (StringUtils.isNotBlank(req.getData().getCategory())) {
queryBuilder.should(QueryBuilders.matchQuery("firstLevel", req.getData().getCategory()))
.should(QueryBuilders.matchQuery("secondLevel", req.getData().getCategory()));
}
// 排序条件
List<SortBuilder<?>> sortList = new LinkedList<>();
if (req.getData().getSort() == 1) {
sortList.add(SortBuilders.fieldSort("sellPrice").order(SortOrder.ASC));
}
if (req.getData().getSort() == 2) {
sortList.add(SortBuilders.fieldSort("sellPrice").order(SortOrder.DESC));
}
sortList.add(SortBuilders.fieldSort("_score").order(SortOrder.DESC));
// 相关性查询 根据相关性相关公式计算 值越大,相关性越强,文档被搜索到越靠前。
// FieldValueFactorFunction.Modifier.LN1P 值越大,计算结果越大,相关性越强
// FieldValueFactorFunction.Modifier.RECIPROCAL 值越大,计算结果越小,相关性越弱
// 参与活动商品脚本相关性
Map<String, Object> activityParams = singletonMap("currentDay", LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli());
Script activityScript = new Script(
ScriptType.INLINE,
"painless",
"if(doc['activityStartTime'].value <= params.currentDay && doc['activityEndTime'].value >= params.currentDay){return 1;}else{return 0;}",
activityParams);
ScoreFunctionBuilder scriptActivityTime = ScoreFunctionBuilders.scriptFunction(activityScript).setWeight(1.5f);
// 库存相关性
ScoreFunctionBuilder scoreGoodsStock =
ScoreFunctionBuilders.fieldValueFactorFunction("goodsStock").modifier(FieldValueFactorFunction.Modifier.NONE).factor(DEFAULT_FACTOR).setWeight(5.0f);
// 手动排序相关性
ScoreFunctionBuilder scoreGoodsOrderNum =
ScoreFunctionBuilders.fieldValueFactorFunction("goodsOrderNum").modifier(FieldValueFactorFunction.Modifier.RECIPROCAL).factor(DEFAULT_FACTOR).setWeight(4.0f);
// GaussDecayFunctionBuilder gaussUpdateTime = ScoreFunctionBuilders.gaussDecayFunction("zestUpStatusTime", System.currentTimeMillis(), 1, 2).setWeight(10f);
// 上架时间相关性
// ScoreFunctionBuilder scoreGoodsUpTime =
// ScoreFunctionBuilders.fieldValueFactorFunction("goodsUpTime").modifier(FieldValueFactorFunction.Modifier.RECIPROCAL).factor(DEFAULT_FACTOR);
Map<String, Object> upTimeParams = singletonMap("currentDay", LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli());
Script upTimeScript = new Script(
ScriptType.INLINE,
"painless",
"if(doc['goodsUpTime'].value != 0 && params.currentDay - doc['goodsUpTime'].value > 0){return 1.0 / ((params.currentDay - doc['goodsUpTime'].value) / (1000 * 60 * 60 * 24) + 1);}else{return 0;}",
upTimeParams);
ScoreFunctionBuilder scriptGoodsUpTime = ScoreFunctionBuilders.scriptFunction(upTimeScript).setWeight(0.5f);
FunctionScoreQueryBuilder.FilterFunctionBuilder[] filterFunctionBuilders = new FunctionScoreQueryBuilder.FilterFunctionBuilder[4];
filterFunctionBuilders[0] = new FunctionScoreQueryBuilder.FilterFunctionBuilder(scoreGoodsStock);
filterFunctionBuilders[1] = new FunctionScoreQueryBuilder.FilterFunctionBuilder(scoreGoodsOrderNum);
filterFunctionBuilders[2] = new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptActivityTime);
filterFunctionBuilders[3] = new FunctionScoreQueryBuilder.FilterFunctionBuilder(scriptGoodsUpTime);
FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(queryBuilder, filterFunctionBuilders).scoreMode(FunctionScoreQuery.ScoreMode.SUM).boostMode(CombineFunction.SUM);
// 搜索高亮显示
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.field("showName");
//多个高亮显示设置为true
highlightBuilder.requireFieldMatch(false);
highlightBuilder.preTags("<span style = 'color:red'>");
highlightBuilder.postTags("</span>");
// List<TestGoodsDoc> list = esSearchSupport.search(queryBuilder, sortList, highlightBuilder, req.getPageNum(), req.getPageSize(), EsGoodsDoc.class, "index_zest_goods");
SearchHits hits = esSearchSupport.search(highlightBuilder, functionScoreQueryBuilder, sortList, req.getPageNum(), req.getPageSize(), "index_test_goods");
SearchHit[] searchHits = hits.getHits();
List<TestGoodsDoc> list = new ArrayList<>();
for (SearchHit hit : searchHits) {
TestGoodsDoc t = JSONUtil.toBean(hit.getSourceAsString(), TestGoodsDoc.class);
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
if (!highlightFields.isEmpty()) {
HighlightField showName = highlightFields.get("showName");
if (showName != null) {
t.setHighlightTitle(Arrays.stream(showName.getFragments()).map(Text::string).collect(Collectors.joining(" ")));
}
}
t.setId(hit.getId()); // 数据的唯一标识
t.setScore(hit.getScore());
list.add(t);
}
long count = esSearchSupport.count(queryBuilder, "index_test_goods");
return PageInfo.of(list, count);
}
TestGoodsReq代码如下:
@Data
public class EsGoodsReq {
// es 搜索商品关键词
private String keyword;
// 商品分类
private String category;
// 排序规则 0 推荐(默认)排序 1 价格升序 2价格降序
private Integer sort;
}
6、elasticsearch搜索自动补全
@Override
public List<String> autoCompletion(String keyword, Integer size) {
Suggest suggest = esSearchSupport.getSearchSuggest("suggest", keyword, size, true, "test_goods_suggest", "index_test_goods");
List<String> keywords = null;
if (suggest != null) {
keywords = new ArrayList<>();
List<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> entries =
suggest.getSuggestion("test_goods_suggest").getEntries();
for (Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option> entry : entries) {
for (Suggest.Suggestion.Entry.Option option : entry.getOptions()) {
String key = option.getText().string();
if (!StringUtils.isEmpty(key)) {
if (keywords.contains(key)) {
continue;
}
keywords.add(key);
if (keywords.size() >= size) {
break;
}
}
}
}
}
return keywords;
}