分组聚合带条件Java RestHighLevelClient 操作
//查询es 书籍日志 SearchRequest searchRequest = new SearchRequest("bookanalysis"); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); List<QueryBuilder> filter = boolQuery.filter(); long start = LocalDateTime.now().plusDays(-1).withHour(0).withMinute(0).withSecond(0).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli(); // long start = LocalDate.now().plusDays(-1).atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli(); long end = LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli(); //开始时间 Long startMilli = start; //结束时间 Long endMilli = end; //时间区间条件 filter.add(rangeQuery(startMilli, endMilli, "create_time")); //表示是阅读书籍数据 filter.add(QueryBuilders.matchQuery("log_type",0)); //这里拼接动态条件 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); //添加条件 searchSourceBuilder.query(boolQuery); //时间分桶 DateHistogramAggregationBuilder builder = AggregationBuilders .dateHistogram("create_time") .field("create_time") .calendarInterval(DateHistogramInterval.HOUR).format("yyyy-MM-dd HH:mm:ss"); //书籍桶 TermsAggregationBuilder book_idField = AggregationBuilders.terms("book_id").field("book_id"); TermsAggregationBuilder chapter_idField = AggregationBuilders.terms("content_id").field("content_id"); //将子桶加入桶中 builder.subAggregation(book_idField.subAggregation(chapter_idField)); //阅读章节数 // book_idField.subAggregation(AggregationBuilders.count("content_id_count").field("content_id")); //阅读UV chapter_idField.subAggregation(AggregationBuilders.cardinality("read_uv").field("user_id")); searchSourceBuilder.aggregation(builder); searchRequest.source(searchSourceBuilder); SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); //每日的 List<AppStatisticsChapterUvEveryday> everydayList = new ArrayList<>(); //累计的 List<AppStatisticsChapterUvTotal> totalList = new ArrayList<>(); ParsedDateHistogram terms = response.getAggregations().get("create_time"); Iterator<? extends Histogram.Bucket> timeIter = terms.getBuckets().iterator(); while (timeIter.hasNext()){ Histogram.Bucket timeBucket = timeIter.next(); Object create_time = timeBucket.getKey(); ParsedLongTerms bookTerms = timeBucket.getAggregations().get("book_id"); Iterator<? extends Terms.Bucket> bookIter = bookTerms.getBuckets().iterator(); while (bookIter.hasNext()){ Terms.Bucket bookBucket = bookIter.next(); int book_id = bookBucket.getKeyAsNumber().intValue(); ParsedLongTerms contentTerms = bookBucket.getAggregations().get("content_id"); Iterator<? extends Terms.Bucket> contentIter = contentTerms.getBuckets().iterator(); while (contentIter.hasNext()) { Terms.Bucket contentBucket = contentIter.next(); int chapter_id = contentBucket.getKeyAsNumber().intValue(); Aggregations aggregations = contentBucket.getAggregations(); ZonedDateTime date = (ZonedDateTime) create_time; AppStatisticsChapterUvEveryday chapterUvEveryday = new AppStatisticsChapterUvEveryday(); Date createTime = new SimpleDateFormat("yyyy-MM-dd").parse(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") .format(date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime())); chapterUvEveryday.setStatistics_date(createTime); //阅读UV ParsedCardinality read_uv = aggregations.get("read_uv"); chapterUvEveryday.setRead_uv((int) read_uv.getValue()); //书籍id chapterUvEveryday.setBook_id(book_id); //章节id chapterUvEveryday.setChapter_id(chapter_id); //章节名称 AppBookChapter appBookChapter = appBookChapterService.chapterInfoById(chapter_id); if (appBookChapter==null){ continue; } chapterUvEveryday.setChapter_name(appBookChapter.getTitle()); everydayList.add(chapterUvEveryday); //查询上次的累计内容 AppStatisticsChapterUvTotal one = appStatisticsChapterUvTotalService.getOne( new QueryWrapper<AppStatisticsChapterUvTotal>() .eq("chapter_id", chapter_id) .orderByDesc("statistics_date") .last("limit 0,1") ); if (one == null) { one = new AppStatisticsChapterUvTotal(); one.setStatistics_date(createTime); //阅读UV one.setRead_uv((int) read_uv.getValue()); //书籍id one.setBook_id(book_id); //章节id one.setChapter_id(chapter_id); //章节名称 one.setChapter_name(appBookChapter.getTitle()); }else{ one.setId(null); one.setStatistics_date(createTime); //阅读UV one.setRead_uv(one.getRead_uv()+ (int) read_uv.getValue()); //书籍id one.setBook_id(book_id); //章节id one.setChapter_id(chapter_id); //章节名称 one.setChapter_name(appBookChapter.getTitle()); } totalList.add(one); } } } appStatisticsChapterUvTotalService.saveBatch(totalList); appStatisticsChapterUvEverydayService.saveBatch(everydayList);获取桶内的真正数据hits
TopHitsAggregationBuilder topHitsAggregationBuilder = AggregationBuilders.topHits("top"); build.subAggregation(topHitsAggregationBuilder); TopHits topHits = Bucket.getAggregations().get("top"); for (SearchHit hit : topHits.getHits().getHits()) { hit.getSourceAsMap().get("device_id"); } //获取超过1w条数据 需要加上 "track_total_hits":true ,不然只能显示出9999条 searchRequest.source(searchSourceBuilder.trackTotalHits(true)); //博客学习 //内容: Top Hits Aggregation //网址: https://blog.csdn.net/ctwy291314/article/details/82773180普通查询,使用查询内容
@Test public void test() throws IOException { SearchRequest searchRequest = new SearchRequest("useranalysis"); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); List<QueryBuilder> filter = boolQuery.filter(); //时间区间条件 filter.add(rangeQuery(1611590400000L, 1611676800000L, "create_time")); filter.add(QueryBuilders.matchQuery("channel_id",50000011104L)); filter.add(QueryBuilders.matchQuery("is_new",1)); //这里拼接动态条件 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); //添加条件 searchSourceBuilder.query(boolQuery); searchRequest.source(searchSourceBuilder.trackTotalHits(true).size(20001)); SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); HashSet<String> userIds = new HashSet<>(); for (SearchHit hit : response.getHits().getHits()) { String user_id = hit.getSourceAsMap().get("user_id").toString(); userIds.add(user_id); } System.out.println(userIds); }使用模板查询(dsl语言)
public int test(List<Integer> userIds) throws IOException { long start = LocalDateTime.now().plusDays(-1).withHour(0).withMinute(0).withSecond(0).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli(); long end = LocalDateTime.now().plusDays(0).withHour(0).withMinute(0).withSecond(0).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli(); SearchTemplateRequest request = new SearchTemplateRequest(); request.setRequest(new SearchRequest("bookanalysis")); request.setScriptType(ScriptType.INLINE); String script = "{\n" + " \"query\": {\n" + " \"bool\": {\n" + " \"must\": [\n" + " {\n" + " \"range\": {\n" + " \"create_time\": {\n" + " \"gte\": \"{{start}}\",\n" + " \"lte\": \"{{end}}\"\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"terms\": {\n" + " \"user_id\":{{user_id}}\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " },\n" + " \"size\": 0,\n" + " \"track_total_hits\": true"+ "}"; request.setScript(script); Map<String, Object> params = new HashMap<>(); params.put("user_id", JSONObject.toJSONString(userIds)); params.put("start", start); params.put("end", end); request.setScriptParams(params); SearchTemplateResponse searchTemplateResponse = restHighLevelClient.searchTemplate(request, RequestOptions.DEFAULT); SearchResponse response = searchTemplateResponse.getResponse(); TotalHits totalHits = response.getHits().getTotalHits(); return (int) totalHits.value; }删除数据
public ReturnT<String> deleteEveryday(String param) throws IOException { //删除指定的传入的索引 try { DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(param); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); List<QueryBuilder> filter = boolQueryBuilder.filter(); long start = LocalDateTime.now().plusDays(-10).withHour(23).withMinute(59).withSecond(59).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli(); long end = LocalDateTime.now().plusDays(-2).withHour(0).withMinute(0).withSecond(0).withNano(0).toInstant(ZoneOffset.of("+8")).toEpochMilli(); XxlJobLogger.log("\t 开始删除数据:("+start+"-"+end+") 索引地址:"+param+"。\t 开始时间:"+LocalDateTime.now().toString()); filter.add(rangeQuery(start, end, "create_time")); deleteByQueryRequest.setQuery(boolQueryBuilder); //并行 5条线程同时执行 deleteByQueryRequest.setSlices(5); //批次大小 一次删除多少 deleteByQueryRequest.setBatchSize(5000); // 更新最大文档数 // deleteByQueryRequest.setSize(10); //使用滚动参数来控制“搜索上下文”存活的时间 因为删除数据前是先查询到这个数据 deleteByQueryRequest.setScroll(TimeValue.timeValueMinutes(10)); //刷新索引 deleteByQueryRequest.setRefresh(true); //超时时间 2小时 deleteByQueryRequest.setTimeout(TimeValue.timeValueHours(2)); BulkByScrollResponse response = restHighLevelClient.deleteByQuery(deleteByQueryRequest,RequestOptions.DEFAULT); XxlJobLogger.log("\t 删除数据:"+response.getTotal()+" 条"+"\t 结束时间:"+LocalDateTime.now().toString()); //使用异步删除 /* Cancellable response = restHighLevelClient.deleteByQueryAsync( deleteByQueryRequest, RequestOptions.DEFAULT, new ActionListener<BulkByScrollResponse>() { @Override public void onResponse(BulkByScrollResponse response) { // XxlJobLogger.log("\t 删除数据:"+response.getTotal()+" 条"+"\t 异步结束时间:"+LocalDateTime.now().toString()); System.out.println("\t 删除数据:"+response.getTotal()+" 条"+"\t 异步结束时间:"+LocalDateTime.now().toString()); } @Override public void onFailure(Exception e) { e.printStackTrace(); // XxlJobLogger.log("\t 异步删除数据报错了........"+LocalDateTime.now().toString()+"\t"+e.getMessage()); System.out.println("\t 异步删除数据报错了........"+LocalDateTime.now().toString()+"\t"+e.getMessage()); } }); //取消删除 // response.cancel(); // Thread.sleep(2000); */ } catch (Exception e) { e.printStackTrace(); XxlJobLogger.log("\t 删除数据报错了........"+LocalDateTime.now().toString()+"\t"+e.getMessage()); return ReturnT.FAIL; } return ReturnT.SUCCESS;