ElasticSearch API使用

原创

DanielMaster 2022-01-13 14:05:14 博主文章分类：ElasticSearch ©著作权

文章标签 elasticsearch json java Echarts 文章分类 代码人生

©著作权归作者所有：来自51CTO博客作者DanielMaster的原创作品，请联系作者获取转载授权，否则将追究法律责任

Maven依赖

<dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.11</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>transport</artifactId>
            <version>6.5.2</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.8</version>
        </dependency>
        <!-- json -->
        <dependency>
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>
            <version>20180813</version>
        </dependency>
</dependencies>

因为方法较多，这里采用junit单元测试

目录结构

ElasticSearch API使用_Echarts

Product.java

package com.daniel.entity;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
 * @Author Daniel
 * @Description 这里需要在idea中安装lombok插件
 **/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Product {
    private String name;
    private String author;
    private String version;
}

ElasticSearchUtil.java

package com.daniel.util;

import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;

import java.io.IOException;
import java.net.InetAddress;
import java.util.LinkedList;
import java.util.Properties;

/**
 * @Author Daniel
 * @Description 连接到ElasticSearch集群
 **/
public class ElasticSearchUtil {
    // 连接池
    private static LinkedList<TransportClient> pool = new LinkedList<>();

    static {
        String CLUSTER_NAME = "cluster.name";
        String CLUSTER_HOSTS_PORT = "cluster.hosts.port";
        Properties properties = new Properties();
        try {
            // 加载配置文件
            properties.load(ElasticSearchUtil.class.getClassLoader().getResourceAsStream("elastic.properties"));

            Settings setting = Settings.builder()
                    // 如果集群的cluster.name和elasticsearch不同，需要手动指定
                    .put(CLUSTER_NAME, properties.getProperty(CLUSTER_NAME))
                    .build();
            // 入口
            TransportClient client;
            for (int i = 0; i < 5; i++) {
                client = new PreBuiltTransportClient(setting);
                // 指定es集群的地址
                String[] hostAndPorts = properties.getProperty(CLUSTER_HOSTS_PORT).split(",");
                for (String hostAndPort : hostAndPorts) {
                    String host = hostAndPort.split(":")[0];
                    int port = Integer.valueOf(hostAndPort.split(":")[1]);
                    TransportAddress trans = new TransportAddress(InetAddress.getByName(host), port);
                    client.addTransportAddress(trans);
                }
                pool.push(client);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public static TransportClient getClient() {
        while (pool.isEmpty()) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        return pool.poll();
    }

    public static void release(TransportClient client) {
        pool.push(client);
    }
}

elastic.properties

cluster.name=bde-es
cluster.hosts.port=hadoop01:9300,hadoop02:9300,hadoop03:9300

1.增删改查

CURD.java

package com.daniel.api;

import com.daniel.entity.Product;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.beans.PropertyDescriptor;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Map;


/**
 * @Author Daniel
 * @Description 增删改查
 **/
public class CURD {
    // 入口：TransportClient
    private TransportClient client;

    @Before
    // 建立连接
    public void setUp() throws UnknownHostException {
        Settings setting = Settings.builder()
                .put("cluster.name", "bde-es")
                .build();

        client = new PreBuiltTransportClient(setting);
        // 指定es集群的地址
        TransportAddress[] trans = {
                // 这里不是9200
                new TransportAddress(InetAddress.getByName("hadoop01"), 9300),
                new TransportAddress(InetAddress.getByName("hadoop02"), 9300),
                new TransportAddress(InetAddress.getByName("hadoop03"), 9300)
        };
        client.addTransportAddresses(trans);

/*        // 获取集群配置
        for(String key : client.settings().keySet()) {
            System.out.println(key + ": " + client.settings().get(key));
        }*/
    }

    final String INDEX = "product";
    final String TYPE = "hadoop";

    @Test
    public void get() {
        // 这里相当于curl -XGET http://hadoop01:9200/{index}/{type}/{id}
        GetResponse response = client.prepareGet(INDEX, TYPE, "1")
                .get();
        Map<String, Object> map = response.getSource();
        map.forEach((k, v) -> System.out.println(k + "--->" + v));
        System.out.println("create json version: " + response.getVersion());
    }

    @Test
    public void createJSON() {
        String json = "{\"name\": \"azkaban\", \"author\": \"apache\", \"version\": \"1.4.2\"}";
        IndexResponse response = client.prepareIndex(INDEX, TYPE, "1")
                // 传入XContentType.JSON是为了避免json数组只能是偶数的报错
                .setSource(json, XContentType.JSON)
                .get();
        System.out.println("create json version: " + response.getVersion());
    }

    @Test
    public void createMap() {
        Map<String, String> map = new HashMap<>();
        map.put("name", "spark");
        map.put("author", "apache");
        map.put("version", "2.7.6");
        IndexResponse response = client.prepareIndex(INDEX, TYPE, "3")
                .setSource(map)
                .get();
        System.out.println("create map version: " + response.getVersion());
    }

    @Test
    public void createBean() throws Exception {
        Product bp = new Product();
        bp.setName("hbase");
        bp.setAuthor("apache");
        bp.setVersion("1.2.1");
        // 通常做法讲bean转换成map或者json字符串
        // Bean to json
//        JSONObject jsonObj = new JSONObject(bp);
        // Bean to map
        Map<String, Object> beanMap = bean2Map(bp);
        IndexResponse response = client.prepareIndex(INDEX, TYPE, "4")
//                                        .setSource(jsonObj.toString(), XContentType.JSON)
                .setSource(beanMap)
                .get();
        System.out.println("create bean version: " + response.getVersion());
    }

    // 基本类型XContentBuilder
    @Test
    public void createXContentBuilder() throws IOException {
        XContentBuilder builder = JsonXContent.contentBuilder();

        builder.startObject()
                .field("name", "elasticsearch")
                .field("author", "elastic")
                .field("version", "6.5.2")
                .endObject();

        IndexResponse response = client.prepareIndex(INDEX, TYPE, "5")
                .setSource(builder)
                .get();
        System.out.println("create bean version: " + response.getVersion());
    }

    // 将对象转换成Map
    public <T> Map<String, Object> bean2Map(T t) throws Exception {
        Map<String, Object> map = new HashMap<>();
        Class<?> clazz = t.getClass();
        Field[] fields = clazz.getDeclaredFields();
        PropertyDescriptor bp = null;
        for (Field field : fields) {
            String name = field.getName();
            bp = new PropertyDescriptor(name, clazz);
            Method getterMethod = bp.getReadMethod();
            Object value = getterMethod.invoke(t);
            map.put(name, value);
        }
        return map;
    }

    @Test
    // 局部修改
    public void update() {
        // 添加或修改一个url字段
        String json = "{\"url\": \"http://www.elastic.io\"}";
        UpdateResponse response = client.prepareUpdate(INDEX, TYPE, "6")
                .setDoc(json, XContentType.JSON)
                .get();
        System.out.println("update index version: " + response.getVersion());
    }

    @Test
    public void Delete() {
        DeleteResponse response = client.prepareDelete(INDEX, TYPE, "7")
                .get();
        System.out.println("delete index version: " + response.getVersion());
    }

    @Test
    // 批量操作
    public void bulk() throws Exception {
        XContentBuilder builder = JsonXContent.contentBuilder();
        // 新建一个json
        builder.startObject()
                .field("name", "elasticsearch")
                .field("author", "elastic")
                .field("version", "6.5.2")
                .endObject();
        String json = "{\"url\": \"http://www.elastic.io\"}";
        BulkResponse itemResponses = client.prepareBulk()
                // 增
                .add(client.prepareIndex(INDEX, TYPE, "7").setSource(builder))
                // 删
                .add(client.prepareDelete(INDEX, TYPE, "6"))
                // 改
                .add(client.prepareUpdate(INDEX, TYPE, "5").setDoc(json, XContentType.JSON))
                // 查
                .get();
        // 拿到这个操作的一些信息
        for (BulkItemResponse response : itemResponses) {
            String id = response.getId();
            long version = response.getVersion();
            System.out.println(id + "--->version: " + version);
        }
    }

    @After
    public void cleanUp() {
        client.close();
    }
}

2.全文索引

全文索引，prepareSearch，需要指定检索的类型 SearchType：

query and fetch(deprecated)
- 向某个索引库中检索数据，并立即返回结果
- 向所有的分片shards进行检索，每个shard都会返回N（分页的页面大小）条记录
- 如果有M个shard，总共返回的数据量就是M * N
- 速度最快
query then fetch（default）
- 向某个索引库中检索数据，并立即返回结果N条记录
- 在返回之前，需要对检索的所数据进行排序，排名，找到相关度最高的N条记录返回
dfs query and fetch
dfs query then fetch

以上两种就比最上面两种多了一个dfs的过程：d：distributed f：frequency s：scatter称为分布式词频率和文档频率散发

常见的查询方式：

matchAllQuery --> select * from t
matchQuery(要求对索引库中的数据进行分词) --> select * from t where name like “%text%”
matchPhraseQuery(查询一个短语，如果结果不是以短语的形式出现，则无法查到)
termQuery(不进行分词) --> select * from t where name = text

FullTextIndex.java

package com.daniel.api;

import com.daniel.util.ElasticSearchUtil;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.util.Map;

/**
 * @Author Daniel
 * @Description 全文索引
 **/
public class FullTextIndex {
    private TransportClient client;

    @Before
    public void setUp() {
        client = ElasticSearchUtil.getClient();
    }

    String[] indices = {"account"};

    @Test
    public void search() {
        // 指定索引库
        SearchResponse response = client.prepareSearch(indices)
                // 指定检索类型
                .setSearchType(SearchType.QUERY_THEN_FETCH)
                // 选择一种查询方式
//                              .setQuery(QueryBuilders.matchAllQuery())
//                              .setQuery(QueryBuilders.matchQuery("address", "Avenue"))
                .setQuery(QueryBuilders.matchPhraseQuery("address", "Clarkson Avenue"))
                // 查什么内容
                // 有啥要求(排序，聚合等等)
                .get();
        // 搜索结果
        SearchHits searchHits = response.getHits();
        long totalHits = searchHits.totalHits;
        System.out.println("Daniel为您找到相关结果约" + totalHits + "个");
        // es会对数据进行评分
        float maxScore = searchHits.getMaxScore();
        System.out.println("最大得分：" + maxScore);
        SearchHit[] hits = searchHits.getHits();
        for (SearchHit hit : hits) {
            System.out.println("-------------------------------------");
            String index = hit.getIndex();
            String type = hit.getType();
            String id = hit.getId();
            float score = hit.getScore();
            String source = hit.getSourceAsString();
            System.out.printf("index:\t%s\n", index);
            System.out.printf("type:\t: %s\n", type);
            System.out.printf("id:\t%s\n", id);
            System.out.printf("score:\t%f\n", score);
            System.out.printf("source:\t%s\n", source);
        }
    }

    @Test
    public void search2() {
        SearchResponse response = client.prepareSearch(indices)
                .setSearchType(SearchType.QUERY_THEN_FETCH)
                .setQuery(QueryBuilders.matchQuery("address", "Avenue"))
                // 分页算法：from=(page - 1) * pageSize
                // 第五页
                .setFrom(20)
                .setSize(5)
                // 排序(二次排序)
                .addSort("age", SortOrder.DESC)
                .addSort("balance", SortOrder.DESC)
                //高亮操作
                .highlighter(
                        SearchSourceBuilder.highlight()
                                // 前置标签
                                .preTags("<font style=\"font-size: 20px;color:blue\">")
                                .field("address")
                                // 后置标签
                                .postTags("</font>")
                )
                .get();
        SearchHits searchHits = response.getHits();
        long totalHits = searchHits.totalHits;
        System.out.println("Daniel为您找到相关结果约" + totalHits + "个");
        float maxScore = searchHits.getMaxScore();
        System.out.println("最大得分：" + maxScore);
        SearchHit[] hits = searchHits.getHits();
        for (SearchHit hit : hits) {
            System.out.println("-------------------------------------");
            String index = hit.getIndex();
            String type = hit.getType();
            String id = hit.getId();
            float score = hit.getScore();
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            //key --> 高亮字段对应的值
            //value --> 高亮字段对应的高亮内容
            String hl = "";
            for (Map.Entry<String, HighlightField> hf : highlightFields.entrySet()) {
                HighlightField highlightField = hf.getValue();
                Text[] texts = highlightField.fragments();
                for (Text text : texts) {
                    hl += text.toString();
                }
            }
            System.out.printf("index:\t%s\n", index);
            System.out.printf("type:\t: %s\n", type);
            System.out.printf("id:\t%s\n", id);
            System.out.printf("score:\t%f\n", score);
            // 将最后的结果输入在html页面上即可以看到效果
            System.out.println("高亮内容：" + hl);
        }
    }

    @After
    public void cleanUp() {
        ElasticSearchUtil.release(client);
    }
}