spark和elasticsearch的整合

下载jar包
https://www.elastic.co/downloads/hadoop

将elasticsearch-spark_2.10-2.4.0.jar加入类路径里面

import org.elasticsearch.spark.sql._
        // DataFrame schema automatically inferred
        val conf = new SparkConf().setAppName("Statistics")
        conf.setMaster("local");
        conf.set("es.nodes", "20.2.1.16")
        conf.set("es.port", "9200")
        val sc = new SparkContext(conf)
        val sqlContext = new SQLContext(sc)

        val df = sqlContext.read.format("es").load("product93_20161024")

        df.printSchema()
        println(df.count())