IntelliJ进行Spark编程之WordCount

项目目录如下：

代码：

import org.apache.spark.{SparkConf, SparkContext}

object WordCount{
  def main(args:Array[String]) : Unit ={
    val conf=new SparkConf().setAppName("word count").setMaster("local");
    val sc=new SparkContext(conf);
    val rdd=sc.textFile("C:\\Users\\Administrator\\Desktop\\words.txt");
    val count=rdd.flatMap(x=>x.split(" ")).map(x=>(x,1)).reduceByKey((x,y)=>x+y).sortBy(x=>x._2,false,1);
    count.foreach(x=>println(x._1+"\t"+x._2));
  }
}

输出：

生成jar包的代码：

package demo
import org.apache.spark.{SparkConf, SparkContext}

object WordCount{
  def main(args:Array[String]) : Unit ={
    val conf=new SparkConf().setAppName("word count");
    val sc=new SparkContext(conf);
    val input=args(0);
    val output=args(1);
    val rdd=sc.textFile(input);
    val count=rdd.flatMap(x=>x.split(" ")).map(x=>(x,1)).reduceByKey((x,y)=>x+y).sortBy(x=>x._2,false,1);
    count.saveAsTextFile(output);
  }
}