package com.immooc.spark
import com.immooc.spark.ReflectionTest.Person
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{Row, SparkSession}
object ProgramSchema {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setMaster("local[2]").setAppName("ProgramSchema")
val ssc = new SparkContext(sparkConf)
val spark = SparkSession
.builder()
.appName("ProgramSchema")
.config("spark.some.config.option", "some-value")
.getOrCreate()
import spark.implicits._
val peopleRDD = spark.sparkContext.textFile("file:usr/local/Cellar/spark-2.3.0/examples/src/main/resources/people.txt")
.map(_.split(","))
.map(line => Row(line(0), line(1).trim.toInt))
val structType = StructType(Array(StructField("name", StringType, true),
StructField("age", IntegerType, true)))
val peopleDF = spark.createDataFrame(peopleRDD, structType)
peopleDF.show()
spark.close()
}
}
spark 编程方式指定dataframe的 Schema
原创
©著作权归作者所有:来自51CTO博客作者fox64194167的原创作品,请联系作者获取转载授权,否则将追究法律责任
提问和评论都可以,用心的回复会被更多人看到
评论
发布评论
相关文章
-
Java实现异步编程的几种方式
Java实现异步编程的几种方式
异步任务 @Async CompletableFuture Future