Spark Transformation算子-＞repartition

增加或减少分区。会产生 shuffle。（多个分区分到一个分区不会产生 shuffle）

1. scala

package transformation

import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable.ListBuffer

/**
 * @Author yqq
 * @Date 2021/12/09 22:37
 * @Version 1.0
 */
object RepartitionTest {
  def main(args: Array[String]): Unit = {
    val context = new SparkContext(
      new SparkConf()
        .setMaster("local")
        .setAppName("repartition")
    )
    context.setLogLevel("Error")
    context.parallelize(Array[String](
      "yqq1", "yqq2", "yqq3", "yqq4",
      "yqq5", "yqq6", "yqq7", "yqq8",
      "yqq9", "yqq10", "yqq11", "yqq12"
    ),3).mapPartitionsWithIndex((index,iter)=>{
      val buffer = new ListBuffer[String]()
      while (iter.hasNext)
        buffer.append(s"before partition index = $index,value = ${iter.next()}")
      buffer.iterator
    }).repartition(2).mapPartitionsWithIndex((index,iter)=>{
      val strings = new ListBuffer[String]()
      while(iter.hasNext)
        strings.append(s"after partition index = $index,value = ${iter.next()}")
      strings.iterator
    }).collect().foreach(println)
  }
}

2. java

package transformations;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

/**
 * @Author yqq
 * @Date 2021/12/09 21:36
 * @Version 1.0
 */
public class RepartitionTest {
    public static void main(String[] args) {
        JavaSparkContext context = new JavaSparkContext(
                new SparkConf()
                        .setMaster("local")
                        .setAppName("repartition")
        );
        context.setLogLevel("Error");
        List<String> list = new ArrayList<>();
        context.parallelize(Arrays.asList(
                "yqq1","yqq2","yqq3","yqq4",
                "yqq5","yqq6","yqq7","yqq8",
                "yqq9","yqq10","yqq11","yqq12"
        ),3).repartition(2).mapPartitionsWithIndex(new Function2<Integer, Iterator<String>, Iterator<String>>() {
            @Override
            public Iterator<String> call(Integer v1, Iterator<String> v2) throws Exception {
                while (v2.hasNext())
                    list.add("repartition partition index = "+"【"+v1+"】"+",value = "+"【"+v2.next()+"】");
                return list.iterator();
            }
        },false).collect().forEach(e-> System.out.println(e));
    }
}