java spark 流式计算 java流式编程常用方法

转载

编程小匠人 2023-09-23 09:51:26

文章标签 java spark 流式计算 stream java Test System 文章分类 Java 后端开发

1、toCollection(collectionFactory)
2、toList()
3、toSet()
4、toMap(keyMapper, valueMapper)
5、toMap(keyMapper, valueMapper, mergeFunction)
6、toMap(keyMapper, valueMapper, mergeFunction, mapSupplier)
7、toConcurrentMap(keyMapper, valueMapper)
8、joining(delimiter, prefix, suffix)
9、mapping( mapper, downstream)
10、collectingAndThen(downstream, finisher)
11、counting()
12、maxBy(comparator)
13、minBy(comparator)
14、summingDouble(mapper)
15、averagingDouble(mapper)
16、summarizingDouble(mapper)
17、reducing(identity,mapper, op)
18、 groupingBy(classifier)
19、groupingBy(classifier,downstream)
20、partitioningBy(predicate, downstream)

总结

前言

Collectors是Collector的工具类，类中提供了很多流收集、归约、分组、分区等方法，方便我们直接使用。

一、创建实体类

@Data
@AllArgsConstructor
public class Student {

    /**
     * 学号
     */
    private long id;
    /**
     * 姓名
     */
    private String name;
    /**
     * 年龄
     */
    private int age;
    /**
     * 年级
     */
    private String grade;
    /**
     * 语文成绩
     */
    private double chineseScore;
    /**
     * 数学成绩
     */
    private double mathScore;
    /**
     * 英语成绩
     */
    private double englishScore;

}

二、测试数据

ImmutableList<Student> students;
    @Before
    public void before() {
        Student a = new Student(1, "a", 16, "高一", 120, 130, 125);
        Student b = new Student(2, "b", 17, "高二", 80, 90, 80);
        Student c = new Student(3, "c", 18, "高三", 130.1, 130, 100);
        Student d = new Student(4, "d", 16, "高一", 100, 140, 110);
        Student e = new Student(5, "e", 17, "高二", 120, 120, 120);
        Student f = new Student(6, "f", 18, "高三", 130.2, 110, 100);
        Student g = new Student(7, "g", 16, "高一", 110, 130, 80);
        Student h = new Student(8, "h", 17, "高二", 90, 120, 900);
        students = ImmutableList.of(a, b, c, d, e, f, g, h);
    }

二、测试方法

1、toCollection(collectionFactory)

@Test
    public void toCollectionTest() {
        //将学生年级放入集合
        TreeSet<String> grades = students.stream().map(Student::getGrade).collect(Collectors.toCollection(TreeSet::new));
        System.out.println(grades.toString());
    }

结果

[高一, 高三, 高二]

2、toList()

@Test
    public void toListTest() {
        //将学生名字放入集合
        List<String> names = students.stream().map(Student::getName).collect(Collectors.toList());
        System.out.println(names.toString());
    }

结果

[a, b, c, d, e, f, g, h]

3、toSet()

@Test
    public void toSetTest() {
        //将学生语文成绩放入集合
        Set<Double> chineseScores = students.stream().map(Student::getChineseScore).collect(Collectors.toSet());
        System.out.println(chineseScores.toString());
    }

结果

[130.0, 80.0, 90.0, 100.0, 110.0, 120.0]

4、toMap(keyMapper, valueMapper)

@Test
    public void toMapTest() {
        //将学生学号和姓名放入map，当出现相同key时，会抛异常
        Map<Long, String> map = students.stream().collect(Collectors.toMap(student -> student.getId(), student -> student.getName()));
        System.out.println(map.toString());
    }

结果

{1=a, 2=b, 3=c, 4=d, 5=e, 6=f, 7=g, 8=h}

5、toMap(keyMapper, valueMapper, mergeFunction)

@Test
    public void toMapWithMergeTest() {
        //将学生学号和班级放入map，当出现相同key时，用新值替换旧值
        Map<Long, String> map = students.stream().collect(Collectors.toMap(student -> student.getId(), student -> student.getGrade(), (oldValue, newValue) -> newValue));
        System.out.println(map.toString());
    }

结果

{1=高一, 2=高二, 3=高三, 4=高一, 5=高二, 6=高三, 7=高一, 8=高二}

6、toMap(keyMapper, valueMapper, mergeFunction, mapSupplier)

@Test
    public void toLinkedHashMapTest() {
        //将学生学号和班级放入自定义的LinkedHashMap
        LinkedHashMap<Long, String> map = students.stream().collect(Collectors.toMap(
                student -> student.getId(),
                student -> student.getGrade(),
                (oldValue, newValue) -> newValue,
                LinkedHashMap::new)
        );
        System.out.println(map.toString());
    }

结果

{1=高一, 2=高二, 3=高三, 4=高一, 5=高二, 6=高三, 7=高一, 8=高二}

7、toConcurrentMap(keyMapper, valueMapper)

@Test
    public void toConcurrentMapTest() {
        //将学生学号和班级放入ConcurrentMap
        ConcurrentMap<Long, String> concurrentMap = students.stream().collect(Collectors.toConcurrentMap(student -> student.getId(), student -> student.getGrade()));
        System.out.println(concurrentMap.toString());
    }

结果

{1=高一, 2=高二, 3=高三, 4=高一, 5=高二, 6=高三, 7=高一, 8=高二}

8、joining(delimiter, prefix, suffix)

@Test
    public void joiningWithPrefixAndSuffixTest() {
        //将学生姓名用逗号连接成一个字符串,并在字符串前加上^^^,在字符串后加上$$$
        String names = students.stream().map(Student::getName).collect(Collectors.joining(",","^^^","$$$"));
        System.out.println(names);
    }

结果

^^^a,b,c,d,e,f,g,h$$$

9、mapping( mapper, downstream)

@Test
    public void mappingTest() {
        //将学生映射成学生名字,并收集到list中
        List<String> names = students.stream().collect(Collectors.mapping(Student::getName, Collectors.toList()));
        System.out.println(names.toString());
    }

结果

[a, b, c, d, e, f, g, h]

10、collectingAndThen(downstream, finisher)

@Test
    public void collectingAndThenTest() {
        //先将学生年级收集到set中，再计算set的大小
        Integer count = students.stream().map(Student::getGrade).collect(Collectors.collectingAndThen(Collectors.toSet(), grade -> grade.size()));
        System.out.println(count);
    }

结果

11、counting()

@Test
    public void countingTest() {
        //计算学生的数量
        Long count = students.stream().collect(Collectors.counting());
        System.out.println(count);
    }

结果

12、maxBy(comparator)

@Test
    public void maxByTest() {
        //获取语文成绩最高分
        Optional<Double> max = students.stream().map(Student::getChineseScore).collect(Collectors.maxBy(Double::compare));
        System.out.println(max.get());
    }

结果

130.2

13、minBy(comparator)

@Test
    public void minByTest() {
        //获取语文成绩最低分
        Optional<Double> max = students.stream().map(Student::getChineseScore).collect(Collectors.minBy(Double::compare));
        System.out.println(max.get());
    }

结果

80.0

14、summingDouble(mapper)

@Test
    public void summingDoubleTest() {
        //获取所有学生语文成绩总得分
        Double summing = students.stream().map(Student::getChineseScore).collect(Collectors.summingDouble(a -> a));
        System.out.println(summing);
    }

结果

880.3

15、averagingDouble(mapper)

@Test
    public void averagingDoubleTest() {
        //获取所有学生语文成绩平均分
        Double summing = students.stream().map(Student::getChineseScore).collect(Collectors.averagingDouble(a -> a));
        System.out.println(summing);
    }

结果

110.0375

16、summarizingDouble(mapper)

@Test
    public void summarizingDoubleTest() {
        //返回统计数据
        DoubleSummaryStatistics statistics = students.stream().map(Student::getChineseScore).collect(Collectors.summarizingDouble(a -> a));
        System.out.println("数量：" + statistics.getCount() +
                "\r\n总成绩：" + statistics.getSum() +
                "\n最大成绩" + statistics.getMax() +
                "\n最小成绩" + statistics.getMin() +
                "\n平均成绩" + statistics.getAverage()
        );
    }

结果：

数量：8
总成绩：880.3
最大成绩130.2
最小成绩80.0
平均成绩110.0375

17、reducing(identity,mapper, op)

@Test
    public void reducingTest() {
        //获取所有学生语文成绩总分
        Optional<Double> sum1 = students.stream().map(Student::getChineseScore).collect(Collectors.reducing(Double::sum));
        Double sum2 = students.stream().map(Student::getChineseScore).collect(Collectors.reducing(0d, Double::sum));
        Double sum3 = students.stream().collect(Collectors.reducing(0d, Student::getChineseScore, Double::sum));
        System.out.println(sum1.get());
    }

结果

880.3

18、 groupingBy(classifier)

@Test
    public void groupingByTest() {
        //按照年级分组
        Map<String, List<Student>> groupByGrade = students.stream().collect(Collectors.groupingBy(Student::getGrade));
        System.out.println(groupByGrade);
    }

结果

{高三=[Student(id=3, name=c, age=18, grade=高三, chineseScore=130.1, mathScore=130.0, englishScore=100.0), Student(id=6, name=f, age=18,grade=高三, chineseScore=130.2, mathScore=110.0, englishScore=100.0)], 
高二=[Student(id=2, name=b, age=17, grade=高二, chineseScore=80.0, mathScore=90.0, englishScore=80.0), Student(id=5, name=e, age=17, grade=高二, chineseScore=120.0, mathScore=120.0, englishScore=120.0), Student(id=8, name=h, age=17, grade=高二, chineseScore=90.0, mathScore=120.0, englishScore=900.0)], 
高一=[Student(id=1, name=a, age=16, grade=高一, chineseScore=120.0, mathScore=130.0, englishScore=125.0), Student(id=4, name=d, age=16, grade=高一, chineseScore=100.0, mathScore=140.0, englishScore=110.0), Student(id=7, name=g, age=16, grade=高一, chineseScore=110.0, mathScore=130.0, englishScore=80.0)]}

19、groupingBy(classifier,downstream)

@Test
    public void groupingBy2Test() {
        //按照年级分组后统计每组中每个学生的总成绩
        Map<String, Map<Long, Double>> map = students.stream().collect(
                Collectors.groupingBy(
                        Student::getGrade,
                        Collectors.toMap(
                                student -> student.getId(),
                                student -> student.getChineseScore() + student.getEnglishScore() + student.getMathScore()
                        )));

        map.forEach((grade, idTotalScore) -> {
            idTotalScore.forEach((id, totalScore) -> {
                System.out.println("grade: " + grade + " id: " + id + " totalScore: " + totalScore);
            });
        });
    }

结果

grade: 高三 id: 3 totalScore: 360.1
grade: 高三 id: 6 totalScore: 340.2
grade: 高二 id: 2 totalScore: 250.0
grade: 高二 id: 5 totalScore: 360.0
grade: 高二 id: 8 totalScore: 1110.0
grade: 高一 id: 1 totalScore: 375.0
grade: 高一 id: 4 totalScore: 350.0
grade: 高一 id: 7 totalScore: 320.0

20、partitioningBy(predicate, downstream)

@Test
    public void partitioningByTest() {
        //按照年级分组后,再根据学生总成绩是否大于300分组
        Map<String, Map<Boolean, List<Student>>> map = students.stream().collect(
                Collectors.groupingBy(Student::getGrade,
                        Collectors.partitioningBy(
                                student -> student.getChineseScore() + student.getEnglishScore() + student.getMathScore() > 300
                        )
                )
        );

        map.forEach((grade, map1) -> map1.forEach((b, list) ->
                list.stream().forEach(student -> System.out.println("grade: " + grade + " 是否大于300分: " + b + " id: " + student.getId())))
        );
    }

结果

grade: 高三 是否大于300分: true id: 3
grade: 高三 是否大于300分: true id: 6
grade: 高二 是否大于300分: false id: 2
grade: 高二 是否大于300分: true id: 5
grade: 高二 是否大于300分: true id: 8
grade: 高一 是否大于300分: true id: 1
grade: 高一 是否大于300分: true id: 4
grade: 高一 是否大于300分: true id: 7