springbatch 链接hive springbatch step

转载

云端梦想实现家 2023-10-11 08:13:52

文章标签 springbatch 链接hive spring batch xml 多线程 文章分类 Hive 大数据

Job执行默认情况使用单个线程完成任务。Spring Batch框架支持为Step配置多个线程，即可以使用多个线程并行执行一个Step，可以提高Step的处理速度。使用tasklet的属性task-executor为Step定义多个线程。

<!--定义名字为billJob2的作业，多线程-->
<batch:job id="billJob2">
    <!--定义一个Step，继承parentStep-->
    <batch:step id="billStep2">
        <!--使用线程池taskExecutor,throttle-limit表示限制能使用的最大的线程数目-->
        <batch:tasklet transaction-manager="transactionManager"
                 task-executor="taskExecutor" throttle-limit="6">
            <!--使用parentStep的读、写操作，使用自己定义的处理操作和commit-interval属性-->
            <batch:chunk reader="csvItemReader" writer="csvItemWriter"                          processor="creditBillProcessor" commit-interval="2">
            </batch:chunk>
        </batch:tasklet>
    </batch:step>
</batch:job>

<!--定义线程池，默认有5个线程，最大线程为15-->
<bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
    <property name="corePoolSize" value="5"/>
    <property name="maxPoolSize" value="15"/>
</bean>

一、项目框架

springbatch 链接hive springbatch step_batch

二、代码实现

CreditBill.java:

package com.xj.demo12;

/**
 * @Author : xjfu
 * @Date : 2021/10/26 19:27
 * @Description :
 */
public class CreditBill {
    //银行卡账户ID
    private String accountID = "";
    //持卡人姓名
    private String name = "";
    //消费金额
    private double amount = 0;
    //消费日期
    private String date = "";
    //消费场所
    private String address = "";

    //get和set方法

    @Override
    public String toString() {
        return this.accountID + "," + this.name + "," + this.amount + "," + this.date + "," + this.address;
    }
}

CreditBillProcessor.java:

package com.xj.demo12;

import org.springframework.batch.item.ItemProcessor;
import java.util.Date;

/**
 * @Author : xjfu
 * @Date : 2021/10/26 19:29
 * @Description :
 */
public class CreditBillProcessor implements ItemProcessor<CreditBill, CreditBill> {
    @Override
    public CreditBill process(CreditBill bill) throws Exception {

        System.out.println(bill.toString());
        //做一些简单的处理
        bill.setAccountID(bill.getAccountID() + "1");
        bill.setName(bill.getName() + "2");
        bill.setAmount(bill.getAmount() + 3);
        bill.setDate(new Date().toString());
        bill.setAddress(bill.getAddress() + 5);

        return bill;
    }
}

Demo12BatchMain.java:

package com.xj.demo12;

import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

/**
 * @Author : xjfu
 * @Date : 2021/12/8 8:52
 * @Description :多线程Step
 */
public class Demo12BatchMain {

    public static void main(String[] args) {
        Demo12BatchMain batchMain = new Demo12BatchMain();
        //单线程调用
        batchMain.executeJob("demo12/job/demo12-job.xml", "billJob1", "jobLauncher", new JobParametersBuilder().toJobParameters());
        //多线程调用
        //batchMain.executeJob("demo12/job/demo12-job.xml", "billJob2", "jobLauncher", new JobParametersBuilder().toJobParameters());
    }

    /**
     *执行Job
     * @param jobXmlPath 配置job的xml文件路径
     * @param jobId job的id
     * @param jobLauncherId jobLauncher的id
     * @param jobParameters 参数
     */
    public void executeJob(String jobXmlPath, String jobId, String jobLauncherId, JobParameters jobParameters){
        ApplicationContext context = new ClassPathXmlApplicationContext(jobXmlPath);
        JobLauncher jobLauncher = (JobLauncher) context.getBean(jobLauncherId);
        //获取要执行的Job
        Job job = (Job)context.getBean(jobId);

        try{
            //开始执行作业Job
            JobExecution jobExecution =  jobLauncher.run(job, jobParameters);
            //输出执行结果
            System.out.println(jobExecution.toString());
        }catch (Exception e){
            e.printStackTrace();
        }
    }
}

demo12-inputFile.csv:

1111111111111,tom,100.00,2013-01-31 01:00:08,Lu lit1
2222222222222,tom,120.00,2013-02-31 02:00:08,Lu Zui2
3333333333333,tom,130.00,2013-03-31 03:00:08,Lu lit3
4444444444444,tom,140.00,2013-04-31 04:00:08,Lu Zui4
5555555555555,tom,150.00,2013-05-31 05:00:08,Lu lit5
6666666666666,tom,160.00,2013-06-31 06:00:08,Lu Zui6

demo12-job.xml:

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:batch="http://www.springframework.org/schema/batch"
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch.xsd">

    <!--导入文件-->
    <import resource="classpath:demo12/job/demo12-jobContext.xml"/>

    <!--定义名字为billJob1的作业,单线程-->
    <batch:job id="billJob1">
        <!--定义一个Step，继承parentStep-->
        <batch:step id="billStep1">
            <batch:tasklet transaction-manager="transactionManager">
                <!--使用parentStep定义的读、写操作，使用自己定义的处理操作和commit-interval属性-->
                <batch:chunk reader="csvItemReader" writer="csvItemWriter" processor="creditBillProcessor" commit-interval="2">
                </batch:chunk>
            </batch:tasklet>
        </batch:step>
    </batch:job>

    <!--定义名字为billJob2的作业，多线程-->
    <batch:job id="billJob2">
        <!--定义一个Step，继承parentStep-->
        <batch:step id="billStep2">
            <!--使用线程池taskExecutor,throttle-limit表示限制能使用的最大的线程数目-->
            <batch:tasklet transaction-manager="transactionManager" task-executor="taskExecutor" throttle-limit="6">
                <!--使用parentStep定义的读、写操作，使用自己定义的处理操作和commit-interval属性-->
                <batch:chunk reader="csvItemReader" writer="csvItemWriter" processor="creditBillProcessor" commit-interval="2">
                </batch:chunk>
            </batch:tasklet>
        </batch:step>
    </batch:job>

    <!--定义线程池，默认有5个线程，最大线程为15-->
    <bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
        <property name="corePoolSize" value="5"/>
        <property name="maxPoolSize" value="15"/>
    </bean>


</beans>

demo12-jobContext.xml:

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">

    <!--定义作业仓库 Job执行期间的元数据存储在内存中-->
    <bean id="jobRepository" class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean">
    </bean>

    <!--定义作业调度器，用来启动job-->
    <bean id="jobLauncher" class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
        <!--注入jobRepository-->
        <property name="jobRepository" ref="jobRepository"/>
    </bean>

    <!--定义事务管理器，用于Spring Batch框架中对数据操作提供事务能力-->
    <bean id="transactionManager" class="org.springframework.batch.support.transaction.ResourcelessTransactionManager"/>

    <!--读取信用卡账单文件，CSV 格式-->
    <!--使用FlatFileItemReader读文本文件-->
    <bean id="csvItemReader" class="org.springframework.batch.item.file.FlatFileItemReader" scope="step">
        <!--指定读取的资源文件-->
        <property name="resource" value="classpath:demo12/data/demo12-inputFile.csv"/>
        <!--通过lineMapper把文本中的一行转换为领域对象creditBill-->
        <property name="lineMapper">
            <bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper">
                <!--lineTokenizer定义文本中每行的分隔符号-->
                <property name="lineTokenizer" ref="lineTokenizer"/>
                <!--fieldSetMapper定义了转换结果映射,即具体映射到哪个Java类对象-->
                <property name="fieldSetMapper">
                    <bean class="org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper">
                        <property name="prototypeBeanName" value="creditBill"/>
                    </bean>
                </property>
            </bean>
        </property>
    </bean>

    <!--lineTokenizer-->
    <bean id="lineTokenizer" class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer">
        <!--按","符号对行进行切割-->
        <property name="delimiter" value=","/>
        <!--属性名称列表,将切割后的行按顺序投入-->
        <property name="names">
            <list>
                <value>accountID</value>
                <value>name</value>
                <value>amount</value>
                <value>date</value>
                <value>address</value>
            </list>
        </property>
    </bean>

    <!--注入实体类-->
    <bean id="creditBill" class="com.xj.demo10.CreditBill" scope="prototype"></bean>

    <!--数据处理类-->
    <bean id="creditBillProcessor" class="com.xj.demo10.CreditBillProcessor" scope="step"></bean>

    <!--写信用卡账单文件，CSV格式-->
    <bean id="csvItemWriter" class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step">
        <!--要写入的文件位置,因为[classpath:]不是一个具体的目录,这里应当用[file:](从项目根目录开始)指明输出位置-->
        <property name="resource" value="file:src/main/resources/demo12/data/demo12-outputFile.csv"/>
        <!--[lineAggregator成员]指明行聚合器,用来将对象输出到文件时构造文件中的每行的格式-->
        <property name="lineAggregator">
            <!--这里使用Spring Batch自带的DelimitedLineAggregator来作为行聚合器(可以拼接一个个属性形成行)-->
            <bean class="org.springframework.batch.item.file.transform.DelimitedLineAggregator">
                <!--使用","拼接-->
                <property name="delimiter" value=","/>
                <!--fieldExtractor成员用来将Java类的属性组成的数组拼接成行字符串-->
                <property name="fieldExtractor">
                    <bean class="org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor">
                        <property name="names" value="accountID,name,amount,date,address">
                        </property>
                    </bean>
                </property>
            </bean>
        </property>
    </bean>
</beans>