SpringBOot高版本 整合es6

一、采用当当网elastic job，实现spingboot启动注入定时任务，任务配置在数据库，可以动态加载任务配置。

上干货

引入jar

<properties>
        <elastic-job.version>2.1.5</elastic-job.version>
    </properties>

        <!-- 引入elastic-job-lite核心模块 -->
        <dependency>
            <artifactId>elastic-job-common-core</artifactId>
            <groupId>com.dangdang</groupId>
            <version>${elastic-job.version}</version>
        </dependency>
        <dependency>
            <artifactId>elastic-job-lite-core</artifactId>
            <groupId>com.dangdang</groupId>
            <version>${elastic-job.version}</version>
        </dependency>
        <dependency>
            <artifactId>elastic-job-lite-spring</artifactId>
            <groupId>com.dangdang</groupId>
            <version>${elastic-job.version}</version>
        </dependency>

定义任务载体

package cn.quantgroup.cashloanflow.config.jobevent.job;

import cn.quantgroup.cashloanflow.service.job.ITimeJobService;
import cn.quantgroup.cashloanflow.util.DateUtil;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.Date;
/**
 * 每一小时扫描CallbackFailRecord表
 */
@Slf4j
@Component
public class CallbackFailSimpleJob implements SimpleJob {

    @Autowired
    private ITimeJobService timeJobService;


    @Override
    public void execute(ShardingContext shardingContext) {
        log.info("[CallbackFailRecord回调失败监控][JobName:{}] [ShardingItem:{}] [ShardingTotalCount:{}] [TaskId:{}] [JobParameter:{}] [Time:{}] start",
                shardingContext.getJobName(), shardingContext.getShardingItem(), shardingContext.getShardingTotalCount(), shardingContext.getTaskId(), shardingContext.getJobParameter(), DateUtil.format(new Date(), "yyyy-MM-dd HH:mm:ss"));
         timeJobService.callbackFailSendMsg();
    }
}

注册加载定时任务

package cn.quantgroup.cashloanflow.config.jobevent;

import cn.quantgroup.cashloanflow.entity.cashloanflow.ElasticJobConfig;
import cn.quantgroup.cashloanflow.repository.cashloanflow.IElasticJobConfigRepository;
import cn.quantgroup.cashloanflow.util.ApplicationContextHolder;
import com.dangdang.ddframe.job.api.ElasticJob;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import com.dangdang.ddframe.job.config.JobCoreConfiguration;
import com.dangdang.ddframe.job.config.JobTypeConfiguration;
import com.dangdang.ddframe.job.config.simple.SimpleJobConfiguration;
import com.dangdang.ddframe.job.event.JobEventConfiguration;
import com.dangdang.ddframe.job.lite.config.LiteJobConfiguration;
import com.dangdang.ddframe.job.lite.spring.api.SpringJobScheduler;
import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperRegistryCenter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.util.List;

/**
 * 可以动态改变任务，（JobConfigUpdateController 提供接口）
 * Created with suntao on 2019/3/19
 */
@Slf4j
@Component
public class ElasticJobLoader2 implements CommandLineRunner {

    @Autowired
    private IElasticJobConfigRepository elasticJobConfigRepository;

    @Resource
    private ZookeeperRegistryCenter zookeeperRegistryCenter;

    @Resource
    private JobEventConfiguration jobEventConfiguration;



    @Override
    public void run(String... args) {

        List<ElasticJobConfig> elasticJobConfigList = elasticJobConfigRepository.findAllActive();
        if (elasticJobConfigList == null || elasticJobConfigList.size() == 0) {
            return;
        }
        elasticJobConfigList.forEach(elasticJobConfig -> registryJob(elasticJobConfig));
        log.info("所有定时任务加载完成");
    }


    /**
     * 重新加载任务
     * @param ids
     */
    public void reloadJobByIds(List<Long> ids) {
        List<ElasticJobConfig> elasticJobConfigList = elasticJobConfigRepository.findAll(ids);
        elasticJobConfigList.forEach(elasticJobConfig -> registryJob(elasticJobConfig));
    }

    private void registryJob(ElasticJobConfig elasticJobConfigBean) {
        try {
            Class<? extends ElasticJob> jobClass = (Class<? extends ElasticJob>) Class
                    .forName(elasticJobConfigBean.getJobClass());
            ElasticJob elasticJob = getInstance(jobClass);
            SpringJobScheduler jobScheduler = jobScheduler(elasticJob, elasticJobConfigBean);
            jobScheduler.init();
            log.info("初始化定时任务 {} ", elasticJobConfigBean.toString());
        } catch (Exception e) {
            log.error("注册Job出错：{} ", elasticJobConfigBean.toString(), e);
        }

    }


    /**
     * @param jobClass
     * @return
     */
    private ElasticJob getInstance(Class<? extends ElasticJob> jobClass) {
        ElasticJob bean = ApplicationContextHolder.getBean(jobClass);
        return bean;
    }

    /**
     * 注册SpringJobScheduler
     *
     * @param elasticJob
     * @param elasticJobConfigBean
     * @return
     */
    private SpringJobScheduler jobScheduler(ElasticJob elasticJob, ElasticJobConfig elasticJobConfigBean) {
        LiteJobConfiguration build = LiteJobConfiguration.newBuilder(jobConfiguration(elasticJob, elasticJobConfigBean))
                .overwrite(true).build();
        SpringJobScheduler springJobScheduler = new SpringJobScheduler(elasticJob, zookeeperRegistryCenter, build,
                jobEventConfiguration);
        return springJobScheduler;
    }

    /**
     * job配置
     *
     * @param elasticJob
     * @param elasticJobConfigBean
     * @return
     */
    private JobTypeConfiguration jobConfiguration(final ElasticJob elasticJob,
                                                  ElasticJobConfig elasticJobConfigBean) {
        JobCoreConfiguration jobCoreConfiguration = JobCoreConfiguration
                .newBuilder(elasticJobConfigBean.getJobName(), elasticJobConfigBean.getCron(),
                        elasticJobConfigBean.getShardingTotalCount())
                .shardingItemParameters(elasticJobConfigBean.getShardingItemParameters())
                //.misfire(Boolean.valueOf(elasticJobConfigBean.getMisfire()))
                .description(elasticJobConfigBean.getDescription())
                //.failover(Boolean.valueOf(elasticJobConfigBean.getFailover()))
                .jobParameter(elasticJobConfigBean.getJobParameter())
                .build();
        if (elasticJob instanceof SimpleJob) {
            return new SimpleJobConfiguration(jobCoreConfiguration, elasticJob.getClass().getCanonicalName());
        }
//        if (elasticJob instanceof DataflowJob) {
//            return new DataflowJobConfiguration(jobCoreConfiguration, elasticJob.getClass().getCanonicalName(),
//                    Boolean.valueOf(elasticJobConfigBean.getStreamingProcess()));
//        }
        throw new RuntimeException("未知类型定时任务：" + elasticJob.getClass().getName());
    }


}

任务配置config，引入DateSource，在加载时，读取数据库任务配置表

package cn.quantgroup.cashloanflow.config.jobevent;

import com.dangdang.ddframe.job.event.JobEventConfiguration;
import com.dangdang.ddframe.job.event.rdb.JobEventRdbConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import javax.annotation.Resource;
import javax.sql.DataSource;

/**
 * 为job配置数据源
 * Created with suntao on 2019/3/19
 */
@Configuration
public class JobEventConfig {

    @Resource
    private DataSource dataSource;

    @Bean
    public JobEventConfiguration jobEventConfiguration() {
        return new JobEventRdbConfiguration(dataSource);
    }
}

sql:

CREATE TABLE `elastic_job_config` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `job_name` varchar(255) NOT NULL COMMENT '作业名称',
  `cron` varchar(50) NOT NULL COMMENT 'cron表达式，用于控制作业触发时间',
  `sharding_total_count` int(3) NOT NULL COMMENT '作业分片总数',
  `sharding_item_parameters` varchar(512) DEFAULT NULL COMMENT '分片序列,列号从0开始,不可大于或等于作业分片总数.如：0=a,1=b,2=c',
  `job_parameter` varchar(512) DEFAULT NULL COMMENT '作业自定义参数',
  `failover` tinyint(4) DEFAULT NULL COMMENT '是否开启任务执行失效转移',
  `misfire` tinyint(4) DEFAULT NULL COMMENT '是否开启错过任务重新执行',
  `description` varchar(512) DEFAULT NULL COMMENT '作业描述信息',
  `job_class` varchar(512) NOT NULL COMMENT '作业实现类',
  `streaming_process` tinyint(4) DEFAULT NULL COMMENT '是否流式处理数据（只有Dataflow类型支持）',
  `job_config` text,
  `is_active` tinyint(4) NOT NULL DEFAULT '1' COMMENT '是否有效，0：否，1：是，default：1',
  `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
  PRIMARY KEY (`id`),
  UNIQUE KEY `jobName` (`job_name`)
) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=utf8;

INSERT INTO `elastic_job_config` (`job_name`, `cron`, `sharding_total_count`, `sharding_item_parameters`, `job_parameter`, `failover`, `misfire`, `description`, `job_class`, `streaming_process`, `job_config`, `is_active`, `created_at`)
VALUES
	('callbackFailSimpleJob', '0 0 0/1 * * ? ', 1, '', NULL, NULL, NULL, '回调失败监控', 'cn.quantgroup.cashloanflow.config.jobevent.job.CallbackFailSimpleJob', NULL, NULL, 1, '2019-04-02 18:05:08'),
	('orderMappingIncrementJob', '0 */10 8-23 * * ?', 1, '', '{\"monitorDays\":7, \"fluctuate\":{\"low\":0.6, \"up\":2, \"absoluteValue\":10}, \"scanMins\":10, \"channelIdList\":[333,158748,158888,159507,159509,159513,159537,159538,159541,159561,159562,159563,159583,159584,159672,159726]}', NULL, NULL, '进件流量监控任务', 'cn.quantgroup.cashloanflow.config.jobevent.job.OrderMappingIncrementSimpleJob', NULL, NULL, 1, '2019-04-03 14:06:35');

任务额外自定义配置可以加载job_parameter中，通过

String jobParameter = shardingContext.getJobParameter();

获取自定义配置

可以通过加载器动态加载任务，当任务需要更改配置时，服务有不想重新启动，通过自定义接口调用reloadJobByIds方法。

/**
     * 重新加载任务
     * @param ids
     */
    public void reloadJobByIds(List<Long> ids) {
        List<ElasticJobConfig> elasticJobConfigList = elasticJobConfigRepository.findAll(ids);
        elasticJobConfigList.forEach(elasticJobConfig -> registryJob(elasticJobConfig));
    }

二、通过当当elastic job二次开发，再次是应用的开发好的jar，这种方式优点是每个定时任务，通过注解方式配置，方便阅读，缺点是不能动态加载任务配置，当定时任务很多时，不方便管理。

1、引入jar

<dependency>
            <groupId>com.github.kuhn-he</groupId>
            <artifactId>elastic-job-lite-spring-boot-starter</artifactId>
            <version>2.1.5</version>
        </dependency>

该jar 是有当当网elastic job封装升级的jar

2、创建job执行类

@Component
@Slf4j
@ElasticSimpleJob(cron="* * * * * ?",jobName="test123",shardingTotalCount=2,jobParameter="测试参数",shardingItemParameters="0=A,1=B")
public class MySimpleJob implements SimpleJob {

    @Override
    public void execute(ShardingContext shardingContext) {
        log.info("itemId={}", shardingContext.getShardingItem());
    }
}

三、启动spring boot，定时任务即启动

1、shardingTotalCount=2，将定时任务分成两个片区，如果一台server执行，那么两个片区都由该server执行

打印结果：

2018-07-13 17:17:14.204  INFO 88070 --- [g.MySimpleJob-1] c.f.t.e.config.MySimpleJob               : itemId=0
2018-07-13 17:17:14.205  INFO 88070 --- [g.MySimpleJob-2] c.f.t.e.config.MySimpleJob               : itemId=1
2018-07-13 17:17:15.175  INFO 88070 --- [g.MySimpleJob-4] c.f.t.e.config.MySimpleJob               : itemId=0
2018-07-13 17:17:15.175  INFO 88070 --- [g.MySimpleJob-3] c.f.t.e.config.MySimpleJob               : itemId=1
2018-07-13 17:17:16.168  INFO 88070 --- [g.MySimpleJob-6] c.f.t.e.config.MySimpleJob               : itemId=0
2018-07-13 17:17:16.168  INFO 88070 --- [g.MySimpleJob-5] c.f.t.e.config.MySimpleJob               : itemId=1

此时只有一个server节点在zk注册，任务全部由该server执行

如果有多台server接入，如 server1，server2

那么server1得到分片itemId=0，server2分片itemid=1.

如果两台server，分成6片，或者7片

shardingItemParameters="0=A,1=B,2=C,3=D,4=E,5=F"

那么分片情况

server1：0，1，2

server2：3，4，5

或者分成7片：shardingItemParameters="0=A,1=B,2=C,3=D,4=E,5=F,6=G"

 

server1：0，1，2

server2：3，4，5，6

如果当server1或者server2宕机，那么会重新选举，重新负载任务到可用server

默认shardingTotalCount=1，由其中一台主server执行，其他为福server，当主server宕机，副server会顶替。

2、elastic-job定位为分布式定时任务，当任务数量级非常大的时候，可以根据需要加机器，解决定时任务原有串行执行效率问题，在解决中可以根据itemId对数据进行分组，解决重复执行问题。