JAVA定时器每日删除ES过期索引
背景:在Linux服务器上,设置一个后台进程,每天0点10分自动删除ELK中Elasticsearch中超过三个月的旧索引,使用Quartz实现。
废话少说,直接上代码!!!
目录:
1、源码:
2、调用方法:
1、源码:
(1)QuartzCli.java
package com.remoa.quartzCli;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.quartz.*;
import org.quartz.impl.StdSchedulerFactory;
/**
* This class is used to use Quartz
* @author remoa
* @since 2017.09.18
*/
public class QuartzCli{
//日志记录
private static final Log LOG = LogFactory.getLog(QuartzCli.class);
//后台流程每天0点10分开始跑
//public static final String INDEX_DELETE_CRON = "0 10 0 * * ?";
//后台流程每10秒开始跑
private static final String INDEX_DELETE_CRON = "*/10 * * * * ?";
//定义Job名字
private static final String JOB_NAME = "Index_quartz_cli";
/**
* 启动定时任务
*/
public void start(){
System.out.println("Start quartz mode: " + INDEX_DELETE_CRON);
LOG.info("Start quartz mode: " + INDEX_DELETE_CRON);
//SchedulerFactory:提供用于获取调度程序实例的客户端可用句柄的机制
SchedulerFactory schedulerFactory = new StdSchedulerFactory();
//Scheduler:调度程序,接口,代表一个独立运行容器,调度程序维护JobDetail和触发器的注册表,一旦注册,调度程序负责执行作业。
Scheduler scheduler = null;
try {
//设置调度程序的实现类
scheduler = schedulerFactory.getScheduler();
} catch (SchedulerException e) {
LOG.error(e.getMessage(), e);
}
//实例化JobDetail调度任务,JobDetail存放作业的状态,JobDetail对象储存作业的侦听器、群组、数据映射、描述以及作业的其它属性。
//newJob方法设置需要执行的Job任务类名
JobBuilder jobBuilder = JobBuilder.newJob(MyJob.class);
//withIdentity设置JobDetail的name和group
jobBuilder.withIdentity(JOB_NAME, "remoaGroup");
//JobDetail是一个接口,必须通过JobBuilder的build方法实例化JobDetail
JobDetail job = jobBuilder.build();
//实例化Trigger触发器,实现对作业的调度。
CronTrigger cronTrigger = (CronTrigger)TriggerBuilder.newTrigger()
.withIdentity("Cron_Trigger" + System.currentTimeMillis(), "remoaGroup")
.withSchedule(CronScheduleBuilder.cronSchedule(INDEX_DELETE_CRON))
.startNow()
.build();
//JobDataMap是java map的具体实现,并添加了一些便利的方法用于存储与读取原生类型数据,可以为Job实例提供属性/配置
job.getJobDataMap()
.put("executor", this);
try {
//为调度程序设置job和trigger
scheduler.scheduleJob(job, cronTrigger);
//调用start开始调度
scheduler.start();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
public static void main(String[] args) {
QuartzCli quartzCli = new QuartzCli();
quartzCli.start();
}
}
(2)MyJob.java
package com.remoa.quartzCli;
import com.remoa.index.IndexProcess;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Properties;
/**
* This class is used to define job
* @author remoa
* @since 2017.09.18
*/
public class MyJob implements Job {
private static final Log LOG = LogFactory.getLog(MyJob.class);
//在Quartz中,所有的任务都必须实现Job接口,Job中只有一个execute(JobExecutionContext jobExecutionContext)实现方法,具体业务实现需要写在这个方法里,这个方法将会在触发器满足调度条件时触发。
@Override
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
System.out.println(sdf.format(new Date()) + "------------- start job");
LOG.info(sdf.format(new Date()) + "------------- start job");
InputStream input = MyJob.class.getClassLoader().getResourceAsStream("indexDelete.properties");
Properties properties = new Properties();
try {
properties.load(input);
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
IndexProcess indexProcess = null;
try {
indexProcess = new IndexProcess(properties.getProperty("keystorePath"), properties.getProperty("truststorePath"), properties.getProperty("keypassword"), properties.getProperty("trustpassword"), properties.getProperty("elasticsearchHost"), properties.getProperty("elasticsearchClusterName"));
} catch (Throwable throwable) {
LOG.error(throwable.getMessage());
throwable.printStackTrace();
}
System.out.println("get index-------------");
List<String> list = indexProcess.getIndex();
System.out.println("Total number of lists :" + list.size());
LOG.info("Total number of lists :" + list.size());
String keyword = indexProcess.getKeyword();
indexProcess.deleteOldIndex(list, keyword);
}
}
(3)IndexProcess.java
package com.remoa.index;
import com.floragunn.searchguard.ssl.SearchGuardSSLPlugin;
import com.floragunn.searchguard.ssl.util.SSLConfigConstants;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Iterator;
import java.util.List;
/**
* This class is used to delete old index more than 100 days in elasticsearch.
* @author remoa
* @since 2017.09.18
*/
public class IndexProcess {
private TransportClient client;
private AdminClient adminClient;
public static final int DATE_NUMBER = -90;
/**
* 构造方法,提供连接Elasticsearch的相关参数并使用传输客户端TransportClient连接到Elasticsearch集群
* @param keystorePath KeyStore签名证书位置
* @param truststorePath Truststore签名证书位置
* @param keypwd KeyStore签名证书密码
* @param trustpwd Truststore签名证书密码
* @param esHost Elasticsearch集群服务器
* @param esClusterName Elasticsearch集群名称
* @throws Throwable 异常不单独处理了,直接在方法中抛出
*/
public IndexProcess(String keystorePath, String truststorePath, String keypwd, String trustpwd, String
esHost, String esClusterName) throws Throwable{
Settings settings = Settings.builder()
//ES集群名称
.put("cluster.name", esClusterName)
.put("searchguard.ssl.transport.enabled", true)
//私钥证书位置
.put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_KEYSTORE_FILEPATH, keystorePath)
//公钥证书位置
.put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_TRUSTSTORE_FILEPATH, truststorePath)
//keystore一般保存我们的私钥,用来加密解密或者为别人做签名。
.put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_KEYSTORE_PASSWORD, keypwd)
//truststore里存放的是只包含公钥的数字证书。
.put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_TRUSTSTORE_PASSWORD, trustpwd)
.put(SSLConfigConstants.SEARCHGUARD_SSL_TRANSPORT_ENFORCE_HOSTNAME_VERIFICATION, false)
.put("path.home", ".")
//使得客户端去嗅探整个集群的状态,把集群中其它机器的IP地址加到客户端中,设置为true则不用手动设置集群里所有机器的IP连接到客户端,它会自动帮助添加,并自动发现新加入集群的机器。
.put("client.transport.sniff", true)
.build();
client = new PreBuiltTransportClient(settings, SearchGuardSSLPlugin.class)
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), 9301))
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), 9300));
adminClient = client.admin();
}
/**
* 根据索引名删除索引
* @param index 索引名
*/
public void deleteIndex(String index){
adminClient.indices().
delete(new DeleteIndexRequest(index)).
actionGet();
}
/**
* 得到全部索引
* @return 返回List封装的全部索引的名字
*/
public List<String> getIndex(){
List<String> list = new ArrayList<String>();
String [] indices = adminClient.indices()
.prepareGetIndex()
.setFeatures()
.get()
.getIndices();
for(int i = 0; i < indices.length; i++){
list.add(indices[i]);
}
return list;
}
/**
* 得到过滤的关键字,即xxxx.xx.xx(年月日字符串),这里设置为90天
* @return 过滤关键字
*/
public String getKeyword(){
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE, DATE_NUMBER);
String year = String.valueOf(calendar.get(Calendar.YEAR));
String month = String.valueOf(calendar.get(Calendar.MONTH) + 1);
String day = String.valueOf(calendar.get(Calendar.DATE));
if(month.length() == 1){
month = "0" + month;
}
if(day.length() == 1){
day = "0" + day;
}
String keyword = year + "." + month + "." + day;
return keyword;
}
/**
* 删除所有的超期的旧索引
* @param list 索引列表
* @param keyword 过滤关键字
*/
public void deleteOldIndex(List<String> list, String keyword){
int count = 0;
for(Iterator<String> iter = list.iterator(); iter.hasNext(); ){
String str = iter.next();
//索引名称字段长度比10小的话就不作为删除对象了
if(str.length() > 10){
String dateStr = str.substring(str.length() - 10 ,str.length());//取出索引名称字段的结尾字段
//索引名称字段最后不是以日期结尾的话也不作为删除对象
String patternStr = "^[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}$";
if(dateStr.matches(patternStr)){
if(str.substring(str.length() - 10 ,str.length()).compareTo(keyword) < 0){
System.out.println("index: " + str + " is deleted");
//this.deleteIndex(str);
count++;
}
}
}
}
System.out.println("The total delete index is: " + count);
}
}
(4)log4j2.xml
<?xml version="1.0" encoding="UTF-8" ?>
<!-- monitorInterval:自动检测修改配置文件和重新配置本身,设置间隔秒数-->
<Configuration stauts="WARN" monitorInterval="60">
<!-- Appender附加器对象,负责输出日志信息到不同的地方,如数据库,文件,控制台等等-->
<Appenders>
<Console name="Console" target="SYSTEM_ERR">
<!--
输出日志的格式
%-5p:用空格右垫,当事件级别的名称少于五个字符
%F:用于输出被发出日志记录请求的文件名,带.java后缀及包名称
%L:用于输出从被发出日志记录请求的行号
%m:输出记录事件的消息内容
%x:用于与产生该日志事件的线程相关联输出的NDC
%n:输出行分隔符或文字
%t:用于输出生成的日志记录事件的线程的名称,比如在main方法中,将输出main
-->
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} [%-5p] --> %F(%L): %m %x %n" />
</Console>
<!-- 每次大小超过size,则size大小的日志会自动存入按年份-月份建立的文件夹下面并进行压缩,作为存档。 -->
<RollingFile name="RollingFileInfo" fileName="/root/remoa/logs/deleteIndex.log" filePattern="/root/remoa/logs/$${date:yyyy-MM}/deleteIndex-%d{yyyy-MM-dd}-%i.log">
<!-- 只输出level及以上级别的事件,其它的直接拒绝 -->
<ThresholdFilter level="warn" onMatch="ACCEPT" onMismatch="DENY" />
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss,SSS} [%C{2}] [%p %L] [%t] %m [ %x] %n" />
<Policies>
<TimeBasedTriggerPolicy/>
<SizeBasedTriggerPolicy size="100MB" />
</Policies>
<!-- 设置默认同一个文件夹下文件数目,默认为7个,这里设置为10个 -->
<DefaultRolloverStrategy max="10" />
</RollingFile>
</Appenders>
<!-- 定义记录器对象,Logger对象负责捕获日志信息及它们存储在一个空间的层次结构 -->
<Loggers>
<!-- 全局的级别为info,可通过具体appender中的ThresholdFilter进行设置别的级别 -->
<Root level="info">
<!-- 只有引入了logger并引入的appender,appender才会生效 -->
<AppenderRef ref="Console" />
<AppenderRef ref="RollingFileInfo" />
</Root>
</Loggers>
</Configuration>
(5)pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.remoa</groupId>
<artifactId>indexTest</artifactId>
<version>1.0-SNAPSHOT</version>
<build>
<finalName>indexTest</finalName>
<!-- 默认源代码和资源文件目录配置 -->
<sourceDirectory>src/main/java </sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<testResources>
<testResource>
<directory>src/test/resources</directory>
</testResource>
</testResources>
<plugins>
<!-- 编译插件,处理maven项目管理因为版本不一致导致编译不通过的问题 -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<!-- 源代码使用的开发版本 -->
<source>1.8</source>
<!-- 需要生成的目标class文件的编译版本 -->
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<!-- 将依赖的某个jar包内部的类或者资源include/exclude掉-->
<!-- 通过maven-shade-plugin插件生成一个uber-jar,它包含所有的依赖jar包-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>jar-with-dependencies</shadedClassifierName>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<!-- https://mvnrepository.com/artifact/org.quartz-scheduler/quartz -->
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
<version>2.2.1</version>
</dependency>
<dependency>
<groupId>com.floragunn</groupId>
<artifactId>search-guard-5</artifactId>
<version>5.2.2-15</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.floragunn/search-guard-ssl -->
<dependency>
<groupId>com.floragunn</groupId>
<artifactId>search-guard-ssl</artifactId>
<version>5.2.2-22</version>
</dependency>
<!-- https://mvnrepository.com/artifact/log4j/log4j -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>5.2.2</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.6.6</version>
</dependency>
</dependencies>
</project>
2、调用方法:
使用mvn进行打包部署后,进入target目录,输入nohup java -cp /root/remoa/indexTest/target/indexTest-1.0-SNAPSHOT-jar-with-dependencies.jar com.remoa.quartzCli.QuartzCli &将其设置为后台进程即可。