声明：本系列博客是根据SGG的视频整理而成，非常适合大家入门学习。
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>flink-learning-connectors</artifactId>
<groupId>com.zhisheng.flink</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>flink-learning-connectors-clickhouse</artifactId>
<properties>
<async.client.version>2.0.39</async.client.version>
<typesafe.config.version>1.3.3</typesafe.config.version>
</properties>
<dependencies>
<dependency>
<groupId>org.asynchttpclient</groupId>
<artifactId>async-http-client</artifactId>
<version>${async.client.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>${typesafe.config.version}</version>
</dependency>
</dependencies>
</project>
工程目录：
src/main
java/com/zhisheng/connectors/clickhouse
applied
ClickhouseSinkBuffer.java
ClickhouseSinkManager.java
ClickhouseSinkScheduledChecker.java
ClickhouseWriter.java
model
ClickhouseClusterSettings.java
ClickhouseRequestBlank.java
ClickhouseSinkCommonParams.java
ClickhouseSinkConsts.java
util
ConfigUtil.java
ThreadUtil.java
ClickhouseSink.java
resources
README.md
pom.xml
代码：
ClickhouseSinkBuffer.java
package com.zhisheng.connectors.clickhouse.applied;
import com.google.common.base.Preconditions;
import com.zhisheng.connectors.clickhouse.model.ClickhouseRequestBlank;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* Desc:
*/
public class ClickhouseSinkBuffer implements AutoCloseable {
private static final Logger logger = LoggerFactory.getLogger(ClickhouseSinkBuffer.class);
private final ClickhouseWriter writer;
private final String targetTable;
private final int maxFlushBufferSize;
private final long timeoutMillis;
private final List<String> localValues;
private volatile long lastAddTimeMillis = 0L;
private ClickhouseSinkBuffer(
ClickhouseWriter chWriter,
long timeout,
int maxBuffer,
String table
) {
writer = chWriter;
localValues = new ArrayList<>();
timeoutMillis = timeout;
maxFlushBufferSize = maxBuffer;
targetTable = table;
logger.info("Instance Clickhouse Sink, target table = {}, buffer size = {}", this.targetTable, this.maxFlushBufferSize);
}
String getTargetTable() {
return targetTable;
}
public void put(String recordAsCSV) {
tryAddToQueue();
localValues.add(recordAsCSV);
lastAddTimeMillis = System.currentTimeMillis();
}
synchronized void tryAddToQueue() {
if (flushCondition()) {
addToQueue();
}
}
private void addToQueue() {
List<String> deepCopy = buildDeepCopy(localValues);
ClickhouseRequestBlank params = ClickhouseRequestBlank.Builder
.aBuilder()
.withValues(deepCopy)
.withTargetTable(targetTable)
.build();
logger.debug("Build blank with params: buffer size = {}, target table = {}", params.getValues().size(), params.getTargetTable());
writer.put(params);
localValues.clear();
}
private boolean flushCondition() {
return localValues.size() > 0 && (checkSize() || checkTime());
}
private boolean checkSize() {
return localValues.size() >= maxFlushBufferSize;
}
private boolean checkTime() {
if (lastAddTimeMillis == 0) {
return false;
}
long current = System.currentTimeMillis();
return current - lastAddTimeMillis > timeoutMillis;
}
private static List<String> buildDeepCopy(List<String> original) {
return Collections.unmodifiableList(new ArrayList<>(original));
}
@Override
public void close() {
if (localValues != null && localValues.size() > 0) {
addToQueue();
}
}
public static final class Builder {
private String targetTable;
private int maxFlushBufferSize;
private int timeoutSec;
private Builder() {
}
public static Builder aClickhouseSinkBuffer() {
return new Builder();
}
public Builder withTargetTable(String targetTable) {
this.targetTable = targetTable;
return this;
}
public Builder withMaxFlushBufferSize(int maxFlushBufferSize) {
this.maxFlushBufferSize = maxFlushBufferSize;
return this;
}
public Builder withTimeoutSec(int timeoutSec) {
this.timeoutSec = timeoutSec;
return this;
}
public ClickhouseSinkBuffer build(ClickhouseWriter writer) {
Preconditions.checkNotNull(targetTable);
Preconditions.checkArgument(maxFlushBufferSize > 0);
Preconditions.checkArgument(timeoutSec > 0);
return new ClickhouseSinkBuffer(
writer,
TimeUnit.SECONDS.toMillis(this.timeoutSec),
this.maxFlushBufferSize,
this.targetTable
);
}
}
}
ClickhouseSinkManager.java
package com.zhisheng.connectors.clickhouse.applied;
import com.google.common.base.Preconditions;
import com.zhisheng.connectors.clickhouse.model.ClickhouseSinkCommonParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
import java.util.Properties;
import static com.zhisheng.connectors.clickhouse.model.ClickhouseSinkConsts.MAX_BUFFER_SIZE;
import static com.zhisheng.connectors.clickhouse.model.ClickhouseSinkConsts.TARGET_TABLE_NAME;
/**
* Desc:
*/
public class ClickhouseSinkManager implements AutoCloseable {
private static final Logger logger = LoggerFactory.getLogger(ClickhouseSinkManager.class);
private final ClickhouseWriter clickhouseWriter;
private final ClickhouseSinkScheduledChecker clickhouseSinkScheduledChecker;
private final ClickhouseSinkCommonParams sinkParams;
private volatile boolean isClosed = false;
public ClickhouseSinkManager(Map<String, String> globalParams) {
sinkParams = new ClickhouseSinkCommonParams(globalParams);
clickhouseWriter = new ClickhouseWriter(sinkParams);
clickhouseSinkScheduledChecker = new ClickhouseSinkScheduledChecker(sinkParams);
logger.info("Build sink writer's manager. params = {}", sinkParams.toString());
}
public ClickhouseSinkBuffer buildBuffer(Properties localProperties) {
String targetTable = localProperties.getProperty(TARGET_TABLE_NAME);
int maxFlushBufferSize = Integer.valueOf(localProperties.getProperty(MAX_BUFFER_SIZE));
return buildBuffer(targetTable, maxFlushBufferSize);
}
public ClickhouseSinkBuffer buildBuffer(String targetTable, int maxBufferSize) {
Preconditions.checkNotNull(clickhouseSinkScheduledChecker);
Preconditions.checkNotNull(clickhouseWriter);
ClickhouseSinkBuffer clickhouseSinkBuffer = ClickhouseSinkBuffer.Builder
.aClickhouseSinkBuffer()
.withTargetTable(targetTable)
.withMaxFlushBufferSize(maxBufferSize)
.withTimeoutSec(sinkParams.getTimeout())
.build(clickhouseWriter);
clickhouseSinkScheduledChecker.addSinkBuffer(clickhouseSinkBuffer);
return clickhouseSinkBuffer;
}
public boolean isClosed() {
return isClosed;
}
@Override
public void close() throws Exception {
clickhouseWriter.close();
clickhouseSinkScheduledChecker.close();
isClosed = true;
}
}
ClickhouseSinkScheduledChecker.java
package com.zhisheng.connectors.clickhouse.applied;
import com.zhisheng.connectors.clickhouse.model.ClickhouseSinkCommonParams;
import com.zhisheng.connectors.clickhouse.util.ThreadUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
/**
* Desc:
*/
public class ClickhouseSinkScheduledChecker implements AutoCloseable {
private static final Logger logger = LoggerFactory.getLogger(ClickhouseSinkScheduledChecker.class);
private final ScheduledExecutorService scheduledExecutorService;
private final List<ClickhouseSinkBuffer> clickhouseSinkBuffers;
private final ClickhouseSinkCommonParams params;
public ClickhouseSinkScheduledChecker(ClickhouseSinkCommonParams props) {
clickhouseSinkBuffers = new ArrayList<>();
params = props;
ThreadFactory factory = ThreadUtil.threadFactory("clickhouse-writer-checker");
scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(factory);
scheduledExecutorService.scheduleWithFixedDelay(getTask(), params.getTimeout(), params.getTimeout(), TimeUnit.SECONDS);
logger.info("Build Sink scheduled checker, timeout (sec) = {}", params.getTimeout());
}
public void addSinkBuffer(ClickhouseSinkBuffer clickhouseSinkBuffer) {
synchronized (this) {
clickhouseSinkBuffers.add(clickhouseSinkBuffer);
}
logger.debug("Add sinkBuffer, target table = {}", clickhouseSinkBuffer.getTargetTable());
}
private Runnable getTask() {
return () -> {
synchronized (this) {
logger.debug("Start checking buffers. Current count of buffers = {}", clickhouseSinkBuffers.size());
clickhouseSinkBuffers.forEach(ClickhouseSinkBuffer::tryAddToQueue);
}
};
}
@Override
public void close() throws Exception {
ThreadUtil.shutdownExecutorService(scheduledExecutorService);
}
}
ClickhouseWriter.java
package com.zhisheng.connectors.clickhouse.applied;
import com.google.common.collect.Lists;
import com.zhisheng.connectors.clickhouse.model.ClickhouseRequestBlank;
import com.zhisheng.connectors.clickhouse.model.ClickhouseSinkCommonParams;
import com.zhisheng.connectors.clickhouse.util.ThreadUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.netty.handler.codec.http.HttpHeaders;
import org.asynchttpclient.*;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.concurrent.*;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午10:09
*/
public class ClickhouseWriter implements AutoCloseable {
private static final Logger logger = LoggerFactory.getLogger(ClickhouseWriter.class);
private ExecutorService service;
private ExecutorService callbackService;
private List<WriterTask> tasks;
private BlockingQueue<ClickhouseRequestBlank> commonQueue;
private AsyncHttpClient asyncHttpClient;
private ClickhouseSinkCommonParams sinkParams;
public ClickhouseWriter(ClickhouseSinkCommonParams sinkParams) {
this.sinkParams = sinkParams;
initDirAndExecutors();
}
private void initDirAndExecutors() {
try {
initDir(sinkParams.getFailedRecordsPath());
buildComponents();
} catch (Exception e) {
logger.error("Error while starting CH writer", e);
throw new RuntimeException(e);
}
}
private static void initDir(String pathName) throws IOException {
Path path = Paths.get(pathName);
Files.createDirectories(path);
}
private void buildComponents() {
asyncHttpClient = Dsl.asyncHttpClient();
int numWriters = sinkParams.getNumWriters();
commonQueue = new LinkedBlockingQueue<>(sinkParams.getQueueMaxCapacity());
ThreadFactory threadFactory = ThreadUtil.threadFactory("clickhouse-writer");
service = Executors.newFixedThreadPool(sinkParams.getNumWriters(), threadFactory);
ThreadFactory callbackServiceFactory = ThreadUtil.threadFactory("clickhouse-writer-callback-executor");
int cores = Runtime.getRuntime().availableProcessors();
int coreThreadsNum = Math.max(cores / 4, 2);
callbackService = new ThreadPoolExecutor(
coreThreadsNum,
Integer.MAX_VALUE,
60L,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
callbackServiceFactory);
tasks = Lists.newArrayList();
for (int i = 0; i < numWriters; i++) {
WriterTask task = new WriterTask(i, asyncHttpClient, commonQueue, sinkParams, callbackService);
tasks.add(task);
service.submit(task);
}
}
public void put(ClickhouseRequestBlank params) {
try {
commonQueue.put(params);
} catch (InterruptedException e) {
logger.error("Interrupted error while putting data to queue", e);
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
}
private void stopWriters() {
if (tasks != null && tasks.size() > 0) {
tasks.forEach(WriterTask::setStopWorking);
}
}
@Override
public void close() throws Exception {
logger.info("Closing clickhouse-writer...");
stopWriters();
ThreadUtil.shutdownExecutorService(service);
ThreadUtil.shutdownExecutorService(callbackService);
asyncHttpClient.close();
logger.info("{} is closed", ClickhouseWriter.class.getSimpleName());
}
static class WriterTask implements Runnable {
private static final Logger logger = LoggerFactory.getLogger(WriterTask.class);
private static final int HTTP_OK = 200;
private final BlockingQueue<ClickhouseRequestBlank> queue;
private final ClickhouseSinkCommonParams sinkSettings;
private final AsyncHttpClient asyncHttpClient;
private final ExecutorService callbackService;
private final int id;
private volatile boolean isWorking;
WriterTask(int id,
AsyncHttpClient asyncHttpClient,
BlockingQueue<ClickhouseRequestBlank> queue,
ClickhouseSinkCommonParams settings,
ExecutorService callbackService
) {
this.id = id;
this.sinkSettings = settings;
this.queue = queue;
this.callbackService = callbackService;
this.asyncHttpClient = asyncHttpClient;
}
@Override
public void run() {
try {
isWorking = true;
logger.info("Start writer task, id = {}", id);
while (isWorking || queue.size() > 0) {
ClickhouseRequestBlank blank = queue.poll(300, TimeUnit.MILLISECONDS);
if (blank != null) {
send(blank);
}
}
} catch (Exception e) {
logger.error("Error while inserting data", e);
throw new RuntimeException(e);
} finally {
logger.info("Task id = {} is finished", id);
}
}
private void send(ClickhouseRequestBlank requestBlank) {
Request request = buildRequest(requestBlank);
logger.debug("Ready to load data to {}, size = {}", requestBlank.getTargetTable(), requestBlank.getValues().size());
ListenableFuture<Response> whenResponse = asyncHttpClient.executeRequest(request);
Runnable callback = responseCallback(whenResponse, requestBlank);
whenResponse.addListener(callback, callbackService);
}
private Request buildRequest(ClickhouseRequestBlank requestBlank) {
String resultCSV = String.join(" , ", requestBlank.getValues());
String query = String.format("INSERT INTO %s VALUES %s", requestBlank.getTargetTable(), resultCSV);
String host = sinkSettings.getClickhouseClusterSettings().getRandomHostUrl();
BoundRequestBuilder builder = asyncHttpClient
.preparePost(host)
.setHeader(HttpHeaders.Names.CONTENT_TYPE, "text/plain; charset=utf-8")
.setBody(query);
if (sinkSettings.getClickhouseClusterSettings().isAuthorizationRequired()) {
builder.setHeader(HttpHeaders.Names.AUTHORIZATION, "Basic " + sinkSettings.getClickhouseClusterSettings().getCredentials());
}
return builder.build();
}
private Runnable responseCallback(ListenableFuture<Response> whenResponse, ClickhouseRequestBlank requestBlank) {
return () -> {
Response response = null;
try {
response = whenResponse.get();
if (response.getStatusCode() != HTTP_OK) {
handleUnsuccessfulResponse(response, requestBlank);
} else {
logger.info("Successful send data to Clickhouse, batch size = {}, target table = {}, current attempt = {}",
requestBlank.getValues().size(),
requestBlank.getTargetTable(),
requestBlank.getAttemptCounter());
}
} catch (Exception e) {
logger.error("Error while executing callback, params = {}", sinkSettings, e);
try {
handleUnsuccessfulResponse(response, requestBlank);
} catch (Exception error) {
logger.error("Error while handle unsuccessful response", error);
}
}
};
}
private void handleUnsuccessfulResponse(Response response, ClickhouseRequestBlank requestBlank) throws Exception {
int currentCounter = requestBlank.getAttemptCounter();
if (currentCounter > sinkSettings.getMaxRetries()) {
logger.warn("Failed to send data to Clickhouse, cause: limit of attempts is exceeded. Clickhouse response = {}. Ready to flush data on disk", response);
logFailedRecords(requestBlank);
} else {
requestBlank.incrementCounter();
logger.warn("Next attempt to send data to Clickhouse, table = {}, buffer size = {}, current attempt num = {}, max attempt num = {}, response = {}",
requestBlank.getTargetTable(),
requestBlank.getValues().size(),
requestBlank.getAttemptCounter(),
sinkSettings.getMaxRetries(),
response);
queue.put(requestBlank);
}
}
private void logFailedRecords(ClickhouseRequestBlank requestBlank) throws Exception {
String filePath = String.format("%s/%s_%s",
sinkSettings.getFailedRecordsPath(),
requestBlank.getTargetTable(),
System.currentTimeMillis());
try (PrintWriter writer = new PrintWriter(filePath)) {
List<String> records = requestBlank.getValues();
records.forEach(writer::println);
writer.flush();
}
logger.info("Successful send data on disk, path = {}, size = {} ", filePath, requestBlank.getValues().size());
}
void setStopWorking() {
isWorking = false;
}
}
}
ClickhouseClusterSettings.java
package com.zhisheng.connectors.clickhouse.model;
import com.google.common.base.Preconditions;
import com.zhisheng.connectors.clickhouse.util.ConfigUtil;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午12:40
*/
public class ClickhouseClusterSettings {
public static final String CLICKHOUSE_HOSTS = "clickhouse.access.hosts";
public static final String CLICKHOUSE_USER = "clickhouse.access.user";
public static final String CLICKHOUSE_PASSWORD = "clickhouse.access.password";
private final List<String> hostsWithPorts;
private final String user;
private final String password;
private final String credentials;
private final boolean authorizationRequired;
private int currentHostId = 0;
public ClickhouseClusterSettings(Map<String, String> parameters) {
Preconditions.checkNotNull(parameters);
String hostsString = parameters.get(CLICKHOUSE_HOSTS);
Preconditions.checkNotNull(hostsString);
hostsWithPorts = buildHostsAndPort(hostsString);
Preconditions.checkArgument(hostsWithPorts.size() > 0);
String usr = parameters.get(CLICKHOUSE_USER);
String pass = parameters.get(CLICKHOUSE_PASSWORD);
if (StringUtils.isNotEmpty(usr) && StringUtils.isNotEmpty(pass)) {
user = parameters.get(CLICKHOUSE_USER);
password = parameters.get(CLICKHOUSE_PASSWORD);
credentials = buildCredentials(user, password);
authorizationRequired = true;
} else {
// avoid NPE
credentials = "";
password = "";
user = "";
authorizationRequired = false;
}
}
private static List<String> buildHostsAndPort(String hostsString) {
return Arrays.stream(hostsString
.split(ConfigUtil.HOST_DELIMITER))
.map(ClickhouseClusterSettings::checkHttpAndAdd)
.collect(Collectors.toList());
}
private static String checkHttpAndAdd(String host) {
String newHost = host.replace(" ", "");
if (!newHost.contains("http")) {
return "http://" + newHost;
}
return newHost;
}
private static String buildCredentials(String user, String password) {
Base64.Encoder x = Base64.getEncoder();
String credentials = String.join(":", user, password);
return new String(x.encode(credentials.getBytes()));
}
public String getRandomHostUrl() {
currentHostId = ThreadLocalRandom.current().nextInt(hostsWithPorts.size());
return hostsWithPorts.get(currentHostId);
}
public String getNextHost() {
if (currentHostId >= hostsWithPorts.size() - 1) {
currentHostId = 0;
} else {
currentHostId += 1;
}
return hostsWithPorts.get(currentHostId);
}
public List<String> getHostsWithPorts() {
return hostsWithPorts;
}
public String getUser() {
return user;
}
public String getPassword() {
return password;
}
public String getCredentials() {
return credentials;
}
public boolean isAuthorizationRequired() {
return authorizationRequired;
}
@Override
public String toString() {
return "ClickhouseClusterSettings{" +
"hostsWithPorts=" + hostsWithPorts +
", credentials='" + credentials + '\'' +
", authorizationRequired=" + authorizationRequired +
", currentHostId=" + currentHostId +
'}';
}
}
ClickhouseRequestBlank.java
package com.zhisheng.connectors.clickhouse.model;
import java.util.List;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午10:14
*/
public class ClickhouseRequestBlank {
private final List<String> values;
private final String targetTable;
private int attemptCounter;
public ClickhouseRequestBlank(List<String> values, String targetTable) {
this.values = values;
this.targetTable = targetTable;
this.attemptCounter = 0;
}
public List<String> getValues() {
return values;
}
public void incrementCounter() {
this.attemptCounter++;
}
public int getAttemptCounter() {
return attemptCounter;
}
public String getTargetTable() {
return targetTable;
}
public static final class Builder {
private List<String> values;
private String targetTable;
private Builder() {
}
public static Builder aBuilder() {
return new Builder();
}
public Builder withValues(List<String> values) {
this.values = values;
return this;
}
public Builder withTargetTable(String targetTable) {
this.targetTable = targetTable;
return this;
}
public ClickhouseRequestBlank build() {
return new ClickhouseRequestBlank(values, targetTable);
}
}
@Override
public String toString() {
return "ClickhouseRequestBlank{" +
"values=" + values +
", targetTable='" + targetTable + '\'' +
", attemptCounter=" + attemptCounter +
'}';
}
}
ClickhouseSinkCommonParams
package com.zhisheng.connectors.clickhouse.model;
import com.google.common.base.Preconditions;
import java.util.Map;
import static com.zhisheng.connectors.clickhouse.model.ClickhouseSinkConsts.*;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午10:05
*/
public class ClickhouseSinkCommonParams {
private final ClickhouseClusterSettings clickhouseClusterSettings;
private final String failedRecordsPath;
private final int numWriters;
private final int queueMaxCapacity;
private final int timeout;
private final int maxRetries;
public ClickhouseSinkCommonParams(Map<String, String> params) {
this.clickhouseClusterSettings = new ClickhouseClusterSettings(params);
this.numWriters = Integer.valueOf(params.get(NUM_WRITERS));
this.queueMaxCapacity = Integer.valueOf(params.get(QUEUE_MAX_CAPACITY));
this.maxRetries = Integer.valueOf(params.get(NUM_RETRIES));
this.timeout = Integer.valueOf(params.get(TIMEOUT_SEC));
this.failedRecordsPath = params.get(FAILED_RECORDS_PATH);
Preconditions.checkNotNull(failedRecordsPath);
Preconditions.checkArgument(queueMaxCapacity > 0);
Preconditions.checkArgument(numWriters > 0);
Preconditions.checkArgument(timeout > 0);
Preconditions.checkArgument(maxRetries > 0);
}
public int getNumWriters() {
return numWriters;
}
public int getQueueMaxCapacity() {
return queueMaxCapacity;
}
public ClickhouseClusterSettings getClickhouseClusterSettings() {
return clickhouseClusterSettings;
}
public int getTimeout() {
return timeout;
}
public int getMaxRetries() {
return maxRetries;
}
public String getFailedRecordsPath() {
return failedRecordsPath;
}
@Override
public String toString() {
return "ClickhouseSinkCommonParams{" +
"clickhouseClusterSettings=" + clickhouseClusterSettings +
", failedRecordsPath='" + failedRecordsPath + '\'' +
", numWriters=" + numWriters +
", queueMaxCapacity=" + queueMaxCapacity +
", timeout=" + timeout +
", maxRetries=" + maxRetries +
'}';
}
}
ClickhouseSinkConsts.java
package com.zhisheng.connectors.clickhouse.model;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午10:03
*/
public final class ClickhouseSinkConsts {
private ClickhouseSinkConsts() {
}
public static final String TARGET_TABLE_NAME = "clickhouse.sink.target-table";
public static final String MAX_BUFFER_SIZE = "clickhouse.sink.max-buffer-size";
public static final String NUM_WRITERS = "clickhouse.sink.num-writers";
public static final String QUEUE_MAX_CAPACITY = "clickhouse.sink.queue-max-capacity";
public static final String TIMEOUT_SEC = "clickhouse.sink.timeout-sec";
public static final String NUM_RETRIES = "clickhouse.sink.retries";
public static final String FAILED_RECORDS_PATH = "clickhouse.sink.failed-records-path";
}
ConfigUtil.java
package com.zhisheng.connectors.clickhouse.util;
import java.util.*;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigValue;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午12:38
*/
public class ConfigUtil {
public static final String HOST_DELIMITER = ", ";
private ConfigUtil(){
}
public static Properties toProperties(Config config) {
Properties properties = new Properties();
config.entrySet().forEach(e -> properties.put(e.getKey(), unwrapped(config.getValue(e.getKey()))));
return properties;
}
public static Map<String, String> toMap(Config config) {
Map<String, String> map = new HashMap<>();
config.entrySet().forEach(e -> map.put(e.getKey(), unwrapped(e.getValue())));
return map;
}
private static String unwrapped(ConfigValue configValue) {
Object object = configValue.unwrapped();
return object.toString();
}
static public String buildStringFromList(List<String> list) {
return String.join(HOST_DELIMITER, list);
}
static public List<String> buildListFromString(String string) {
return Arrays.asList(string.split(" "));
}
}
ThreadUtil.java
package com.zhisheng.connectors.clickhouse.util;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午12:39
*/
public class ThreadUtil {
private ThreadUtil() {
}
public static ThreadFactory threadFactory(String threadName, boolean isDaemon) {
return new ThreadFactoryBuilder()
.setNameFormat(threadName + "-%d")
.setDaemon(isDaemon)
.build();
}
public static ThreadFactory threadFactory(String threadName) {
return threadFactory(threadName, true);
}
public static void shutdownExecutorService(ExecutorService executorService) throws InterruptedException {
shutdownExecutorService(executorService, 5);
}
public static void shutdownExecutorService(ExecutorService executorService, int timeoutS) throws InterruptedException {
if (executorService != null && !executorService.isShutdown()) {
executorService.shutdown();
if (!executorService.awaitTermination(timeoutS, TimeUnit.SECONDS)) {
executorService.shutdownNow();
executorService.awaitTermination(timeoutS, TimeUnit.SECONDS);
}
}
}
}
ClickhouseSink.java
package com.zhisheng.connectors.clickhouse;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午12:38
*/
import com.zhisheng.connectors.clickhouse.applied.ClickhouseSinkBuffer;
import com.zhisheng.connectors.clickhouse.applied.ClickhouseSinkManager;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
import java.util.Properties;
/**
* Desc:
* Created by zhisheng on 2019/9/28 上午12:38
*/
public class ClickhouseSink extends RichSinkFunction<String> {
private static final Logger logger = LoggerFactory.getLogger(ClickhouseSink.class);
private static final Object DUMMY_LOCK = new Object();
private final Properties localProperties;
private volatile static transient ClickhouseSinkManager sinkManager;
private transient ClickhouseSinkBuffer clickhouseSinkBuffer;
public ClickhouseSink(Properties properties) {
this.localProperties = properties;
}
@Override
public void open(Configuration config) {
if (sinkManager == null) {
synchronized (DUMMY_LOCK) {
if (sinkManager == null) {
Map<String, String> params = getRuntimeContext()
.getExecutionConfig()
.getGlobalJobParameters()
.toMap();
sinkManager = new ClickhouseSinkManager(params);
}
}
}
clickhouseSinkBuffer = sinkManager.buildBuffer(localProperties);
}
/**
* Add csv to buffer
*
* @param recordAsCSV csv-event
*/
@Override
public void invoke(String recordAsCSV, Context context) {
try {
clickhouseSinkBuffer.put(recordAsCSV);
} catch (Exception e) {
logger.error("Error while sending data to Clickhouse, record = {}", recordAsCSV, e);
throw new RuntimeException(e);
}
}
@Override
public void close() throws Exception {
if (clickhouseSinkBuffer != null) {
clickhouseSinkBuffer.close();
}
if (sinkManager != null) {
if (!sinkManager.isClosed()) {
synchronized (DUMMY_LOCK) {
if (!sinkManager.isClosed()) {
sinkManager.close();
}
}
}
}
super.close();
}
}
