业务场景

最近项目中场景需要get一批key的value，因为redis的get操作(不单单是get命令)是阻塞的，如果循环取值的话，就算是内网，耗时也是巨大的。所以想到了redis的pipeline命令。

pipeline简介

非pipeline：client一个请求，redis server一个响应，期间client阻塞

Pipeline：redis的管道命令，允许client将多个请求依次发给服务器(redis的客户端，如jedisCluster，lettuce等都实现了对pipeline的封装)，过程中而不需要等待请求的回复，在最后再一并读取结果即可。

单机版

单机版比较简单，直接上代码

//换成真实的redis实例
Jedis jedis = new Jedis();
//获取管道
Pipeline p = jedis.pipelined();
for (int i = 0; i < 10000; i++) {
p.get(i + "");
}
//获取结果
List results = p.syncAndReturnAll();

集群版

因为 JedisCluster 本身不支持 pipeline ，所以我们需要对 JedisCluster 进行一些封装。

还是一样，直接上代码

import lombok.extern.slf4j.Slf4j;
import redis.clients.jedis.*;
import redis.clients.jedis.exceptions.JedisMovedDataException;
import redis.clients.jedis.exceptions.JedisRedirectionException;
import redis.clients.util.JedisClusterCRC16;
import redis.clients.util.SafeEncoder;
import java.io.Closeable;
import java.lang.reflect.Field;
import java.util.*;
import java.util.function.BiConsumer;
@Slf4j
public class JedisClusterPipeline extends PipelineBase implements Closeable {
/**
* 用于获取 JedisClusterInfoCache
*/
private JedisSlotBasedConnectionHandler connectionHandler;
/**
* 根据hash值获取连接
*/
private JedisClusterInfoCache clusterInfoCache;
/**
* 也可以去继承JedisCluster和JedisSlotBasedConnectionHandler来提供访问接口
* JedisCluster继承于BinaryJedisCluster
* 在BinaryJedisCluster，connectionHandler属性protected修饰的，所以需要反射
*
*
* 而 JedisClusterInfoCache 属性在JedisClusterConnectionHandler中，但是这个类是抽象类，
* 但它有一个实现类JedisSlotBasedConnectionHandler
*/
private static final Field FIELD_CONNECTION_HANDLER;
private static final Field FIELD_CACHE;
static {
FIELD_CONNECTION_HANDLER = getField(BinaryJedisCluster.class, "connectionHandler");
FIELD_CACHE = getField(JedisClusterConnectionHandler.class, "cache");
}
/**
* 根据顺序存储每个命令对应的Client
*/
private Queue clients = new LinkedList<>();
/**
* 用于缓存连接
* 一次pipeline过程中使用到的jedis缓存
*/
private Map jedisMap = new HashMap<>();
/**
* 是否有数据在缓存区
*/
private boolean hasDataInBuf = false;
/**
* 根据jedisCluster实例生成对应的JedisClusterPipeline
* 通过此方式获取pipeline进行操作的话必须调用close()关闭管道
* 调用本类里pipelineXX方法则不用close(),但建议最好还是在finally里调用一下close()
* @param
* @return
*/
public static JedisClusterPipeline pipelined(JedisCluster jedisCluster) {
JedisClusterPipeline pipeline = new JedisClusterPipeline();
pipeline.setJedisCluster(jedisCluster);
return pipeline;
}
public JedisClusterPipeline() {
}
public void setJedisCluster(JedisCluster jedis) {
connectionHandler = getValue(jedis, FIELD_CONNECTION_HANDLER);
clusterInfoCache = getValue(connectionHandler, FIELD_CACHE);
}
/**
* 刷新集群信息，当集群信息发生变更时调用
* @param
* @return
*/
public void refreshCluster() {
connectionHandler.renewSlotCache();
}
/**
* 同步读取所有数据. 与syncAndReturnAll()相比，sync()只是没有对数据做反序列化
*/
public void sync() {
innerSync(null);
}
/**
* 同步读取所有数据 并按命令顺序返回一个列表
*
* @return 按照命令的顺序返回所有的数据
*/
public List syncAndReturnAll() {
List responseList = new ArrayList<>();
innerSync(responseList);
return responseList;
}
@Override
public void close() {
clean();
clients.clear();
for (Jedis jedis : jedisMap.values()) {
if (hasDataInBuf) {
flushCachedData(jedis);
}
jedis.close();
}
jedisMap.clear();
hasDataInBuf = false;
}
private void flushCachedData(Jedis jedis) {
try {
jedis.getClient().getAll();
} catch (RuntimeException ex) {
}
}
@Override
protected Client getClient(String key) {
byte[] bKey = SafeEncoder.encode(key);
return getClient(bKey);
}
@Override
protected Client getClient(byte[] key) {
Jedis jedis = getJedis(JedisClusterCRC16.getSlot(key));
Client client = jedis.getClient();
clients.add(client);
return client;
}
private Jedis getJedis(int slot) {
JedisPool pool = clusterInfoCache.getSlotPool(slot);
// 根据pool从缓存中获取Jedis
Jedis jedis = jedisMap.get(pool);
if (null == jedis) {
jedis = pool.getResource();
jedisMap.put(pool, jedis);
}
hasDataInBuf = true;
return jedis;
}
public static void pipelineSetEx(String[] keys, String[] values, int[] exps,JedisCluster jedisCluster) {
operate(new Command() {
@Override
public List execute() {
JedisClusterPipeline p = pipelined(jedisCluster);
for (int i = 0, len = keys.length; i < len; i++) {
p.setex(keys[i], exps[i], values[i]);
}
return p.syncAndReturnAll();
}
});
}
public static List> pipelineHgetAll(String[] keys,JedisCluster jedisCluster) {
return operate(new Command() {
@Override
public List execute() {
JedisClusterPipeline p = pipelined(jedisCluster);
for (int i = 0, len = keys.length; i < len; i++) {
p.hgetAll(keys[i]);
}
return p.syncAndReturnAll();
}
});
}
public static List pipelineSismember(String[] keys, String members,JedisCluster jedisCluster) {
return operate(new Command() {
@Override
public List execute() {
JedisClusterPipeline p = pipelined(jedisCluster);
for (int i = 0, len = keys.length; i < len; i++) {
p.sismember(keys[i], members);
}
return p.syncAndReturnAll();
}
});
}
public static List pipeline(BiConsumer function, O obj,JedisCluster jedisCluster) {
return operate(new Command() {
@Override
public List execute() {
JedisClusterPipeline jcp = JedisClusterPipeline.pipelined(jedisCluster);
function.accept(obj, jcp);
return jcp.syncAndReturnAll();
}
});
}
private void innerSync(List formatted) {
HashSet clientSet = new HashSet<>();
try {
for (Client client : clients) {
// 在sync()调用时其实是不需要解析结果数据的，但是如果不调用get方法，发生了JedisMovedDataException这样的错误应用是不知道的，因此需要调用get()来触发错误。
// 其实如果Response的data属性可以直接获取，可以省掉解析数据的时间，然而它并没有提供对应方法，要获取data属性就得用反射，不想再反射了，所以就这样了
Object data = generateResponse(client.getOne()).get();
if (null != formatted) {
formatted.add(data);
}
// size相同说明所有的client都已经添加，就不用再调用add方法了
if (clientSet.size() != jedisMap.size()) {
clientSet.add(client);
}
}
} catch (JedisRedirectionException jre) {
if (jre instanceof JedisMovedDataException) {
// if MOVED redirection occurred, rebuilds cluster's slot cache,
// recommended by Redis cluster specification
refreshCluster();
}
throw jre;
} finally {
if (clientSet.size() != jedisMap.size()) {
// 所有还没有执行过的client要保证执行(flush)，防止放回连接池后后面的命令被污染
for (Jedis jedis : jedisMap.values()) {
if (clientSet.contains(jedis.getClient())) {
continue;
}
flushCachedData(jedis);
}
}
hasDataInBuf = false;
close();
}
}
private static Field getField(Class> cls, String fieldName) {
try {
Field field = cls.getDeclaredField(fieldName);
field.setAccessible(true);
return field;
} catch (NoSuchFieldException | SecurityException e) {
throw new RuntimeException("cannot find or access field '" + fieldName + "' from " + cls.getName(), e);
}
}
@SuppressWarnings({"unchecked" })
private static T getValue(Object obj, Field field) {
try {
return (T)field.get(obj);
} catch (IllegalArgumentException | IllegalAccessException e) {
log.error("get value fail", e);
throw new RuntimeException(e);
}
}
private static T operate(Command command) {
try {
return command.execute();
} catch (Exception e) {
log.error("redis operate error");
throw new RuntimeException(e);
}
}
interface Command {
/**
* 具体执行命令
*
* @param 
* @return
*/
 T execute();
}
}
使用demo
public Object testPipelineOperate() {
// String[] keys = {"dylan1","dylan2"};
// String[] values = {"dylan1-v1","dylan2-v2"};
// int[] exps = {100,200};
// JedisClusterPipeline.pipelineSetEx(keys, values, exps, jedisCluster);
long start = System.currentTimeMillis();
List keyList = new ArrayList<>();
for (int i = 0; i < 1000; i++) {
keyList.add(i + "");
}
// List pipeline = JedisClusterPipeline.pipeline(this::getValue, keyList, jedisCluster);
// List pipeline = JedisClusterPipeline.pipeline(this::getHashValue, keyList, jedisCluster);
String[] keys = {"dylan-test1", "dylan-test2"};
List> all = JedisClusterPipeline.pipelineHgetAll(keys, jedisCluster);
long end = System.currentTimeMillis();
System.out.println("testPipelineOperate cost:" + (end-start));
return Response.success(all);
}