分布式RPC(Distributed RPC,DRPC)
Storm 里面引入 DRPC 主要是利用 storm 的实时计算能力来并行优化 cpu 密集型的计算任务。DRPC 的 storm Topology 以函数的参数流作为输入,而把这些函数调用的返回值作为 Topology 的输出流。
DRPC 其实不能算是 storm 本身的一个特性,他是通过组合 storm 的原语 storm,spout,bolt,topology 而成的一种模式。本应该把 DRPC 单独打包,但是 DRPC 真的太有用了,所以把他和 storm 捆绑在一起。
DRPC 是通过一个 DRPC Server 来实现的。
DRPC Server 整体工作过程如下:
- 就收一个 rpc 请求
- 发送请求到 storm topology
- 从 storm topology 接受结果
- 把结果发回给等待的客户端
6.2 DRPC 配置
Storm 提供了一个称作 LinearDRPCTopologyBuilder 的 topology builder,它把实现 DRPC 的几乎所有步骤都自简化了。
6.2.1 实现步骤
1. 需要修改 storm.yaml 文件内容为(分别修改每台机器配置):
drpc.server:
- "xxx.xxx.xxx.xxx"
2. 需要启动 storm 的 drpc 服务,命令:storm drpc &
3. 把相应的 topology 代码上传到 storm 服务器上
storm jar xxx.jar 包名.类名 参数
storm jar storm-drpc-1.0.0-SNAPSHOT.jar com.ws.drpc.topology.BasicDRPCTopology storm-drpc-demo
4. 在本地调用远程的 topology 即可
6.3 代码
BasicDRPCTopology
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.ws.drpc.topology;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.LocalDRPC;
import org.apache.storm.StormSubmitter;
import org.apache.storm.drpc.LinearDRPCTopologyBuilder;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
/**
* This topology is a basic example of doing distributed RPC on top of Storm. It implements a function that appends a
* "!" to any string you send the DRPC function.
*
* @see <a href="http://storm.apache.org/documentation/Distributed-RPC.html">Distributed RPC</a>
*/
public class BasicDRPCTopology {
public static class ExclaimBolt extends BaseBasicBolt {
private static final long serialVersionUID = 3330175993027037816L;
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
String input = tuple.getString(1);
collector.emit(new Values(tuple.getValue(0), input + "!"));
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("id", "result"));
}
}
public static void main(String[] args) throws Exception {
LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
builder.addBolt(new ExclaimBolt(), 3);
Config conf = new Config();
if (args == null || args.length == 0) {
LocalDRPC drpc = new LocalDRPC();
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));
for (String word : new String[] { "hello", "goodbye" }) {
System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
}
Thread.sleep(100000000);
drpc.shutdown();
cluster.shutdown();
} else {
conf.setNumWorkers(3);
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
}
}
}
DRPCExclam
package com.ws.drpc.topology;
import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.utils.DRPCClient;
import org.apache.storm.utils.Utils;
public class DRPCExclam {
public static void main(String[] args) throws Exception {
Config conf = new Config();
conf.setDebug(false);
Map config = Utils.readDefaultConfig();
DRPCClient client = new DRPCClient(config, "192.168.136.175", 3772);
for (String word : new String[] { "hello", "goodbye" }) {
System.out.println(client.execute("exclamation", word));
}
}
}