采用Storm统计网站的PV，需要从两个方面考虑（1） 性能问题 （2） 线程安全考虑
一、需求分析
（1）网站最常用访问量指标
PV(page views)： count (session_id)
（2）多线程下，注意线程安全问题
PV统计
方案分析
如下是否可行？
1、定义static long pv， Synchronized 控制累计操作
Synchronized 和 Lock在单JVM下有效，但在多JVM下无效
常用的代码一般为：设置
static变量和通过
synchronized关键字处理，代码为：
static private long pv = 0; //多线程情况下共享变量，单JVM没有问题。分布式系统中该变量统计结果存在问题
/**
* Process a single tuple of input.
* @param input The input tuple to be processed.
*/
@Override
public void execute(Tuple input ) {
String line = input .getStringByField("line" );
synchronized (this ) {//多线程情况下代码块异步处理，单JVM没有问题
if (StringUtils.isNotBlank( line)){
pv ++;
}
}
Thread currentThread = Thread.currentThread();
System. out .println(currentThread .getName() + "[" +currentThread .getId()+ "]" + "->" + pv );
}
可行的两个方案：
1、shuffleGrouping下，pv * Executer并发数
2、bolt1进行多并发局部汇总，bolt2单线程进行全局汇总
线程安全：多线程处理的结果和单线程一致
二、统计PV的流程图以及Storm代码
采用Storm进行数据汇总的大致步骤： 通过以及bolt高并发多线程情况下统计出来部分的数据，然后通过单线程二级bolt进行整体汇总，请求结果
PVTopology主程序
package com.yun.storm.pv;
import java.util.HashMap;
import java.util.Map;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
/**
* 实时统计PV拓扑
* @author shenfl
* @version V1.0
*/
public class PVTopology {
public final static String SPOUT_ID = PVSpout.class.getSimpleName();
public final static String PVBOLT_ID = PVBolt.class.getSimpleName();
public final static String PVTOPOLOGY_ID = PVTopology.class.getSimpleName();
public final static String PVSUMBOLT_ID = PVSumBolt.class.getSimpleName();
public static void main(String[] args) {
TopologyBuilder builder = new TopologyBuilder();
/*//表示kafka使用的zookeeper的地址
String brokerZkStr = "192.168.35:2181,192.168.36:2181,192.168.37:2181";
ZkHosts zkHosts = new ZkHosts(brokerZkStr);
//表示的是kafak中存储数据的主题名称
String topic = "pvtopic";
//指定zookeeper中的一个根目录，里面存储kafkaspout读取数据的位置等信息
String zkRoot = "/kafkaspout";
String id = UUID.randomUUID().toString();
SpoutConfig spoutconf = new SpoutConfig(zkHosts, topic, zkRoot, id);
builder.setSpout(SPOUT_ID , new KafkaSpout(spoutconf),1);//单线程*/
builder.setSpout( SPOUT_ID, new PVSpout(),1);
builder.setBolt( PVBOLT_ID, new PVBolt(), 4).setNumTasks(8).shuffleGrouping(SPOUT_ID );//2个线程，4个task实例
builder.setBolt( PVSUMBOLT_ID, new PVSumBolt(), 1).shuffleGrouping(PVBOLT_ID );//单线程汇总
Map<String,Object> conf = new HashMap<String,Object>();
conf.put(Config. TOPOLOGY_RECEIVER_BUFFER_SIZE , 8);
conf.put(Config. TOPOLOGY_TRANSFER_BUFFER_SIZE , 32);
conf.put(Config. TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE , 16384);
conf.put(Config. TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE , 16384);
/* try {
StormSubmitter.submitTopology(PVTOPOLOGY_ID, conf, builder.createTopology());
} catch (Exception e) {
e.printStackTrace();
} */
LocalCluster cluster = new LocalCluster();
cluster.submitTopology( PVTOPOLOGY_ID, conf ,builder.createTopology());
}
}
模拟数据源：
package com.yun.storm.pv;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import com.yun.redis.PropertyReader;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
/**
* 实时读取PV用户行为日志数据 可以从数据库读取数据，可以从kafka读取数据，可以从文件系统读取数据
*
* @author shenfl
* @version V1.0
*/
public class PVSpout extends BaseRichSpout {
/**
*
*/
private static final long serialVersionUID = 1L;
private SpoutOutputCollector collector;
private Map stormConf;
/**
* 当PVSpout初始化时候调用一次
*
* @param conf
* The Storm configuration for this spout.
* @param context
* 可以获取每个任务的TaskID
* @param collector
* The collector is used to emit tuples from this spout.
*/
@Override
public void open(Map stormConf, TopologyContext context, SpoutOutputCollector collector) {
this.collector = collector;
this.stormConf = stormConf;
}
/**
* 死循环，一直会调用
*/
@Override
public void nextTuple() {
// 获取数据源
try {
String dataDir = PropertyReader.getProperty("parameter.properties", "data.dir");
File file = new File(dataDir);
//获取文件列表
Collection<File> listFiles = FileUtils.listFiles(file, new String[]{"log"},true);
for (File f : listFiles) {
//处理文件
List<String> readLines = FileUtils.readLines(f);
for (String line : readLines) {
this.collector.emit(new Values(line));
}
// 文件已经处理完成
try {
File srcFile = f.getAbsoluteFile();
File destFile = new File(srcFile + ".done." + System.currentTimeMillis());
FileUtils.moveFile(srcFile, destFile);
} catch (IOException e) {
e.printStackTrace();
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Declare the output schema for all the streams of this topology.
*
* @param declarer
* this is used to declare output stream ids, output fields, and
* whether or not each output stream is a direct stream
*/
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("line"));
}
}
(2) 一级bolt，高并发情况局部汇总
package com.yun.storm.pv;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
/**
* 获取PVSpout发送的数据，PVTopology开启多线程。 给出每个线程处理的PV数
*
* 在多线程情况下，对PV数据只能局部汇总，不能整体汇总，可以把局部汇总的结果给一个单线程的BOLT进行整体汇总（PVSumBolt）
*
* @author shenfl
* @version V1.0
*/
public class PVBolt extends BaseRichBolt {
/**
*
*/
private static final long serialVersionUID = 1L;
private OutputCollector collector;
private TopologyContext context;
/**
* 实例初始化的时候调用一次
*
* @param stormConf
* The Storm configuration for this bolt.
* @param context
* This object can be used to get information about this task's
* place within the topology, including the task id and component
* id of this task, input and output information, etc.
* @param collector
* The collector is used to emit tuples from this bolt
*/
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector ) {
this.context = context ;
this.collector = collector ;
}
private long pv = 0;
/**
* Process a single tuple of input.
*
* @param input
* The input tuple to be processed.
*/
@Override
public void execute(Tuple input ) {
try {
String line = input.getStringByField("line" );
if (StringUtils.isNotBlank( line)) {
pv++;
}
//System.out.println(Thread.currentThread().getName() + "[" + Thread.currentThread().getId() + "]" +context.getThisTaskId()+ "->" + pv);
//this.collector.emit(new Values(Thread.currentThread().getId(),pv));//仅适合一个线程和一个task情况
this.collector .emit(new Values(context.getThisTaskId(),pv ));// 一个线程和1或多个task的情况,TaskId唯一
this.collector .ack(input );
} catch(Exception e ){
e.printStackTrace();
this.collector .fail(input );
}
}
/**
* Declare the output schema for all the streams of this topology.
*
* @param declarer
* this is used to declare output stream ids, output fields, and
* whether or not each output stream is a direct stream
*/
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer ) {
declarer.declare( new Fields("taskId","pv" ));
}
}
(3)二 级bolt，单线程汇总
package com.yun.storm.pv;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.collections.MapUtils;
import org.apache.hadoop.hbase.client.Result;
import com.yun.hbase.HBaseUtils;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;
/**
* 汇总PVBolt多个线程的结果
* @author shenfl
*
*/
public class PVSumBolt extends BaseRichBolt{
/**
*
*/
private static final long serialVersionUID = 1L;
private OutputCollector collector;
private Map<Integer,Long> map = new HashMap<Integer,Long>();//<日期,PV数>
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
}
@Override
public void execute(Tuple input) {
try {
Integer taskId = input.getIntegerByField("taskId");
Long pv = input.getLongByField("pv");
map.put(taskId, pv);//map个数为task实例数
long sum = 0;//获取总数，遍历map 的values，进行sum
for (Entry<Integer, Long> e : map.entrySet()) {
sum += e.getValue();
}
System.out.println("当前时间："+System.currentTimeMillis()+"pv汇总结果：" + "->" + sum);
this.collector.ack(input);
}catch(Exception e){
e.printStackTrace();
this.collector.fail(input);
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
}
