本人在跟着导师做大数据分析,需要用到kafka,在Producer端,用Python(用的是kafka-python)将读取的数据经过训练好的机器学习模型计算之后,发送到指定的服务器(IP)的指定topic,因为需要将数据在web端做展示,所以Consumer端用Java语言来写。这几天在网上参考了很多帖子,现在经过实际运行无误之后,决定将代码分享给大家,下面是Websocket结合kafka的Java API来实时更新数据的代码。
package servlet;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;import javax.websocket.EncodeException;
import javax.websocket.OnClose;
import javax.websocket.OnError;
import javax.websocket.OnMessage;
import javax.websocket.OnOpen;
import javax.websocket.RemoteEndpoint;
import javax.websocket.Session;
import javax.websocket.server.ServerEndpoint;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;import net.sf.json.JSONArray;
@ServerEndpoint("/getServer2")
public class websocket {
//静态变量,用来记录当前在线连接数。应该把它设计成线程安全的。
private static int onlineCount = 0; //concurrent包的线程安全Set,用来存放每个客户端对应的MyWebSocket对象。若要实现服务端与单一客户端通信的话,可以使用Map来存放,其中Key可以为用户标识
private static CopyOnWriteArraySet<websocket> webSocketSet = new CopyOnWriteArraySet<websocket>(); //与某个客户端的连接会话,需要通过它来给客户端发送数据
private Session session; /**
* 连接建立成功调用的方法
* @param session 可选的参数。session为与某个客户端的连接会话,需要通过它来给客户端发送数据
*/
@OnOpen
public void onOpen(Session session) throws EncodeException {
//1.配置属性
Properties properties = new Properties();
//配置kafka计算节点的地址
properties.put("bootstrap.servers", "172.17.0.4:9092");//172.17.0.4是Producer发送给服务器的IP地址,根据自己的实际情况修改
//设置消费组
properties.put("group.id", "g1");
//是否自动确认offset
properties.put("enable.auto.commit", "true");
//接收消息的反序列化
//key
properties.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
//value
properties.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
//2.创建消费则实例
final KafkaConsumer< String,String> consumer = new KafkaConsumer<String, String>(properties);
//订阅消息主题
consumer.subscribe(Arrays.asList("test"));//test是topic的名字
//4.释放资源
//启动一个守护线程,在虚拟机要挂的时候或者执行不了了
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
public void run() {
if(consumer != null){
consumer.close();
}
}
}));
System.out.println("web");
this.session = session;
webSocketSet.add(this); //加入set中
addOnlineCount(); //在线数加1
System.out.println("有新连接加入!当前在线人数为" + getOnlineCount());
final RemoteEndpoint.Basic basic = session.getBasicRemote();
Runnable runnable = new Runnable() {
@Override
public void run() {
try {
ConsumerRecords<String, String> records = consumer.poll(10);
//拿到的消息很多,要遍历
for(ConsumerRecord<String, String> record : records){
basic.sendText(record.value());
System.out.println(record.offset()+"******"+record.key()+"******"+record.value());
}
} catch (IOException e) {
e.printStackTrace();
}
}
};
ScheduledExecutorService sc = Executors.newSingleThreadScheduledExecutor();
// 第二个参数为首次执行的延时时间,第三个参数为定时执行的间隔时间
sc.scheduleAtFixedRate(runnable, 1, 10, TimeUnit.SECONDS);
}
@OnError
public void onError(Throwable throwable, Session session) {
System.out.println("pathParams:" + session.getPathParameters());
System.out.println("requestParams" + session.getRequestParameterMap());
System.out.print("onError" + throwable.toString());
} /**
* 关闭连接时触发
*
* @param relationId
* @param userCode
* @param session
*/
@OnClose
public void onClose(Session session) {
webSocketSet.remove(this); //从set中删除
subOnlineCount(); //在线数减1
System.out.println("pathParams:" + session.getPathParameters());
System.out.println("requestParams" + session.getRequestParameterMap());
System.out.print("onClose ");
}
/**
* 接收前端传过来的数据。
* 虽然在实现推送逻辑中并不需要接收前端数据,但是作为一个webSocket的教程或叫备忘,还是将接收数据的逻辑加上了。
*/
@OnMessage
public void onMessage(String message ,Session session){
System.out.println(message + "from " + session.getId());
}
public static synchronized int getOnlineCount() {
return onlineCount;
} public static synchronized void addOnlineCount() {
websocket.onlineCount++;
} public static synchronized void subOnlineCount() {
websocket.onlineCount--;
}
}
下面是JSP页面展示的代码:
<%@ page language="java" contentType="text/html; charset=UTF-8"
pageEncoding="UTF-8"%>
<%
String path = request.getContextPath();
String socPath="ws://"+request.getServerName()+":"+request.getServerPort()+path+"/";
%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>Insert title here</title>
<script type="text/javascript" src="<%=path %>/js/jquery-1.12.3.js"></script>
<script type="text/javascript" src="<%=path %>/js/echarts.min.js"></script>
<script type="text/javascript">
$(function(){
var wsuri = "<%=socPath%>getServer2";
if ('WebSocket' in window)
websocket = new WebSocket(wsuri);
else if ('MozWebSocket' in window)
websocket = new MozWebSocket(wsuri);
else {
alert('当前浏览器 Not support websocket')
}
//连接发生错误的回调方法
websocket.onerror = function () {
setMessageInnerHTML("WebSocket连接发生错误");
};
//连接成功建立的回调方法
websocket.onopen = function (event) {
setMessageInnerHTML("WebSocket连接成功");
}
//接收到消息的回调方法
websocket.onmessage = function (event) {
setMessageInnerHTML(event.data);
} //连接关闭的回调方法
websocket.onclose = function () {
setMessageInnerHTML("WebSocket连接关闭");
} //监听窗口关闭事件,当窗口关闭时,主动去关闭websocket连接,防止连接还没断开就关闭窗口,server端会抛异常。
window.onbeforeunload = function () {
closeWebSocket();
} //将消息显示在网页上
function setMessageInnerHTML(innerHTML) {
document.getElementById('sp').innerHTML += innerHTML + '<br/>';
} //关闭WebSocket连接
function closeWebSocket() {
websocket.close();
}
function sendMsg() {
socket.send("This is a client message ");
}
//发送消息
/* function send() {
var message = document.getElementById('text').value;
websocket.send(message);
} */
<%-- $.ajax({
url: '<%=path %>/cum',
type: 'post',
success:function(res){
document.getElementById('sp').innerHTML = 'hahahahah';
alert("hahahah")
alert(res)
}
}) --%>
})
</script>
<body>
<span id="sp"></span>
</body>
</html>