接收远端消息处理主函数
/* Handle remote connections */
void HandleRemote(int uid)
{
const int position = logr.position;
int recv_timeout; //timeout in seconds waiting for a client reply
int send_timeout;
char * str_protocol = NULL;
recv_timeout = getDefine_Int("remoted", "recv_timeout", 1, 60);
send_timeout = getDefine_Int("remoted", "send_timeout", 1, 60);
tcp_keepidle = getDefine_Int("remoted", "tcp_keepidle", 1, 7200);
tcp_keepintvl = getDefine_Int("remoted", "tcp_keepintvl", 1, 100);
tcp_keepcnt = getDefine_Int("remoted", "tcp_keepcnt", 1, 50);
/* If syslog connection and allowips is not defined, exit */
if (logr.conn[position] == SYSLOG_CONN) {
if (logr.allowips == NULL) {
minfo(NO_SYSLOG);
exit(0);
} else {
os_ip **tmp_ips;
tmp_ips = logr.allowips;
while (*tmp_ips) {
minfo("Remote syslog allowed from: '%s'", (*tmp_ips)->ip);
tmp_ips++;
}
}
}
// Set resource limit for file descriptors
{
nofile = getDefine_Int("remoted", "rlimit_nofile", 1024, 1048576);
struct rlimit rlimit = { nofile, nofile };
if (setrlimit(RLIMIT_NOFILE, &rlimit) < 0) {
merror("Could not set resource limit for file descriptors to %d: %s (%d)", (int)nofile, strerror(errno), errno);
}
}
/* If TCP is enabled then bind the TCP socket */
if (logr.proto[position] & REMOTED_NET_PROTOCOL_TCP) {
logr.tcp_sock = OS_Bindporttcp(logr.port[position], logr.lip[position], logr.ipv6[position]);
if (logr.tcp_sock < 0) {
merror_exit(BIND_ERROR, logr.port[position], errno, strerror(errno));
}
else if (logr.conn[position] == SECURE_CONN) {
if (OS_SetKeepalive(logr.tcp_sock) < 0) {
merror("OS_SetKeepalive failed with error '%s'", strerror(errno));
}
#ifndef CLIENT
else {
OS_SetKeepalive_Options(logr.tcp_sock, tcp_keepidle, tcp_keepintvl, tcp_keepcnt);
}
#endif
if (OS_SetRecvTimeout(logr.tcp_sock, recv_timeout, 0) < 0) {
merror("OS_SetRecvTimeout failed with error '%s'", strerror(errno));
}
if (OS_SetSendTimeout(logr.tcp_sock, send_timeout) < 0) {
merror("OS_SetSendTimeout failed with error '%s'", strerror(errno));
}
}
}
/* If UDP is enabled then bind the UDP socket */
if (logr.proto[position] & REMOTED_NET_PROTOCOL_UDP) {
/* Using UDP. Fast, unreliable... perfect */
logr.udp_sock = OS_Bindportudp(logr.port[position], logr.lip[position], logr.ipv6[position]);
if (logr.udp_sock < 0) {
merror_exit(BIND_ERROR, logr.port[position], errno, strerror(errno));
}
}
/* Revoke privileges */
if (Privsep_SetUser(uid) < 0) {
merror_exit(SETUID_ERROR, USER, errno, strerror(errno));
}
/* Create PID */
if (CreatePID(ARGV0, getpid()) < 0) {
merror_exit(PID_ERROR);
}
/* Start up message */
// If TCP is enabled
if (logr.proto[position] & REMOTED_NET_PROTOCOL_TCP) {
wm_strcat(&str_protocol, REMOTED_NET_PROTOCOL_TCP_STR, WM_STRCAT_NO_SEPARATOR);
}
// If UDP is enabled
if (logr.proto[position] & REMOTED_NET_PROTOCOL_UDP) {
wm_strcat(&str_protocol, REMOTED_NET_PROTOCOL_UDP_STR, (str_protocol == NULL) ? WM_STRCAT_NO_SEPARATOR : ',');
}
/* This should never happen */
if (str_protocol == NULL) {
merror_exit(REMOTED_NET_PROTOCOL_NOT_SET);
}
minfo(STARTUP_MSG " Listening on port %d/%s (%s).",
(int)getpid(),
logr.port[position],
str_protocol,
logr.conn[position] == SECURE_CONN ? "secure" : "syslog");
os_free(str_protocol);
/* If secure connection, deal with it */
if (logr.conn[position] == SECURE_CONN) {
HandleSecure();
}
else if (logr.proto[position] == REMOTED_NET_PROTOCOL_TCP) {
HandleSyslogTCP();
}
else { /* If not, deal with syslog */
HandleSyslog();
}
}
配置获取，是通过ossec.conf配置文件来的，默认情况下是tcp、连接方式是secure
<remote>
<connection>secure</connection>
<port>1514</port>
<protocol>tcp</protocol>
<queue_size>131072</queue_size>
</remote>
/* Return 0 if not configured */
if (RemotedConfig(cfg, &logr) < 0) {
merror_exit(CONFIG_ERROR, cfg);
}
由于配置中是secure 所有我们看HandleSecure()，这里我们可以看到函数前半部分创建很多线程，接收消息的是从while (1) {开始一直到该函数结束，通过epoll来监听网络事件、然后通过nb_recv接收tcp数据。
/* Handle secure connections */
void HandleSecure()
{
const int protocol = logr.proto[logr.position];
int sock_client;
int n_events = 0;
char buffer[OS_MAXSTR + 1];
int recv_b;
struct sockaddr_in peer_info;
memset(&peer_info, 0, sizeof(struct sockaddr_in));
wnotify_t * notify = NULL;
/* Initialize manager */
manager_init();
// Initialize messag equeue
rem_msginit(logr.queue_size);
/* Initialize the agent key table mutex */
key_lock_init();
/* Create shared file updating thread */
w_create_thread(update_shared_files, NULL);
/* Create Active Response forwarder thread */
w_create_thread(AR_Forward, NULL);
/* Create Security configuration assessment forwarder thread */
w_create_thread(SCFGA_Forward, NULL);
// Create Request listener thread
w_create_thread(req_main, NULL);
// Create State writer thread
w_create_thread(rem_state_main, NULL);
key_request_queue = queue_init(1024);
// Create key request thread
w_create_thread(w_key_request_thread, NULL);
/* Create wait_for_msgs threads */
{
int i;
sender_pool = getDefine_Int("remoted", "sender_pool", 1, 64);
mdebug2("Creating %d sender threads.", sender_pool);
for (i = 0; i < sender_pool; i++) {
w_create_thread(wait_for_msgs, NULL);
}
}
// Reset all the agents' connection status in Wazuh DB
// The master will disconnect and alert the agents on its own DB. Thus, synchronization is not required.
if (OS_SUCCESS != wdb_reset_agents_connection("synced", NULL))
mwarn("Unable to reset the agents' connection status. Possible incorrect statuses until the agents get connected to the manager.");
// Create message handler thread pool
{
int worker_pool = getDefine_Int("remoted", "worker_pool", 1, 16);
// Initialize FD list and counter.
global_counter = 0;
rem_initList(FD_LIST_INIT_VALUE);
while (worker_pool > 0) {
w_create_thread(rem_handler_main, NULL);
worker_pool--;
}
}
/* Connect to the message queue
* Exit if it fails.
*/
if ((logr.m_queue = StartMQ(DEFAULTQUEUE, WRITE, INFINITE_OPENQ_ATTEMPTS)) < 0) {
merror_exit(QUEUE_FATAL, DEFAULTQUEUE);
}
/* Read authentication keys */
minfo(ENC_READ);
OS_ReadKeys(&keys, W_ENCRYPTION_KEY, 0);
OS_StartCounter(&keys);
// Key reloader thread
w_create_thread(rem_keyupdate_main, NULL);
// fp closer thread
w_create_thread(close_fp_main, &keys);
/* Set up peer size */
logr.peer_size = sizeof(peer_info);
/* Initialize some variables */
memset(buffer, '\0', OS_MAXSTR + 1);
/* Events watcher is started (is used to monitor sockets events) */
if (notify = wnotify_init(MAX_EVENTS), !notify) {
merror_exit("wnotify_init(): %s (%d)", strerror(errno), errno);
}
/* If TCP is set on the config, then the corresponding sockets is added to the watching list */
if (protocol & REMOTED_NET_PROTOCOL_TCP) {
if (wnotify_add(notify, logr.tcp_sock) < 0) {
merror_exit("wnotify_add(%d): %s (%d)", logr.tcp_sock, strerror(errno), errno);
}
}
/* If UDP is set on the config, then the corresponding sockets is added to the watching list */
if (protocol & REMOTED_NET_PROTOCOL_UDP) {
if (wnotify_add(notify, logr.udp_sock) < 0) {
merror_exit("wnotify_add(%d): %s (%d)", logr.udp_sock, strerror(errno), errno);
}
}
while (1) {
/* It waits for a socket event */
if (n_events = wnotify_wait(notify, EPOLL_MILLIS), n_events < 0) {
if (errno != EINTR) {
merror("Waiting for connection: %s (%d)", strerror(errno), errno);
sleep(1);
}
continue;
}
int i;
for (i = 0; i < n_events; i++) {
// Returns the fd of the socket that recived a message
int fd = wnotify_get(notify, i);
// In case of failure or unexpected file descriptor
if (fd <= 0) {
merror("Unexpected file descriptor: %d, %s (%d)", fd, strerror(errno), errno);
continue;
}
// If a new TCP connection was received and TCP is enabled
else if ((fd == logr.tcp_sock) && (protocol & REMOTED_NET_PROTOCOL_TCP)) {
sock_client = accept(logr.tcp_sock, (struct sockaddr *) &peer_info, &logr.peer_size);
if (sock_client < 0) {
switch (errno) {
case ECONNABORTED:
mdebug1(ACCEPT_ERROR, strerror(errno), errno);
break;
default:
merror(ACCEPT_ERROR, strerror(errno), errno);
}
continue;
}
nb_open(&netbuffer, sock_client, &peer_info);
rem_inc_tcp();
mdebug1("New TCP connection at %s [%d]", inet_ntoa(peer_info.sin_addr), sock_client);
if (wnotify_add(notify, sock_client) < 0) {
merror("wnotify_add(%d, %d): %s (%d)", notify->fd, sock_client, strerror(errno), errno);
_close_sock(&keys, sock_client);
}
}
// If a new UDP connection was received and UDP is enabled
else if (fd == logr.udp_sock && protocol & REMOTED_NET_PROTOCOL_UDP) {
recv_b = recvfrom(logr.udp_sock, buffer, OS_MAXSTR, 0, (struct sockaddr *) &peer_info, &logr.peer_size);
/* Nothing received */
if (recv_b <= 0) {
continue;
} else {
rem_msgpush(buffer, recv_b, &peer_info, USING_UDP_NO_CLIENT_SOCKET);
rem_add_recv((unsigned long) recv_b);
}
}
// If a message was received through a TCP client and tcp is enabled
else if (protocol & REMOTED_NET_PROTOCOL_TCP) {
sock_client = fd;
switch (recv_b = nb_recv(&netbuffer, sock_client), recv_b) {
case -2:
mwarn("Too big message size from %s [%d].", inet_ntoa(peer_info.sin_addr), sock_client);
_close_sock(&keys, sock_client);
continue;
case -1:
switch (errno) {
case ECONNRESET:
case ENOTCONN:
case EAGAIN:
#if EAGAIN != EWOULDBLOCK
case EWOULDBLOCK:
#endif
#if ETIMEDOUT
case ETIMEDOUT:
#endif
mdebug2("TCP peer [%d] at %s: %s (%d)", sock_client,
inet_ntoa(peer_info.sin_addr), strerror(errno), errno);
break;
default:
merror("TCP peer [%d] at %s: %s (%d)", sock_client,
inet_ntoa(peer_info.sin_addr), strerror(errno), errno);
}
fallthrough;
case 0:
_close_sock(&keys, sock_client);
continue;
default:
rem_add_recv((unsigned long) recv_b);
}
}
}
}
manager_free();
}
tcp消息接收函数，接收到数据之后通过字节序转换，然后rem_msgpush(sockbuf->data + cur_offset, cur_len, &sockbuf->peer_info, sock);往队列（双向循环链表）中存
/*
* Receive available data from the network and push as many message as possible
* Returns -2 on data corruption at application layer (header).
* Returns -1 on system call error: recv().
* Returns 0 if no data was available in the socket.
* Returns the number of bytes received on success.
*/
int nb_recv(netbuffer_t * buffer, int sock) {
sockbuffer_t * sockbuf = &buffer->buffers[sock];
unsigned long data_ext = sockbuf->data_len + receive_chunk;
long recv_len;
unsigned long i;
unsigned long cur_offset;
uint32_t cur_len;
w_mutex_lock(&mutex);
// Extend data buffer
if (data_ext > sockbuf->data_size) {
os_realloc(sockbuf->data, data_ext, sockbuf->data);
sockbuf->data_size = data_ext;
}
// Receive and append
recv_len = recv(sock, sockbuf->data + sockbuf->data_len, receive_chunk, 0);
if (recv_len <= 0) {
goto end;
}
sockbuf->data_len += recv_len;
// Dispatch as most messages as possible
for (i = 0; i + sizeof(uint32_t) <= sockbuf->data_len; i = cur_offset + cur_len) {
cur_len = wnet_order(*(uint32_t *)(sockbuf->data + i));
if (cur_len > OS_MAXSTR) {
recv_len = -2;
goto end;
}
cur_offset = i + sizeof(uint32_t);
if (cur_offset + cur_len > sockbuf->data_len) {
break;
}
rem_msgpush(sockbuf->data + cur_offset, cur_len, &sockbuf->peer_info, sock);
}
// Move remaining data to data start
if (i > 0) {
if (i < sockbuf->data_len) {
memcpy(sockbuf->data, sockbuf->data + i, sockbuf->data_len - i);
}
sockbuf->data_len -= i;
switch (buffer_relax) {
case 0:
// Do not deallocate memory.
break;
case 1:
// Shrink memory to fit the current buffer or the receive chunk.
sockbuf->data_size = sockbuf->data_len > receive_chunk ? sockbuf->data_len : receive_chunk;
os_realloc(sockbuf->data, sockbuf->data_size, sockbuf->data);
break;
default:
// Full memory deallocation.
sockbuf->data_size = sockbuf->data_len;
if (sockbuf->data_size) {
os_realloc(sockbuf->data, sockbuf->data_size, sockbuf->data);
} else {
os_free(sockbuf->data);
}
}
}
end:
w_mutex_unlock(&mutex);
return recv_len;
}
消息出队列rem_msgpop();然后处理，释放内存
// Message handler thread
void * rem_handler_main(__attribute__((unused)) void * args) {
message_t * message;
char buffer[OS_MAXSTR + 1] = "";
int wdb_sock = -1;
mdebug1("Message handler thread started.");
while (1) {
message = rem_msgpop();
if (message->sock == USING_UDP_NO_CLIENT_SOCKET || message->counter > rem_getCounter(message->sock)) {
memcpy(buffer, message->buffer, message->size);
HandleSecureMessage(buffer, message->size, &message->addr, message->sock, &wdb_sock);
} else {
rem_inc_dequeued();
}
rem_msgfree(message);
}
return NULL;
}
HandleSecureMessage获取有效agentid，keys的管理用读写锁，并且key是被建立成一个红黑树进行管理。对消息进行解码，解压缩，新老格式兼容处理。解析完消息之后通过SendMSG(logr.m_queue, tmp_msg, srcmsg, SECURE_MQ) < 0) ，这里的m_queue是通过unix domain socket实现的，提供给其他进程读，通过SendMSG函数最后一个参数来标记发送的消息类型。
STATIC void HandleSecureMessage(char *buffer, int recv_b, struct sockaddr_in *peer_info, int sock_client, int *wdb_sock) {
int agentid;
const int protocol = (sock_client == USING_UDP_NO_CLIENT_SOCKET) ? REMOTED_NET_PROTOCOL_UDP : REMOTED_NET_PROTOCOL_TCP;
char cleartext_msg[OS_MAXSTR + 1];
char srcmsg[OS_FLSIZE + 1];
char srcip[IPSIZE + 1] = {0};
char agname[KEYSIZE + 1] = {0};
char *tmp_msg;
size_t msg_length;
char ip_found = 0;
int r;
/* Set the source IP */
inet_ntop(peer_info->sin_family, &peer_info->sin_addr, srcip, IPSIZE);
/* Initialize some variables */
memset(cleartext_msg, '\0', OS_MAXSTR + 1);
memset(srcmsg, '\0', OS_FLSIZE + 1);
tmp_msg = NULL;
/* Get a valid agent id */
...
/* Decrypt the message */
...
/* Check if it is a control message */
if (IsValidHeader(tmp_msg)) {
/* We need to save the peerinfo if it is a control msg */
keys.keyentries[agentid]->net_protocol = protocol;
memcpy(&keys.keyentries[agentid]->peer_info, peer_info, logr.peer_size);
keyentry * key = OS_DupKeyEntry(keys.keyentries[agentid]);
r = (protocol == REMOTED_NET_PROTOCOL_TCP) ? OS_AddSocket(&keys, agentid, sock_client) : REMOTED_USING_UDP;
keys.keyentries[agentid]->rcvd = time(0);
switch (r) {
case OS_ADDSOCKET_ERROR:
merror("Couldn't add TCP socket to keystore.");
break;
case OS_ADDSOCKET_KEY_UPDATED:
mdebug2("TCP socket %d already in keystore. Updating...", sock_client);
break;
case OS_ADDSOCKET_KEY_ADDED:
mdebug2("TCP socket %d added to keystore.", sock_client);
break;
case REMOTED_USING_UDP:
keys.keyentries[agentid]->sock = USING_UDP_NO_CLIENT_SOCKET;
break;
default:
;
}
key_unlock();
// The critical section for readers closes within this function
save_controlmsg(key, tmp_msg, msg_length - 3, wdb_sock);
rem_inc_ctrl_msg();
OS_FreeKey(key);
return;
}
/* Generate srcmsg */
snprintf(srcmsg, OS_FLSIZE, "[%s] (%s) %s", keys.keyentries[agentid]->id,
keys.keyentries[agentid]->name, keys.keyentries[agentid]->ip->ip);
key_unlock();
/* If we can't send the message, try to connect to the
* socket again. If it not exit.
*/
if (SendMSG(logr.m_queue, tmp_msg, srcmsg,
SECURE_MQ) < 0) {
merror(QUEUE_ERROR, DEFAULTQUEUE, strerror(errno));
// Try to reconnect infinitely
logr.m_queue = StartMQ(DEFAULTQUEUE, WRITE, INFINITE_OPENQ_ATTEMPTS);
minfo("Successfully reconnected to '%s'", DEFAULTQUEUE);
if (SendMSG(logr.m_queue, tmp_msg, srcmsg, SECURE_MQ) < 0) {
// Something went wrong sending a message after an immediate reconnection...
merror(QUEUE_ERROR, DEFAULTQUEUE, strerror(errno));
}
} else {
rem_inc_evt();
}
}remoted进程开启debug日志
echo "remoted.debug=2" > /var/ossec/etc/local_internal_options.conf
systemctl restart wazuh-manager
在/var/ossec/logs/ossec.log中可以看到remoted相关debug日志
