之前有重点介绍过，我们的智能外呼+智能ivr+本次的智能视频ivr，均采用了freeswitch 二次开发自定义application完成的。
在我们继续如何实现智能视频IVR之前，我们来详细解读一下media bug。
freeswitch使用media bug来实现旁路输出媒体，在录音、监听、ASR、信号检测等等场景都使用到它。
我们看一下switch_core_media_bug_add 用来添加一个bug， freeswitch eavesdrop 源码中是如何使用的呢？
if (switch_core_media_bug_add(tsession, "eavesdrop", uuid,
eavesdrop_callback, ep, 0,
read_flags | write_flags | SMBF_READ_PING | SMBF_THREAD_LOCK | SMBF_NO_PAUSE,
&bug) != SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Cannot attach bug\n");
goto end;
}这里的 callback函数
static switch_bool_t eavesdrop_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type)
{
struct eavesdrop_pvt *ep = (struct eavesdrop_pvt *) user_data;
uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
switch_frame_t frame = { 0 };
switch_core_session_t *session = switch_core_media_bug_get_session(bug);
switch_channel_t *e_channel = switch_core_session_get_channel(ep->eavesdropper);
int show_spy = 0;
switch_frame_t *nframe = NULL;
frame.data = data;
frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
show_spy = switch_core_media_bug_test_flag(bug, SMBF_SPY_VIDEO_STREAM) || switch_core_media_bug_test_flag(bug, SMBF_SPY_VIDEO_STREAM_BLEG);
if (show_spy) {
if (!ep->set_decoded_read) {
ep->set_decoded_read = 1;
switch_channel_set_flag_recursive(e_channel, CF_VIDEO_DECODED_READ);
switch_core_session_request_video_refresh(ep->eavesdropper);
}
} else {
if (ep->set_decoded_read) {
ep->set_decoded_read = 0;
switch_channel_clear_flag_recursive(e_channel, CF_VIDEO_DECODED_READ);
switch_core_session_request_video_refresh(ep->eavesdropper);
}
}
switch (type) {
case SWITCH_ABC_TYPE_INIT:
if (switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_STREAM) ||
switch_core_media_bug_test_flag(bug, SMBF_WRITE_VIDEO_STREAM) ||
switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_PING)) {
switch_core_session_set_video_read_callback(ep->eavesdropper, video_eavesdrop_callback, (void *)bug);
switch_channel_set_flag_recursive(switch_core_session_get_channel(session), CF_VIDEO_DECODED_READ);
}
break;
case SWITCH_ABC_TYPE_CLOSE:
if (ep->set_decoded_read) {
switch_channel_clear_flag_recursive(e_channel, CF_VIDEO_DECODED_READ);
}
if (switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_STREAM) ||
switch_core_media_bug_test_flag(bug, SMBF_WRITE_VIDEO_STREAM) ||
switch_core_media_bug_test_flag(bug, SMBF_READ_VIDEO_PING)) {
switch_core_session_set_video_read_callback(ep->eavesdropper, NULL, NULL);
}
switch_channel_clear_flag_recursive(switch_core_session_get_channel(session), CF_VIDEO_DECODED_READ);
break;
case SWITCH_ABC_TYPE_TAP_NATIVE_WRITE:
nframe = switch_core_media_bug_get_native_write_frame(bug);
break;
case SWITCH_ABC_TYPE_TAP_NATIVE_READ:
nframe = switch_core_media_bug_get_native_read_frame(bug);
break;
case SWITCH_ABC_TYPE_WRITE:
break;
case SWITCH_ABC_TYPE_READ_PING:
if (ep->buffer) {
if (switch_core_media_bug_read(bug, &frame, SWITCH_FALSE) != SWITCH_STATUS_FALSE) {
switch_buffer_lock(ep->buffer);
switch_buffer_zwrite(ep->buffer, frame.data, frame.datalen);
switch_buffer_unlock(ep->buffer);
}
}
break;
case SWITCH_ABC_TYPE_READ:
break;
case SWITCH_ABC_TYPE_READ_REPLACE:
{
if (switch_test_flag(ep, ED_MUX_READ)) {
switch_frame_t *rframe = switch_core_media_bug_get_read_replace_frame(bug);
if (switch_buffer_inuse(ep->r_buffer) >= rframe->datalen) {
uint32_t bytes;
int channels = rframe->channels ? rframe->channels : 1;
switch_buffer_lock(ep->r_buffer);
bytes = (uint32_t) switch_buffer_read(ep->r_buffer, ep->data, rframe->datalen);
rframe->datalen = switch_merge_sln(rframe->data, rframe->samples, (int16_t *) ep->data, bytes / 2, channels) * 2 * channels;
rframe->samples = rframe->datalen / 2;
ep->demux_frame.data = ep->data;
ep->demux_frame.datalen = bytes;
ep->demux_frame.samples = bytes / 2;
ep->demux_frame.channels = rframe->channels;
switch_buffer_unlock(ep->r_buffer);
switch_core_media_bug_set_read_replace_frame(bug, rframe);
switch_core_media_bug_set_read_demux_frame(bug, &ep->demux_frame);
}
}
}
break;
case SWITCH_ABC_TYPE_WRITE_REPLACE:
{
if (switch_test_flag(ep, ED_MUX_WRITE)) {
switch_frame_t *rframe = switch_core_media_bug_get_write_replace_frame(bug);
if (switch_buffer_inuse(ep->w_buffer) >= rframe->datalen) {
uint32_t bytes;
int channels = rframe->channels ? rframe->channels : 1;
switch_buffer_lock(ep->w_buffer);
bytes = (uint32_t) switch_buffer_read(ep->w_buffer, data, rframe->datalen);
rframe->datalen = switch_merge_sln(rframe->data, rframe->samples, (int16_t *) data, bytes / 2, channels) * 2 * channels;
rframe->samples = rframe->datalen / 2;
switch_buffer_unlock(ep->w_buffer);
switch_core_media_bug_set_write_replace_frame(bug, rframe);
}
}
}
break;
case SWITCH_ABC_TYPE_READ_VIDEO_PING:
case SWITCH_ABC_TYPE_STREAM_VIDEO_PING:
{
if (!bug->video_ping_frame || !bug->video_ping_frame->img) {
break;
}
if (ep->eavesdropper && switch_core_session_read_lock(ep->eavesdropper) == SWITCH_STATUS_SUCCESS) {
if (switch_core_session_write_video_frame(ep->eavesdropper, bug->video_ping_frame, SWITCH_IO_FLAG_NONE, 0) != SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error writing video to %s\n", switch_core_session_get_name(ep->eavesdropper));
ep->errs++;
if (ep->errs > 10) {
switch_channel_hangup(e_channel, SWITCH_CAUSE_DESTINATION_OUT_OF_ORDER);
switch_core_session_reset(ep->eavesdropper, SWITCH_TRUE, SWITCH_TRUE);
switch_core_session_rwunlock(ep->eavesdropper);
return SWITCH_FALSE;
}
} else {
ep->errs = 0;
}
switch_core_session_rwunlock(ep->eavesdropper);
}
}
break;
default:
break;
}
if (nframe) {
switch_frame_t frame = *nframe;
uint8_t buf[SWITCH_RECOMMENDED_BUFFER_SIZE] = "";
frame.data = buf;
frame.codec = nframe->codec;
memcpy(frame.data, nframe->data, nframe->datalen);
if (switch_core_session_write_frame(ep->eavesdropper, nframe, SWITCH_IO_FLAG_NONE, 0) != SWITCH_STATUS_SUCCESS) {
return SWITCH_FALSE;
}
}
return SWITCH_TRUE;
}源码比较长，我们从eavesdrop 的监听就知道了，每当freeswitch读到一个frame时候，都会触发调用callback回调。
我们来浓缩一下callback函数，假设我们自定义一个mediabug，
static void start_media_bug(switch_core_session_t *session) {
switch_media_bug_t *bug = NULL;
if (switch_core_media_bug_add(session, "my_bug", NULL, bug_callback, handler, 0, SMBF_READ_REPLACE, → &bug) != SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Failed to create media → bug\n");
}
/* save bug to channel for later removal */
switch_channel_set_private(switch_core_session_get_channel(session), "__my_bug__", bug); }
static void stop_media_bug(switch_core_session_t *session) {
/* get saved bug from channel */
switch_media_bug_t *bug = switch_channel_get_private(switch_core_session_get_channel(session), → "__my_bug__", bug);
if (bug) {
switch_core_media_bug_remove(session, &bug);
} }即使用switch_core_media_bug_add 方法为session添加media bug
那么我们的callback函数应当如下：
static switch_bool_t robot_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type)
{
// switch_codec_t *read_codec;
switch_frame_t *frame;
switch (type) {
case SWITCH_ABC_TYPE_INIT:
break;
case SWITCH_ABC_TYPE_READ_REPLACE:
frame = switch_core_media_bug_get_read_replace_frame(bug);
process_data(frame,bug);
break;
case SWITCH_ABC_TYPE_CLOSE:
break;
default:
break;
}
return SWITCH_TRUE;
}当有媒体流读入时候，
type SWITCH_ABC_TYPE_READ_REPLACE ，同样的 SWITCH_ABC_TYPE_INIT， 和 SWITCH_ABC_TYPE_CLOSE正如他字面意思描述那样，代表着初始化和关闭。
我们就可以调用 frame =switch_core_media_bug_get_read_replace_frame(bug); 将frame获取到，并进行我们自己的处理。
因此实现媒体流的旁路输出，是一个非常简单的事情，当然这全都仰仗freeswitch对media bug的封装。
我们直接定位源码，文件 switch_core_media_bug.c 来仔细看看freeswitch都为我们带来了哪些方法和实现。
switch_core_media_bug_get_session 非常重要
switch_core_media_bug_get_text
switch_core_media_bug_get_video_ping_frame
switch_core_media_bug_get_write_replace_frame 获取媒体frame
switch_core_media_bug_set_write_replace_frame 写入
switch_core_media_bug_add 添加media bug
switch_core_media_bug_close
switch_core_media_bug_remove这些方法都非常重要， 我们可以根据以上方法在freeswitch源码中查到对应的调用，那么自己写起来就可以模仿（和我一样不熟悉c语音的同学），那么我们一样能写出可以实现自身意图的功能。
我们再一次总结一下，freeswitch的media bug，首先我们今天先介绍了什么是freeswitch的media bug
1：什么是media bug
2：media bug 在freeswitch中有哪些应该，我们在哪些场景可以采纳
3：media bug在源码中的使用，包括eavesdrop 中的使用
4：浓缩丸media bug的callback代码，我们在实践中需要如何处理
5：查询switch_core_media_bug.c 中定义了哪些非常重要的方法
今天的回顾结束，下一节，我们将结束如何将media bug中的媒体进行转发处理，对接ASR。
在这之前我们选择使用UDP服务将media实时传到我们熟悉的应用开发语言上。
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
