之前做了linux x86_64上的摄像头采集、屏幕采集和麦克风等采集，并把采集到的音视频数据实时编码输出为RTMP/RTSP流, 现在国产arm64位设备越来越多，最近对linux arm64也做了相应的支持.
Linux上摄像头采集使用V4L2相关接口，查看摄像头设备文件可以使用(ls -l /dev/|grep video). 打开设备使用open接口就行, 例如:open("/dev/videoxx", flag).
屏幕采集用X相关接口实现，如果是Wayland协议, 用PipeWire相关接口实现采集就好.
麦克风采集使用ALSA或者PulseAudio.
采集播放音频用PulseAudio.
采集实现代码很多, 已封装好接口,下面是调用demo:
/*
* 问题沟通微信:ldxevt
*/
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <signal.h>
#include <assert.h>
#include <string.h>
#include <poll.h>
#include <errno.h>
#include <string>
#include <vector>
#include <memory>
#include <sstream>
#include <X11/Xlib.h>
#include <X11/keysym.h>
#include "nt_sdk_linux_smart_log.h"
#include "nt_common_media_define.h"
#include "nt_linux_smart_publisher_sdk.h"
#include "nt_layer_conf_wrapper.h"
namespace
{
int EventPoll(int fd, bool is_write, int timeout_ms)
{
int result;
do
{
struct pollfd info;
info.fd = fd;
if (is_write)
{
info.events = POLLOUT;
}
else
{
info.events = POLLIN | POLLPRI;
}
result = poll(&info, 1, timeout_ms);
} while (result < 0 && errno == EINTR);
return result;
}
bool MY_X11_Pending(Display* display, int timeout_ms)
{
XFlush(display);
if (XEventsQueued(display, QueuedAlready) > 0)
{
return true;
}
if (EventPoll(ConnectionNumber(display), false, timeout_ms))
{
if (XPending(display) > 0)
{
return true;
}
}
return false;
}
Window CreateSubWindow(Display* display, int screen, Window parent)
{
XWindowAttributes parent_win_att;
XGetWindowAttributes(display, parent, &parent_win_att);
fprintf(stdout, "parent w:%d, h:%d\n", parent_win_att.width, parent_win_att.height);
XSetWindowAttributes swa;
swa.border_pixel = WhitePixel(display, screen);
swa.event_mask = KeyPressMask | StructureNotifyMask;
return XCreateWindow(display, parent, 0, 0, parent_win_att.width - 4, parent_win_att.height - 4,
2, parent_win_att.depth, InputOutput, parent_win_att.visual, CWEventMask | CWBorderPixel, &swa);
}
class CameraInfo
{
public:
std::string name_;
std::string id_;
std::vector<NT_PB_VideoCaptureCapability> capabilities_;
};
volatile bool g_is_exit = false;
void OnSaSigaction(int signo, siginfo_t* s_info, void*)
{
if (SIGINT == signo)
{
g_is_exit = true;
fprintf(stdout, "OnSaSigaction SIGINT si_pid=%d, pid=%d\n", (int)s_info->si_pid, (int)getpid());
}
else if (SIGFPE == signo)
{
fprintf(stderr, "OnSaSigaction SIGFPE si_pid=%d, pid=%d, addr=%p\n", (int)s_info->si_pid, (int)getpid(), s_info->si_addr);
switch (s_info->si_code)
{
case FPE_INTDIV:
fprintf(stderr, "OnSaSigaction SIGFPE FPE_INTDIV\n");
break;
case FPE_INTOVF:
fprintf(stderr, "OnSaSigaction SIGFPE FPE_INTOVF\n");
break;
case FPE_FLTDIV:
fprintf(stderr, "OnSaSigaction SIGFPE FPE_FLTDIV\n");
break;
case FPE_FLTOVF:
fprintf(stderr, "OnSaSigaction SIGFPE FPE_FLTOVF\n");
break;
case FPE_FLTUND:
fprintf(stderr, "OnSaSigaction SIGFPE FPE_FLTUND\n");
break;
case FPE_FLTRES:
fprintf(stderr, "OnSaSigaction SIGFPE FPE_FLTRES\n");
break;
case FPE_FLTINV:
fprintf(stderr, "OnSaSigaction SIGFPE FPE_FLTINV\n");
break;
case FPE_FLTSUB:
fprintf(stderr, "OnSaSigaction SIGFPE FPE_FLTSUB\n");
break;
default:
break;
}
}
}
void OnSDKEventHandle(NT_HANDLE handle, NT_PVOID user_data,
NT_UINT32 event_id,
NT_INT64 param1,
NT_INT64 param2,
NT_UINT64 param3,
NT_UINT64 param4,
NT_PCSTR param5,
NT_PCSTR param6,
NT_PVOID param7
)
{
if (NT_PB_E_EVENT_ID_CAPTURE_WINDOW_INVALID == event_id)
{
fprintf(stdout, "X Window is invalid, wid=%lld, handle=%p\n", param1, handle);
}
}
void LogInit()
{
SmartLogAPI log_api;
memset(&log_api, 0, sizeof(log_api));
GetSmartLogAPI(&log_api);
log_api.SetLevel(SL_INFO_LEVEL);
log_api.SetPath((NT_PVOID)"./");
}
bool PushSDKInit(NT_SmartPublisherSDKAPI& push_api)
{
memset(&push_api, 0, sizeof(push_api));
NT_GetSmartPublisherSDKAPI(&push_api);
auto ret = push_api.Init(0, nullptr);
if (NT_ERC_OK != ret)
{
fprintf(stderr, "push_api.Init failed!\n");
return false;
}
else
{
fprintf(stdout, "push_api.Init ok!\n");
}
return true;
}
std::vector<NT_UINT64> x_win_list;
void GetCameraInfo(NT_SmartPublisherSDKAPI* push_api, std::vector<CameraInfo>& cameras)
{
NT_INT32 device_number = 0;
if (NT_ERC_OK != push_api->GetVideoCaptureDeviceNumber(&device_number))
{
return;
}
if (device_number < 1)
{
return;
}
for (auto i = 0; i < device_number; ++i)
{
CameraInfo info;
char name[256] = { 0 };
char id[1024] = { 0 };
if (NT_ERC_OK != push_api->GetVideoCaptureDeviceInfo(i,
name, sizeof(name) / sizeof(char),
id, sizeof(id) / sizeof(char)))
{
continue;
}
info.name_ = name;
info.id_ = id;
NT_INT32 capability_number = 0;
if (NT_ERC_OK != push_api->GetVideoCaptureDeviceCapabilityNumber(
id, &capability_number))
{
continue;
}
auto is_failed = false;
for (auto i = 0; i < capability_number; ++i)
{
NT_PB_VideoCaptureCapability capability = { 0, 0, 0 };
if (NT_ERC_OK != push_api->GetVideoCaptureDeviceCapability(
id, i, &capability))
{
is_failed = true;
break;
}
info.capabilities_.push_back(capability);
}
if (!is_failed)
{
cameras.push_back(info);
}
}
}
NT_HANDLE StartPush(NT_SmartPublisherSDKAPI* push_api, const std::string& rtmp_url, int dst_fps)
{
NT_INT32 pulse_device_number = 0;
if (NT_ERC_OK == push_api->GetAuidoInputDeviceNumber(2, &pulse_device_number))
{
fprintf(stdout, "Pulse device num:%d\n", pulse_device_number);
char device_name[512];
for (auto i = 0; i < pulse_device_number; ++i)
{
if (NT_ERC_OK == push_api->GetAuidoInputDeviceName(2, i, device_name, 512))
{
fprintf(stdout, "index:%d name:%s\n", i, device_name);
}
}
}
NT_INT32 alsa_device_number = 0;
if (pulse_device_number < 1)
{
if (NT_ERC_OK == push_api->GetAuidoInputDeviceNumber(1, &alsa_device_number))
{
fprintf(stdout, "Alsa device num:%d\n", alsa_device_number);
char device_name[512];
for (auto i = 0; i < alsa_device_number; ++i)
{
if (NT_ERC_OK == push_api->GetAuidoInputDeviceName(1, i, device_name, 512))
{
fprintf(stdout, "index:%d name:%s\n", i, device_name);
}
}
}
}
NT_INT32 capture_speaker_flag = 0;
if ( NT_ERC_OK == push_api->IsCanCaptureSpeaker(2, &capture_speaker_flag) )
{
if (capture_speaker_flag)
fprintf(stdout, "Support speaker capture\n");
else
fprintf(stdout, "UnSupport speaker capture\n");
}
NT_INT32 is_support_window_capture = 0;
if (NT_ERC_OK == push_api->IsCaptureXWindowSupported(NULL, &is_support_window_capture))
{
if (is_support_window_capture)
fprintf(stdout, "Support window capture\n");
else
fprintf(stdout, "UnSupport window capture\n");
}
if (is_support_window_capture)
{
NT_INT32 win_count = 0;
if (NT_ERC_OK == push_api->UpdateCaptureXWindowList(NULL, &win_count) && win_count > 0 )
{
fprintf(stdout, "X Capture Winows list++\n");
for (auto i = 0; i < win_count; ++i)
{
NT_UINT64 wid;
char title[512];
if (NT_ERC_OK == push_api->GetCaptureXWindowInfo(i, &wid, title, sizeof(title) / sizeof(char)))
{
x_win_list.push_back(wid);
fprintf(stdout, "wid:%llu, title:%s\n", wid, title);
}
}
fprintf(stdout, "X Capture Winows list--\n");
}
}
std::vector<CameraInfo> cameras;
GetCameraInfo(push_api, cameras);
if (!cameras.empty())
{
fprintf(stdout, "cameras count:%d\n", (int)cameras.size());
for (const auto& c : cameras)
{
fprintf(stdout, "camera name:%s, id:%s, cap_num:%d\n", c.name_.c_str(), c.id_.c_str(), (int)c.capabilities_.size());
for (const auto& i : c.capabilities_)
{
fprintf(stdout, "cap w:%d, h:%d, fps:%d\n", i.width_, i.height_, i.max_frame_rate_);
}
}
}
NT_UINT32 auido_option = NT_PB_E_AUDIO_OPTION_NO_AUDIO;
if (pulse_device_number > 0 || alsa_device_number > 0)
{
auido_option = NT_PB_E_AUDIO_OPTION_CAPTURE_MIC;
}
else if (capture_speaker_flag)
{
auido_option = NT_PB_E_AUDIO_OPTION_CAPTURE_SPEAKER;
}
//auido_option = NT_PB_E_AUDIO_OPTION_CAPTURE_MIC_SPEAKER_MIXER;
NT_UINT32 video_option = NT_PB_E_VIDEO_OPTION_SCREEN;
if (!cameras.empty())
{
video_option = NT_PB_E_VIDEO_OPTION_CAMERA;
}
else if (is_support_window_capture)
{
video_option = NT_PB_E_VIDEO_OPTION_WINDOW;
}
NT_HANDLE push_handle = nullptr;
if (NT_ERC_OK != push_api->Open(&push_handle, video_option, auido_option, 0, NULL))
{
return nullptr;
}
push_api->SetEventCallBack(push_handle, nullptr, OnSDKEventHandle);
if (video_option == NT_PB_E_VIDEO_OPTION_CAMERA)
{
if (!cameras.empty())
{
push_api->SetVideoCaptureDeviceBaseParameter(push_handle, cameras.front().id_.c_str(),
640, 480);
//push_api->FlipVerticalCamera(push_handle, 1);
//push_api->FlipHorizontalCamera(push_handle, 1);
//push_api->RotateCamera(push_handle, 0);
}
}
if (video_option == NT_PB_E_VIDEO_OPTION_WINDOW)
{
if (!x_win_list.empty())
{
//push_api->SetCaptureXWindow(push_handle, x_win_list[0]);
push_api->SetCaptureXWindow(push_handle, x_win_list.back());
}
}
push_api->SetFrameRate(push_handle, dst_fps);
push_api->SetVideoEncoder(push_handle, 0, 1, NT_MEDIA_CODEC_ID_H264, 0);
push_api->SetVideoBitRate(push_handle, 2000);
push_api->SetVideoQuality(push_handle, 26);
push_api->SetVideoMaxBitRate(push_handle, 4000);
push_api->SetVideoKeyFrameInterval(push_handle, dst_fps*2);
push_api->SetVideoEncoderProfile(push_handle, 3); // H264 high
push_api->SetVideoEncoderSpeed(push_handle, 3);
if (pulse_device_number > 0)
{
push_api->SetAudioInputLayer(push_handle, 2);
push_api->SetAuidoInputDeviceId(push_handle, 0);
}
else if (alsa_device_number > 0)
{
push_api->SetAudioInputLayer(push_handle, 1);
push_api->SetAuidoInputDeviceId(push_handle, 0);
}
push_api->SetPublisherAudioCodecType(push_handle, 1);
//push_api->SetMute(push_handle, 1);
if ( NT_ERC_OK != push_api->SetURL(push_handle, rtmp_url.c_str(), NULL) )
{
push_api->Close(push_handle);
push_handle = nullptr;
return nullptr;
}
if ( NT_ERC_OK != push_api->StartPublisher(push_handle, NULL) )
{
push_api->Close(push_handle);
push_handle = nullptr;
return nullptr;
}
return push_handle;
}
}
int main(int argc, char *argv[])
{
struct sigaction act;
sigemptyset(&act.sa_mask);
act.sa_sigaction = OnSaSigaction;
act.sa_flags = SA_SIGINFO;
sigaction(SIGINT, &act, NULL);
sigaction(SIGFPE, &act, NULL);
XInitThreads(); // X支持多线程, 必须调用
auto display = XOpenDisplay(nullptr);
if (!display)
{
fprintf(stderr, "Cannot connect to X server\n");
return 0;
}
auto screen = DefaultScreen(display);
auto root = XRootWindow(display, screen);
XWindowAttributes root_win_att;
if (!XGetWindowAttributes(display, root, &root_win_att))
{
fprintf(stderr, "Get Root window attri failed\n");
XCloseDisplay(display);
return 0;
}
int main_w = root_win_att.width / 2, main_h = root_win_att.height / 2;
auto black_pixel = BlackPixel(display, screen);
auto white_pixel = WhitePixel(display, screen);
auto main_wid = XCreateSimpleWindow(display, root, 0, 0, main_w, main_h, 0, white_pixel, black_pixel);
if (!main_wid)
{
fprintf(stderr, "Cannot Create Main Window\n");
XCloseDisplay(display);
return 0;
}
XSelectInput(display, main_wid, StructureNotifyMask | KeyPressMask);
auto sub_wid = CreateSubWindow(display, screen, main_wid);
if (!sub_wid)
{
fprintf(stderr, "Cannot Create Render Window\n");
XDestroyWindow(display, main_wid);
XCloseDisplay(display);
return 0;
}
XMapWindow(display, main_wid);
XStoreName(display, main_wid, "Video Preview");
XMapWindow(display, sub_wid);
LogInit();
NT_SmartPublisherSDKAPI push_api;
if (!PushSDKInit(push_api))
{
XDestroyWindow(display, sub_wid);
XDestroyWindow(display, main_wid);
XCloseDisplay(display);
return 0;
}
auto push_handle = StartPush(&push_api, "rtmp://192.168.0.131:1935/live/test", 25);
if (!push_handle)
{
fprintf(stderr, "start push failed.\n");
XDestroyWindow(display, sub_wid);
XDestroyWindow(display, main_wid);
XCloseDisplay(display);
push_api.UnInit();
return 0;
}
// 开启预览，也可以不开启, 根据需求来
push_api.SetPreviewXWindow(push_handle, "", sub_wid);
push_api.StartPreview(push_handle, 0, nullptr);
while (!g_is_exit)
{
while (MY_X11_Pending(display, 10))
{
XEvent xev;
memset(&xev, 0, sizeof(xev));
XNextEvent(display, &xev);
if (xev.type == ConfigureNotify)
{
if (xev.xconfigure.window == main_wid)
{
if (xev.xconfigure.width != main_w || xev.xconfigure.height != main_h)
{
main_w = xev.xconfigure.width;
main_h = xev.xconfigure.height;
XMoveResizeWindow(display, sub_wid, 0, 0, main_w - 4, main_h - 4);
}
}
}
else if (xev.type == KeyPress)
{
if (xev.xkey.keycode == XKeysymToKeycode(display, XK_Escape))
{
fprintf(stdout, "ESC Key Press\n");
g_is_exit = true;
}
}
if (g_is_exit)
break;
}
}
fprintf(stdout, "exit run loop, is_exit:%d\n", g_is_exit);
push_api.StopPreview(push_handle);
push_api.StopPublisher(push_handle);
push_api.Close(push_handle);
push_handle = nullptr;
XDestroyWindow(display, sub_wid);
XDestroyWindow(display, main_wid);
XCloseDisplay(display);
push_api.UnInit();
fprintf(stdout, "SDK UnInit..\n");
return 0;
}上面的代码音视频采集都有，视频采集有屏幕、窗口和摄像头, 音频采集麦克风、扬声器(两者混音也支持). 然后使用rtmp协议推送出去, 这里没输出rtsp流的代码，需要的话沟通就好, 不依赖QT, 但可以集成到QT项目将中, Linux arm64和x86_64都支持。
