技术背景
我们在对接Windows平台RTSP|RTMP直播播放模块的时候，有开发者提出来这样的技术需求，他们做驾考、全景摄像头、多路会议录制等场景的时候，希望把多路视频流数据，合并到一路保存或者对外推送到RTMP服务。
技术实现
多路RTSP|RTMP流合流，实际上我们2016年就有这块demo，当时合流的数据是本地采集的摄像头或屏幕数据，和外部RTSP、RTMP流，合成后输出（类似于传统意义的连麦操作）。这里大概说下思路，外部的RTSP|RTMP流数据，解码后，把YUV或RGB数据回调上来，然后，按照图层的形式，分别贴摄像头、屏幕数据或解码后的流数据。
本次以四路RTSP摄像头数据合流为例：
开始播放：
/* * SmartPlayerDemo.cs * Author: daniusdk.com * QQ: 89030985 */ private void btn_play1_Click(object sender, EventArgs e) { if (btn_play1.Text == "播放") { String url = textBox_url1.Text; if (!InitCommonSDKParam(player1_handle_, url)) { MessageBox.Show("设置参数错误!"); return; } bool is_support_d3d_render = false; Int32 in_support_d3d_render = 0; if (NT.NTBaseCodeDefine.NT_ERC_OK == NTSmartPlayerSDK.NT_SP_IsSupportD3DRender(player1_handle_, playWnd1.Handle, ref in_support_d3d_render)) { if (1 == in_support_d3d_render) { is_support_d3d_render = true; } } if (is_support_d3d_render) { // 支持d3d绘制的话，就用D3D绘制 NTSmartPlayerSDK.NT_SP_SetRenderWindow(player1_handle_, playWnd1.Handle); if (btn_check_render_scale_mode.Checked) NTSmartPlayerSDK.NT_SP_SetRenderScaleMode(player1_handle_, 1); else NTSmartPlayerSDK.NT_SP_SetRenderScaleMode(player1_handle_, 0); } video_frame_call_back1_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack); NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player1_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back1_); UInt32 ret_start = NTSmartPlayerSDK.NT_SP_StartPlay(player1_handle_); if (ret_start != 0) { MessageBox.Show("播放失败.."); return; } btn_play1.Text = "停止"; } else { StopPlayback1(); } }
其中InitCommonSDKParam()主要完成一些初始化参数设置：
private bool InitCommonSDKParam(IntPtr handle, String url) { if (IntPtr.Zero == handle) return false; if (String.IsNullOrEmpty(url)) return false; Int32 buffer_time = int.Parse(textBox_buffer_time.Text); NTSmartPlayerSDK.NT_SP_SetBuffer(handle, buffer_time); // 设置rtsp tcp模式，rtmp不使用, 可以不设置 if (checkBox_rtsp_tcp.Checked) NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(handle, 1); else NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(handle, 0); //RTSP timeout设置 Int32 rtsp_timeout = 10; NTSmartPlayerSDK.NT_SP_SetRtspTimeout(handle, rtsp_timeout); //RTSP TCP/UDP自动切换设置 Int32 is_auto_switch_tcp_udp = 1; NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(handle, is_auto_switch_tcp_udp); if (checkBox_mute.Checked) NTSmartPlayerSDK.NT_SP_SetMute(handle, 1); else NTSmartPlayerSDK.NT_SP_SetMute(handle, 0); if (checkBox_fast_startup.Checked) NTSmartPlayerSDK.NT_SP_SetFastStartup(handle, 1); else NTSmartPlayerSDK.NT_SP_SetFastStartup(handle, 0); if (checkBox_hardware_decoder.Checked) { NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(handle, is_support_h264_hardware_decoder_ ? 1 : 0, 0); NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(handle, is_support_h265_hardware_decoder_ ? 1 : 0, 0); } else { NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(handle, 0, 0); NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(handle, 0, 0); } // 设置是否只解码关键帧 if (btn_check_only_decode_video_key_frame.Checked) NTSmartPlayerSDK.NT_SP_SetOnlyDecodeVideoKeyFrame(handle, 1); else NTSmartPlayerSDK.NT_SP_SetOnlyDecodeVideoKeyFrame(handle, 0); // 设置低延迟模式 if (checkBox_low_latency.Checked) NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(handle, 1); else NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(handle, 0); NTSmartPlayerSDK.NT_SP_SetRotation(handle, rotate_degrees_); NTSmartPlayerSDK.NT_SP_SetAudioVolume(handle, slider_audio_volume.Value); NTSmartPlayerSDK.NT_SP_SetReportDownloadSpeed(handle, 1, 1); NTSmartPlayerSDK.NT_SP_SetURL(handle, url); return true; }
开始播放之前，我们设置YUV数据回调：
video_frame_call_back1_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack); NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player1_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back1_);
回调处理如下，如果是多个图层，通过推送端，把yuv或rgb数据，投递给推送端，video frame数据回调，可以根据handle区分不同的图层或实例：
public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame) { if (frame == IntPtr.Zero) return; NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame)); if (publisher_wrapper_ != null) { int video_layer_index; if (handle == player_handle_) video_layer_index = publisher_wrapper_.get_external_video_layer0_index(); else if (handle == player1_handle_) video_layer_index = publisher_wrapper_.get_external_video_layer1_index(); else if (handle == player2_handle_) video_layer_index = publisher_wrapper_.get_external_video_layer2_index(); else if (handle == player3_handle_) video_layer_index = publisher_wrapper_.get_external_video_layer3_index(); else video_layer_index = -1; if (video_layer_index > -1) { publisher_wrapper_.post_i420_layer_image(video_layer_index, video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_, video_frame.plane2_, video_frame.stride2_, video_frame.width_, video_frame.height_); } } }
推送端，目前以四路合成为例，另外加个实时文字水印，图层设计如下：
public bool config_layers(bool is_add_rgbx_zero_layer) { if (video_option_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER) return false; if (is_empty_handle()) return false; int w = video_width_; int h = video_height_; if ((w & 0x1) != 0) --w; if ((h & 0x1) != 0) --h; if (w < 2 || h < 2) return false; NTSmartPublisherSDK.NT_PB_ClearLayersConfig(handle_, 0, 0, IntPtr.Zero); int type, index = 0; if (is_add_rgbx_zero_layer) { NT_PB_RGBARectangleLayerConfig rgba_layer = new NT_PB_RGBARectangleLayerConfig(); type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE; fill_layer_base(rgba_layer, out rgba_layer.base_, type, index, true, 0, 0, w, h); rgba_layer.red_ = 0; rgba_layer.green_ = 0; rgba_layer.blue_ = 0; rgba_layer.alpha_ = 255; if (add_layer_config(rgba_layer, type)) index++; } NT_PB_ExternalVideoFrameLayerConfig external_video_layer0 = new NT_PB_ExternalVideoFrameLayerConfig(); type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME; fill_layer_base(external_video_layer0, out external_video_layer0.base_, type, index, true, 0, 0, w/2, h/2); if (add_layer_config(external_video_layer0, type)) external_video_layer0_index_ = index++; NT_PB_ExternalVideoFrameLayerConfig external_video_layer1 = new NT_PB_ExternalVideoFrameLayerConfig(); type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME; fill_layer_base(external_video_layer1, out external_video_layer1.base_, type, index, true, w / 2, 0, w / 2, h / 2); if (add_layer_config(external_video_layer1, type)) external_video_layer1_index_ = index++; NT_PB_ExternalVideoFrameLayerConfig external_video_layer2 = new NT_PB_ExternalVideoFrameLayerConfig(); type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME; fill_layer_base(external_video_layer2, out external_video_layer2.base_, type, index, true, 0, h / 2, w / 2, h / 2); if (add_layer_config(external_video_layer2, type)) external_video_layer2_index_ = index++; NT_PB_ExternalVideoFrameLayerConfig external_video_layer3 = new NT_PB_ExternalVideoFrameLayerConfig(); type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME; fill_layer_base(external_video_layer3, out external_video_layer3.base_, type, index, true, w / 2, h / 2, w / 2, h / 2); if (add_layer_config(external_video_layer3, type)) external_video_layer3_index_ = index++; //叠加的文本层 NT_PB_ExternalVideoFrameLayerConfig text_layer = new NT_PB_ExternalVideoFrameLayerConfig(); type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME; fill_layer_base(text_layer, out text_layer.base_, type, index, false, w / 2, h / 2, 64, 64); if (add_layer_config(text_layer, type)) text_layer_index_ = index++; return index > 0; }
合成后数据，可以对外推送到RTMP服务，也可以注入到本地RTSP服务，或者本地直接录制MP4文件,录制出来四宫格效果如下：
总结
多路RTSP|RTMP数据合流，在多媒体处理、实时监控、驾考、教育等各个行业，应用非常广泛，除了视频外，音频如果需要合成，可以以采集系统扬声器的形式合流出来。多路合流，可以事先做好排版编辑，如果期间不希望显示某一路数据，可以点隐藏图层，实时对图层进行隐藏。感兴趣的开发者，可以单独跟我沟通交流。
