目前市面上大多一对一互动都是基于WebRTC,缺点如下:
- 服务器部署非常复杂,不利于私有部署,在一些私密性高的场景下,无法使用,如公安、市政等体系;
- 传输基于UDP,很难保证传输质量,由于UDP是不可靠的传输协议,在复杂的公网网络环境下,各种突发流量、偶尔的传输错误、网络抖动、超时等等都会引起丢包异常,都会在一定程度上影响音视频通信的质量;
- 难以应对复杂的互联网环境,如跨区跨运营商、低带宽、高丢包等场景;
- 整个框架体系不够灵活,代码复杂度高,行话说的好:从demo到实用,中间还差1万个WebRTC。
RTMP一对一互动技术特点:
- 基于现有RTMP推拉流体系,产品稳定度高,整体延迟低;
- 加入噪音抑制、回音消除、自动增益控制等特性,确保通话效果;
- 采用通用的RTMP和RTSP服务器,如nginx、SRS,更有利于私有部署;
- 支持H.264的扩展SEI消息发送机制;
- 支持H.265编码和H.264可变码率设定;
- 支持H.265解码,直播播放器支持的功能,一对一互动模块都可以有选择的支持;
- 适用于应急指挥、教育培训等领域。
废话不多说,上封装代码:
基于 https://github.com/daniulive/SmarterStreaming/ 拉流端封装的代码:
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Runtime.InteropServices; using System.Windows.Forms; using System.Drawing; using System.Drawing.Drawing2D; using NT; namespace SmartEchoCancellationDemo { public delegate void DelGetPlayerEventMsg(String msg); public delegate void DelGetVideoSize(String size); class nt_player_wrapper : IDisposable { [DllImport("kernel32", EntryPoint = "CopyMemory")] static extern void CopyMemory(IntPtr Destination, IntPtr Source, uint Length); private bool disposed_ = false; private IntPtr player_handle_ = IntPtr.Zero; private System.Windows.Forms.Control render_wnd_ = null; private System.Windows.Forms.PaintEventHandler render_wnd_paint_event_ = null; private bool is_playing_ = false; private bool is_mute_ = false; private int play_buffer_ = 100; private NT_SP_VideoFrame cur_video_frame_ = new NT_SP_VideoFrame(); private WeakReference sync_invoke_ = null; //分辨率信息回调 delegate void ResolutionNotifyCallback(Int32 width, Int32 height); ResolutionNotifyCallback resolution_notify_callback_; SP_SDKVideoSizeCallBack video_size_call_back_; //视频数据回调 SP_SDKVideoFrameCallBack video_frame_call_back_; delegate void VideoFrameCallBack(UInt32 status, NT_SP_VideoFrame frame); VideoFrameCallBack set_video_frame_call_back_; //event事件回调 //拉流端事件 SP_SDKEventCallBack pull_event_call_back_; delegate void SetPullEventCallBack(UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, [MarshalAs(UnmanagedType.LPStr)] String param4, [MarshalAs(UnmanagedType.LPStr)] String param5, IntPtr param6); SetPullEventCallBack set_pull_event_call_back_; private UInt32 connection_status_ = 0; private UInt32 buffer_status_ = 0; private Int32 buffer_percent_ = 0; private Int32 download_speed_ = -1; public event DelGetPlayerEventMsg EventGetPlayerEventMsg; public event DelGetVideoSize EventGetVideoSize; public nt_player_wrapper(System.Windows.Forms.Control render_wnd, System.ComponentModel.ISynchronizeInvoke sync_invoke) { render_wnd_ = render_wnd; sync_invoke_ = new WeakReference(sync_invoke); set_pull_event_call_back_ = new SetPullEventCallBack(PullEventCallBack); if (render_wnd_ != null) { render_wnd_paint_event_ = new System.Windows.Forms.PaintEventHandler(this.OnRenderWindowPaint); render_wnd_.Paint += render_wnd_paint_event_; } } public void Dispose() { Dispose(true); // This object will be cleaned up by the Dispose method. // Therefore, you should call GC.SupressFinalize to // take this object off the finalization queue // and prevent finalization code for this object // from executing a second time. // GC.SuppressFinalize(this); } protected virtual void Dispose(bool disposing) { // Check to see if Dispose has already been called. if (!this.disposed_) { if (disposing) { } if (IsPlaying()) { StopPlay(false); } if (render_wnd_ != null && render_wnd_paint_event_ != null) { render_wnd_.Paint -= render_wnd_paint_event_; } render_wnd_paint_event_ = null; if (cur_video_frame_.plane0_ != IntPtr.Zero) { Marshal.FreeHGlobal(cur_video_frame_.plane0_); cur_video_frame_.plane0_ = IntPtr.Zero; } // Note disposing has been done. disposed_ = true; } } ~nt_player_wrapper() { Dispose(false); } public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame) { if (frame == IntPtr.Zero) { return; } //如需直接处理RGB数据,请参考以下流程 NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame)); if (video_frame.format_ != (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FORMAT_RGB32) return; NT_SP_VideoFrame pVideoFrame = new NT_SP_VideoFrame(); pVideoFrame.format_ = video_frame.format_; pVideoFrame.width_ = video_frame.width_; pVideoFrame.height_ = video_frame.height_; pVideoFrame.timestamp_ = video_frame.timestamp_; pVideoFrame.stride0_ = video_frame.stride0_; pVideoFrame.stride1_ = video_frame.stride1_; pVideoFrame.stride2_ = video_frame.stride2_; pVideoFrame.stride3_ = video_frame.stride3_; if (sync_invoke_ != null) { System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke; if (sync_invoke_target != null) { Int32 argb_size = video_frame.stride0_ * video_frame.height_; pVideoFrame.plane0_ = Marshal.AllocHGlobal(argb_size); CopyMemory(pVideoFrame.plane0_, video_frame.plane0_, (UInt32)argb_size); if (sync_invoke_target.InvokeRequired) { sync_invoke_target.BeginInvoke(set_video_frame_call_back_, new object[] { status, pVideoFrame }); } else { set_video_frame_call_back_(status, pVideoFrame); } } } } public void SDKVideoFrameCallBack(UInt32 status, NT_SP_VideoFrame frame) { if (cur_video_frame_.plane0_ != IntPtr.Zero) { Marshal.FreeHGlobal(cur_video_frame_.plane0_); cur_video_frame_.plane0_ = IntPtr.Zero; } cur_video_frame_ = frame; if (render_wnd_ != null) { render_wnd_.Invalidate(); } } public void SDKPullEventCallBack(IntPtr handle, IntPtr user_data, UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, [MarshalAs(UnmanagedType.LPStr)] String param4, [MarshalAs(UnmanagedType.LPStr)] String param5, IntPtr param6) { if (sync_invoke_ != null) { System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke; if (sync_invoke_target != null ) { if (sync_invoke_target.InvokeRequired) { sync_invoke_target.BeginInvoke(set_pull_event_call_back_, new object[] { event_id, param1, param2, param3, param4, param5, param6 }); } else { set_pull_event_call_back_(event_id, param1, param2, param3, param4, param5, param6); } } } } private void PullEventCallBack(UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, [MarshalAs(UnmanagedType.LPStr)] String param4, [MarshalAs(UnmanagedType.LPStr)] String param5, IntPtr param6) { if (!is_playing_) { return; } String show_str = ""; if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_PLAYBACK_REACH_EOS == event_id) { StopPlay(); return; } else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_RTSP_STATUS_CODE == event_id) { int status_code = (int)param1; show_str = "RTSP incorrect status code received: " + status_code.ToString() + ", 请确保用户名/密码正确"; } if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTING == event_id || (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTION_FAILED == event_id || (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTED == event_id || (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_DISCONNECTED == event_id) { connection_status_ = event_id; } if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_START_BUFFERING == event_id || (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_BUFFERING == event_id || (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_STOP_BUFFERING == event_id) { buffer_status_ = event_id; if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_BUFFERING == event_id) { buffer_percent_ = (Int32)param1; } } if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_DOWNLOAD_SPEED == event_id) { download_speed_ = (Int32)param1; } if (connection_status_ != 0) { show_str += "连接状态: "; if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTING == connection_status_) { show_str += "连接中"; } else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTION_FAILED == connection_status_) { show_str += "连接失败"; } else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTED == connection_status_) { show_str += "连接成功"; } else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_DISCONNECTED == connection_status_) { show_str += "断开连接"; } } if (download_speed_ != -1) { String ss = " 下载速度: " + (download_speed_ * 8 / 1000).ToString() + "kbps " + (download_speed_ / 1024).ToString() + "KB/s"; show_str += ss; } if (buffer_status_ != 0) { show_str += " 缓冲状态: "; if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_START_BUFFERING == buffer_status_) { show_str += "开始缓冲"; } else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_BUFFERING == buffer_status_) { String ss = "缓冲中 " + buffer_percent_.ToString() + "%"; show_str += ss; } else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_STOP_BUFFERING == buffer_status_) { show_str += "结束缓冲"; } } if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_NEED_KEY == event_id) { show_str = "RTMP加密流,请设置播放需要的Key.."; } else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_KEY_ERROR == event_id) { show_str = "RTMP加密流,Key错误,请重新设置.."; } EventGetPlayerEventMsg(show_str); } public void SetMute(bool is_mute) { is_mute_ = is_mute; if ( !is_playing_ ) return; NTSmartPlayerSDK.NT_SP_SetMute(player_handle_, is_mute ? 1 : 0); } public void SetBuffer(int buffer_time) { if (buffer_time >= 0) { play_buffer_ = buffer_time; } } public bool IsPlaying() { return is_playing_; } public bool OpenPullHandle(String url, bool is_rtsp_tcp_mode, bool is_mute) { if ( player_handle_ != IntPtr.Zero ) return true; if ( String.IsNullOrEmpty(url) ) return false; IntPtr pull_handle = IntPtr.Zero; if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPlayerSDK.NT_SP_Open(out pull_handle, IntPtr.Zero, 0, IntPtr.Zero)) { return false; } if (pull_handle == IntPtr.Zero) { return false; } pull_event_call_back_ = new SP_SDKEventCallBack(SDKPullEventCallBack); NTSmartPlayerSDK.NT_SP_SetEventCallBack(pull_handle, IntPtr.Zero, pull_event_call_back_); resolution_notify_callback_ = new ResolutionNotifyCallback(PlaybackWindowResized); set_video_frame_call_back_ = new VideoFrameCallBack(SDKVideoFrameCallBack); NTSmartPlayerSDK.NT_SP_SetBuffer(pull_handle, play_buffer_); NTSmartPlayerSDK.NT_SP_SetFastStartup(pull_handle, 1); NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(pull_handle, 1); NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(pull_handle, is_rtsp_tcp_mode ? 1 : 0); NTSmartPlayerSDK.NT_SP_SetMute(pull_handle, is_mute_ ? 1 : 0); //RTSP timeout设置 Int32 rtsp_timeout = 10; NTSmartPlayerSDK.NT_SP_SetRtspTimeout(pull_handle, rtsp_timeout); //RTSP TCP/UDP自动切换设置 Int32 is_auto_switch_tcp_udp = 1; NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(pull_handle, is_auto_switch_tcp_udp); NTSmartPlayerSDK.NT_SP_SetMute(pull_handle, is_mute ? 1 : 0); if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPlayerSDK.NT_SP_SetURL(pull_handle, url)) { NTSmartPlayerSDK.NT_SP_Close(pull_handle); pull_handle = IntPtr.Zero; return false; } player_handle_ = pull_handle; return true; } private void PlaybackWindowResized(Int32 width, Int32 height) { String resolution = width + "*" + height; EventGetVideoSize(resolution); } public void SP_SDKVideoSizeHandle(IntPtr handle, IntPtr userData, Int32 width, Int32 height) { if (null == sync_invoke_) return; System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke; if (sync_invoke_target != null) { if (sync_invoke_target.InvokeRequired) { sync_invoke_target.BeginInvoke(resolution_notify_callback_, new object[] { width, height }); } else { resolution_notify_callback_(width, height); } } } public bool StartPlay(String url, bool is_rtsp_tcp_mode, bool is_mute) { if ( is_playing_ ) return false; if ( !OpenPullHandle(url, is_rtsp_tcp_mode, is_mute) ) return false; NTSmartPlayerSDK.NT_SP_SetMute(player_handle_, is_mute ? 1 : 0); //video resolution callback video_size_call_back_ = new SP_SDKVideoSizeCallBack(SP_SDKVideoSizeHandle); NTSmartPlayerSDK.NT_SP_SetVideoSizeCallBack(player_handle_, IntPtr.Zero, video_size_call_back_); bool is_support_d3d_render = false; Int32 in_support_d3d_render = 0; if (NT.NTBaseCodeDefine.NT_ERC_OK == NTSmartPlayerSDK.NT_SP_IsSupportD3DRender(player_handle_, render_wnd_.Handle, ref in_support_d3d_render)) { if (1 == in_support_d3d_render) { is_support_d3d_render = true; } } // is_support_d3d_render = false; if (is_support_d3d_render) { // 支持d3d绘制的话,就用D3D绘制 NTSmartPlayerSDK.NT_SP_SetRenderWindow(player_handle_, render_wnd_.Handle); NTSmartPlayerSDK.NT_SP_SetRenderScaleMode(player_handle_, 1); } else { // 不支持D3D就让播放器吐出数据来,用GDI绘制,本demo仅用来展示一对一互动使用,具体可参考播放端的demo //video frame callback (YUV/RGB) //format请参见 NT_SP_E_VIDEO_FRAME_FORMAT,如需回调YUV,请设置为 NT_SP_E_VIDEO_FRAME_FROMAT_I420 video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack); NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FORMAT_RGB32, IntPtr.Zero, video_frame_call_back_); } uint ret = NTSmartPlayerSDK.NT_SP_StartPlay(player_handle_); if ( NTBaseCodeDefine.NT_ERC_OK != ret ) { NTSmartPlayerSDK.NT_SP_Close(player_handle_); player_handle_ = IntPtr.Zero; return false; } is_playing_ = true; return true; } public void StopPlay(bool is_update_ui =true) { if ( !is_playing_ ) return; NTSmartPlayerSDK.NT_SP_StopPlay(player_handle_); NTSmartPlayerSDK.NT_SP_Close(player_handle_); player_handle_ = IntPtr.Zero; is_playing_ = false; if (is_update_ui && render_wnd_ != null) { render_wnd_.Invalidate(); } } private void GetRenderRect(int limtWidth, int limtHeight, int image_w, int image_h, ref int left_offset, ref int top_offset, ref int dw, ref int dh) { if (limtWidth < 1 || limtHeight < 1) { left_offset = 0; top_offset = 0; dw = limtWidth; dh = limtHeight; return; } if (image_w < 1 || image_h < 1) { left_offset = 0; top_offset = 0; dw = limtWidth; dh = limtHeight; return; } // 按比例 double limit_ratio = limtWidth * 1.0 / limtHeight; double video_ratio = image_w * 1.0 / image_h; if (video_ratio > limit_ratio) { dw = limtWidth; dh = (int)(dw * image_h * 1.0 / image_w); if (dh > limtHeight) dh = limtHeight; } else { dh = limtHeight; dw = (int)(dh * image_w * 1.0 / image_h); if (dw > limtWidth) dw = limtWidth; } left_offset = limtWidth / 2 - dw / 2; if (left_offset < 0) left_offset = 0; top_offset = limtHeight / 2 - dh / 2; if (top_offset < 0) top_offset = 0; } private void OnRenderWindowPaint(object sender, PaintEventArgs e) { if (render_wnd_.Width < 1 || render_wnd_.Height < 1) return; Graphics g = e.Graphics; Brush brush = new SolidBrush(Color.Black); g.FillRectangle(brush, 0, 0, render_wnd_.Width, render_wnd_.Height); if ( IsPlaying() && cur_video_frame_.plane0_ != IntPtr.Zero) { g.SmoothingMode = SmoothingMode.HighSpeed; int image_width = cur_video_frame_.width_; int image_height = cur_video_frame_.height_; Bitmap bitmap = new Bitmap(image_width, image_height, cur_video_frame_.stride0_, System.Drawing.Imaging.PixelFormat.Format32bppRgb, cur_video_frame_.plane0_); int d_w = 0, d_h = 0; int left_offset = 0; int top_offset = 0; GetRenderRect(render_wnd_.Width, render_wnd_.Height, image_width, image_height, ref left_offset, ref top_offset, ref d_w, ref d_h); g.DrawImage(bitmap, left_offset, top_offset, d_w, d_h); //在窗体的画布中绘画出内存中的图像 } } } }
基于 https://github.com/daniulive/SmarterStreaming/ 推流端封装的代码:
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Runtime.InteropServices; using System.Windows.Forms; using System.Drawing; using System.Drawing.Drawing2D; using NT; namespace SmartEchoCancellationDemo { public delegate void DelGetPublisherEventMsg(String msg); //推送端Event消息 public struct NT_VideoFrame { public Int32 width_; // 图像宽 public Int32 height_; // 图像高 public IntPtr plane_; public Int32 stride_; } public struct CameraInfo { public String name_; public String id_; public List<NT_PB_VideoCaptureCapability> capabilities_; }; class nt_publisher_wrapper : IDisposable { [DllImport("kernel32", EntryPoint = "CopyMemory")] static extern void CopyMemory(IntPtr Destination, IntPtr Source, uint Length); private bool disposed_ = false; private IntPtr publisher_handle_ = IntPtr.Zero; private System.Windows.Forms.Control render_wnd_ = null; private System.Windows.Forms.PaintEventHandler render_wnd_paint_event_ = null; private int publisher_handle_count_; private bool is_publishing_ = false; private bool is_previewing_ = false; private WeakReference sync_invoke_ = null; //event事件回调 NT_PB_SDKEventCallBack pb_event_call_back_; delegate void PbSetEventCallBack(UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, UInt64 param4, [MarshalAs(UnmanagedType.LPStr)] String param5, [MarshalAs(UnmanagedType.LPStr)] String param6, IntPtr param7); PbSetEventCallBack pb_set_event_call_back_; //预览数据回调 NT_PB_SDKVideoPreviewImageCallBack video_preview_image_callback_; delegate void SetVideoPreviewImageCallBack(NT_VideoFrame frame); SetVideoPreviewImageCallBack set_video_preview_image_callback_; private NT_VideoFrame cur_image_ = new NT_VideoFrame(); public event DelGetPublisherEventMsg EventGetPublisherEventMsg; public nt_publisher_wrapper(System.Windows.Forms.Control render_wnd, System.ComponentModel.ISynchronizeInvoke sync_invoke) { render_wnd_ = render_wnd; sync_invoke_ = new WeakReference(sync_invoke); ; pb_set_event_call_back_ = new PbSetEventCallBack(PbEventCallBack); if (render_wnd_ != null) { render_wnd_paint_event_ = new System.Windows.Forms.PaintEventHandler(this.OnRenderWindowPaint); render_wnd_.Paint += render_wnd_paint_event_; } } public void Dispose() { Dispose(true); // This object will be cleaned up by the Dispose method. // Therefore, you should call GC.SupressFinalize to // take this object off the finalization queue // and prevent finalization code for this object // from executing a second time. // GC.SuppressFinalize(this); } protected virtual void Dispose(bool disposing) { // Check to see if Dispose has already been called. if (!this.disposed_) { if (disposing) { } if (render_wnd_ != null && render_wnd_paint_event_ != null) { render_wnd_.Paint -= render_wnd_paint_event_; } render_wnd_paint_event_ = null; if (cur_image_.plane_ != IntPtr.Zero) { Marshal.FreeHGlobal(cur_image_.plane_); cur_image_.plane_ = IntPtr.Zero; } // Note disposing has been done. disposed_ = true; } } ~nt_publisher_wrapper() { Dispose(false); } private void PbEventCallBack(UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, UInt64 param4, [MarshalAs(UnmanagedType.LPStr)] String param5, [MarshalAs(UnmanagedType.LPStr)] String param6, IntPtr param7) { String event_log = ""; switch (event_id) { case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTING: event_log = "连接中"; if (!String.IsNullOrEmpty(param5)) { event_log = event_log + " url:" + param5; } break; case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTION_FAILED: event_log = "连接失败"; if (!String.IsNullOrEmpty(param5)) { event_log = event_log + " url:" + param5; } break; case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTED: event_log = "已连接"; if (!String.IsNullOrEmpty(param5)) { event_log = event_log + " url:" + param5; } break; case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_DISCONNECTED: event_log = "断开连接"; if (!String.IsNullOrEmpty(param5)) { event_log = event_log + " url:" + param5; } break; default: break; } EventGetPublisherEventMsg(event_log); } public int CalBitRate(int frame_rate, int w, int h) { int kbit_rate = 2000; int area = w * h; if (area <= (320 * 300)) { kbit_rate = 280; } else if (area <= (360 * 320)) { kbit_rate = 360; } else if (area <= (640 * 480)) { kbit_rate = 580; } else if (area <= (800 * 600)) { kbit_rate = 620; } else if (area <= (900 * 700)) { kbit_rate = 820; } else if (area <= (1280 * 720)) { kbit_rate = 1600; } else if (area <= (1366 * 768)) { kbit_rate = 2000; } else if (area <= (1600 * 900)) { kbit_rate = 2300; } else if (area <= (1600 * 1050)) { kbit_rate = 2500; } else { kbit_rate = 2800; } kbit_rate = kbit_rate * frame_rate / 25; if (kbit_rate < 80) kbit_rate = 80; return kbit_rate; } public int CalMaxKBitRate(int frame_rate, int w, int h, bool is_var_bitrate) { int max_kbit_rate = 2000; int area = w * h; if (area <= (320 * 300)) { max_kbit_rate = is_var_bitrate ? 320 : 600; } else if (area <= (360 * 320)) { max_kbit_rate = is_var_bitrate ? 400 : 800; } else if (area <= (640 * 360)) { max_kbit_rate = is_var_bitrate ? 600 : 1000; } else if (area <= (640 * 480)) { max_kbit_rate = is_var_bitrate ? 680 : 1300; } else if (area <= (800 * 600)) { max_kbit_rate = is_var_bitrate ? 700 : 1500; } else if (area <= (900 * 700)) { max_kbit_rate = is_var_bitrate ? 920 : 2200; } else if (area <= (1280 * 720)) { max_kbit_rate = is_var_bitrate ? 1600 : 3000; } else if (area <= (1366 * 768)) { max_kbit_rate = is_var_bitrate ? 1700 : 3300; } else if (area <= (1600 * 900)) { max_kbit_rate = is_var_bitrate ? 2400 : 3400; } else if (area <= (1600 * 1050)) { max_kbit_rate = is_var_bitrate ? 2600 : 3600; } else if (area <= (1920 * 1080)) { max_kbit_rate = is_var_bitrate ? 2900 : 3800; } else { max_kbit_rate = is_var_bitrate ? 3500 : 5500; } max_kbit_rate = max_kbit_rate * frame_rate / 25; if (area <= (320 * 240)) { if (max_kbit_rate < 150) max_kbit_rate = 150; } else if (area <= (640 * 480)) { if (max_kbit_rate < 300) max_kbit_rate = 300; } else if (area <= (1280 * 720)) { if (max_kbit_rate < 600) max_kbit_rate = 600; } else if (area <= (1920 * 1080)) { if (max_kbit_rate < 960) max_kbit_rate = 960; } else { if (max_kbit_rate < 1500) max_kbit_rate = 1500; } return max_kbit_rate; } public int CalVideoQuality(int w, int h, bool is_h264) { int area = w * h; int quality = is_h264 ? 23 : 28; if (area <= (320 * 240)) { quality = is_h264 ? 23 : 27; } else if (area <= (640 * 360)) { quality = is_h264 ? 25 : 28; } else if (area <= (640 * 480)) { quality = is_h264 ? 25 : 28; } else if (area <= (960 * 600)) { quality = is_h264 ? 26 : 28; } else if (area <= (1280 * 720)) { quality = is_h264 ? 27 : 29; } else if (area <= (1600 * 900)) { quality = is_h264 ? 28 : 30; } else if (area <= (1920 * 1080)) { quality = is_h264 ? 29 : 31; } else { quality = is_h264 ? 30 : 32; } return quality; } public int CalVideoEncoderSpeed(int w, int h, bool is_h264) { if (is_h264) return 3; int area = w * h; if (area <= (960 * 600)) { return 3; } else if (area <= (1280 * 720)) { return 2; } else { return 1; } } public int GetAudioInputDeviceNumber() { int auido_devices = 0; NTSmartPublisherSDK.NT_PB_GetAuidoInputDeviceNumber(ref auido_devices); return auido_devices; } public List<String> GetAudioInputDeviceName(int auido_devices) { List<String> audio_device_name = new List<string>(); if (auido_devices > 0) { for (int i = 0; i < auido_devices; ++i) { byte[] deviceNameBuffer = new byte[512]; string name = ""; if (NTBaseCodeDefine.NT_ERC_OK == NTSmartPublisherSDK.NT_PB_GetAuidoInputDeviceName((uint)i, deviceNameBuffer, 512)) { int count = 0; for (int j = 0; j < deviceNameBuffer.Length; ++j) { if (deviceNameBuffer[j] != 0) { count++; } else { break; } } if (count > 0) { name = Encoding.UTF8.GetString(deviceNameBuffer, 0, count); } } var audio_name = ""; if (name.Length == 0) { audio_name = "音频采集设备-"; } else { audio_name = name + "-"; } audio_name = audio_name + (i + 1); audio_device_name.Add(name); } } return audio_device_name; } public bool IsCanCaptureSpeaker() { int is_capture_speader = 0; if (NTBaseCodeDefine.NT_ERC_OK == NTSmartPublisherSDK.NT_PB_IsCanCaptureSpeaker(ref is_capture_speader)) { if (1 == is_capture_speader) { return true; } } return false; } public bool OpenPublisherHandle(uint video_option, uint audio_option) { if (publisher_handle_ != IntPtr.Zero) { return true; } publisher_handle_count_ = 0; if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_Open(out publisher_handle_, video_option, audio_option, 0, IntPtr.Zero)) { return false; } if (publisher_handle_ != IntPtr.Zero) { pb_event_call_back_ = new NT_PB_SDKEventCallBack(PbSDKEventCallBack); NTSmartPublisherSDK.NT_PB_SetEventCallBack(publisher_handle_, IntPtr.Zero, pb_event_call_back_); set_video_preview_image_callback_ = new SetVideoPreviewImageCallBack(VideoPreviewImageCallBack); return true; } else { return false; } } public void PbSDKEventCallBack(IntPtr handle, IntPtr user_data, UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, UInt64 param4, [MarshalAs(UnmanagedType.LPStr)] String param5, [MarshalAs(UnmanagedType.LPStr)] String param6, IntPtr param7) { if (sync_invoke_ != null) { System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke; if (sync_invoke_target != null) { if (sync_invoke_target.InvokeRequired) { sync_invoke_target.BeginInvoke(pb_set_event_call_back_, new object[] { event_id, param1, param2, param3, param4, param5, param6, param7 }); } else { pb_set_event_call_back_(event_id, param1, param2, param3, param4, param5, param6, param7); } } } } //预览数据回调 public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image) { NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image)); NT_VideoFrame pVideoFrame = new NT_VideoFrame(); pVideoFrame.width_ = pb_image.width_; pVideoFrame.height_ = pb_image.height_; pVideoFrame.stride_ = pb_image.stride_[0]; Int32 argb_size = pb_image.stride_[0] * pb_image.height_; pVideoFrame.plane_ = Marshal.AllocHGlobal(argb_size); CopyMemory(pVideoFrame.plane_, pb_image.plane_[0], (UInt32)argb_size); if (sync_invoke_ != null) { System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke; if (sync_invoke_target != null) { if (sync_invoke_target.InvokeRequired) { sync_invoke_target.BeginInvoke(set_video_preview_image_callback_, new object[] { pVideoFrame }); } else { set_video_preview_image_callback_(pVideoFrame); } } } } public void VideoPreviewImageCallBack(NT_VideoFrame frame) { if (cur_image_.plane_ != IntPtr.Zero) { Marshal.FreeHGlobal(cur_image_.plane_); cur_image_.plane_ = IntPtr.Zero; } cur_image_ = frame; if ( render_wnd_ != null) { render_wnd_.Invalidate(); } } public List<CameraInfo> GetCameraInfos() { List<CameraInfo> cameras = new List<CameraInfo>(); int device_number = 0; if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_GetVideoCaptureDeviceNumber(ref device_number)) { return cameras; } if (device_number < 1) { return cameras; } for (int i = 0; i < device_number; ++i) { CameraInfo info = new CameraInfo(); info.capabilities_ = new List<NT_PB_VideoCaptureCapability>(); StringBuilder name = new StringBuilder(256); StringBuilder id = new StringBuilder(1024); if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_GetVideoCaptureDeviceInfo(i, name, 256, id, 1024)) { continue; } info.name_ = name.ToString(); info.id_ = id.ToString(); int capability_number = 0; if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_GetVideoCaptureDeviceCapabilityNumber( id.ToString(), ref capability_number)) { continue; } bool is_failed = false; for (int j = 0; j < capability_number; ++j) { NT_PB_VideoCaptureCapability capability = new NT_PB_VideoCaptureCapability(); if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_GetVideoCaptureDeviceCapability( id.ToString(), j, ref capability)) { is_failed = true; break; } info.capabilities_.Add(capability); } if (!is_failed) { cameras.Add(info); } } return cameras; } public bool StartPreview() { video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack); NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(publisher_handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_); if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(publisher_handle_, 0, IntPtr.Zero)) { if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } return false; } publisher_handle_count_++; is_previewing_ = true; return true; } public void StopPreview() { is_previewing_ = false; publisher_handle_count_--; NTSmartPublisherSDK.NT_PB_StopPreview(publisher_handle_); if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } if (render_wnd_ != null) { render_wnd_.Invalidate(); } } public bool StartPublisher(String url) { if (publisher_handle_ == IntPtr.Zero) { return false; } if (!String.IsNullOrEmpty(url)) { NTSmartPublisherSDK.NT_PB_SetURL(publisher_handle_, url, IntPtr.Zero); } if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(publisher_handle_, IntPtr.Zero)) { if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } is_publishing_ = false; return false; } publisher_handle_count_++; is_publishing_ = true; return true; } public void StopPublisher() { publisher_handle_count_--; NTSmartPublisherSDK.NT_PB_StopPublisher(publisher_handle_); if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } is_publishing_ = false; } public void Close() { if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } } public bool IsPreviewing() { return is_previewing_; } public bool IsPublishing() { return is_publishing_; } public bool IsPublisherHandleAvailable() { return publisher_handle_ != IntPtr.Zero ? true : false; } public int GetPublisherHandleCount() { return publisher_handle_count_; } public void SetVideoCaptureDeviceBaseParameter(String camera_id, UInt32 width, UInt32 height) { NTSmartPublisherSDK.NT_PB_SetVideoCaptureDeviceBaseParameter(publisher_handle_, camera_id, width, height); } public void SetFrameRate(UInt32 frame_rate) { NTSmartPublisherSDK.NT_PB_SetFrameRate(publisher_handle_, frame_rate); } public void SetVideoEncoderType(Int32 encode_type) { NTSmartPublisherSDK.NT_PB_SetVideoEncoderType(publisher_handle_, encode_type); } public void SetVideoQualityV2(Int32 quality) { NTSmartPublisherSDK.NT_PB_SetVideoQualityV2(publisher_handle_, quality); } public void SetVideoMaxBitRate(Int32 kbit_rate) { NTSmartPublisherSDK.NT_PB_SetVideoMaxBitRate(publisher_handle_, kbit_rate); } public void SetVideoKeyFrameInterval(Int32 interval) { NTSmartPublisherSDK.NT_PB_SetVideoKeyFrameInterval(publisher_handle_, interval); } public void SetVideoEncoderProfile(Int32 profile) { NTSmartPublisherSDK.NT_PB_SetVideoEncoderProfile(publisher_handle_, profile); } public void SetVideoEncoderSpeed(Int32 speed) { NTSmartPublisherSDK.NT_PB_SetVideoEncoderSpeed(publisher_handle_, speed); } public void SetAuidoInputDeviceId(UInt32 device_id) { NTSmartPublisherSDK.NT_PB_SetAuidoInputDeviceId(publisher_handle_, device_id); } public void SetPublisherAudioCodecType(Int32 type) { NTSmartPublisherSDK.NT_PB_SetPublisherAudioCodecType(publisher_handle_, type); } public void SetPublisherMute(bool is_mute) { NTSmartPublisherSDK.NT_PB_SetMute(publisher_handle_, is_mute ? 1 : 0); } public void SetEchoCancellation(Int32 isCancel, Int32 delay) { NTSmartPublisherSDK.NT_PB_SetEchoCancellation(publisher_handle_, isCancel, delay); } public void SetNoiseSuppression(Int32 isNS) { NTSmartPublisherSDK.NT_PB_SetNoiseSuppression(publisher_handle_, isNS); } public void SetAGC(Int32 isAGC) { NTSmartPublisherSDK.NT_PB_SetAGC(publisher_handle_, isAGC); } public void SetVAD(Int32 isVAD) { NTSmartPublisherSDK.NT_PB_SetVAD(publisher_handle_, isVAD); } private void GetRenderRect(int limtWidth, int limtHeight, int image_w, int image_h, ref int left_offset, ref int top_offset, ref int dw, ref int dh) { if (limtWidth < 1 || limtHeight < 1) { left_offset = 0; top_offset = 0; dw = limtWidth; dh = limtHeight; return; } if (image_w < 1 || image_h < 1) { left_offset = 0; top_offset = 0; dw = limtWidth; dh = limtHeight; return; } // 按比例 double limit_ratio = limtWidth * 1.0 / limtHeight; double video_ratio = image_w * 1.0 / image_h; if (video_ratio > limit_ratio) { dw = limtWidth; dh = (int)(dw * image_h * 1.0 / image_w); if (dh > limtHeight) dh = limtHeight; } else { dh = limtHeight; dw = (int)(dh * image_w * 1.0 / image_h); if (dw > limtWidth) dw = limtWidth; } left_offset = limtWidth / 2 - dw / 2; if (left_offset < 0) left_offset = 0; top_offset = limtHeight / 2 - dh / 2; if (top_offset < 0) top_offset = 0; } private void OnRenderWindowPaint(object sender, PaintEventArgs e) { if (render_wnd_.Width < 1 || render_wnd_.Height < 1) return; Graphics g = e.Graphics; Brush brush = new SolidBrush(Color.Black); g.FillRectangle(brush, 0, 0, render_wnd_.Width, render_wnd_.Height); if (is_previewing_ && cur_image_.plane_ != IntPtr.Zero) { g.SmoothingMode = SmoothingMode.HighSpeed; int image_width = cur_image_.width_; int image_height = cur_image_.height_; Bitmap bitmap = new Bitmap(image_width, image_height, cur_image_.stride_, System.Drawing.Imaging.PixelFormat.Format32bppRgb, cur_image_.plane_); int d_w = 0, d_h = 0; int left_offset = 0; int top_offset = 0; GetRenderRect(render_wnd_.Width, render_wnd_.Height, image_width, image_height, ref left_offset, ref top_offset, ref d_w, ref d_h); g.DrawImage(bitmap, left_offset, top_offset, d_w, d_h); //在窗体的画布中绘画出内存中的图像 } } } }
由于RTMP在0缓冲下,延迟在200-400毫秒区间,常规的对延迟不是非常苛刻的场景下,足够用了。