技术背景
在探讨Unity平台RTMP或RTSP直播流数据播放和录制之前，我们先简单回顾下RTSP或RTMP直播流数据在Unity平台的播放流程：
- 通过Native RTSP或RTSP直播播放SDK回调RGB/YUV420/NV12等其中的一种未压缩的图像格式；
- Unity下创建相应的RGB/YUV420等Shader；
- Unity从各个平台获取图像数据来填充纹理即可。
Unity3D环境下，我们之前已覆盖以下模块：
- Windows平台RTMP直播推送模块（采集Unity窗体、摄像头或屏幕）;
- Windows平台RTMP|RTSP直播播放模块；
- Linux平台RTMP直播推送模块（采集Unity窗体、Unity声音）;
- Linux平台RTMP|RTSP直播播放模块；
- Android平台RTMP直播推送模块（采集Unity窗体、麦克风或Unity声音）；
- Android平台RTMP|RTSP直播播放模块；
- iOS平台RTMP|RTSP直播播放模块。
技术实现
本文主要介绍流数据录制，实际上，录制相对直播播放来说，更简单一些，毕竟不需要解码绘制。本文以Windows平台下Unity为例，介绍下流数据的录制流程，无图无真相：
开始录像
我们可指定文件录制规则，如是否需要录制纯音频或纯视频、单个录制文件大小限制、文件录制目录等参数设定，并设置录像回调事件：
/* * SmartPlayerWinMono.cs * * WebSite: http://daniusdk.com * Created by DaniuSDK on 2018/05/10. */ private void StartRecorder(int sel) { Debug.Log("StartRecorder++, sel: " + sel); if (videoctrl[sel].is_recording_) { Debug.Log("StartRecorder, already started.. sel: " + sel); return; } if (!videoctrl[sel].is_playing_) { if (!OpenPlayerHandle(sel)) { Debug.LogError("call OpenPlayerHandle failed.."); return; } } bool is_rec_video = true; bool is_rec_audio = true; NTSmartPlayerSDK.NT_SP_SetRecorderVideo(videoctrl[sel].player_handle_, is_rec_video ? 1 : 0); NTSmartPlayerSDK.NT_SP_SetRecorderAudio(videoctrl[sel].player_handle_, is_rec_audio ? 1 : 0); String rec_dir = "D:\\Rec"; //录像目录可自行指定 String rec_name_file_prefix_= "daniu" + sel.ToString(); UInt32 max_file_size = 200 * 1024; // 单位是KByte, 默认200MB bool is_append_date = true; bool is_append_time = true; bool is_audio_transcode_aac = true; UInt32 ret = NTSmartPlayerSDK.NT_SP_SetRecorderDirectory(videoctrl[sel].player_handle_, rec_dir); if (NT.NTBaseCodeDefine.NT_ERC_OK != ret) { Debug.LogError("设置录像目录失败，请确保目录存在且是英文目录"); return; } NTSmartPlayerSDK.NT_SP_SetRecorderFileMaxSize(videoctrl[sel].player_handle_, max_file_size); NT_SP_RecorderFileNameRuler rec_name_ruler = new NT_SP_RecorderFileNameRuler(); rec_name_ruler.type_ = 0; rec_name_ruler.file_name_prefix_ = rec_name_file_prefix_; rec_name_ruler.append_date_ = is_append_date ? 1 : 0; rec_name_ruler.append_time_ = is_append_time ? 1 : 0; NTSmartPlayerSDK.NT_SP_SetRecorderFileNameRuler(videoctrl[sel].player_handle_, ref rec_name_ruler); NTSmartPlayerSDK.NT_SP_SetRecorderAudioTranscodeAAC(videoctrl[sel].player_handle_, is_audio_transcode_aac ? 1 : 0); videoctrl[sel].record_call_back_ = new SP_SDKRecorderCallBack(NT_SP_SDKRecorderCallBack); NTSmartPlayerSDK.NT_SP_SetRecorderCallBack(videoctrl[sel].player_handle_, IntPtr.Zero, videoctrl[sel].record_call_back_); videoctrl[sel].set_record_call_back_ = new VideoControl.SetRecordCallBack(RecordCallBack); if (NT.NTBaseCodeDefine.NT_ERC_OK != NTSmartPlayerSDK.NT_SP_StartRecorder(videoctrl[sel].player_handle_)) { Debug.LogError("call NT_SP_StartRecorder failed.."); return; } videoctrl[sel].is_recording_ = true; }
这里需要注意的是，OpenPlayerHandle()针对播放端的设定，同一个实例，播放和录像仅需设置一次，具体实现如下：
private bool OpenPlayerHandle(int sel) { if (videoctrl[sel].player_handle_ != IntPtr.Zero) return true; window_handle_ = IntPtr.Zero; if (videoctrl[sel].player_handle_ == IntPtr.Zero) { videoctrl[sel].player_handle_ = new IntPtr(); UInt32 ret_open = NTSmartPlayerSDK.NT_SP_Open(out videoctrl[sel].player_handle_, window_handle_, 0, IntPtr.Zero); if (ret_open != 0) { videoctrl[sel].player_handle_ = IntPtr.Zero; Debug.LogError("call NT_SP_Open failed, sel: " + sel); return false; } } videoctrl[sel].event_call_back_ = new SP_SDKEventCallBack(NT_SP_SDKEventCallBack); NTSmartPlayerSDK.NT_SP_SetEventCallBack(videoctrl[sel].player_handle_, window_handle_, videoctrl[sel].event_call_back_); videoctrl[sel].sdk_event_call_back_ = new VideoControl.SetEventCallBack(SDKEventCallBack); if (IntPtr.Zero == videoctrl[sel].player_handle_) return false; /* ++ 播放前参数配置可加在此处 ++ */ int play_buffer_time_ = 100; NTSmartPlayerSDK.NT_SP_SetBuffer(videoctrl[sel].player_handle_, play_buffer_time_); //设置buffer time int is_using_tcp = 0; //TCP模式 NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(videoctrl[sel].player_handle_, is_using_tcp); int timeout = 10; NTSmartPlayerSDK.NT_SP_SetRtspTimeout(videoctrl[sel].player_handle_, timeout); int is_auto_switch_tcp_udp = 1; NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(videoctrl[sel].player_handle_, is_auto_switch_tcp_udp); Boolean is_mute_ = false; NTSmartPlayerSDK.NT_SP_SetMute(videoctrl[sel].player_handle_, is_mute_ ? 1 : 0); //是否启动播放的时候静音 int is_fast_startup = 1; NTSmartPlayerSDK.NT_SP_SetFastStartup(videoctrl[sel].player_handle_, is_fast_startup); //设置快速启动模式 Boolean is_low_latency_ = false; NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(videoctrl[sel].player_handle_, is_low_latency_ ? 1 : 0); //设置是否启用低延迟模式 //设置旋转角度(设置0， 90， 180， 270度有效，其他值无效) int rotate_degrees = 0; NTSmartPlayerSDK.NT_SP_SetRotation(videoctrl[sel].player_handle_, rotate_degrees); int volume = 100; NTSmartPlayerSDK.NT_SP_SetAudioVolume(videoctrl[sel].player_handle_, volume); //设置播放音量, 范围是[0, 100], 0是静音，100是最大音量, 默认是100 // 设置上传下载报速度 int is_report = 0; int report_interval = 2; NTSmartPlayerSDK.NT_SP_SetReportDownloadSpeed(videoctrl[sel].player_handle_, is_report, report_interval); //设置播放URL NTSmartPlayerSDK.NT_SP_SetURL(videoctrl[sel].player_handle_, videoctrl[sel].playback_url_); /* -- 播放前参数配置可加在此处 -- */ return true; }
停止录像
停止录制的时候，如果当前流数据没有在播放，可直接调用Close接口，释放实例并把player_handle置null。
private void StopRecorder(int sel) { Debug.Log("StopRecorder++, sel: " + sel); if (videoctrl[sel].player_handle_ == IntPtr.Zero) { return; } NTSmartPlayerSDK.NT_SP_StopRecorder(videoctrl[sel].player_handle_); videoctrl[sel].is_recording_ = false; if (!videoctrl[sel].is_playing_) { NTSmartPlayerSDK.NT_SP_Close(videoctrl[sel].player_handle_); videoctrl[sel].player_handle_ = IntPtr.Zero; } }
开始播放
接下来，我们回顾下播放流程，播放端我们以回调YUV数据为例：
public void StartPlayer(int sel) { Debug.Log("StartPlayer++, sel: " + sel); if (videoctrl[sel].is_playing_) { Debug.Log("StartPlayer, already started.. sel: " + sel); return; } lock (videoctrl[sel].frame_lock_) { videoctrl[sel].cur_video_frame_ = null; } if (!videoctrl[sel].is_recording_) { if (!OpenPlayerHandle(sel)) { Debug.LogError("call OpenPlayerHandle failed, sel:" + sel); return; } } if (is_enable_hardware_decoder_) { NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(videoctrl[sel].player_handle_, is_support_h264_hardware_decoder_ ? 1 : 0, 0); NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(videoctrl[sel].player_handle_, is_support_h265_hardware_decoder_ ? 1 : 0, 0); } else { NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(videoctrl[sel].player_handle_, 0, 0); NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(videoctrl[sel].player_handle_, 0, 0); } //video frame callback (YUV/RGB) videoctrl[sel].sdk_video_frame_call_back_ = new VideoControl.SetVideoFrameCallBack(SDKVideoFrameCallBack); videoctrl[sel].video_frame_call_back_ = new SP_SDKVideoFrameCallBack(NT_SP_SetVideoFrameCallBack); NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(videoctrl[sel].player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, window_handle_, videoctrl[sel].video_frame_call_back_); UInt32 flag = NTSmartPlayerSDK.NT_SP_StartPlay(videoctrl[sel].player_handle_); if (flag == DANIULIVE_RETURN_OK) { videoctrl[sel].is_need_get_frame_ = true; Debug.Log("NT_SP_StartPlay succeed, sel:" + sel); } else { videoctrl[sel].is_need_get_frame_ = false; Debug.LogError("NT_SP_StartPlay failed, sel:" + sel); } videoctrl[sel].is_playing_ = true; }
停止播放
停止播放的时候和录像一样，如果当前流数据没有在录制状态，可直接调用Close接口，释放实例并把player_handle置null。
private void StopPlayer(int sel) { Debug.Log("StopPlayer++, sel: " + sel); videoctrl[sel].is_need_get_frame_ = false; videoctrl[sel].is_need_init_texture_ = false; if (videoctrl[sel].player_handle_ == IntPtr.Zero) { return; } UInt32 flag = NTSmartPlayerSDK.NT_SP_StopPlay(videoctrl[sel].player_handle_); if (flag == DANIULIVE_RETURN_OK) { Debug.Log("call NT_SP_StopPlay succeed, sel: " + sel); } else { Debug.LogError("call NT_SP_StopPlay failed, sel: " + sel); } if (!videoctrl[sel].is_recording_) { NTSmartPlayerSDK.NT_SP_Close(videoctrl[sel].player_handle_); videoctrl[sel].player_handle_ = IntPtr.Zero; } videoctrl[sel].is_playing_ = false; }
总结
Unity下实现RTMP或RTSP流录像，相对来说技术难度不大，甚至不需要操作UI即可完成。需要注意的是，Windows平台录制特别是多路录制受限于磁盘IO写入等，如果录制多路，可能会存在硬件性能瓶颈。此外，录像和播放，共用一个player实例即可，播放和录像，设计需要考虑逻辑完全分离，比如可只播放或只录制，亦或播放的过程中随时录制，或者录像的过程中随时播放。
