技术背景
好多开发者,希望我们能探讨下Unity平台RTMP或RTSP直播流数据播放和录制相关的模块,实际上,这块流程我们已经聊过多次,无非就是通过原生的RTMP或者RTSP模块,先从协议层拉取到数据,并解包解码,回调YUV或RGB数据,然后,在Unity创建响应的shader,获取图像数据填充纹理即可,说起来流程很简单,但是每个环节,如果做到极致体验,都非常难。简单来说,多一次拷贝,都会增大性能瓶颈或延迟。
目前,Unity3D下,我们覆盖了以下常用的模块:
- Windows平台RTMP直播推送模块(采集Unity窗体、摄像头或屏幕);
- Windows平台轻量级RTSP服务模块(采集Unity窗体、摄像头或屏幕);
- Windows平台RTMP|RTSP直播播放模块;
- Linux平台RTMP直播推送模块(采集Unity窗体、Unity声音),也可扩展轻量级RTSP服务模块;
- Linux平台RTMP|RTSP直播播放模块;
- Android平台RTMP直播推送模块(采集Unity窗体、麦克风或Unity声音);
- Android平台轻量级RTSP服务模块(采集Unity窗体、麦克风或Unity声音);
- Android平台RTMP|RTSP直播播放模块;
- iOS平台RTMP|RTSP直播播放模块。
下图系Linux平台RTMP播放图,可以看到,延迟非常低。
技术实现
本文以Android平台RTMP、RTSP播放模块为例,介绍下Unity相关接口设置和逻辑处理:
开始播放
public void Play() { if (is_running) { Debug.Log("已经在播放。。"); return; } //获取输入框的url string videoUrl = input_url_.text.Trim(); OpenPlayer(); if ( player_handle_ == 0 ) return; NT_U3D_Set_Game_Object(player_handle_, game_object_); /* ++ 播放前参数配置可加在此处 ++ */ int is_using_tcp = 0; //TCP/UDP模式设置 NT_U3D_SetRTSPTcpMode(player_handle_, is_using_tcp); int is_report = 0; int report_interval = 1; NT_U3D_SetReportDownloadSpeed(player_handle_, is_report, report_interval); //下载速度回调 NT_U3D_SetBuffer(player_handle_, play_buffer_time_); //设置buffer time NT_U3D_SetPlayerLowLatencyMode(player_handle_, is_low_latency_ ? 1 : 0); //设置是否启用低延迟模式 NT_U3D_SetMute(player_handle_, is_mute_ ? 1 : 0); //是否启动播放的时候静音 NT_U3D_SetAudioVolume(player_handle_, cur_audio_volume_); //设置播放音量 NT_U3D_SetVideoDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0); //设置H.264软硬解模式 NT_U3D_SetVideoHevcDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0); //设置H.265软硬解模式 int is_fast_startup = 1; NT_U3D_SetFastStartup(player_handle_, is_fast_startup); //设置快速启动模式 int rtsp_timeout = 10; NT_U3D_SetRTSPTimeout(player_handle_, rtsp_timeout); //设置RTSP超时时间 int is_auto_switch_tcp_udp = 1; NT_U3D_SetRTSPAutoSwitchTcpUdp(player_handle_, is_auto_switch_tcp_udp); //设置TCP/UDP模式自动切换 int is_audiotrack = 1; NT_U3D_SetAudioOutputType(player_handle_, is_audiotrack); //设置音频输出模式: if 0: 自动选择; if with 1: audiotrack模式 NT_U3D_SetUrl(player_handle_, videoUrl); /* -- 播放前参数配置可加在此处 -- */ int flag = NT_U3D_StartPlay(player_handle_); if (flag == DANIULIVE_RETURN_OK) { is_need_get_frame_ = true; Debug.Log("播放成功"); } else { is_need_get_frame_ = false; Debug.LogError("播放失败"); } is_running = true; }
这里调用了OpenPlayer()设计如下:
private void OpenPlayer() { if ( java_obj_cur_activity_ == null ) { Debug.LogError("getApplicationContext is null"); return; } player_handle_ = NT_U3D_Open(); if (player_handle_ != 0) Debug.Log("open success"); else Debug.LogError("open fail"); }
停止播放:
private void ClosePlayer() { is_need_get_frame_ = false; is_need_init_texture_ = false; int flag = NT_U3D_StopPlay(player_handle_); if (flag == DANIULIVE_RETURN_OK) { Debug.Log("停止成功"); } else { Debug.LogError("停止失败"); } flag = NT_U3D_Close(player_handle_); if (flag == DANIULIVE_RETURN_OK) { Debug.Log("关闭成功"); } else { Debug.LogError("关闭失败"); } player_handle_ = 0; NT_U3D_UnInit(); is_running = false; video_width_ = 0; video_height_ = 0; }
Event状态回调处理:
/// <summary> /// android 传递过来 code /// </summary> /// <param name="event_message"></param> public void onNTSmartEvent(string event_message) { if (null == event_message || event_message.Length < 1) return; string[] strs = event_message.Split(','); if (null == strs || strs.Length < 6) return; string player_handle =strs[0]; string code = strs[1]; string param1 = strs[2]; string param2 = strs[3]; string param3 = strs[4]; string param4 = strs[5]; Debug.Log("[onNTSmartEvent] code: 0x" + Convert.ToString(Convert.ToInt32(code), 16)); String player_event = ""; switch (Convert.ToInt32(code)) { case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STARTED: player_event = "开始.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING: player_event = "连接中.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED: player_event = "连接失败.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED: player_event = "连接成功.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED: player_event = "连接断开.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP: player_event = "停止播放.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO: player_event = "分辨率信息: width: " + Convert.ToInt32(param1) + ", height: " + Convert.ToInt32(param2); break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED: player_event = "收不到媒体数据,可能是url错误.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL: player_event = "切换播放URL.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE: player_event = "快照: " + param1 + " 路径:" + param3; if (Convert.ToInt32(param1) == 0) { player_event = "截取快照成功.."; } else { player_event = "截取快照失败.."; } break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE: player_event = "[record]开始一个新的录像文件 : " + param3; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED: player_event = "[record]已生成一个录像文件 : " + param3; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING: player_event = "Start_Buffering.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING: player_event = "Buffering: " + Convert.ToInt32(param1); break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING: player_event = "Stop_Buffering.."; break; case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED: player_event = "download_speed:" + param1 + "Byte/s" + ", " + (Convert.ToInt32(param1) * 8 / 1000) + "kbps" + ", " + (Convert.ToInt32(param1) / 1024) + "KB/s"; break; } Debug.Log(player_event); player_event = null; strs = null; }
如果想扩展录像,实际上,我们也针对播放端录像做了接口的封装设计,整体接口设计如下:
/// <summary> /// SmartPlayer Unity Interface /// Author: daniusdk.com /// </summary> /// <summary> /// Init /// </summary> public int NT_U3D_Init() { return player_obj_.Call<int>("Init", java_obj_cur_activity_); } /// <summary> /// 开始 /// 返回播放句柄 /// </summary> public long NT_U3D_Open() { return player_obj_.Call<long>("Open"); } /// <summary> /// Register Game Object,用于消息传递 /// </summary> public int NT_U3D_Set_Game_Object(long handle, string gameObjectName) { return player_obj_.Call<int>("SetGameObject", handle, gameObjectName); } /// <summary> /// 设置H.264解码方式 false 软件解码 true 硬件解码 默认为false /// </summary> /// <param name="isHwDecoder"></param> public int NT_U3D_SetVideoDecoderMode(long handle, int isHwDecoder) { return player_obj_.Call<int>("SetPlayerVideoHWDecoder", handle, isHwDecoder); } /// <summary> /// 设置H.265 解码方式 false 软件解码 true 硬件解码 默认为false /// </summary> /// <param name="isHevcHwDecoder"></param> public int NT_U3D_SetVideoHevcDecoderMode(long handle, int isHevcHwDecoder) { return player_obj_.Call<int>("SetPlayerVideoHevcHWDecoder", handle, isHevcHwDecoder); } /// <summary> /// 设置音频输出模式: if 0: 自动选择; if with 1: audiotrack模式 /// </summary> /// <param name="use_audiotrack"></param> public int NT_U3D_SetAudioOutputType(long handle, int use_audiotrack) { return player_obj_.Call<int>("SetAudioOutputType", handle, use_audiotrack); } /// <summary> /// 设置播放端缓存大小, 默认200毫秒 /// </summary> /// <param name="buffer"></param> public int NT_U3D_SetBuffer(long handle, int buffer) { return player_obj_.Call<int>("SetBuffer", handle, buffer); } /// <summary> /// 接口可实时调用:设置是否实时静音,1:静音; 0: 取消静音 /// </summary> /// <param name="is_mute"></param> public int NT_U3D_SetMute(long handle, int is_mute) { return player_obj_.Call<int>("SetMute", handle, is_mute); } /// <summary> /// 接口可实时调用:设置播放音量,范围是[0, 100], 0是静音,100是最大音量, 默认是100 /// </summary> /// <param name="audio_volume"></param> public int NT_U3D_SetAudioVolume(long handle, int audio_volume) { return player_obj_.Call<int>("SetAudioVolume", handle, audio_volume); } /// <summary> /// 设置RTSP TCP模式, 1: TCP; 0: UDP /// </summary> /// <param name="is_using_tcp"></param> public int NT_U3D_SetRTSPTcpMode(long handle, int is_using_tcp) { return player_obj_.Call<int>("SetRTSPTcpMode", handle, is_using_tcp); } /// <summary> /// 设置RTSP超时时间, timeout单位为秒,必须大于0 /// </summary> /// <param name="timeout"></param> public int NT_U3D_SetRTSPTimeout(long handle, int timeout) { return player_obj_.Call<int>("SetRTSPTimeout", handle, timeout); } /// <summary> /// 设置RTSP TCP/UDP自动切换 /// NOTE: 对于RTSP来说,有些可能支持rtp over udp方式,有些可能支持使用rtp over tcp方式. /// 为了方便使用,有些场景下可以开启自动尝试切换开关, 打开后如果udp无法播放,sdk会自动尝试tcp, 如果tcp方式播放不了,sdk会自动尝试udp. /// </summary> /// <param name="timeout"></param> /// timeout:如果设置1的话, sdk将在tcp和udp之间尝试切换播放,如果设置为0,则不尝试切换. public int NT_U3D_SetRTSPAutoSwitchTcpUdp(long handle, int is_auto_switch_tcp_udp) { return player_obj_.Call<int>("SetRTSPAutoSwitchTcpUdp", handle, is_auto_switch_tcp_udp); } /// <summary> /// 设置快速启动该模式, /// </summary> /// <param name="is_fast_startup"></param> public int NT_U3D_SetFastStartup(long handle, int is_fast_startup) { return player_obj_.Call<int>("SetFastStartup", handle, is_fast_startup); } /// <summary> /// 设置超低延迟模式 false不开启 true开启 默认false /// </summary> /// <param name="mode"></param> public int NT_U3D_SetPlayerLowLatencyMode(long handle, int mode) { return player_obj_.Call<int>("SetPlayerLowLatencyMode", handle, mode); } /// <summary> /// 设置视频垂直反转 /// is_flip: 0: 不反转, 1: 反转 /// </summary> /// <param name="is_flip"></param> public int NT_U3D_SetFlipVertical(long handle, int is_flip) { return player_obj_.Call<int>("SetFlipVertical", handle, is_flip); } /// <summary> /// 设置视频水平反转 /// is_flip: 0: 不反转, 1: 反转 /// </summary> /// <param name="is_flip"></param> public int NT_U3D_SetFlipHorizontal(long handle, int is_flip) { return player_obj_.Call<int>("SetFlipHorizontal", handle, is_flip); } /// <summary> /// 设置顺时针旋转, 注意除了0度之外, 其他角度都会额外消耗性能 /// degress: 当前支持 0度,90度, 180度, 270度 旋转 /// </summary> /// <param name="degress"></param> public int NT_U3D_SetRotation(long handle, int degress) { return player_obj_.Call<int>("SetRotation", handle, degress); } /// <summary> /// 设置是否回调下载速度 /// is_report: if 1: 上报下载速度, 0: 不上报. /// report_interval: 上报间隔,以秒为单位,>0. /// </summary> /// <param name="is_report"></param> /// <param name="report_interval"></param> public int NT_U3D_SetReportDownloadSpeed(long handle, int is_report, int report_interval) { return player_obj_.Call<int>("SetReportDownloadSpeed", handle, is_report, report_interval); } /// <summary> /// 设置是否需要在播放或录像过程中快照 /// </summary> /// <param name="is_save_image"></param> public int NT_U3D_SetSaveImageFlag(long handle, int is_save_image) { return player_obj_.Call<int>("SetSaveImageFlag", handle, is_save_image); } /// <summary> /// 播放或录像过程中快照 /// </summary> /// <param name="imageName"></param> public int NT_U3D_SaveCurImage(long handle, string imageName) { return player_obj_.Call<int>("SaveCurImage", handle, imageName); } /// <summary> /// 播放或录像过程中,快速切换url /// </summary> /// <param name="uri"></param> public int NT_U3D_SwitchPlaybackUrl(long handle, string uri) { return player_obj_.Call<int>("SwitchPlaybackUrl", handle, uri); } /// <summary> /// 创建录像存储路径 /// </summary> /// <param name="path"></param> public int NT_U3D_CreateFileDirectory(string path) { return player_obj_.Call<int>("CreateFileDirectory", path); } /// <summary> /// 设置录像存储路径 /// </summary> /// <param name="path"></param> public int NT_U3D_SetRecorderDirectory(long handle, string path) { return player_obj_.Call<int>("SetRecorderDirectory", handle, path); } /// <summary> /// 设置单个录像文件大小 /// </summary> /// <param name="size"></param> public int NT_U3D_SetRecorderFileMaxSize(long handle, int size) { return player_obj_.Call<int>("SetRecorderFileMaxSize", handle, size); } /// <summary> /// 设置录像时音频转AAC编码的开关 /// aac比较通用,sdk增加其他音频编码(比如speex, pcmu, pcma等)转aac的功能. /// 注意: 转码会增加性能消耗 /// </summary> /// <param name="is_transcode"></param> /// is_transcode:设置为1的话,如果音频编码不是aac,则转成aac,如果是aac,则不做转换. 设置为0的话,则不做任何转换. 默认是0. public int NT_U3D_SetRecorderAudioTranscodeAAC(long handle, int is_transcode) { return player_obj_.Call<int>("SetRecorderAudioTranscodeAAC", handle, is_transcode); } /// <summary> /// 设置播放路径 /// </summary> public int NT_U3D_SetUrl(long handle, string url) { return player_obj_.Call<int>("SetUrl", handle, url); } /// <summary> /// 开始播放 /// </summary> public int NT_U3D_StartPlay(long handle) { return player_obj_.Call<int>("StartPlay", handle); } /// <summary> /// 获取YUV数据 /// </summary> public AndroidJavaObject NT_U3D_GetVideoFrame(long handle) { return player_obj_.Call<AndroidJavaObject>("GetVideoFrame", handle); } /// <summary> /// 停止播放 /// </summary> public int NT_U3D_StopPlay(long handle) { return player_obj_.Call<int>("StopPlay", handle); } /// <summary> /// 开始录像 /// </summary> public int NT_U3D_StartRecorder(long handle) { return player_obj_.Call<int>("StartRecorder", handle); } /// <summary> /// 停止录像 /// </summary> public int NT_U3D_StopRecorder(long handle) { return player_obj_.Call<int>("StopRecorder", handle); } /// <summary> /// 关闭播放 /// </summary> public int NT_U3D_Close(long handle) { return player_obj_.Call<int>("Close", handle); } /// <summary> /// UnInit Player /// </summary> public int NT_U3D_UnInit() { return DANIULIVE_RETURN_OK; }
总结
Unity下实现RTMP或RTSP无论是播放还是录像,甚至快照,说难不难,但是做好真的比较难,特别是移动端,Unity和原生层交互的时候,数据交互效率相对较低,需要尽可能减少拷贝。录像的话,还需要考虑硬件性能瓶颈。此外,还需要逻辑分离,确保播放和录像相互不影响,以上是抛砖引玉,感兴趣的开发者,可以自行参考实现,如果需要单独和我交流的,可以相互交流。