技术背景
VR（虚拟现实技术）给我们带来身临其境的视觉体验，广泛的应用于城市规划、教育培训、工业仿真、房地产、水利电力、室内设计、文旅、军事等众多领域，常用的行业比如：
- 教育行业：VR头显可以用于教育培训，提供沉浸式的教学体验，例如虚拟实验室、虚拟课堂等，帮助学生更好地理解和掌握知识。
- 医疗行业：VR头显可以用于医疗训练和治疗，例如手术模拟、康复训练等，提高医疗效果和质量。
- 文旅行业：VR头显可以用于旅游娱乐，提供沉浸式的旅游体验，例如虚拟旅游、文化遗产展示等。
- 房地产行业：VR头显可以用于房地产展示，提供更加真实、直观的房屋展示和体验，帮助客户更好地了解和选择房屋。
- 展览展示行业：VR头显可以用于展览展示，提供沉浸式的展览体验，例如虚拟展厅、虚拟展品等，吸引观众的注意和参与。
- 军事行业：VR头显可以用于军事训练和作战指挥，提供更加真实、逼真的军事训练环境。
技术实现
如何在VR头显实现RTMP或RTSP播放？
VR头显播放RTMP或RTSP流数据，简单来说，通过jni层打通RTMP或RTSP流传输，解包并解码回调给Unity YUV或RGB数据，Unity场景下，绘制即可，本文以大牛直播SDK的Unity平台RTMP、RTSP播放为例，介绍下具体技术实现：
开始播放：
publicvoidPlay() { if (is_running) { Debug.Log("已经在播放。。"); return; } //获取输入框的urlstringurl=input_url_.text.Trim(); OpenPlayer(); if ( player_handle_==0 ) return; NT_U3D_Set_Game_Object(player_handle_, game_object_); /* ++ 播放前参数配置可加在此处 ++ */intis_using_tcp=0; //TCP/UDP模式设置NT_U3D_SetRTSPTcpMode(player_handle_, is_using_tcp); intis_report=0; intreport_interval=1; NT_U3D_SetReportDownloadSpeed(player_handle_, is_report, report_interval); //下载速度回调NT_U3D_SetBuffer(player_handle_, play_buffer_time_); //设置buffer timeNT_U3D_SetPlayerLowLatencyMode(player_handle_, is_low_latency_?1 : 0); //设置是否启用低延迟模式NT_U3D_SetMute(player_handle_, is_mute_?1 : 0); //是否启动播放的时候静音NT_U3D_SetAudioVolume(player_handle_, cur_audio_volume_); //设置播放音量NT_U3D_SetVideoDecoderMode(player_handle_, is_hw_decode_?1 : 0); //设置H.264软硬解模式NT_U3D_SetVideoHevcDecoderMode(player_handle_, is_hw_decode_?1 : 0); //设置H.265软硬解模式intis_fast_startup=1; NT_U3D_SetFastStartup(player_handle_, is_fast_startup); //设置快速启动模式intrtsp_timeout=10; NT_U3D_SetRTSPTimeout(player_handle_, rtsp_timeout); //设置RTSP超时时间intis_auto_switch_tcp_udp=1; NT_U3D_SetRTSPAutoSwitchTcpUdp(player_handle_, is_auto_switch_tcp_udp); //设置TCP/UDP模式自动切换intis_audiotrack=1; NT_U3D_SetAudioOutputType(player_handle_, is_audiotrack); //设置音频输出模式: if 0: 自动选择; if with 1: audiotrack模式NT_U3D_SetUrl(player_handle_, videoUrl); /* -- 播放前参数配置可加在此处 -- */intflag=NT_U3D_StartPlay(player_handle_); if (flag==DANIULIVE_RETURN_OK) { is_need_get_frame_=true; Debug.Log("播放成功"); } else { is_need_get_frame_=false; Debug.LogError("播放失败"); } is_running=true; }
Close Player：
privatevoidClosePlayer() { is_need_get_frame_=false; is_need_init_texture_=false; intflag=NT_U3D_StopPlay(player_handle_); if (flag==DANIULIVE_RETURN_OK) { Debug.Log("停止成功"); } else { Debug.LogError("停止失败"); } flag=NT_U3D_Close(player_handle_); if (flag==DANIULIVE_RETURN_OK) { Debug.Log("关闭成功"); } else { Debug.LogError("关闭失败"); } player_handle_=0; NT_U3D_UnInit(); is_running=false; video_width_=0; video_height_=0; }
Event事件回调处理：
/// <summary>/// android 传递过来 code/// </summary>/// <param name="event_message"></param>publicvoidonNTSmartEvent(stringevent_message) { if (null==event_message||event_message.Length<1) return; string[] strs=event_message.Split(','); if (null==strs||strs.Length<6) return; stringplayer_handle=strs[0]; stringcode=strs[1]; stringparam1=strs[2]; stringparam2=strs[3]; stringparam3=strs[4]; stringparam4=strs[5]; Debug.Log("[daiusdk] code: 0x"+Convert.ToString(Convert.ToInt32(code), 16)); Stringplayer_event=""; switch (Convert.ToInt32(code)) { caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_STARTED: player_event="开始.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING: player_event="连接中.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED: player_event="连接失败.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED: player_event="连接成功.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED: player_event="连接断开.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP: player_event="停止播放.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO: player_event="分辨率信息: width: "+Convert.ToInt32(param1) +", height: "+Convert.ToInt32(param2); break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED: player_event="收不到媒体数据，可能是url错误.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL: player_event="切换播放URL.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE: player_event="快照: "+param1+" 路径："+param3; if (Convert.ToInt32(param1) ==0) { player_event="截取快照成功.."; } else { player_event="截取快照失败.."; } break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE: player_event="[record]开始一个新的录像文件 : "+param3; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED: player_event="[record]已生成一个录像文件 : "+param3; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING: player_event="Start_Buffering.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING: player_event="Buffering: "+Convert.ToInt32(param1); break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING: player_event="Stop_Buffering.."; break; caseEVENTID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED: player_event="download_speed:"+param1+"Byte/s"+", "+ (Convert.ToInt32(param1) *8/1000) +"kbps"+", "+ (Convert.ToInt32(param1) /1024) +"KB/s"; break; } Debug.Log(player_event); player_event=null; strs=null; }
如何封装实现原生jni层交互：
/// SmartPlayerAndroidMono.cs/// Author: daniusdk.com///Created on 2018/05/10/// <summary>/// Init/// </summary>publicintNT_U3D_Init() { returnplayer_obj_.Call<int>("Init", java_obj_cur_activity_); } /// <summary>/// 开始/// 返回播放句柄/// </summary>publiclongNT_U3D_Open() { returnplayer_obj_.Call<long>("Open"); } /// <summary>/// Register Game Object,用于消息传递/// </summary>publicintNT_U3D_Set_Game_Object(longhandle, stringgameObjectName) { returnplayer_obj_.Call<int>("SetGameObject", handle, gameObjectName); } /// <summary>/// 设置H.264解码方式 false 软件解码 true 硬件解码 默认为false/// </summary>/// <param name="isHwDecoder"></param>publicintNT_U3D_SetVideoDecoderMode(longhandle, intisHwDecoder) { returnplayer_obj_.Call<int>("SetPlayerVideoHWDecoder", handle, isHwDecoder); } /// <summary>/// 设置H.265 解码方式 false 软件解码 true 硬件解码 默认为false/// </summary>/// <param name="isHevcHwDecoder"></param>publicintNT_U3D_SetVideoHevcDecoderMode(longhandle, intisHevcHwDecoder) { returnplayer_obj_.Call<int>("SetPlayerVideoHevcHWDecoder", handle, isHevcHwDecoder); } /// <summary>/// 设置音频输出模式: if 0: 自动选择; if with 1: audiotrack模式/// </summary>/// <param name="use_audiotrack"></param>publicintNT_U3D_SetAudioOutputType(longhandle, intuse_audiotrack) { returnplayer_obj_.Call<int>("SetAudioOutputType", handle, use_audiotrack); } /// <summary>/// 设置播放端缓存大小, 默认200毫秒/// </summary>/// <param name="buffer"></param>publicintNT_U3D_SetBuffer(longhandle, intbuffer) { returnplayer_obj_.Call<int>("SetBuffer", handle, buffer); } /// <summary>/// 接口可实时调用：设置是否实时静音，1：静音; 0: 取消静音/// </summary>/// <param name="is_mute"></param>publicintNT_U3D_SetMute(longhandle, intis_mute) { returnplayer_obj_.Call<int>("SetMute", handle, is_mute); } /// <summary>/// 接口可实时调用：设置播放音量，范围是[0, 100], 0是静音，100是最大音量, 默认是100/// </summary>/// <param name="audio_volume"></param>publicintNT_U3D_SetAudioVolume(longhandle, intaudio_volume) { returnplayer_obj_.Call<int>("SetAudioVolume", handle, audio_volume); } /// <summary>/// 设置RTSP TCP模式, 1: TCP; 0: UDP/// </summary>/// <param name="is_using_tcp"></param>publicintNT_U3D_SetRTSPTcpMode(longhandle, intis_using_tcp) { returnplayer_obj_.Call<int>("SetRTSPTcpMode", handle, is_using_tcp); } /// <summary>/// 设置RTSP超时时间, timeout单位为秒，必须大于0/// </summary>/// <param name="timeout"></param>publicintNT_U3D_SetRTSPTimeout(longhandle, inttimeout) { returnplayer_obj_.Call<int>("SetRTSPTimeout", handle, timeout); } /// <summary>/// 设置RTSP TCP/UDP自动切换/// NOTE: 对于RTSP来说，有些可能支持rtp over udp方式，有些可能支持使用rtp over tcp方式./// 为了方便使用，有些场景下可以开启自动尝试切换开关, 打开后如果udp无法播放，sdk会自动尝试tcp, 如果tcp方式播放不了,sdk会自动尝试udp./// </summary>/// <param name="timeout"></param>/// timeout：如果设置1的话, sdk将在tcp和udp之间尝试切换播放，如果设置为0，则不尝试切换.publicintNT_U3D_SetRTSPAutoSwitchTcpUdp(longhandle, intis_auto_switch_tcp_udp) { returnplayer_obj_.Call<int>("SetRTSPAutoSwitchTcpUdp", handle, is_auto_switch_tcp_udp); } /// <summary>/// 设置快速启动该模式，/// </summary>/// <param name="is_fast_startup"></param>publicintNT_U3D_SetFastStartup(longhandle, intis_fast_startup) { returnplayer_obj_.Call<int>("SetFastStartup", handle, is_fast_startup); } /// <summary>/// 设置超低延迟模式 false不开启 true开启 默认false/// </summary>/// <param name="mode"></param>publicintNT_U3D_SetPlayerLowLatencyMode(longhandle, intmode) { returnplayer_obj_.Call<int>("SetPlayerLowLatencyMode", handle, mode); } /// <summary>/// 设置视频垂直反转/// is_flip： 0: 不反转, 1: 反转/// </summary>/// <param name="is_flip"></param>publicintNT_U3D_SetFlipVertical(longhandle, intis_flip) { returnplayer_obj_.Call<int>("SetFlipVertical", handle, is_flip); } /// <summary>/// 设置视频水平反转/// is_flip： 0: 不反转, 1: 反转/// </summary>/// <param name="is_flip"></param>publicintNT_U3D_SetFlipHorizontal(longhandle, intis_flip) { returnplayer_obj_.Call<int>("SetFlipHorizontal", handle, is_flip); } /// <summary>/// 设置顺时针旋转, 注意除了0度之外， 其他角度都会额外消耗性能/// degress： 当前支持 0度，90度, 180度, 270度 旋转/// </summary>/// <param name="degress"></param>publicintNT_U3D_SetRotation(longhandle, intdegress) { returnplayer_obj_.Call<int>("SetRotation", handle, degress); } /// <summary>/// 设置是否回调下载速度/// is_report: if 1: 上报下载速度, 0: 不上报./// report_interval: 上报间隔，以秒为单位，>0./// </summary>/// <param name="is_report"></param>/// <param name="report_interval"></param>publicintNT_U3D_SetReportDownloadSpeed(longhandle, intis_report, intreport_interval) { returnplayer_obj_.Call<int>("SetReportDownloadSpeed", handle, is_report, report_interval); } /// <summary>/// 设置是否需要在播放或录像过程中快照/// </summary>/// <param name="is_save_image"></param>publicintNT_U3D_SetSaveImageFlag(longhandle, intis_save_image) { returnplayer_obj_.Call<int>("SetSaveImageFlag", handle, is_save_image); } /// <summary>/// 播放或录像过程中快照/// </summary>/// <param name="imageName"></param>publicintNT_U3D_SaveCurImage(longhandle, stringimageName) { returnplayer_obj_.Call<int>("SaveCurImage", handle, imageName); } /// <summary>/// 播放或录像过程中，快速切换url/// </summary>/// <param name="uri"></param>publicintNT_U3D_SwitchPlaybackUrl(longhandle, stringuri) { returnplayer_obj_.Call<int>("SwitchPlaybackUrl", handle, uri); } /// <summary>/// 创建录像存储路径/// </summary>/// <param name="path"></param>publicintNT_U3D_CreateFileDirectory(stringpath) { returnplayer_obj_.Call<int>("CreateFileDirectory", path); } /// <summary>/// 设置录像存储路径/// </summary>/// <param name="path"></param>publicintNT_U3D_SetRecorderDirectory(longhandle, stringpath) { returnplayer_obj_.Call<int>("SetRecorderDirectory", handle, path); } /// <summary>/// 设置单个录像文件大小/// </summary>/// <param name="size"></param>publicintNT_U3D_SetRecorderFileMaxSize(longhandle, intsize) { returnplayer_obj_.Call<int>("SetRecorderFileMaxSize", handle, size); } /// <summary>/// 设置录像时音频转AAC编码的开关/// aac比较通用，sdk增加其他音频编码(比如speex, pcmu, pcma等)转aac的功能./// 注意: 转码会增加性能消耗/// </summary>/// <param name="is_transcode"></param>/// is_transcode：设置为1的话，如果音频编码不是aac，则转成aac，如果是aac，则不做转换. 设置为0的话，则不做任何转换. 默认是0.publicintNT_U3D_SetRecorderAudioTranscodeAAC(longhandle, intis_transcode) { returnplayer_obj_.Call<int>("SetRecorderAudioTranscodeAAC", handle, is_transcode); } /// <summary>/// 设置播放路径/// </summary>publicintNT_U3D_SetUrl(longhandle, stringurl) { returnplayer_obj_.Call<int>("SetUrl", handle, url); } /// <summary>/// 开始播放/// </summary>publicintNT_U3D_StartPlay(longhandle) { returnplayer_obj_.Call<int>("StartPlay", handle); } /// <summary>/// 获取YUV数据/// </summary>publicAndroidJavaObjectNT_U3D_GetVideoFrame(longhandle) { returnplayer_obj_.Call<AndroidJavaObject>("GetVideoFrame", handle); } /// <summary>/// 停止播放/// </summary>publicintNT_U3D_StopPlay(longhandle) { returnplayer_obj_.Call<int>("StopPlay", handle); } /// <summary>/// 开始录像/// </summary>publicintNT_U3D_StartRecorder(longhandle) { returnplayer_obj_.Call<int>("StartRecorder", handle); } /// <summary>/// 停止录像/// </summary>publicintNT_U3D_StopRecorder(longhandle) { returnplayer_obj_.Call<int>("StopRecorder", handle); } /// <summary>/// 关闭播放/// </summary>publicintNT_U3D_Close(longhandle) { returnplayer_obj_.Call<int>("Close", handle); } /// <summary>/// UnInit Player/// </summary>publicintNT_U3D_UnInit() { returnDANIULIVE_RETURN_OK; }
技术总结
通过实际测试来看，VR头显端，如果设备性能尚可的话，播放RTMP或RTSP，可实现毫秒级的延迟，可满足大多数有交互诉求的技术场景，此外，如果头显端支持硬解码的话，可以优先考虑硬解码。
