背景
随着智能门禁等物联网产品的普及，越来越多的开发者对音视频互动体验提出了更高的要求。目前市面上大多一对一互动都是基于WebRTC，优点不再赘述，我们这里先说说可能需要面临的问题：WebRTC的服务器部署非常复杂，可以私有部署，但是非常复杂。传输基于UDP，很难保证传输质量，由于UDP是不可靠的传输协议，在复杂的公网网络环境下，各种突发流量、偶尔的传输错误、网络抖动、超时等等都会引起丢包异常，都会在一定程度上影响音视频通信的质量，难以应对复杂的互联网环境，如跨区跨运营商、低带宽、高丢包等场景，行话说的好：从demo到实用,中间还差1万个WebRTC。
其他技术方案
- 内网环境下的RTSP轻量级服务；
- 基于RTMP的公网或内网技术方案。
本方案系基于现有RTMP或内置RTSP服务、RTMP/RTSP直播播放模块，产品稳定度高，在保证超低延迟的基础上，加入噪音抑制、回音消除、自动增益控制等特性，确保通话体验（如需更好的消除效果，亦可考虑如麦克风阵列等技术方案），采用通用的RTMP服务器（如nginx、SRS）或自身的轻量级RTSP服务，更有利于私有部署，便于支持H.264的扩展SEI消息发送机制，方便扩展特定机型H.265编码支持。
技术实现
废话不多说，先上图：
关键demo代码说明：
拉流播放：
btnPlaybackStartStopPlayback.setOnClickListener(new Button.OnClickListener() { // @Override public void onClick(View v) { if(isPlaybackViewStarted) { Log.i(PLAY_TAG, "Stop playback stream++"); btnPlaybackStartStopPlayback.setText("开始播放 "); //btnPopInputText.setEnabled(true); btnPlaybackPopInputUrl.setEnabled(true); btnPlaybackHardwareDecoder.setEnabled(true); btnPlaybackSetPlayBuffer.setEnabled(true); btnPlaybackFastStartup.setEnabled(true); if ( playerHandle != 0 ) { libPlayer.SmartPlayerStopPlay(playerHandle); libPlayer.SmartPlayerClose(playerHandle); playerHandle = 0; } isPlaybackViewStarted = false; Log.i(PLAY_TAG, "Stop playback stream--"); } else { Log.i(PLAY_TAG, "Start playback stream++"); playerHandle = libPlayer.SmartPlayerOpen(curContext); if(playerHandle == 0) { Log.e(PLAY_TAG, "surfaceHandle with nil.."); return; } libPlayer.SetSmartPlayerEventCallbackV2(playerHandle, new EventHandePlayerV2()); libPlayer.SmartPlayerSetSurface(playerHandle, playerSurfaceView); //if set the second param with null, it means it will playback audio only.. // libPlayer.SmartPlayerSetSurface(playerHandle, null); libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1); // External Render test //libPlayer.SmartPlayerSetExternalRender(playerHandle, new RGBAExternalRender()); //libPlayer.SmartPlayerSetExternalRender(playerHandle, new I420ExternalRender()); libPlayer.SmartPlayerSetExternalAudioOutput(playerHandle, new PlayerExternalPcmOutput()); libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1); libPlayer.SmartPlayerSetBuffer(playerHandle, playbackBuffer); libPlayer.SmartPlayerSetFastStartup(playerHandle, isPlaybackFastStartup?1:0); if ( isPlaybackMute ) { libPlayer.SmartPlayerSetMute(playerHandle, isPlaybackMute?1:0); } if (isPlaybackHardwareDecoder) { int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle,1); int isSupportH264HwDecoder = libPlayer .SetSmartPlayerVideoHWDecoder(playerHandle,1); Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder); } if( playbackUrl == null ) { Log.e(PLAY_TAG, "playback URL with NULL..."); return; } libPlayer.SmartPlayerSetAudioVolume(playerHandle, curAudioVolume); libPlayer.SmartPlayerSetUrl(playerHandle, playbackUrl); int iPlaybackRet = libPlayer.SmartPlayerStartPlay(playerHandle); if( iPlaybackRet != 0 ) { libPlayer.SmartPlayerClose(playerHandle); playerHandle = 0; Log.e(PLAY_TAG, "StartPlayback strem failed.."); return; } btnPlaybackStartStopPlayback.setText("停止播放 "); btnPlaybackPopInputUrl.setEnabled(false); btnPlaybackHardwareDecoder.setEnabled(false); btnPlaybackSetPlayBuffer.setEnabled(false); btnPlaybackFastStartup.setEnabled(false); isPlaybackViewStarted = true; Log.i(PLAY_TAG, "Start playback stream--"); } } });
拉流端实时音量调节：
audioVolumeBar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { @Override public void onProgressChanged(SeekBar seekBar, int i, boolean b) { } @Override public void onStartTrackingTouch(SeekBar seekBar) { //Log.i(TAG, "开始拖动"); } @Override public void onStopTrackingTouch(SeekBar seekBar) { Log.i(TAG, "停止拖动, CurProgress: " + seekBar.getProgress()); curAudioVolume = seekBar.getProgress(); audioVolumeText.setText("当前音量: " + curAudioVolume); if(playerHandle != 0) { libPlayer.SmartPlayerSetAudioVolume(playerHandle, curAudioVolume); } } }); }
回调后的PCM数据，传给推送端，用于音频处理
class NTAudioRecordV2CallbackImpl implements NTAudioRecordV2Callback { @Override public void onNTAudioRecordV2Frame(ByteBuffer data, int size, int sampleRate, int channel, int per_channel_sample_number) { /* Log.i(TAG, "onNTAudioRecordV2Frame size=" + size + " sampleRate=" + sampleRate + " channel=" + channel + " per_channel_sample_number=" + per_channel_sample_number); */ if ( (isPushingRtmp || isRTSPPublisherRunning) && publisherHandle != 0 ) { libPublisher.SmartPublisherOnPCMData(publisherHandle, data, size, sampleRate, channel, per_channel_sample_number); } } }
class PlayerExternalPcmOutput implements NTExternalAudioOutput { private int sample_rate_ = 0; private int channel_ = 0; private int sample_size = 0; private int buffer_size = 0; private ByteBuffer pcm_buffer_ = null; @Override public ByteBuffer getPcmByteBuffer(int size) { //Log.i("getPcmByteBuffer", "size: " + size); if(size < 1) { return null; } if(buffer_size != size) { buffer_size = size; pcm_buffer_ = ByteBuffer.allocateDirect(buffer_size); } return pcm_buffer_; } public void onGetPcmFrame(int ret, int sampleRate, int channel, int sampleSize, int is_low_latency) { /*Log.i("onGetPcmFrame", "ret: " + ret + ", sampleRate: " + sampleRate + ", channel: " + channel + ", sampleSize: " + sampleSize + ",is_low_latency:" + is_low_latency + " buffer_size：" + buffer_size);*/ if ( pcm_buffer_ == null) return; pcm_buffer_.rewind(); if ( ret == 0 && (isPushingRtmp || isRTSPPublisherRunning)) { libPublisher.SmartPublisherOnFarEndPCMData(publisherHandle, pcm_buffer_, sampleRate, channel, sampleSize, is_low_latency); if (is_audio_mix_) { libPublisher.SmartPublisherOnMixPCMData(publisherHandle, 1, pcm_buffer_, 0, buffer_size, sampleRate, channel, sampleSize); } /* java.nio.ByteOrder old_order = pcm_buffer_.order(); pcm_buffer_.order(java.nio.ByteOrder.nativeOrder()); java.nio.ShortBuffer short_buffer = pcm_buffer_.asShortBuffer(); pcm_buffer_.order(old_order); short[] short_array = new short[short_buffer.remaining()]; short_buffer.get(short_array); libPublisher.SmartPublisherOnMixPCMShortArray(publisherHandle, 1, short_array, 0, short_array.length, sampleRate, channel, sampleSize); */ } // test /* byte[] test_buffer = new byte[16]; pcm_buffer_.get(test_buffer); Log.i(TAG, "onGetPcmFrame data:" + bytesToHexString(test_buffer)); */ } }
推送端：
RTMP推送：
class ButtonPushStartListener implements OnClickListener { public void onClick(View v) { if (isPushingRtmp) { stopPush(); if (!isRTSPPublisherRunning) { ConfigControlEnable(true); } btnPushStartStop.setText("推送RTMP"); isPushingRtmp = false; return; } Log.i(PUSH_TAG, "onClick start push rtmp.."); if (libPublisher == null) return; if (!isRTSPPublisherRunning) { InitPusherAndSetConfig(); } if ( inputPushURL != null && inputPushURL.length() > 1 ) { publishURL = inputPushURL; Log.i(PUSH_TAG, "start, input publish url:" + publishURL); } else { publishURL = basePushURL + String.valueOf((int)( System.currentTimeMillis() % 1000000)); Log.i(PUSH_TAG, "start, generate random url:" + publishURL); } printPushText = "URL:" + publishURL; Log.i(PUSH_TAG, printPushText); textPushCurURL = (TextView)findViewById(R.id.txt_push_cur_url); textPushCurURL.setText(printPushText); Log.i(PUSH_TAG, "videoWidth: "+ pushVideoWidth + " videoHeight: " + pushVideoHeight + " pushType:" + pushType); if ( libPublisher.SmartPublisherSetURL(publisherHandle, publishURL) != 0 ) { Log.e(PUSH_TAG, "Failed to set rtmp pusher URL.."); } int startRet = libPublisher.SmartPublisherStartPublisher(publisherHandle); if (startRet != 0) { isPushingRtmp = false; Log.e(TAG, "Failed to start push stream.."); return; } if ( !isRTSPPublisherRunning ) { if (pushType == 0 || pushType == 1) { CheckInitAudioRecorder(); //enable pure video publisher.. } ConfigControlEnable(false); } btnPushStartStop.setText("停止推送 "); isPushingRtmp = true; } };
轻量级RTSP服务模式：
//启动/停止RTSP服务 class ButtonRtspServiceListener implements OnClickListener { public void onClick(View v) { if (isRTSPServiceRunning) { stopRtspService(); btnRtspService.setText("启动RTSP服务"); btnRtspPublisher.setEnabled(false); isRTSPServiceRunning = false; return; } Log.i(TAG, "onClick start rtsp service.."); rtsp_handle_ = libPublisher.OpenRtspServer(0); if (rtsp_handle_ == 0) { Log.e(TAG, "创建rtsp server实例失败! 请检查SDK有效性"); } else { int port = 8554; if (libPublisher.SetRtspServerPort(rtsp_handle_, port) != 0) { libPublisher.CloseRtspServer(rtsp_handle_); rtsp_handle_ = 0; Log.e(TAG, "创建rtsp server端口失败! 请检查端口是否重复或者端口不在范围内!"); } //String user_name = "admin"; //String password = "12345"; //libPublisher.SetRtspServerUserNamePassword(rtsp_handle_, user_name, password); if (libPublisher.StartRtspServer(rtsp_handle_, 0) == 0) { Log.i(TAG, "启动rtsp server 成功!"); } else { libPublisher.CloseRtspServer(rtsp_handle_); rtsp_handle_ = 0; Log.e(TAG, "启动rtsp server失败! 请检查设置的端口是否被占用!"); } btnRtspService.setText("停止RTSP服务"); btnRtspPublisher.setEnabled(true); isRTSPServiceRunning = true; } } } //发布/停止RTSP流 class ButtonRtspPublisherListener implements OnClickListener { public void onClick(View v) { if (isRTSPPublisherRunning) { stopRtspPublisher(); if (!isPushingRtmp) { ConfigControlEnable(true); } btnRtspPublisher.setText("发布RTSP流"); btnGetRtspSessionNumbers.setEnabled(false); btnRtspService.setEnabled(true); isRTSPPublisherRunning = false; return; } Log.i(TAG, "onClick start rtsp publisher.."); if (!isPushingRtmp) { InitPusherAndSetConfig(); } if (publisherHandle == 0) { Log.e(TAG, "Start rtsp publisher, publisherHandle is null.."); return; } String rtsp_stream_name = "stream1"; libPublisher.SetRtspStreamName(publisherHandle, rtsp_stream_name); libPublisher.ClearRtspStreamServer(publisherHandle); libPublisher.AddRtspStreamServer(publisherHandle, rtsp_handle_, 0); if (libPublisher.StartRtspStream(publisherHandle, 0) != 0) { Log.e(TAG, "调用发布rtsp流接口失败!"); return; } if (!isPushingRtmp) { if (pushType == 0 || pushType == 1) { CheckInitAudioRecorder(); //enable pure video publisher.. } ConfigControlEnable(false); } btnRtspPublisher.setText("停止RTSP流"); btnGetRtspSessionNumbers.setEnabled(true); btnRtspService.setEnabled(false); isRTSPPublisherRunning = true; } } ;
RTMP推送和轻量级RTSP服务，可以在一个实例里面处理，所以推送参数的初始化，只需要调用一次即可。
private void InitPusherAndSetConfig() { Log.i(TAG, "videoWidth: " + pushVideoWidth + " videoHeight: " + pushVideoHeight + " pushType:" + pushType); int audio_opt = 1; int video_opt = 1; if ( pushType == 1 ) { video_opt = 0; } else if (pushType == 2 ) { audio_opt = 0; } publisherHandle = libPublisher.SmartPublisherOpen(curContext, audio_opt, video_opt, pushVideoWidth, pushVideoHeight); if ( publisherHandle == 0 ) { return; } if(videoEncodeType == 1) { int h264HWKbps = setHardwareEncoderKbps(true, pushVideoWidth, pushVideoHeight); Log.i(TAG, "h264HWKbps: " + h264HWKbps); int isSupportH264HWEncoder = libPublisher .SetSmartPublisherVideoHWEncoder(publisherHandle, h264HWKbps); if (isSupportH264HWEncoder == 0) { Log.i(TAG, "Great, it supports h.264 hardware encoder!"); } } else if (videoEncodeType == 2) { int hevcHWKbps = setHardwareEncoderKbps(false, pushVideoWidth, pushVideoHeight); Log.i(TAG, "hevcHWKbps: " + hevcHWKbps); int isSupportHevcHWEncoder = libPublisher .SetSmartPublisherVideoHevcHWEncoder(publisherHandle, hevcHWKbps); if (isSupportHevcHWEncoder == 0) { Log.i(TAG, "Great, it supports hevc hardware encoder!"); } } if(is_sw_vbr_mode) { int is_enable_vbr = 1; int video_quality = CalVideoQuality(pushVideoWidth, pushVideoHeight, true); int vbr_max_bitrate = CalVbrMaxKBitRate(pushVideoWidth, pushVideoHeight); libPublisher.SmartPublisherSetSwVBRMode(publisherHandle, is_enable_vbr, video_quality, vbr_max_bitrate); } libPublisher.SetSmartPublisherEventCallbackV2(publisherHandle, new EventHandePublisherV2()); //如果想和时间显示在同一行，请去掉'\n' String watermarkText = "大牛直播(daniulive)\n\n"; String path = pushLogoPath; if( pushWatemarkType == 0 ) { if ( isPushWritelogoFileSuccess ) libPublisher.SmartPublisherSetPictureWatermark(publisherHandle, path, WATERMARK.WATERMARK_POSITION_TOPRIGHT, 160, 160, 10, 10); } else if( pushWatemarkType == 1 ) { if ( isPushWritelogoFileSuccess ) libPublisher.SmartPublisherSetPictureWatermark(publisherHandle, path, WATERMARK.WATERMARK_POSITION_TOPRIGHT, 160, 160, 10, 10); libPublisher.SmartPublisherSetTextWatermark(publisherHandle, watermarkText, 1, WATERMARK.WATERMARK_FONTSIZE_BIG, WATERMARK.WATERMARK_POSITION_BOTTOMRIGHT, 10, 10); //libPublisher.SmartPublisherSetTextWatermarkFontFileName("/system/fonts/DroidSansFallback.ttf"); //libPublisher.SmartPublisherSetTextWatermarkFontFileName("/sdcard/DroidSansFallback.ttf"); } else if(pushWatemarkType == 2) { libPublisher.SmartPublisherSetTextWatermark(publisherHandle, watermarkText, 1, WATERMARK.WATERMARK_FONTSIZE_BIG, WATERMARK.WATERMARK_POSITION_BOTTOMRIGHT, 10, 10); //libPublisher.SmartPublisherSetTextWatermarkFontFileName("/system/fonts/DroidSansFallback.ttf"); } else { Log.i(TAG, "no watermark settings.."); } //end if ( !is_push_speex ) { // set AAC encoder libPublisher.SmartPublisherSetAudioCodecType(publisherHandle, 1); } else { // set Speex encoder libPublisher.SmartPublisherSetAudioCodecType(publisherHandle, 2); libPublisher.SmartPublisherSetSpeexEncoderQuality(publisherHandle, 8); } libPublisher.SmartPublisherSetNoiseSuppression(publisherHandle, is_push_noise_suppression?1:0); libPublisher.SmartPublisherSetAGC(publisherHandle, is_push_agc?1:0); libPublisher.SmartPublisherSetEchoCancellation(publisherHandle, 1, echoCancelDelay); libPublisher.SmartPublisherSetAudioMix(publisherHandle, is_audio_mix_?1:0); libPublisher.SmartPublisherSetInputAudioVolume(publisherHandle, 0 , mic_audio_volume_); if ( is_audio_mix_ ) { libPublisher.SmartPublisherSetInputAudioVolume(publisherHandle, 1 , mix_audio_volume_); } libPublisher.SmartPublisherSetClippingMode(publisherHandle, 0); libPublisher.SmartPublisherSetSWVideoEncoderProfile(publisherHandle, push_sw_video_encoder_profile); //libPublisher.SetRtmpPublishingType(0); //libPublisher.SmartPublisherSetGopInterval(publisherHandle, 18*3); //libPublisher.SmartPublisherSetFPS(publisherHandle, 18); libPublisher.SmartPublisherSetSWVideoEncoderSpeed(publisherHandle, sw_video_encoder_speed); //libPublisher.SmartPublisherSetSWVideoBitRate(600, 1200); }
相关封装：
//停止rtmp推送 private void stopPush() { if(!isPushingRtmp) { return; } if ( !isRTSPPublisherRunning) { if (audioRecord_ != null) { Log.i(TAG, "stopPush, call audioRecord_.StopRecording.."); audioRecord_.Stop(); if (audioRecordCallback_ != null) { audioRecord_.RemoveCallback(audioRecordCallback_); audioRecordCallback_ = null; } audioRecord_ = null; } } if (libPublisher != null) { libPublisher.SmartPublisherStopPublisher(publisherHandle); } if (!isRTSPPublisherRunning) { if (publisherHandle != 0) { if (libPublisher != null) { libPublisher.SmartPublisherClose(publisherHandle); publisherHandle = 0; } } } } //停止发布RTSP流 private void stopRtspPublisher() { if(!isRTSPPublisherRunning) { return; } if (!isPushingRtmp) { if (audioRecord_ != null) { Log.i(TAG, "stopRtspPublisher, call audioRecord_.StopRecording.."); audioRecord_.Stop(); if (audioRecordCallback_ != null) { audioRecord_.RemoveCallback(audioRecordCallback_); audioRecordCallback_ = null; } audioRecord_ = null; } } if (libPublisher != null) { libPublisher.StopRtspStream(publisherHandle); } if (!isPushingRtmp) { if (publisherHandle != 0) { if (libPublisher != null) { libPublisher.SmartPublisherClose(publisherHandle); publisherHandle = 0; } } } } //停止RTSP服务 private void stopRtspService() { if(!isRTSPServiceRunning) { return; } if (libPublisher != null && rtsp_handle_ != 0) { libPublisher.StopRtspServer(rtsp_handle_); libPublisher.CloseRtspServer(rtsp_handle_); rtsp_handle_ = 0; } }
传递采集到的视频数据，摄像头数据采集，也可选用camera2的接口，对焦和体验更好：
@Override public void onPreviewFrame(byte[] data, Camera camera) { pushFrameCount++; if ( pushFrameCount % 3000 == 0 ) { Log.i("OnPre", "gc+"); System.gc(); Log.i("OnPre", "gc-"); } if (data == null) { Parameters params = camera.getParameters(); Size size = params.getPreviewSize(); int bufferSize = (((size.width|0x1f)+1) * size.height * ImageFormat.getBitsPerPixel(params.getPreviewFormat())) / 8; camera.addCallbackBuffer(new byte[bufferSize]); } else { if(isPushingRtmp || isRTSPPublisherRunning) { libPublisher.SmartPublisherOnCaptureVideoData(publisherHandle, data, data.length, pushCurrentCameraType, currentPushOrigentation); } camera.addCallbackBuffer(data); } }
如果内网环境下，用轻量级RTSP服务的话，需判断对方有没有播放自己的流数据的话，可以通过获取RTSP会话数来判断是否链接。
//当前RTSP会话数弹出框 private void PopRtspSessionNumberDialog(int session_numbers) { final EditText inputUrlTxt = new EditText(this); inputUrlTxt.setFocusable(true); inputUrlTxt.setEnabled(false); String session_numbers_tag = "RTSP服务当前客户会话数: " + session_numbers; inputUrlTxt.setText(session_numbers_tag); AlertDialog.Builder builderUrl = new AlertDialog.Builder(this); builderUrl .setTitle("内置RTSP服务") .setView(inputUrlTxt).setNegativeButton("确定", null); builderUrl.show(); } //获取RTSP会话数 class ButtonGetRtspSessionNumbersListener implements OnClickListener { public void onClick(View v) { if (libPublisher != null && rtsp_handle_ != 0) { int session_numbers = libPublisher.GetRtspServerClientSessionNumbers(rtsp_handle_); Log.i(TAG, "GetRtspSessionNumbers: " + session_numbers); PopRtspSessionNumberDialog(session_numbers); } } };
总结
Android平台的一对一互动，除了WebRTC外，在保证低延迟的前提下，RTMP或RTSP技术方案也是非常不错的选择。
