技术背景
我们在做执法记录仪或指挥系统的时候，会遇到这样的情况，大多场景下，我们是不需要把设备端的数据，实时传给国标平台端的，默认只需要本地录像留底，如果指挥中心需要查看前端设备实时数据的时候，发起视频播放请求，设备侧再推送数据到平台侧，如需语音广播，只要发起语音广播（broadcast），GB28181设备接入侧响应，然后发送INVITE请求等，完成语音广播和语音对讲。此外，考虑到设备侧的上行带宽瓶颈，一般来说，本地录像需要尽可能清晰（比如1920*1080分辨率），上传视频数据，传输1280*720分辨率，也就是我们传统意义提到的双码流编码。
技术实现
带着这些问题，以Android平台设备接入模块为例，我们来逐一分析解决：
按需编码
按需编码，只需要Android平台GB28181设备接入端，完成设备到平台的注册（register），然后平台侧发起catalog查询设备，并维持心跳信息，如果订阅了实时位置信息，按照设定间隔，实时上报位置信息即可。
在需要录像或指挥中心需要播放前端设备实时音视频数据的时候，我们才编码音视频数据，这样保证，待机时，最小化的资源占用。
以上图为例，只需点击“启动GB28181”即可，对应的代码实现如下：
classButtonGB28181AgentListenerimplementsView.OnClickListener { publicvoidonClick(Viewv) { stopAudioPlayer(); destoryRTPReceiver(); gb_broadcast_source_id_=null; gb_broadcast_target_id_=null; btnGB28181AudioBroadcast.setText("GB28181语音广播"); btnGB28181AudioBroadcast.setEnabled(false); stopGB28181Stream(); destoryRTPSender(); if (null==gb28181_agent_ ) { if( !initGB28181Agent() ) return; } if (gb28181_agent_.isRunning()) { gb28181_agent_.terminateAllPlays(true);// 目前测试下来,发送BYE之后,有些服务器会立即发送INVITE,是否发送BYE根据实际情况看gb28181_agent_.stop(); btnGB28181Agent.setText("启动GB28181"); } else { if ( gb28181_agent_.start() ) { btnGB28181Agent.setText("停止GB28181"); } } } }
initGB28181Agent()实现如下：
privatebooleaninitGB28181Agent() { if ( gb28181_agent_!=null ) returntrue; getLocation(context_); Stringlocal_ip_addr=IPAddrUtils.getIpAddress(context_); Log.i(TAG, "initGB28181Agent local ip addr: "+local_ip_addr); if ( local_ip_addr==null||local_ip_addr.isEmpty() ) { Log.e(TAG, "initGB28181Agent local ip is empty"); returnfalse; } gb28181_agent_=GBSIPAgentFactory.getInstance().create(); if ( gb28181_agent_==null ) { Log.e(TAG, "initGB28181Agent create agent failed"); returnfalse; } gb28181_agent_.addListener(this); gb28181_agent_.addPlayListener(this); gb28181_agent_.addAudioBroadcastListener(this); gb28181_agent_.addDeviceControlListener(this); gb28181_agent_.addQueryCommandListener(this); // 必填信息gb28181_agent_.setLocalAddress(local_ip_addr); gb28181_agent_.setServerParameter(gb28181_sip_server_addr_, gb28181_sip_server_port_, gb28181_sip_server_id_, gb28181_sip_domain_); gb28181_agent_.setUserInfo(gb28181_sip_username_, gb28181_sip_password_); //gb28181_agent_.setUserInfo(gb28181_sip_username_, gb28181_sip_username_, gb28181_sip_password_);// 可选参数gb28181_agent_.setUserAgent(gb28181_sip_user_agent_filed_); gb28181_agent_.setTransportProtocol(gb28181_sip_trans_protocol_==0?"UDP":"TCP"); // GB28181配置gb28181_agent_.config(gb28181_reg_expired_, gb28181_heartbeat_interval_, gb28181_heartbeat_count_); com.gb.ntsignalling.Devicegb_device=newcom.gb.ntsignalling.Device("34020000001380000001", "安卓测试设备", Build.MANUFACTURER, Build.MODEL, "宇宙","火星1","火星", true); if (mLongitude!=null&&mLatitude!=null) { com.gb.ntsignalling.DevicePositiondevice_pos=newcom.gb.ntsignalling.DevicePosition(); device_pos.setTime(mLocationTime); device_pos.setLongitude(mLongitude); device_pos.setLatitude(mLatitude); gb_device.setPosition(device_pos); gb_device.setSupportMobilePosition(true); // 设置支持移动位置上报 } gb28181_agent_.addDevice(gb_device); if (!gb28181_agent_.createSipStack()) { gb28181_agent_=null; Log.e(TAG, "initGB28181Agent gb28181_agent_.createSipStack failed."); returnfalse; } booleanis_bind_local_port_ok=false; // 最多尝试5000个端口inttry_end_port=gb28181_sip_local_port_base_+5000; try_end_port=try_end_port>65536?65536: try_end_port; for (inti=gb28181_sip_local_port_base_; i<try_end_port; ++i) { if (gb28181_agent_.bindLocalPort(i)) { is_bind_local_port_ok=true; break; } } if (!is_bind_local_port_ok) { gb28181_agent_.releaseSipStack(); gb28181_agent_=null; Log.e(TAG, "initGB28181Agent gb28181_agent_.bindLocalPort failed."); returnfalse; } if (!gb28181_agent_.initialize()) { gb28181_agent_.unBindLocalPort(); gb28181_agent_.releaseSipStack(); gb28181_agent_=null; Log.e(TAG, "initGB28181Agent gb28181_agent_.initialize failed."); returnfalse; } returntrue; }
注册和心跳异常处理如下：
publicvoidntsRegisterOK(StringdateString) { Log.i(TAG, "ntsRegisterOK Date: "+ (dateString!=null?dateString : "")); } publicvoidntsRegisterTimeout() { Log.e(TAG, "ntsRegisterTimeout"); } publicvoidntsRegisterTransportError(StringerrorInfo) { Log.e(TAG, "ntsRegisterTransportError error:"+ (errorInfo!=null?errorInfo :"")); } publicvoidntsOnHeartBeatException(intexceptionCount, StringlastExceptionInfo) { Log.e(TAG, "ntsOnHeartBeatException heart beat timeout count reached, count:"+exceptionCount+", exception info:"+ (lastExceptionInfo!=null?lastExceptionInfo:"")); // 停止信令, 然后重启handler_.postDelayed(newRunnable() { publicvoidrun() { Log.i(TAG, "gb28281_heart_beart_timeout"); stopAudioPlayer(); destoryRTPReceiver(); if (gb_broadcast_source_id_!=null&&gb_broadcast_target_id_!=null&&gb28181_agent_!=null) gb28181_agent_.byeAudioBroadcast(gb_broadcast_source_id_, gb_broadcast_target_id_); gb_broadcast_source_id_=null; gb_broadcast_target_id_=null; btnGB28181AudioBroadcast.setText("GB28181语音广播"); btnGB28181AudioBroadcast.setEnabled(false); stopGB28181Stream(); destoryRTPSender(); if (gb28181_agent_!=null) { gb28181_agent_.terminateAllPlays(true); Log.i(TAG, "gb28281_heart_beart_timeout sip stop"); gb28181_agent_.stop(); Stringlocal_ip_addr=IPAddrUtils.getIpAddress(context_); if (local_ip_addr!=null&&!local_ip_addr.isEmpty() ) { Log.i(TAG, "gb28281_heart_beart_timeout get local ip addr: "+local_ip_addr); gb28181_agent_.setLocalAddress(local_ip_addr); } Log.i(TAG, "gb28281_heart_beart_timeout sip start"); gb28181_agent_.start(); } } },0); }
ntsOnAckPlay()的时候，我们才开始编码：
publicvoidntsOnAckPlay(StringdeviceId) { handler_.postDelayed(newRunnable() { publicvoidrun() { Log.i(TAG,"ntsOnACKPlay, device_id:"+device_id_); if (!isRTSPPublisherRunning&&!isPushingRtmp&&!isRecording) { InitAndSetConfig(); } libPublisher.SetGB28181RTPSender(publisherHandle, gb28181_rtp_sender_handle_, gb28181_rtp_payload_type_, gb28181_rtp_encoding_name_); //libPublisher.SetGBTCPConnectTimeout(publisherHandle, 10*60*1000);//libPublisher.SetGBInitialTCPReconnectInterval(publisherHandle, 1000);//libPublisher.SetGBInitialTCPMaxReconnectAttempts(publisherHandle, 3);intstartRet=libPublisher.StartGB28181MediaStream(publisherHandle); if (startRet!=0) { if (!isRTSPPublisherRunning&&!isPushingRtmp&&!isRecording) { if (publisherHandle!=0) { longhandle=publisherHandle; publisherHandle=0; libPublisher.SmartPublisherClose(handle); } } destoryRTPSender(); Log.e(TAG, "Failed to start GB28181 service.."); return; } if (!isRTSPPublisherRunning&&!isPushingRtmp&&!isRecording) { CheckInitAudioRecorder(); } startLayerPostThread(); isGB28181StreamRunning=true; } privateStringdevice_id_; publicRunnableset(Stringdevice_id) { this.device_id_=device_id; returnthis; } }.set(deviceId),0); }
其中，InitAndSetConfig()完成基础参数设定，比如数据源类型、软硬编码设置、帧率关键帧间隔码率等参数设置：
privatevoidInitAndSetConfig() { intaudio_opt=1; intfps=18; intgop=fps*2; Log.i(TAG, "InitAndSetConfig video_width: "+video_width_+" cur_video_height"+video_height_+" imageRotationDegree:"+cameraImageRotationDegree_); publisherHandle=libPublisher.SmartPublisherOpen(context_, audio_opt, 3, video_width_, video_height_); if (publisherHandle==0) { Log.e(TAG, "sdk open failed!"); return; } Log.i(TAG, "publisherHandle="+publisherHandle); if(videoEncodeType==1) { inth264HWKbps=setHardwareEncoderKbps(true, video_width_, video_height_); h264HWKbps=h264HWKbps*fps/25; Log.i(TAG, "h264HWKbps: "+h264HWKbps); intisSupportH264HWEncoder=libPublisher .SetSmartPublisherVideoHWEncoder(publisherHandle, h264HWKbps); if (isSupportH264HWEncoder==0) { libPublisher.SetNativeMediaNDK(publisherHandle, 0); libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 1); // 0:CQ, 1:VBR, 2:CBRlibPublisher.SetVideoHWEncoderQuality(publisherHandle, 39); libPublisher.SetAVCHWEncoderProfile(publisherHandle, 0x08); // 0x01: Baseline, 0x02: Main, 0x08: High// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x200); // Level 3.1// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x400); // Level 3.2// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x800); // Level 4libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x1000); // Level 4.1 多数情况下,这个够用了//libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x2000); // Level 4.2// libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)h264HWKbps)*1300);Log.i(TAG, "Great, it supports h.264 hardware encoder!"); } } elseif (videoEncodeType==2) { inthevcHWKbps=setHardwareEncoderKbps(false, video_width_, video_height_); hevcHWKbps=hevcHWKbps*fps/25; Log.i(TAG, "hevcHWKbps: "+hevcHWKbps); intisSupportHevcHWEncoder=libPublisher .SetSmartPublisherVideoHevcHWEncoder(publisherHandle, hevcHWKbps); if (isSupportHevcHWEncoder==0) { libPublisher.SetNativeMediaNDK(publisherHandle, 0); libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 0); // 0:CQ, 1:VBR, 2:CBRlibPublisher.SetVideoHWEncoderQuality(publisherHandle, 39); // libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)hevcHWKbps)*1200);Log.i(TAG, "Great, it supports hevc hardware encoder!"); } } booleanis_sw_vbr_mode=true; if(is_sw_vbr_mode) //H.264 software encoder { intis_enable_vbr=1; intvideo_quality=CalVideoQuality(video_width_, video_height_, true); intvbr_max_bitrate=CalVbrMaxKBitRate(video_width_, video_height_); libPublisher.SmartPublisherSetSwVBRMode(publisherHandle, is_enable_vbr, video_quality, vbr_max_bitrate); } if (is_pcma_) { libPublisher.SmartPublisherSetAudioCodecType(publisherHandle, 3); } else { libPublisher.SmartPublisherSetAudioCodecType(publisherHandle, 1); } libPublisher.SetSmartPublisherEventCallbackV2(publisherHandle, newEventHandlerPublisherV2().set(handler_, recorder_io_executor_)); libPublisher.SmartPublisherSetSWVideoEncoderProfile(publisherHandle, 3); libPublisher.SmartPublisherSetSWVideoEncoderSpeed(publisherHandle, 2); libPublisher.SmartPublisherSetGopInterval(publisherHandle, gop); libPublisher.SmartPublisherSetFPS(publisherHandle, fps); // libPublisher.SmartPublisherSetSWVideoBitRate(publisherHandle, 600, 1200);booleanis_noise_suppression=true; libPublisher.SmartPublisherSetNoiseSuppression(publisherHandle, is_noise_suppression?1 : 0); booleanis_agc=false; libPublisher.SmartPublisherSetAGC(publisherHandle, is_agc?1 : 0); intecho_cancel_delay=0; libPublisher.SmartPublisherSetEchoCancellation(publisherHandle, 1, echo_cancel_delay); libPublisher.SmartPublisherSaveImageFlag(publisherHandle, 1); }
双码流编码
以采集摄像头采集为例，如果需要双码流编码，采集数据源时，以大分辨率作为采集基准分辨率，如采集1920*1080的，那么如果需要上传实时视频数据的时候，只需要缩放，得到1280*720分辨率的编码数据：
publicvoidonCameraImageData(Imageimage) { if (null==libPublisher) return; if (isPushingRtmp||isRTSPPublisherRunning||isGB28181StreamRunning||isRecording) { if (0==publisherHandle) return; Image.Plane[] planes=image.getPlanes(); intw=image.getWidth(), h=image.getHeight(); inty_offset=0, u_offset=0, v_offset=0; Rectcrop_rect=image.getCropRect(); if (crop_rect!=null&&!crop_rect.isEmpty()) { w=crop_rect.width(); h=crop_rect.height(); y_offset+=crop_rect.top*planes[0].getRowStride() +crop_rect.left*planes[0].getPixelStride(); u_offset+= (crop_rect.top/2) *planes[1].getRowStride() + (crop_rect.left/2) *planes[1].getPixelStride(); v_offset+= (crop_rect.top/2) *planes[2].getRowStride() + (crop_rect.left/2) *planes[2].getPixelStride(); ; // Log.i(TAG, "crop w:" + w + " h:" + h + " y_offset:"+ y_offset + " u_offset:" + u_offset + " v_offset:" + v_offset); } intscale_w=0, scale_h=0, scale_filter_mode=0; scale_filter_mode=3; introtation_degree=cameraImageRotationDegree_; if (rotation_degree<0) { Log.i(TAG, "onCameraImageData rotation_degree < 0, may need to set orientation_ to 0, 90, 180 or 270"); return; } if (!post_image_lock_.tryLock()) { Log.i(TAG, "post_image_lock_.tryLock return false"); return; } try { if (publisherHandle!=0) { if (isPushingRtmp||isRTSPPublisherRunning||isGB28181StreamRunning||isRecording) { libPublisher.PostLayerImageYUV420888ByteBuffer(publisherHandle, 0, 0, 0, planes[0].getBuffer(), y_offset, planes[0].getRowStride(), planes[1].getBuffer(), u_offset, planes[1].getRowStride(), planes[2].getBuffer(), v_offset, planes[2].getRowStride(), planes[1].getPixelStride(), w, h, 0, 0, scale_w, scale_h, scale_filter_mode, rotation_degree); } } }catch (Exceptione) { Log.e(TAG, "onCameraImageData Exception:", e); }finally { post_image_lock_.unlock(); } } }
PostLayerImageYUV420888ByteBuffer()设计如下：
/* * SmartPublisherJniV2.java * SmartPublisherJniV2 * * Author: https://daniusdk.com * Created by DaniuLive on 2015/09/20. */ /** * 投递层YUV420888图像, 专门为android.media.Image的android.graphics.ImageFormat.YUV_420_888格式提供的接口 * * @param index: 层索引, 必须大于等于0 * * @param left: 层叠加的左上角坐标, 对于第0层的话传0 * * @param top: 层叠加的左上角坐标, 对于第0层的话传0 * * @param y_plane: 对应android.media.Image.Plane[0].getBuffer() * * @param y_offset: 图像偏移, 这个主要目的是用来做clip的,一般传0 * * @param y_row_stride: 对应android.media.Image.Plane[0].getRowStride() * * @param u_plane: android.media.Image.Plane[1].getBuffer() * * @param u_offset: 图像偏移, 这个主要目的是用来做clip的,一般传0 * * @param u_row_stride: android.media.Image.Plane[1].getRowStride() * * @param v_plane: 对应android.media.Image.Plane[2].getBuffer() * * @param v_offset: 图像偏移, 这个主要目的是用来做clip的,一般传0 * * @param v_row_stride: 对应android.media.Image.Plane[2].getRowStride() * * @param uv_pixel_stride: 对应android.media.Image.Plane[1].getPixelStride() * * @param width: width, 必须大于1, 且必须是偶数 * * @param height: height, 必须大于1, 且必须是偶数 * * @param is_vertical_flip: 是否垂直翻转, 0不翻转, 1翻转 * * @param is_horizontal_flip：是否水平翻转, 0不翻转, 1翻转 * * @param scale_width: 缩放宽，必须是偶数, 0或负数不缩放 * * @param scale_height: 缩放高, 必须是偶数, 0或负数不缩放 * * @param scale_filter_mode: 缩放质量, 传0使用默认速度,可选等级范围是:[1,3],值越大缩放质量越好, 但速度越慢 * * @param rotation_degree: 顺时针旋转, 必须是0, 90, 180, 270, 注意:旋转是在缩放, 垂直/水品反转之后再做, 请留意顺序 * * @return {0} if successful */ public native int PostLayerImageYUV420888ByteBuffer(long handle, int index, int left, int top, ByteBuffer y_plane, int y_offset, int y_row_stride, ByteBuffer u_plane, int u_offset, int u_row_stride, ByteBuffer v_plane, int v_offset, int v_row_stride, int uv_pixel_stride, int width, int height, int is_vertical_flip, int is_horizontal_flip, int scale_width, int scale_height, int scale_filter_mode, int rotation_degree);
上述接口参数，scale_width和scale_height可以指定缩放宽高，甚至如果摄像头采集的方向不对，可以设置rotation_degree接口，来实现视频数据的旋转。
接口参数第一个是实例句柄，如果需要两路编码，势必对应两个推送实例，也就是两个handle，一个用来录像，一个用来gb28181上行数据推送。
需要注意的是，如果需要同时两个实例编码，需要投递数据的时候，两个实例，分别调用PostLayerImageYUV420888ByteBuffer()实现数据源到底层模块的投递。
本地录像操作如下：
classButtonStartRecorderListenerimplementsView.OnClickListener { publicvoidonClick(Viewv) { if (isRecording) { stopRecorder(); if (!isPushingRtmp&&!isRTSPPublisherRunning&&!isGB28181StreamRunning) { ConfigControlEnable(true); } btnStartRecorder.setText("实时录像"); btnPauseRecorder.setText("暂停录像"); btnPauseRecorder.setEnabled(false); isPauseRecording=true; return; } Log.i(TAG, "onClick start recorder.."); if (libPublisher==null) return; if (!isPushingRtmp&&!isRTSPPublisherRunning&&!isGB28181StreamRunning) { InitAndSetConfig(); } ConfigRecorderParam(); intstartRet=libPublisher.SmartPublisherStartRecorder(publisherHandle); if (startRet!=0) { if (!isPushingRtmp&&!isRTSPPublisherRunning&&!isGB28181StreamRunning) { if (publisherHandle!=0) { longhandle=publisherHandle; publisherHandle=0; libPublisher.SmartPublisherClose(handle); } } Log.e(TAG, "Failed to start recorder."); return; } if (!isPushingRtmp&&!isRTSPPublisherRunning&&!isGB28181StreamRunning) { CheckInitAudioRecorder(); ConfigControlEnable(false); } startLayerPostThread(); btnStartRecorder.setText("停止录像"); isRecording=true; btnPauseRecorder.setEnabled(true); isPauseRecording=true; } }
停止录像：
//停止录像privatevoidstopRecorder() { if(!isRecording) return; isRecording=false; if (!isPushingRtmp&&!isRTSPPublisherRunning&&!isGB28181StreamRunning) stopLayerPostThread(); if (!isPushingRtmp&&!isRTSPPublisherRunning&&!isGB28181StreamRunning) { if (audioRecord_!=null) { Log.i(TAG, "stopRecorder, call audioRecord_.StopRecording.."); audioRecord_.Stop(); if (audioRecordCallback_!=null) { audioRecord_.RemoveCallback(audioRecordCallback_); audioRecordCallback_=null; } audioRecord_=null; } } if (null==libPublisher||0==publisherHandle) return; libPublisher.SmartPublisherStopRecorder(publisherHandle); if (!isPushingRtmp&&!isRTSPPublisherRunning&&!isGB28181StreamRunning) { releasePublisherHandle(); } }
技术总结
按需编码，可以只是本地录像或上行数据推送，对应一个实例完成，如果双码流编码，势必需要两个实例，对应不同的编码参数，输出不同的分辨率的H.264/H.265数据。此外，音频数据回调的地方，两个实例也调用音频投递接口，传下去。需要注意的是，两路视频编码，尽管可以硬编码，对设备性能依然提了更高的要求。
