技术背景
我们在做Android平台RTSP、RTMP播放器的时候，遇到这样的技术诉求，开发者除了希望低延迟的播放外，还想把数据回调上来，然后做视觉算法分析。
单纯地回调数据，不难，需要保证的是，在不影响播放、录像、快照等常规功能的前提下，尽可能高效的数据回调。
技术实现
以大牛直播SDK的SmartPlayer为例，点开始播放之前，初始化参数的时候，我们设置YUV或RGB数据回调：
设置YUV或RGB数据回调：
btnStartStopPlayback.setOnClickListener(new Button.OnClickListener() { // @Override public void onClick(View v) { if (isPlaying) { Log.i(TAG, "Stop playback stream++"); int iRet = libPlayer.SmartPlayerStopPlay(playerHandle); if (iRet != 0) { Log.e(TAG, "Call SmartPlayerStopPlay failed.."); return; } btnHardwareDecoder.setEnabled(true); btnLowLatency.setEnabled(true); if (!isRecording) { btnPopInputUrl.setEnabled(true); btnPopInputKey.setEnabled(true); btnSetPlayBuffer.setEnabled(true); btnFastStartup.setEnabled(true); btnRecoderMgr.setEnabled(true); libPlayer.SmartPlayerClose(playerHandle); playerHandle = 0; } isPlaying = false; btnStartStopPlayback.setText("开始播放 "); if (is_enable_hardware_render_mode && sSurfaceView != null) { sSurfaceView.setVisibility(View.GONE); sSurfaceView.setVisibility(View.VISIBLE); } Log.i(TAG, "Stop playback stream--"); } else { Log.i(TAG, "Start playback stream++"); if (!isRecording) { InitAndSetConfig(); } // 如果第二个参数设置为null，则播放纯音频 libPlayer.SmartPlayerSetSurface(playerHandle, sSurfaceView); libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1); //int render_format = 1; //libPlayer.SmartPlayerSetSurfaceRenderFormat(playerHandle, render_format); //int is_enable_anti_alias = 1; //libPlayer.SmartPlayerSetSurfaceAntiAlias(playerHandle, is_enable_anti_alias); if (isHardwareDecoder && is_enable_hardware_render_mode) { libPlayer.SmartPlayerSetHWRenderMode(playerHandle, 1); } // External Render test //libPlayer.SmartPlayerSetExternalRender(playerHandle, new RGBAExternalRender(imageSavePath)); libPlayer.SmartPlayerSetExternalRender(playerHandle, new I420ExternalRender(imageSavePath)); libPlayer.SmartPlayerSetUserDataCallback(playerHandle, new UserDataCallback()); //libPlayer.SmartPlayerSetSEIDataCallback(playerHandle, new SEIDataCallback()); libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1); if (isMute) { libPlayer.SmartPlayerSetMute(playerHandle, isMute ? 1 : 0); } if (isHardwareDecoder) { int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle, 1); int isSupportH264HwDecoder = libPlayer .SetSmartPlayerVideoHWDecoder(playerHandle, 1); Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder); } libPlayer.SmartPlayerSetLowLatencyMode(playerHandle, isLowLatency ? 1 : 0); libPlayer.SmartPlayerSetFlipVertical(playerHandle, is_flip_vertical ? 1 : 0); libPlayer.SmartPlayerSetFlipHorizontal(playerHandle, is_flip_horizontal ? 1 : 0); libPlayer.SmartPlayerSetRotation(playerHandle, rotate_degrees); libPlayer.SmartPlayerSetAudioVolume(playerHandle, curAudioVolume); int iPlaybackRet = libPlayer .SmartPlayerStartPlay(playerHandle); if (iPlaybackRet != 0) { Log.e(TAG, "Call SmartPlayerStartPlay failed.."); return; } btnStartStopPlayback.setText("停止播放 "); btnPopInputUrl.setEnabled(false); btnPopInputKey.setEnabled(false); btnHardwareDecoder.setEnabled(false); btnSetPlayBuffer.setEnabled(false); btnLowLatency.setEnabled(false); btnFastStartup.setEnabled(false); btnRecoderMgr.setEnabled(false); isPlaying = true; Log.i(TAG, "Start playback stream--"); } } });
对应的设置如下：
// External Render test libPlayer.SmartPlayerSetExternalRender(playerHandle, new RGBAExternalRender(imageSavePath)); libPlayer.SmartPlayerSetExternalRender(playerHandle, new I420ExternalRender(imageSavePath));
如果是RGBA数据，处理如下：
/* * RGBA数据回调处理 * Author: daniusdk.com */ private static class RGBAExternalRender implements NTExternalRender { // public static final int NT_FRAME_FORMAT_RGBA = 1; // public static final int NT_FRAME_FORMAT_ABGR = 2; // public static final int NT_FRAME_FORMAT_I420 = 3; private final String image_path_; private long last_save_image_time_ms_; private int width_; private int height_; private int row_bytes_; private ByteBuffer rgba_buffer_; public RGBAExternalRender(String image_path) { this.image_path_ = image_path; } public int getNTFrameFormat() { Log.i(TAG, "RGBAExternalRender::getNTFrameFormat return " + NT_FRAME_FORMAT_RGBA); return NT_FRAME_FORMAT_RGBA; } public void onNTFrameSizeChanged(int width, int height) { width_ = width; height_ = height; row_bytes_ = width_ * 4; rgba_buffer_ = ByteBuffer.allocateDirect(row_bytes_ * height_); Log.i(TAG, "RGBAExternalRender::onNTFrameSizeChanged width_:" + width_ + " height_:" + height_); } public ByteBuffer getNTPlaneByteBuffer(int index) { if (index == 0) return rgba_buffer_; Log.e(TAG, "RGBAExternalRender::getNTPlaneByteBuffer index error:" + index); return null; } public int getNTPlanePerRowBytes(int index) { if (index == 0) return row_bytes_; Log.e(TAG, "RGBAExternalRender::getNTPlanePerRowBytes index error:" + index); return 0; } public void onNTRenderFrame(int width, int height, long timestamp) { if (rgba_buffer_ == null) return; rgba_buffer_.rewind(); // copy buffer // test // byte[] test_buffer = new byte[16]; // rgba_buffer_.get(test_buffer); Log.i(TAG, "RGBAExternalRender:onNTRenderFrame " + width + "*" + height + ", t:" + timestamp); // Log.i(TAG, "RGBAExternalRender:onNTRenderFrame rgba:" + // bytesToHexString(test_buffer)); } }
如果是I420数据：
/* *YUV数据回调处理 * Author: daniusdk.com */ private static class I420ExternalRender implements NTExternalRender { // public static final int NT_FRAME_FORMAT_RGBA = 1; // public static final int NT_FRAME_FORMAT_ABGR = 2; // public static final int NT_FRAME_FORMAT_I420 = 3; private final String image_path_; private long last_save_image_time_ms_; private int width_; private int height_; private int y_row_bytes_; private int u_row_bytes_; private int v_row_bytes_; private ByteBuffer y_buffer_; private ByteBuffer u_buffer_; private ByteBuffer v_buffer_; public I420ExternalRender(String image_path) { this.image_path_ = image_path; } public int getNTFrameFormat() { Log.i(TAG, "I420ExternalRender::getNTFrameFormat return " + NT_FRAME_FORMAT_I420); return NT_FRAME_FORMAT_I420; } public void onNTFrameSizeChanged(int width, int height) { width_ = width; height_ = height; y_row_bytes_ = width; u_row_bytes_ = (width+1)/2; v_row_bytes_ = (width+1)/2; y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_*height_); u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_*((height_ + 1) / 2)); v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_*((height_ + 1) / 2)); Log.i(TAG, "I420ExternalRender::onNTFrameSizeChanged width_=" + width_ + " height_=" + height_ + " y_row_bytes_=" + y_row_bytes_ + " u_row_bytes_=" + u_row_bytes_ + " v_row_bytes_=" + v_row_bytes_); } public ByteBuffer getNTPlaneByteBuffer(int index) { switch (index) { case 0: return y_buffer_; case 1: return u_buffer_; case 2: return v_buffer_; default: Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index); return null; } } public int getNTPlanePerRowBytes(int index) { switch (index) { case 0: return y_row_bytes_; case 1: return u_row_bytes_; case 2: return v_row_bytes_; default: Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index); return 0; } } public void onNTRenderFrame(int width, int height, long timestamp) { if (null == y_buffer_ || null == u_buffer_ || null == v_buffer_) return; y_buffer_.rewind(); u_buffer_.rewind(); v_buffer_.rewind(); Log.i(TAG, "I420ExternalRender::onNTRenderFrame " + width + "*" + height + ", t:" + timestamp); // copy buffer // test // byte[] test_buffer = new byte[16]; // y_buffer_.get(test_buffer); // Log.i(TAG, "I420ExternalRender::onNTRenderFrame y data:" + bytesToHexString(test_buffer)); // u_buffer_.get(test_buffer); // Log.i(TAG, "I420ExternalRender::onNTRenderFrame u data:" + bytesToHexString(test_buffer)); // v_buffer_.get(test_buffer); // Log.i(TAG, "I420ExternalRender::onNTRenderFrame v data:" + bytesToHexString(test_buffer)); } }
感兴趣的开发者，可以参考看看，如果需要测试，可以私信探讨。
