I am implementing raw h264 player using MediaCodec - h.264

Question

I am implementing raw h264 player.
The following codes work slowly. (There is mutex problems, But ignore it.)
Also bpkt.bytes is sent each frame unit named end frame from other server via tcp.
But mH264Queue is slower for consuming data than filling data to mH264Queue.
The size of mH264Queue is increased forever.
Please see my codes.
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.Queue;
import android.app.Activity;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaFormat;
import android.os.Bundle;
import android.os.Environment;
import android.os.Handler;
import android.util.Log;
import android.view.Surface;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.widget.Toast;
public class RawH264Activity extends Activity implements SurfaceHolder.Callback
{
// private static final String filePath = Environment.getExternalStorageDirectory()+ "/ksoo.h264"; // + "/video_encoded.263";//"/video_encoded.264";
private PlayerThread mPlayer = null;
Handler handler = null;
// public static byte[] SPS = null;
// public static byte[] PPS = null;
Socket mMediaSocket = null;
BufferedInputStream mMediaSocketIn = null;
OutputStream mMediaSocketOut = null;
Thread mSocketTh = null;
Queue<SocketTool.BinaryPkt> mH264Queue = new LinkedList<SocketTool.BinaryPkt>();
#Override
protected void onCreate(Bundle savedInstanceState)
{
super.onCreate(savedInstanceState);
SurfaceView sv = new SurfaceView(this);
handler = new Handler();
sv.getHolder().addCallback(this);
setContentView(sv);
}
#Override
public void surfaceCreated(SurfaceHolder holder)
{
Log.d("DecodeActivity", "in surfaceCreated");
}
#Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height)
{
Log.d("DecodeActivity", "in surfaceChanged");
if (mPlayer == null)
{
Toast.makeText(getApplicationContext(), "in surfaceChanged. creating playerthread", Toast.LENGTH_SHORT).show();
mSocketTh = new Thread(new Runnable() {
#Override
public void run() {
try {
mMediaSocket = new Socket(CurrentState.get().mRemoteAddress, CurrentState.get().mRemoteMediaPort);
mMediaSocketIn = new BufferedInputStream(mMediaSocket.getInputStream());
mMediaSocketOut = mMediaSocket.getOutputStream();
mPlayer = new PlayerThread(holder.getSurface());
mPlayer.start();
while(!mSocketTh.isInterrupted()) {
SocketTool.BinaryPkt bpkt = SocketTool.readPKTBinary(mMediaSocketIn);
mH264Queue.add(bpkt);
if(mH264Queue.size() > 2000)
break;
}
try {
mMediaSocketIn.close();
mMediaSocketOut.close();
mMediaSocket.close();
} catch (IOException e) {
e.printStackTrace();
}
// Log.i("bpkt", "" + bpkt.mCmdType);
// }
} catch (IOException e) {
// 서버 접속 끊기
e.printStackTrace();
}
}
});
mSocketTh.start();
}
}
#Override
public void surfaceDestroyed(SurfaceHolder holder)
{
if (mPlayer != null)
{
mPlayer.interrupt();
}
}
private class PlayerThread extends Thread
{
//private MediaExtractor extractor;
private MediaCodec decoder;
private Surface surface;
public PlayerThread(Surface surface)
{
this.surface = surface;
}
#Override
public void run()
{
handler.post(new Runnable()
{
#Override
public void run()
{
try {
decoder = MediaCodec.createDecoderByType("video/avc");
} catch (IOException e) {
e.printStackTrace();
}
MediaFormat mediaFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720);
// Video_format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, interval);
// mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
// mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 60);
// mediaFormat.setByteBuffer("csd-0", ByteBuffer.wrap(new byte[] {
// 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0x00, 0x1F, (byte)0x9D, (byte)0xA8, 0x14, 0x01, 0x6E, (byte)0x9B, (byte)0x80,
// (byte)0x80, (byte)0x80, (byte)0x81}));
// mediaFormat.setByteBuffer("csd-1", ByteBuffer.wrap(new byte[] {
// 0x00, 0x00, 0x00, 0x01, 0x68, (byte)0xCE, 0x3C, (byte)0x80 }));
// byte[] header_sps = { 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, (byte)0x80, 0x0C, (byte)0xE4, 0x40, (byte)0xA0, (byte)0xFD, 0x00, (byte)0xDA, 0x14, 0x26, (byte)0xA0 };
// byte[] header_pps = {0x00, 0x00, 0x00, 0x01, 0x68, (byte)0xCE, 0x38, (byte)0x80 };
// mediaFormat.setByteBuffer("csd-0", ByteBuffer.wrap(header_sps));
// mediaFormat.setByteBuffer("csd-1", ByteBuffer.wrap(header_pps));
decoder.configure(mediaFormat, surface /* surface */, null /* crypto */, 0 /* flags */);
if (decoder == null)
{
Log.e("DecodeActivity", "Can't find video info!");
return;
}
decoder.start();
Log.d("DecodeActivity", "decoder.start() called");
ByteBuffer[] inputBuffers = decoder.getInputBuffers();
ByteBuffer[] outputBuffers = decoder.getOutputBuffers();
long startMs = System.currentTimeMillis();
int i = 0;
while(!Thread.interrupted())
{
int inIndex = 0;
while ((inIndex = decoder.dequeueInputBuffer(1)) < 0)
;
if (inIndex >= 0)
{
ByteBuffer buffer = inputBuffers[inIndex];
buffer.clear();
Log.i("queue", String.valueOf(mH264Queue.size()));
SocketTool.BinaryPkt bpkt = mH264Queue.poll();
int sampleSize = 0;
if (bpkt.bytes == null) {
Log.d("DecodeActivity", "InputBuffer BUFFER_FLAG_END_OF_STREAM");
decoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
break;
}
else
{
sampleSize = bpkt.bytes.length;
buffer.clear();
buffer.put(bpkt.bytes);
decoder.queueInputBuffer(inIndex, 0, sampleSize, 0, 0);
}
BufferInfo info = new BufferInfo();
int outIndex = decoder.dequeueOutputBuffer(info, 10000);
switch (outIndex)
{
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d("DecodeActivity", "INFO_OUTPUT_BUFFERS_CHANGED");
outputBuffers = decoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
// Log.d("DecodeActivity", "New format " + decoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
// Log.d("DecodeActivity", "dequeueOutputBuffer timed out!");
try {
sleep(1);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
break;
default:
ByteBuffer outbuffer = outputBuffers[outIndex];
// Log.d("DecodeActivity", "We can't use this buffer but render it due to the API limit, " + outbuffer);
decoder.releaseOutputBuffer(outIndex, true);
break;
}
i++;
}
}
decoder.stop();
decoder.release();
}
});
}
}
}

Answer 1

The problem here is you only drain the output buffer when input buffer is available.
The fact is the output of mediacodec is often (if not always) not available instantly after queueing the input buffer.
So instead you should do something like this
while(!endReached) {
// Try drain decoder output first
BufferInfo info = new BufferInfo();
int outIndex = decoder.dequeueOutputBuffer(info, 10000);
switch (outIndex) {
...
}
// Feed decoder input
if (inputAvailable) {
int inIndex = decoder.dequeueInputBuffer(10000);
...
}
}