Android MediaCodec with MediaProjection - android-mediacodec

Question

i would like to record screen with MediaCodec and MediaProjection. After MediaCodec starts, i expect to see the display image. but i got wrong display image in onOutputBufferAvailable. also i am using Galaxy S20 for testing. what is wrong with my code?
private fun startRecording(){
val displayManager = getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
val defaultDisplay: Display = displayManager.getDisplay(Display.DEFAULT_DISPLAY)
val metrics = resources.displayMetrics
prepareMediaCodec(metrics.widthPixels, metrics.heightPixels, metrics.densityDpi)
}
private fun prepareMediaCodec(screenWidth: Int, screenHeight: Int, screenDensity : Int) {
val mediaFormat =
MediaFormat.createVideoFormat("video/avc", screenWidth, screenHeight).apply {
setInteger(MediaFormat.KEY_BIT_RATE, 6000000)
setInteger(MediaFormat.KEY_FRAME_RATE, 30)
setInteger(MediaFormat.KEY_CAPTURE_RATE, 30)
setInteger(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 1000000 / 30)
setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1)
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
}
mediaCodec = MediaCodec.createEncoderByType("video/avc")
mediaCodec.setCallback(object : MediaCodec.Callback() {
override fun onOutputBufferAvailable( p0: MediaCodec,outputBufferId: Int, info: MediaCodec.BufferInfo) {
val encodedData: ByteBuffer = mediaCodec.getOutputBuffer(outputBufferId)
encodedData.position(info.offset)
encodedData.limit(info.offset + info.size)
val data = ByteArray(encodedData.remaining())
encodedData.get(data)
val baos =ByteArrayOutputStream()
val yuvImage = YuvImage(data, ImageFormat.NV21, screenWidth, screenHeight, null)
yuvImage.compressToJpeg(Rect(0, 0, screenWidth, screenHeight), 80, baos)
val response = baos.toByteArray()
val bitmap: Bitmap? = BitmapFactory.decodeByteArray(response, 0, response.size)
Log.e(TAG, bitmap.toString()) // check bitmap
mediaCodec.releaseOutputBuffer(outputBufferId, false)
}
...
})
mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
mediaCodec.start()
mediaProjection.createVirtualDisplay(
"Record", screenWidth, screenHeight, screenDensity,
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
mediaCodec.createInputSurface(), null)
}

Answer 1

The output buffer that you received from onOutputBufferAvailable of an encoder is in h264 format and is not NV21 buffer so you couldn't not save it to bitmap as it is.
What you need to do is use MediaMuxer to save it as a playable video format such as mp4 files.