Экспорт видео на Android с MediaCodec - воспроизведение слишком быстрое
Я пытаюсь отладить приложение, которое объединяет два фрагмента видео и экспортирует их как одно. Получающееся видео намного короче и воспроизводится слишком быстро, хотя метаданные mp4 показывают его как 25fps.
Приложение представляет собой приложение Unity с внешним Java-jar для обработки кодирования видео на Android. Если оставить бит соединения, то та же проблема существует, если мы просто берем загруженное видео и экспортируем его. Так что теоретически это должно получиться точно так же, но это не так.
Итак, функция Init выглядит следующим образом:
public void Init(String VideoPath, int inWidth, int inHeight) {
// start our exporting dialog
mExportDialog = new CustomAlertDialog();
mExportDialog.CreateCustom("Exporting", "Exporting video\nPlease Wait",
2);
if (mDecodeName == "") {
Log.d(UnityAppPlayer.TAG, "getting filename: "
+ VideoFile.Instance(UnityPlayer.currentActivity)
.GetFileName());
mDecodeName = VideoFile.Instance(UnityPlayer.currentActivity)
.GetFileName();
}
// get the update value step for our progress bar
// divide 1 by the total number of frames in the original video.
// Multiply it by 0.85f
mFrameProgressStep = (1.0f / (VideoFile.Instance(
UnityPlayer.currentActivity).GetDuration() * 25.0f * 0.85f));
// before disposing of our video get the current point of the video
// we're at
mCurrentSeekPoint = VideoFile.Instance(UnityPlayer.currentActivity)
.GetCurrentTime();
// also get the video URI so we can reinitialise the video player
// afterwards
mVideoURI = VideoFile.Instance(UnityPlayer.currentActivity)
.GetVideoURI();
// Dispose
VideoFile.Instance(UnityPlayer.currentActivity).Dispose();
mVideoName = VideoPath;
// create the video file output stream at the video path supplied
try {
mFileStream = new BufferedOutputStream(new FileOutputStream(
Environment.getExternalStorageDirectory().getPath() + "/"
+ mVideoName + ".h264"));
} catch (FileNotFoundException e) {
Log.e(UnityAppPlayer.TAG, "Unable to open video file");
e.printStackTrace();
}
mWidth = inWidth;
mHeight = inHeight;
mMediaCodec = MediaCodec.createEncoderByType("video/avc");
// Find a code that supports the mime type
int numCodecs = MediaCodecList.getCodecCount();
MediaCodecInfo codecInfo = null;
for (int i = 0; i < numCodecs && codecInfo == null; i++) {
MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
if (!info.isEncoder()) {
continue;
}
String[] types = info.getSupportedTypes();
boolean found = false;
for (int j = 0; j < types.length && !found; j++) {
if (types[j].equals("video/avc"))
found = true;
}
if (!found)
continue;
codecInfo = info;
}
Log.d(UnityAppPlayer.TAG, "Found " + codecInfo.getName() + " supporting " + "video/avc");
// Find a color profile that the codec supports
mColourFormat = 0;
MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType("video/avc");
for (int i = 0; i < capabilities.colorFormats.length && mColourFormat == 0; i++) {
int format = capabilities.colorFormats[i];
switch (format) {
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
mColourFormat = format;
break;
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
mColourFormat = format;
break;
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
mColourFormat = format;
break;
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
mColourFormat = format;
break;
case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
mColourFormat = format;
break;
default:
Log.d(UnityAppPlayer.TAG, "Skipping unsupported color format " + format);
break;
}
}
// setup the media format
MediaFormat mediaFormat = MediaFormat.createVideoFormat("video/avc", mWidth, mHeight);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 4000000);
// set the frame rate to 25FPS
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 25);
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, mColourFormat);
// add a key frame every second
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
MediaCodecList.getCodecCount();
// configure the media codec as an encoder, we don't have an input
// surface and we're not encrypting the video
mMediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
// start the encoder
mMediaCodec.start();
mExportDialog.SetProgress(5);
}
Тогда функция кодирования выглядит следующим образом:
public void Encode() {
Log.d(UnityAppPlayer.TAG, "Encode Orig Video: " + mDecodeName);
mExportDialog.SetProgress(15);
float currentProgress = 15.0f;
int VIDEO_FPS = 25;
MediaCodec decoder = null;
Log.d(UnityAppPlayer.TAG, "creating media extractor");
MediaExtractor extractor = new MediaExtractor();
try {
extractor.setDataSource(mDecodeName);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// create our output surface ready for our decoder to render on
mOutputSurface = new CodecOutputSurface(mWidth, mHeight);
Log.d(UnityAppPlayer.TAG, "getting track count");
int numTracks = extractor.getTrackCount();
Log.d(UnityAppPlayer.TAG, "track count is: " + numTracks);
for (int i = 0; i < numTracks; ++i) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
Log.d(UnityAppPlayer.TAG, "Found Correct track");
extractor.selectTrack(i);
// Decoder
decoder = MediaCodec.createDecoderByType(mime);
// configure our decoder to use the mOutputSurface
decoder.configure(format, mOutputSurface.getSurface(), null, 0);
break;
}
}
if (decoder == null) {
Log.e("DecodeActivity", "Can't find video info");
return;
} else
Log.d(UnityAppPlayer.TAG, "Decoder is Fine");
// Start Decoder
Log.d(UnityAppPlayer.TAG, "Start Decoder");
decoder.start();
// Get the average duration of a frame
long averageDuration = Math.round((1.0f / 25.0f) * 1000000.0f);
// Variables
//boolean specialFrame = false;
float addition = 0;
long timeStamp = 0;
// Byte Buffer
Log.d(UnityAppPlayer.TAG, "Assigning memory for buffer");
ByteBuffer[] inputBuffers = decoder.getInputBuffers();
ByteBuffer[] outputBuffers = decoder.getOutputBuffers();
// Gets filled with buffer Meta Data
BufferInfo info = new BufferInfo();
Log.d(UnityAppPlayer.TAG, "Starting read Loop. Analysis List size: "
+ analysisPoints.size());
// create a variable for our presentation stamp and the one from the
// last frame
long presentationTimeUs = 0;
long prevPresentationTimeUs = 0;
int outIndex = -1;
while (true) {
int sampleSize = 0;
int trackIndex = extractor.getSampleTrackIndex();
presentationTimeUs = extractor.getSampleTime();
// Get Decoder Index
int inputBufIndex = decoder.dequeueInputBuffer(-1);
if (inputBufIndex >= 0) {
// free buffer
inputBuffers[inputBufIndex].clear();
// if(VERBOSE) Log.v(UnityAppPlayer.TAG, "adding: " +
// presentationTimeUs + " - " + prevPresentationTimeUs +
// "to timestamp");
// Add the new time to time stamp
// timeStamp += (presentationTimeUs - prevPresentationTimeUs);
timeStamp = (presentationTimeUs);
if ((sampleSize = extractor.readSampleData(
inputBuffers[inputBufIndex], 0)) < 0)
break;
// inputBuffers[inputBufIndex].put(readData);
// Decoding
decoder.queueInputBuffer(inputBufIndex, 0, sampleSize,
presentationTimeUs, 0);
while(true){
// Get Outputbuffer Index
outIndex = decoder.dequeueOutputBuffer(info, 10000);
// create a flag so we only render the image when we have data
// on the dequeued output buffer
boolean doRender = true;// (info.size != 0);
if(outIndex == MediaCodec.INFO_TRY_AGAIN_LATER){
Log.d(UnityAppPlayer.TAG,"outIndex == INFO_TRY_AGAIN_LATER");
break;
}
else if (outIndex >= 0) {
Log.d(UnityAppPlayer.TAG,"outIndex >= 0");
// Release buffer
decoder.releaseOutputBuffer(outIndex, doRender);
// Log.d(UnityAppPlayer.TAG,
// "extractor Advance! bufferIndex: " + inputBufIndex);
extractor.advance();
// increment current progress
currentProgress += mFrameProgressStep;
// set the export bar value
mExportDialog.SetProgress(Math.round(currentProgress));
if (analysisPoints.size() > 0
&& (presentationTimeUs / 100000) == analysisPoints.get(0).startTime) {
Log.d(UnityAppPlayer.TAG, "Found Special frame");
/* // addition = analysisPoints.get(0).duration;
// Clock that we're inserting a special frame
specialFrame = true;
}
if (specialFrame) {
*/
Log.d(UnityAppPlayer.TAG, "Encoding Special frame"
+ " at time: " + timeStamp);
boolean isCutPaste = (analysisPoints.get(0).copyPastes.size() > 0);
int textureID[] = { -1, -1 };
// create a texture from the analysispoint's frame data
CreateTexture(textureID);
//The Index we're icnrementing to get copy paste Index
int positionIndex = 0;
float[] position = {-1.0f,-1.0f};
int numberFrames = VIDEO_FPS * (int) analysisPoints.get(0).duration;
for (int i = 0; i < numberFrames; i++)
{
//Draw main Analysis image
mOutputSurface.drawAnalysisImage(textureID[0], true);
if (isCutPaste)
{
int length = analysisPoints.get(0).copyPastes.size();
for (int copyPasteCounter = 0; copyPasteCounter < length; ++copyPasteCounter)
{
// Get the Copy and Paste item
CopyPasteData copyPasteData = analysisPoints.get(0).copyPastes.get(copyPasteCounter);
float[] vertices = {
// X, Y, Z, U, V
-.10f, -.10f, 0, 0.f, 0.f,
.10f, -.10f, 0, 1.f, 0.f,
-.10f, .10f, 0, 0.f, 1.f,
.10f, .10f, 0, 1.f, 1.f, };
// Create Float buffer
ByteBuffer vbb = ByteBuffer.allocateDirect(vertices.length * 4);
vbb.order(ByteOrder.nativeOrder()); // use the device hardware's native byte order
FloatBuffer fb = vbb.asFloatBuffer(); // create a floating point buffer from the ByteBuffer
fb.put(vertices); // add the coordinates to the FloatBuffer
fb.position(0); // set the buffer to read the first coordinate
if(positionIndex < copyPasteData.Positions.length)
position[0] = copyPasteData.Positions[positionIndex++];
if(positionIndex < copyPasteData.Positions.length)
position[1] = copyPasteData.Positions[positionIndex++];
//Log.d(UnityAppPlayer.TAG,"mOutputSurface.drawCutPasteImage Position x: " + position[0] + " y: " + position[1]);
mOutputSurface.drawCutPasteImage(textureID[1], position, fb, true);
}
}
//render Image to buffer
ByteBuffer decodedBuffer = mOutputSurface.renderToBuffer();
byte[] newArray = new byte[decodedBuffer.remaining()];
//reqind Array
decodedBuffer.rewind();
decodedBuffer.get(newArray);
// the image data should in the size of YUV which is a 3byte
// variable
byte[] imageData = new byte[(mWidth * mHeight * 3) / 2];
covertToYUV(imageData, newArray, mWidth, mHeight);
timeStamp += averageDuration;
// Perform Encoding
EncodeFrame(imageData, timeStamp + (long) addition);
}
// update the addition
addition += analysisPoints.get(0).duration;
//specialFrame = false;
// remove used frame data
analysisPoints.get(0).byteBuffer.clear();
analysisPoints.remove(0);
GLES20.glDeleteTextures(textureID.length, textureID, 0);
} else {
// Log.d(UnityAppPlayer.TAG,
// "Encoding Standard frame! SampleSize: " + sampleSize +
// " at time: " + timeStamp);
// Check Outputbuffer Index is valid and we have an image to
// render
//if (outIndex >= 0 && doRender) {
// if(VERBOSE) Log.d(UnityAppPlayer.TAG,
// "Awaiting new image");
mOutputSurface.awaitNewImage();
// if(VERBOSE) Log.d(UnityAppPlayer.TAG,
// "Draw new image");
mOutputSurface.drawImage(false);
// if(VERBOSE) Log.d(UnityAppPlayer.TAG,
// "Render to buffer");
// get the buffer from the outputSurface
ByteBuffer decodedBuffer = mOutputSurface
.renderToBuffer();
// Convert to byte array
byte[] newArray = new byte[decodedBuffer.remaining()];
decodedBuffer.rewind();
decodedBuffer.get(newArray);
// the image data should in the size of YUV which is a
// 3byte variable
byte[] imageData = new byte[(mWidth * mHeight * 3) / 2];
// if(VERBOSE) Log.d(UnityAppPlayer.TAG,
// "Encoding RGBA to YUV");
covertToYUV(imageData, newArray, mWidth, mHeight);
// if(VERBOSE) Log.d(UnityAppPlayer.TAG,
// "Perfom encode");
// Perform Encoding
EncodeFrame(imageData, timeStamp + (long) addition);
//}
/*else {
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(UnityAppPlayer.TAG,
"INFO_OUTPUT_BUFFERS_CHANGED");
outputBuffers = decoder.getOutputBuffers();
Log.d(UnityAppPlayer.TAG,
"New output buffer size = "
+ outputBuffers.length);
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.d(UnityAppPlayer.TAG,
"New format " + decoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.d(UnityAppPlayer.TAG,
"dequeueOutputBuffer timed out! Index: "
+ outIndex);
break;
default:
Log.d(UnityAppPlayer.TAG,
"Found output buffer Index! " + outIndex);
break;
}
}*/
}
}
else{
Log.d(UnityAppPlayer.TAG,"outIndex: "+outIndex);
}
}
}
// update the previous presentation stamp here
prevPresentationTimeUs = presentationTimeUs;
// Log
//Log.d(UnityAppPlayer.TAG, "Looping");
}
Log.d(UnityAppPlayer.TAG, "Finished Encoding");
extractor.release();
extractor = null;
Log.d(UnityAppPlayer.TAG, "Time to hint at some garbage collection");
System.gc();
UnityPlayer.currentActivity.runOnUiThread(new Runnable() {
@Override
public void run() {
mExportDialog.SetProgress(100);
}
});
}
Глядя на PresentationTime исходного видео из extractor.getSampleTime(), я вижу, что время между кадрами составляет 0,16 с. Для видео со скоростью 25 кадров в секунду я бы не ожидал увидеть 0,04 секунды между кадрами? Я пытался изменить время презентации, но это не имело значения.
Проблема, по-видимому, связана с функцией кодирования, а не с конечным MUX-mp4, поскольку необработанный файл H264 показывает ту же проблему, то есть воспроизводит слишком быстро.
Кто-нибудь может подсказать, что не так с кодом выше?
РЕДАКТИРОВАТЬ: добавлен код функции EncodeFrame:
void EncodeFrame(byte[] data, long inPTS) {
// Log.d(UnityAppPlayer.TAG, "Getiing input/output buffers");
ByteBuffer[] inputBuffers = mMediaCodec.getInputBuffers();
ByteBuffer[] outputBuffers = mMediaCodec.getOutputBuffers();
// get the input buffer index to use
int inBuffIndex = mMediaCodec.dequeueInputBuffer(-1);
// if we have an invalid buffer index then there's something wrong
if (inBuffIndex < 0) {
Log.e(UnityAppPlayer.TAG, "No input buffer available");
return;
}
// Log
// Log.d(UnityAppPlayer.TAG, "Clearing input buffer");
// put our frame data into the the input buffer we've been given
inputBuffers[inBuffIndex].clear();
// Log
// Log.d(UnityAppPlayer.TAG, "Putting data of size: " + data.length +
// " into buffer, index: " + inBuffIndex + " and size:" +
// inputBuffers[inBuffIndex].limit());
// Insert the Data into the buffer
inputBuffers[inBuffIndex].put(data);
// Log
// Log.d(UnityAppPlayer.TAG, "Queuing input data");
// queue the inputbuffer
mMediaCodec.queueInputBuffer(inBuffIndex, 0, data.length, inPTS, 0);
// For testing increase the pts by 1/FPS * 1,000,000 to convert the
// value into microseconds
// mCurrentPts += Math.round((1.0f/25.0f)*1000000.0f);
MediaCodec.BufferInfo buffInfo = new MediaCodec.BufferInfo();
int outBuffIndex = mMediaCodec.dequeueOutputBuffer(buffInfo, 0);
// use a do while loop as we need to check if the output buffers have
// changed
do {
// if we have a valid buffer index
if (outBuffIndex >= 0) {
// get the data from our output buffer
// Log.d(UnityAppPlayer.TAG,
// "Getting data from buffer, with index: "+ outBuffIndex
// +" and size:" + outputBuffers[outBuffIndex].limit());
byte[] outData = new byte[buffInfo.size];
outputBuffers[outBuffIndex].get(outData);
try {
// if we have an offset write to the file using the offset
if (buffInfo.offset != 0) {
Log.d(UnityAppPlayer.TAG, "Writing data with offset");
mFileStream.write(outData, buffInfo.offset,
outData.length);
} else {
// Log.d(UnityAppPlayer.TAG, "Writing data");
mFileStream.write(outData);
}
mFileStream.flush();
// Log.d(UnityAppPlayer.TAG, "Releasing output Buffer");
// release the output buffer
mMediaCodec.releaseOutputBuffer(outBuffIndex, false);
// Log.d(UnityAppPlayer.TAG, "Getting next output index");
// check if there's another buffer that has an output for us
outBuffIndex = mMediaCodec.dequeueOutputBuffer(buffInfo, 0);
} catch (IOException e) {
Log.e(UnityAppPlayer.TAG, "Error writing to output file");
e.printStackTrace();
}
}
// if the buffer index isn't valid, check if it means the output
// buffers have changed
else if (outBuffIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
// Log.d(UnityAppPlayer.TAG, "Output buffers have changed");
// get the output buffers again
outputBuffers = mMediaCodec.getOutputBuffers();
}
} while (outBuffIndex >= 0);
}
1 ответ
Отметки времени представления для каждого кадра находятся в мультиплексированном файле.mp4. Они не существуют в файле H.264. MediaCodec просто передает метку времени вместе с кадром, чтобы сохранить связь; это важно для кодеков, которые могут создавать кадры не по порядку.
Код, который вам нужно посмотреть, это та часть, которая вызывает MediaMuxer#writeSampleData() в частности, значение времени представления в BufferInfo объект. Убедитесь, что вы генерируете значения PTS для исходных кадров, и что PTS, связанный с каждым кадром, переданным кодировщику, получается с закодированным кадром и пересылается в MediaMuxer.