[graphics/krita/krita/6.0] /: [android] Animation audio exporting
Carsten Hartenfels <[email protected]>
| Newsgroups | gmane.comp.kde.cvs |
|---|---|
| Message-ID | <[email protected]> |
Git commit 466f33c8641df73d2d0865c492b60fbec7187fca by Carsten Hartenfels.
Committed on 27/07/2026 at 17:19.
Pushed by hartenfels into branch 'krita/6.0'.
[android] Animation audio exporting
Uses MLT to read the samples from the input audio file and pipes that
data into the Android media encoder. The sample format, sampling rate,
and channel count are taken straight from the input file, we don't
attempt to resample them.
M +644 -136 libs/ui/animation/KisAndroidMediaEncoderRunnable.cpp
M +30 -12 libs/ui/animation/KisAndroidMediaEncoderRunnable.h
M +7 -0 libs/ui/animation/KisMediaEncoderWrapper.h
M +21 -0 libs/ui/animation/KisVideoSaver.cpp
M +533 -140 packaging/android/apk/src/org/krita/android/VideoEncoder.java
M +2 -0 plugins/dockers/recorder/recorder_export.cpp
https://invent.kde.org/graphics/krita/-/commit/466f33c8641df73d2d0865c492b60fbec7187fca
diff --git a/libs/ui/animation/KisAndroidMediaEncoderRunnable.cpp b/libs/ui/animation/KisAndroidMediaEncoderRunnable.cpp
index 7175f5d7acb..28aae9c036a 100644
--- a/libs/ui/animation/KisAndroidMediaEncoderRunnable.cpp
+++ b/libs/ui/animation/KisAndroidMediaEncoderRunnable.cpp
@@ -10,6 +10,7 @@
#include <QImage>
#include <QSpinBox>
#include <QTemporaryFile>
+#include <memory>
#if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)
#include <QJniEnvironment>
@@ -26,6 +27,8 @@ using QJniObject = QAndroidJniObject;
#include <KisAndroidUtils.h>
#include <kis_debug.h>
+#include <mlt++/Mlt.h>
+
extern "C" {
#include <libavutil/imgutils.h>
#include <libavutil/pixfmt.h>
@@ -53,22 +56,12 @@ public:
bool hardware;
};
- Format(int formatId, const QVector<Encoder> &encoders)
+ Format(int formatId, const QVector<Encoder> &videoEncoders, const QVector<Encoder> &audioEncoders)
: KisMediaEncoderFormat()
, m_formatId(formatId)
{
- // Prefer software encoders, because hardware encoders are often busted.
- m_encoders.reserve(encoders.size());
- for (const Encoder &encoder : encoders) {
- if (!encoder.hardware) {
- m_encoders.append(encoder);
- }
- }
- for (const Encoder &encoder : encoders) {
- if (encoder.hardware) {
- m_encoders.append(encoder);
- }
- }
+ fillEncoders(m_videoEncoders, videoEncoders);
+ fillEncoders(m_audioEncoders, audioEncoders);
}
int formatId() const
@@ -98,21 +91,19 @@ public:
bool supportsAudio() const override
{
- return true;
+ return !m_audioEncoders.isEmpty();
}
QWidget *createPreferencesWidget(const QVariantMap &preferences) const override
{
- KisAndroidMediaEncoderPreferencesWidget *pw = new KisAndroidMediaEncoderPreferencesWidget;
+ KisAndroidMediaEncoderPreferencesWidget *pw = new KisAndroidMediaEncoderPreferencesWidget(this);
- for (const Encoder &encoder : m_encoders) {
- QString title;
- if (encoder.hardware) {
- title = i18n("%1 (hardware)", encoder.name);
- } else {
- title = i18n("%1 (software)", encoder.name);
- }
- pw->addEncoderOption(title, encoder.name);
+ for (const Encoder &videoEncoder : m_videoEncoders) {
+ pw->addVideoEncoderOption(makeEncoderTitle(videoEncoder), videoEncoder.name);
+ }
+
+ for (const Encoder &audioEncoder : m_audioEncoders) {
+ pw->addAudioEncoderOption(makeEncoderTitle(audioEncoder), audioEncoder.name);
}
applyPreferencesToWidget(pw, preferences);
@@ -131,36 +122,105 @@ public:
QVariantMap preferences;
KisAndroidMediaEncoderPreferencesWidget *pw = qobject_cast<KisAndroidMediaEncoderPreferencesWidget *>(widget);
KIS_SAFE_ASSERT_RECOVER_RETURN_VALUE(pw, preferences);
- preferences.insert(QStringLiteral("encoder"), pw->encoder());
- preferences.insert(QStringLiteral("bitrate"), pw->bitrate());
+ preferences.insert(QStringLiteral("encoder"), pw->videoEncoder());
+ preferences.insert(QStringLiteral("bitrate"), pw->videoBitrate());
+ if (supportsAudio()) {
+ preferences.insert(QStringLiteral("aencoder"), pw->audioEncoder());
+ preferences.insert(QStringLiteral("abitrate"), pw->audioBitrate());
+ }
return preferences;
}
- QString getEncoderPreference(const QVariantMap &preferences) const
+ QString getVideoEncoderPreference(const QVariantMap &preferences) const
{
- QString encoder = preferences.value(QStringLiteral("encoder")).toString();
- if (encoder.isEmpty() && !m_encoders.isEmpty()) {
- return m_encoders.constFirst().name;
+ QString videoEncoder = preferences.value(QStringLiteral("encoder")).toString();
+ if (videoEncoder.isEmpty() && !m_videoEncoders.isEmpty()) {
+ return m_videoEncoders.constFirst().name;
} else {
- return encoder;
+ return videoEncoder;
}
}
- int getBitratePreference(const QVariantMap &preferences) const
+ int getVideoBitratePreference(const QVariantMap &preferences) const
{
- int bitrate = preferences.value(QStringLiteral("bitrate")).toInt();
- if (bitrate <= 0) {
- return defaultBitrateForFormatId(m_formatId);
+ int videoBitrate = preferences.value(QStringLiteral("bitrate")).toInt();
+ if (videoBitrate <= 0) {
+ return defaultVideoBitrateForFormatId(m_formatId);
} else {
- return bitrate;
+ return videoBitrate;
+ }
+ }
+
+ QString getAudioEncoderPreference(const QVariantMap &preferences) const
+ {
+ QString audioEncoder = preferences.value(QStringLiteral("aencoder")).toString();
+ if (audioEncoder.isEmpty() && !m_videoEncoders.isEmpty()) {
+ return m_audioEncoders.constFirst().name;
+ } else {
+ return audioEncoder;
+ }
+ }
+
+ int getAudioBitratePreference(const QVariantMap &preferences) const
+ {
+ int audioBitrate = preferences.value(QStringLiteral("abitrate")).toInt();
+ if (audioBitrate <= 0) {
+ return defaultAudioBitrateForFormatId(m_formatId);
+ } else {
+ return audioBitrate;
+ }
+ }
+
+ int audioSampleRate() const
+ {
+ // While creating an audio track on Android lets you pass it a sample
+ // rate, that doesn't actually give you an audio track with the given
+ // sample rate. For WEBM with Vorbis, you always get a 48kHz track. MP4
+ // with AAC seems more forgiving, but let's go with known good values.
+ switch (m_formatId) {
+ case FORMAT_MP4_H264:
+ case FORMAT_MP4_AV1:
+ return 44100;
+ case FORMAT_WEBM_VP8:
+ return 48000;
}
+ return 44100;
}
private:
+ static void fillEncoders(QVector<Encoder> &outEncoders, const QVector<Encoder> &encoders)
+ {
+ // Prefer software encoders, because hardware encoders are often busted.
+ outEncoders.reserve(encoders.size());
+ for (const Encoder &encoder : encoders) {
+ if (!encoder.hardware) {
+ outEncoders.append(encoder);
+ }
+ }
+ for (const Encoder &encoder : encoders) {
+ if (encoder.hardware) {
+ outEncoders.append(encoder);
+ }
+ }
+ }
+
+ static QString makeEncoderTitle(const Encoder &encoder)
+ {
+ if (encoder.hardware) {
+ return i18n("%1 (hardware)", encoder.name);
+ } else {
+ return i18n("%1 (software)", encoder.name);
+ }
+ }
+
void applyPreferencesToWidget(KisAndroidMediaEncoderPreferencesWidget *pw, const QVariantMap &preferences) const
{
- pw->setEncoder(getEncoderPreference(preferences));
- pw->setBitrate(getBitratePreference(preferences));
+ pw->setVideoEncoder(getVideoEncoderPreference(preferences));
+ pw->setVideoBitrate(getVideoBitratePreference(preferences));
+ if (supportsAudio()) {
+ pw->setAudioEncoder(getAudioEncoderPreference(preferences));
+ pw->setAudioBitrate(getAudioBitratePreference(preferences));
+ }
}
static QString keyForFormatId(int formatId)
@@ -201,7 +261,7 @@ private:
return QString();
}
- static int defaultBitrateForFormatId(int formatId)
+ static int defaultVideoBitrateForFormatId(int formatId)
{
switch (formatId) {
case FORMAT_MP4_H264:
@@ -214,7 +274,20 @@ private:
return 6000000;
}
- QVector<Encoder> m_encoders;
+ static int defaultAudioBitrateForFormatId(int formatId)
+ {
+ switch (formatId) {
+ case FORMAT_MP4_H264:
+ case FORMAT_MP4_AV1:
+ return 128000;
+ case FORMAT_WEBM_VP8:
+ return 96000;
+ }
+ return 128000;
+ }
+
+ QVector<Encoder> m_videoEncoders;
+ QVector<Encoder> m_audioEncoders;
int m_formatId;
};
@@ -275,11 +348,18 @@ public:
{
if (checkException(title)) {
return true;
- } else if (result == STATUS_ERROR_START_ENCODER) {
- warnFile << "Start encoder error" << result;
+ } else if (result == STATUS_ERROR_START_VIDEO_ENCODER) {
+ warnFile << "Start video encoder error" << result;
setErrorMessage(i18n("Unsupported video parameters, try lowering the video FPS or size"));
return true;
- } else if (result == STATUS_ERROR_DRAIN_MUXER_ADD_TRACK) {
+ } else if (result == STATUS_ERROR_START_AUDIO_FORMAT || result == STATUS_ERROR_START_AUDIO_ENCODER) {
+ warnFile << "Start audio encoder error" << result;
+ setErrorMessage(
+ i18n("Unsupported audio parameters, try to re-encode your audio file with a more common sample format, "
+ "sampling rate and channel count"));
+ return true;
+ } else if (result == STATUS_ERROR_DRAIN_VIDEO_MUXER_ADD_TRACK
+ || result == STATUS_ERROR_DRAIN_AUDIO_MUXER_ADD_TRACK) {
warnFile << "Muxer track error" << result;
setErrorMessage(i18n("Unsupported format"));
return true;
@@ -350,6 +430,194 @@ private:
AVPixelFormat m_imageFormat = AV_PIX_FMT_NONE;
};
+class KisAndroidMediaEncoderRunnable::Audio
+{
+public:
+ static constexpr int PULL_FRAME_OK = 0;
+ static constexpr int PULL_FRAME_END_OF_STREAM = 1;
+ static constexpr int PULL_FRAME_ERROR = 2;
+
+ Audio(mlt_audio_format format, int sampleRate, int channelCount)
+ : m_format(format)
+ , m_sampleRate(sampleRate)
+ , m_channelCount(channelCount)
+ , m_sampleSize(mlt_audio_format_size(format, 1, 1))
+ {
+ }
+
+ QString formatName() const
+ {
+ return QString::fromUtf8(mlt_audio_format_name(m_format));
+ }
+
+ int sampleRate() const
+ {
+ return m_sampleRate;
+ }
+
+ int channelCount() const
+ {
+ return m_channelCount;
+ }
+
+ bool isFinished() const
+ {
+ return m_finished;
+ }
+
+ bool open(Context &ctx, const KisMediaEncoderWrapperSettings &settings)
+ {
+ m_audioFileBytes = settings.audioFile.toUtf8();
+
+ m_profile = std::make_unique<Mlt::Profile>();
+ m_profile->set_frame_rate(settings.outputFps, 1);
+
+ m_producer = std::make_unique<Mlt::Producer>(*m_profile, "avformat", m_audioFileBytes.constData());
+ if (!m_producer->is_valid()) {
+ ctx.setInternalErrorMessage(QStringLiteral("failed to open MLT producer for '%1'").arg(settings.audioFile));
+ return false;
+ }
+
+ m_filter = std::make_unique<Mlt::Filter>(*m_profile, "swresample");
+ if (!m_filter->is_valid()) {
+ ctx.setInternalErrorMessage(QStringLiteral("failed to create MLT filter"));
+ return false;
+ }
+
+ int result = m_producer->attach(*m_filter);
+ if (result != 0) {
+ ctx.setInternalErrorMessage(QStringLiteral("error %1 attaching MLT filter").arg(result));
+ return false;
+ }
+
+ m_producer->seek(settings.audioSeekFrame);
+
+ return true;
+ }
+
+ int pullFrame(Context &ctx)
+ {
+ if (m_frame) {
+ return PULL_FRAME_OK;
+ }
+
+ m_frame.reset(m_producer->get_frame());
+ if (!m_frame || !m_frame->is_valid()) {
+ m_frame.reset();
+ return PULL_FRAME_END_OF_STREAM;
+ }
+
+ mlt_audio_format format = m_format;
+ int sampleRate = m_sampleRate;
+ int channelCount = m_channelCount;
+
+ float fps = float(m_profile->fps());
+ int64_t position = m_frame->get_position();
+ int sampleCount = mlt_audio_calculate_frame_samples(fps, sampleRate, position);
+ const void *sampleData = m_frame->get_audio(format, sampleRate, channelCount, sampleCount);
+ if (sampleCount <= 0 || !sampleData) {
+ m_frame.reset();
+ return PULL_FRAME_END_OF_STREAM;
+ }
+
+ if (format != m_format || sampleRate != m_sampleRate || channelCount != m_channelCount) {
+ ctx.setInternalErrorMessage(QStringLiteral("bad MLT frame format %1:%2:%3 != not %4:%5:%6")
+ .arg(int(format))
+ .arg(sampleRate)
+ .arg(channelCount)
+ .arg(int(m_format))
+ .arg(m_sampleRate)
+ .arg(m_channelCount));
+ m_frame.reset();
+ return PULL_FRAME_ERROR;
+ }
+
+ m_sampleCount = sampleCount;
+ m_sampleData = reinterpret_cast<const unsigned char *>(sampleData);
+ m_samplePos = 0;
+ return PULL_FRAME_OK;
+ }
+
+ bool pushSamples(Context &ctx)
+ {
+ void *buffer;
+ int availableSize;
+ if (!readAudioInputBuffer(ctx, buffer, availableSize)) {
+ return false;
+ }
+
+ int availableSamples = sizeToSamples(availableSize);
+ if (availableSamples <= 0) {
+ ctx.setInternalErrorMessage(QStringLiteral("no available samples from size %1").arg(availableSize));
+ return false;
+ }
+
+ int remainingSamples = m_sampleCount - m_samplePos;
+ int chunkSamples = qMin(availableSamples, remainingSamples);
+ int chunkSize = samplesToSize(chunkSamples);
+ memcpy(buffer, m_sampleData + samplesToSize(m_samplePos), chunkSize);
+
+ int commitResult =
+ int(ctx.encoder().callMethod<jint>("commitAudio", "(II)I", jint(chunkSamples), jint(chunkSize)));
+ if (ctx.checkResult(QStringLiteral("commitAudio"), commitResult)) {
+ return false;
+ }
+
+ m_samplePos += chunkSamples;
+ return true;
+ }
+
+ bool isSampleDataRemaining() const
+ {
+ return m_samplePos < m_sampleCount;
+ }
+
+ void finishFrame()
+ {
+ m_frame.reset();
+ m_sampleData = nullptr;
+ m_sampleCount = 0;
+ m_samplePos = 0;
+ }
+
+ bool finish(Context &ctx)
+ {
+ if (!m_finished) {
+ int finishAudioResult = int(ctx.encoder().callMethod<jint>("finishAudio", "()I"));
+ if (ctx.checkResult(QStringLiteral("finishAudio"), finishAudioResult)) {
+ return false;
+ }
+ m_finished = true;
+ }
+ return true;
+ }
+
+private:
+ int sizeToSamples(int size) const
+ {
+ return size / (m_channelCount * m_sampleSize);
+ }
+
+ int samplesToSize(int samples) const
+ {
+ return samples * m_channelCount * m_sampleSize;
+ }
+
+ QByteArray m_audioFileBytes;
+ std::unique_ptr<Mlt::Profile> m_profile;
+ std::unique_ptr<Mlt::Producer> m_producer;
+ std::unique_ptr<Mlt::Filter> m_filter;
+ std::unique_ptr<Mlt::Frame> m_frame;
+ const unsigned char *m_sampleData = nullptr;
+ const mlt_audio_format m_format;
+ const int m_sampleRate;
+ const int m_channelCount;
+ const int m_sampleSize;
+ int m_sampleCount = 0;
+ int m_samplePos = 0;
+ bool m_finished = false;
+};
+
KisAndroidMediaEncoderRunnable *KisAndroidMediaEncoderRunnable::create(const KisMediaEncoderWrapperSettings &settings,
QObject *parent)
{
@@ -393,6 +661,33 @@ KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::encode(QSt
int outputWidth = settings().outputSize.width();
int outputHeight = settings().outputSize.height();
+ // Set up audio if we were given some.
+ std::unique_ptr<Audio> audio;
+ if (!settings().audioFile.isEmpty()) {
+ // We always use these fixed parameters: 16 bit PCM, 2 channels and a
+ // sample rate according to the format. The Android media encoder makes
+ // it appear like you can create audio tracks with other parameters, but
+ // if it doesn't like the parameters it just gives you something else,
+ // which obviously just results in nonsense. We'll go with known good
+ // parameters instead, they'll also play back consistently everywhere.
+ audio = std::make_unique<Audio>(mlt_audio_s16, format->audioSampleRate(), 2);
+ if (!audio->open(ctx, settings())) {
+ return EncodeResult::Failed;
+ }
+
+ switch (audio->pullFrame(ctx)) {
+ case Audio::PULL_FRAME_OK:
+ break;
+ case Audio::PULL_FRAME_END_OF_STREAM:
+ // There's no frames, which can happen if our animation range start
+ // is beyond the end of the file. Just export without audio then.
+ audio.reset();
+ break;
+ default:
+ return EncodeResult::Failed;
+ }
+ }
+
// Set up the encoder.
{
QJniObject outputPath = QJniObject::fromString(settings().outputFile);
@@ -405,19 +700,42 @@ KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::encode(QSt
return EncodeResult::Failed;
}
- QJniObject encoderName = QJniObject::fromString(format->getEncoderPreference(settings().formatPreferences));
- ctx.checkException(QStringLiteral("encoderName"));
+ QJniObject videoEncoderName =
+ QJniObject::fromString(format->getVideoEncoderPreference(settings().formatPreferences));
+ ctx.checkException(QStringLiteral("videoEncoderName"));
+
+ jint audioSampleRate;
+ jint audioChannelCount;
+ QJniObject audioEncoderName;
+ QJniObject audioFormatName;
+ if (audio) {
+ audioSampleRate = jint(audio->sampleRate());
+ audioChannelCount = jint(audio->channelCount());
+ audioEncoderName = QJniObject::fromString(format->getAudioEncoderPreference(settings().formatPreferences));
+ ctx.checkException(QStringLiteral("audioEncoderName"));
+ audioFormatName = QJniObject::fromString(audio->formatName());
+ ctx.checkException(QStringLiteral("audioFormatName"));
+ } else {
+ audioSampleRate = jint(0);
+ audioChannelCount = jint(0);
+ }
- ctx.setEncoder(QJniObject("org/krita/android/VideoEncoder",
- "(IIIFLjava/lang/String;Ljava/lang/String;Ljava/lang/String;I)V",
- jint(format->formatId()),
- jint(outputWidth),
- jint(outputHeight),
- jfloat(settings().outputFps),
- outputPath.object<jstring>(),
- tempPath.object<jstring>(),
- encoderName.object<jstring>(),
- jint(format->getBitratePreference(settings().formatPreferences))));
+ ctx.setEncoder(QJniObject(
+ "org/krita/android/VideoEncoder",
+ "(IIIFLjava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/lang/String;Ljava/lang/String;III)V",
+ jint(format->formatId()),
+ jint(outputWidth),
+ jint(outputHeight),
+ jfloat(settings().outputFps),
+ outputPath.object<jstring>(),
+ tempPath.object<jstring>(),
+ videoEncoderName.object<jstring>(),
+ jint(format->getVideoBitratePreference(settings().formatPreferences)),
+ audio ? audioEncoderName.object<jstring>() : nullptr,
+ audio ? audioFormatName.object<jstring>() : nullptr,
+ jint(audioSampleRate),
+ jint(audioChannelCount),
+ jint(format->getAudioBitratePreference(settings().formatPreferences))));
if (ctx.checkObject(QStringLiteral("encoder"), ctx.encoder())) {
return EncodeResult::Failed;
}
@@ -453,17 +771,38 @@ KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::encode(QSt
// Grab the next frame from disk.
QImage inputImage;
+ if (!frame.readImage(inputImage)) {
+ // If we don't have audio, we can just skip the frame and hope it's
+ // okay. In timelapses it often is, you're not gonna notice a single
+ // frame missing in the video. However, if we have an animation with
+ // audio, skipping a frame would make a mess and be hard to resync,
+ // so in that case we just give up and bail out.
+ if (audio) {
+ ctx.setInternalErrorMessage(QStringLiteral("failed to read frame %1").arg(frame.path()));
+ return EncodeResult::Failed;
+ } else {
+ continue;
+ }
+ }
+
AVPixelFormat inputPixelFormat;
- if (!frame.readImage(inputImage) || !ctx.convertFrame(inputImage, inputPixelFormat)) {
- continue; // Keep going, some frames may be corrupted.
+ if (!ctx.convertFrame(inputImage, inputPixelFormat)) {
+ // Dito to the above, try to skip frames we don't understand, don't
+ // bother when there's audio involved.
+ if (audio) {
+ ctx.setInternalErrorMessage(QStringLiteral("failed to convert frame %1").arg(frame.path()));
+ return EncodeResult::Failed;
+ } else {
+ continue;
+ }
}
int instances = frame.instances();
for (int i = 0; i < instances; ++i) {
// Grab a buffer from the encoder.
- EncodeResult prepareResult = prepare(ctx);
- if (prepareResult != EncodeResult::Completed) {
- return prepareResult;
+ EncodeResult prepareVideoResult = prepareVideo(ctx);
+ if (prepareVideoResult != EncodeResult::Completed) {
+ return prepareVideoResult;
}
// Retrieve the buffer layout.
@@ -566,10 +905,40 @@ KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::encode(QSt
outputHeight);
}
- int commitResult = int(ctx.encoder().callMethod<jint>("commit", "()I"));
- if (ctx.checkResult(QStringLiteral("commit"), commitResult)) {
+ int commitResult = int(ctx.encoder().callMethod<jint>("commitVideo", "()I"));
+ if (ctx.checkResult(QStringLiteral("commitVideo"), commitResult)) {
return EncodeResult::Failed;
}
+
+ if (audio && !audio->isFinished()) {
+ int pullFrameResult = audio->pullFrame(ctx);
+ if (pullFrameResult == Audio::PULL_FRAME_OK) {
+ do {
+ EncodeResult prepareAudioResult = prepareAudio(ctx);
+ if (prepareAudioResult != EncodeResult::Completed) {
+ return prepareAudioResult;
+ }
+
+ if (!audio->pushSamples(ctx)) {
+ return EncodeResult::Failed;
+ }
+ } while (audio->isSampleDataRemaining());
+ audio->finishFrame();
+
+ } else if (pullFrameResult == Audio::PULL_FRAME_END_OF_STREAM) {
+ EncodeResult prepareAudioResult = prepareAudio(ctx);
+ if (prepareAudioResult != EncodeResult::Completed) {
+ return prepareAudioResult;
+ }
+
+ if (!audio->finish(ctx)) {
+ return EncodeResult::Failed;
+ }
+
+ } else {
+ return EncodeResult::Failed;
+ }
+ }
}
}
@@ -577,34 +946,63 @@ KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::encode(QSt
return EncodeResult::Cancelled;
}
- // Finish the encoder stream.
+ // Finish the encoder streams.
{
// Need a buffer from the encoder to tell it that it's done.
- EncodeResult prepareResult = prepare(ctx);
- if (prepareResult != EncodeResult::Completed) {
- return prepareResult;
+ EncodeResult prepareVideoResult = prepareVideo(ctx);
+ if (prepareVideoResult != EncodeResult::Completed) {
+ return prepareVideoResult;
}
- // Hand an empty buffer back with the end of stream flag set.
- int finishResult = int(ctx.encoder().callMethod<jint>("finish", "()I"));
- if (ctx.checkResult(QStringLiteral("finish"), finishResult)) {
+
+ // Hand empty buffer back with the end of stream flag set.
+ int finishVideoResult = int(ctx.encoder().callMethod<jint>("finishVideo", "()I"));
+ if (ctx.checkResult(QStringLiteral("finishVideo"), finishVideoResult)) {
return EncodeResult::Failed;
}
- }
- // Drain all remaining frames out of the encoder.
- while (true) {
- int drainResult = drain(ctx, 1000000LL);
- if (drainResult == DRAIN_END_OF_STREAM) {
- break;
- } else if (drainResult == DRAIN_ERROR) {
- return EncodeResult::Failed;
- } else if (drainResult == DRAIN_CANCELLED) {
- return EncodeResult::Cancelled;
- } else {
- KIS_SAFE_ASSERT_RECOVER_NOOP(drainResult >= 0);
+ // Same procedure with audio, but it might already have finished.
+ if (audio && !audio->isFinished()) {
+ EncodeResult prepareAudioResult = prepareAudio(ctx);
+ if (prepareAudioResult != EncodeResult::Completed) {
+ return prepareAudioResult;
+ }
+
+ if (!audio->finish(ctx)) {
+ return EncodeResult::Failed;
+ }
}
}
+ // Drain all remaining frames and samples out of the encoders.
+ bool videoStreamEnded = false;
+ bool audioStreamEnded = !audio;
+ do {
+ if (!videoStreamEnded) {
+ int drainVideoResult = drainVideo(ctx, 1000000LL);
+ if (drainVideoResult == DRAIN_END_OF_STREAM) {
+ videoStreamEnded = true;
+ } else if (drainVideoResult == DRAIN_ERROR) {
+ return EncodeResult::Failed;
+ } else if (drainVideoResult == DRAIN_CANCELLED) {
+ return EncodeResult::Cancelled;
+ } else {
+ KIS_SAFE_ASSERT_RECOVER_NOOP(drainVideoResult >= 0);
+ }
+ }
+ if (!audioStreamEnded) {
+ int drainAudioResult = drainAudio(ctx, 1000000LL);
+ if (drainAudioResult == DRAIN_END_OF_STREAM) {
+ audioStreamEnded = true;
+ } else if (drainAudioResult == DRAIN_ERROR) {
+ return EncodeResult::Failed;
+ } else if (drainAudioResult == DRAIN_CANCELLED) {
+ return EncodeResult::Cancelled;
+ } else {
+ KIS_SAFE_ASSERT_RECOVER_NOOP(drainAudioResult >= 0);
+ }
+ }
+ } while (!videoStreamEnded || !audioStreamEnded);
+
// Close the encoder, copy the temporary file to the output file if needed.
{
int closeResult = int(ctx.encoder().callMethod<jint>("close", "()I"));
@@ -628,21 +1026,35 @@ KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::encode(QSt
return EncodeResult::Completed;
}
-KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::prepare(Context &ctx)
+KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::prepareVideo(Context &ctx)
{
+ return prepare(ctx, false);
+}
+
+KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::prepareAudio(Context &ctx)
+{
+ return prepare(ctx, true);
+}
+
+KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::prepare(Context &ctx, bool audio)
+{
+ const char *methodName = audio ? "prepareAudio" : "prepareVideo";
+ QString title = audio ? QStringLiteral("prepareAudio") : QStringLiteral("prepareVideo");
+
while (true) {
if (isCancelled()) {
return EncodeResult::Cancelled;
}
- int prepareResult = int(ctx.encoder().callMethod<jint>("prepare", "(J)I", jlong(100000LL)));
- if (ctx.checkResult(QStringLiteral("prepare"), prepareResult)) {
+ int prepareResult = int(ctx.encoder().callMethod<jint>(methodName, "(J)I", jlong(100000LL)));
+ if (ctx.checkResult(title, prepareResult)) {
return EncodeResult::Failed;
} else if (prepareResult == STATUS_TIMEOUT) {
- int drainResult = drain(ctx, 0LL);
+ int drainResult = drain(ctx, 0LL, audio);
if (drainResult == DRAIN_END_OF_STREAM) {
- ctx.setInternalErrorMessage(QStringLiteral("unexpected end of stream"));
+ ctx.setInternalErrorMessage(QStringLiteral("unexpected end of %1 stream")
+ .arg(audio ? QStringLiteral("audio") : QStringLiteral("video")));
return EncodeResult::Failed;
} else if (drainResult == DRAIN_ERROR) {
return EncodeResult::Failed;
@@ -660,8 +1072,21 @@ KisMediaEncoderRunnable::EncodeResult KisAndroidMediaEncoderRunnable::prepare(Co
return EncodeResult::Completed;
}
-int KisAndroidMediaEncoderRunnable::drain(Context &ctx, long long initialTimeout)
+int KisAndroidMediaEncoderRunnable::drainVideo(Context &ctx, long long initialTimeout)
+{
+ return drain(ctx, initialTimeout, false);
+}
+
+int KisAndroidMediaEncoderRunnable::drainAudio(Context &ctx, long long initialTimeout)
+{
+ return drain(ctx, initialTimeout, true);
+}
+
+int KisAndroidMediaEncoderRunnable::drain(Context &ctx, long long initialTimeout, bool audio)
{
+ const char *methodName = audio ? "drainAudio" : "drainVideo";
+ QString title = audio ? QStringLiteral("drainAudio") : QStringLiteral("drainVideo");
+
int count = 0;
long long timeout = initialTimeout;
while (true) {
@@ -669,9 +1094,9 @@ int KisAndroidMediaEncoderRunnable::drain(Context &ctx, long long initialTimeout
return DRAIN_CANCELLED;
}
- int drainResult = int(ctx.encoder().callMethod<jint>("drain", "(J)I", jlong(timeout)));
+ int drainResult = int(ctx.encoder().callMethod<jint>(methodName, "(J)I", jlong(timeout)));
- if (ctx.checkResult(QStringLiteral("drain"), drainResult)) {
+ if (ctx.checkResult(title, drainResult)) {
return DRAIN_ERROR;
} else if (drainResult == STATUS_TIMEOUT) {
@@ -742,95 +1167,178 @@ bool KisAndroidMediaEncoderRunnable::readPlanePixelStride(Context &ctx, int inde
return true;
}
+bool KisAndroidMediaEncoderRunnable::readAudioInputBuffer(Context &ctx, void *&outBuffer, int &outSize)
+{
+ QJniObject audioBuffer = ctx.encoder().callObjectMethod("getInputAudioBuffer", "()Ljava/nio/ByteBuffer;", jint());
+ if (ctx.checkObject(QStringLiteral("audioBuffer"), audioBuffer)) {
+ return false;
+ }
+
+ void *buffer = ctx.env()->GetDirectBufferAddress(audioBuffer.object<jobject>());
+ if (!buffer) {
+ ctx.setInternalErrorMessage(QStringLiteral("null audio buffer"));
+ return false;
+ }
+
+ jint size = audioBuffer.callMethod<jint>("remaining", "()I");
+ if (ctx.checkException(QStringLiteral("remaining"))) {
+ return false;
+ }
+
+ if (size <= 0) {
+ ctx.setInternalErrorMessage(QStringLiteral("audio buffer with size %1").arg(size));
+ return false;
+ }
+
+ outBuffer = buffer;
+ outSize = size;
+ return true;
+}
+
void KisAndroidMediaEncoderRunnable::checkFormatSupport(Context &ctx,
int formatId,
QVector<KisMediaEncoderFormat *> &outSupportedFormats)
{
- QJniObject supports = QJniObject::callStaticObjectMethod("org/krita/android/VideoEncoder",
- "getSupportsForFormat",
- "(I)Ljava/util/List;",
- jint(formatId));
- if (ctx.checkObject(QStringLiteral("supports"), supports)) {
- return;
- }
+ QVector<Format::Encoder> videoEncoders;
+ QVector<Format::Encoder> audioEncoders;
- jint count = supports.callMethod<jint>("size", "()I");
- if (ctx.checkException(QStringLiteral("size"))) {
- return;
- }
+ const QPair<QVector<Format::Encoder> *, const char *> ps[] = {
+ {&videoEncoders, "getSupportsForVideoFormat"},
+ {&audioEncoders, "getSupportsForAudioFormat"},
+ };
- QVector<Format::Encoder> encoders;
- for (jint i = 0; i < count; ++i) {
- QJniObject entry = supports.callObjectMethod("get", "(I)Ljava/lang/Object;", i);
- if (ctx.checkObject(QStringLiteral("entry"), entry)) {
- continue;
+ for (const QPair<QVector<Format::Encoder> *, const char *> &p : ps) {
+ QJniObject supports = QJniObject::callStaticObjectMethod("org/krita/android/VideoEncoder",
+ p.second,
+ "(I)Ljava/util/List;",
+ jint(formatId));
+ if (ctx.checkObject(QStringLiteral("supports"), supports)) {
+ return;
}
- QJniObject name = entry.getObjectField("name", "Ljava/lang/String;");
- if (ctx.checkObject(QStringLiteral("name"), name)) {
- continue;
+ jint count = supports.callMethod<jint>("size", "()I");
+ if (ctx.checkException(QStringLiteral("size"))) {
+ return;
}
- QString nameString = name.toString();
- if (ctx.checkException(QStringLiteral("nameString")) || nameString.isEmpty()) {
- continue;
- }
+ for (jint i = 0; i < count; ++i) {
+ QJniObject entry = supports.callObjectMethod("get", "(I)Ljava/lang/Object;", i);
+ if (ctx.checkObject(QStringLiteral("entry"), entry)) {
+ continue;
+ }
- bool hardware = entry.getField<jboolean>("hardware");
- if (ctx.checkException(QStringLiteral("hardware"))) {
- continue;
- }
+ QJniObject name = entry.getObjectField("name", "Ljava/lang/String;");
+ if (ctx.checkObject(QStringLiteral("name"), name)) {
+ continue;
+ }
+
+ QString nameString = name.toString();
+ if (ctx.checkException(QStringLiteral("nameString")) || nameString.isEmpty()) {
+ continue;
+ }
- encoders.append({nameString, hardware});
+ bool hardware = entry.getField<jboolean>("hardware");
+ if (ctx.checkException(QStringLiteral("hardware"))) {
+ continue;
+ }
+
+ p.first->append({nameString, hardware});
+ }
}
- if (!encoders.isEmpty()) {
- outSupportedFormats.append(new Format(formatId, encoders));
+ if (!videoEncoders.isEmpty()) {
+ outSupportedFormats.append(new Format(formatId, videoEncoders, audioEncoders));
}
}
-KisAndroidMediaEncoderPreferencesWidget::KisAndroidMediaEncoderPreferencesWidget(QWidget *parent)
+KisAndroidMediaEncoderPreferencesWidget::KisAndroidMediaEncoderPreferencesWidget(const KisMediaEncoderFormat *format,
+ QWidget *parent)
: QWidget(parent)
{
QFormLayout *form = new QFormLayout(this);
- m_cmbEncoder = new QComboBox;
- form->addRow(i18n("Encoder:"), m_cmbEncoder);
+ m_cmbVideoEncoder = new QComboBox;
+ form->addRow(i18n("Video encoder:"), m_cmbVideoEncoder);
+
+ m_intVideoBitrate = new QSpinBox;
+ m_intVideoBitrate->setRange(1, 999999999);
+ form->addRow(i18n("Video bitrate:"), m_intVideoBitrate);
+
+ if (format->supportsAudio()) {
+ m_cmbAudioEncoder = new QComboBox;
+ form->addRow(i18n("Audio encoder:"), m_cmbAudioEncoder);
+
+ m_intAudioBitrate = new QSpinBox;
+ m_intAudioBitrate->setRange(1, 999999999);
+ form->addRow(i18n("Audio bitrate:"), m_intAudioBitrate);
+ } else {
+ m_cmbAudioEncoder = nullptr;
+ m_intAudioBitrate = nullptr;
+ }
+}
+
+void KisAndroidMediaEncoderPreferencesWidget::addVideoEncoderOption(const QString &title, const QString &key)
+{
+ m_cmbVideoEncoder->addItem(title, QVariant(key));
+}
+
+void KisAndroidMediaEncoderPreferencesWidget::addAudioEncoderOption(const QString &title, const QString &key)
+{
+ m_cmbAudioEncoder->addItem(title, QVariant(key));
+}
+
+QString KisAndroidMediaEncoderPreferencesWidget::videoEncoder() const
+{
+ return m_cmbVideoEncoder->currentData().toString();
+}
+
+void KisAndroidMediaEncoderPreferencesWidget::setVideoEncoder(const QString &key)
+{
+ int index = 0;
+ int count = m_cmbVideoEncoder->count();
+ for (int i = 0; i < count; ++i) {
+ if (m_cmbVideoEncoder->itemData(i).toString() == key) {
+ index = i;
+ break;
+ }
+ }
+ m_cmbVideoEncoder->setCurrentIndex(index);
+}
- m_intBitrate = new QSpinBox;
- m_intBitrate->setRange(1, 999999999);
- form->addRow(i18n("Bitrate:"), m_intBitrate);
+int KisAndroidMediaEncoderPreferencesWidget::videoBitrate() const
+{
+ return m_intVideoBitrate->value();
}
-void KisAndroidMediaEncoderPreferencesWidget::addEncoderOption(const QString &title, const QString &key)
+void KisAndroidMediaEncoderPreferencesWidget::setVideoBitrate(int videoBitrate)
{
- m_cmbEncoder->addItem(title, QVariant(key));
+ m_intVideoBitrate->setValue(videoBitrate);
}
-QString KisAndroidMediaEncoderPreferencesWidget::encoder() const
+QString KisAndroidMediaEncoderPreferencesWidget::audioEncoder() const
{
- return m_cmbEncoder->currentData().toString();
+ return m_cmbAudioEncoder->currentData().toString();
}
-void KisAndroidMediaEncoderPreferencesWidget::setEncoder(const QString &key)
+void KisAndroidMediaEncoderPreferencesWidget::setAudioEncoder(const QString &key)
{
int index = 0;
- int count = m_cmbEncoder->count();
+ int count = m_cmbAudioEncoder->count();
for (int i = 0; i < count; ++i) {
- if (m_cmbEncoder->itemData(i).toString() == key) {
+ if (m_cmbAudioEncoder->itemData(i).toString() == key) {
index = i;
break;
}
}
- m_cmbEncoder->setCurrentIndex(index);
+ m_cmbAudioEncoder->setCurrentIndex(index);
}
-int KisAndroidMediaEncoderPreferencesWidget::bitrate() const
+int KisAndroidMediaEncoderPreferencesWidget::audioBitrate() const
{
- return m_intBitrate->value();
+ return m_intAudioBitrate->value();
}
-void KisAndroidMediaEncoderPreferencesWidget::setBitrate(int bitrate)
+void KisAndroidMediaEncoderPreferencesWidget::setAudioBitrate(int videoBitrate)
{
- m_intBitrate->setValue(bitrate);
+ m_intAudioBitrate->setValue(videoBitrate);
}
diff --git a/libs/ui/animation/KisAndroidMediaEncoderRunnable.h b/libs/ui/animation/KisAndroidMediaEncoderRunnable.h
index d9ecd39b95e..568403a8418 100644
--- a/libs/ui/animation/KisAndroidMediaEncoderRunnable.h
+++ b/libs/ui/animation/KisAndroidMediaEncoderRunnable.h
@@ -36,22 +36,31 @@ private:
static constexpr int STATUS_TIMEOUT = 1;
static constexpr int STATUS_END_OF_STREAM = 2;
static constexpr int STATUS_NEEDS_COPY = 3;
- static constexpr int STATUS_ERROR_START_ENCODER = 104;
- static constexpr int STATUS_ERROR_DRAIN_MUXER_ADD_TRACK = 404;
+ static constexpr int STATUS_ERROR_START_VIDEO_ENCODER = 104;
+ static constexpr int STATUS_ERROR_START_AUDIO_FORMAT = 107;
+ static constexpr int STATUS_ERROR_START_AUDIO_ENCODER = 108;
+ static constexpr int STATUS_ERROR_DRAIN_VIDEO_MUXER_ADD_TRACK = 404;
+ static constexpr int STATUS_ERROR_DRAIN_AUDIO_MUXER_ADD_TRACK = 704;
struct EncoderImage;
class Format;
class Context;
+ class Audio;
- EncodeResult prepare(Context &ctx);
+ EncodeResult prepareVideo(Context &ctx);
+ EncodeResult prepareAudio(Context &ctx);
+ EncodeResult prepare(Context &ctx, bool audio);
// Returns number of frames drained or one of the DRAIN_* values above.
- int drain(Context &ctx, long long initialTimeout);
+ int drainVideo(Context &ctx, long long initialTimeout);
+ int drainAudio(Context &ctx, long long initialTimeout);
+ int drain(Context &ctx, long long initialTimeout, bool audio);
static bool readEncoderImage(Context &ctx, EncoderImage &outImage);
static bool readPlaneBuffer(Context &ctx, int index, uint8_t *&outBuffer);
static bool readPlaneRowStride(Context &ctx, int index, int &outRowStride);
static bool readPlanePixelStride(Context &ctx, int index, int &outPixelStride);
+ static bool readAudioInputBuffer(Context &ctx, void *&outBuffer, int &outSize);
static void checkFormatSupport(Context &ctx, int formatId, QVector<KisMediaEncoderFormat *> &outSupportedFormats);
};
@@ -60,19 +69,28 @@ class KisAndroidMediaEncoderPreferencesWidget : public QWidget
{
Q_OBJECT
public:
- explicit KisAndroidMediaEncoderPreferencesWidget(QWidget *parent = nullptr);
+ explicit KisAndroidMediaEncoderPreferencesWidget(const KisMediaEncoderFormat *format, QWidget *parent = nullptr);
- void addEncoderOption(const QString &title, const QString &key);
+ void addVideoEncoderOption(const QString &title, const QString &key);
+ void addAudioEncoderOption(const QString &title, const QString &key);
- QString encoder() const;
- void setEncoder(const QString &key);
+ QString videoEncoder() const;
+ void setVideoEncoder(const QString &key);
- int bitrate() const;
- void setBitrate(int bitrate);
+ int videoBitrate() const;
+ void setVideoBitrate(int videoBitrate);
+
+ QString audioEncoder() const;
+ void setAudioEncoder(const QString &key);
+
+ int audioBitrate() const;
+ void setAudioBitrate(int audioBitrate);
private:
- QComboBox *m_cmbEncoder;
- QSpinBox *m_intBitrate;
+ QComboBox *m_cmbVideoEncoder;
+ QSpinBox *m_intVideoBitrate;
+ QComboBox *m_cmbAudioEncoder;
+ QSpinBox *m_intAudioBitrate;
};
#endif
diff --git a/libs/ui/animation/KisMediaEncoderWrapper.h b/libs/ui/animation/KisMediaEncoderWrapper.h
index 30e4a22b0ec..0b5adf3c875 100644
--- a/libs/ui/animation/KisMediaEncoderWrapper.h
+++ b/libs/ui/animation/KisMediaEncoderWrapper.h
@@ -46,6 +46,7 @@ protected:
struct KRITAUI_EXPORT KisMediaEncoderWrapperSettings {
QString outputFile;
QStringList inputFiles;
+ QString audioFile;
KisMediaEncoderFormat *format;
QVariantMap formatPreferences;
QString scaleFilter;
@@ -54,6 +55,7 @@ struct KRITAUI_EXPORT KisMediaEncoderWrapperSettings {
int outputFps;
int firstFrameSec;
int lastFrameSec;
+ int audioSeekFrame;
};
// Intentionally not exported, this is only needed internally.
@@ -90,6 +92,11 @@ protected:
{
}
+ const QString &path() const
+ {
+ return m_path;
+ }
+
int instances() const
{
return m_instances;
diff --git a/libs/ui/animation/KisVideoSaver.cpp b/libs/ui/animation/KisVideoSaver.cpp
index c26cf038506..80eff7eb394 100644
--- a/libs/ui/animation/KisVideoSaver.cpp
+++ b/libs/ui/animation/KisVideoSaver.cpp
@@ -31,7 +31,9 @@
#ifdef Q_OS_ANDROID
#include "animation/KisMediaEncoderWrapper.h"
+#include <KisAndroidUtils.h>
#include <QJsonDocument>
+#include <QTemporaryFile>
#else
#include "animation/KisFFMpegWrapper.h"
#endif
@@ -71,6 +73,7 @@ KisImportExportErrorCode KisAnimationVideoSaver::encode(const QString &framesDir
KisMediaEncoderWrapperSettings settings = {
options.videoFileName,
QStringList(),
+ QString(),
format,
formatPreferences,
options.scaleFilter,
@@ -79,6 +82,7 @@ KisImportExportErrorCode KisAnimationVideoSaver::encode(const QString &framesDir
options.frameRate,
0,
0,
+ 0,
};
QString inputDir = framesDirectory;
@@ -91,6 +95,23 @@ KisImportExportErrorCode KisAnimationVideoSaver::encode(const QString &framesDir
settings.inputFiles.append(inputDir + savedFile);
}
+ // MLT can't read from Android content URIs, so we have to copy the file
+ // into temporary storage and get a real path for it.
+ QTemporaryFile audioFile;
+ if (format->supportsAudio() && options.includeAudio) {
+ QVector<QFileInfo> audioFiles = m_doc->getAudioTracks();
+ if (!audioFiles.isEmpty()) {
+ QString audioPath = audioFiles.first().filePath();
+ QString errorMessage;
+ if (KisAndroidUtils::copyFileToTemporary(audioPath, audioFile, &errorMessage)) {
+ settings.audioFile = audioFile.fileName();
+ settings.audioSeekFrame = options.firstFrame;
+ } else {
+ return KisImportExportErrorCode(i18n("Error preparing audio file: %1", errorMessage));
+ }
+ }
+ }
+
// This whole batchMode business is either a vestige or was never actually
// implemented to completion. It's *supposed* to not show dialogs if Krita
// is run in batch mode via the command line, but it's not actually wired up
diff --git a/packaging/android/apk/src/org/krita/android/VideoEncoder.java b/packaging/android/apk/src/org/krita/android/VideoEncoder.java
index 7abb81778f2..0275b1aaaa3 100644
--- a/packaging/android/apk/src/org/krita/android/VideoEncoder.java
+++ b/packaging/android/apk/src/org/krita/android/VideoEncoder.java
@@ -3,6 +3,7 @@ package org.krita.android;
import android.content.ContentResolver;
import android.content.Context;
+import android.media.AudioFormat;
import android.media.Image;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
@@ -31,6 +32,19 @@ public class VideoEncoder {
}
}
+ private static class BufferedSample {
+ public final byte[] data;
+ public final MediaCodec.BufferInfo bufferInfo;
+ public final boolean audio;
+
+ private BufferedSample(ByteBuffer buffer, MediaCodec.BufferInfo bufferInfo, boolean audio) {
+ this.data = new byte[bufferInfo.size];
+ this.bufferInfo = bufferInfo;
+ this.audio = audio;
+ buffer.get(data);
+ }
+ }
+
private static final String TAG = "krita.VideoEncoder";
// Keep these formats in sync with KisAndroidMediaEncoderRunnable!
@@ -43,51 +57,87 @@ public class VideoEncoder {
private static final int STATUS_END_OF_STREAM = 2;
private static final int STATUS_NEEDS_TEMP_TO_OUTPUT_COPY = 3;
private static final int STATUS_ERROR_START_ALREADY_STARTED = 101;
- private static final int STATUS_ERROR_START_UNKNOWN_FORMAT = 102;
- private static final int STATUS_ERROR_START_FORMAT = 103;
- private static final int STATUS_ERROR_START_ENCODER = 104;
+ private static final int STATUS_ERROR_START_UNKNOWN_VIDEO_FORMAT = 102;
+ private static final int STATUS_ERROR_START_VIDEO_FORMAT = 103;
+ private static final int STATUS_ERROR_START_VIDEO_ENCODER = 104;
private static final int STATUS_ERROR_START_MUXER = 105;
+ private static final int STATUS_ERROR_START_UNKNOWN_AUDIO_FORMAT = 106;
+ private static final int STATUS_ERROR_START_AUDIO_FORMAT = 107;
+ private static final int STATUS_ERROR_START_AUDIO_ENCODER = 108;
private static final int STATUS_ERROR_PREPARE_DEQUEUE = 201;
private static final int STATUS_ERROR_PREPARE_GET_IMAGE = 202;
private static final int STATUS_ERROR_PREPARE_NULL_IMAGE = 203;
+ private static final int STATUS_ERROR_PREPARE_GET_AUDIO_BUFFER = 204;
+ private static final int STATUS_ERROR_PREPARE_NULL_AUDIO_BUFFER = 205;
private static final int STATUS_ERROR_COMMIT_QUEUE = 301;
- private static final int STATUS_ERROR_DRAIN_DEQUEUE = 401;
- private static final int STATUS_ERROR_DRAIN_MUXER_ALREADY_STARTED = 402;
- private static final int STATUS_ERROR_DRAIN_ENCODER_GET_OUTPUT_FORMAT = 403;
- private static final int STATUS_ERROR_DRAIN_MUXER_ADD_TRACK = 404;
- private static final int STATUS_ERROR_DRAIN_MUXER_START = 405;
- private static final int STATUS_ERROR_DRAIN_ENCODER_GET_OUTPUT_BUFFER = 406;
- private static final int STATUS_ERROR_DRAIN_ENCODER_NULL_OUTPUT_BUFFER = 407;
- private static final int STATUS_ERROR_DRAIN_MUXER_NOT_STARTED = 408;
- private static final int STATUS_ERROR_DRAIN_ENCODER_RELEASE_OUTPUT_BUFFER = 409;
- private static final int STATUS_ERROR_DRAIN_OUTPUT_BUFFER_POSITION = 410;
- private static final int STATUS_ERROR_DRAIN_OUTPUT_BUFFER_LIMIT = 411;
- private static final int STATUS_ERROR_DRAIN_MUXER_WRITE = 412;
- private static final int STATUS_ERROR_DRAIN_MUXER_NEVER_STARTED = 413;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_DEQUEUE = 401;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_TRACK_ALREADY_INITIALIZED = 402;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_ENCODER_GET_OUTPUT_FORMAT = 403;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_MUXER_ADD_TRACK = 404;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_MUXER_START = 405;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_ENCODER_GET_OUTPUT_BUFFER = 406;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_ENCODER_NULL_OUTPUT_BUFFER = 407;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_TRACK_NOT_INITIALIZED = 408;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_ENCODER_RELEASE_OUTPUT_BUFFER = 409;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_OUTPUT_BUFFER_POSITION = 410;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_OUTPUT_BUFFER_LIMIT = 411;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_MUXER_WRITE = 412;
+ private static final int STATUS_ERROR_DRAIN_VIDEO_MUXER_BUFFER = 413;
private static final int STATUS_ERROR_FINISH_QUEUE = 501;
private static final int STATUS_ERROR_CLOSE_TEAR_DOWN = 601;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_DEQUEUE = 701;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_TRACK_ALREADY_INITIALIZED = 702;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_ENCODER_GET_OUTPUT_FORMAT = 703;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_MUXER_ADD_TRACK = 704;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_MUXER_START = 705;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_ENCODER_GET_OUTPUT_BUFFER = 706;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_ENCODER_NULL_OUTPUT_BUFFER = 707;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_TRACK_NOT_INITIALIZED = 708;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_ENCODER_RELEASE_OUTPUT_BUFFER = 709;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_OUTPUT_BUFFER_POSITION = 710;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_OUTPUT_BUFFER_LIMIT = 711;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_MUXER_WRITE = 712;
+ private static final int STATUS_ERROR_DRAIN_AUDIO_MUXER_BUFFER = 713;
private final int mFormat;
private final int mWidth;
private final int mHeight;
private final float mFramerate;
private final double mFrameDurationUs;
+ private final int mAudioSampleRate;
+ private final int mAudioChannelCount;
private final String mOutputPath;
private final String mTempPath;
- private final String mEncoderName;
- private final int mBitrate;
- private final MediaCodec.BufferInfo mBufferInfo = new MediaCodec.BufferInfo();
- private MediaCodec mEncoder = null;
+ private final String mVideoEncoderName;
+ private final String mAudioEncoderName;
+ private final String mAudioFormatName;
+ private final int mVideoBitrate;
+ private final int mAudioBitrate;
+ private MediaCodec.BufferInfo mVideoBufferInfo = null;
+ private MediaCodec.BufferInfo mAudioBufferInfo = null;
+ private MediaCodec mVideoEncoder = null;
+ private MediaCodec mAudioEncoder = null;
private MediaMuxer mMuxer = null;
private boolean mUsesTempPath = false;
private boolean mMuxerStarted = false;
- private int mFrameIndex = 0;
+ private int mVideoFrameIndex = 0;
+ private int mAudioSamplesWritten = 0;
private int mVideoTrack = -1;
- private int mInputBufferIndex = -1;
- private Image mInputImage = null;
+ private int mAudioTrack = -1;
+ private int mVideoInputBufferIndex = -1;
+ private int mAudioInputBufferIndex = -1;
+ private Image mVideoInputImage = null;
+ private ByteBuffer mAudioInputBuffer = null;
+ private boolean mVideoFinished = false;
+ private boolean mAudioFinished = false;
+ private boolean mVideoStreamEnded = false;
+ private boolean mAudioStreamEnded = false;
+ private final List<BufferedSample> mBufferedSamples = new ArrayList<>();
public VideoEncoder(int format, int width, int height, float framerate, String outputPath,
- String tempPath, String encoderName, int bitrate) {
+ String tempPath, String videoEncoderName, int videoBitrate,
+ String audioEncoderName, String audioFormatName, int audioSampleRate,
+ int audioChannelCount, int audioBitrate) {
mFormat = format;
mWidth = width;
mHeight = height;
@@ -95,19 +145,24 @@ public class VideoEncoder {
mFrameDurationUs = 1000000.0 / ((double) framerate);
mOutputPath = outputPath;
mTempPath = tempPath;
- mEncoderName = encoderName;
- mBitrate = bitrate;
+ mVideoEncoderName = videoEncoderName;
+ mVideoBitrate = videoBitrate;
+ mAudioEncoderName = audioEncoderName;
+ mAudioFormatName = audioFormatName;
+ mAudioSampleRate = audioSampleRate;
+ mAudioChannelCount = audioChannelCount;
+ mAudioBitrate = audioBitrate;
}
public int start(Context context) {
- if (mEncoder != null || mMuxer != null) {
+ if (mVideoEncoder != null || mMuxer != null) {
Log.e(TAG, "Attempting to start an already started video encoder");
return STATUS_ERROR_START_ALREADY_STARTED;
}
- String videoMimeType = getFormatMimeType(mFormat);
+ String videoMimeType = getFormatVideoMimeType(mFormat);
if (videoMimeType == null) {
- return STATUS_ERROR_START_UNKNOWN_FORMAT;
+ return STATUS_ERROR_START_UNKNOWN_VIDEO_FORMAT;
}
MediaFormat videoFormat;
@@ -115,14 +170,41 @@ public class VideoEncoder {
videoFormat = initializeVideoFormat(videoMimeType);
} catch (Exception e) {
Log.e(TAG, "Error initializing video format", e);
- return STATUS_ERROR_START_FORMAT;
+ return STATUS_ERROR_START_VIDEO_FORMAT;
+ }
+
+ String audioMimeType = null;
+ if (mAudioSampleRate > 0 && mAudioChannelCount > 0 && mAudioBitrate > 0) {
+ audioMimeType = getFormatAudioMimeType(mFormat);
+ if (audioMimeType == null) {
+ return STATUS_ERROR_START_UNKNOWN_AUDIO_FORMAT;
+ }
+ }
+
+ MediaFormat audioFormat = null;
+ if (audioMimeType != null) {
+ try {
+ audioFormat = initializeAudioFormat(audioMimeType);
+ } catch (Exception e) {
+ Log.e(TAG, "Error initializing audio format", e);
+ return STATUS_ERROR_START_AUDIO_FORMAT;
+ }
}
try {
- mEncoder = initializeEncoder(videoMimeType, videoFormat);
+ mVideoEncoder = initializeEncoder(mVideoEncoderName, videoMimeType, videoFormat);
} catch (Exception e) {
- Log.e(TAG, "Error initializing encoder", e);
- return STATUS_ERROR_START_ENCODER;
+ Log.e(TAG, "Error initializing video encoder", e);
+ return STATUS_ERROR_START_VIDEO_ENCODER;
+ }
+
+ if (audioFormat != null) {
+ try {
+ mAudioEncoder = initializeEncoder(mAudioEncoderName, audioMimeType, audioFormat);
+ } catch (Exception e) {
+ Log.e(TAG, "Error initializing audio encoder", e);
+ return STATUS_ERROR_START_AUDIO_ENCODER;
+ }
}
try {
@@ -135,39 +217,74 @@ public class VideoEncoder {
return STATUS_OK;
}
- public int prepare(long timeoutUs) {
- int inputBufferIndex;
- try {
- inputBufferIndex = mEncoder.dequeueInputBuffer(timeoutUs);
- } catch (Exception e) {
- Log.e(TAG, "Error dequeuing input buffer", e);
- return STATUS_ERROR_PREPARE_DEQUEUE;
- }
+ public int prepareVideo(long timeoutUs) {
+ if (mVideoInputBufferIndex == -1) {
+ int videoInputBufferIndex;
+ try {
+ videoInputBufferIndex = mVideoEncoder.dequeueInputBuffer(timeoutUs);
+ } catch (Exception e) {
+ Log.e(TAG, "Error dequeuing video input buffer", e);
+ return STATUS_ERROR_PREPARE_DEQUEUE;
+ }
- if (inputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
- return STATUS_TIMEOUT;
- }
+ if (videoInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
+ return STATUS_TIMEOUT;
+ }
- Image inputImage;
- try {
- inputImage = mEncoder.getInputImage(inputBufferIndex);
- } catch (Exception e) {
- Log.e(TAG, "Error getting input image " + inputBufferIndex, e);
- return STATUS_ERROR_PREPARE_GET_IMAGE;
- }
- if (inputImage == null) {
- Log.e(TAG, "No input image " + inputBufferIndex);
- return STATUS_ERROR_PREPARE_NULL_IMAGE;
+ Image inputImage;
+ try {
+ inputImage = mVideoEncoder.getInputImage(videoInputBufferIndex);
+ } catch (Exception e) {
+ Log.e(TAG, "Error getting input image " + videoInputBufferIndex, e);
+ return STATUS_ERROR_PREPARE_GET_IMAGE;
+ }
+ if (inputImage == null) {
+ Log.e(TAG, "No input image " + videoInputBufferIndex);
+ return STATUS_ERROR_PREPARE_NULL_IMAGE;
+ }
+
+ mVideoInputBufferIndex = videoInputBufferIndex;
+ mVideoInputImage = inputImage;
}
+ return STATUS_OK;
+ }
- mInputBufferIndex = inputBufferIndex;
- mInputImage = inputImage;
+ public int prepareAudio(long timeoutUs) {
+ if (mAudioInputBufferIndex == -1) {
+ int audioInputBufferIndex;
+ try {
+ audioInputBufferIndex = mAudioEncoder.dequeueInputBuffer(timeoutUs);
+ } catch (Exception e) {
+ Log.e(TAG, "Error dequeuing audio input buffer", e);
+ return STATUS_ERROR_PREPARE_DEQUEUE;
+ }
+
+ if (audioInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
+ return STATUS_TIMEOUT;
+ }
+
+ ByteBuffer audioInputBuffer;
+ try {
+ audioInputBuffer = mAudioEncoder.getInputBuffer(audioInputBufferIndex);
+ } catch (Exception e) {
+ Log.e(TAG, "Error getting audio input buffer " + audioInputBufferIndex, e);
+ return STATUS_ERROR_PREPARE_GET_AUDIO_BUFFER;
+ }
+ if (audioInputBuffer == null) {
+ Log.e(TAG, "No audio input buffer " + audioInputBufferIndex);
+ return STATUS_ERROR_PREPARE_NULL_AUDIO_BUFFER;
+ }
+
+ mAudioInputBufferIndex = audioInputBufferIndex;
+ mAudioInputBuffer = audioInputBuffer;
+ mAudioInputBuffer.clear();
+ }
return STATUS_OK;
}
public ByteBuffer getInputImagePlaneBuffer(int index) {
try {
- Image.Plane[] planes = mInputImage.getPlanes();
+ Image.Plane[] planes = mVideoInputImage.getPlanes();
Image.Plane plane = planes[index];
return plane.getBuffer();
} catch (Exception e) {
@@ -178,7 +295,7 @@ public class VideoEncoder {
public int getInputImagePlaneRowStride(int index) {
try {
- Image.Plane[] planes = mInputImage.getPlanes();
+ Image.Plane[] planes = mVideoInputImage.getPlanes();
Image.Plane plane = planes[index];
return plane.getRowStride();
} catch (Exception e) {
@@ -189,7 +306,7 @@ public class VideoEncoder {
public int getInputImagePlanePixelStride(int index) {
try {
- Image.Plane[] planes = mInputImage.getPlanes();
+ Image.Plane[] planes = mVideoInputImage.getPlanes();
Image.Plane plane = planes[index];
return plane.getPixelStride();
} catch (Exception e) {
@@ -198,138 +315,341 @@ public class VideoEncoder {
}
}
- public int commit() {
+ public ByteBuffer getInputAudioBuffer() {
+ return mAudioInputBuffer;
+ }
+
+ public int commitVideo() {
+ try {
+ ByteBuffer inputBuffer = mVideoEncoder.getInputBuffer(mVideoInputBufferIndex);
+ mVideoEncoder.queueInputBuffer(mVideoInputBufferIndex, 0, inputBuffer == null ? 0 :
+ inputBuffer.capacity(), getVideoPresentationTimeUs(), 0);
+ } catch (Exception e) {
+ Log.e(TAG, "Error committing video frame " + mVideoFrameIndex, e);
+ return STATUS_ERROR_COMMIT_QUEUE;
+ }
+ mVideoInputBufferIndex = -1;
+ mVideoInputImage = null;
+ ++mVideoFrameIndex;
+ return STATUS_OK;
+ }
+
+ public int commitAudio(int chunkSamples, int chunkSize) {
try {
- ByteBuffer inputBuffer = mEncoder.getInputBuffer(mInputBufferIndex);
- mEncoder.queueInputBuffer(mInputBufferIndex, 0, inputBuffer == null ? 0 :
- inputBuffer.capacity(), getPresentationTimeUs(), 0);
+ mAudioEncoder.queueInputBuffer(mAudioInputBufferIndex, 0, chunkSize,
+ getAudioPresentationTimeUs(), 0);
} catch (Exception e) {
- Log.e(TAG, "Error committing frame " + mFrameIndex, e);
+ Log.e(TAG, "Error committing samples at " + mAudioSamplesWritten, e);
return STATUS_ERROR_COMMIT_QUEUE;
}
- mInputBufferIndex = -1;
- mInputImage = null;
- ++mFrameIndex;
+ mAudioInputBufferIndex = -1;
+ mAudioInputBuffer = null;
+ mAudioSamplesWritten += chunkSamples;
return STATUS_OK;
}
- public int drain(long timeoutUs) {
+ public int drainVideo(long timeoutUs) {
+ if (mVideoStreamEnded) {
+ return STATUS_END_OF_STREAM;
+ }
+
+ if (mVideoBufferInfo == null) {
+ mVideoBufferInfo = new MediaCodec.BufferInfo();
+ }
+
int outputBufferIndex;
+ int flags;
try {
- outputBufferIndex = mEncoder.dequeueOutputBuffer(mBufferInfo, timeoutUs);
+ outputBufferIndex = mVideoEncoder.dequeueOutputBuffer(mVideoBufferInfo, timeoutUs);
+ flags = mVideoBufferInfo.flags;
} catch (Exception e) {
- Log.e(TAG, "Error dequeuing output buffer", e);
- return STATUS_ERROR_DRAIN_DEQUEUE;
+ Log.e(TAG, "Error dequeuing video output buffer", e);
+ return STATUS_ERROR_DRAIN_VIDEO_DEQUEUE;
}
if (outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
return STATUS_TIMEOUT;
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
- if (mMuxerStarted) {
- Log.e(TAG, "Format changed, but muxer already started");
- return STATUS_ERROR_DRAIN_MUXER_ALREADY_STARTED;
+ if (mVideoTrack != -1) {
+ Log.e(TAG, "Video format changed, but track already initialized");
+ return STATUS_ERROR_DRAIN_VIDEO_TRACK_ALREADY_INITIALIZED;
}
MediaFormat outputFormat;
try {
- outputFormat = mEncoder.getOutputFormat();
+ outputFormat = mVideoEncoder.getOutputFormat();
} catch (Exception e) {
- Log.e(TAG, "Error getting encoder output format", e);
- return STATUS_ERROR_DRAIN_ENCODER_GET_OUTPUT_FORMAT;
+ Log.e(TAG, "Error getting video encoder output format", e);
+ return STATUS_ERROR_DRAIN_VIDEO_ENCODER_GET_OUTPUT_FORMAT;
}
try {
mVideoTrack = mMuxer.addTrack(outputFormat);
} catch (Exception e) {
- Log.e(TAG, "Error adding muxer track", e);
- return STATUS_ERROR_DRAIN_MUXER_ADD_TRACK;
+ Log.e(TAG, "Error adding video muxer track", e);
+ return STATUS_ERROR_DRAIN_VIDEO_MUXER_ADD_TRACK;
}
- try {
- mMuxer.start();
- } catch (Exception e) {
- Log.e(TAG, "Error starting muxer", e);
- return STATUS_ERROR_DRAIN_MUXER_START;
+ if (!tryStartMuxer()) {
+ return STATUS_ERROR_DRAIN_VIDEO_MUXER_START;
}
- mMuxerStarted = true;
} else if (outputBufferIndex >= 0) {
ByteBuffer outputBuffer;
try {
- outputBuffer = mEncoder.getOutputBuffer(outputBufferIndex);
+ outputBuffer = mVideoEncoder.getOutputBuffer(outputBufferIndex);
} catch (Exception e) {
- Log.e(TAG, "Error getting encoder output buffer " + outputBufferIndex, e);
- return STATUS_ERROR_DRAIN_ENCODER_GET_OUTPUT_BUFFER;
+ Log.e(TAG, "Error getting video encoder output buffer " + outputBufferIndex, e);
+ return STATUS_ERROR_DRAIN_VIDEO_ENCODER_GET_OUTPUT_BUFFER;
}
if (outputBuffer == null) {
- Log.e(TAG, "Null encoder output buffer " + outputBufferIndex);
- return STATUS_ERROR_DRAIN_ENCODER_NULL_OUTPUT_BUFFER;
+ Log.e(TAG, "Null video encoder output buffer " + outputBufferIndex);
+ return STATUS_ERROR_DRAIN_VIDEO_ENCODER_NULL_OUTPUT_BUFFER;
}
- if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0 && mBufferInfo.size != 0) {
- if (!mMuxerStarted) {
+ if ((flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0 && mVideoBufferInfo.size != 0) {
+ if (mVideoTrack == -1) {
Log.e(TAG,
- "Got output buffer " + outputBufferIndex + " with size " + mBufferInfo.size + " while muxer is not started");
- return STATUS_ERROR_DRAIN_MUXER_NOT_STARTED;
+ "Got video output buffer " + outputBufferIndex + " with size " + mVideoBufferInfo.size + " while track is not initialized");
+ return STATUS_ERROR_DRAIN_VIDEO_TRACK_NOT_INITIALIZED;
}
- int position = mBufferInfo.offset;
+ int position = mVideoBufferInfo.offset;
try {
outputBuffer.position(position);
} catch (Exception e) {
Log.e(TAG,
- "Error setting output buffer " + outputBufferIndex + " position " + position, e);
- return STATUS_ERROR_DRAIN_OUTPUT_BUFFER_POSITION;
+ "Error setting video output buffer " + outputBufferIndex + " position " + position, e);
+ return STATUS_ERROR_DRAIN_VIDEO_OUTPUT_BUFFER_POSITION;
}
- int limit = position + mBufferInfo.size;
+ int limit = position + mVideoBufferInfo.size;
try {
outputBuffer.limit(limit);
} catch (Exception e) {
Log.e(TAG,
- "Error setting output buffer " + outputBufferIndex + " limit " + limit, e);
- return STATUS_ERROR_DRAIN_OUTPUT_BUFFER_LIMIT;
+ "Error setting video output buffer " + outputBufferIndex + " limit " + limit, e);
+ return STATUS_ERROR_DRAIN_VIDEO_OUTPUT_BUFFER_LIMIT;
}
- try {
- mMuxer.writeSampleData(mVideoTrack, outputBuffer, mBufferInfo);
- } catch (Exception e) {
- Log.e(TAG,
- "Error writing sample data from output buffer " + outputBufferIndex + " to track " + mVideoTrack, e);
- return STATUS_ERROR_DRAIN_MUXER_WRITE;
+ if (mMuxerStarted) {
+ try {
+ mMuxer.writeSampleData(mVideoTrack, outputBuffer, mVideoBufferInfo);
+ } catch (Exception e) {
+ Log.e(TAG,
+ "Error writing video sample data from output buffer " + outputBufferIndex + " to track " + mVideoTrack, e);
+ return STATUS_ERROR_DRAIN_VIDEO_MUXER_WRITE;
+ }
+ } else {
+ try {
+ mBufferedSamples.add(new BufferedSample(outputBuffer, mVideoBufferInfo, false));
+ } catch (Exception e) {
+ Log.e(TAG, "Error buffering video sample data from output buffer " + outputBufferIndex, e);
+ return STATUS_ERROR_DRAIN_VIDEO_MUXER_BUFFER;
+ }
+ mVideoBufferInfo = null;
}
}
try {
- mEncoder.releaseOutputBuffer(outputBufferIndex, false);
+ mVideoEncoder.releaseOutputBuffer(outputBufferIndex, false);
+ } catch (Exception e) {
+ Log.e(TAG, "Error releasing video encoder output buffer " + outputBufferIndex, e);
+ return STATUS_ERROR_DRAIN_VIDEO_ENCODER_RELEASE_OUTPUT_BUFFER;
+ }
+
+ if ((flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
+ mVideoStreamEnded = true;
+ return STATUS_END_OF_STREAM;
+ }
+
+ } else {
+ Log.d(TAG, "Unhandled video output buffer index value " + outputBufferIndex);
+ }
+
+ return STATUS_OK;
+ }
+
+ public int drainAudio(long timeoutUs) {
+ if (mAudioStreamEnded) {
+ return STATUS_END_OF_STREAM;
+ }
+
+ if (mAudioBufferInfo == null) {
+ mAudioBufferInfo = new MediaCodec.BufferInfo();
+ }
+
+ int outputBufferIndex;
+ int flags;
+ try {
+ outputBufferIndex = mAudioEncoder.dequeueOutputBuffer(mAudioBufferInfo, timeoutUs);
+ flags = mAudioBufferInfo.flags;
+ } catch (Exception e) {
+ Log.e(TAG, "Error dequeuing audio output buffer", e);
+ return STATUS_ERROR_DRAIN_AUDIO_DEQUEUE;
+ }
+
+ if (outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
+ return STATUS_TIMEOUT;
+
+ } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+ if (mAudioTrack != -1) {
+ Log.e(TAG, "Audio format changed, but track already initialized");
+ return STATUS_ERROR_DRAIN_AUDIO_TRACK_ALREADY_INITIALIZED;
+ }
+
+ MediaFormat outputFormat;
+ try {
+ outputFormat = mAudioEncoder.getOutputFormat();
+ } catch (Exception e) {
+ Log.e(TAG, "Error getting audio encoder output format", e);
+ return STATUS_ERROR_DRAIN_AUDIO_ENCODER_GET_OUTPUT_FORMAT;
+ }
+ try {
+ mAudioTrack = mMuxer.addTrack(outputFormat);
+ } catch (Exception e) {
+ Log.e(TAG, "Error adding audio muxer track", e);
+ return STATUS_ERROR_DRAIN_AUDIO_MUXER_ADD_TRACK;
+ }
+
+ if (!tryStartMuxer()) {
+ return STATUS_ERROR_DRAIN_AUDIO_MUXER_START;
+ }
+
+ } else if (outputBufferIndex >= 0) {
+ ByteBuffer outputBuffer;
+ try {
+ outputBuffer = mAudioEncoder.getOutputBuffer(outputBufferIndex);
} catch (Exception e) {
- Log.e(TAG, "Error releasing encoder output buffer " + outputBufferIndex, e);
- return STATUS_ERROR_DRAIN_ENCODER_RELEASE_OUTPUT_BUFFER;
+ Log.e(TAG, "Error getting audio encoder output buffer " + outputBufferIndex, e);
+ return STATUS_ERROR_DRAIN_AUDIO_ENCODER_GET_OUTPUT_BUFFER;
+ }
+ if (outputBuffer == null) {
+ Log.e(TAG, "Null audio encoder output buffer " + outputBufferIndex);
+ return STATUS_ERROR_DRAIN_AUDIO_ENCODER_NULL_OUTPUT_BUFFER;
}
- if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
+ if ((flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0 && mAudioBufferInfo.size != 0) {
+ if (mAudioTrack == -1) {
+ Log.e(TAG,
+ "Got audio output buffer " + outputBufferIndex + " with size " + mAudioBufferInfo.size + " while track is not initialized");
+ return STATUS_ERROR_DRAIN_AUDIO_TRACK_NOT_INITIALIZED;
+ }
+
+ int position = mAudioBufferInfo.offset;
+ try {
+ outputBuffer.position(position);
+ } catch (Exception e) {
+ Log.e(TAG,
+ "Error setting audio output buffer " + outputBufferIndex + " position " + position, e);
+ return STATUS_ERROR_DRAIN_AUDIO_OUTPUT_BUFFER_POSITION;
+ }
+
+ int limit = position + mAudioBufferInfo.size;
+ try {
+ outputBuffer.limit(limit);
+ } catch (Exception e) {
+ Log.e(TAG,
+ "Error setting audio output buffer " + outputBufferIndex + " limit " + limit, e);
+ return STATUS_ERROR_DRAIN_AUDIO_OUTPUT_BUFFER_LIMIT;
+ }
+
if (mMuxerStarted) {
- return STATUS_END_OF_STREAM;
+ try {
+ mMuxer.writeSampleData(mAudioTrack, outputBuffer, mAudioBufferInfo);
+ } catch (Exception e) {
+ Log.e(TAG,
+ "Error writing audio sample data from output buffer " + outputBufferIndex + " to track " + mAudioTrack, e);
+ return STATUS_ERROR_DRAIN_AUDIO_MUXER_WRITE;
+ }
} else {
- Log.e(TAG, "Got end of stream without the muxer ever starting");
- return STATUS_ERROR_DRAIN_MUXER_NEVER_STARTED;
+ try {
+ mBufferedSamples.add(new BufferedSample(outputBuffer, mAudioBufferInfo, true));
+ } catch (Exception e) {
+ Log.e(TAG, "Error buffering audio sample data from output buffer " + outputBufferIndex, e);
+ return STATUS_ERROR_DRAIN_AUDIO_MUXER_BUFFER;
+ }
+ mAudioBufferInfo = null;
}
}
+ try {
+ mAudioEncoder.releaseOutputBuffer(outputBufferIndex, false);
+ } catch (Exception e) {
+ Log.e(TAG, "Error releasing audio encoder output buffer " + outputBufferIndex, e);
+ return STATUS_ERROR_DRAIN_AUDIO_ENCODER_RELEASE_OUTPUT_BUFFER;
+ }
+
+ if ((flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
+ mAudioStreamEnded = true;
+ return STATUS_END_OF_STREAM;
+ }
+
} else {
- Log.d(TAG, "Unhandled output buffer index value " + outputBufferIndex);
+ Log.d(TAG, "Unhandled audio output buffer index value " + outputBufferIndex);
}
return STATUS_OK;
}
- public int finish() {
- try {
- mEncoder.queueInputBuffer(mInputBufferIndex, 0, 0, getPresentationTimeUs(),
- MediaCodec.BUFFER_FLAG_END_OF_STREAM);
- } catch (Exception e) {
- Log.e(TAG, "Error committing end of stream frame " + mFrameIndex, e);
- return STATUS_ERROR_FINISH_QUEUE;
+ private boolean tryStartMuxer() {
+ if (mVideoTrack != -1 && (mAudioEncoder == null || mAudioTrack != -1)) {
+ try {
+ mMuxer.start();
+ } catch (Exception e) {
+ Log.e(TAG, "Error starting muxer", e);
+ return false;
+ }
+ mMuxerStarted = true;
+
+ for (BufferedSample bs : mBufferedSamples) {
+ String title = bs.audio ? "audio" : "video";
+ int track = bs.audio ? mAudioTrack : mVideoTrack;
+
+ ByteBuffer buf;
+ try {
+ buf = ByteBuffer.wrap(bs.data);
+ } catch (Exception e) {
+ Log.e(TAG, "Error wrapping buffered " + title + " sample", e);
+ return false;
+ }
+
+ try {
+ mMuxer.writeSampleData(track, buf, bs.bufferInfo);
+ } catch (Exception e) {
+ Log.e(TAG, "Error writing buffered " + title + " sample data to track " + track, e);
+ return false;
+ }
+ }
+ mBufferedSamples.clear();
+ }
+ return true;
+ }
+
+ public int finishVideo() {
+ if (!mVideoFinished) {
+ try {
+ mVideoEncoder.queueInputBuffer(mVideoInputBufferIndex, 0, 0,
+ getVideoPresentationTimeUs(), MediaCodec.BUFFER_FLAG_END_OF_STREAM);
+ } catch (Exception e) {
+ Log.e(TAG, "Error committing video end of stream frame " + mVideoFrameIndex, e);
+ return STATUS_ERROR_FINISH_QUEUE;
+ }
+ mVideoFinished = true;
+ }
+ return STATUS_OK;
+ }
+
+ public int finishAudio() {
+ if (mAudioEncoder != null && !mAudioFinished) {
+ try {
+ mAudioEncoder.queueInputBuffer(mAudioInputBufferIndex, 0, 0,
+ getAudioPresentationTimeUs(), MediaCodec.BUFFER_FLAG_END_OF_STREAM);
+ } catch (Exception e) {
+ Log.e(TAG, "Error committing audio end of stream at " + mAudioSamplesWritten, e);
+ return STATUS_ERROR_FINISH_QUEUE;
+ }
+ mAudioFinished = true;
}
return STATUS_OK;
}
@@ -353,20 +673,36 @@ public class VideoEncoder {
private boolean tearDown() {
boolean ok = true;
- if (mEncoder != null) {
+ if (mAudioEncoder != null) {
+ try {
+ mAudioEncoder.stop();
+ } catch (Exception e) {
+ Log.e(TAG, "Error stopping audio encoder", e);
+ ok = false;
+ }
+ try {
+ mAudioEncoder.release();
+ } catch (Exception e) {
+ Log.e(TAG, "Error releasing audio encoder", e);
+ ok = false;
+ }
+ mAudioEncoder = null;
+ }
+
+ if (mVideoEncoder != null) {
try {
- mEncoder.stop();
+ mVideoEncoder.stop();
} catch (Exception e) {
- Log.e(TAG, "Error stopping encoder", e);
+ Log.e(TAG, "Error stopping video encoder", e);
ok = false;
}
try {
- mEncoder.release();
+ mVideoEncoder.release();
} catch (Exception e) {
- Log.e(TAG, "Error releasing encoder", e);
+ Log.e(TAG, "Error releasing video encoder", e);
ok = false;
}
- mEncoder = null;
+ mVideoEncoder = null;
}
if (mMuxer != null) {
@@ -397,7 +733,7 @@ public class VideoEncoder {
MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 2);
videoFormat.setFloat(MediaFormat.KEY_FRAME_RATE, mFramerate);
- videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, mBitrate);
+ videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, mVideoBitrate);
switch (mFormat) {
case FORMAT_MP4_H264:
case FORMAT_WEBM_VP8:
@@ -417,19 +753,51 @@ public class VideoEncoder {
return videoFormat;
}
- private MediaCodec initializeEncoder(String videoMimeType, MediaFormat videoFormat) throws Exception {
- MediaCodec encoder = createEncoder(videoMimeType);
- encoder.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+ private MediaFormat initializeAudioFormat(String audioMimeType) throws Exception {
+ int pcmEncoding = getPcmEncodingForAudioFormatName(mAudioFormatName);
+ // Note that this doesn't necessarily actually result in a track with
+ // the given parameters! See the notes in the C++ code about only using
+ // known good parameters to actually get something useful here.
+ MediaFormat audioFormat =
+ MediaFormat.createAudioFormat(audioMimeType, mAudioSampleRate, mAudioChannelCount);
+ audioFormat.setInteger(MediaFormat.KEY_PCM_ENCODING, pcmEncoding);
+ audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, mAudioBitrate);
+ if (audioMimeType.equals(MediaFormat.MIMETYPE_AUDIO_AAC)) {
+ audioFormat.setInteger(
+ MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
+ }
+ return audioFormat;
+ }
+
+ private static int getPcmEncodingForAudioFormatName(String name) {
+ int pcmEncoding;
+ if ("s16".equals(name)) {
+ pcmEncoding = AudioFormat.ENCODING_PCM_16BIT;
+ } else if ("float".equals(name) || "f32le".equals(name)) {
+ pcmEncoding = AudioFormat.ENCODING_PCM_FLOAT;
+ } else if ("u8".equals(name)) {
+ pcmEncoding = AudioFormat.ENCODING_PCM_8BIT;
+ } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && ("s32".equals(name) || "s32le".equals(name))) {
+ pcmEncoding = AudioFormat.ENCODING_PCM_32BIT;
+ } else {
+ throw new IllegalArgumentException("Unhandled audio format " + name);
+ }
+ return pcmEncoding;
+ }
+
+ private MediaCodec initializeEncoder(String encoderName, String mimeType, MediaFormat format) throws Exception {
+ MediaCodec encoder = createEncoder(encoderName, mimeType);
+ encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
encoder.start();
return encoder;
}
- private MediaCodec createEncoder(String videoMimeType) throws Exception {
- if (mEncoderName != null && !mEncoderName.isEmpty()) {
+ private MediaCodec createEncoder(String encoderName, String videoMimeType) throws Exception {
+ if (encoderName != null && !encoderName.isEmpty()) {
try {
- return MediaCodec.createByCodecName(mEncoderName);
+ return MediaCodec.createByCodecName(encoderName);
} catch (Exception e) {
- Log.w(TAG, "Failed to create encoder " + mEncoderName, e);
+ Log.w(TAG, "Failed to create encoder " + encoderName, e);
}
}
return MediaCodec.createEncoderByType(videoMimeType);
@@ -469,12 +837,25 @@ public class VideoEncoder {
return new MediaMuxer(mTempPath, outputFormat);
}
- private long getPresentationTimeUs() {
- return Math.round(((double) mFrameIndex) * mFrameDurationUs);
+ private long getVideoPresentationTimeUs() {
+ return Math.round(((double) mVideoFrameIndex) * mFrameDurationUs);
}
- public static List<Support> getSupportsForFormat(int format) {
- String mimeType = getFormatMimeType(format);
+ private long getAudioPresentationTimeUs() {
+ return (mAudioSamplesWritten * 1000000L) / mAudioSampleRate;
+ }
+
+ public static List<Support> getSupportsForVideoFormat(int format) {
+ String mimeType = getFormatVideoMimeType(format);
+ if (mimeType == null) {
+ return Collections.emptyList();
+ } else {
+ return getSupportsForMimeType(mimeType);
+ }
+ }
+
+ public static List<Support> getSupportsForAudioFormat(int format) {
+ String mimeType = getFormatAudioMimeType(format);
if (mimeType == null) {
return Collections.emptyList();
} else {
@@ -529,7 +910,7 @@ public class VideoEncoder {
return lowerName.startsWith("omx.google.") || lowerName.startsWith("c2.android.");
}
- private static String getFormatMimeType(int format) {
+ private static String getFormatVideoMimeType(int format) {
switch (format) {
case FORMAT_MP4_H264:
return MediaFormat.MIMETYPE_VIDEO_AVC;
@@ -545,4 +926,16 @@ public class VideoEncoder {
return null;
}
}
+
+ private static String getFormatAudioMimeType(int format) {
+ switch (format) {
+ case FORMAT_MP4_H264:
+ case FORMAT_MP4_AV1:
+ return MediaFormat.MIMETYPE_AUDIO_AAC;
+ case FORMAT_WEBM_VP8:
+ return MediaFormat.MIMETYPE_AUDIO_OPUS;
+ default:
+ return null;
+ }
+ }
}
diff --git a/plugins/dockers/recorder/recorder_export.cpp b/plugins/dockers/recorder/recorder_export.cpp
index 9210446abed..76d5847569a 100644
--- a/plugins/dockers/recorder/recorder_export.cpp
+++ b/plugins/dockers/recorder/recorder_export.cpp
@@ -298,6 +298,7 @@ public:
KisMediaEncoderWrapperSettings exporterSettings = {
settings->videoFilePath,
settings->inputFilePaths,
+ QString(),
KisMediaEncoderWrapper::getFormatByKey(settings->selectedFormat),
settings->formatPreferences.value(settings->selectedFormat).toMap(),
QString(),
@@ -306,6 +307,7 @@ public:
settings->fps,
settings->firstFrameSec,
settings->lastFrameSec,
+ 0,
};
#else
const RecorderProfile &profile = settings->profiles[settings->profileIndex];