diff --git a/src/jobs/audiothumbjob.cpp b/src/jobs/audiothumbjob.cpp
index 74f2b87fc..7e9b305f1 100644
--- a/src/jobs/audiothumbjob.cpp
+++ b/src/jobs/audiothumbjob.cpp
@@ -1,433 +1,439 @@
/***************************************************************************
* Copyright (C) 2017 by Nicolas Carion *
* This file is part of Kdenlive. See www.kdenlive.org. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) version 3 or any later version accepted by the *
* membership of KDE e.V. (or its successor approved by the membership *
* of KDE e.V.), which shall act as a proxy defined in Section 14 of *
* version 3 of the license. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see . *
***************************************************************************/
#include "audiothumbjob.hpp"
#include "bin/projectclip.h"
#include "bin/projectitemmodel.h"
#include "core.h"
#include "doc/kdenlivedoc.h"
#include "doc/kthumb.h"
#include "kdenlivesettings.h"
#include "klocalizedstring.h"
#include "lib/audio/audioStreamInfo.h"
#include "macros.hpp"
#include "utils/thumbnailcache.hpp"
#include
#include
#include
#include
#include
AudioThumbJob::AudioThumbJob(const QString &binId)
: AbstractClipJob(AUDIOTHUMBJOB, binId)
, m_ffmpegProcess(nullptr)
{
}
const QString AudioThumbJob::getDescription() const
{
return i18n("Extracting audio thumb from clip %1", m_clipId);
}
bool AudioThumbJob::computeWithMlt()
{
m_audioLevels.clear();
m_errorMessage.clear();
// MLT audio thumbs: slower but safer
QString service = m_prod->get("mlt_service");
if (service == QLatin1String("avformat-novalidate")) {
service = QStringLiteral("avformat");
} else if (service.startsWith(QLatin1String("xml"))) {
service = QStringLiteral("xml-nogl");
}
QScopedPointer audioProducer(new Mlt::Producer(*m_prod->profile(), service.toUtf8().constData(), m_prod->get("resource")));
if (!audioProducer->is_valid()) {
m_errorMessage.append(i18n("Audio thumbs: cannot open file %1", m_prod->get("resource")));
return false;
}
audioProducer->set("video_index", "-1");
Mlt::Filter chans(*m_prod->profile(), "audiochannels");
Mlt::Filter converter(*m_prod->profile(), "audioconvert");
Mlt::Filter levels(*m_prod->profile(), "audiolevel");
audioProducer->attach(chans);
audioProducer->attach(converter);
audioProducer->attach(levels);
int last_val = 0;
double framesPerSecond = audioProducer->get_fps();
mlt_audio_format audioFormat = mlt_audio_s16;
QStringList keys;
keys.reserve(m_channels);
for (int i = 0; i < m_channels; i++) {
keys << "meta.media.audio_level." + QString::number(i);
}
double maxLevel = 1;
QVector mltLevels;
for (int z = 0; z < m_lengthInFrames; ++z) {
int val = (int)(100.0 * z / m_lengthInFrames);
if (last_val != val) {
emit jobProgress(val);
last_val = val;
}
QScopedPointer mltFrame(audioProducer->get_frame());
if ((mltFrame != nullptr) && mltFrame->is_valid() && (mltFrame->get_int("test_audio") == 0)) {
int samples = mlt_sample_calculator(float(framesPerSecond), m_frequency, z);
mltFrame->get_audio(audioFormat, m_frequency, m_channels, samples);
for (int channel = 0; channel < m_channels; ++channel) {
double lev = mltFrame->get_double(keys.at(channel).toUtf8().constData());
mltLevels << lev;
maxLevel = qMax(lev, maxLevel);
}
} else if (!mltLevels.isEmpty()) {
for (int channel = 0; channel < m_channels; channel++) {
mltLevels << mltLevels.last();
}
}
}
// Normalize
for (double &v : mltLevels) {
m_audioLevels << 255 * v / maxLevel;
}
m_done = true;
return true;
}
bool AudioThumbJob::computeWithFFMPEG()
{
QString filePath = m_prod->get("kdenlive:originalurl");
if (filePath.isEmpty()) {
filePath = m_prod->get("resource");
}
m_ffmpegProcess.reset(new QProcess);
QString thumbPath = m_binClip->getAudioThumbPath(m_audioStream, true);
int audioStreamIndex = m_binClip->getAudioStreamFfmpegIndex(m_audioStream);
if (!QFile::exists(thumbPath)) {
// Generate thumbnail used in monitor overlay
QStringList args;
args << QStringLiteral("-hide_banner") << QStringLiteral("-y")<< QStringLiteral("-i") << QUrl::fromLocalFile(filePath).toLocalFile() << QString("-filter_complex");
if (m_audioStream >= 0) {
args << QString("[a:%1]showwavespic=s=%2x%3:split_channels=1:scale=cbrt:colors=0xffdddd|0xddffdd").arg(audioStreamIndex).arg(m_thumbSize.width()).arg(m_thumbSize.height());
} else {
// Only 1 audio stream in clip
args << QString("[a]showwavespic=s=%2x%3:split_channels=1:scale=cbrt:colors=0xffdddd|0xddffdd").arg(m_thumbSize.width()).arg(m_thumbSize.height());
}
args << QStringLiteral("-frames:v") << QStringLiteral("1");
args << thumbPath;
qDebug()<<"=== FFARGS: "<kill();
}
m_done = true;
m_successful = false;
});
m_ffmpegProcess->start(KdenliveSettings::ffmpegpath(), args);
m_ffmpegProcess->waitForFinished(-1);
disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
if (m_ffmpegProcess->exitStatus() != QProcess::CrashExit) {
if (m_dataInCache || !KdenliveSettings::audiothumbnails()) {
m_done = true;
return true;
}
}
}
if (!m_dataInCache && !m_done && KdenliveSettings::audiothumbnails()) {
// Generate timeline audio thumbnail data
m_audioLevels.clear();
std::vector> channelFiles;
for (int i = 0; i < m_channels; i++) {
std::unique_ptr channelTmpfile(new QTemporaryFile());
if (!channelTmpfile->open()) {
m_errorMessage.append(i18n("Audio thumbs: cannot create temporary file, check disk space and permissions\n"));
return false;
}
channelTmpfile->close();
channelFiles.emplace_back(std::move(channelTmpfile));
}
// Always create audio thumbs from the original source file, because proxy
// can have a different audio config (channels / mono/ stereo)
QStringList args {QStringLiteral("-hide_banner"), QStringLiteral("-i"), QUrl::fromLocalFile(filePath).toLocalFile(), QStringLiteral("-progress")};
#ifdef Q_OS_WIN
args << QStringLiteral("-");
#else
args << QStringLiteral("/dev/stdout");
#endif
bool isFFmpeg = KdenliveSettings::ffmpegpath().contains(QLatin1String("ffmpeg"));
args << QStringLiteral("-filter_complex");
if (m_channels == 1) {
if (m_audioStream >= 0) {
args << QStringLiteral("[a:%1]aformat=channel_layouts=mono,%2=100").arg(audioStreamIndex).arg(isFFmpeg ? "aresample=async" : "sample_rates");
} else {
args << QStringLiteral("[a]aformat=channel_layouts=mono,%1=100").arg(isFFmpeg ? "aresample=async" : "sample_rates");
}
/*args << QStringLiteral("-map") << QStringLiteral("0:a%1").arg(m_audioStream > 0 ? ":" + QString::number(audioStreamIndex) : QString())*/
args << QStringLiteral("-c:a") << QStringLiteral("pcm_s16le") << QStringLiteral("-frames:v")
<< QStringLiteral("1") << QStringLiteral("-y") << QStringLiteral("-f") << QStringLiteral("data")
<< channelFiles[0]->fileName();
} else {
QString aformat = QStringLiteral("[a%1]%2=100,channelsplit=channel_layout=%3")
.arg(audioStreamIndex >= 0 ? ":" + QString::number(audioStreamIndex) : QString())
.arg(isFFmpeg ? "aresample=async" : "aformat=sample_rates")
.arg(m_channels > 2 ? "5.1" : "stereo");
for (int i = 0; i < m_channels; ++i) {
aformat.append(QStringLiteral("[0:%1]").arg(i));
}
args << aformat;
args << QStringLiteral("-frames:v") << QStringLiteral("1");
for (int i = 0; i < m_channels; i++) {
// Channel 1
args << QStringLiteral("-map") << QStringLiteral("[0:%1]").arg(i) << QStringLiteral("-c:a") << QStringLiteral("pcm_s16le") << QStringLiteral("-y")
<< QStringLiteral("-f") << QStringLiteral("data") << channelFiles[size_t(i)]->fileName();
}
}
m_ffmpegProcess.reset(new QProcess);
connect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress, Qt::UniqueConnection);
connect(this, &AudioThumbJob::jobCanceled, [&]() {
if (m_ffmpegProcess) {
disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
m_ffmpegProcess->kill();
+ m_successful = false;
}
});
m_ffmpegProcess->start(KdenliveSettings::ffmpegpath(), args);
m_ffmpegProcess->waitForFinished(-1);
disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
- if (m_ffmpegProcess->exitStatus() != QProcess::CrashExit) {
+ if (m_successful && m_ffmpegProcess->exitStatus() != QProcess::CrashExit) {
int dataSize = 0;
std::vector rawChannels;
std::vector sourceChannels;
for (auto &channelFile : channelFiles) {
channelFile->open();
sourceChannels.emplace_back(channelFile->readAll());
QByteArray &res = sourceChannels.back();
channelFile->close();
if (dataSize == 0) {
dataSize = res.size();
}
if (res.isEmpty() || res.size() != dataSize) {
// Something went wrong, abort
m_errorMessage.append(i18n("Audio thumbs: error reading audio thumbnail created with FFmpeg\n"));
return false;
}
rawChannels.emplace_back(reinterpret_cast(res.constData()));
}
int progress = 0;
std::vector channelsData;
double offset = (double)dataSize / (2.0 * m_lengthInFrames);
int intraOffset = 1;
if (offset > 1000) {
intraOffset = offset / 60;
} else if (offset > 250) {
intraOffset = offset / 10;
}
long maxLevel = 1;
QVector ffmpegLevels;
for (int i = 0; i < m_lengthInFrames; i++) {
channelsData.resize((size_t)rawChannels.size());
std::fill(channelsData.begin(), channelsData.end(), 0);
int pos = (int)(i * offset);
int steps = 0;
for (int j = 0; j < (int)offset && (pos + j < dataSize); j += intraOffset) {
steps++;
for (size_t k = 0; k < rawChannels.size(); k++) {
channelsData[k] += abs(rawChannels[k][pos + j]);
}
}
for (long &k : channelsData) {
if (steps != 0) {
k /= steps;
}
maxLevel = qMax(k, maxLevel);
ffmpegLevels << k;
}
int p = 80 + (i * 20 / m_lengthInFrames);
if (p != progress) {
emit jobProgress(p);
progress = p;
}
}
for (long &v : ffmpegLevels) {
m_audioLevels << (uint8_t) (255 * v / maxLevel);
}
m_done = true;
return true;
}
}
if (!KdenliveSettings::audiothumbnails()) {
// We only wanted the thumb generation
return true;
}
QString err = m_ffmpegProcess->readAllStandardError();
m_ffmpegProcess.reset();
// m_errorMessage += err;
// m_errorMessage.append(i18n("Failed to create FFmpeg audio thumbnails, we now try to use MLT"));
qWarning() << "Failed to create FFmpeg audio thumbs:\n" << err << "\n---------------------";
return m_done;
}
void AudioThumbJob::updateFfmpegProgress()
{
if (m_ffmpegProcess == nullptr) {
return;
}
QString result = m_ffmpegProcess->readAllStandardOutput();
const QStringList lines = result.split(QLatin1Char('\n'));
for (const QString &data : lines) {
if (data.startsWith(QStringLiteral("out_time_ms"))) {
double ms = data.section(QLatin1Char('='), 1).toDouble();
emit jobProgress((int)(ms / m_binClip->duration().ms() / 10));
} else {
m_logDetails += data + QStringLiteral("\n");
}
}
}
bool AudioThumbJob::startJob()
{
if (m_done) {
return true;
}
m_dataInCache = false;
m_binClip = pCore->projectItemModel()->getClipByBinID(m_clipId);
if (m_binClip == nullptr) {
// Clip was deleted
return false;
}
if (m_binClip->audioChannels() == 0 || m_binClip->audioThumbCreated()) {
// nothing to do
m_done = true;
m_successful = true;
return true;
}
m_thumbSize = QSize(1000, 1000 / pCore->getCurrentDar());
m_prod = m_binClip->originalProducer();
m_frequency = m_binClip->audioInfo()->samplingRate();
m_frequency = m_frequency <= 0 ? 48000 : m_frequency;
m_channels = m_binClip->audioInfo()->channels();
m_channels = m_channels <= 0 ? 2 : m_channels;
m_lengthInFrames = m_prod->get_length(); // Multiply this if we want more than 1 sample per frame
QMap streams = m_binClip->audioInfo()->streams();
if ((m_prod == nullptr) || !m_prod->is_valid()) {
m_errorMessage.append(i18n("Audio thumbs: cannot open project file %1", m_binClip->url()));
m_done = true;
m_successful = false;
return false;
}
QMapIterator st(streams);
while (st.hasNext()) {
st.next();
int stream = st.key();
// Generate one thumb per stream
m_audioStream = stream;
m_cachePath = m_binClip->getAudioThumbPath(stream);
// checking for cached thumbs
QImage image(m_cachePath);
if (!image.isNull()) {
// Audio cache already exists
continue;
}
m_done = false;
bool ok = m_binClip->clipType() == ClipType::Playlist ? (KdenliveSettings::audiothumbnails() ? false : true) : computeWithFFMPEG();
if (!m_done) {
ok = ok ? ok : computeWithMlt();
}
Q_ASSERT(ok == m_done);
+ if (!m_successful) {
+ // Job was aborted
+ m_done = true;
+ return false;
+ }
if (ok && m_done && !m_audioLevels.isEmpty()) {
// Put into an image for caching.
int count = m_audioLevels.size();
image = QImage((int)lrint((count + 3) / 4.0 / m_channels), m_channels, QImage::Format_ARGB32);
int n = image.width() * image.height();
for (int i = 0; i < n; i++) {
QRgb p;
if ((4 * i + 3) < count) {
p = qRgba(m_audioLevels.at(4 * i), m_audioLevels.at(4 * i + 1), m_audioLevels.at(4 * i + 2),
m_audioLevels.at(4 * i + 3));
} else {
int last = m_audioLevels.last();
int r = (4 * i + 0) < count ? m_audioLevels.at(4 * i + 0) : last;
int g = (4 * i + 1) < count ? m_audioLevels.at(4 * i + 1) : last;
int b = (4 * i + 2) < count ? m_audioLevels.at(4 * i + 2) : last;
int a = last;
p = qRgba(r, g, b, a);
}
image.setPixel(i / m_channels, i % m_channels, p);
}
image.save(m_cachePath);
}
m_audioLevels.clear();
}
if (m_done || !KdenliveSettings::audiothumbnails()) {
m_successful = true;
return true;
}
m_done = true;
m_successful = false;
return false;
}
bool AudioThumbJob::commitResult(Fun &undo, Fun &redo)
{
Q_ASSERT(!m_resultConsumed);
m_ffmpegProcess.reset();
if (!m_done) {
qDebug() << "ERROR: Trying to consume invalid results";
return false;
}
m_resultConsumed = true;
if (!m_successful) {
return false;
}
QImage oldImage;
QImage result;
if (m_binClip->clipType() == ClipType::Audio) {
oldImage = m_binClip->thumbnail(m_thumbSize.width(), m_thumbSize.height()).toImage();
result = ThumbnailCache::get()->getAudioThumbnail(m_clipId);
}
// note that the image is moved into lambda, it won't be available from this class anymore
auto operation = [clip = m_binClip, image = std::move(result)]() {
clip->updateAudioThumbnail();
if (!image.isNull() && clip->clipType() == ClipType::Audio) {
clip->setThumbnail(image);
}
return true;
};
auto reverse = [clip = m_binClip, image = std::move(oldImage)]() {
clip->updateAudioThumbnail();
if (!image.isNull() && clip->clipType() == ClipType::Audio) {
clip->setThumbnail(image);
}
return true;
};
bool ok = operation();
if (ok) {
UPDATE_UNDO_REDO_NOLOCK(operation, reverse, undo, redo);
}
return ok;
}