diff --git a/src/externalextractor.cpp b/src/externalextractor.cpp index 50b4f2b..9c2f353 100644 --- a/src/externalextractor.cpp +++ b/src/externalextractor.cpp @@ -1,174 +1,174 @@ /* * This file is part of the KFileMetaData project * Copyright (C) 2016 Varun Joshi * Copyright (C) 2015 Boudhayan Gupta * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "externalextractor.h" #include "kfilemetadata_debug.h" #include "properties.h" #include "propertyinfo.h" #include "typeinfo.h" #include #include #include #include #include #define EXTRACTOR_TIMEOUT_MS 30000 using namespace KFileMetaData; class Q_DECL_HIDDEN ExternalExtractor::ExternalExtractorPrivate { public: QString path; QStringList writeMimetypes; QString mainPath; }; ExternalExtractor::ExternalExtractor(QObject* parent) : ExtractorPlugin(parent), d_ptr(new ExternalExtractorPrivate) { } ExternalExtractor::ExternalExtractor(const QString& pluginPath) : ExtractorPlugin(nullptr), d_ptr(new ExternalExtractorPrivate) { Q_D(ExternalExtractor); d->path = pluginPath; QDir pluginDir(pluginPath); QStringList pluginDirContents = pluginDir.entryList(QDir::Files | QDir::NoDotAndDotDot); if (!pluginDirContents.contains(QStringLiteral("manifest.json"))) { qCDebug(KFILEMETADATA_LOG) << pluginPath << "does not seem to contain a valid plugin"; return; } QFile manifest(pluginDir.filePath(QStringLiteral("manifest.json"))); manifest.open(QIODevice::ReadOnly); QJsonDocument manifestDoc = QJsonDocument::fromJson(manifest.readAll()); if (!manifestDoc.isObject()) { qCDebug(KFILEMETADATA_LOG) << "Manifest does not seem to be a valid JSON Object"; return; } QJsonObject rootObject = manifestDoc.object(); const QJsonArray mimetypesArray = rootObject.value(QStringLiteral("mimetypes")).toArray(); QStringList mimetypes; mimetypes.reserve(mimetypesArray.count()); - for (const QVariant &mimetype : mimetypesArray) { + for (const QJsonValue &mimetype : mimetypesArray) { mimetypes << mimetype.toString(); } d->writeMimetypes.append(mimetypes); d->mainPath = pluginDir.filePath(rootObject[QStringLiteral("main")].toString()); } ExternalExtractor::~ExternalExtractor() { delete d_ptr; } QStringList ExternalExtractor::mimetypes() const { Q_D(const ExternalExtractor); return d->writeMimetypes; } void ExternalExtractor::extract(ExtractionResult* result) { Q_D(ExternalExtractor); QJsonDocument writeData; QJsonObject writeRootObject; QByteArray output; QByteArray errorOutput; writeRootObject[QStringLiteral("path")] = QJsonValue(result->inputUrl()); writeRootObject[QStringLiteral("mimetype")] = result->inputMimetype(); writeData.setObject(writeRootObject); QProcess extractorProcess; extractorProcess.start(d->mainPath, QStringList(), QIODevice::ReadWrite); bool started = extractorProcess.waitForStarted(); if (!started) { qCWarning(KFILEMETADATA_LOG) << "External extractor" << d->mainPath << "failed to start:" << extractorProcess.errorString(); return; } extractorProcess.write(writeData.toJson()); extractorProcess.closeWriteChannel(); extractorProcess.waitForFinished(EXTRACTOR_TIMEOUT_MS); output = extractorProcess.readAll(); errorOutput = extractorProcess.readAllStandardError(); if (extractorProcess.exitStatus()) { qCWarning(KFILEMETADATA_LOG) << "External extractor" << d->mainPath << "failed to index" << result->inputUrl() << "-" << errorOutput; return; } // now we read in the output (which is a standard json format) into the // ExtractionResult QJsonDocument extractorData = QJsonDocument::fromJson(output); if (!extractorData.isObject()) { return; } QJsonObject rootObject = extractorData.object(); QJsonObject propertiesObject = rootObject[QStringLiteral("properties")].toObject(); const auto propertiesObjectEnd = propertiesObject.constEnd(); auto i = propertiesObject.constBegin(); for (; i != propertiesObjectEnd; ++i) { if (i.key() == QStringLiteral("typeInfo")) { TypeInfo info = TypeInfo::fromName(propertiesObject.value(i.key()).toString()); result->addType(info.type()); continue; } // for plaintext extraction if (i.key() == QStringLiteral("text")) { result->append(propertiesObject.value(i.key()).toString(QStringLiteral(""))); continue; } PropertyInfo info = PropertyInfo::fromName(i.key()); if (info.name() != i.key()) { continue; } result->add(info.property(), i.value().toVariant()); } if (rootObject[QStringLiteral("status")].toString() != QStringLiteral("OK")) { qCDebug(KFILEMETADATA_LOG) << rootObject[QStringLiteral("error")].toString(); } } diff --git a/src/externalwriter.cpp b/src/externalwriter.cpp index 5496901..38e33f3 100644 --- a/src/externalwriter.cpp +++ b/src/externalwriter.cpp @@ -1,147 +1,147 @@ /* * This file is part of the KFileMetaData project * Copyright (C) 2016 Varun Joshi * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "externalwriter.h" #include "properties.h" #include "propertyinfo.h" #include "kfilemetadata_debug.h" #include #include #include #include #include #define WRITER_TIMEOUT_MS 30000 using namespace KFileMetaData; class Q_DECL_HIDDEN ExternalWriter::ExternalWriterPrivate { public: QString path; QStringList writeMimetypes; QString mainPath; }; ExternalWriter::ExternalWriter(QObject* parent) : WriterPlugin(parent), d_ptr(new ExternalWriterPrivate) { } ExternalWriter::~ExternalWriter() { delete d_ptr; } ExternalWriter::ExternalWriter(const QString& pluginPath) : WriterPlugin(new QObject()), d_ptr(new ExternalWriterPrivate) { Q_D(ExternalWriter); d->path = pluginPath; QDir pluginDir(pluginPath); QStringList pluginDirContents = pluginDir.entryList(); if (!pluginDirContents.contains(QStringLiteral("manifest.json"))) { qCDebug(KFILEMETADATA_LOG) << "Path does not seem to contain a valid plugin"; return; } QFile manifest(pluginDir.filePath(QStringLiteral("manifest.json"))); manifest.open(QIODevice::ReadOnly); QJsonDocument manifestDoc = QJsonDocument::fromJson(manifest.readAll()); if (!manifestDoc.isObject()) { qCDebug(KFILEMETADATA_LOG) << "Manifest does not seem to be a valid JSON Object"; return; } QJsonObject rootObject = manifestDoc.object(); const QJsonArray mimetypesArray = rootObject.value(QStringLiteral("mimetypes")).toArray(); QStringList mimetypes; - for (const QVariant &mimetype : mimetypesArray) { + for (const QJsonValue &mimetype : mimetypesArray) { mimetypes << mimetype.toString(); } d->writeMimetypes.append(mimetypes); d->mainPath = pluginDir.filePath(rootObject[QStringLiteral("main")].toString()); } QStringList ExternalWriter::writeMimetypes() const { Q_D(const ExternalWriter); return d->writeMimetypes; } void ExternalWriter::write(const WriteData& data) { Q_D(ExternalWriter); QJsonDocument writeData; QJsonObject rootObject; QJsonObject propertiesObject; QByteArray output; QByteArray errorOutput; const QMap properties = data.getAllProperties(); QMap::const_iterator i = properties.constBegin(); while (i != properties.constEnd()) { PropertyInfo propertyInfo(i.key()); propertiesObject[propertyInfo.name()] = QJsonValue::fromVariant(properties[i.key()]); } rootObject[QStringLiteral("path")] = QJsonValue(data.inputUrl()); rootObject[QStringLiteral("mimetype")] = data.inputMimetype(); rootObject[QStringLiteral("properties")] = propertiesObject; writeData.setObject(rootObject); QProcess writerProcess; writerProcess.start(d->mainPath, QStringList(), QIODevice::ReadWrite); writerProcess.write(writeData.toJson()); writerProcess.closeWriteChannel(); writerProcess.waitForFinished(WRITER_TIMEOUT_MS); errorOutput = writerProcess.readAllStandardError(); if (writerProcess.exitStatus()) { qCDebug(KFILEMETADATA_LOG) << "Something went wrong while trying to write data"; qCDebug(KFILEMETADATA_LOG) << errorOutput; return; } output = writerProcess.readAll(); QJsonDocument writerExitData = QJsonDocument::fromJson(output); if (!writerExitData.isObject()) { return; } QJsonObject outputRootObject = writerExitData.object(); if (outputRootObject[QStringLiteral("status")].toString() != QStringLiteral("OK")) { qCDebug(KFILEMETADATA_LOG) << outputRootObject[QStringLiteral("error")].toString(); qCDebug(KFILEMETADATA_LOG) << errorOutput; } } diff --git a/src/extractorcollection.cpp b/src/extractorcollection.cpp index 5b93267..7768ff0 100644 --- a/src/extractorcollection.cpp +++ b/src/extractorcollection.cpp @@ -1,220 +1,220 @@ /* * Copyright (C) 2012 Vishesh Handa * Copyright (C) 2016 Varun Joshi * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "extractorcollection.h" #include "extractor.h" #include "extractor_p.h" #include "extractorplugin.h" #include "externalextractor.h" #include "kfilemetadata_debug.h" #include "config-kfilemetadata.h" #include #include #include #include #include using namespace KFileMetaData; class Q_DECL_HIDDEN ExtractorCollection::Private { public: QHash m_mimeExtractors; std::vector m_allExtractors; void findExtractors(); QList getExtractors(const QString& mimetype); }; ExtractorCollection::ExtractorCollection() : d(new Private) { d->findExtractors(); } ExtractorCollection::~ExtractorCollection() { delete d; } QList ExtractorCollection::allExtractors() { QList plugins; for (auto& ex : d->m_allExtractors) { if (ex.d->initPlugin()) { plugins.push_back(&ex); } } return plugins; } void ExtractorCollection::Private::findExtractors() { QStringList plugins; QStringList externalPlugins; const QStringList paths = QCoreApplication::libraryPaths(); for (const QString& libraryPath : paths) { QString path(libraryPath + QStringLiteral("/kf5/kfilemetadata")); QDir dir(path); qCDebug(KFILEMETADATA_LOG) << "Searching for extractors:" << dir.path(); if (!dir.exists()) { continue; } const QStringList entryList = dir.entryList(QDir::Files | QDir::NoDotAndDotDot); for (const QString& fileName : entryList) { // Make sure the same plugin is not loaded twice, even if it is // installed in two different locations if (plugins.contains(fileName)) { qCDebug(KFILEMETADATA_LOG) << "Skipping duplicate - " << path << ":" << fileName; continue; } plugins << fileName; Extractor extractor; extractor.setAutoDeletePlugin(Extractor::DoNotDeletePlugin); auto pluginPath = dir.absoluteFilePath(fileName); QPluginLoader loader(pluginPath); auto metadata = loader.metaData().value(QLatin1String("MetaData")); if (metadata.type() == QJsonValue::Object) { qCDebug(KFILEMETADATA_LOG) << "Found plugin with metadata:" << metadata.toObject(); auto pluginProperties = metadata.toObject().toVariantMap(); extractor.setMetaData(pluginProperties); extractor.d->m_pluginPath = pluginPath; m_allExtractors.push_back(std::move(extractor)); } else { qCDebug(KFILEMETADATA_LOG) << "Found plugin without metadata:" << pluginPath; extractor.d->m_pluginPath = pluginPath; if (extractor.d->initPlugin() && !extractor.mimetypes().isEmpty()) { m_allExtractors.push_back(std::move(extractor)); } } } } plugins.clear(); QDir externalPluginDir(QStringLiteral(LIBEXEC_INSTALL_DIR) + QStringLiteral("/kfilemetadata/externalextractors")); qCDebug(KFILEMETADATA_LOG) << "Searching for external extractors:" << externalPluginDir.path(); // For external plugins, we look into the directories const QStringList externalPluginEntryList = externalPluginDir.entryList(QDir::Dirs | QDir::NoDotAndDotDot); for (const QString& externalPlugin : externalPluginEntryList) { if (externalPlugins.contains(externalPlugin)) { qCDebug(KFILEMETADATA_LOG) << "Skipping duplicate - " << externalPluginDir.path() << ":" << externalPlugin; continue; } qCDebug(KFILEMETADATA_LOG) << "Adding plugin - " << externalPluginDir.path() << ":" << externalPlugin; externalPlugins << externalPlugin; Extractor extractor; auto pluginPath = externalPluginDir.absoluteFilePath(externalPlugin); ExternalExtractor *plugin = new ExternalExtractor(pluginPath); if (plugin && !plugin->mimetypes().isEmpty()) { extractor.setExtractorPlugin(plugin); extractor.setAutoDeletePlugin(Extractor::AutoDeletePlugin); m_allExtractors.push_back(std::move(extractor)); } } externalPlugins.clear(); for (Extractor& extractor : m_allExtractors) { auto pluginProperties = extractor.extractorProperties(); if (!pluginProperties.isEmpty()) { auto mimetypeProperties = pluginProperties[QLatin1String("MimeTypes")]; const auto mimetypes = mimetypeProperties.toMap().keys(); - for (QString mimetype : mimetypes) { + for (const QString &mimetype : mimetypes) { m_mimeExtractors.insertMulti(mimetype, &extractor); } } else if (extractor.d->m_plugin) { const auto mimetypes = extractor.mimetypes(); - for (QString mimetype : mimetypes) { + for (const QString &mimetype : mimetypes) { m_mimeExtractors.insertMulti(mimetype, &extractor); } } } } QList ExtractorCollection::Private::getExtractors(const QString& mimetype) { QList extractors = m_mimeExtractors.values(mimetype); if (extractors.isEmpty()) { qCDebug(KFILEMETADATA_LOG) << "No extractor for" << mimetype; return extractors; } qCDebug(KFILEMETADATA_LOG) << "Fetching extractors for" << mimetype; Extractor* failed = nullptr; for (auto ex : extractors) { if (!ex->d->initPlugin()) { failed = ex; break; } } if (!failed) { return extractors; } auto it = m_mimeExtractors.begin(); while (it != m_mimeExtractors.end()) { if (it.value() == failed) { it = m_mimeExtractors.erase(it); } else { ++it; } } return getExtractors(mimetype); } QList ExtractorCollection::fetchExtractors(const QString& mimetype) const { QList plugins = d->getExtractors(mimetype); if (!plugins.isEmpty()) { return plugins; } // try to find the best matching more generic extractor by mimetype inheritance QMimeDatabase db; auto type = db.mimeTypeForName(mimetype); const QStringList ancestors = type.allAncestors(); - for (auto ancestor : ancestors) { + for (const auto &ancestor : ancestors) { if (ancestor == QLatin1String("application/octet-stream")) { continue; } QList plugins = d->getExtractors(ancestor); if (!plugins.isEmpty()) { qCDebug(KFILEMETADATA_LOG) << "Using inherited mimetype" << ancestor << "for" << mimetype; return plugins; } } return plugins; } diff --git a/src/mimeutils.cpp b/src/mimeutils.cpp index 7ea4dc8..d09d5da 100644 --- a/src/mimeutils.cpp +++ b/src/mimeutils.cpp @@ -1,50 +1,50 @@ /* * This file is part of KFileMetaData * Copyright (C) 2019 Stefan Brüns * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "mimeutils.h" namespace KFileMetaData { namespace MimeUtils { QMimeType strictMimeType(const QString& filePath, const QMimeDatabase& db) { auto extensionMimes = db.mimeTypesForFileName(filePath); auto contentMime = db.mimeTypeForFile(filePath, QMimeDatabase::MatchContent); if (extensionMimes.contains(contentMime)) { // content based mime type is one of the types for the file extension, e.g.: // *.ogg -> [ audio/ogg, audio/x-vorbis+ogg, ...] // content -> audio/x-vorbis+ogg return contentMime; } - for (auto mime : extensionMimes) { + for (const auto &mime : extensionMimes) { // check if the content is generic and the extension is more specific, e.g.: // *.mkv -> [ video/matroska ] // content -> application/matroska if (mime.inherits(contentMime.name())) { return mime; } } // content mime type does not match the extension, trust the content return contentMime; } }} // namespace KFileMetaData::MimeUtils