diff --git a/core/libs/database/collection/collectionmanager.h b/core/libs/database/collection/collectionmanager.h index 65543d6b56..f6f920fc41 100644 --- a/core/libs/database/collection/collectionmanager.h +++ b/core/libs/database/collection/collectionmanager.h @@ -1,345 +1,351 @@ /* ============================================================ * * This file is a part of digiKam project * https://www.digikam.org * * Date : 2007-04-09 * Description : Collection location management * * Copyright (C) 2007-2009 by Marcel Wiesweg * * This program is free software; you can redistribute it * and/or modify it under the terms of the GNU General * Public License as published by the Free Software Foundation; * either version 2, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * ============================================================ */ #ifndef DIGIKAM_COLLECTION_MANAGER_H #define DIGIKAM_COLLECTION_MANAGER_H // Qt includes #include #include #include #include // Local includes #include "digikam_export.h" namespace Digikam { class CollectionLocation; class AlbumRootChangeset; class DIGIKAM_DATABASE_EXPORT CollectionManager : public QObject { Q_OBJECT public: enum LocationCheckResult { /// The check did not succeed, status unknown LocationInvalidCheck, /// All right. The accompanying message may be empty. LocationAllRight, /// Location can be added, but the user should be aware of a problem LocationHasProblems, /// Adding the location will fail (e.g. there is already a location for the path) LocationNotAllowed }; public: static CollectionManager* instance(); static void cleanUp(); /** * Disables the collection watch. * It will be reenabled as soon as refresh() is called * or any other action triggered. */ void setWatchDisabled(); /** * Clears all locations and re-reads the lists of collection locations. * Enables the watch. */ void refresh(); private Q_SLOTS: void deviceAdded(const QString&); void deviceRemoved(const QString&); void accessibilityChanged(bool, const QString&); private: CollectionManager(); ~CollectionManager(); void clearLocations(); Q_PRIVATE_SLOT(d, void slotTriggerUpdateVolumesList()) Q_SIGNALS: // internal use only with slotTriggerUpdateVolumesList() void triggerUpdateVolumesList(); // ----------------------------------------------------------------------------- /** @name Operations on Collection Location */ //@{ public: /** * Add the given file system location as new collection location. * Type and availability will be detected. * On failure returns null. This would be the case if the given * url is already contained in another collection location. * You may pass an optional user-visible label that will be stored in the database. * The label has no further meaning and can be freely chosen. * * CollectionLocation objects returned are simple data containers. * If the corresponding location is returned, the data is still safe to access, * but does not represent anything. * Therefore, do not store returned objects, but prefer to retrieve them freshly. */ CollectionLocation addLocation(const QUrl& fileUrl, const QString& label = QString()); CollectionLocation addNetworkLocation(const QUrl& fileUrl, const QString& label = QString()); CollectionLocation refreshLocation(const CollectionLocation& location, int newType, const QUrl& fileUrl, const QString& label = QString()); /** * Analyzes the given file path. Creates an info message * describing the result of identification or possible problems. * The text is i18n'ed and can be presented to the user. * The returned result enum describes the test result. */ LocationCheckResult checkLocation(const QUrl& fileUrl, QList assumeDeleted, QString* message = nullptr, QString* suggestedMessageIconName = nullptr); LocationCheckResult checkNetworkLocation(const QUrl& fileUrl, QList assumeDeleted, QString* message = nullptr, QString* suggestedMessageIconName = nullptr); /** * Removes the given location. This means that all images contained on the * location will be removed from the database, all tags will be lost. */ void removeLocation(const CollectionLocation& location); /** * Sets the label of the given location */ void setLabel(const CollectionLocation& location, const QString& label); /** * Changes the CollectionLocation::Type of the given location */ void changeType(const CollectionLocation& location, int type); /** * Checks the locations of type HardWired. If one of these is not available currently, * it is added to the list of disappeared locations. * This case may happen if a file system is changed, a backup restored or other actions * taken that change the UUID, although the data may still be available and mounted. * If there are hard-wired volumes available which are candidates for a newly appeared * volume (in fact those that do not contain any collections currently), they are * added to the map, identifier -> i18n'ed user presentable description. * The identifier can be used for changeVolume. */ QList checkHardWiredLocations(); /** * For a given disappeared location (retrieved from checkHardWiredLocations()) * retrieve a user-presentable technical description (excluding the CollectionLocation's label) * and a list of identifiers and corresponding user presentable strings of candidates * to where the given location may have been moved. */ void migrationCandidates(const CollectionLocation& disappearedLocation, QString* const technicalDescription, QStringList* const candidateIdentifiers, QStringList* const candidateDescriptions); /** * Migrates the existing collection to a new volume, identified by an internal identifier * as returned by checkHardWiredLocations(). * Use this _only_ to react to changes like those detailed for checkHardWiredLocations; * the actual data pointed to shall be unchanged. */ void migrateToVolume(const CollectionLocation& location, const QString& identifier); /** * Returns a list of all CollectionLocations stored in the database */ QList allLocations(); /** * Returns a list of all currently available CollectionLocations */ QList allAvailableLocations(); /** * Returns the location for the given album root id */ CollectionLocation locationForAlbumRootId(int id); /** * Returns the CollectionLocation that contains the given album root. * The path must be an album root with isAlbumRoot() == true. * Returns 0 if no collection location matches. * Only available (or hidden, but available) locations are guaranteed to be found. */ CollectionLocation locationForAlbumRoot(const QUrl& fileUrl); CollectionLocation locationForAlbumRootPath(const QString& albumRootPath); /** * Returns the CollectionLocation that contains the given path. * Equivalent to calling locationForAlbumRoot(albumRoot(fileUrl)). * Only available (or hidden, but available) locations are guaranteed to be found. */ CollectionLocation locationForUrl(const QUrl& fileUrl); CollectionLocation locationForPath(const QString& filePath); private: void updateLocations(); Q_SIGNALS: /** * Emitted when the status of a collection location changed. * This means that the location became available, hidden or unavailable. * * An added location will change its status after addition, * from Null to Available, Hidden or Unavailable. * * A removed location will change its status to Deleted * during the removal; in this case, you shall not use the object * passed with this signal with any method of CollectionManager. * * The second signal argument is of type CollectionLocation::Status * and describes the status before the state change occurred */ void locationStatusChanged(const CollectionLocation& location, int oldStatus); /** * Emitted when the label of a collection location is changed */ void locationPropertiesChanged(const CollectionLocation& location); //@} // ----------------------------------------------------------------------------- /** @name Operations on Albums */ //@{ public: /** * Returns a list of the paths of all currently available root paths */ QStringList allAvailableAlbumRootPaths(); /** * Returns the album root path with the given id. * Returns a null QString if the root path does not exist or is not available. */ QString albumRootPath(int id); /** * Returns the album root label with the given id. * Returns a null QString if the root path does not exist or is not available. */ QString albumRootLabel(int id); + //@{ /** * For a given path, the part of the path that forms the album root is returned, * ending without a slash. Example: "/media/fotos/Paris 2007" gives "/media/fotos". * Only available (or hidden, but available) album roots are guaranteed to be found. */ QUrl albumRoot(const QUrl& fileUrl); QString albumRootPath(const QUrl& fileUrl); QString albumRootPath(const QString& filePath); + //@} /** * Returns true if the given path forms an album root. * It will return false if the path is a path below an album root, * or if the path does not belong to an album root. * Example: "/media/fotos/Paris 2007" is an album with album root "/media/fotos". * "/media/fotos" returns true, "/media/fotos/Paris 2007" and "/media" return false. * Only available (or hidden, but available) album roots are guaranteed to be found. */ bool isAlbumRoot(const QUrl& fileUrl); /** * The file path should not end with the directory slash. Using CoreDbUrl's method is fine. */ bool isAlbumRoot(const QString& filePath); + //@{ /** * Returns the album part of the given file path, i.e. the album root path * at the beginning is removed and the second part, starting with "/", ending without a slash, * is returned. Example: "/media/fotos/Paris 2007" gives "/Paris 2007" * Returns a null QString if the file path is not located in an album root. * Returns "/" if the file path is an album root. * Note that trailing slashes are removed in the return value, regardless if there was * one or not. * Note that you have to feed a path/url pointing to a directory. File names cannot * be recognized as such by this method, and will be treated as a directory. */ QString album(const QUrl& fileUrl); QString album(const QString& filePath); QString album(const CollectionLocation& location, const QUrl& fileUrl); QString album(const CollectionLocation& location, const QString& filePath); + //@} + //@{ /** * Returns just one album root, out of the list of available location, * the one that is most suitable to serve as a default, e.g. * to suggest as default place when the user wants to add files. */ QUrl oneAlbumRoot(); QString oneAlbumRootPath(); + //@} private Q_SLOTS: void slotAlbumRootChange(const AlbumRootChangeset& changeset); //@} public: class Private; private: static CollectionManager* m_instance; Private* const d; friend class Private; friend class CoreDbWatch; friend class CoreDbAccess; }; } // namespace Digikam #endif // DIGIKAM_COLLECTION_MANAGER_H diff --git a/core/libs/database/collection/collectionmanager_p.h b/core/libs/database/collection/collectionmanager_p.h index e3763f95e0..948c4b309c 100644 --- a/core/libs/database/collection/collectionmanager_p.h +++ b/core/libs/database/collection/collectionmanager_p.h @@ -1,299 +1,299 @@ /* ============================================================ * * This file is a part of digiKam project * https://www.digikam.org * * Date : 2007-04-09 * Description : Collection location management - private containers. * * Copyright (C) 2007-2009 by Marcel Wiesweg * * This program is free software; you can redistribute it * and/or modify it under the terms of the GNU General * Public License as published by the Free Software Foundation; * either version 2, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * ============================================================ */ #ifndef DIGIKAM_COLLECTION_MANAGER_P_H #define DIGIKAM_COLLECTION_MANAGER_P_H #include "collectionmanager.h" // Qt includes #include #include #include #include #include #include #include // KDE includes #include // Solid includes #if defined(Q_CC_CLANG) # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wnonportable-include-path" #endif #include #include #include #include #include #include #include #include #if defined(Q_CC_CLANG) # pragma clang diagnostic pop #endif // Local includes #include "digikam_debug.h" #include "coredbaccess.h" #include "coredbchangesets.h" #include "coredbtransaction.h" #include "coredb.h" #include "collectionscanner.h" #include "collectionlocation.h" namespace Digikam { class Q_DECL_HIDDEN AlbumRootLocation : public CollectionLocation { public: AlbumRootLocation() : available(false), hidden(false) { } explicit AlbumRootLocation(const AlbumRootInfo& info) { qCDebug(DIGIKAM_DATABASE_LOG) << "Creating new Location " << info.specificPath << " uuid " << info.identifier; m_id = info.id; m_type = (Type)info.type; QString path = info.specificPath; if (path != QLatin1String("/") && path.endsWith(QLatin1Char('/'))) { path.chop(1); } specificPath = path; identifier = info.identifier; m_label = info.label; m_path.clear(); setStatus((CollectionLocation::Status)info.status); } void setStatusFromFlags() { if (hidden) { m_status = CollectionLocation::LocationHidden; } else { if (available) { m_status = CollectionLocation::LocationAvailable; } else { m_status = CollectionLocation::LocationUnavailable; } } } void setStatus(CollectionLocation::Status s) { m_status = s; // status is exclusive, and Hidden wins // but really both states are independent // - a hidden location might or might not be available if (m_status == CollectionLocation::LocationAvailable) { available = true; hidden = false; } else if (m_status == CollectionLocation::LocationHidden) { available = false; hidden = true; } else // Unavailable, Null, Deleted { available = false; hidden = false; } } void setId(int id) { m_id = id; } void setAbsolutePath(const QString& path) { m_path = path; } void setType(Type type) { m_type = type; } void setLabel(const QString& label) { m_label = label; } public: QString identifier; QString specificPath; bool available; bool hidden; }; // ------------------------------------------------- class Q_DECL_HIDDEN SolidVolumeInfo { public: SolidVolumeInfo() : isRemovable(false), isOpticalDisc(false), isMounted(false) { } bool isNull() const { return path.isNull(); } public: - QString udi; // Solid device UDI of the StorageAccess device - QString path; // mount path of volume, with trailing slash - QString uuid; // UUID as from Solid - QString label; // volume label (think of CDs) - bool isRemovable; // may be removed - bool isOpticalDisc; // is an optical disk device as CD/DVD/BR - bool isMounted; // is mounted on File System. + QString udi; ///< Solid device UDI of the StorageAccess device + QString path; ///< mount path of volume, with trailing slash + QString uuid; ///< UUID as from Solid + QString label; ///< volume label (think of CDs) + bool isRemovable; ///< may be removed + bool isOpticalDisc; ///< is an optical disk device as CD/DVD/BR + bool isMounted; ///< is mounted on File System. }; // ------------------------------------------------- class Q_DECL_HIDDEN CollectionManager::Private { public: explicit Private(CollectionManager* const s); // hack for Solid's threading problems QList actuallyListVolumes(); void slotTriggerUpdateVolumesList(); QList volumesListCache; /// Access Solid and return a list of storage volumes QList listVolumes(); /** * Find from a given list (usually the result of listVolumes) the volume * corresponding to the location */ SolidVolumeInfo findVolumeForLocation(const AlbumRootLocation* location, const QList volumes); /** * Find from a given list (usually the result of listVolumes) the volume * on which the file path specified by the url is located. */ SolidVolumeInfo findVolumeForUrl(const QUrl& fileUrl, const QList volumes); /// Create the volume identifier for the given volume info static QString volumeIdentifier(const SolidVolumeInfo& info); /// Create a volume identifier based on the path only QString volumeIdentifier(const QString& path); /// Create a network share identifier based on the mountpath QString networkShareIdentifier(const QString& path); /// Return the path, if location has a path-only identifier. Else returns a null string. QString pathFromIdentifier(const AlbumRootLocation* location); /// Return the path, if location has a path-only identifier. Else returns a null string. QStringList networkShareMountPathsFromIdentifier(const AlbumRootLocation* location); /// Create an MD5 hash of the top-level entries (file names, not file content) of the given path static QString directoryHash(const QString& path); /// Check if a location for specified path exists, assuming the given list of locations was deleted bool checkIfExists(const QString& path, QList assumeDeleted); /// Make a user presentable description, regardless of current location status QString technicalDescription(const AlbumRootLocation* location); public: QReadWriteLock lock; QMap locations; bool changingDB; QStringList udisToWatch; bool watchEnabled; CollectionManager* s; }; // ------------------------------------------------- class Q_DECL_HIDDEN ChangingDB { public: explicit ChangingDB(CollectionManager::Private* const d) : d(d) { d->changingDB = true; } ~ChangingDB() { d->changingDB = false; } public: CollectionManager::Private* const d; }; } // namespace Digikam #endif // DIGIKAM_COLLECTION_MANAGER_P_H diff --git a/core/libs/database/collection/collectionscanner.h b/core/libs/database/collection/collectionscanner.h index 4a7c07b412..f58a98c60a 100644 --- a/core/libs/database/collection/collectionscanner.h +++ b/core/libs/database/collection/collectionscanner.h @@ -1,291 +1,298 @@ /* ============================================================ * * This file is a part of digiKam project * https://www.digikam.org * * Date : 2007-03-21 * Description : Collection scanning to database. * * Copyright (C) 2005 by Renchi Raju * Copyright (C) 2005-2006 by Tom Albers * Copyright (C) 2007-2011 by Marcel Wiesweg * Copyright (C) 2009-2019 by Gilles Caulier * * This program is free software; you can redistribute it * and/or modify it under the terms of the GNU General * Public License as published by the Free Software Foundation; * either version 2, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * ============================================================ */ #ifndef DIGIKAM_COLLECTION_SCANNER_H #define DIGIKAM_COLLECTION_SCANNER_H // Qt includes #include #include #include // Local includes #include "digikam_export.h" #include "coredbaccess.h" #include "coredbalbuminfo.h" #include "collectionscannerhints.h" class QFileInfo; namespace Digikam { class DIGIKAM_DATABASE_EXPORT CollectionScanner : public QObject { Q_OBJECT public: enum FileScanMode { /** * The file will be scanned like it is done for any usual scan. * If it was not modified, no further action is taken. * If the file is not known yet, it will be fully scanned, or, * if an identical file is found, this data will be copied. */ NormalScan, /** * The file will scanned like a modified file. Only a selected portion * of the metadata will be updated into the database. * If the file is not known yet, it will be fully scanned, or, * if an identical file is found, this data will be copied. */ ModifiedScan, /** * The file will be scanned like a completely new file. * The complete metadata is re-read into the database. * No search for identical files will be done. */ Rescan }; public: explicit CollectionScanner(); virtual ~CollectionScanner(); + //@{ /** * Hints give the scanner additional info about things that happened in the past * carried out by higher level which the collection scanner cannot know. * They allow to carry out optimizations. * Record hints in a container, and provide the container to the collection scanner. * The Container set in setHintContainer must be one created by createContainer. */ static CollectionScannerHintContainer* createHintContainer(); void setHintContainer(CollectionScannerHintContainer* const container); void setUpdateHashHint(bool hint = true); + //@} /** * Call this to enable the progress info signals. * Default is off. */ void setSignalsEnabled(bool on); /** * Call this to enable emitting the total files to scan * (for progress info) before a complete collection scan. * Default is off. If on, setSignalEnabled() must be on to take effect. */ void setNeedFileCount(bool on); /** * Set an observer to be able to cancel a running scan */ void setObserver(CollectionScannerObserver* const observer); void setDeferredFileScanning(bool defer); QStringList deferredAlbumPaths() const; // ----------------------------------------------------------------------------- /** @name Scan operations */ //@{ public: /** * Carries out a full scan on all available parts of the collection. * Only a full scan can finally remove deleted files from the database, * only a full scan will mark the database as scanned. * The database will be locked while running (Note: this is not done for partialScans). */ void completeScan(); /** * If you enable deferred file scanning for a completeScan(), new files * will not be scanned. The relevant albums are available from * deferredAlbumPaths() when completeScan() has finished. * You need to call finishCompleteScan() afterwards with the list * to get the same complete scan than undeferred completeScan(). */ void finishCompleteScan(const QStringList& albumPaths); /** * Returns if the initial scan of the database has been done. * This is the first complete scan after creation of a new database file * (or update requiring a rescan) */ static bool databaseInitialScanDone(); /** * Carries out a partial scan on the specified path of the collection. * The includes scanning for new files + albums and updating modified file data. * Files no longer found in the specified path however are not completely * removed, but only marked as removed. They will be removed only after a complete scan. */ void partialScan(const QString& filePath); /** * Same procedure as above, but albumRoot and album is provided. */ void partialScan(const QString& albumRoot, const QString& album); /** * The given file will be scanned according to the given mode. * Returns the image id of the file. */ qlonglong scanFile(const QString& filePath, FileScanMode mode = ModifiedScan); /** * Same procedure as above, but albumRoot and album is provided. * If you already have this info it need not be retrieved. * Returns the image id of the file, or -1 on failure. */ qlonglong scanFile(const QString& albumRoot, const QString& album, const QString& fileName, FileScanMode mode = ModifiedScan); /** * The given file represented by the ItemInfo will be scanned according to mode */ void scanFile(const ItemInfo& info, FileScanMode mode = ModifiedScan); protected: void scanForStaleAlbums(const QList& locations); void scanForStaleAlbums(const QList& locationIdsToScan); void scanAlbumRoot(const CollectionLocation& location); void scanAlbum(const CollectionLocation& location, const QString& album); void scanExistingFile(const QFileInfo& fi, qlonglong id); void scanFileNormal(const QFileInfo& info, const ItemScanInfo& scanInfo, bool checkSidecar = true); void scanModifiedFile(const QFileInfo& info, const ItemScanInfo& scanInfo); void scanFileUpdateHashReuseThumbnail(const QFileInfo& fi, const ItemScanInfo& scanInfo, bool fileWasEdited); void rescanFile(const QFileInfo& info, const ItemScanInfo& scanInfo); void completeScanCleanupPart(); void completeHistoryScanning(); void finishHistoryScanning(); void historyScanningStage2(const QList& ids); void historyScanningStage3(const QList& ids); qlonglong scanFile(const QFileInfo& fi, int albumId, qlonglong id, FileScanMode mode); qlonglong scanNewFile(const QFileInfo& info, int albumId); qlonglong scanNewFileFullScan(const QFileInfo& info, int albumId); //@} // ----------------------------------------------------------------------------- /** @name Scan utilities */ //@{ public: /** * Prepare the given albums to be removed, * typically by setting the albums as orphan * and removing all entries from the albums */ void safelyRemoveAlbums(const QList& albumIds); /** * When a file is derived from another file, typically through editing, * copy all relevant attributes from source file to the new file. */ static void copyFileProperties(const ItemInfo& source, const ItemInfo& dest); protected: void markDatabaseAsScanned(); void mainEntryPoint(bool complete); int checkAlbum(const CollectionLocation& location, const QString& album); void itemsWereRemoved(const QList& removedIds); void updateRemovedItemsTime(); void incrementDeleteRemovedCompleteScanCount(); void resetDeleteRemovedSettings(); bool checkDeleteRemoved(); void loadNameFilters(); int countItemsInFolder(const QString& directory); DatabaseItem::Category category(const QFileInfo& info); //@} Q_SIGNALS: /** * Emitted once in scanAlbums(), the scan() methods, and updateItemsWithoutDate(). * Gives the number of the files that need to be scanned. */ void totalFilesToScan(int count); + //@{ /** * Notifies the begin of the scanning of the specified album root, * album, of stale files, or of the whole collection (after stale files) */ void startScanningAlbumRoot(const QString& albumRoot); void startScanningAlbum(const QString& albumRoot, const QString& album); void startScanningForStaleAlbums(); void startScanningAlbumRoots(); void startCompleteScan(); + //@} + //@{ /** * Emitted when the scanning has finished. * Note that start/finishScanningAlbum may be emitted recursively. */ void finishedScanningAlbumRoot(const QString& albumRoot); void finishedScanningAlbum(const QString& albumRoot, const QString& album, int filesScanned); void finishedScanningForStaleAlbums(); void finishedCompleteScan(); + //@} /** * Emitted between startScanningAlbum and finishedScanningAlbum. * In between these two signals, the sum of filesScanned of all sent signals * equals the one reported by finishedScanningAlbum() */ void scannedFiles(int filesScanned); + /** * Emitted when the observer told to cancel the scan */ void cancelled(); private: class Private; Private* const d; }; } // namespace Digikam #endif // DIGIKAM_COLLECTION_SCANNER_H diff --git a/core/libs/database/collection/collectionscanner_scan.cpp b/core/libs/database/collection/collectionscanner_scan.cpp index 892b93fbaf..937acbba03 100644 --- a/core/libs/database/collection/collectionscanner_scan.cpp +++ b/core/libs/database/collection/collectionscanner_scan.cpp @@ -1,1042 +1,1042 @@ /* ============================================================ * * This file is a part of digiKam project * https://www.digikam.org * * Date : 2007-03-21 * Description : Collection scanning to database - scan operations. * * Copyright (C) 2005-2006 by Tom Albers * Copyright (C) 2007-2011 by Marcel Wiesweg * Copyright (C) 2009-2019 by Gilles Caulier * * This program is free software; you can redistribute it * and/or modify it under the terms of the GNU General * Public License as published by the Free Software Foundation; * either version 2, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * ============================================================ */ #include "collectionscanner_p.h" namespace Digikam { void CollectionScanner::completeScan() { QTime time; time.start(); emit startCompleteScan(); { // lock database CoreDbTransaction transaction; mainEntryPoint(true); d->resetRemovedItemsTime(); } //TODO: Implement a mechanism to watch for album root changes while we keep this list QList allLocations = CollectionManager::instance()->allAvailableLocations(); if (d->wantSignals && d->needTotalFiles) { // count for progress info int count = 0; foreach (const CollectionLocation& location, allLocations) { count += countItemsInFolder(location.albumRootPath()); } emit totalFilesToScan(count); } if (!d->checkObserver()) { emit cancelled(); return; } // if we have no hints to follow, clean up all stale albums if (!d->hints || !d->hints->hasAlbumHints()) { CoreDbAccess().db()->deleteStaleAlbums(); } scanForStaleAlbums(allLocations); if (!d->checkObserver()) { emit cancelled(); return; } if (d->wantSignals) { emit startScanningAlbumRoots(); } foreach (const CollectionLocation& location, allLocations) { scanAlbumRoot(location); } // do not continue to clean up without a complete scan! if (!d->checkObserver()) { emit cancelled(); return; } if (d->deferredFileScanning) { qCDebug(DIGIKAM_DATABASE_LOG) << "Complete scan (file scanning deferred) took:" << time.elapsed() << "msecs."; emit finishedCompleteScan(); return; } CoreDbTransaction transaction; completeScanCleanupPart(); qCDebug(DIGIKAM_DATABASE_LOG) << "Complete scan took:" << time.elapsed() << "msecs."; } void CollectionScanner::finishCompleteScan(const QStringList& albumPaths) { emit startCompleteScan(); { // lock database CoreDbTransaction transaction; mainEntryPoint(true); d->resetRemovedItemsTime(); } if (!d->checkObserver()) { emit cancelled(); return; } if (d->wantSignals) { emit startScanningAlbumRoots(); } // remove subalbums from list if parent album is already contained QStringList sortedPaths = albumPaths; std::sort(sortedPaths.begin(), sortedPaths.end()); QStringList::iterator it, it2; for (it = sortedPaths.begin() ; it != sortedPaths.end() ; ) { // remove all following entries as long as they have the same beginning (= are subalbums) for (it2 = it + 1 ; it2 != sortedPaths.end() && it2->startsWith(*it) ; ) { it2 = sortedPaths.erase(it2); } it = it2; } if (d->wantSignals && d->needTotalFiles) { // count for progress info int count = 0; foreach (const QString& path, sortedPaths) { count += countItemsInFolder(path); } emit totalFilesToScan(count); } foreach (const QString& path, sortedPaths) { CollectionLocation location = CollectionManager::instance()->locationForPath(path); QString album = CollectionManager::instance()->album(path); if (album == QLatin1String("/")) { scanAlbumRoot(location); } else { scanAlbum(location, album); } } // do not continue to clean up without a complete scan! if (!d->checkObserver()) { emit cancelled(); return; } CoreDbTransaction transaction; completeScanCleanupPart(); } void CollectionScanner::completeScanCleanupPart() { completeHistoryScanning(); updateRemovedItemsTime(); // Items may be set to status removed, without being definitely deleted. // This deletion shall be done after a certain time, as checked by checkedDeleteRemoved if (checkDeleteRemoved()) { // Mark items that are old enough and have the status trashed as obsolete // Only do this in a complete scan! CoreDbAccess access; QList trashedItems = access.db()->getImageIds(DatabaseItem::Status::Trashed); foreach (const qlonglong& item, trashedItems) { access.db()->setItemStatus(item, DatabaseItem::Status::Obsolete); } resetDeleteRemovedSettings(); } else { // increment the count of complete scans during which removed items were not deleted incrementDeleteRemovedCompleteScanCount(); } markDatabaseAsScanned(); emit finishedCompleteScan(); } void CollectionScanner::partialScan(const QString& filePath) { QString albumRoot = CollectionManager::instance()->albumRootPath(filePath); QString album = CollectionManager::instance()->album(filePath); partialScan(albumRoot, album); } void CollectionScanner::partialScan(const QString& albumRoot, const QString& album) { if (albumRoot.isNull() || album.isEmpty()) { // If you want to scan the album root, pass "/" qCWarning(DIGIKAM_DATABASE_LOG) << "partialScan(QString, QString) called with invalid values"; return; } /* if (CoreDbAccess().backend()->isInTransaction()) { // Install ScanController::instance()->suspendCollectionScan around your CoreDbTransaction qCDebug(DIGIKAM_DATABASE_LOG) << "Detected an active database transaction when starting a collection scan. " "Please report this error."; return; } */ mainEntryPoint(false); d->resetRemovedItemsTime(); CollectionLocation location = CollectionManager::instance()->locationForAlbumRootPath(albumRoot); if (location.isNull()) { qCWarning(DIGIKAM_DATABASE_LOG) << "Did not find a CollectionLocation for album root path " << albumRoot; return; } // if we have no hints to follow, clean up all stale albums // Hint: Rethink with next major db update if (!d->hints || !d->hints->hasAlbumHints()) { CoreDbAccess().db()->deleteStaleAlbums(); } // Usually, we can restrict stale album scanning to our own location. // But when there are album hints from a second location to this location, // also scan the second location QSet locationIdsToScan; locationIdsToScan << location.id(); if (d->hints) { QReadLocker locker(&d->hints->lock); QHash::const_iterator it; for (it = d->hints->albumHints.constBegin() ; it != d->hints->albumHints.constEnd() ; ++it) { if (it.key().albumRootId == location.id()) { locationIdsToScan << it.key().albumRootId; } } } scanForStaleAlbums(locationIdsToScan.toList()); if (!d->checkObserver()) { emit cancelled(); return; } if (album == QLatin1String("/")) { scanAlbumRoot(location); } else { scanAlbum(location, album); } finishHistoryScanning(); if (!d->checkObserver()) { emit cancelled(); return; } updateRemovedItemsTime(); } qlonglong CollectionScanner::scanFile(const QString& filePath, FileScanMode mode) { QFileInfo info(filePath); QString dirPath = info.path(); // strip off filename QString albumRoot = CollectionManager::instance()->albumRootPath(dirPath); if (albumRoot.isNull()) { return -1; } QString album = CollectionManager::instance()->album(dirPath); return scanFile(albumRoot, album, info.fileName(), mode); } qlonglong CollectionScanner::scanFile(const QString& albumRoot, const QString& album, const QString& fileName, FileScanMode mode) { if (album.isEmpty() || fileName.isEmpty()) { qCWarning(DIGIKAM_DATABASE_LOG) << "scanFile(QString, QString, QString) called with empty album or empty filename"; return -1; } CollectionLocation location = CollectionManager::instance()->locationForAlbumRootPath(albumRoot); if (location.isNull()) { qCWarning(DIGIKAM_DATABASE_LOG) << "Did not find a CollectionLocation for album root path " << albumRoot; return -1; } QDir dir(location.albumRootPath() + album); QFileInfo fi(dir, fileName); if (!fi.exists()) { qCWarning(DIGIKAM_DATABASE_LOG) << "File given to scan does not exist" << albumRoot << album << fileName; return -1; } int albumId = checkAlbum(location, album); qlonglong imageId = CoreDbAccess().db()->getImageId(albumId, fileName); imageId = scanFile(fi, albumId, imageId, mode); return imageId; } void CollectionScanner::scanFile(const ItemInfo& info, FileScanMode mode) { if (info.isNull()) { return; } QFileInfo fi(info.filePath()); scanFile(fi, info.albumId(), info.id(), mode); } qlonglong CollectionScanner::scanFile(const QFileInfo& fi, int albumId, qlonglong imageId, FileScanMode mode) { mainEntryPoint(false); if (imageId == -1) { switch (mode) { case NormalScan: case ModifiedScan: imageId = scanNewFile(fi, albumId); break; case Rescan: imageId = scanNewFileFullScan(fi, albumId); break; } } else { ItemScanInfo scanInfo = CoreDbAccess().db()->getItemScanInfo(imageId); switch (mode) { case NormalScan: scanFileNormal(fi, scanInfo); break; case ModifiedScan: scanModifiedFile(fi, scanInfo); break; case Rescan: rescanFile(fi, scanInfo); break; } } finishHistoryScanning(); return imageId; } void CollectionScanner::scanAlbumRoot(const CollectionLocation& location) { if (d->wantSignals) { emit startScanningAlbumRoot(location.albumRootPath()); } /* QDir dir(location.albumRootPath()); QStringList fileList(dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot)); for (QStringList::iterator fileIt = fileList.begin(); fileIt != fileList.end(); ++fileIt) { scanAlbum(location, '/' + (*fileIt)); } */ // scan album that covers the root directory of this album root, // all contained albums, and their subalbums recursively. scanAlbum(location, QLatin1String("/")); if (d->wantSignals) { emit finishedScanningAlbumRoot(location.albumRootPath()); } } void CollectionScanner::scanForStaleAlbums(const QList& locations) { QList locationIdsToScan; foreach (const CollectionLocation& location, locations) { locationIdsToScan << location.id(); } scanForStaleAlbums(locationIdsToScan); } void CollectionScanner::scanForStaleAlbums(const QList& locationIdsToScan) { if (d->wantSignals) { emit startScanningForStaleAlbums(); } QList albumList = CoreDbAccess().db()->getAlbumShortInfos(); QList toBeDeleted; /* // See bug #231598 QHash albumRoots; foreach (const CollectionLocation& location, locations) { albumRoots[location.id()] = location; } */ - QList::const_iterator it; + QList::const_iterator it3; - for (it = albumList.constBegin() ; it != albumList.constEnd() ; ++it) + for (it3 = albumList.constBegin() ; it3 != albumList.constEnd() ; ++it3) { - if (!locationIdsToScan.contains((*it).albumRootId)) + if (!locationIdsToScan.contains((*it3).albumRootId)) { continue; } - CollectionLocation location = CollectionManager::instance()->locationForAlbumRootId((*it).albumRootId); + CollectionLocation location = CollectionManager::instance()->locationForAlbumRootId((*it3).albumRootId); // Only handle albums on available locations if (location.isAvailable()) { - QFileInfo fileInfo(location.albumRootPath() + (*it).relativePath); + QFileInfo fileInfo(location.albumRootPath() + (*it3).relativePath); bool dirExist = (fileInfo.exists() && fileInfo.isDir()); #ifdef Q_OS_WIN - if (dirExist && !(*it).relativePath.endsWith(QLatin1Char('/'))) + if (dirExist && !(*it3).relativePath.endsWith(QLatin1Char('/'))) { QDir dir(fileInfo.dir()); dirExist = dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot) .contains(fileInfo.fileName()); } #endif // let digikam think that ignored directories got deleted // (if they already exist in the database, this will delete them) if (!dirExist || d->ignoreDirectory.contains(fileInfo.fileName())) { - toBeDeleted << (*it).id; - d->scannedAlbums << (*it).id; + toBeDeleted << (*it3).id; + d->scannedAlbums << (*it3).id; } } } // At this point, it is important to handle album renames. // We can still copy over album attributes later, but we cannot identify // the former album of removed images. // Just renaming the album is also much cheaper than rescanning all files. if (!toBeDeleted.isEmpty() && d->hints) { // shallow copy for reading without caring for locks QHash albumHints; { QReadLocker locker(&d->hints->lock); albumHints = d->hints->albumHints; } // go through all album copy/move hints QHash::const_iterator it; int toBeDeletedIndex; for (it = albumHints.constBegin() ; it != albumHints.constEnd() ; ++it) { // if the src entry of a hint is found in toBeDeleted, we have a move/rename, no copy. Handle these here. toBeDeletedIndex = toBeDeleted.indexOf(it.value().albumId); // We must double check that not, for some reason, the target album has already been scanned. QList::const_iterator it2; for (it2 = albumList.constBegin() ; it2 != albumList.constEnd() ; ++it2) { if (it2->albumRootId == it.key().albumRootId && it2->relativePath == it.key().relativePath) { toBeDeletedIndex = -1; break; } } if (toBeDeletedIndex != -1) { // check for existence of target CollectionLocation location = CollectionManager::instance()->locationForAlbumRootId(it.key().albumRootId); if (location.isAvailable()) { QFileInfo fileInfo(location.albumRootPath() + it.key().relativePath); bool dirExist = (fileInfo.exists() && fileInfo.isDir()); #ifdef Q_OS_WIN if (dirExist && !it.key().relativePath.endsWith(QLatin1Char('/'))) { QDir dir(fileInfo.dir()); dirExist = dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot) .contains(fileInfo.fileName()); } #endif // Make sure ignored directories are not used in renaming operations if (dirExist && d->ignoreDirectory.contains(fileInfo.fileName())) { // Just set a new root/relativePath to the album. Further scanning will care for all cases or error. CoreDbAccess().db()->renameAlbum(it.value().albumId, it.key().albumRootId, it.key().relativePath); // No need any more to delete the album toBeDeleted.removeAt(toBeDeletedIndex); } } } } } safelyRemoveAlbums(toBeDeleted); if (d->wantSignals) { emit finishedScanningForStaleAlbums(); } } void CollectionScanner::scanAlbum(const CollectionLocation& location, const QString& album) { // + Adds album if it does not yet exist in the db. // + Recursively scans subalbums of album. // + Adds files if they do not yet exist in the db. // + Marks stale files as removed QDir dir(location.albumRootPath() + album); if (!dir.exists() || !dir.isReadable()) { qCWarning(DIGIKAM_DATABASE_LOG) << "Folder does not exist or is not readable: " << dir.path(); return; } if (d->wantSignals) { emit startScanningAlbum(location.albumRootPath(), album); } int albumID = checkAlbum(location, album); MetaEngineSettingsContainer settings = MetaEngineSettings::instance()->settings(); const QList& scanInfos = CoreDbAccess().db()->getItemScanInfos(albumID); // create a QHash filename -> index in list QHash fileNameIndexHash; QSet itemIdSet; for (int i = 0 ; i < scanInfos.size() ; ++i) { fileNameIndexHash[scanInfos.at(i).itemName] = i; itemIdSet << scanInfos.at(i).id; } const QStringList& list = dir.entryList(QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot, QDir::Name | QDir::DirsLast); const QString xmpExt(QLatin1String(".xmp")); int counter = -1; foreach (const QString& entry, list) { if (!d->checkObserver()) { return; // return directly, do not go to cleanup code after loop! } ++counter; if (d->wantSignals && counter && (counter % 100 == 0)) { emit scannedFiles(counter); counter = 0; } QFileInfo info(dir, entry); if (info.isFile()) { // filter with name filter QString suffix(info.suffix().toLower()); if (!d->nameFilters.contains(suffix)) { continue; } int index = fileNameIndexHash.value(info.fileName(), -1); if (index != -1) { // mark item as "seen" itemIdSet.remove(scanInfos.at(index).id); bool hasSidecar = (settings.useXMPSidecar4Reading && (list.contains(info.fileName() + xmpExt) || list.contains(info.completeBaseName() + xmpExt))); scanFileNormal(info, scanInfos.at(index), hasSidecar); } else if (info.completeSuffix().contains(QLatin1String("digikamtempfile."))) { // ignore temp files we created ourselves continue; } else { //qCDebug(DIGIKAM_DATABASE_LOG) << "Adding item " << info.fileName(); scanNewFile(info, albumID); // emit signals for scanned files with much higher granularity if (d->wantSignals && counter && (counter % 2 == 0)) { emit scannedFiles(counter); counter = 0; } } } else if (info.isDir()) { #ifdef Q_OS_WIN //Hide album that starts with a dot, as under Linux. if (info.fileName().startsWith(QLatin1Char('.'))) { continue; } #endif if (d->ignoreDirectory.contains(info.fileName())) { continue; } QString subAlbum = album; if (subAlbum != QLatin1String("/")) { subAlbum += QLatin1Char('/'); } scanAlbum(location, subAlbum + info.fileName()); } } if (d->wantSignals && counter) { emit scannedFiles(counter); } // Mark items in the db which we did not see on disk. if (!itemIdSet.isEmpty()) { QList ids = itemIdSet.toList(); CoreDbOperationGroup group; CoreDbAccess().db()->removeItems(ids, QList() << albumID); itemsWereRemoved(ids); } // mark album as scanned d->scannedAlbums << albumID; if (d->wantSignals) { emit finishedScanningAlbum(location.albumRootPath(), album, list.count()); } } void CollectionScanner::scanFileNormal(const QFileInfo& fi, const ItemScanInfo& scanInfo, bool checkSidecar) { bool hasAnyHint = d->hints && d->hints->hasAnyNormalHint(scanInfo.id); // if the date is null, this signals a full rescan if (scanInfo.modificationDate.isNull() || (hasAnyHint && d->hints->hasRescanHint(scanInfo.id))) { if (hasAnyHint) { QWriteLocker locker(&d->hints->lock); d->hints->rescanItemHints.remove(scanInfo.id); } rescanFile(fi, scanInfo); return; } else if (hasAnyHint && d->hints->hasModificationHint(scanInfo.id)) { { QWriteLocker locker(&d->hints->lock); d->hints->modifiedItemHints.remove(scanInfo.id); } scanModifiedFile(fi, scanInfo); return; } else if (hasAnyHint) // metadata adjustment hints { if (d->hints->hasMetadataAboutToAdjustHint(scanInfo.id)) { // postpone scan return; } else // hasMetadataAdjustedHint { { QWriteLocker locker(&d->hints->lock); d->hints->metadataAdjustedHints.remove(scanInfo.id); } scanFileUpdateHashReuseThumbnail(fi, scanInfo, true); return; } } else if (d->updatingHashHint) { // if the file need not be scanned because of modification, update the hash if (s_modificationDateEquals(fi.lastModified(), scanInfo.modificationDate) && fi.size() == scanInfo.fileSize) { scanFileUpdateHashReuseThumbnail(fi, scanInfo, false); return; } } MetaEngineSettingsContainer settings = MetaEngineSettings::instance()->settings(); QDateTime modificationDate = fi.lastModified(); if (checkSidecar && settings.updateFileTimeStamp && settings.useXMPSidecar4Reading && DMetadata::hasSidecar(fi.filePath())) { QString filePath = DMetadata::sidecarPath(fi.filePath()); QDateTime sidecarDate = QFileInfo(filePath).lastModified(); if (sidecarDate > modificationDate) { modificationDate = sidecarDate; } } if (!s_modificationDateEquals(modificationDate, scanInfo.modificationDate) || fi.size() != scanInfo.fileSize) { if (settings.rescanImageIfModified) { rescanFile(fi, scanInfo); } else { scanModifiedFile(fi, scanInfo); } } } qlonglong CollectionScanner::scanNewFile(const QFileInfo& info, int albumId) { if (d->checkDeferred(info)) { return -1; } ItemScanner scanner(info); scanner.setCategory(category(info)); // Check copy/move hints for single items qlonglong srcId = 0; if (d->hints) { QReadLocker locker(&d->hints->lock); srcId = d->hints->itemHints.value(NewlyAppearedFile(albumId, info.fileName())); } if (srcId != 0) { scanner.copiedFrom(albumId, srcId); } else { // Check copy/move hints for whole albums int srcAlbum = d->establishedSourceAlbums.value(albumId); if (srcAlbum) { // if we have one source album, find out if there is a file with the same name srcId = CoreDbAccess().db()->getImageId(srcAlbum, info.fileName()); } if (srcId != 0) { scanner.copiedFrom(albumId, srcId); } else { // Establishing identity with the unique hsah scanner.newFile(albumId); } } d->finishScanner(scanner); return scanner.id(); } qlonglong CollectionScanner::scanNewFileFullScan(const QFileInfo& info, int albumId) { if (d->checkDeferred(info)) { return -1; } ItemScanner scanner(info); scanner.setCategory(category(info)); scanner.newFileFullScan(albumId); d->finishScanner(scanner); return scanner.id(); } void CollectionScanner::scanModifiedFile(const QFileInfo& info, const ItemScanInfo& scanInfo) { if (d->checkDeferred(info)) { return; } ItemScanner scanner(info, scanInfo); scanner.setCategory(category(info)); scanner.fileModified(); d->finishScanner(scanner); } void CollectionScanner::scanFileUpdateHashReuseThumbnail(const QFileInfo& info, const ItemScanInfo& scanInfo, bool fileWasEdited) { QString oldHash = scanInfo.uniqueHash; qlonglong oldSize = scanInfo.fileSize; // same code as scanModifiedFile ItemScanner scanner(info, scanInfo); scanner.setCategory(category(info)); scanner.fileModified(); QString newHash = scanner.itemScanInfo().uniqueHash; qlonglong newSize = scanner.itemScanInfo().fileSize; if (ThumbsDbAccess::isInitialized()) { if (fileWasEdited) { // The file was edited in such a way that we know that the pixel content did not change, so we can reuse the thumbnail. // We need to add a link to the thumbnail data with the new hash/file size _and_ adjust // the file modification date in the data table. ThumbsDbInfo thumbDbInfo = ThumbsDbAccess().db()->findByHash(oldHash, oldSize); if (thumbDbInfo.id != -1) { ThumbsDbAccess().db()->insertUniqueHash(newHash, newSize, thumbDbInfo.id); ThumbsDbAccess().db()->updateModificationDate(thumbDbInfo.id, scanner.itemScanInfo().modificationDate); // TODO: also update details thumbnails (by file path and URL scheme) } } else { ThumbsDbAccess().db()->replaceUniqueHash(oldHash, oldSize, newHash, newSize); } } d->finishScanner(scanner); } void CollectionScanner::rescanFile(const QFileInfo& info, const ItemScanInfo& scanInfo) { if (d->checkDeferred(info)) { return; } ItemScanner scanner(info, scanInfo); scanner.setCategory(category(info)); scanner.rescan(); d->finishScanner(scanner); } void CollectionScanner::completeHistoryScanning() { // scan tagged images int needResolvingTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::needResolvingHistory()); int needTaggingTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::needTaggingHistoryGraph()); QList ids = CoreDbAccess().db()->getItemIDsInTag(needResolvingTag); historyScanningStage2(ids); ids = CoreDbAccess().db()->getItemIDsInTag(needTaggingTag); qCDebug(DIGIKAM_DATABASE_LOG) << "items to tag" << ids; historyScanningStage3(ids); } void CollectionScanner::finishHistoryScanning() { // scan recorded ids QList ids; // stage 2 ids = d->needResolveHistorySet.toList(); d->needResolveHistorySet.clear(); historyScanningStage2(ids); if (!d->checkObserver()) { return; } // stage 3 ids = d->needTaggingHistorySet.toList(); d->needTaggingHistorySet.clear(); historyScanningStage3(ids); } void CollectionScanner::historyScanningStage2(const QList& ids) { foreach (const qlonglong& id, ids) { if (!d->checkObserver()) { return; } CoreDbOperationGroup group; if (d->recordHistoryIds) { QList needTaggingIds; ItemScanner::resolveImageHistory(id, &needTaggingIds); foreach (const qlonglong& needTag, needTaggingIds) { d->needTaggingHistorySet << needTag; } } else { ItemScanner::resolveImageHistory(id); } } } void CollectionScanner::historyScanningStage3(const QList& ids) { foreach (const qlonglong& id, ids) { if (!d->checkObserver()) { return; } CoreDbOperationGroup group; ItemScanner::tagItemHistoryGraph(id); } } bool CollectionScanner::databaseInitialScanDone() { CoreDbAccess access; return !access.db()->getSetting(QLatin1String("Scanned")).isEmpty(); } } // namespace Digikam diff --git a/core/libs/database/item/scanner/itemscanner.h b/core/libs/database/item/scanner/itemscanner.h index 96c88cf881..506f8df411 100644 --- a/core/libs/database/item/scanner/itemscanner.h +++ b/core/libs/database/item/scanner/itemscanner.h @@ -1,375 +1,377 @@ /* ============================================================ * * This file is a part of digiKam project * https://www.digikam.org * * Date : 2007-09-19 * Description : Scanning a single item. * * Copyright (C) 2007-2013 by Marcel Wiesweg * Copyright (C) 2013-2019 by Gilles Caulier * * This program is free software; you can redistribute it * and/or modify it under the terms of the GNU General * Public License as published by the Free Software Foundation; * either version 2, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * ============================================================ */ #ifndef DIGIKAM_ITEM_SCANNER_H #define DIGIKAM_ITEM_SCANNER_H // Qt includes #include // Local includes #include "dimg.h" #include "iteminfo.h" #include "dmetadata.h" #include "coredbalbuminfo.h" #include "coredbinfocontainers.h" namespace Digikam { class DIGIKAM_DATABASE_EXPORT ItemScanner { public: enum ScanMode { NewScan, ModifiedScan, Rescan }; public: /** * Construct an ItemScanner object from an existing QFileInfo * and ItemScanInfo object. * This constructor shall be used with fileModified() or fullScan(). */ ItemScanner(const QFileInfo& info, const ItemScanInfo& Iteminfo); /** * Construct an ItemScanner from an existing QFileInfo object. * Use this constructor if you intend to call newFile(). */ explicit ItemScanner(const QFileInfo& info); /** * Construct an ItemScanner for an image in the database. * File info, Scan info and the category will be retrieved from the database. */ explicit ItemScanner(qlonglong imageid); ~ItemScanner(); /** * Inform the scanner about the category of the file. * Required at least for newFile() calls, recommended for calls with the * first constructor above as well. */ void setCategory(DatabaseItem::Category category); /** * Provides access to the information retrieved by scanning. * The validity depends on the previously executed scan. */ const ItemScanInfo& itemScanInfo() const; /** * Loads data from disk (metadata, image file properties). * This method is called from any of the main entry points above. * You can call it before if you want to control the time when it is executed. * Calling it a second time with data already loaded will do nothing. */ void loadFromDisk(); /** * Helper method to translate enum values to user presentable strings */ static QString formatToString(const QString& format); private: // Hidden copy constructor and assignment operator. ItemScanner(const ItemScanner&); ItemScanner& operator=(const ItemScanner&); static bool hasValidField(const QVariantList& list); static bool lessThanForIdentity(const ItemScanInfo& a, const ItemScanInfo& b); // ----------------------------------------------------------------------------- /** @name Operations with Database */ //@{ public: /** * Call this when you want ItemScanner to add a new file to the database * and read all information into the database. */ void newFile(int albumId); /** * Call this when you want ItemScanner to add a new file to the database * and read all information into the database. This variant will not use * the unique hash to establish identify with an existing entry, but * read all information newly from the file. */ void newFileFullScan(int albumId); /** * Call this to take an existing image in the database, but re-read * all information from the file into the database, possibly overwriting * information there. */ void rescan(); /** * Commits the scanned information to the database. * You must call this after scanning was done for any changes to take effect. * Only this method will perform write operations to the database. */ void commit(); /** * Returns the image id of the scanned file, if (yet) available. */ qlonglong id() const; /** * Similar to newFile. * Call this when you want ItemScanner to add a new file to the database * which is a copy of another file, copying attributes from the src * and rescanning other attributes as appropriate. * Give the id of the album of the new file, and the id of the src file. */ void copiedFrom(int albumId, qlonglong srcId); /** * Sort a list of infos by proximity to the given subject. * Infos are near if they are e.g. in the same album. * They are not near if they are e.g. in different collections. */ static void sortByProximity(QList& infos, const ItemInfo& subject); protected: bool copyFromSource(qlonglong src); void commitCopyImageAttributes(); void prepareAddImage(int albumId); bool commitAddImage(); //@} // ----------------------------------------------------------------------------- /** @name Operations on File Metadata */ //@{ public: /** * Call this when you have detected that a file in the database has been * modified on disk. Only two groups of fields will be updated in the database: * - filesystem specific properties (those that signaled you that the file has been modified * because their state on disk differed from the state in the database) * - image specific properties, for which a difference in the database independent from * the actual file does not make sense (width/height, bit depth, color model) */ void fileModified(); /** * Returns File-metadata container with user-presentable information. * These methods provide the reverse service: Not writing into the db, but reading from the db. */ static void fillCommonContainer(qlonglong imageid, ImageCommonContainer* const container); /** * Returns a suitable creation date from file system information. * Use this as a fallback if metadata is not available. */ static QDateTime creationDateFromFilesystem(const QFileInfo& info); protected: void prepareUpdateImage(); void commitUpdateImage(); bool scanFromIdenticalFile(); void scanFile(ScanMode mode); void scanItemInformation(); void commitItemInformation(); //@} // ----------------------------------------------------------------------------- /** @name Operations on Photo Metadata */ //@{ public: /** * Returns Photo-metadata container with user-presentable information. * These methods provide the reverse service: Not writing into the db, but reading from the db. */ static void fillMetadataContainer(qlonglong imageid, ImageMetadataContainer* const container); /** * Helper method to return official property name by which * IPTC core properties are stored in the database (ItemCopyright and ImageProperties table). * Allowed arguments: All MetadataInfo::Fields starting with "IptcCore..." */ static QString iptcCorePropertyName(MetadataInfo::Field field); static MetadataFields allImageMetadataFields(); protected: QString detectImageFormat() const; void scanImageMetadata(); void commitImageMetadata(); void scanItemPosition(); void commitItemPosition(); void scanItemComments(); void commitItemComments(); void scanItemCopyright(); void commitItemCopyright(); void scanIPTCCore(); void commitIPTCCore(); void scanTags(); void commitTags(); void scanFaces(); void commitFaces(); bool checkRatingFromMetadata(const QVariant& ratingFromMetadata) const; void checkCreationDateFromMetadata(QVariant& dateFromMetadata) const; //@} // ----------------------------------------------------------------------------- /** @name Operations on Video Metadata */ //@{ public: /** * Returns Video container with user-presentable information. * These methods provide the reverse service: Not writing into the db, but reading from the db. */ static void fillVideoMetadataContainer(qlonglong imageid, VideoMetadataContainer* const container); protected: void scanVideoInformation(); void scanVideoMetadata(); void commitVideoMetadata(); QString detectVideoFormat() const; QString detectAudioFormat() const; static MetadataFields allVideoMetadataFields(); //@} // ----------------------------------------------------------------------------- /** @name Operations on History Metadata */ //@{ public: /** * Returns true if this file has been marked as needing history resolution at a later stage */ bool hasHistoryToResolve() const; + //@{ /** * Resolves the image history of the image id by filling the ImageRelations table * for all contained referred images. * If needTaggingIds is given, all ids marked for needing tagging of the history graph are added. */ static bool resolveImageHistory(qlonglong id, QList* needTaggingIds = nullptr); static bool resolveImageHistory(qlonglong imageId, const QString& historyXml, QList* needTaggingIds = nullptr); + //@} /** * Takes the history graph reachable from the given image, and assigns * versioning tags to all entries based on history image types and graph structure */ static void tagItemHistoryGraph(qlonglong id); /** * All referred images of the given history will be resolved. * In the returned history, the actions are the same, while each * referred image actually exists in the collection * (if mustBeAvailable is true, it is even in a currently available collection). * That means the number of referred images may be less or greater than initially. * Note that this history may have peculiar properties, like multiple Original or Current entries * (if the source entry resolves to multiple collection images), so this history * is only for internal use, not for storage. */ static DImageHistory resolvedImageHistory(const DImageHistory& history, bool mustBeAvailable = false); /** * Determines if the two ids refer to the same image. * Does not check if such a referred image really exists. */ static bool sameReferredImage(const HistoryImageId& id1, const HistoryImageId& id2); /** * Returns all image ids fulfilling the given image id. */ static QList resolveHistoryImageId(const HistoryImageId& historyId); protected: void scanImageHistory(); void commitImageHistory(); void scanImageHistoryIfModified(); QString uniqueHash() const; //@} public: /** * @brief scanBalooInfo - retrieve tags, comments and rating from Baloo Desktop service. */ void scanBalooInfo(); private: class Private; Private* const d; }; } // namespace Digikam #endif // DIGIKAM_ITEM_SCANNER_H diff --git a/core/libs/database/item/scanner/itemscanner_history.cpp b/core/libs/database/item/scanner/itemscanner_history.cpp index 229144bc21..0c57e804e7 100644 --- a/core/libs/database/item/scanner/itemscanner_history.cpp +++ b/core/libs/database/item/scanner/itemscanner_history.cpp @@ -1,434 +1,434 @@ /* ============================================================ * * This file is a part of digiKam project * https://www.digikam.org * * Date : 2007-09-19 * Description : Scanning a single item - history metadata helper. * * Copyright (C) 2007-2013 by Marcel Wiesweg * Copyright (C) 2013-2019 by Gilles Caulier * * This program is free software; you can redistribute it * and/or modify it under the terms of the GNU General * Public License as published by the Free Software Foundation; * either version 2, or (at your option) * any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * ============================================================ */ #include "itemscanner_p.h" namespace Digikam { - + void ItemScanner::scanImageHistory() { /** Stage 1 of history scanning */ d->commit.historyXml = d->metadata.getItemHistory(); d->commit.uuid = d->metadata.getItemUniqueId(); } void ItemScanner::commitImageHistory() { if (!d->commit.historyXml.isEmpty()) { CoreDbAccess().db()->setItemHistory(d->scanInfo.id, d->commit.historyXml); // Delay history resolution by setting this tag: // Resolution depends on the presence of other images, possibly only when the scanning process has finished CoreDbAccess().db()->addItemTag(d->scanInfo.id, TagsCache::instance()-> getOrCreateInternalTag(InternalTagName::needResolvingHistory())); d->hasHistoryToResolve = true; } if (!d->commit.uuid.isNull()) { CoreDbAccess().db()->setImageUuid(d->scanInfo.id, d->commit.uuid); } } void ItemScanner::scanImageHistoryIfModified() { // If a file has a modified history, it must have a new UUID QString previousUuid = CoreDbAccess().db()->getImageUuid(d->scanInfo.id); QString currentUuid = d->metadata.getItemUniqueId(); if (!currentUuid.isEmpty() && previousUuid != currentUuid) { scanImageHistory(); } } bool ItemScanner::resolveImageHistory(qlonglong id, QList* needTaggingIds) { ImageHistoryEntry history = CoreDbAccess().db()->getItemHistory(id); return resolveImageHistory(id, history.history, needTaggingIds); } bool ItemScanner::resolveImageHistory(qlonglong imageId, const QString& historyXml, QList* needTaggingIds) { /** Stage 2 of history scanning */ if (historyXml.isNull()) { return true; // "true" means nothing is left to resolve } DImageHistory history = DImageHistory::fromXml(historyXml); if (history.isNull()) { return true; } ItemHistoryGraph graph; graph.addScannedHistory(history, imageId); if (!graph.hasEdges()) { return true; } QPair, QList > cloud = graph.relationCloudParallel(); CoreDbAccess().db()->addImageRelations(cloud.first, cloud.second, DatabaseRelation::DerivedFrom); int needResolvingTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::needResolvingHistory()); int needTaggingTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::needTaggingHistoryGraph()); // remove the needResolvingHistory tag from all images in graph CoreDbAccess().db()->removeTagsFromItems(graph.allImageIds(), QList() << needResolvingTag); // mark a single image from the graph (sufficient for find the full relation cloud) QList roots = graph.rootImages(); if (!roots.isEmpty()) { CoreDbAccess().db()->addItemTag(roots.first().id(), needTaggingTag); if (needTaggingIds) { *needTaggingIds << roots.first().id(); } } return !graph.hasUnresolvedEntries(); } void ItemScanner::tagItemHistoryGraph(qlonglong id) { /** Stage 3 of history scanning */ ItemInfo info(id); if (info.isNull()) { return; } //qCDebug(DIGIKAM_DATABASE_LOG) << "tagItemHistoryGraph" << id; // Load relation cloud, history of info and of all leaves of the tree into the graph, fully resolved ItemHistoryGraph graph = ItemHistoryGraph::fromInfo(info, ItemHistoryGraph::LoadAll, ItemHistoryGraph::NoProcessing); qCDebug(DIGIKAM_DATABASE_LOG) << graph; int originalVersionTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::originalVersion()); int currentVersionTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::currentVersion()); int intermediateVersionTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::intermediateVersion()); int needTaggingTag = TagsCache::instance()->getOrCreateInternalTag(InternalTagName::needTaggingHistoryGraph()); // Remove all relevant tags CoreDbAccess().db()->removeTagsFromItems(graph.allImageIds(), QList() << originalVersionTag << currentVersionTag << intermediateVersionTag << needTaggingTag); if (!graph.hasEdges()) { return; } // get category info - QList originals, intermediates, currents; - QHash types = graph.categorize(); + QList originals, intermediates, currents; + QHash grpTypes = graph.categorize(); QHash::const_iterator it; - for (it = types.constBegin() ; it != types.constEnd() ; ++it) + for (it = grpTypes.constBegin() ; it != grpTypes.constEnd() ; ++it) { qCDebug(DIGIKAM_DATABASE_LOG) << "Image" << it.key().id() << "type" << it.value(); HistoryImageId::Types types = it.value(); if (types & HistoryImageId::Original) { originals << it.key().id(); } if (types & HistoryImageId::Intermediate) { intermediates << it.key().id(); } if (types & HistoryImageId::Current) { currents << it.key().id(); } } if (!originals.isEmpty()) { CoreDbAccess().db()->addTagsToItems(originals, QList() << originalVersionTag); } if (!intermediates.isEmpty()) { CoreDbAccess().db()->addTagsToItems(intermediates, QList() << intermediateVersionTag); } if (!currents.isEmpty()) { CoreDbAccess().db()->addTagsToItems(currents, QList() << currentVersionTag); } } DImageHistory ItemScanner::resolvedImageHistory(const DImageHistory& history, bool mustBeAvailable) { DImageHistory h; foreach (const DImageHistory::Entry& e, history.entries()) { // Copy entry, without referredImages DImageHistory::Entry entry; entry.action = e.action; // resolve referredImages foreach (const HistoryImageId& id, e.referredImages) { QList imageIds = resolveHistoryImageId(id); // append each image found in collection to referredImages foreach (const qlonglong& imageId, imageIds) { ItemInfo info(imageId); if (info.isNull()) { continue; } if (mustBeAvailable) { CollectionLocation location = CollectionManager::instance()->locationForAlbumRootId(info.albumRootId()); if (!location.isAvailable()) { continue; } } HistoryImageId newId = info.historyImageId(); newId.setType(id.m_type); entry.referredImages << newId; } } // add to history h.entries() << entry; } return h; } bool ItemScanner::sameReferredImage(const HistoryImageId& id1, const HistoryImageId& id2) { if (!id1.isValid() || !id2.isValid()) { return false; } /* * We give the UUID the power of equivalence that none of the other criteria has: * For two images a,b with uuids x,y, where x and y not null, * a (same image as) b <=> x == y */ if (id1.hasUuid() && id2.hasUuid()) { return id1.m_uuid == id2.m_uuid; } if (id1.hasUniqueHashIdentifier() && id1.m_uniqueHash == id2.m_uniqueHash && id1.m_fileSize == id2.m_fileSize) { return true; } if (id1.hasFileName() && id1.hasCreationDate() && id1.m_fileName == id2.m_fileName && id1.m_creationDate == id2.m_creationDate) { return true; } if (id1.hasFileOnDisk() && id1.m_filePath == id2.m_filePath && id1.m_fileName == id2.m_fileName) { return true; } return false; } // Returns true if both have the same UUID, or at least one of the two has no UUID // Returns false iff both have a UUID and the UUIDs differ static bool uuidDoesNotDiffer(const HistoryImageId& referenceId, qlonglong id) { if (referenceId.hasUuid()) { QString uuid = CoreDbAccess().db()->getImageUuid(id); if (!uuid.isEmpty()) { return referenceId.m_uuid == uuid; } } return true; } static QList mergedIdLists(const HistoryImageId& referenceId, const QList& uuidList, const QList& candidates) { QList results; // uuidList are definite results results = uuidList; // Add a candidate if it has the same UUID, or either reference or candidate have a UUID // (other way round: do not add a candidate which positively has a different UUID) foreach (const qlonglong& candidate, candidates) { if (results.contains(candidate)) { continue; // already in list, skip } if (uuidDoesNotDiffer(referenceId, candidate)) { results << candidate; } } return results; } QList ItemScanner::resolveHistoryImageId(const HistoryImageId& historyId) { // first and foremost: UUID QList uuidList; if (historyId.hasUuid()) { uuidList = CoreDbAccess().db()->getItemsForUuid(historyId.m_uuid); // If all images had a UUID, we would be finished and could return here with a result: /* if (!uuidList.isEmpty()) { return uuidList; } */ // But as identical images may have no UUID yet, we need to continue } // Second: uniqueHash + fileSize. Sufficient to assume that a file is identical, but subject to frequent change. if (historyId.hasUniqueHashIdentifier() && CoreDbAccess().db()->isUniqueHashV2()) { QList infos = CoreDbAccess().db()->getIdenticalFiles(historyId.m_uniqueHash, historyId.m_fileSize); if (!infos.isEmpty()) { QList ids; foreach (const ItemScanInfo& info, infos) { if (info.status != DatabaseItem::Status::Trashed && info.status != DatabaseItem::Status::Obsolete) { ids << info.id; } } return mergedIdLists(historyId, uuidList, ids); } } // As a third combination, we try file name and creation date. Susceptible to renaming, // but not to metadata changes. if (historyId.hasFileName() && historyId.hasCreationDate()) { QList ids = CoreDbAccess().db()->findByNameAndCreationDate(historyId.m_fileName, historyId.m_creationDate); if (!ids.isEmpty()) { return mergedIdLists(historyId, uuidList, ids); } } // Another possibility: If the original UUID is given, we can find all relations for the image with this UUID, // and make an assumption from this group of images. Currently not implemented. // resolve old-style by full file path if (historyId.hasFileOnDisk()) { QFileInfo file(historyId.filePath()); if (file.exists()) { CollectionLocation location = CollectionManager::instance()->locationForPath(historyId.path()); if (!location.isNull()) { QString album = CollectionManager::instance()->album(file.path()); QString name = file.fileName(); ItemShortInfo info = CoreDbAccess().db()->getItemShortInfo(location.id(), album, name); if (info.id) { return mergedIdLists(historyId, uuidList, QList() << info.id); } } } } return uuidList; } bool ItemScanner::hasHistoryToResolve() const { return d->hasHistoryToResolve; } QString ItemScanner::uniqueHash() const { // the QByteArray is an ASCII hex string if (d->scanInfo.category == DatabaseItem::Image) { if (CoreDbAccess().db()->isUniqueHashV2()) return QString::fromUtf8(d->img.getUniqueHashV2()); else return QString::fromUtf8(d->img.getUniqueHash()); } else { if (CoreDbAccess().db()->isUniqueHashV2()) return QString::fromUtf8(DImg::getUniqueHashV2(d->fileInfo.filePath())); else return QString::fromUtf8(DImg::getUniqueHash(d->fileInfo.filePath())); } } } // namespace Digikam