diff --git a/DB/NewImageFinder.cpp b/DB/NewImageFinder.cpp index 26495a1c..2dcf571a 100644 --- a/DB/NewImageFinder.cpp +++ b/DB/NewImageFinder.cpp @@ -1,752 +1,747 @@ /* Copyright (C) 2003-2019 The KPhotoAlbum Development Team This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; see the file COPYING. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "NewImageFinder.h" #include "FastDir.h" #include "ImageDB.h" #include "ImageScout.h" #include "Logging.h" #include "MD5Map.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace DB; /***************************************************************** * * NOTES ON PERFORMANCE * ===== == =========== * * - Robert Krawitz 2018-05-24 * * * GENERAL NOTES ON STORAGE I/O * ------- ----- -- ------- --- * * The two main gates to loading new images are: * * 1) I/O (how fast can we read images off mass storage) * * Different I/O devices have different characteristics in terms of * througput, media latency, and protocol latency. * * - Throughput is the raw speed at which data can be transferred, * limited by the physical and/or electronic characteristics of * the medium and the interface. Short of reducing the amount of * data that's transferred, or clever games with using the most * efficient part of the medium (the outer tracks only for HDD's, * a practice referred to as "short stroking" because it reduces * the distance the head has to seek, at the cost of wasting a * lot of capacity), there's nothing that can be done about this. * * - Media latency is the latency component due to characteristics * of the underlying storage medium. For spinning disks, this is * a function of rotational latency and sek latency. In some * cases, particularly with hard disks, it is possible to reduce * media latency by arranging to access the data in a way that * reduces seeking. See DB/FastDir.cpp for an example of this. * * While media latency can sometimes be hidden by overlapping * I/O, generally not possible to avoid it. Sometimes trying too * hard can actually increase media latency if it results in I/O * operations competing against each other requiring additional * seeks. * * Overlapping I/O with computation is another matter; that can * easily yield benefit, especially if it eliminates rotational * latency. * * - Protocol latency. This refers to things like SATA overhead, * network overhead (for images stored on a network), and so * forth. This can encompass multiple things, and often they can * be pipelined by means of multiple queued I/O operations. For * example, multiple commands can be issued to modern interfaces * (SATA, NVMe) and many network interfaces without waiting for * earlier operations to return. * * If protocol latency is high compared with media latency, * having multiple requests outstanding simultaneously can * yield significant benefits. * * iostat is a valuable tool for investigating throughput and * looking for possible optimizations. The IO/sec and data * read/written per second when compared against known media * characteristics (disk and SSD throughput, network bandwidth) * provides valuable information about whether we're getting close * to full performance from the I/O, and user and system CPU time * give us additional clues about whether we're I/O-bound or * CPU-bound. * * Historically in the computer field, operations that require * relatively simple processing on large volumes of data are I/O * bound. But with very fast I/O devices such as NVMe SSDs, some * of which reach 3 GB/sec, that's not always the case. * * 2) Image (mostly JPEG) loading. * * This is a function of image characteristics and image processing * libraries. Sometimes it's possible to apply parameters to * the underlying image loader to speed it up. This shows up as user * CPU time. Usually the only way to improve this performance * characteristic is to use more or faster CPU cores (sometimes GPUs * can assist here) or use better image loading routines (better * libraries). * * * DESCRIPTION OF KPHOTOALBUM IMAGE LOAD PROCESS * ----------- -- ----------- ----- ---- ------- * * KPhotoAlbum, when it loads an image, performs three processing steps: * * 1) Compute the MD5 checksum * * 2) Extract the Exif metadata * * 3) Generate a thumbnail * * Previous to this round of performance tuning, the first two steps * were performed in the first pass, and thumbnails were generated in * a separate pass. Assuming that the set of new images is large enough * that they cannot all fit in RAM buffers, this results in the I/O * being performed twice. The rewrite results in I/O being performed once. * * In addition, I have made many other changes: * * 1) Prior to the MD5 calculation step, a new thread, called a "scout * thread", reads the files into memory. While this memory is not * directly used in the later computations, it results in the images * being in RAM when they are later needed, making the I/O very fast * (copying data in memory rather than reading it from storage). * * This is a way to overlap I/O with computation. * * 2) The MD5 checksum uses its own I/O to read the data in in larger * chunks than the Qt MD5 routine does. The Qt routine reads it in * in 4KiB chunks; my experimentation has found that 256KiB chunks * are more efficient, even with a scout thread (it reduces the * number of system calls). * * 3) When searching for other images to stack with the image being * loaded, the new image loader no longer attempts to determine * whether other candidate filenames are present, nor does it * compute the MD5 checksum of any such files it does find. Rather, * it only checks for files that are already in KPhotoAlbum, either * previously or as a result of the current load. Merely checking * for the presence of another file is not cheap, and it's not * necessary; if an image will belong to a stack, we'll either know * it now or when other images that can be stacked are loaded. * * 4) The Exif metadata extraction is now done only once; previously * it was performed several times at different stages of the loading * process. * * 5) The thumbnail index is now written out incrementally rather than * the entire index (which can be many megabytes in a large image * database) being rewritten frequently. The index is fully rewritten * prior to exit. * * * BASELINE PERFORMANCE * -------- ----------- * * These measurements were all taken on a Lenovo ThinkPad P70 with 32 * GB of dual-channel DDR4-2400 DRAM, a Xeon E3-1505M CPU (4 cores/8 * total hyperthreads, 2.8-3.7 GHz Skylake; usually runs around * 3.1-3.2 GHz in practice), a Seagate ST2000LM015-2E8174 2TB HDD, and * a Crucial MX300 1TB SATA SSD. Published numbers and measurements I * took otherwise indicate that the HDD can handle about 105-110 * MB/sec with a maximum of 180 IO/sec (in a favorable case). The SSD * is rated to handle 530 MB/sec read, 510 MB/sec write, 92K random * reads/sec, and 83K random writes/sec. * * The image set I used for all measurements, except as noted, * consists of 10839 total files of which about 85% are 20 MP JPEG and * the remainder (with a few exceptions are 20 MP RAW files from a * Canon EOS 7D mkII camera. The total dataset is about 92 GB in * size. * * I baselined both drives by reading the same dataset by means of * * % ls | xargs cat | dd bs=1048576 of=/dev/null * * The HDD required between 850 and 870 seconds (14'10" to 14'30") to * perform this operation, yielding about 105-108 MB/sec. The SSD * achieved about 271 MB/sec, which is well under its rated throughput * (hdparm -Tt yields 355 MB/sec, which is likewise nowhere close to * its rated throughput). hdparm -Tt on the HDD yields about 120 * MB/sec, but throughput to an HDD depends upon which part of the * disk is being read. The outer tracks have a greater angular * density to achieve the same linear density (in other words, the * circumference of an outer track is longer than that of an inner * track, and the data is stored at a constant linear density). So * hdparm isn't very useful on an HDD except as a best case. * * Note also that hdparm does a single stream read from the device. * It does not take advantage of the ability to queue multiple * requests. * * * ANALYSIS OF KPHOTOALBUM LOAD PERFORMANCE * -------- -- ----------- ---- ----------- * * I analyzed the following cases, with images stored both on the * HDD and the SSD: * * 1) Images loaded (All, JPEG only, RAW only) * * B) Thumbnail creation (Including, Excluding) * * C) Scout threads (0, 1, 2, 3) * * The JPG image set constitutes 9293 images totaling about 55 GB. The * JPEG files are mostly 20 MP high quality files, in the range of * 6-10 MB. * The RAW image set constitutes 1544 images totaling about 37 GB. The * RAW files are 20 MP files, in the range of 25 MB. * The ALL set consists of 10839 or 10840 images totaling about 92 GB * (the above set plus 2 .MOV files and in some cases one additional * JPEG file). * * Times are elapsed times; CPU consumption is approximate user+system * CPU consumption. Numbers in parentheses are with thumbnail * building disabled. Note that in the cases with no scout threads on * the SSD the times were reproducibly shorter with thumbnail building * enabled (reasons are not determined at this time). * * Cases building RAW thumbnails generally consumed somewhat more * system CPU (in the range of 10-15%) than JPEG-only cases. This may * be due to custom I/O routines used for generating thumbnails with * JPEG files; RAW files used the I/O provided by libkdcraw, which * uses smaller I/O operations. * * Estimating CPU time for mixed workloads proved very problematic, * as there were significant changes over time. * * Elapsed Time * ------- ---- * * SSD HDD * * JPG - 0 scouts 4:03 (3:59) * JPG - 1 scout 2:46 (2:44) * JPG - 2 scouts 2:20 (2:07) * JPG - 3 scouts 2:21 (1:58) * * ALL - 0 scouts 6:32 (7:03) 16:01 * ALL - 1 scout 4:33 (4:33) 15:01 * ALL - 2 scouts 3:37 (3:28) 16:59 * ALL - 3 scouts 3:36 (3:15) * * RAW - 0 scouts 2:18 (2:46) * RAW - 1 scout 1:46 (1:46) * RAW - 2 scouts 1:17 (1:17) * RAW - 3 scouts 1:13 (1:13) * * User+System CPU * ----------- --- * * SSD HDD * * JPG - 0 scouts 40% (12%) * JPG - 1 scout 70% (20%) * JPG - 2 scouts 85% (15%) * JPG - 3 scouts 85% (15%) * * RAW - 0 scouts 15% (10%) * RAW - 1 scout 18% (12%) * RAW - 2 scouts 25% (15%) * RAW - 3 scouts 25% (15%) * * I also used kcachegrind to measure CPU consumption on smaller * subsets of images (with and without thumbnail creation). In terms * of user CPU consumption, thumbnail creation constitutes the large * majority of CPU cycles for processing JPEG files, followed by MD5 * computation, with Exif parsing lagging far behind. For RAW files, * MD5 computation consumes more cycles, likely in part due to the * larger size of RAW files but possibly also related to the smaller * filesize of embedded thumbnails (on the Canon 7D mkII, the embedded * thumbnail is full size but low quality). * * With thumbnail generation: * ---- --------- ----------- * * RAW JPEG * * Thumbnail generation 44% 82% * libjpeg processing 43% 82% * MD5 computation 51% 13% * Read Exif 1% 1.0% * * Without thumbnail generation: * ------- --------- ----------- * * RAW JPEG * * MD5 computation 92% 80% * Read Exif 4% 10% * * * CONCLUSIONS * ----------- * * For loading files from hard disk (likely the most common case), * there's no reason to consider any loading method other than using a * single scout thread and computing thumbnails concurrently. Even * with thumbnail computation, there is very little CPU utilization. * * Loading from SATA SSD benefits from two scout threads, and possibly * more. For minimal time to regain control, there is some benefit * seen from separating thumbnail generation from the rest of the * processing stages at the cost of more total elapsed time. This is * more evident with JPEG files than with RAW files in this test case. * RAW files typically have smaller thumbnail images which can be * extracted and processed more quickly than full-size JPEG files. On * a slower CPU, it may be desirable to return control to the user * even if the thumbnails are not built yet. * * Two other cases would be NVMe (or other very fast) SSDs and network * storage. Since we're seeing evidence of CPU saturation on SATA * SSDs, we would likely see this even more strongly with NVMe; with * large numbers of images it may be desirable to separate the * thumbnail building from the rest of the processing. It may also be * beneficial to use more scout threads. * * Network storage presents a different problem. It is likely to have * lower throughput -- and certainly much higher latency -- than even * HDD, unless the underlying storage medium is SSD and the data is * located on a very fast, low latency network. So there would be no * benefit to separating thumbnail processing. However, due to * protocol vs. media latency discussed above, it may well work to use * more scout threads. However, this may saturate the network and the * storage, to the detriment of other users, and there's probably no * general (or easily discoverable) optimum for this. * * It's my judgment that most images will be stored on HDDs for at * least the next few years, so tuning for that use case is probably * the best single choice to be made. * *****************************************************************/ namespace { -// Number of scout threads for preloading images. More than one scout thread -// yields about 10% less performance with higher IO/sec but lower I/O throughput, -// most probably due to thrashing. -constexpr int IMAGE_SCOUT_THREAD_COUNT = 4; -constexpr int PRELOAD_MD5 = 1; bool canReadImage(const DB::FileName &fileName) { bool fastMode = !Settings::SettingsData::instance()->ignoreFileExtension(); QMimeDatabase::MatchMode mode = fastMode ? QMimeDatabase::MatchExtension : QMimeDatabase::MatchDefault; QMimeDatabase db; QMimeType mimeType = db.mimeTypeForFile(fileName.absolute(), mode); return QImageReader::supportedMimeTypes().contains(mimeType.name().toUtf8()) || ImageManager::ImageDecoder::mightDecode(fileName); } } bool NewImageFinder::findImages() { // Load the information from the XML file. DB::FileNameSet loadedFiles; QElapsedTimer timer; timer.start(); // TODO: maybe the databas interface should allow to query if it // knows about an image ? Here we've to iterate through all of them and it // might be more efficient do do this in the database without fetching the // whole info. for (const DB::FileName &fileName : DB::ImageDB::instance()->images()) { loadedFiles.insert(fileName); } m_pendingLoad.clear(); searchForNewFiles(loadedFiles, Settings::SettingsData::instance()->imageDirectory()); int filesToLoad = m_pendingLoad.count(); loadExtraFiles(); qCDebug(TimingLog) << "Loaded " << filesToLoad << " images in " << timer.elapsed() / 1000.0 << " seconds"; // Man this is not super optimal, but will be changed onces the image finder moves to become a background task. if (MainWindow::FeatureDialog::hasVideoThumbnailer()) { BackgroundTaskManager::JobManager::instance()->addJob( new BackgroundJobs::SearchForVideosWithoutVideoThumbnailsJob); } // To avoid deciding if the new images are shown in a given thumbnail view or in a given search // we rather just go to home. return (!m_pendingLoad.isEmpty()); // returns if new images was found. } void NewImageFinder::searchForNewFiles(const DB::FileNameSet &loadedFiles, QString directory) { qApp->processEvents(QEventLoop::AllEvents); directory = Utilities::stripEndingForwardSlash(directory); const QString imageDir = Utilities::stripEndingForwardSlash(Settings::SettingsData::instance()->imageDirectory()); FastDir dir(directory); const QStringList dirList = dir.entryList(); ImageManager::RAWImageDecoder rawDec; QStringList excluded; excluded << Settings::SettingsData::instance()->excludeDirectories(); excluded = excluded.at(0).split(QString::fromLatin1(",")); bool skipSymlinks = Settings::SettingsData::instance()->skipSymlinks(); // Keep files within a directory more local by processing all files within the // directory, and then all subdirectories. QStringList subdirList; for (QStringList::const_iterator it = dirList.constBegin(); it != dirList.constEnd(); ++it) { const DB::FileName file = DB::FileName::fromAbsolutePath(directory + QString::fromLatin1("/") + *it); if ((*it) == QString::fromLatin1(".") || (*it) == QString::fromLatin1("..") || excluded.contains((*it)) || loadedFiles.contains(file) || rawDec.fileCanBeSkipped(loadedFiles, file) || (*it) == QString::fromLatin1("CategoryImages")) continue; QFileInfo fi(file.absolute()); if (!fi.isReadable()) continue; if (skipSymlinks && fi.isSymLink()) continue; if (fi.isFile()) { if (!DB::ImageDB::instance()->isBlocking(file)) { if (canReadImage(file)) m_pendingLoad.append(qMakePair(file, DB::Image)); else if (Utilities::isVideo(file)) m_pendingLoad.append(qMakePair(file, DB::Video)); } } else if (fi.isDir()) { subdirList.append(file.absolute()); } } for (QStringList::const_iterator it = subdirList.constBegin(); it != subdirList.constEnd(); ++it) searchForNewFiles(loadedFiles, *it); } void NewImageFinder::loadExtraFiles() { // FIXME: should be converted to a threadpool for SMP stuff and whatnot :] QProgressDialog dialog; QElapsedTimer timeSinceProgressUpdate; dialog.setLabelText(i18n("

Loading information from new files

" "

Depending on the number of images, this may take some time.
" "However, there is only a delay when new images are found.

")); QProgressBar *progressBar = new QProgressBar; progressBar->setFormat(QLatin1String("%v/%m")); dialog.setBar(progressBar); dialog.setMaximum(m_pendingLoad.count()); dialog.setMinimumDuration(1000); QAtomicInt loadedCount = 0; setupFileVersionDetection(); int count = 0; MD5::resetMD5Cache(); ImageScoutQueue asyncPreloadQueue; for (LoadList::Iterator it = m_pendingLoad.begin(); it != m_pendingLoad.end(); ++it) { asyncPreloadQueue.enqueue((*it).first); } - ImageScout scout(asyncPreloadQueue, loadedCount, IMAGE_SCOUT_THREAD_COUNT); - if (PRELOAD_MD5) + ImageScout scout(asyncPreloadQueue, loadedCount, Settings::SettingsData::instance()->getPreloadThreadCount()); + if (Settings::SettingsData::instance()->getOverlapLoadMD5()) scout.setPreloadFunc(DB::PreloadMD5Sum); scout.start(); Exif::Database::instance()->startInsertTransaction(); dialog.setValue(count); // ensure to call setProgress(0) timeSinceProgressUpdate.start(); for (LoadList::Iterator it = m_pendingLoad.begin(); it != m_pendingLoad.end(); ++it, ++count) { qApp->processEvents(QEventLoop::AllEvents); if (dialog.wasCanceled()) { m_pendingLoad.clear(); Exif::Database::instance()->abortInsertTransaction(); return; } // (*it).first: DB::FileName // (*it).second: DB::MediaType loadExtraFile((*it).first, (*it).second); loadedCount++; // Atomic if (timeSinceProgressUpdate.elapsed() >= 1000) { dialog.setValue(count); timeSinceProgressUpdate.restart(); } } dialog.setValue(count); // loadExtraFile() has already inserted all images into the // database, but without committing the changes DB::ImageDB::instance()->commitDelayedImages(); Exif::Database::instance()->commitInsertTransaction(); ImageManager::ThumbnailBuilder::instance()->save(); } void NewImageFinder::setupFileVersionDetection() { // should be cached because loading once per image is expensive m_modifiedFileCompString = Settings::SettingsData::instance()->modifiedFileComponent(); m_modifiedFileComponent = QRegExp(m_modifiedFileCompString); m_originalFileComponents << Settings::SettingsData::instance()->originalFileComponent(); m_originalFileComponents = m_originalFileComponents.at(0).split(QString::fromLatin1(";")); } void NewImageFinder::loadExtraFile(const DB::FileName &newFileName, DB::MediaType type) { MD5 sum = MD5Sum(newFileName); if (handleIfImageHasBeenMoved(newFileName, sum)) return; // check to see if this is a new version of a previous image // We'll get the Exif data later, when we get the MD5 checksum. ImageInfoPtr info = ImageInfoPtr(new ImageInfo(newFileName, type, false, false)); ImageInfoPtr originalInfo; DB::FileName originalFileName; if (Settings::SettingsData::instance()->detectModifiedFiles()) { // requires at least *something* in the modifiedFileComponent if (m_modifiedFileCompString.length() >= 0 && newFileName.relative().contains(m_modifiedFileComponent)) { for (QStringList::const_iterator it = m_originalFileComponents.constBegin(); it != m_originalFileComponents.constEnd(); ++it) { QString tmp = newFileName.relative(); tmp.replace(m_modifiedFileComponent, (*it)); originalFileName = DB::FileName::fromRelativePath(tmp); MD5 originalSum; if (newFileName == originalFileName) originalSum = sum; else if (DB::ImageDB::instance()->md5Map()->containsFile(originalFileName)) originalSum = DB::ImageDB::instance()->md5Map()->lookupFile(originalFileName); else // Do *not* attempt to compute the checksum here. It forces a filesystem // lookup on a file that may not exist and substantially degrades // performance by about 25% on an SSD and about 30% on a spinning disk. // If one of these other files exist, it will be found later in // the image search at which point we'll detect the modified file. continue; if (DB::ImageDB::instance()->md5Map()->contains(originalSum)) { // we have a previous copy of this file; copy it's data // from the original. originalInfo = DB::ImageDB::instance()->info(originalFileName); if (!originalInfo) { qCDebug(DBLog) << "Original info not found by name for " << originalFileName.absolute() << ", trying by MD5 sum."; originalFileName = DB::ImageDB::instance()->md5Map()->lookup(originalSum); if (!originalFileName.isNull()) { qCDebug(DBLog) << "Substitute image " << originalFileName.absolute() << " found."; originalInfo = DB::ImageDB::instance()->info(originalFileName); } if (!originalInfo) { qCWarning(DBLog, "How did that happen? We couldn't find info for the original image %s; can't copy the original data to %s", qPrintable(originalFileName.absolute()), qPrintable(newFileName.absolute())); continue; } } info->copyExtraData(*originalInfo); /* if requested to move, then delete old data from original */ if (Settings::SettingsData::instance()->moveOriginalContents()) { originalInfo->removeExtraData(); } break; } } } } ImageInfoList newImages; newImages.append(info); DB::ImageDB::instance()->addImages(newImages, false); // also inserts image into exif db if present: info->setMD5Sum(sum); DB::ImageDB::instance()->md5Map()->insert(sum, info->fileName()); if (originalInfo && Settings::SettingsData::instance()->autoStackNewFiles()) { // stack the files together DB::FileName olderfile = originalFileName; DB::FileName newerfile = info->fileName(); DB::FileNameList tostack; // the newest file should go to the top of the stack tostack.append(newerfile); DB::FileNameList oldStack; if ((oldStack = DB::ImageDB::instance()->getStackFor(olderfile)).isEmpty()) { tostack.append(olderfile); } else { for (const DB::FileName &tmp : oldStack) { tostack.append(tmp); } } DB::ImageDB::instance()->stack(tostack); MainWindow::Window::theMainWindow()->setStackHead(newerfile); // ordering: XXX we ideally want to place the new image right // after the older one in the list. } markUnTagged(info); ImageManager::ThumbnailBuilder::instance()->buildOneThumbnail(info); if (info->isVideo() && MainWindow::FeatureDialog::hasVideoThumbnailer()) { // needs to be done *after* insertion into database BackgroundTaskManager::JobManager::instance()->addJob( new BackgroundJobs::ReadVideoLengthJob(info->fileName(), BackgroundTaskManager::BackgroundVideoPreviewRequest)); } } bool NewImageFinder::handleIfImageHasBeenMoved(const FileName &newFileName, const MD5 &sum) { if (DB::ImageDB::instance()->md5Map()->contains(sum)) { const DB::FileName matchedFileName = DB::ImageDB::instance()->md5Map()->lookup(sum); QFileInfo fi(matchedFileName.absolute()); if (!fi.exists()) { // The file we had a collapse with didn't exists anymore so it is likely moved to this new name ImageInfoPtr info = DB::ImageDB::instance()->info(matchedFileName); if (!info) qCWarning(DBLog, "How did that happen? We couldn't find info for the images %s", qPrintable(matchedFileName.relative())); else { fi = QFileInfo(matchedFileName.relative()); if (info->label() == fi.completeBaseName()) { fi = QFileInfo(newFileName.absolute()); info->setLabel(fi.completeBaseName()); } DB::ImageDB::instance()->renameImage(info, newFileName); // We need to insert the new name into the MD5 map, // as it is a map, the value for the moved file will automatically be deleted. DB::ImageDB::instance()->md5Map()->insert(sum, info->fileName()); Exif::Database::instance()->remove(matchedFileName); Exif::Database::instance()->add(newFileName); ImageManager::ThumbnailBuilder::instance()->buildOneThumbnail(info); return true; } } } return false; // The image wasn't just moved } bool NewImageFinder::calculateMD5sums( const DB::FileNameList &list, DB::MD5Map *md5Map, bool *wasCanceled) { // FIXME: should be converted to a threadpool for SMP stuff and whatnot :] QProgressDialog dialog; dialog.setLabelText( i18np("

Calculating checksum for %1 file

", "

Calculating checksums for %1 files

", list.size()) + i18n("

By storing a checksum for each image " "KPhotoAlbum is capable of finding images " "even when you have moved them on the disk.

")); dialog.setMaximum(list.size()); dialog.setMinimumDuration(1000); int count = 0; DB::FileNameList cantRead; bool dirty = false; for (const FileName &fileName : list) { if (count % 10 == 0) { dialog.setValue(count); // ensure to call setProgress(0) qApp->processEvents(QEventLoop::AllEvents); if (dialog.wasCanceled()) { if (wasCanceled) *wasCanceled = true; return dirty; } } MD5 md5 = MD5Sum(fileName); if (md5.isNull()) { cantRead << fileName; continue; } ImageInfoPtr info = ImageDB::instance()->info(fileName); if (info->MD5Sum() != md5) { info->setMD5Sum(md5); dirty = true; ImageManager::ThumbnailCache::instance()->removeThumbnail(fileName); } md5Map->insert(md5, fileName); ++count; } if (wasCanceled) *wasCanceled = false; if (!cantRead.empty()) KMessageBox::informationList(nullptr, i18n("Following files could not be read:"), cantRead.toStringList(DB::RelativeToImageRoot)); return dirty; } void DB::NewImageFinder::markUnTagged(ImageInfoPtr info) { if (Settings::SettingsData::instance()->hasUntaggedCategoryFeatureConfigured()) { info->addCategoryInfo(Settings::SettingsData::instance()->untaggedCategory(), Settings::SettingsData::instance()->untaggedTag()); } } // vi:expandtab:tabstop=4 shiftwidth=4: diff --git a/ImageManager/AsyncLoader.cpp b/ImageManager/AsyncLoader.cpp index e16cf90a..e8b73e9c 100644 --- a/ImageManager/AsyncLoader.cpp +++ b/ImageManager/AsyncLoader.cpp @@ -1,245 +1,249 @@ /* Copyright (C) 2003-2010 Jesper K. Pedersen This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; see the file COPYING. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "AsyncLoader.h" #include "CancelEvent.h" #include "ImageClientInterface.h" #include "ImageEvent.h" #include "ImageLoaderThread.h" #include "ThumbnailBuilder.h" #include "ThumbnailCache.h" #include #include #include +#include #include #include #include ImageManager::AsyncLoader *ImageManager::AsyncLoader::s_instance = nullptr; // -- Manager -- ImageManager::AsyncLoader *ImageManager::AsyncLoader::instance() { if (!s_instance) { s_instance = new AsyncLoader; s_instance->init(); } return s_instance; } // We need this as a separate method as the s_instance variable will otherwise not be initialized // corrected before the thread starts. void ImageManager::AsyncLoader::init() { // Use up to three cores for thumbnail generation. No more than three as that // likely will make it less efficient due to three cores hitting the harddisk at the same time. // This might limit the throughput on SSD systems, but we likely have a few years before people // put all of their pictures on SSDs. // rlk 20180515: with improvements to the thumbnail generation code, I've conducted // experiments demonstrating benefit even at 2x the number of hyperthreads, even on // an HDD. However, we need to reserve a thread for the UI or it gets very sluggish // We need one more core in the computer for the GUI thread, but we won't dedicate it to GUI, // as that'd mean that a dual-core box would only have one core decoding images, which would be // suboptimal. // In case of only one core in the computer, use one core for thumbnail generation // TODO(isilmendil): It seems that many people have their images on NFS-mounts. // Should we somehow detect this and allocate less threads there? // rlk 20180515: IMO no; if anything, we need more threads to hide // the latency of NFS. - const int cores = qMax(1, qMin(16, QThread::idealThreadCount() - 1)); + int desiredThreads = Settings::SettingsData::instance()->getThumbnailBuilderThreadCount(); + if (desiredThreads == 0) { + desiredThreads = qMax(1, qMin(16, QThread::idealThreadCount() - 1)); + } m_exitRequested = false; - for (int i = 0; i < cores; ++i) { + for (int i = 0; i < desiredThreads; ++i) { ImageLoaderThread *imageLoader = new ImageLoaderThread(); // The thread is set to the lowest priority to ensure that it doesn't starve the GUI thread. m_threadList << imageLoader; imageLoader->start(QThread::IdlePriority); } } bool ImageManager::AsyncLoader::load(ImageRequest *request) { if (m_exitRequested) return false; // rlk 2018-05-15: Skip this check here. Even if the check // succeeds at this point, it may fail later, and if we're suddenly // processing a lot of requests (e. g. a thumbnail build), // this may be very I/O-intensive since it actually has to // read the inode. // silently ignore images not (currently) on disk: // if ( ! request->fileSystemFileName().exists() ) // return false; if (Utilities::isVideo(request->fileSystemFileName())) { if (!loadVideo(request)) return false; } else { loadImage(request); } return true; } bool ImageManager::AsyncLoader::loadVideo(ImageRequest *request) { if (m_exitRequested) return false; if (!MainWindow::FeatureDialog::hasVideoThumbnailer()) return false; BackgroundTaskManager::Priority priority = (request->priority() > ThumbnailInvisible) ? BackgroundTaskManager::ForegroundThumbnailRequest : BackgroundTaskManager::BackgroundVideoThumbnailRequest; BackgroundTaskManager::JobManager::instance()->addJob( new BackgroundJobs::HandleVideoThumbnailRequestJob(request, priority)); return true; } void ImageManager::AsyncLoader::loadImage(ImageRequest *request) { QMutexLocker dummy(&m_lock); if (m_exitRequested) return; QSet::const_iterator req = m_currentLoading.find(request); if (req != m_currentLoading.end() && m_loadList.isRequestStillValid(request)) { // The last part of the test above is needed to not fail on a race condition from AnnotationDialog::ImagePreview, where the preview // at startup request the same image numerous time (likely from resize event). Q_ASSERT(*req != request); delete request; return; // We are currently loading it, calm down and wait please ;-) } // Try harder to find a pending request. Unfortunately, we can't simply use // m_currentLoading.contains() because that will compare pointers // when we want to compare values. for (req = m_currentLoading.begin(); req != m_currentLoading.end(); req++) { ImageRequest *r = *req; if (*request == *r) { delete request; return; // We are currently loading it, calm down and wait please ;-) } } // if request is "fresh" (not yet pending): if (m_loadList.addRequest(request)) m_sleepers.wakeOne(); } void ImageManager::AsyncLoader::stop(ImageClientInterface *client, StopAction action) { // remove from pending map. QMutexLocker requestLocker(&m_lock); m_loadList.cancelRequests(client, action); // PENDING(blackie) Reintroduce this // VideoManager::instance().stop( client, action ); // Was implemented as m_pending.cancelRequests( client, action ); // Where m_pending is the RequestQueue } int ImageManager::AsyncLoader::activeCount() const { QMutexLocker dummy(&m_lock); return m_currentLoading.count(); } bool ImageManager::AsyncLoader::isExiting() const { return m_exitRequested; } ImageManager::ImageRequest *ImageManager::AsyncLoader::next() { QMutexLocker dummy(&m_lock); ImageRequest *request = nullptr; while (!(request = m_loadList.popNext())) m_sleepers.wait(&m_lock); m_currentLoading.insert(request); return request; } void ImageManager::AsyncLoader::requestExit() { m_exitRequested = true; ImageManager::ThumbnailBuilder::instance()->cancelRequests(); m_sleepers.wakeAll(); // TODO(jzarl): check if we can just connect the finished() signal of the threads to deleteLater() // and exit this function without waiting for (QList::iterator it = m_threadList.begin(); it != m_threadList.end(); ++it) { while (!(*it)->isFinished()) { QThread::msleep(10); } delete (*it); } } void ImageManager::AsyncLoader::customEvent(QEvent *ev) { if (ev->type() == ImageEventID) { ImageEvent *iev = dynamic_cast(ev); if (!iev) { Q_ASSERT(iev); return; } ImageRequest *request = iev->loadInfo(); QMutexLocker requestLocker(&m_lock); const bool requestStillNeeded = m_loadList.isRequestStillValid(request); m_loadList.removeRequest(request); m_currentLoading.remove(request); requestLocker.unlock(); QImage image = iev->image(); if (!request->loadedOK()) { if (m_brokenImage.size() != request->size()) { // we can ignore the krazy warning here because we have a valid fallback QIcon brokenFileIcon = QIcon::fromTheme(QLatin1String("file-broken")); // krazy:exclude=iconnames if (brokenFileIcon.isNull()) { brokenFileIcon = QIcon::fromTheme(QLatin1String("image-x-generic")); } m_brokenImage = brokenFileIcon.pixmap(request->size()).toImage(); } image = m_brokenImage; } if (request->isThumbnailRequest()) ImageManager::ThumbnailCache::instance()->insert(request->databaseFileName(), image); if (requestStillNeeded && request->client()) { request->client()->pixmapLoaded(request, image); } delete request; } else if (ev->type() == CANCELEVENTID) { CancelEvent *cancelEvent = dynamic_cast(ev); cancelEvent->request()->client()->requestCanceled(); } } // vi:expandtab:tabstop=4 shiftwidth=4: diff --git a/ImageManager/ThumbnailBuilder.cpp b/ImageManager/ThumbnailBuilder.cpp index 1d536ce6..9bf6febb 100644 --- a/ImageManager/ThumbnailBuilder.cpp +++ b/ImageManager/ThumbnailBuilder.cpp @@ -1,224 +1,225 @@ /* Copyright (C) 2003-2019 The KPhotoAlbum Development Team This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; see the file COPYING. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "ThumbnailBuilder.h" #include "AsyncLoader.h" #include "Logging.h" #include "PreloadRequest.h" #include "ThumbnailCache.h" #include #include #include #include +#include #include #include #include #include #include ImageManager::ThumbnailBuilder *ImageManager::ThumbnailBuilder::s_instance = nullptr; ImageManager::ThumbnailBuilder::ThumbnailBuilder(MainWindow::StatusBar *statusBar, QObject *parent) : QObject(parent) , m_statusBar(statusBar) , m_count(0) , m_isBuilding(false) , m_loadedCount(0) , m_preloadQueue(nullptr) , m_scout(nullptr) { connect(m_statusBar, &MainWindow::StatusBar::cancelRequest, this, &ThumbnailBuilder::cancelRequests); s_instance = this; // Make sure that this is created early, in the main thread, so it // can receive signals. ThumbnailCache::instance(); m_startBuildTimer = new QTimer(this); m_startBuildTimer->setSingleShot(true); connect(m_startBuildTimer, &QTimer::timeout, this, &ThumbnailBuilder::doThumbnailBuild); } void ImageManager::ThumbnailBuilder::cancelRequests() { ImageManager::AsyncLoader::instance()->stop(this, ImageManager::StopAll); m_isBuilding = false; m_statusBar->setProgressBarVisible(false); m_startBuildTimer->stop(); } void ImageManager::ThumbnailBuilder::terminateScout() { if (m_scout) { delete m_scout; m_scout = nullptr; } if (m_preloadQueue) { delete m_preloadQueue; m_preloadQueue = nullptr; } } void ImageManager::ThumbnailBuilder::pixmapLoaded(ImageManager::ImageRequest *request, const QImage & /*image*/) { const DB::FileName fileName = request->databaseFileName(); const QSize fullSize = request->fullSize(); DB::ImageInfoPtr info = DB::ImageDB::instance()->info(fileName); // We probably shouldn't do this at all, since the "full size" // of the request could be the size of the embedded thumbnail // or even a scaled-down such. But if this hasn't been // set orrectly earlier, we have nothing else to go on. if (fullSize.width() != -1 && info->size().width() == -1) { info->setSize(fullSize); } m_loadedCount++; m_statusBar->setProgress(++m_count); if (m_count >= m_expectedThumbnails) { terminateScout(); } } void ImageManager::ThumbnailBuilder::buildAll(ThumbnailBuildStart when) { QMessageBox msgBox; msgBox.setText(i18n("Building all thumbnails may take a long time.")); msgBox.setInformativeText(i18n("Do you want to rebuild all of your thumbnails?")); msgBox.setStandardButtons(QMessageBox::Yes | QMessageBox::No); msgBox.setDefaultButton(QMessageBox::No); int ret = msgBox.exec(); if (ret == QMessageBox::Yes) { ImageManager::ThumbnailCache::instance()->flush(); scheduleThumbnailBuild(DB::ImageDB::instance()->images(), when); } } ImageManager::ThumbnailBuilder *ImageManager::ThumbnailBuilder::instance() { Q_ASSERT(s_instance); return s_instance; } ImageManager::ThumbnailBuilder::~ThumbnailBuilder() { terminateScout(); } void ImageManager::ThumbnailBuilder::buildMissing() { const DB::FileNameList images = DB::ImageDB::instance()->images(); DB::FileNameList needed; for (const DB::FileName &fileName : images) { if (!ImageManager::ThumbnailCache::instance()->contains(fileName)) needed.append(fileName); } scheduleThumbnailBuild(needed, StartDelayed); } void ImageManager::ThumbnailBuilder::scheduleThumbnailBuild(const DB::FileNameList &list, ThumbnailBuildStart when) { if (list.count() == 0) return; if (m_isBuilding) cancelRequests(); DB::OptimizedFileList files(list); m_thumbnailsToBuild = files.optimizedDbFiles(); m_startBuildTimer->start(when == StartNow ? 0 : 5000); } void ImageManager::ThumbnailBuilder::buildOneThumbnail(const DB::ImageInfoPtr &info) { ImageManager::ImageRequest *request = new ImageManager::PreloadRequest(info->fileName(), ThumbnailView::CellGeometry::preferredIconSize(), info->angle(), this); request->setIsThumbnailRequest(true); request->setPriority(ImageManager::BuildThumbnails); ImageManager::AsyncLoader::instance()->load(request); } void ImageManager::ThumbnailBuilder::doThumbnailBuild() { m_isBuilding = true; int numberOfThumbnailsToBuild = 0; terminateScout(); m_count = 0; m_loadedCount = 0; m_preloadQueue = new DB::ImageScoutQueue; for (const DB::FileName &fileName : m_thumbnailsToBuild) { m_preloadQueue->enqueue(fileName); } qCDebug(ImageManagerLog) << "thumbnail builder starting scout"; - m_scout = new DB::ImageScout(*m_preloadQueue, m_loadedCount, 1); + m_scout = new DB::ImageScout(*m_preloadQueue, m_loadedCount, Settings::SettingsData::instance()->getThumbnailPreloadThreadCount()); m_scout->setMaxSeekAhead(10); m_scout->setReadLimit(10 * 1048576); m_scout->start(); m_statusBar->startProgress(i18n("Building thumbnails"), qMax(m_thumbnailsToBuild.size() - 1, 1)); // We'll update this later. Meanwhile, we want to make sure that the scout // isn't prematurely terminated because the expected number of thumbnails // is less than (i. e. zero) the number of thumbnails actually built. m_expectedThumbnails = m_thumbnailsToBuild.size(); for (const DB::FileName &fileName : m_thumbnailsToBuild) { DB::ImageInfoPtr info = fileName.info(); if (ImageManager::AsyncLoader::instance()->isExiting()) { cancelRequests(); break; } if (info->isNull()) { m_loadedCount++; m_count++; continue; } ImageManager::ImageRequest *request = new ImageManager::PreloadRequest(fileName, ThumbnailView::CellGeometry::preferredIconSize(), info->angle(), this); request->setIsThumbnailRequest(true); request->setPriority(ImageManager::BuildThumbnails); if (ImageManager::AsyncLoader::instance()->load(request)) ++numberOfThumbnailsToBuild; } m_expectedThumbnails = numberOfThumbnailsToBuild; if (numberOfThumbnailsToBuild == 0) { m_statusBar->setProgressBarVisible(false); terminateScout(); } } void ImageManager::ThumbnailBuilder::save() { ImageManager::ThumbnailCache::instance()->save(); } void ImageManager::ThumbnailBuilder::requestCanceled() { m_statusBar->setProgress(++m_count); m_loadedCount++; if (m_count >= m_expectedThumbnails) { terminateScout(); } } // vi:expandtab:tabstop=4 shiftwidth=4: diff --git a/Settings/FileVersionDetectionPage.cpp b/Settings/FileVersionDetectionPage.cpp index 5150f8f6..a2ba5a17 100644 --- a/Settings/FileVersionDetectionPage.cpp +++ b/Settings/FileVersionDetectionPage.cpp @@ -1,224 +1,322 @@ /* Copyright (C) 2003-2010 Jesper K. Pedersen This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; see the file COPYING. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "FileVersionDetectionPage.h" #include "SettingsData.h" +#include #include #include #include #include #include +#include #include Settings::FileVersionDetectionPage::FileVersionDetectionPage(QWidget *parent) : QWidget(parent) { QVBoxLayout *topLayout = new QVBoxLayout(this); QString txt; // General file searching { QGroupBox *generalBox = new QGroupBox(i18n("New File Searches"), this); topLayout->addWidget(generalBox); QVBoxLayout *layout = new QVBoxLayout(generalBox); // Search for images on startup m_searchForImagesOnStart = new QCheckBox(i18n("Search for new images and videos on startup"), generalBox); layout->addWidget(m_searchForImagesOnStart); m_ignoreFileExtension = new QCheckBox(i18n("Ignore file extensions when searching for new images and videos"), generalBox); layout->addWidget(m_ignoreFileExtension); m_skipSymlinks = new QCheckBox(i18n("Skip symbolic links when searching for new images"), generalBox); layout->addWidget(m_skipSymlinks); m_skipRawIfOtherMatches = new QCheckBox(i18n("Do not read RAW files if a matching JPEG/TIFF file exists"), generalBox); layout->addWidget(m_skipRawIfOtherMatches); // Exclude directories from search QLabel *excludeDirectoriesLabel = new QLabel(i18n("Directories to exclude from new file search:"), generalBox); layout->addWidget(excludeDirectoriesLabel); m_excludeDirectories = new QLineEdit(generalBox); layout->addWidget(m_excludeDirectories); excludeDirectoriesLabel->setBuddy(m_excludeDirectories); txt = i18n("

KPhotoAlbum is capable of searching for new images and videos when started, this does, " "however, take some time, so instead you may wish to manually tell KPhotoAlbum to search for new images " "using Maintenance->Rescan for new images

"); m_searchForImagesOnStart->setWhatsThis(txt); txt = i18n("

KPhotoAlbum will normally search new images and videos by their file extension. " "If this option is set, all files neither in the database nor in the block list " "will be checked by their Mime type, regardless of their extension. This will take " "significantly longer than finding files by extension!

"); m_ignoreFileExtension->setWhatsThis(txt); txt = i18n("

KPhotoAlbum attempts to read all image files whether actual files or symbolic links. If you " "wish to ignore symbolic links, check this option. This is useful if for some reason you have e.g. " "both the original files and symbolic links to these files within your image directory.

"); m_skipSymlinks->setWhatsThis(txt); txt = i18n("

KPhotoAlbum is capable of reading certain kinds of RAW images. " "Some cameras store both a RAW image and a matching JPEG or TIFF image. " "This causes duplicate images to be stored in KPhotoAlbum, which may be undesirable. " "If this option is checked, KPhotoAlbum will not read RAW files for which matching image files also exist.

"); m_skipRawIfOtherMatches->setWhatsThis(txt); txt = i18n("

Directories defined here (separated by comma ,) are " "skipped when searching for new photos. Thumbnail directories of different " "tools should be configured here. E.g. xml,ThumbNails,.thumbs,.thumbnails.

"); excludeDirectoriesLabel->setWhatsThis(txt); } // Original/Modified File Support { QGroupBox *modifiedBox = new QGroupBox(i18n("File Version Detection Settings"), this); topLayout->addWidget(modifiedBox); QVBoxLayout *layout = new QVBoxLayout(modifiedBox); m_detectModifiedFiles = new QCheckBox(i18n("Try to detect multiple versions of files"), modifiedBox); layout->addWidget(m_detectModifiedFiles); QLabel *modifiedFileComponentLabel = new QLabel(i18n("File versions search regexp:"), modifiedBox); layout->addWidget(modifiedFileComponentLabel); m_modifiedFileComponent = new QLineEdit(modifiedBox); layout->addWidget(m_modifiedFileComponent); QLabel *originalFileComponentLabel = new QLabel(i18n("Original file replacement text:"), modifiedBox); layout->addWidget(originalFileComponentLabel); m_originalFileComponent = new QLineEdit(modifiedBox); layout->addWidget(m_originalFileComponent); m_moveOriginalContents = new QCheckBox(i18n("Move meta-data (i.e. delete tags from the original)"), modifiedBox); layout->addWidget(m_moveOriginalContents); m_autoStackNewFiles = new QCheckBox(i18n("Automatically stack new versions of images"), modifiedBox); layout->addWidget(m_autoStackNewFiles); txt = i18n("

When KPhotoAlbum searches for new files and finds a file that matches the " "modified file search regexp it is assumed that an original version of " "the image may exist. The regexp pattern will be replaced with the original " "file replacement text and if that file exists, all associated metadata (category " "information, ratings, etc) will be copied or moved from the original file to the new one.

"); m_detectModifiedFiles->setWhatsThis(txt); txt = i18n("

A perl regular expression that should match a modified file. " "

  • A dot matches a single character (\\. matches a dot)
  • " "
  • You can use the quantifiers *,+,?, or you can " " match multiple occurrences of an expression by using curly brackets (e.g. " "e{0,1} matches 0 or 1 occurrences of the character \"e\").
  • " "
  • You can group parts of the expression using parenthesis.
  • " "
Example: -modified\\.(jpg|tiff)

"); modifiedFileComponentLabel->setWhatsThis(txt); m_modifiedFileComponent->setWhatsThis(txt); txt = i18n("

A string that is used to replace the match from the File versions search regexp. " "This can be a semicolon (;) separated list. Each string is used to replace the match " "in the new file's name until an original file is found or we run out of options.

"); originalFileComponentLabel->setWhatsThis(txt); m_originalFileComponent->setWhatsThis(txt); txt = i18n("

The tagging is moved from the original file to the new file. This way " "only the latest version of an image is tagged.

"); m_moveOriginalContents->setWhatsThis(txt); txt = i18n("

If this option is set, new versions of an image are automatically stacked " "and placed to the top of the stack. This way the new image is shown when the " "stack is in collapsed state - the default state in KPhotoAlbum.

"); m_autoStackNewFiles->setWhatsThis(txt); } // Copy File Support { QGroupBox *copyBox = new QGroupBox(i18nc("Configure the feature to make a copy of a file first and then open the copied file with an external application", "Copy File and Open with an External Application"), this); topLayout->addWidget(copyBox); QVBoxLayout *layout = new QVBoxLayout(copyBox); QLabel *copyFileComponentLabel = new QLabel(i18n("Copy file search regexp:"), copyBox); layout->addWidget(copyFileComponentLabel); m_copyFileComponent = new QLineEdit(copyBox); layout->addWidget(m_copyFileComponent); QLabel *copyFileReplacementComponentLabel = new QLabel(i18n("Copy file replacement text:"), copyBox); layout->addWidget(copyFileReplacementComponentLabel); m_copyFileReplacementComponent = new QLineEdit(copyBox); layout->addWidget(m_copyFileReplacementComponent); txt = i18n("

KPhotoAlbum can make a copy of an image before opening it with an external application. This configuration defines how the new file is named.

" "

The regular expression defines the part of the original file name that is replaced with the replacement text. " "E.g. regexp \"\\.(jpg|png)\" and replacement text \"-mod.\\1\" would copy test.jpg to test-mod.jpg and open the new file in selected application.

"); copyFileComponentLabel->setWhatsThis(txt); m_copyFileComponent->setWhatsThis(txt); copyFileReplacementComponentLabel->setWhatsThis(txt); m_copyFileReplacementComponent->setWhatsThis(txt); } + + // Loader Optimization Setting (prototype) + { + QGroupBox *loadOptimizationBox = new QGroupBox(i18n("EXPERIMENTAL: Tune new image loading for best performance."), this); + topLayout->addWidget(loadOptimizationBox); + QGridLayout *layout = new QGridLayout(loadOptimizationBox); + int row = 0; + + // Loader preset + QLabel *loadOptimizationPresetLabel = new QLabel(i18n("Type of media on which your images are stored.")); + layout->addWidget(loadOptimizationPresetLabel, row, 0); + layout->setColumnStretch(0, 1); + + m_loadOptimizationPreset = new KComboBox; + m_loadOptimizationPreset->addItems(QStringList() << i18n("Hard Disk") + << i18n("Network") + << i18n("SATA SSD") + << i18n("Slow PCIe/NVMe") + << i18n("Fast PCIe/NVMe") + << i18n("Manual Settings")); + layout->addWidget(m_loadOptimizationPreset, row, 1); + + txt = i18n("

Tune image loading for best performance based on the type of storage your image database resides on. If your image database resides on multiple media, choose the slowet media type used.

" + "

Use Manual Settings to configure details of how the loading is performed.

"); + loadOptimizationPresetLabel->setWhatsThis(txt); + m_loadOptimizationPreset->setWhatsThis(txt); + + // Overlap load with MD5 computation + ++row; + QLabel *overlapLoadMD5Label = new QLabel(i18n("Calculate MD5 checksum while images are being preloaded")); + layout->addWidget(overlapLoadMD5Label, row, 0); + m_overlapLoadMD5 = new QCheckBox; + layout->addWidget(m_overlapLoadMD5, row, 1); + + txt = i18n("

Calculate MD5 checksums while images are being preloaded. This works well if the storage is very fast, such as an NVMe drive. If the storage is slow, this degrades performance as the checksum calculation has to wait longer for the images to be preloaded.

"); + overlapLoadMD5Label->setWhatsThis(txt); + m_overlapLoadMD5->setWhatsThis(txt); + + // Threads for preload + ++row; + QLabel *preloadThreadCountLabel = new QLabel(i18n("Number of threads to use for preloading (scouting) images")); + layout->addWidget(preloadThreadCountLabel, row, 0); + + m_preloadThreadCount = new QSpinBox; + m_preloadThreadCount->setRange(1, 16); + m_preloadThreadCount->setSingleStep(1); + layout->addWidget(m_preloadThreadCount, row, 1); + + txt = i18n("

Number of threads to use for preloading images to have them in memory when their checksums are calculated. This should generally be set higher for faster storage, but not more than the number of cores in your CPU.<. Default is 1, which works well for mechanical hard disks./p>"); + preloadThreadCountLabel->setWhatsThis(txt); + m_preloadThreadCount->setWhatsThis(txt); + + // Threads for thumbnailPreload + ++row; + QLabel *thumbnailPreloadThreadCountLabel = new QLabel(i18n("Number of threads to use for thumbnailPreloading (scouting) images")); + layout->addWidget(thumbnailPreloadThreadCountLabel, row, 0); + + m_thumbnailPreloadThreadCount = new QSpinBox; + m_thumbnailPreloadThreadCount->setRange(1, 16); + m_thumbnailPreloadThreadCount->setSingleStep(1); + layout->addWidget(m_thumbnailPreloadThreadCount, row, 1); + + txt = i18n("

Number of threads to use for preloading images prior to building thumbnails. Normally this should be set to 1; the exception might be if you have very fast storage.

"); + thumbnailPreloadThreadCountLabel->setWhatsThis(txt); + m_thumbnailPreloadThreadCount->setWhatsThis(txt); + + // Threads for thumbnailBuilder + ++row; + QLabel *thumbnailBuilderThreadCountLabel = new QLabel(i18n("Number of threads to use for building thumbnails")); + layout->addWidget(thumbnailBuilderThreadCountLabel, row, 0); + + m_thumbnailBuilderThreadCount = new QSpinBox(loadOptimizationBox); + m_thumbnailBuilderThreadCount->setRange(0, 16); + m_thumbnailBuilderThreadCount->setSingleStep(1); + layout->addWidget(m_thumbnailBuilderThreadCount, row, 1); + + txt = i18n("

Number of threads to use for building thumbnails. If set to 0 this will be set automatically one less than the number of cores, at least one and no more than three. If you have fast storage and a CPU with many cores, you may see benefit from setting this to a larger value.

" + "

KPhotoAlbum must be restarted for changes to take effect.

"); + thumbnailBuilderThreadCountLabel->setWhatsThis(txt); + m_thumbnailBuilderThreadCount->setWhatsThis(txt); + } } Settings::FileVersionDetectionPage::~FileVersionDetectionPage() { delete m_searchForImagesOnStart; delete m_ignoreFileExtension; delete m_skipSymlinks; delete m_skipRawIfOtherMatches; delete m_excludeDirectories; delete m_detectModifiedFiles; delete m_modifiedFileComponent; delete m_originalFileComponent; delete m_moveOriginalContents; delete m_autoStackNewFiles; delete m_copyFileComponent; delete m_copyFileReplacementComponent; + delete m_loadOptimizationPreset; + delete m_overlapLoadMD5; + delete m_preloadThreadCount; + delete m_thumbnailPreloadThreadCount; + delete m_thumbnailBuilderThreadCount; } void Settings::FileVersionDetectionPage::loadSettings(Settings::SettingsData *opt) { m_searchForImagesOnStart->setChecked(opt->searchForImagesOnStart()); m_ignoreFileExtension->setChecked(opt->ignoreFileExtension()); m_skipSymlinks->setChecked(opt->skipSymlinks()); m_skipRawIfOtherMatches->setChecked(opt->skipRawIfOtherMatches()); m_excludeDirectories->setText(opt->excludeDirectories()); m_detectModifiedFiles->setChecked(opt->detectModifiedFiles()); m_modifiedFileComponent->setText(opt->modifiedFileComponent()); m_originalFileComponent->setText(opt->originalFileComponent()); m_moveOriginalContents->setChecked(opt->moveOriginalContents()); m_autoStackNewFiles->setChecked(opt->autoStackNewFiles()); m_copyFileComponent->setText(opt->copyFileComponent()); m_copyFileReplacementComponent->setText(opt->copyFileReplacementComponent()); + m_loadOptimizationPreset->setCurrentIndex(opt->loadOptimizationPreset()); + m_overlapLoadMD5->setChecked(opt->overlapLoadMD5()); + m_preloadThreadCount->setValue(opt->preloadThreadCount()); + m_thumbnailPreloadThreadCount->setValue(opt->thumbnailPreloadThreadCount()); + m_thumbnailBuilderThreadCount->setValue(opt->thumbnailBuilderThreadCount()); } void Settings::FileVersionDetectionPage::saveSettings(Settings::SettingsData *opt) { opt->setSearchForImagesOnStart(m_searchForImagesOnStart->isChecked()); opt->setIgnoreFileExtension(m_ignoreFileExtension->isChecked()); opt->setSkipSymlinks(m_skipSymlinks->isChecked()); opt->setSkipRawIfOtherMatches(m_skipRawIfOtherMatches->isChecked()); opt->setExcludeDirectories(m_excludeDirectories->text()); opt->setDetectModifiedFiles(m_detectModifiedFiles->isChecked()); opt->setModifiedFileComponent(m_modifiedFileComponent->text()); opt->setOriginalFileComponent(m_originalFileComponent->text()); opt->setAutoStackNewFiles(m_autoStackNewFiles->isChecked()); opt->setCopyFileComponent(m_copyFileComponent->text()); opt->setCopyFileReplacementComponent(m_copyFileReplacementComponent->text()); + opt->setLoadOptimizationPreset(m_loadOptimizationPreset->currentIndex()); + opt->setOverlapLoadMD5(m_overlapLoadMD5->isChecked()); + opt->setPreloadThreadCount(m_preloadThreadCount->value()); + opt->setThumbnailPreloadThreadCount(m_thumbnailPreloadThreadCount->value()); + opt->setThumbnailBuilderThreadCount(m_thumbnailBuilderThreadCount->value()); } // vi:expandtab:tabstop=4 shiftwidth=4: diff --git a/Settings/FileVersionDetectionPage.h b/Settings/FileVersionDetectionPage.h index da8f0917..c414c3f0 100644 --- a/Settings/FileVersionDetectionPage.h +++ b/Settings/FileVersionDetectionPage.h @@ -1,58 +1,63 @@ /* Copyright (C) 2003-2010 Jesper K. Pedersen This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; see the file COPYING. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef FILEVERSIONDETECTIONPAGE_H #define FILEVERSIONDETECTIONPAGE_H #include class KComboBox; class QLineEdit; class QSpinBox; class QComboBox; class QCheckBox; namespace Settings { class SettingsData; class FileVersionDetectionPage : public QWidget { public: explicit FileVersionDetectionPage(QWidget *parent); ~FileVersionDetectionPage() override; void loadSettings(Settings::SettingsData *); void saveSettings(Settings::SettingsData *); private: QCheckBox *m_searchForImagesOnStart; QCheckBox *m_ignoreFileExtension; QCheckBox *m_skipSymlinks; QCheckBox *m_skipRawIfOtherMatches; QLineEdit *m_excludeDirectories; // Directories to exclude QCheckBox *m_detectModifiedFiles; QLineEdit *m_modifiedFileComponent; QLineEdit *m_originalFileComponent; QCheckBox *m_moveOriginalContents; QCheckBox *m_autoStackNewFiles; QLineEdit *m_copyFileComponent; QLineEdit *m_copyFileReplacementComponent; + KComboBox *m_loadOptimizationPreset; + QCheckBox *m_overlapLoadMD5; + QSpinBox *m_preloadThreadCount; + QSpinBox *m_thumbnailPreloadThreadCount; + QSpinBox *m_thumbnailBuilderThreadCount; }; } #endif /* FILEVERSIONDETECTIONPAGE_H */ // vi:expandtab:tabstop=4 shiftwidth=4: diff --git a/Settings/SettingsData.cpp b/Settings/SettingsData.cpp index bd761bd2..7cc4d7b4 100644 --- a/Settings/SettingsData.cpp +++ b/Settings/SettingsData.cpp @@ -1,534 +1,614 @@ /* Copyright (C) 2003-2019 The KPhotoAlbum Development Team This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License or (at your option) version 3 or any later version accepted by the membership of KDE e. V. (or its successor approved by the membership of KDE e. V.), which shall act as a proxy defined in Section 14 of version 3 of the license. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ #include "SettingsData.h" #include #include #include #include #include #include #include #include #include #include +#include #include #define STR(x) QString::fromLatin1(x) #define value(GROUP, OPTION, DEFAULT) \ KSharedConfig::openConfig()->group(GROUP).readEntry(OPTION, DEFAULT) #define setValue(GROUP, OPTION, VALUE) \ { \ KConfigGroup group = KSharedConfig::openConfig()->group(GROUP); \ group.writeEntry(OPTION, VALUE); \ group.sync(); \ } #define getValueFunc_(TYPE, FUNC, GROUP, OPTION, DEFAULT) \ TYPE SettingsData::FUNC() const \ { \ return (TYPE)value(GROUP, OPTION, DEFAULT); \ } #define setValueFunc_(FUNC, TYPE, GROUP, OPTION, VALUE) \ void SettingsData::FUNC(const TYPE v) \ { \ setValue(GROUP, OPTION, VALUE); \ } #define getValueFunc(TYPE, FUNC, GROUP, DEFAULT) getValueFunc_(TYPE, FUNC, #GROUP, #FUNC, DEFAULT) #define setValueFunc(FUNC, TYPE, GROUP, OPTION) setValueFunc_(FUNC, TYPE, #GROUP, #OPTION, v) // TODO(mfwitten): document parameters. #define property_(GET_TYPE, GET_FUNC, GET_VALUE, SET_FUNC, SET_TYPE, SET_VALUE, GROUP, OPTION, GET_DEFAULT_1, GET_DEFAULT_2, GET_DEFAULT_2_TYPE) \ GET_TYPE SettingsData::GET_FUNC() const \ { \ KConfigGroup g = KSharedConfig::openConfig()->group(GROUP); \ \ if (!g.hasKey(OPTION)) \ return GET_DEFAULT_1; \ \ GET_DEFAULT_2_TYPE v = g.readEntry(OPTION, (GET_DEFAULT_2_TYPE)GET_DEFAULT_2); \ return (GET_TYPE)GET_VALUE; \ } \ setValueFunc_(SET_FUNC, SET_TYPE, GROUP, OPTION, SET_VALUE) #define property(GET_TYPE, GET_FUNC, SET_FUNC, SET_TYPE, SET_VALUE, GROUP, OPTION, GET_DEFAULT) \ getValueFunc_(GET_TYPE, GET_FUNC, GROUP, OPTION, GET_DEFAULT) \ setValueFunc_(SET_FUNC, SET_TYPE, GROUP, OPTION, SET_VALUE) #define property_copy(GET_FUNC, SET_FUNC, TYPE, GROUP, GET_DEFAULT) \ property(TYPE, GET_FUNC, SET_FUNC, TYPE, v, #GROUP, #GET_FUNC, GET_DEFAULT) #define property_ref_(GET_FUNC, SET_FUNC, TYPE, GROUP, GET_DEFAULT) \ property(TYPE, GET_FUNC, SET_FUNC, TYPE &, v, GROUP, #GET_FUNC, GET_DEFAULT) #define property_ref(GET_FUNC, SET_FUNC, TYPE, GROUP, GET_DEFAULT) \ property(TYPE, GET_FUNC, SET_FUNC, TYPE &, v, #GROUP, #GET_FUNC, GET_DEFAULT) #define property_enum(GET_FUNC, SET_FUNC, TYPE, GROUP, GET_DEFAULT) \ property(TYPE, GET_FUNC, SET_FUNC, TYPE, (int)v, #GROUP, #GET_FUNC, (int)GET_DEFAULT) #define property_sset(GET_FUNC, SET_FUNC, GROUP, GET_DEFAULT) \ property_(StringSet, GET_FUNC, v.toSet(), SET_FUNC, StringSet &, v.toList(), #GROUP, #GET_FUNC, GET_DEFAULT, QStringList(), QStringList) /** * smoothScale() is called from the image loading thread, therefore we need * to cache it this way, rather than going to KConfig. */ static bool _smoothScale = true; using namespace Settings; const WindowType Settings::MainWindow = "MainWindow"; const WindowType Settings::AnnotationDialog = "AnnotationDialog"; SettingsData *SettingsData::s_instance = nullptr; SettingsData *SettingsData::instance() { if (!s_instance) qFatal("instance called before loading a setup!"); return s_instance; } bool SettingsData::ready() { return s_instance; } void SettingsData::setup(const QString &imageDirectory) { if (!s_instance) s_instance = new SettingsData(imageDirectory); } SettingsData::SettingsData(const QString &imageDirectory) { m_hasAskedAboutTimeStamps = false; QString s = STR("/"); m_imageDirectory = imageDirectory.endsWith(s) ? imageDirectory : imageDirectory + s; _smoothScale = value("Viewer", "smoothScale", true); // Split the list of Exif comments that should be stripped automatically to a list QStringList commentsToStrip = value("General", "commentsToStrip", QString::fromLatin1("Exif_JPEG_PICTURE-,-OLYMPUS DIGITAL CAMERA-,-JENOPTIK DIGITAL CAMERA-,-")).split(QString::fromLatin1("-,-"), QString::SkipEmptyParts); for (QString &comment : commentsToStrip) comment.replace(QString::fromLatin1(",,"), QString::fromLatin1(",")); m_EXIFCommentsToStrip = commentsToStrip; } ///////////////// //// General //// ///////////////// // clang-format off property_copy(useEXIFRotate, setUseEXIFRotate, bool, General, true) property_copy(useEXIFComments, setUseEXIFComments, bool, General, true) property_copy(stripEXIFComments, setStripEXIFComments, bool, General, true) property_copy(commentsToStrip, setCommentsToStrip, QString, General, "" /* see constructor */) property_copy(searchForImagesOnStart, setSearchForImagesOnStart, bool, General, true) property_copy(ignoreFileExtension, setIgnoreFileExtension, bool, General, false) property_copy(skipSymlinks, setSkipSymlinks, bool, General, false) property_copy(skipRawIfOtherMatches, setSkipRawIfOtherMatches, bool, General, false) property_copy(useRawThumbnail, setUseRawThumbnail, bool, General, true) property_copy(useRawThumbnailSize, setUseRawThumbnailSize, QSize, General, QSize(1024, 768)) property_copy(useCompressedIndexXML, setUseCompressedIndexXML, bool, General, true) property_copy(compressBackup, setCompressBackup, bool, General, true) property_copy(showSplashScreen, setShowSplashScreen, bool, General, true) property_copy(showHistogram, setShowHistogram, bool, General, true) property_copy(autoSave, setAutoSave, int, General, 5) property_copy(backupCount, setBackupCount, int, General, 5) property_enum(tTimeStamps, setTTimeStamps, TimeStampTrust, General, Always) property_copy(excludeDirectories, setExcludeDirectories, QString, General, QString::fromLatin1("xml,ThumbNails,.thumbs")) #ifdef KPA_ENABLE_REMOTECONTROL property_copy(recentAndroidAddress, setRecentAndroidAddress, QString, General, QString()) property_copy(listenForAndroidDevicesOnStartup, setListenForAndroidDevicesOnStartup, bool, General, false) #endif getValueFunc(QSize, histogramSize, General, QSize(15, 30)) getValueFunc(ViewSortType, viewSortType, General, (int)SortLastUse) getValueFunc(AnnotationDialog::MatchType, matchType, General, (int)AnnotationDialog::MatchFromWordStart) getValueFunc(bool, histogramUseLinearScale, General, false) // clang-format on void SettingsData::setHistogramUseLinearScale(const bool useLinearScale) { if (useLinearScale == histogramUseLinearScale()) return; setValue("General", "histogramUseLinearScale", useLinearScale); emit histogramScaleChanged(); } void SettingsData::setHistogramSize(const QSize &size) { if (size == histogramSize()) return; setValue("General", "histogramSize", size); emit histogramSizeChanged(size); } void SettingsData::setViewSortType(const ViewSortType tp) { if (tp == viewSortType()) return; setValue("General", "viewSortType", (int)tp); emit viewSortTypeChanged(tp); } void SettingsData::setMatchType(const AnnotationDialog::MatchType mt) { if (mt == matchType()) return; setValue("General", "matchType", (int)mt); emit matchTypeChanged(mt); } bool SettingsData::trustTimeStamps() { if (tTimeStamps() == Always) return true; else if (tTimeStamps() == Never) return false; else { if (!m_hasAskedAboutTimeStamps) { QApplication::setOverrideCursor(Qt::ArrowCursor); QString txt = i18n("When reading time information of images, their Exif info is used. " "Exif info may, however, not be supported by your KPhotoAlbum installation, " "or no valid information may be in the file. " "As a backup, KPhotoAlbum may use the timestamp of the image - this may, " "however, not be valid in case the image is scanned in. " "So the question is, should KPhotoAlbum trust the time stamp on your images?"); int answer = KMessageBox::questionYesNo(nullptr, txt, i18n("Trust Time Stamps?")); QApplication::restoreOverrideCursor(); if (answer == KMessageBox::Yes) m_trustTimeStamps = true; else m_trustTimeStamps = false; m_hasAskedAboutTimeStamps = true; } return m_trustTimeStamps; } } //////////////////////////////// //// File Version Detection //// //////////////////////////////// // clang-format off property_copy(detectModifiedFiles, setDetectModifiedFiles, bool, FileVersionDetection, true) property_copy(modifiedFileComponent, setModifiedFileComponent, QString, FileVersionDetection, "^(.*)-edited.([^.]+)$") property_copy(originalFileComponent, setOriginalFileComponent, QString, FileVersionDetection, "\\1.\\2") property_copy(moveOriginalContents, setMoveOriginalContents, bool, FileVersionDetection, false) property_copy(autoStackNewFiles, setAutoStackNewFiles, bool, FileVersionDetection, true) property_copy(copyFileComponent, setCopyFileComponent, QString, FileVersionDetection, "(.[^.]+)$") property_copy(copyFileReplacementComponent, setCopyFileReplacementComponent, QString, FileVersionDetection, "-edited\\1") +property_copy(loadOptimizationPreset, setLoadOptimizationPreset, int, FileVersionDetection, 0) +property_copy(overlapLoadMD5, setOverlapLoadMD5, bool, FileVersionDetection, false) +property_copy(preloadThreadCount, setPreloadThreadCount, int, FileVersionDetection, 1) +property_copy(thumbnailPreloadThreadCount, setThumbnailPreloadThreadCount, int, FileVersionDetection, 1) +property_copy(thumbnailBuilderThreadCount, setThumbnailBuilderThreadCount, int, FileVersionDetection, 0) // clang-format on //////////////////// //// Thumbnails //// //////////////////// // clang-format off property_copy(displayLabels, setDisplayLabels, bool, Thumbnails, true) property_copy(displayCategories, setDisplayCategories, bool, Thumbnails, false) property_copy(autoShowThumbnailView, setAutoShowThumbnailView, int, Thumbnails, 20) property_copy(showNewestThumbnailFirst, setShowNewestFirst, bool, Thumbnails, false) property_copy(thumbnailDisplayGrid, setThumbnailDisplayGrid, bool, Thumbnails, false) property_copy(previewSize, setPreviewSize, int, Thumbnails, 256) property_copy(thumbnailSpace, setThumbnailSpace, int, Thumbnails, 4) // not available via GUI, but should be consistent (and maybe confgurable for powerusers): property_copy(minimumThumbnailSize, setMinimumThumbnailSize, int, Thumbnails, 32) property_copy(maximumThumbnailSize, setMaximumThumbnailSize, int, Thumbnails, 4096) property_enum(thumbnailAspectRatio, setThumbnailAspectRatio, ThumbnailAspectRatio, Thumbnails, Aspect_3_2) property_ref(backgroundColor, setBackgroundColor, QString, Thumbnails, QColor(Qt::darkGray).name()) property_copy(incrementalThumbnails, setIncrementalThumbnails, bool, Thumbnails, true) // database specific so that changing it doesn't invalidate the thumbnail cache for other databases: getValueFunc_(int, thumbnailSize, groupForDatabase("Thumbnails"), "thumbSize", 256) // clang-format on void SettingsData::setThumbnailSize(int value) { // enforce limits: value = qBound(minimumThumbnailSize(), value, maximumThumbnailSize()); if (value != thumbnailSize()) emit thumbnailSizeChanged(value); setValue(groupForDatabase("Thumbnails"), "thumbSize", value); } int SettingsData::actualThumbnailSize() const { // this is database specific since it's a derived value of thumbnailSize int retval = value(groupForDatabase("Thumbnails"), "actualThumbSize", 0); // if no value has been set, use thumbnailSize if (retval == 0) retval = thumbnailSize(); return retval; } void SettingsData::setActualThumbnailSize(int value) { QPixmapCache::clear(); // enforce limits: value = qBound(minimumThumbnailSize(), value, thumbnailSize()); if (value != actualThumbnailSize()) { setValue(groupForDatabase("Thumbnails"), "actualThumbSize", value); emit actualThumbnailSizeChanged(value); } } //////////////// //// Viewer //// //////////////// // clang-format off property_ref(viewerSize, setViewerSize, QSize, Viewer, QSize(1024, 768)) property_ref(slideShowSize, setSlideShowSize, QSize, Viewer, QSize(1024, 768)) property_copy(launchViewerFullScreen, setLaunchViewerFullScreen, bool, Viewer, false) property_copy(launchSlideShowFullScreen, setLaunchSlideShowFullScreen, bool, Viewer, true) property_copy(showInfoBox, setShowInfoBox, bool, Viewer, true) property_copy(showLabel, setShowLabel, bool, Viewer, true) property_copy(showDescription, setShowDescription, bool, Viewer, true) property_copy(showDate, setShowDate, bool, Viewer, true) property_copy(showImageSize, setShowImageSize, bool, Viewer, true) property_copy(showRating, setShowRating, bool, Viewer, true) property_copy(showTime, setShowTime, bool, Viewer, true) property_copy(showFilename, setShowFilename, bool, Viewer, false) property_copy(showEXIF, setShowEXIF, bool, Viewer, true) property_copy(slideShowInterval, setSlideShowInterval, int, Viewer, 5) property_copy(viewerCacheSize, setViewerCacheSize, int, Viewer, 195) property_copy(infoBoxWidth, setInfoBoxWidth, int, Viewer, 400) property_copy(infoBoxHeight, setInfoBoxHeight, int, Viewer, 300) property_enum(infoBoxPosition, setInfoBoxPosition, Position, Viewer, Bottom) property_enum(viewerStandardSize, setViewerStandardSize, StandardViewSize, Viewer, FullSize) // clang-format on bool SettingsData::smoothScale() const { return _smoothScale; } void SettingsData::setSmoothScale(bool b) { _smoothScale = b; setValue("Viewer", "smoothScale", b); } //////////////////// //// Categories //// //////////////////// setValueFunc(setAlbumCategory, QString &, General, albumCategory) QString SettingsData::albumCategory() const { QString category = value("General", "albumCategory", STR("")); if (!DB::ImageDB::instance()->categoryCollection()->categoryNames().contains(category)) { category = DB::ImageDB::instance()->categoryCollection()->categoryNames()[0]; const_cast(this)->setAlbumCategory(category); } return category; } // clang-format off property_ref(untaggedCategory, setUntaggedCategory, QString, General, i18n("Events")) property_ref(untaggedTag, setUntaggedTag, QString, General, i18n("untagged")) property_copy(untaggedImagesTagVisible, setUntaggedImagesTagVisible, bool, General, false) // clang-format on ////////////// //// Exif //// ////////////// // clang-format off property_sset(exifForViewer, setExifForViewer, Exif, StringSet()) property_sset(exifForDialog, setExifForDialog, Exif, Exif::Info::instance()->standardKeys()) property_ref(iptcCharset, setIptcCharset, QString, Exif, QString()) // clang-format on ///////////////////// //// Exif Import //// ///////////////////// // clang-format off property_copy(updateExifData, setUpdateExifData, bool, ExifImport, true) property_copy(updateImageDate, setUpdateImageDate, bool, ExifImport, false) property_copy(useModDateIfNoExif, setUseModDateIfNoExif, bool, ExifImport, true) property_copy(updateOrientation, setUpdateOrientation, bool, ExifImport, false) property_copy(updateDescription, setUpdateDescription, bool, ExifImport, false) // clang-format on /////////////////////// //// Miscellaneous //// /////////////////////// // clang-format off property_copy(delayLoadingPlugins, setDelayLoadingPlugins, bool, Plug - ins, true) property_ref_(HTMLBaseDir, setHTMLBaseDir, QString, groupForDatabase("HTML Settings"), QString::fromLocal8Bit(qgetenv("HOME")) + STR("/public_html")) property_ref_(HTMLBaseURL, setHTMLBaseURL, QString, groupForDatabase("HTML Settings"), STR("file://") + HTMLBaseDir()) property_ref_(HTMLDestURL, setHTMLDestURL, QString, groupForDatabase("HTML Settings"), STR("file://") + HTMLBaseDir()) property_ref_(HTMLCopyright, setHTMLCopyright, QString, groupForDatabase("HTML Settings"), STR("")) property_ref_(HTMLDate, setHTMLDate, int, groupForDatabase("HTML Settings"), true) property_ref_(HTMLTheme, setHTMLTheme, int, groupForDatabase("HTML Settings"), -1) property_ref_(HTMLKimFile, setHTMLKimFile, int, groupForDatabase("HTML Settings"), true) property_ref_(HTMLInlineMovies, setHTMLInlineMovies, int, groupForDatabase("HTML Settings"), true) property_ref_(HTML5Video, setHTML5Video, int, groupForDatabase("HTML Settings"), true) property_ref_(HTML5VideoGenerate, setHTML5VideoGenerate, int, groupForDatabase("HTML Settings"), true) property_ref_(HTMLThumbSize, setHTMLThumbSize, int, groupForDatabase("HTML Settings"), 128) property_ref_(HTMLNumOfCols, setHTMLNumOfCols, int, groupForDatabase("HTML Settings"), 5) property_ref_(HTMLSizes, setHTMLSizes, QString, groupForDatabase("HTML Settings"), STR("")) property_ref_(HTMLIncludeSelections, setHTMLIncludeSelections, QString, groupForDatabase("HTML Settings"), STR("")) property_ref_(password, setPassword, QString, groupForDatabase("Privacy Settings"), STR("")) // clang-format on QDate SettingsData::fromDate() const { QString date = value("Miscellaneous", "fromDate", STR("")); return date.isEmpty() ? QDate(QDate::currentDate().year(), 1, 1) : QDate::fromString(date, Qt::ISODate); } void SettingsData::setFromDate(const QDate &date) { if (date.isValid()) setValue("Miscellaneous", "fromDate", date.toString(Qt::ISODate)); } QDate SettingsData::toDate() const { QString date = value("Miscellaneous", "toDate", STR("")); return date.isEmpty() ? QDate(QDate::currentDate().year() + 1, 1, 1) : QDate::fromString(date, Qt::ISODate); } void SettingsData::setToDate(const QDate &date) { if (date.isValid()) setValue("Miscellaneous", "toDate", date.toString(Qt::ISODate)); } QString SettingsData::imageDirectory() const { return m_imageDirectory; } QString SettingsData::groupForDatabase(const char *setting) const { return STR("%1 - %2").arg(STR(setting)).arg(imageDirectory()); } DB::ImageSearchInfo SettingsData::currentLock() const { return DB::ImageSearchInfo::loadLock(); } void SettingsData::setCurrentLock(const DB::ImageSearchInfo &info, bool exclude) { info.saveLock(); setValue(groupForDatabase("Privacy Settings"), "exclude", exclude); } bool SettingsData::lockExcludes() const { return value(groupForDatabase("Privacy Settings"), "exclude", false); } getValueFunc_(bool, locked, groupForDatabase("Privacy Settings"), "locked", false) void SettingsData::setLocked(bool lock, bool force) { if (lock == locked() && !force) return; setValue(groupForDatabase("Privacy Settings"), "locked", lock); emit locked(lock, lockExcludes()); } void SettingsData::setWindowGeometry(WindowType win, const QRect &geometry) { setValue("Window Geometry", win, geometry); } QRect SettingsData::windowGeometry(WindowType win) const { return value("Window Geometry", win, QRect(0, 0, 800, 600)); } bool Settings::SettingsData::hasUntaggedCategoryFeatureConfigured() const { return DB::ImageDB::instance()->categoryCollection()->categoryNames().contains(untaggedCategory()) && DB::ImageDB::instance()->categoryCollection()->categoryForName(untaggedCategory())->items().contains(untaggedTag()); } double Settings::SettingsData::getThumbnailAspectRatio() const { double ratio = 1.0; switch (Settings::SettingsData::instance()->thumbnailAspectRatio()) { case Settings::Aspect_16_9: ratio = 9.0 / 16; break; case Settings::Aspect_4_3: ratio = 3.0 / 4; break; case Settings::Aspect_3_2: ratio = 2.0 / 3; break; case Settings::Aspect_9_16: ratio = 16 / 9.0; break; case Settings::Aspect_3_4: ratio = 4 / 3.0; break; case Settings::Aspect_2_3: ratio = 3 / 2.0; break; case Settings::Aspect_1_1: ratio = 1.0; break; } return ratio; } QStringList Settings::SettingsData::EXIFCommentsToStrip() { return m_EXIFCommentsToStrip; } void Settings::SettingsData::setEXIFCommentsToStrip(QStringList EXIFCommentsToStrip) { m_EXIFCommentsToStrip = EXIFCommentsToStrip; } +bool Settings::SettingsData::getOverlapLoadMD5() const +{ + switch (Settings::SettingsData::instance()->loadOptimizationPreset()) { + case Settings::LoadOptimizationSlowNVME: + case Settings::LoadOptimizationFastNVME: + return true; + break; + case Settings::LoadOptimizationManual: + return Settings::SettingsData::instance()->overlapLoadMD5(); + break; + case Settings::LoadOptimizationHardDisk: + case Settings::LoadOptimizationNetwork: + case Settings::LoadOptimizationSataSSD: + default: + return false; + break; + } +} + +int Settings::SettingsData::getPreloadThreadCount() const +{ + switch (Settings::SettingsData::instance()->loadOptimizationPreset()) { + case Settings::LoadOptimizationManual: + return Settings::SettingsData::instance()->preloadThreadCount(); + break; + case Settings::LoadOptimizationSlowNVME: + case Settings::LoadOptimizationFastNVME: + case Settings::LoadOptimizationSataSSD: + return qMax(1, qMin(16, QThread::idealThreadCount())); + break; + case Settings::LoadOptimizationHardDisk: + case Settings::LoadOptimizationNetwork: + default: + return 1; + break; + } +} + +int Settings::SettingsData::getThumbnailPreloadThreadCount() const +{ + switch (Settings::SettingsData::instance()->loadOptimizationPreset()) { + case Settings::LoadOptimizationManual: + return Settings::SettingsData::instance()->thumbnailPreloadThreadCount(); + break; + case Settings::LoadOptimizationSlowNVME: + case Settings::LoadOptimizationFastNVME: + case Settings::LoadOptimizationSataSSD: + return qMax(1, qMin(16, QThread::idealThreadCount() / 2)); + break; + case Settings::LoadOptimizationHardDisk: + case Settings::LoadOptimizationNetwork: + default: + return 1; + break; + } +} + +int Settings::SettingsData::getThumbnailBuilderThreadCount() const +{ + switch (Settings::SettingsData::instance()->loadOptimizationPreset()) { + case Settings::LoadOptimizationManual: + return Settings::SettingsData::instance()->thumbnailBuilderThreadCount(); + break; + case Settings::LoadOptimizationSlowNVME: + case Settings::LoadOptimizationFastNVME: + case Settings::LoadOptimizationSataSSD: + case Settings::LoadOptimizationHardDisk: + case Settings::LoadOptimizationNetwork: + default: + return qMax(1, qMin(16, QThread::idealThreadCount() - 1)); + break; + } +} + // vi:expandtab:tabstop=4 shiftwidth=4: diff --git a/Settings/SettingsData.h b/Settings/SettingsData.h index 60260317..da44c085 100644 --- a/Settings/SettingsData.h +++ b/Settings/SettingsData.h @@ -1,273 +1,289 @@ /* Copyright (C) 2003-2019 The KPhotoAlbum Development Team This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; see the file COPYING. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef SETTINGS_SETTINGSDATA_H #define SETTINGS_SETTINGSDATA_H #include #include #include #include #include #define property(GET_TYPE, GET_FUNC, SET_FUNC, SET_TYPE) \ GET_TYPE GET_FUNC() const; \ void SET_FUNC(const SET_TYPE) #define property_copy(GET_FUNC, SET_FUNC, TYPE) property(TYPE, GET_FUNC, SET_FUNC, TYPE) #define property_ref(GET_FUNC, SET_FUNC, TYPE) property(TYPE, GET_FUNC, SET_FUNC, TYPE &) namespace DB { class CategoryCollection; } namespace Settings { using Utilities::StringSet; enum Position { Bottom, Top, Left, Right, TopLeft, TopRight, BottomLeft, BottomRight }; enum ViewSortType { SortLastUse, SortAlphaTree, SortAlphaFlat }; enum TimeStampTrust { Always, Ask, Never }; enum StandardViewSize { FullSize, NaturalSize, NaturalSizeIfFits }; enum ThumbnailAspectRatio { Aspect_1_1, Aspect_4_3, Aspect_3_2, Aspect_16_9, Aspect_3_4, Aspect_2_3, Aspect_9_16 }; +enum LoadOptimizationPreset { LoadOptimizationHardDisk, + LoadOptimizationNetwork, + LoadOptimizationSataSSD, + LoadOptimizationSlowNVME, + LoadOptimizationFastNVME, + LoadOptimizationManual }; typedef const char *WindowType; extern const WindowType MainWindow, AnnotationDialog; class SettingsData : public QObject { Q_OBJECT public: static SettingsData *instance(); static bool ready(); static void setup(const QString &imageDirectory); ///////////////// //// General //// ///////////////// property_ref(histogramSize, setHistogramSize, QSize); property_copy(useEXIFRotate, setUseEXIFRotate, bool); property_copy(useEXIFComments, setUseEXIFComments, bool); property_copy(stripEXIFComments, setStripEXIFComments, bool); property_copy(commentsToStrip, setCommentsToStrip, QString); property_copy(searchForImagesOnStart, setSearchForImagesOnStart, bool); property_copy(ignoreFileExtension, setIgnoreFileExtension, bool); property_copy(skipSymlinks, setSkipSymlinks, bool); property_copy(skipRawIfOtherMatches, setSkipRawIfOtherMatches, bool); property_copy(useRawThumbnail, setUseRawThumbnail, bool); property_copy(useRawThumbnailSize, setUseRawThumbnailSize, QSize); property_copy(useCompressedIndexXML, setUseCompressedIndexXML, bool); property_copy(compressBackup, setCompressBackup, bool); property_copy(showSplashScreen, setShowSplashScreen, bool); property_copy(showHistogram, setShowHistogram, bool); property_copy(histogramUseLinearScale, setHistogramUseLinearScale, bool); property_copy(autoSave, setAutoSave, int); property_copy(backupCount, setBackupCount, int); property_copy(viewSortType, setViewSortType, ViewSortType); property_copy(matchType, setMatchType, AnnotationDialog::MatchType); property_copy(tTimeStamps, setTTimeStamps, TimeStampTrust); property_copy(excludeDirectories, setExcludeDirectories, QString); #ifdef KPA_ENABLE_REMOTECONTROL property_copy(recentAndroidAddress, setRecentAndroidAddress, QString); property_copy(listenForAndroidDevicesOnStartup, setListenForAndroidDevicesOnStartup, bool); #endif //////////////////////////////// //// File Version Detection //// //////////////////////////////// property_copy(detectModifiedFiles, setDetectModifiedFiles, bool); property_copy(modifiedFileComponent, setModifiedFileComponent, QString); property_copy(originalFileComponent, setOriginalFileComponent, QString); property_copy(moveOriginalContents, setMoveOriginalContents, bool); property_copy(autoStackNewFiles, setAutoStackNewFiles, bool); property_copy(copyFileComponent, setCopyFileComponent, QString); property_copy(copyFileReplacementComponent, setCopyFileReplacementComponent, QString); + property_copy(loadOptimizationPreset, setLoadOptimizationPreset, int); + property_copy(overlapLoadMD5, setOverlapLoadMD5, bool); + property_copy(preloadThreadCount, setPreloadThreadCount, int); + property_copy(thumbnailPreloadThreadCount, setThumbnailPreloadThreadCount, int); + property_copy(thumbnailBuilderThreadCount, setThumbnailBuilderThreadCount, int); bool trustTimeStamps(); //////////////////// //// Thumbnails //// //////////////////// property_copy(displayLabels, setDisplayLabels, bool); property_copy(displayCategories, setDisplayCategories, bool); property_copy(autoShowThumbnailView, setAutoShowThumbnailView, int); property_copy(showNewestThumbnailFirst, setShowNewestFirst, bool); property_copy(thumbnailDisplayGrid, setThumbnailDisplayGrid, bool); property_copy(previewSize, setPreviewSize, int); property_ref(backgroundColor, setBackgroundColor, QString); property_copy(incrementalThumbnails, setIncrementalThumbnails, bool); // Border space around thumbnails. property_copy(thumbnailSpace, setThumbnailSpace, int); property_copy(thumbnailSize, setThumbnailSize, int); property_copy(minimumThumbnailSize, setMinimumThumbnailSize, int); property_copy(maximumThumbnailSize, setMaximumThumbnailSize, int); property_copy(actualThumbnailSize, setActualThumbnailSize, int); property_copy(thumbnailAspectRatio, setThumbnailAspectRatio, ThumbnailAspectRatio); //////////////// //// Viewer //// //////////////// property_ref(viewerSize, setViewerSize, QSize); property_ref(slideShowSize, setSlideShowSize, QSize); property_copy(launchViewerFullScreen, setLaunchViewerFullScreen, bool); property_copy(launchSlideShowFullScreen, setLaunchSlideShowFullScreen, bool); property_copy(showInfoBox, setShowInfoBox, bool); property_copy(showLabel, setShowLabel, bool); property_copy(showDescription, setShowDescription, bool); property_copy(showDate, setShowDate, bool); property_copy(showImageSize, setShowImageSize, bool); property_copy(showRating, setShowRating, bool); property_copy(showTime, setShowTime, bool); property_copy(showFilename, setShowFilename, bool); property_copy(showEXIF, setShowEXIF, bool); property_copy(smoothScale, setSmoothScale, bool); property_copy(slideShowInterval, setSlideShowInterval, int); property_copy(viewerCacheSize, setViewerCacheSize, int); property_copy(infoBoxWidth, setInfoBoxWidth, int); property_copy(infoBoxHeight, setInfoBoxHeight, int); property_copy(infoBoxPosition, setInfoBoxPosition, Position); property_copy(viewerStandardSize, setViewerStandardSize, StandardViewSize); //////////////////// //// Categories //// //////////////////// property_ref(albumCategory, setAlbumCategory, QString); property_ref(untaggedCategory, setUntaggedCategory, QString); property_ref(untaggedTag, setUntaggedTag, QString); bool hasUntaggedCategoryFeatureConfigured() const; property_copy(untaggedImagesTagVisible, setUntaggedImagesTagVisible, bool); ////////////// //// Exif //// ////////////// property_ref(exifForViewer, setExifForViewer, StringSet); property_ref(exifForDialog, setExifForDialog, StringSet); property_ref(iptcCharset, setIptcCharset, QString); ///////////////////// //// Exif Import //// ///////////////////// property_copy(updateExifData, setUpdateExifData, bool); property_copy(updateImageDate, setUpdateImageDate, bool); property_copy(useModDateIfNoExif, setUseModDateIfNoExif, bool); property_copy(updateOrientation, setUpdateOrientation, bool); property_copy(updateDescription, setUpdateDescription, bool); /////////////////////// //// Miscellaneous //// /////////////////////// property_copy(delayLoadingPlugins, setDelayLoadingPlugins, bool); property_ref(password, setPassword, QString); property_ref(HTMLBaseDir, setHTMLBaseDir, QString); property_ref(HTMLBaseURL, setHTMLBaseURL, QString); property_ref(HTMLDestURL, setHTMLDestURL, QString); property_ref(HTMLCopyright, setHTMLCopyright, QString); property_ref(HTMLDate, setHTMLDate, int); property_ref(HTMLTheme, setHTMLTheme, int); property_ref(HTMLKimFile, setHTMLKimFile, int); property_ref(HTMLInlineMovies, setHTMLInlineMovies, int); property_ref(HTML5Video, setHTML5Video, int); property_ref(HTML5VideoGenerate, setHTML5VideoGenerate, int); property_ref(HTMLThumbSize, setHTMLThumbSize, int); property_ref(HTMLNumOfCols, setHTMLNumOfCols, int); property_ref(HTMLSizes, setHTMLSizes, QString); property_ref(HTMLIncludeSelections, setHTMLIncludeSelections, QString); property_ref(fromDate, setFromDate, QDate); property_ref(toDate, setToDate, QDate); QString imageDirectory() const; QString groupForDatabase(const char *setting) const; DB::ImageSearchInfo currentLock() const; void setCurrentLock(const DB::ImageSearchInfo &, bool exclude); bool lockExcludes() const; bool locked() const; void setLocked(bool locked, bool force); void setWindowGeometry(WindowType, const QRect &geometry); QRect windowGeometry(WindowType) const; double getThumbnailAspectRatio() const; QStringList EXIFCommentsToStrip(); void setEXIFCommentsToStrip(QStringList EXIFCommentsToStrip); + bool getOverlapLoadMD5() const; + int getPreloadThreadCount() const; + int getThumbnailPreloadThreadCount() const; + int getThumbnailBuilderThreadCount() const; + signals: void locked(bool lock, bool exclude); void viewSortTypeChanged(Settings::ViewSortType); void matchTypeChanged(AnnotationDialog::MatchType); void histogramSizeChanged(const QSize &); void thumbnailSizeChanged(int); void actualThumbnailSizeChanged(int); void histogramScaleChanged(); private: SettingsData(const QString &imageDirectory); bool m_trustTimeStamps; bool m_hasAskedAboutTimeStamps; QString m_imageDirectory; static SettingsData *s_instance; friend class DB::CategoryCollection; QStringList m_EXIFCommentsToStrip; }; } // end of namespace #undef property #undef property_copy #undef property_ref #endif /* SETTINGS_SETTINGSDATA_H */ // vi:expandtab:tabstop=4 shiftwidth=4: