diff --git a/autotests/kprocesstest_helper.cpp b/autotests/kprocesstest_helper.cpp index f97097c..83dd073 100644 --- a/autotests/kprocesstest_helper.cpp +++ b/autotests/kprocesstest_helper.cpp @@ -1,28 +1,32 @@ /* This file is part of the KDE libraries SPDX-FileCopyrightText: 2007 Oswald Buddenhagen SPDX-License-Identifier: LGPL-2.0-or-later */ #include #include "kprocesstest_helper.h" #include #include int main(int argc, char **argv) { + if (argc < 2) { + printf("Missing parameter"); + return -1; + } KProcess p; p.setShellCommand(QString::fromLatin1("echo " EOUT "; echo " EERR " >&2")); p.setOutputChannelMode(static_cast(atoi(argv[1]))); fputs(POUT, stdout); fflush(stdout); p.execute(); fputs(ROUT, stdout); fputs(p.readAllStandardOutput().constData(), stdout); fputs(RERR, stdout); fputs(p.readAllStandardError().constData(), stdout); return 0; } diff --git a/src/lib/caching/kshareddatacache.cpp b/src/lib/caching/kshareddatacache.cpp index 13d4a14..25c278a 100644 --- a/src/lib/caching/kshareddatacache.cpp +++ b/src/lib/caching/kshareddatacache.cpp @@ -1,1722 +1,1729 @@ /* This file is part of the KDE project. SPDX-FileCopyrightText: 2010, 2012 Michael Pyne SPDX-FileCopyrightText: 2012 Ralf Jung SPDX-License-Identifier: LGPL-2.0-only This library includes "MurmurHash" code from Austin Appleby, which is placed in the public domain. See http://sites.google.com/site/murmurhash/ */ #include "kshareddatacache.h" #include "kshareddatacache_p.h" // Various auxiliary support code #include "kcoreaddons_debug.h" #include "qstandardpaths.h" #include #include #include #include #include #include #include #include #include #include #include #include /// The maximum number of probes to make while searching for a bucket in /// the presence of collisions in the cache index table. static const uint MAX_PROBE_COUNT = 6; /** * A very simple class whose only purpose is to be thrown as an exception from * underlying code to indicate that the shared cache is apparently corrupt. * This must be caught by top-level library code and used to unlink the cache * in this circumstance. * * @internal */ class KSDCCorrupted { public: KSDCCorrupted() { qCritical() << "Error detected in cache, re-generating"; } }; //----------------------------------------------------------------------------- // MurmurHashAligned, by Austin Appleby // (Released to the public domain, or licensed under the MIT license where // software may not be released to the public domain. See // http://sites.google.com/site/murmurhash/) // Same algorithm as MurmurHash, but only does aligned reads - should be safer // on certain platforms. static unsigned int MurmurHashAligned(const void *key, int len, unsigned int seed) { const unsigned int m = 0xc6a4a793; const int r = 16; const unsigned char *data = reinterpret_cast(key); unsigned int h = seed ^ (len * m); int align = reinterpret_cast(data) & 3; if (align && len >= 4) { // Pre-load the temp registers unsigned int t = 0, d = 0; switch (align) { case 1: t |= data[2] << 16; + Q_FALLTHROUGH(); case 2: t |= data[1] << 8; + Q_FALLTHROUGH(); case 3: t |= data[0]; } t <<= (8 * align); data += 4 - align; len -= 4 - align; int sl = 8 * (4 - align); int sr = 8 * align; // Mix while (len >= 4) { d = *reinterpret_cast(data); t = (t >> sr) | (d << sl); h += t; h *= m; h ^= h >> r; t = d; data += 4; len -= 4; } // Handle leftover data in temp registers int pack = len < align ? len : align; d = 0; switch (pack) { case 3: d |= data[2] << 16; + Q_FALLTHROUGH(); case 2: d |= data[1] << 8; + Q_FALLTHROUGH(); case 1: d |= data[0]; + Q_FALLTHROUGH(); case 0: h += (t >> sr) | (d << sl); h *= m; h ^= h >> r; } data += pack; len -= pack; } else { while (len >= 4) { h += *reinterpret_cast(data); h *= m; h ^= h >> r; data += 4; len -= 4; } } //---------- // Handle tail bytes switch (len) { case 3: h += data[2] << 16; + Q_FALLTHROUGH(); case 2: h += data[1] << 8; + Q_FALLTHROUGH(); case 1: h += data[0]; h *= m; h ^= h >> r; }; h *= m; h ^= h >> 10; h *= m; h ^= h >> 17; return h; } /** * This is the hash function used for our data to hopefully make the * hashing used to place the QByteArrays as efficient as possible. */ static quint32 generateHash(const QByteArray &buffer) { // The final constant is the "seed" for MurmurHash. Do *not* change it // without incrementing the cache version. return MurmurHashAligned(buffer.data(), buffer.size(), 0xF0F00F0F); } // Alignment concerns become a big deal when we're dealing with shared memory, // since trying to access a structure sized at, say 8 bytes at an address that // is not evenly divisible by 8 is a crash-inducing error on some // architectures. The compiler would normally take care of this, but with // shared memory the compiler will not necessarily know the alignment expected, // so make sure we account for this ourselves. To do so we need a way to find // out the expected alignment. Enter ALIGNOF... #ifndef ALIGNOF #if defined(Q_CC_GNU) || defined(Q_CC_SUN) #define ALIGNOF(x) (__alignof__ (x)) // GCC provides what we want directly #else #include // offsetof template struct __alignmentHack { char firstEntry; T obj; static const size_t size = offsetof(__alignmentHack, obj); }; #define ALIGNOF(x) (__alignmentHack::size) #endif // Non gcc #endif // ALIGNOF undefined // Returns a pointer properly aligned to handle size alignment. // size should be a power of 2. start is assumed to be the lowest // permissible address, therefore the return value will be >= start. template T *alignTo(const void *start, uint size = ALIGNOF(T)) { quintptr mask = size - 1; // Cast to int-type to handle bit-twiddling quintptr basePointer = reinterpret_cast(start); // If (and only if) we are already aligned, adding mask into basePointer // will not increment any of the bits in ~mask and we get the right answer. basePointer = (basePointer + mask) & ~mask; return reinterpret_cast(basePointer); } /** * Returns a pointer to a const object of type T, assumed to be @p offset * *BYTES* greater than the base address. Note that in order to meet alignment * requirements for T, it is possible that the returned pointer points greater * than @p offset into @p base. */ template const T *offsetAs(const void *const base, qint32 offset) { const char *ptr = reinterpret_cast(base); return alignTo(ptr + offset); } // Same as above, but for non-const objects template T *offsetAs(void *const base, qint32 offset) { char *ptr = reinterpret_cast(base); return alignTo(ptr + offset); } /** * @return the smallest integer greater than or equal to (@p a / @p b). * @param a Numerator, should be ≥ 0. * @param b Denominator, should be > 0. */ static unsigned intCeil(unsigned a, unsigned b) { // The overflow check is unsigned and so is actually defined behavior. if (Q_UNLIKELY(b == 0 || ((a + b) < a))) { throw KSDCCorrupted(); } return (a + b - 1) / b; } /** * @return number of set bits in @p value (see also "Hamming weight") */ static unsigned countSetBits(unsigned value) { // K&R / Wegner's algorithm used. GCC supports __builtin_popcount but we // expect there to always be only 1 bit set so this should be perhaps a bit // faster 99.9% of the time. unsigned count = 0; for (count = 0; value != 0; count++) { value &= (value - 1); // Clears least-significant set bit. } return count; } typedef qint32 pageID; // ========================================================================= // Description of the cache: // // The shared memory cache is designed to be handled as two separate objects, // all contained in the same global memory segment. First off, there is the // basic header data, consisting of the global header followed by the // accounting data necessary to hold items (described in more detail // momentarily). Following the accounting data is the start of the "page table" // (essentially just as you'd see it in an Operating Systems text). // // The page table contains shared memory split into fixed-size pages, with a // configurable page size. In the event that the data is too large to fit into // a single logical page, it will need to occupy consecutive pages of memory. // // The accounting data that was referenced earlier is split into two: // // 1. index table, containing a fixed-size list of possible cache entries. // Each index entry is of type IndexTableEntry (below), and holds the various // accounting data and a pointer to the first page. // // 2. page table, which is used to speed up the process of searching for // free pages of memory. There is one entry for every page in the page table, // and it contains the index of the one entry in the index table actually // holding the page (or <0 if the page is free). // // The entire segment looks like so: // ?════════?═════════════?════════════?═══════?═══════?═══════?═══════?═══? // ? Header │ Index Table │ Page Table ? Pages │ │ │ │...? // ?════════?═════════════?════════════?═══════?═══════?═══════?═══════?═══? // ========================================================================= // All elements of this struct must be "plain old data" (POD) types since it // will be in shared memory. In addition, no pointers! To point to something // you must use relative offsets since the pointer start addresses will be // different in each process. struct IndexTableEntry { uint fileNameHash; uint totalItemSize; // in bytes mutable uint useCount; time_t addTime; mutable time_t lastUsedTime; pageID firstPage; }; // Page table entry struct PageTableEntry { // int so we can use values <0 for unassigned pages. qint32 index; }; // Each individual page contains the cached data. The first page starts off with // the utf8-encoded key, a null '\0', and then the data follows immediately // from the next byte, possibly crossing consecutive page boundaries to hold // all of the data. // There is, however, no specific struct for a page, it is simply a location in // memory. // This is effectively the layout of the shared memory segment. The variables // contained within form the header, data contained afterwards is pointed to // by using special accessor functions. struct SharedMemory { /** * Note to downstream packagers: This version flag is intended to be * machine-specific. The KDE-provided source code will not set the lower * two bits to allow for distribution-specific needs, with the exception * of version 1 which was already defined in KDE Platform 4.5. * e.g. the next version bump will be from 4 to 8, then 12, etc. */ enum { PIXMAP_CACHE_VERSION = 12, MINIMUM_CACHE_SIZE = 4096 }; // Note to those who follow me. You should not, under any circumstances, ever // re-arrange the following two fields, even if you change the version number // for later revisions of this code. QAtomicInt ready; ///< DO NOT INITIALIZE quint8 version; // See kshareddatacache_p.h SharedLock shmLock; uint cacheSize; uint cacheAvail; QAtomicInt evictionPolicy; // pageSize and cacheSize determine the number of pages. The number of // pages determine the page table size and (indirectly) the index table // size. QAtomicInt pageSize; // This variable is added to reserve space for later cache timestamping // support. The idea is this variable will be updated when the cache is // written to, to allow clients to detect a changed cache quickly. QAtomicInt cacheTimestamp; /** * Converts the given average item size into an appropriate page size. */ static unsigned equivalentPageSize(unsigned itemSize) { if (itemSize == 0) { return 4096; // Default average item size. } int log2OfSize = 0; while ((itemSize >>= 1) != 0) { log2OfSize++; } // Bound page size between 512 bytes and 256 KiB. // If this is adjusted, also alter validSizeMask in cachePageSize log2OfSize = qBound(9, log2OfSize, 18); return (1 << log2OfSize); } // Returns pageSize in unsigned format. unsigned cachePageSize() const { #if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) unsigned _pageSize = static_cast(pageSize.load()); #else unsigned _pageSize = static_cast(pageSize.loadRelaxed()); #endif // bits 9-18 may be set. static const unsigned validSizeMask = 0x7FE00u; // Check for page sizes that are not a power-of-2, or are too low/high. if (Q_UNLIKELY(countSetBits(_pageSize) != 1 || (_pageSize & ~validSizeMask))) { throw KSDCCorrupted(); } return _pageSize; } /** * This is effectively the class ctor. But since we're in shared memory, * there's a few rules: * * 1. To allow for some form of locking in the initial-setup case, we * use an atomic int, which will be initialized to 0 by mmap(). Then to * take the lock we atomically increment the 0 to 1. If we end up calling * the QAtomicInt constructor we can mess that up, so we can't use a * constructor for this class either. * 2. Any member variable you add takes up space in shared memory as well, * so make sure you need it. */ bool performInitialSetup(uint _cacheSize, uint _pageSize) { if (_cacheSize < MINIMUM_CACHE_SIZE) { qCritical() << "Internal error: Attempted to create a cache sized < " << MINIMUM_CACHE_SIZE; return false; } if (_pageSize == 0) { qCritical() << "Internal error: Attempted to create a cache with 0-sized pages."; return false; } shmLock.type = findBestSharedLock(); if (shmLock.type == LOCKTYPE_INVALID) { qCritical() << "Unable to find an appropriate lock to guard the shared cache. " << "This *should* be essentially impossible. :("; return false; } bool isProcessShared = false; QSharedPointer tempLock(createLockFromId(shmLock.type, shmLock)); if (!tempLock->initialize(isProcessShared)) { qCritical() << "Unable to initialize the lock for the cache!"; return false; } if (!isProcessShared) { qCWarning(KCOREADDONS_DEBUG) << "Cache initialized, but does not support being" << "shared across processes."; } // These must be updated to make some of our auxiliary functions // work right since their values will be based on the cache size. cacheSize = _cacheSize; pageSize = _pageSize; version = PIXMAP_CACHE_VERSION; cacheTimestamp = static_cast(::time(nullptr)); clearInternalTables(); // Unlock the mini-lock, and introduce a total memory barrier to make // sure all changes have propagated even without a mutex. ready.ref(); return true; } void clearInternalTables() { // Assumes we're already locked somehow. cacheAvail = pageTableSize(); // Setup page tables to point nowhere PageTableEntry *table = pageTable(); for (uint i = 0; i < pageTableSize(); ++i) { table[i].index = -1; } // Setup index tables to be accurate. IndexTableEntry *indices = indexTable(); for (uint i = 0; i < indexTableSize(); ++i) { indices[i].firstPage = -1; indices[i].useCount = 0; indices[i].fileNameHash = 0; indices[i].totalItemSize = 0; indices[i].addTime = 0; indices[i].lastUsedTime = 0; } } const IndexTableEntry *indexTable() const { // Index Table goes immediately after this struct, at the first byte // where alignment constraints are met (accounted for by offsetAs). return offsetAs(this, sizeof(*this)); } const PageTableEntry *pageTable() const { const IndexTableEntry *base = indexTable(); base += indexTableSize(); // Let's call wherever we end up the start of the page table... return alignTo(base); } const void *cachePages() const { const PageTableEntry *tableStart = pageTable(); tableStart += pageTableSize(); // Let's call wherever we end up the start of the data... return alignTo(tableStart, cachePageSize()); } const void *page(pageID at) const { if (static_cast(at) >= pageTableSize()) { return nullptr; } // We must manually calculate this one since pageSize varies. const char *pageStart = reinterpret_cast(cachePages()); pageStart += (at * cachePageSize()); return reinterpret_cast(pageStart); } // The following are non-const versions of some of the methods defined // above. They use const_cast<> because I feel that is better than // duplicating the code. I suppose template member functions (?) // may work, may investigate later. IndexTableEntry *indexTable() { const SharedMemory *that = const_cast(this); return const_cast(that->indexTable()); } PageTableEntry *pageTable() { const SharedMemory *that = const_cast(this); return const_cast(that->pageTable()); } void *cachePages() { const SharedMemory *that = const_cast(this); return const_cast(that->cachePages()); } void *page(pageID at) { const SharedMemory *that = const_cast(this); return const_cast(that->page(at)); } uint pageTableSize() const { return cacheSize / cachePageSize(); } uint indexTableSize() const { // Assume 2 pages on average are needed -> the number of entries // would be half of the number of pages. return pageTableSize() / 2; } /** * @return the index of the first page, for the set of contiguous * pages that can hold @p pagesNeeded PAGES. */ pageID findEmptyPages(uint pagesNeeded) const { if (Q_UNLIKELY(pagesNeeded > pageTableSize())) { return pageTableSize(); } // Loop through the page table, find the first empty page, and just // makes sure that there are enough free pages. const PageTableEntry *table = pageTable(); uint contiguousPagesFound = 0; pageID base = 0; for (pageID i = 0; i < static_cast(pageTableSize()); ++i) { if (table[i].index < 0) { if (contiguousPagesFound == 0) { base = i; } contiguousPagesFound++; } else { contiguousPagesFound = 0; } if (contiguousPagesFound == pagesNeeded) { return base; } } return pageTableSize(); } // left < right? static bool lruCompare(const IndexTableEntry &l, const IndexTableEntry &r) { // Ensure invalid entries migrate to the end if (l.firstPage < 0 && r.firstPage >= 0) { return false; } if (l.firstPage >= 0 && r.firstPage < 0) { return true; } // Most recently used will have the highest absolute time => // least recently used (lowest) should go first => use left < right return l.lastUsedTime < r.lastUsedTime; } // left < right? static bool seldomUsedCompare(const IndexTableEntry &l, const IndexTableEntry &r) { // Ensure invalid entries migrate to the end if (l.firstPage < 0 && r.firstPage >= 0) { return false; } if (l.firstPage >= 0 && r.firstPage < 0) { return true; } // Put lowest use count at start by using left < right return l.useCount < r.useCount; } // left < right? static bool ageCompare(const IndexTableEntry &l, const IndexTableEntry &r) { // Ensure invalid entries migrate to the end if (l.firstPage < 0 && r.firstPage >= 0) { return false; } if (l.firstPage >= 0 && r.firstPage < 0) { return true; } // Oldest entries die first -- they have the lowest absolute add time, // so just like the others use left < right return l.addTime < r.addTime; } void defragment() { if (cacheAvail * cachePageSize() == cacheSize) { return; // That was easy } qCDebug(KCOREADDONS_DEBUG) << "Defragmenting the shared cache"; // Just do a linear scan, and anytime there is free space, swap it // with the pages to its right. In order to meet the precondition // we need to skip any used pages first. pageID currentPage = 0; pageID idLimit = static_cast(pageTableSize()); PageTableEntry *pages = pageTable(); if (Q_UNLIKELY(!pages || idLimit <= 0)) { throw KSDCCorrupted(); } // Skip used pages while (currentPage < idLimit && pages[currentPage].index >= 0) { ++currentPage; } pageID freeSpot = currentPage; // Main loop, starting from a free page, skip to the used pages and // move them back. while (currentPage < idLimit) { // Find the next used page while (currentPage < idLimit && pages[currentPage].index < 0) { ++currentPage; } if (currentPage >= idLimit) { break; } // Found an entry, move it. qint32 affectedIndex = pages[currentPage].index; if (Q_UNLIKELY(affectedIndex < 0 || affectedIndex >= idLimit || indexTable()[affectedIndex].firstPage != currentPage)) { throw KSDCCorrupted(); } indexTable()[affectedIndex].firstPage = freeSpot; // Moving one page at a time guarantees we can use memcpy safely // (in other words, the source and destination will not overlap). while (currentPage < idLimit && pages[currentPage].index >= 0) { const void *const sourcePage = page(currentPage); void *const destinationPage = page(freeSpot); // We always move used pages into previously-found empty spots, // so check ordering as well for logic errors. if (Q_UNLIKELY(!sourcePage || !destinationPage || sourcePage < destinationPage)) { throw KSDCCorrupted(); } ::memcpy(destinationPage, sourcePage, cachePageSize()); pages[freeSpot].index = affectedIndex; pages[currentPage].index = -1; ++currentPage; ++freeSpot; // If we've just moved the very last page and it happened to // be at the very end of the cache then we're done. if (currentPage >= idLimit) { break; } // We're moving consecutive used pages whether they belong to // our affected entry or not, so detect if we've started moving // the data for a different entry and adjust if necessary. if (affectedIndex != pages[currentPage].index) { indexTable()[pages[currentPage].index].firstPage = freeSpot; } affectedIndex = pages[currentPage].index; } // At this point currentPage is on a page that is unused, and the // cycle repeats. However, currentPage is not the first unused // page, freeSpot is, so leave it alone. } } /** * Finds the index entry for a given key. * @param key UTF-8 encoded key to search for. * @return The index of the entry in the cache named by @p key. Returns * <0 if no such entry is present. */ qint32 findNamedEntry(const QByteArray &key) const { uint keyHash = generateHash(key); uint position = keyHash % indexTableSize(); uint probeNumber = 1; // See insert() for description // Imagine 3 entries A, B, C in this logical probing chain. If B is // later removed then we can't find C either. So, we must keep // searching for probeNumber number of tries (or we find the item, // obviously). while (indexTable()[position].fileNameHash != keyHash && probeNumber < MAX_PROBE_COUNT) { position = (keyHash + (probeNumber + probeNumber * probeNumber) / 2) % indexTableSize(); probeNumber++; } if (indexTable()[position].fileNameHash == keyHash) { pageID firstPage = indexTable()[position].firstPage; if (firstPage < 0 || static_cast(firstPage) >= pageTableSize()) { return -1; } const void *resultPage = page(firstPage); if (Q_UNLIKELY(!resultPage)) { throw KSDCCorrupted(); } const char *utf8FileName = reinterpret_cast(resultPage); if (qstrncmp(utf8FileName, key.constData(), cachePageSize()) == 0) { return position; } } return -1; // Not found, or a different one found. } // Function to use with QSharedPointer in removeUsedPages below... static void deleteTable(IndexTableEntry *table) { delete [] table; } /** * Removes the requested number of pages. * * @param numberNeeded the number of pages required to fulfill a current request. * This number should be <0 and <= the number of pages in the cache. * @return The identifier of the beginning of a consecutive block of pages able * to fill the request. Returns a value >= pageTableSize() if no such * request can be filled. * @internal */ uint removeUsedPages(uint numberNeeded) { if (numberNeeded == 0) { qCritical() << "Internal error: Asked to remove exactly 0 pages for some reason."; throw KSDCCorrupted(); } if (numberNeeded > pageTableSize()) { qCritical() << "Internal error: Requested more space than exists in the cache."; qCritical() << numberNeeded << "requested, " << pageTableSize() << "is the total possible."; throw KSDCCorrupted(); } // If the cache free space is large enough we will defragment first // instead since it's likely we're highly fragmented. // Otherwise, we will (eventually) simply remove entries per the // eviction order set for the cache until there is enough room // available to hold the number of pages we need. qCDebug(KCOREADDONS_DEBUG) << "Removing old entries to free up" << numberNeeded << "pages," << cacheAvail << "are already theoretically available."; if (cacheAvail > 3 * numberNeeded) { defragment(); uint result = findEmptyPages(numberNeeded); if (result < pageTableSize()) { return result; } else { qCritical() << "Just defragmented a locked cache, but still there" << "isn't enough room for the current request."; } } // At this point we know we'll have to free some space up, so sort our // list of entries by whatever the current criteria are and start // killing expired entries. QSharedPointer tablePtr(new IndexTableEntry[indexTableSize()], deleteTable); if (!tablePtr) { qCritical() << "Unable to allocate temporary memory for sorting the cache!"; clearInternalTables(); throw KSDCCorrupted(); } // We use tablePtr to ensure the data is destroyed, but do the access // via a helper pointer to allow for array ops. IndexTableEntry *table = tablePtr.data(); ::memcpy(table, indexTable(), sizeof(IndexTableEntry) * indexTableSize()); // Our entry ID is simply its index into the // index table, which qSort will rearrange all willy-nilly, so first // we'll save the *real* entry ID into firstPage (which is useless in // our copy of the index table). On the other hand if the entry is not // used then we note that with -1. for (uint i = 0; i < indexTableSize(); ++i) { table[i].firstPage = table[i].useCount > 0 ? static_cast(i) : -1; } // Declare the comparison function that we'll use to pass to qSort, // based on our cache eviction policy. bool (*compareFunction)(const IndexTableEntry &, const IndexTableEntry &); #if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) switch (evictionPolicy.load()) { #else switch (evictionPolicy.loadRelaxed()) { #endif case KSharedDataCache::EvictLeastOftenUsed: case KSharedDataCache::NoEvictionPreference: default: compareFunction = seldomUsedCompare; break; case KSharedDataCache::EvictLeastRecentlyUsed: compareFunction = lruCompare; break; case KSharedDataCache::EvictOldest: compareFunction = ageCompare; break; } std::sort(table, table + indexTableSize(), compareFunction); // Least recently used entries will be in the front. // Start killing until we have room. // Note on removeEntry: It expects an index into the index table, // but our sorted list is all jumbled. But we stored the real index // in the firstPage member. // Remove entries until we've removed at least the required number // of pages. uint i = 0; while (i < indexTableSize() && numberNeeded > cacheAvail) { int curIndex = table[i++].firstPage; // Really an index, not a page // Removed everything, still no luck (or curIndex is set but too high). if (curIndex < 0 || static_cast(curIndex) >= indexTableSize()) { qCritical() << "Trying to remove index" << curIndex << "out-of-bounds for index table of size" << indexTableSize(); throw KSDCCorrupted(); } qCDebug(KCOREADDONS_DEBUG) << "Removing entry of" << indexTable()[curIndex].totalItemSize << "size"; removeEntry(curIndex); } // At this point let's see if we have freed up enough data by // defragmenting first and seeing if we can find that free space. defragment(); pageID result = pageTableSize(); while (i < indexTableSize() && (static_cast(result = findEmptyPages(numberNeeded))) >= pageTableSize()) { int curIndex = table[i++].firstPage; if (curIndex < 0) { // One last shot. defragment(); return findEmptyPages(numberNeeded); } if (Q_UNLIKELY(static_cast(curIndex) >= indexTableSize())) { throw KSDCCorrupted(); } removeEntry(curIndex); } // Whew. return result; } // Returns the total size required for a given cache size. static uint totalSize(uint cacheSize, uint effectivePageSize) { uint numberPages = intCeil(cacheSize, effectivePageSize); uint indexTableSize = numberPages / 2; // Knowing the number of pages, we can determine what addresses we'd be // using (properly aligned), and from there determine how much memory // we'd use. IndexTableEntry *indexTableStart = offsetAs(static_cast(nullptr), sizeof(SharedMemory)); indexTableStart += indexTableSize; PageTableEntry *pageTableStart = reinterpret_cast(indexTableStart); pageTableStart = alignTo(pageTableStart); pageTableStart += numberPages; // The weird part, we must manually adjust the pointer based on the page size. char *cacheStart = reinterpret_cast(pageTableStart); cacheStart += (numberPages * effectivePageSize); // ALIGNOF gives pointer alignment cacheStart = alignTo(cacheStart, ALIGNOF(void *)); // We've traversed the header, index, page table, and cache. // Wherever we're at now is the size of the enchilada. return static_cast(reinterpret_cast(cacheStart)); } uint fileNameHash(const QByteArray &utf8FileName) const { return generateHash(utf8FileName) % indexTableSize(); } void clear() { clearInternalTables(); } void removeEntry(uint index); }; // The per-instance private data, such as map size, whether // attached or not, pointer to shared memory, etc. class Q_DECL_HIDDEN KSharedDataCache::Private { public: Private(const QString &name, unsigned defaultCacheSize, unsigned expectedItemSize ) : m_cacheName(name) , shm(nullptr) , m_lock() , m_mapSize(0) , m_defaultCacheSize(defaultCacheSize) , m_expectedItemSize(expectedItemSize) , m_expectedType(LOCKTYPE_INVALID) { mapSharedMemory(); } // Put the cache in a condition to be able to call mapSharedMemory() by // completely detaching from shared memory (such as to respond to an // unrecoverable error). // m_mapSize must already be set to the amount of memory mapped to shm. void detachFromSharedMemory() { // The lock holds a reference into shared memory, so this must be // cleared before shm is removed. m_lock.clear(); if (shm && 0 != ::munmap(shm, m_mapSize)) { qCritical() << "Unable to unmap shared memory segment" << static_cast(shm) << ":" << ::strerror(errno); } shm = nullptr; m_mapSize = 0; } // This function does a lot of the important work, attempting to connect to shared // memory, a private anonymous mapping if that fails, and failing that, nothing (but // the cache remains "valid", we just don't actually do anything). void mapSharedMemory() { // 0-sized caches are fairly useless. unsigned cacheSize = qMax(m_defaultCacheSize, uint(SharedMemory::MINIMUM_CACHE_SIZE)); unsigned pageSize = SharedMemory::equivalentPageSize(m_expectedItemSize); // Ensure that the cache is sized such that there is a minimum number of // pages available. (i.e. a cache consisting of only 1 page is fairly // useless and probably crash-prone). cacheSize = qMax(pageSize * 256, cacheSize); // The m_cacheName is used to find the file to store the cache in. const QString cacheDir = QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation); QString cacheName = cacheDir + QLatin1String("/") + m_cacheName + QLatin1String(".kcache"); QFile file(cacheName); QFileInfo fileInfo(file); if (!QDir().mkpath(fileInfo.absolutePath())) { return; } // The basic idea is to open the file that we want to map into shared // memory, and then actually establish the mapping. Once we have mapped the // file into shared memory we can close the file handle, the mapping will // still be maintained (unless the file is resized to be shorter than // expected, which we don't handle yet :-( ) // size accounts for the overhead over the desired cacheSize uint size = SharedMemory::totalSize(cacheSize, pageSize); void *mapAddress = MAP_FAILED; if (size < cacheSize) { qCritical() << "Asked for a cache size less than requested size somehow -- Logic Error :("; return; } // We establish the shared memory mapping here, only if we will have appropriate // mutex support (systemSupportsProcessSharing), then we: // Open the file and resize to some sane value if the file is too small. if (file.open(QIODevice::ReadWrite) && (file.size() >= size || (file.resize(size) && ensureFileAllocated(file.handle(), size)))) { // Use mmap directly instead of QFile::map since the QFile (and its // shared mapping) will disappear unless we hang onto the QFile for no // reason (see the note below, we don't care about the file per se...) mapAddress = QT_MMAP(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, file.handle(), 0); // So... it is possible that someone else has mapped this cache already // with a larger size. If that's the case we need to at least match // the size to be able to access every entry, so fixup the mapping. if (mapAddress != MAP_FAILED) { SharedMemory *mapped = reinterpret_cast(mapAddress); // First make sure that the version of the cache on disk is // valid. We also need to check that version != 0 to // disambiguate against an uninitialized cache. if (mapped->version != SharedMemory::PIXMAP_CACHE_VERSION && mapped->version > 0) { qCWarning(KCOREADDONS_DEBUG) << "Deleting wrong version of cache" << cacheName; // CAUTION: Potentially recursive since the recovery // involves calling this function again. m_mapSize = size; shm = mapped; recoverCorruptedCache(); return; } else if (mapped->cacheSize > cacheSize) { // This order is very important. We must save the cache size // before we remove the mapping, but unmap before overwriting // the previous mapping size... cacheSize = mapped->cacheSize; unsigned actualPageSize = mapped->cachePageSize(); ::munmap(mapAddress, size); size = SharedMemory::totalSize(cacheSize, actualPageSize); mapAddress = QT_MMAP(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, file.handle(), 0); } } } // We could be here without the mapping established if: // 1) Process-shared synchronization is not supported, either at compile or run time, // 2) Unable to open the required file. // 3) Unable to resize the file to be large enough. // 4) Establishing the mapping failed. // 5) The mapping succeeded, but the size was wrong and we were unable to map when // we tried again. // 6) The incorrect version of the cache was detected. // 7) The file could be created, but posix_fallocate failed to commit it fully to disk. // In any of these cases, attempt to fallback to the // better-supported anonymous private page style of mmap. This memory won't // be shared, but our code will still work the same. // NOTE: We never use the on-disk representation independently of the // shared memory. If we don't get shared memory the disk info is ignored, // if we do get shared memory we never look at disk again. if (mapAddress == MAP_FAILED) { qCWarning(KCOREADDONS_DEBUG) << "Failed to establish shared memory mapping, will fallback" << "to private memory -- memory usage will increase"; mapAddress = QT_MMAP(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); } // Well now we're really hosed. We can still work, but we can't even cache // data. if (mapAddress == MAP_FAILED) { qCritical() << "Unable to allocate shared memory segment for shared data cache" << cacheName << "of size" << cacheSize; return; } m_mapSize = size; // We never actually construct shm, but we assign it the same address as the // shared memory we just mapped, so effectively shm is now a SharedMemory that // happens to be located at mapAddress. shm = reinterpret_cast(mapAddress); // If we were first to create this memory map, all data will be 0. // Therefore if ready == 0 we're not initialized. A fully initialized // header will have ready == 2. Why? // Because 0 means "safe to initialize" // 1 means "in progress of initing" // 2 means "ready" uint usecSleepTime = 8; // Start by sleeping for 8 microseconds #if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) while (shm->ready.load() != 2) { #else while (shm->ready.loadRelaxed() != 2) { #endif if (Q_UNLIKELY(usecSleepTime >= (1 << 21))) { // Didn't acquire within ~8 seconds? Assume an issue exists qCritical() << "Unable to acquire shared lock, is the cache corrupt?"; file.remove(); // Unlink the cache in case it's corrupt. detachFromSharedMemory(); return; // Fallback to QCache (later) } if (shm->ready.testAndSetAcquire(0, 1)) { if (!shm->performInitialSetup(cacheSize, pageSize)) { qCritical() << "Unable to perform initial setup, this system probably " "does not really support process-shared pthreads or " "semaphores, even though it claims otherwise."; file.remove(); detachFromSharedMemory(); return; } } else { usleep(usecSleepTime); // spin // Exponential fallback as in Ethernet and similar collision resolution methods usecSleepTime *= 2; } } m_expectedType = shm->shmLock.type; m_lock = QSharedPointer(createLockFromId(m_expectedType, shm->shmLock)); bool isProcessSharingSupported = false; if (!m_lock->initialize(isProcessSharingSupported)) { qCritical() << "Unable to setup shared cache lock, although it worked when created."; detachFromSharedMemory(); } } // Called whenever the cache is apparently corrupt (for instance, a timeout trying to // lock the cache). In this situation it is safer just to destroy it all and try again. void recoverCorruptedCache() { KSharedDataCache::deleteCache(m_cacheName); detachFromSharedMemory(); // Do this even if we weren't previously cached -- it might work now. mapSharedMemory(); } // This should be called for any memory access to shared memory. This // function will verify that the bytes [base, base+accessLength) are // actually mapped to d->shm. The cache itself may have incorrect cache // page sizes, incorrect cache size, etc. so this function should be called // despite the cache data indicating it should be safe. // // If the access is /not/ safe then a KSDCCorrupted exception will be // thrown, so be ready to catch that. void verifyProposedMemoryAccess(const void *base, unsigned accessLength) const { quintptr startOfAccess = reinterpret_cast(base); quintptr startOfShm = reinterpret_cast(shm); if (Q_UNLIKELY(startOfAccess < startOfShm)) { throw KSDCCorrupted(); } quintptr endOfShm = startOfShm + m_mapSize; quintptr endOfAccess = startOfAccess + accessLength; // Check for unsigned integer wraparound, and then // bounds access if (Q_UNLIKELY((endOfShm < startOfShm) || (endOfAccess < startOfAccess) || (endOfAccess > endOfShm))) { throw KSDCCorrupted(); } } bool lock() const { if (Q_LIKELY(shm && shm->shmLock.type == m_expectedType)) { return m_lock->lock(); } // No shm or wrong type --> corrupt! throw KSDCCorrupted(); } void unlock() const { m_lock->unlock(); } class CacheLocker { mutable Private *d; bool cautiousLock() { int lockCount = 0; // Locking can fail due to a timeout. If it happens too often even though // we're taking corrective action assume there's some disastrous problem // and give up. while (!d->lock() && !isLockedCacheSafe()) { d->recoverCorruptedCache(); if (!d->shm) { qCWarning(KCOREADDONS_DEBUG) << "Lost the connection to shared memory for cache" << d->m_cacheName; return false; } if (lockCount++ > 4) { qCritical() << "There is a very serious problem with the KDE data cache" << d->m_cacheName << "giving up trying to access cache."; d->detachFromSharedMemory(); return false; } } return true; } // Runs a quick battery of tests on an already-locked cache and returns // false as soon as a sanity check fails. The cache remains locked in this // situation. bool isLockedCacheSafe() const { // Note that cachePageSize() itself runs a check that can throw. uint testSize = SharedMemory::totalSize(d->shm->cacheSize, d->shm->cachePageSize()); if (Q_UNLIKELY(d->m_mapSize != testSize)) { return false; } if (Q_UNLIKELY(d->shm->version != SharedMemory::PIXMAP_CACHE_VERSION)) { return false; } #if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) switch (d->shm->evictionPolicy.load()) { #else switch (d->shm->evictionPolicy.loadRelaxed()) { #endif case NoEvictionPreference: // fallthrough case EvictLeastRecentlyUsed: // fallthrough case EvictLeastOftenUsed: // fallthrough case EvictOldest: break; default: return false; } return true; } public: CacheLocker(const Private *_d) : d(const_cast(_d)) { if (Q_UNLIKELY(!d || !d->shm || !cautiousLock())) { d = nullptr; } } ~CacheLocker() { if (d && d->shm) { d->unlock(); } } CacheLocker(const CacheLocker &) = delete; CacheLocker &operator=(const CacheLocker &) = delete; bool failed() const { return !d || d->shm == nullptr; } }; QString m_cacheName; SharedMemory *shm; QSharedPointer m_lock; uint m_mapSize; uint m_defaultCacheSize; uint m_expectedItemSize; SharedLockId m_expectedType; }; // Must be called while the lock is already held! void SharedMemory::removeEntry(uint index) { if (index >= indexTableSize() || cacheAvail > pageTableSize()) { throw KSDCCorrupted(); } PageTableEntry *pageTableEntries = pageTable(); IndexTableEntry *entriesIndex = indexTable(); // Update page table first pageID firstPage = entriesIndex[index].firstPage; if (firstPage < 0 || static_cast(firstPage) >= pageTableSize()) { qCDebug(KCOREADDONS_DEBUG) << "Trying to remove an entry which is already invalid. This " << "cache is likely corrupt."; throw KSDCCorrupted(); } if (index != static_cast(pageTableEntries[firstPage].index)) { qCritical() << "Removing entry" << index << "but the matching data" << "doesn't link back -- cache is corrupt, clearing."; throw KSDCCorrupted(); } uint entriesToRemove = intCeil(entriesIndex[index].totalItemSize, cachePageSize()); uint savedCacheSize = cacheAvail; for (uint i = firstPage; i < pageTableSize() && static_cast(pageTableEntries[i].index) == index; ++i) { pageTableEntries[i].index = -1; cacheAvail++; } if ((cacheAvail - savedCacheSize) != entriesToRemove) { qCritical() << "We somehow did not remove" << entriesToRemove << "when removing entry" << index << ", instead we removed" << (cacheAvail - savedCacheSize); throw KSDCCorrupted(); } // For debugging #ifdef NDEBUG void *const startOfData = page(firstPage); if (startOfData) { QByteArray str((const char *) startOfData); str.prepend(" REMOVED: "); str.prepend(QByteArray::number(index)); str.prepend("ENTRY "); ::memcpy(startOfData, str.constData(), str.size() + 1); } #endif // Update the index entriesIndex[index].fileNameHash = 0; entriesIndex[index].totalItemSize = 0; entriesIndex[index].useCount = 0; entriesIndex[index].lastUsedTime = 0; entriesIndex[index].addTime = 0; entriesIndex[index].firstPage = -1; } KSharedDataCache::KSharedDataCache(const QString &cacheName, unsigned defaultCacheSize, unsigned expectedItemSize) : d(nullptr) { try { d = new Private(cacheName, defaultCacheSize, expectedItemSize); } catch (KSDCCorrupted) { KSharedDataCache::deleteCache(cacheName); // Try only once more try { d = new Private(cacheName, defaultCacheSize, expectedItemSize); } catch (KSDCCorrupted) { qCritical() << "Even a brand-new cache starts off corrupted, something is" << "seriously wrong. :-("; d = nullptr; // Just in case } } } KSharedDataCache::~KSharedDataCache() { // Note that there is no other actions required to separate from the // shared memory segment, simply unmapping is enough. This makes things // *much* easier so I'd recommend maintaining this ideal. if (!d) { return; } if (d->shm) { #ifdef KSDC_MSYNC_SUPPORTED ::msync(d->shm, d->m_mapSize, MS_INVALIDATE | MS_ASYNC); #endif ::munmap(d->shm, d->m_mapSize); } // Do not delete d->shm, it was never constructed, it's just an alias. d->shm = nullptr; delete d; } bool KSharedDataCache::insert(const QString &key, const QByteArray &data) { try { Private::CacheLocker lock(d); if (lock.failed()) { return false; } QByteArray encodedKey = key.toUtf8(); uint keyHash = generateHash(encodedKey); uint position = keyHash % d->shm->indexTableSize(); // See if we're overwriting an existing entry. IndexTableEntry *indices = d->shm->indexTable(); // In order to avoid the issue of a very long-lived cache having items // with a use count of 1 near-permanently, we attempt to artifically // reduce the use count of long-lived items when there is high load on // the cache. We do this randomly, with a weighting that makes the event // impossible if load < 0.5, and guaranteed if load >= 0.96. const static double startCullPoint = 0.5l; const static double mustCullPoint = 0.96l; // cacheAvail is in pages, cacheSize is in bytes. double loadFactor = 1.0 - (1.0l * d->shm->cacheAvail * d->shm->cachePageSize() / d->shm->cacheSize); bool cullCollisions = false; if (Q_UNLIKELY(loadFactor >= mustCullPoint)) { cullCollisions = true; } else if (loadFactor > startCullPoint) { const int tripWireValue = RAND_MAX * (loadFactor - startCullPoint) / (mustCullPoint - startCullPoint); if (KRandom::random() >= tripWireValue) { cullCollisions = true; } } // In case of collisions in the index table (i.e. identical positions), use // quadratic chaining to attempt to find an empty slot. The equation we use // is: // position = (hash + (i + i*i) / 2) % size, where i is the probe number. uint probeNumber = 1; while (indices[position].useCount > 0 && probeNumber < MAX_PROBE_COUNT) { // If we actually stumbled upon an old version of the key we are // overwriting, then use that position, do not skip over it. if (Q_UNLIKELY(indices[position].fileNameHash == keyHash)) { break; } // If we are "culling" old entries, see if this one is old and if so // reduce its use count. If it reduces to zero then eliminate it and // use its old spot. if (cullCollisions && (::time(nullptr) - indices[position].lastUsedTime) > 60) { indices[position].useCount >>= 1; if (indices[position].useCount == 0) { qCDebug(KCOREADDONS_DEBUG) << "Overwriting existing old cached entry due to collision."; d->shm->removeEntry(position); // Remove it first break; } } position = (keyHash + (probeNumber + probeNumber * probeNumber) / 2) % d->shm->indexTableSize(); probeNumber++; } if (indices[position].useCount > 0 && indices[position].firstPage >= 0) { //qCDebug(KCOREADDONS_DEBUG) << "Overwriting existing cached entry due to collision."; d->shm->removeEntry(position); // Remove it first } // Data will be stored as fileNamefoo\0PNGimagedata..... // So total size required is the length of the encoded file name + 1 // for the trailing null, and then the length of the image data. uint fileNameLength = 1 + encodedKey.length(); uint requiredSize = fileNameLength + data.size(); uint pagesNeeded = intCeil(requiredSize, d->shm->cachePageSize()); uint firstPage(-1); if (pagesNeeded >= d->shm->pageTableSize()) { qCWarning(KCOREADDONS_DEBUG) << key << "is too large to be cached."; return false; } // If the cache has no room, or the fragmentation is too great to find // the required number of consecutive free pages, take action. if (pagesNeeded > d->shm->cacheAvail || (firstPage = d->shm->findEmptyPages(pagesNeeded)) >= d->shm->pageTableSize()) { // If we have enough free space just defragment uint freePagesDesired = 3 * qMax(1u, pagesNeeded / 2); if (d->shm->cacheAvail > freePagesDesired) { // TODO: How the hell long does this actually take on real // caches? d->shm->defragment(); firstPage = d->shm->findEmptyPages(pagesNeeded); } else { // If we already have free pages we don't want to remove a ton // extra. However we can't rely on the return value of // removeUsedPages giving us a good location since we're not // passing in the actual number of pages that we need. d->shm->removeUsedPages(qMin(2 * freePagesDesired, d->shm->pageTableSize()) - d->shm->cacheAvail); firstPage = d->shm->findEmptyPages(pagesNeeded); } if (firstPage >= d->shm->pageTableSize() || d->shm->cacheAvail < pagesNeeded) { qCritical() << "Unable to free up memory for" << key; return false; } } // Update page table PageTableEntry *table = d->shm->pageTable(); for (uint i = 0; i < pagesNeeded; ++i) { table[firstPage + i].index = position; } // Update index indices[position].fileNameHash = keyHash; indices[position].totalItemSize = requiredSize; indices[position].useCount = 1; indices[position].addTime = ::time(nullptr); indices[position].lastUsedTime = indices[position].addTime; indices[position].firstPage = firstPage; // Update cache d->shm->cacheAvail -= pagesNeeded; // Actually move the data in place void *dataPage = d->shm->page(firstPage); if (Q_UNLIKELY(!dataPage)) { throw KSDCCorrupted(); } // Verify it will all fit d->verifyProposedMemoryAccess(dataPage, requiredSize); // Cast for byte-sized pointer arithmetic uchar *startOfPageData = reinterpret_cast(dataPage); ::memcpy(startOfPageData, encodedKey.constData(), fileNameLength); ::memcpy(startOfPageData + fileNameLength, data.constData(), data.size()); return true; } catch (KSDCCorrupted) { d->recoverCorruptedCache(); return false; } } bool KSharedDataCache::find(const QString &key, QByteArray *destination) const { try { Private::CacheLocker lock(d); if (lock.failed()) { return false; } // Search in the index for our data, hashed by key; QByteArray encodedKey = key.toUtf8(); qint32 entry = d->shm->findNamedEntry(encodedKey); if (entry >= 0) { const IndexTableEntry *header = &d->shm->indexTable()[entry]; const void *resultPage = d->shm->page(header->firstPage); if (Q_UNLIKELY(!resultPage)) { throw KSDCCorrupted(); } d->verifyProposedMemoryAccess(resultPage, header->totalItemSize); header->useCount++; header->lastUsedTime = ::time(nullptr); // Our item is the key followed immediately by the data, so skip // past the key. const char *cacheData = reinterpret_cast(resultPage); cacheData += encodedKey.size(); cacheData++; // Skip trailing null -- now we're pointing to start of data if (destination) { *destination = QByteArray(cacheData, header->totalItemSize - encodedKey.size() - 1); } return true; } } catch (KSDCCorrupted) { d->recoverCorruptedCache(); } return false; } void KSharedDataCache::clear() { try { Private::CacheLocker lock(d); if (!lock.failed()) { d->shm->clear(); } } catch (KSDCCorrupted) { d->recoverCorruptedCache(); } } bool KSharedDataCache::contains(const QString &key) const { try { Private::CacheLocker lock(d); if (lock.failed()) { return false; } return d->shm->findNamedEntry(key.toUtf8()) >= 0; } catch (KSDCCorrupted) { d->recoverCorruptedCache(); return false; } } void KSharedDataCache::deleteCache(const QString &cacheName) { QString cachePath = QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/") + cacheName + QLatin1String(".kcache"); // Note that it is important to simply unlink the file, and not truncate it // smaller first to avoid SIGBUS errors and similar with shared memory // attached to the underlying inode. qCDebug(KCOREADDONS_DEBUG) << "Removing cache at" << cachePath; QFile::remove(cachePath); } unsigned KSharedDataCache::totalSize() const { try { Private::CacheLocker lock(d); if (lock.failed()) { return 0u; } return d->shm->cacheSize; } catch (KSDCCorrupted) { d->recoverCorruptedCache(); return 0u; } } unsigned KSharedDataCache::freeSize() const { try { Private::CacheLocker lock(d); if (lock.failed()) { return 0u; } return d->shm->cacheAvail * d->shm->cachePageSize(); } catch (KSDCCorrupted) { d->recoverCorruptedCache(); return 0u; } } KSharedDataCache::EvictionPolicy KSharedDataCache::evictionPolicy() const { if (d && d->shm) { return static_cast(d->shm->evictionPolicy.fetchAndAddAcquire(0)); } return NoEvictionPreference; } void KSharedDataCache::setEvictionPolicy(EvictionPolicy newPolicy) { if (d && d->shm) { d->shm->evictionPolicy.fetchAndStoreRelease(static_cast(newPolicy)); } } unsigned KSharedDataCache::timestamp() const { if (d && d->shm) { return static_cast(d->shm->cacheTimestamp.fetchAndAddAcquire(0)); } return 0; } void KSharedDataCache::setTimestamp(unsigned newTimestamp) { if (d && d->shm) { d->shm->cacheTimestamp.fetchAndStoreRelease(static_cast(newTimestamp)); } } diff --git a/src/lib/io/kdirwatch.cpp b/src/lib/io/kdirwatch.cpp index 6b0a53b..a9f371b 100644 --- a/src/lib/io/kdirwatch.cpp +++ b/src/lib/io/kdirwatch.cpp @@ -1,2051 +1,2053 @@ /* This file is part of the KDE libraries SPDX-FileCopyrightText: 1998 Sven Radej SPDX-FileCopyrightText: 2006 Dirk Mueller SPDX-FileCopyrightText: 2007 Flavio Castelli SPDX-FileCopyrightText: 2008 Rafal Rzepecki SPDX-FileCopyrightText: 2010 David Faure SPDX-License-Identifier: LGPL-2.0-only */ // CHANGES: // Jul 30, 2008 - Don't follow symlinks when recursing to avoid loops (Rafal) // Aug 6, 2007 - KDirWatch::WatchModes support complete, flags work fine also // when using FAMD (Flavio Castelli) // Aug 3, 2007 - Handled KDirWatch::WatchModes flags when using inotify, now // recursive and file monitoring modes are implemented (Flavio Castelli) // Jul 30, 2007 - Substituted addEntry boolean params with KDirWatch::WatchModes // flag (Flavio Castelli) // Oct 4, 2005 - Inotify support (Dirk Mueller) // February 2002 - Add file watching and remote mount check for STAT // Mar 30, 2001 - Native support for Linux dir change notification. // Jan 28, 2000 - Usage of FAM service on IRIX (Josef.Weidendorfer@in.tum.de) // May 24. 1998 - List of times introduced, and some bugs are fixed. (sven) // May 23. 1998 - Removed static pointer - you can have more instances. // It was Needed for KRegistry. KDirWatch now emits signals and doesn't // call (or need) KFM. No more URL's - just plain paths. (sven) // Mar 29. 1998 - added docs, stop/restart for particular Dirs and // deep copies for list of dirs. (sven) // Mar 28. 1998 - Created. (sven) #include "kdirwatch.h" #include "kdirwatch_p.h" #include "kfilesystemtype.h" #include "kcoreaddons_debug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include // QT_LSTAT, QT_STAT, QT_STATBUF #include #include #if HAVE_SYS_INOTIFY_H #include #include #include #ifndef IN_DONT_FOLLOW #define IN_DONT_FOLLOW 0x02000000 #endif #ifndef IN_ONLYDIR #define IN_ONLYDIR 0x01000000 #endif // debug #include #include #endif // HAVE_SYS_INOTIFY_H Q_DECLARE_LOGGING_CATEGORY(KDIRWATCH) // logging category for this framework, default: log stuff >= warning Q_LOGGING_CATEGORY(KDIRWATCH, "kf5.kcoreaddons.kdirwatch", QtWarningMsg) // set this to true for much more verbose debug output static bool s_verboseDebug = false; static QThreadStorage dwp_self; static KDirWatchPrivate *createPrivate() { if (!dwp_self.hasLocalData()) { dwp_self.setLocalData(new KDirWatchPrivate); } return dwp_self.localData(); } // Convert a string into a watch Method static KDirWatch::Method methodFromString(const QByteArray &method) { if (method == "Fam") { return KDirWatch::FAM; } else if (method == "Stat") { return KDirWatch::Stat; } else if (method == "QFSWatch") { return KDirWatch::QFSWatch; } else { #if defined(HAVE_SYS_INOTIFY_H) // inotify supports delete+recreate+modify, which QFSWatch doesn't support return KDirWatch::INotify; #else return KDirWatch::QFSWatch; #endif } } static const char *methodToString(KDirWatch::Method method) { switch (method) { case KDirWatch::FAM: return "Fam"; case KDirWatch::INotify: return "INotify"; case KDirWatch::Stat: return "Stat"; case KDirWatch::QFSWatch: return "QFSWatch"; } // not reached return nullptr; } static const char s_envNfsPoll[] = "KDIRWATCH_NFSPOLLINTERVAL"; static const char s_envPoll[] = "KDIRWATCH_POLLINTERVAL"; static const char s_envMethod[] = "KDIRWATCH_METHOD"; static const char s_envNfsMethod[] = "KDIRWATCH_NFSMETHOD"; // // Class KDirWatchPrivate (singleton) // /* All entries (files/directories) to be watched in the * application (coming from multiple KDirWatch instances) * are registered in a single KDirWatchPrivate instance. * * At the moment, the following methods for file watching * are supported: * - Polling: All files to be watched are polled regularly * using stat (more precise: QFileInfo.lastModified()). * The polling frequency is determined from global kconfig * settings, defaulting to 500 ms for local directories * and 5000 ms for remote mounts * - FAM (File Alternation Monitor): first used on IRIX, SGI * has ported this method to LINUX. It uses a kernel part * (IMON, sending change events to /dev/imon) and a user * level daemon (fam), to which applications connect for * notification of file changes. For NFS, the fam daemon * on the NFS server machine is used; if IMON is not built * into the kernel, fam uses polling for local files. * - INOTIFY: In LINUX 2.6.13, inode change notification was * introduced. You're now able to watch arbitrary inode's * for changes, and even get notification when they're * unmounted. */ KDirWatchPrivate::KDirWatchPrivate() : timer(), freq(3600000), // 1 hour as upper bound statEntries(0), delayRemove(false), rescan_all(false), rescan_timer(), #if HAVE_SYS_INOTIFY_H mSn(nullptr), #endif _isStopped(false) { // Debug unittest on CI if (qAppName() == QLatin1String("kservicetest") || qAppName() == QLatin1String("filetypestest")) { s_verboseDebug = true; } timer.setObjectName(QStringLiteral("KDirWatchPrivate::timer")); connect(&timer, SIGNAL(timeout()), this, SLOT(slotRescan())); m_nfsPollInterval = qEnvironmentVariableIsSet(s_envNfsPoll) ? qEnvironmentVariableIntValue(s_envNfsPoll) : 5000; m_PollInterval = qEnvironmentVariableIsSet(s_envPoll) ? qEnvironmentVariableIntValue(s_envPoll) : 500; m_preferredMethod = methodFromString(qEnvironmentVariableIsSet(s_envMethod) ? qgetenv(s_envMethod) : "inotify"); // The nfs method defaults to the normal (local) method m_nfsPreferredMethod = methodFromString(qEnvironmentVariableIsSet(s_envNfsMethod) ? qgetenv(s_envNfsMethod) : "Fam"); QList availableMethods; availableMethods << "Stat"; // used for FAM and inotify rescan_timer.setObjectName(QStringLiteral("KDirWatchPrivate::rescan_timer")); rescan_timer.setSingleShot(true); connect(&rescan_timer, SIGNAL(timeout()), this, SLOT(slotRescan())); #if HAVE_FAM availableMethods << "FAM"; use_fam = true; sn = nullptr; #endif #if HAVE_SYS_INOTIFY_H supports_inotify = true; m_inotify_fd = inotify_init(); if (m_inotify_fd <= 0) { qCDebug(KDIRWATCH) << "Can't use Inotify, kernel doesn't support it:" << strerror(errno); supports_inotify = false; } //qCDebug(KDIRWATCH) << "INotify available: " << supports_inotify; if (supports_inotify) { availableMethods << "INotify"; (void)fcntl(m_inotify_fd, F_SETFD, FD_CLOEXEC); mSn = new QSocketNotifier(m_inotify_fd, QSocketNotifier::Read, this); connect(mSn, SIGNAL(activated(int)), this, SLOT(inotifyEventReceived())); } #endif #if HAVE_QFILESYSTEMWATCHER availableMethods << "QFileSystemWatcher"; fsWatcher = nullptr; #endif if (s_verboseDebug) { qCDebug(KDIRWATCH) << "Available methods: " << availableMethods << "preferred=" << methodToString(m_preferredMethod); } } // This is called on app exit (deleted by QThreadStorage) KDirWatchPrivate::~KDirWatchPrivate() { timer.stop(); #if HAVE_FAM if (use_fam && sn) { FAMClose(&fc); } #endif #if HAVE_SYS_INOTIFY_H if (supports_inotify) { QT_CLOSE(m_inotify_fd); } #endif #if HAVE_QFILESYSTEMWATCHER delete fsWatcher; #endif } void KDirWatchPrivate::inotifyEventReceived() { //qCDebug(KDIRWATCH); #if HAVE_SYS_INOTIFY_H if (!supports_inotify) { return; } int pending = -1; int offsetStartRead = 0; // where we read into buffer char buf[8192]; assert(m_inotify_fd > -1); ioctl(m_inotify_fd, FIONREAD, &pending); while (pending > 0) { const int bytesToRead = qMin(pending, sizeof(buf) - offsetStartRead); int bytesAvailable = read(m_inotify_fd, &buf[offsetStartRead], bytesToRead); pending -= bytesAvailable; bytesAvailable += offsetStartRead; offsetStartRead = 0; int offsetCurrent = 0; while (bytesAvailable >= int(sizeof(struct inotify_event))) { const struct inotify_event *const event = reinterpret_cast(&buf[offsetCurrent]); const int eventSize = sizeof(struct inotify_event) + event->len; if (bytesAvailable < eventSize) { break; } bytesAvailable -= eventSize; offsetCurrent += eventSize; QString path; // strip trailing null chars, see inotify_event documentation // these must not end up in the final QString version of path int len = event->len; while (len > 1 && !event->name[len - 1]) { --len; } QByteArray cpath(event->name, len); if (len) { path = QFile::decodeName(cpath); } if (!path.isEmpty() && isNoisyFile(cpath.data())) { continue; } // Is set to true if the new event is a directory, false otherwise. This prevents a stat call in clientsForFileOrDir const bool isDir = (event->mask & (IN_ISDIR)); Entry *e = m_inotify_wd_to_entry.value(event->wd); if (!e) { continue; } const bool wasDirty = e->dirty; e->dirty = true; const QString tpath = e->path + QLatin1Char('/') + path; if (s_verboseDebug) { qCDebug(KDIRWATCH).nospace() << "got event 0x" << qPrintable(QString::number(event->mask, 16)) << " for " << e->path; } if (event->mask & IN_DELETE_SELF) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << "-->got deleteself signal for" << e->path; } e->m_status = NonExistent; m_inotify_wd_to_entry.remove(e->wd); e->wd = -1; e->m_ctime = invalid_ctime; emitEvent(e, Deleted, e->path); // If the parent dir was already watched, tell it something changed Entry *parentEntry = entry(e->parentDirectory()); if (parentEntry) { parentEntry->dirty = true; } // Add entry to parent dir to notice if the entry gets recreated addEntry(nullptr, e->parentDirectory(), e, true /*isDir*/); } if (event->mask & IN_IGNORED) { // Causes bug #207361 with kernels 2.6.31 and 2.6.32! //e->wd = -1; } if (event->mask & (IN_CREATE | IN_MOVED_TO)) { Entry *sub_entry = e->findSubEntry(tpath); if (s_verboseDebug) { qCDebug(KDIRWATCH) << "-->got CREATE signal for" << (tpath) << "sub_entry=" << sub_entry; qCDebug(KDIRWATCH) << *e; } // The code below is very similar to the one in checkFAMEvent... if (sub_entry) { // We were waiting for this new file/dir to be created sub_entry->dirty = true; rescan_timer.start(0); // process this asap, to start watching that dir } else if (e->isDir && !e->m_clients.empty()) { const QList clients = e->inotifyClientsForFileOrDir(isDir); // See discussion in addEntry for why we don't addEntry for individual // files in WatchFiles mode with inotify. if (isDir) { for (const Client *client : clients) { addEntry(client->instance, tpath, nullptr, isDir, isDir ? client->m_watchModes : KDirWatch::WatchDirOnly); } } if (!clients.isEmpty()) { emitEvent(e, Created, tpath); qCDebug(KDIRWATCH).nospace() << clients.count() << " instance(s) monitoring the new " << (isDir ? "dir " : "file ") << tpath; } e->m_pendingFileChanges.append(e->path); if (!rescan_timer.isActive()) { rescan_timer.start(m_PollInterval); // singleshot } } } if (event->mask & (IN_DELETE | IN_MOVED_FROM)) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << "-->got DELETE signal for" << tpath; } if ((e->isDir) && (!e->m_clients.empty())) { // A file in this directory has been removed. It wasn't an explicitly // watched file as it would have its own watch descriptor, so // no addEntry/ removeEntry bookkeeping should be required. Emit // the event immediately if any clients are interested. KDirWatch::WatchModes flag = isDir ? KDirWatch::WatchSubDirs : KDirWatch::WatchFiles; int counter = 0; for (const Client &client : e->m_clients) { if (client.m_watchModes & flag) { counter++; } } if (counter != 0) { emitEvent(e, Deleted, tpath); } } } if (event->mask & (IN_MODIFY | IN_ATTRIB)) { if ((e->isDir) && (!e->m_clients.empty())) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << "-->got MODIFY signal for" << (tpath); } // A file in this directory has been changed. No // addEntry/ removeEntry bookkeeping should be required. // Add the path to the list of pending file changes if // there are any interested clients. //QT_STATBUF stat_buf; //QByteArray tpath = QFile::encodeName(e->path+'/'+path); //QT_STAT(tpath, &stat_buf); //bool isDir = S_ISDIR(stat_buf.st_mode); // The API doc is somewhat vague as to whether we should emit // dirty() for implicitly watched files when WatchFiles has // not been specified - we'll assume they are always interested, // regardless. // Don't worry about duplicates for the time // being; this is handled in slotRescan. e->m_pendingFileChanges.append(tpath); // Avoid stat'ing the directory if only an entry inside it changed. e->dirty = (wasDirty || (path.isEmpty() && (event->mask & IN_ATTRIB))); } } if (!rescan_timer.isActive()) { rescan_timer.start(m_PollInterval); // singleshot } } if (bytesAvailable > 0) { // copy partial event to beginning of buffer memmove(buf, &buf[offsetCurrent], bytesAvailable); offsetStartRead = bytesAvailable; } } #endif } KDirWatchPrivate::Entry::~Entry() { } /* In FAM mode, only entries which are marked dirty are scanned. * We first need to mark all yet nonexistent, but possible created * entries as dirty... */ void KDirWatchPrivate::Entry::propagate_dirty() { for (Entry *sub_entry : qAsConst(m_entries)) { if (!sub_entry->dirty) { sub_entry->dirty = true; sub_entry->propagate_dirty(); } } } /* A KDirWatch instance is interested in getting events for * this file/Dir entry. */ void KDirWatchPrivate::Entry::addClient(KDirWatch *instance, KDirWatch::WatchModes watchModes) { if (instance == nullptr) { return; } for (Client &client : m_clients) { if (client.instance == instance) { client.count++; client.m_watchModes = watchModes; return; } } m_clients.emplace_back(instance, watchModes); } void KDirWatchPrivate::Entry::removeClient(KDirWatch *instance) { auto it = m_clients.begin(); const auto end = m_clients.end(); for (; it != end; ++it) { Client &client = *it; if (client.instance == instance) { client.count--; if (client.count == 0) { m_clients.erase(it); } return; } } } /* get number of clients */ int KDirWatchPrivate::Entry::clientCount() const { int clients = 0; for (const Client &client : m_clients) { clients += client.count; } return clients; } QString KDirWatchPrivate::Entry::parentDirectory() const { return QDir::cleanPath(path + QLatin1String("/..")); } QList KDirWatchPrivate::Entry::clientsForFileOrDir(const QString &tpath, bool *isDir) const { QList ret; QFileInfo fi(tpath); if (fi.exists()) { *isDir = fi.isDir(); const KDirWatch::WatchModes flag = *isDir ? KDirWatch::WatchSubDirs : KDirWatch::WatchFiles; for (const Client &client : m_clients) { if (client.m_watchModes & flag) { ret.append(&client); } } } else { // Happens frequently, e.g. ERROR: couldn't stat "/home/dfaure/.viminfo.tmp" //qCDebug(KDIRWATCH) << "ERROR: couldn't stat" << tpath; // In this case isDir is not set, but ret is empty anyway // so isDir won't be used. } return ret; } // inotify specific function that doesn't call KDE::stat to figure out if we have a file or folder. // isDir is determined through inotify's "IN_ISDIR" flag in KDirWatchPrivate::inotifyEventReceived QList KDirWatchPrivate::Entry::inotifyClientsForFileOrDir(bool isDir) const { QList ret; const KDirWatch::WatchModes flag = isDir ? KDirWatch::WatchSubDirs : KDirWatch::WatchFiles; for (const Client &client : m_clients) { if (client.m_watchModes & flag) { ret.append(&client); } } return ret; } QDebug operator<<(QDebug debug, const KDirWatchPrivate::Entry &entry) { debug.nospace() << "[ Entry for " << entry.path << ", " << (entry.isDir ? "dir" : "file"); if (entry.m_status == KDirWatchPrivate::NonExistent) { debug << ", non-existent"; } debug << ", using " << ((entry.m_mode == KDirWatchPrivate::FAMMode) ? "FAM" : (entry.m_mode == KDirWatchPrivate::INotifyMode) ? "INotify" : (entry.m_mode == KDirWatchPrivate::QFSWatchMode) ? "QFSWatch" : (entry.m_mode == KDirWatchPrivate::StatMode) ? "Stat" : "Unknown Method"); #if HAVE_SYS_INOTIFY_H if (entry.m_mode == KDirWatchPrivate::INotifyMode) { debug << " inotify_wd=" << entry.wd; } #endif debug << ", has " << entry.m_clients.size() << " clients"; debug.space(); if (!entry.m_entries.isEmpty()) { debug << ", nonexistent subentries:"; for (KDirWatchPrivate::Entry *subEntry : qAsConst(entry.m_entries)) { debug << subEntry << subEntry->path; } } debug << ']'; return debug; } KDirWatchPrivate::Entry *KDirWatchPrivate::entry(const QString &_path) { // we only support absolute paths if (_path.isEmpty() || QDir::isRelativePath(_path)) { return nullptr; } QString path(_path); if (path.length() > 1 && path.endsWith(QLatin1Char('/'))) { path.chop(1); } EntryMap::Iterator it = m_mapEntries.find(path); if (it == m_mapEntries.end()) { return nullptr; } else { return &(*it); } } // set polling frequency for a entry and adjust global freq if needed void KDirWatchPrivate::useFreq(Entry *e, int newFreq) { e->freq = newFreq; // a reasonable frequency for the global polling timer if (e->freq < freq) { freq = e->freq; if (timer.isActive()) { timer.start(freq); } qCDebug(KDIRWATCH) << "Global Poll Freq is now" << freq << "msec"; } } #if HAVE_FAM // setup FAM notification, returns false if not possible bool KDirWatchPrivate::useFAM(Entry *e) { if (!use_fam) { return false; } if (!sn) { if (FAMOpen(&fc) == 0) { sn = new QSocketNotifier(FAMCONNECTION_GETFD(&fc), QSocketNotifier::Read, this); connect(sn, SIGNAL(activated(int)), this, SLOT(famEventReceived())); } else { use_fam = false; return false; } } // handle FAM events to avoid deadlock // (FAM sends back all files in a directory when monitoring) famEventReceived(); e->m_mode = FAMMode; e->dirty = false; e->m_famReportedSeen = false; bool startedFAMMonitor = false; if (e->isDir) { if (e->m_status == NonExistent) { // If the directory does not exist we watch the parent directory addEntry(nullptr, e->parentDirectory(), e, true); } else { int res = FAMMonitorDirectory(&fc, QFile::encodeName(e->path).data(), &(e->fr), e); startedFAMMonitor = true; if (res < 0) { e->m_mode = UnknownMode; use_fam = false; delete sn; sn = nullptr; return false; } qCDebug(KDIRWATCH).nospace() << " Setup FAM (Req " << FAMREQUEST_GETREQNUM(&(e->fr)) << ") for " << e->path; } } else { if (e->m_status == NonExistent) { // If the file does not exist we watch the directory addEntry(nullptr, QFileInfo(e->path).absolutePath(), e, true); } else { int res = FAMMonitorFile(&fc, QFile::encodeName(e->path).data(), &(e->fr), e); startedFAMMonitor = true; if (res < 0) { e->m_mode = UnknownMode; use_fam = false; delete sn; sn = nullptr; return false; } qCDebug(KDIRWATCH).nospace() << " Setup FAM (Req " << FAMREQUEST_GETREQNUM(&(e->fr)) << ") for " << e->path; } } // handle FAM events to avoid deadlock // (FAM sends back all files in a directory when monitoring) do { famEventReceived(); if (startedFAMMonitor && !e->m_famReportedSeen) { // 50 is ~half the time it takes to setup a watch. If gamin's latency // gets better, this can be reduced. QThread::msleep(50); } } while (startedFAMMonitor &&!e->m_famReportedSeen); return true; } #endif #if HAVE_SYS_INOTIFY_H // setup INotify notification, returns false if not possible bool KDirWatchPrivate::useINotify(Entry *e) { //qCDebug(KDIRWATCH) << "trying to use inotify for monitoring"; e->wd = -1; e->dirty = false; if (!supports_inotify) { return false; } e->m_mode = INotifyMode; if (e->m_status == NonExistent) { addEntry(nullptr, e->parentDirectory(), e, true); return true; } // May as well register for almost everything - it's free! int mask = IN_DELETE | IN_DELETE_SELF | IN_CREATE | IN_MOVE | IN_MOVE_SELF | IN_DONT_FOLLOW | IN_MOVED_FROM | IN_MODIFY | IN_ATTRIB; if ((e->wd = inotify_add_watch(m_inotify_fd, QFile::encodeName(e->path).data(), mask)) != -1) { m_inotify_wd_to_entry.insert(e->wd, e); if (s_verboseDebug) { qCDebug(KDIRWATCH) << "inotify successfully used for monitoring" << e->path << "wd=" << e->wd; } return true; } if (errno == ENOSPC) { // Inotify max_user_watches was reached (/proc/sys/fs/inotify/max_user_watches) // See man inotify_add_watch, https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers qCWarning(KDIRWATCH) << "inotify failed for monitoring" << e->path << "\n" << "Because it reached its max_user_watches,\n" << "you can increase the maximum number of file watches per user,\n"<< "by setting an appropriate fs.inotify.max_user_watches parameter in your /etc/sysctl.conf"; } else { qCDebug(KDIRWATCH) << "inotify failed for monitoring" << e->path << ":" << strerror(errno) << " (errno:" << errno << ")"; } return false; } #endif #if HAVE_QFILESYSTEMWATCHER bool KDirWatchPrivate::useQFSWatch(Entry *e) { e->m_mode = QFSWatchMode; e->dirty = false; if (e->m_status == NonExistent) { addEntry(nullptr, e->parentDirectory(), e, true /*isDir*/); return true; } //qCDebug(KDIRWATCH) << "fsWatcher->addPath" << e->path; if (!fsWatcher) { fsWatcher = new QFileSystemWatcher(); connect(fsWatcher, SIGNAL(directoryChanged(QString)), this, SLOT(fswEventReceived(QString))); connect(fsWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fswEventReceived(QString))); } fsWatcher->addPath(e->path); return true; } #endif bool KDirWatchPrivate::useStat(Entry *e) { if (KFileSystemType::fileSystemType(e->path) == KFileSystemType::Nfs) { // TODO: or Smbfs? useFreq(e, m_nfsPollInterval); } else { useFreq(e, m_PollInterval); } if (e->m_mode != StatMode) { e->m_mode = StatMode; statEntries++; if (statEntries == 1) { // if this was first STAT entry (=timer was stopped) timer.start(freq); // then start the timer qCDebug(KDIRWATCH) << " Started Polling Timer, freq " << freq; } } qCDebug(KDIRWATCH) << " Setup Stat (freq " << e->freq << ") for " << e->path; return true; } /* If !=0, this KDirWatch instance wants to watch at <_path>, * providing in the type of the entry to be watched. * Sometimes, entries are dependent on each other: if !=0, * this entry needs another entry to watch itself (when notExistent). */ void KDirWatchPrivate::addEntry(KDirWatch *instance, const QString &_path, Entry *sub_entry, bool isDir, KDirWatch::WatchModes watchModes) { QString path(_path); if (path.startsWith(QLatin1String(":/"))) { qCWarning(KDIRWATCH) << "Cannot watch QRC-like path" << path; return; } if (path.isEmpty() #ifndef Q_OS_WIN || path == QLatin1String("/dev") || (path.startsWith(QLatin1String("/dev/")) && !path.startsWith(QLatin1String("/dev/.")) && !path.startsWith(QLatin1String("/dev/shm"))) #endif ) { return; // Don't even go there. } if (path.length() > 1 && path.endsWith(QLatin1Char('/'))) { path.chop(1); } EntryMap::Iterator it = m_mapEntries.find(path); if (it != m_mapEntries.end()) { if (sub_entry) { (*it).m_entries.append(sub_entry); if (s_verboseDebug) { qCDebug(KDIRWATCH) << "Added already watched Entry" << path << "(for" << sub_entry->path << ")"; } } else { (*it).addClient(instance, watchModes); if (s_verboseDebug) { qCDebug(KDIRWATCH) << "Added already watched Entry" << path << "(now" << (*it).clientCount() << "clients)" << QStringLiteral("[%1]").arg(instance->objectName()); } } return; } // we have a new path to watch QT_STATBUF stat_buf; bool exists = (QT_STAT(QFile::encodeName(path).constData(), &stat_buf) == 0); EntryMap::iterator newIt = m_mapEntries.insert(path, Entry()); // the insert does a copy, so we have to use now Entry *e = &(*newIt); if (exists) { e->isDir = (stat_buf.st_mode & QT_STAT_MASK) == QT_STAT_DIR; #ifndef Q_OS_WIN if (e->isDir && !isDir) { if (QT_LSTAT(QFile::encodeName(path).constData(), &stat_buf) == 0) { if ((stat_buf.st_mode & QT_STAT_MASK) == QT_STAT_LNK) { // if it's a symlink, don't follow it e->isDir = false; } } } #endif if (e->isDir && !isDir) { qCWarning(KCOREADDONS_DEBUG) << "KDirWatch:" << path << "is a directory. Use addDir!"; } else if (!e->isDir && isDir) { qCWarning(KCOREADDONS_DEBUG) << "KDirWatch:" << path << "is a file. Use addFile!"; } if (!e->isDir && (watchModes != KDirWatch::WatchDirOnly)) { qCWarning(KCOREADDONS_DEBUG) << "KDirWatch:" << path << "is a file. You can't use recursive or " "watchFiles options"; watchModes = KDirWatch::WatchDirOnly; } #ifdef Q_OS_WIN // ctime is the 'creation time' on windows - use mtime instead e->m_ctime = stat_buf.st_mtime; #else e->m_ctime = stat_buf.st_ctime; #endif e->m_status = Normal; e->m_nlink = stat_buf.st_nlink; e->m_ino = stat_buf.st_ino; } else { e->isDir = isDir; e->m_ctime = invalid_ctime; e->m_status = NonExistent; e->m_nlink = 0; e->m_ino = 0; } e->path = path; if (sub_entry) { e->m_entries.append(sub_entry); } else { e->addClient(instance, watchModes); } if (s_verboseDebug) { qCDebug(KDIRWATCH).nospace() << "Added " << (e->isDir ? "Dir " : "File ") << path << (e->m_status == NonExistent ? " NotExisting" : "") << " for " << (sub_entry ? sub_entry->path : QString()) << " [" << (instance ? instance->objectName() : QString()) << "]"; } // now setup the notification method e->m_mode = UnknownMode; e->msecLeft = 0; if (isNoisyFile(QFile::encodeName(path).data())) { return; } if (exists && e->isDir && (watchModes != KDirWatch::WatchDirOnly)) { QFlags filters = QDir::NoDotAndDotDot; if ((watchModes & KDirWatch::WatchSubDirs) && (watchModes & KDirWatch::WatchFiles)) { filters |= (QDir::Dirs | QDir::Files); } else if (watchModes & KDirWatch::WatchSubDirs) { filters |= QDir::Dirs; } else if (watchModes & KDirWatch::WatchFiles) { filters |= QDir::Files; } #if HAVE_SYS_INOTIFY_H if (e->m_mode == INotifyMode || (e->m_mode == UnknownMode && m_preferredMethod == KDirWatch::INotify)) { //qCDebug(KDIRWATCH) << "Ignoring WatchFiles directive - this is implicit with inotify"; // Placing a watch on individual files is redundant with inotify // (inotify gives us WatchFiles functionality "for free") and indeed // actively harmful, so prevent it. WatchSubDirs is necessary, though. filters &= ~QDir::Files; } #endif QDir basedir(e->path); const QFileInfoList contents = basedir.entryInfoList(filters); for (QFileInfoList::const_iterator iter = contents.constBegin(); iter != contents.constEnd(); ++iter) { const QFileInfo &fileInfo = *iter; // treat symlinks as files--don't follow them. bool isDir = fileInfo.isDir() && !fileInfo.isSymLink(); addEntry(instance, fileInfo.absoluteFilePath(), nullptr, isDir, isDir ? watchModes : KDirWatch::WatchDirOnly); } } addWatch(e); } void KDirWatchPrivate::addWatch(Entry *e) { // If the watch is on a network filesystem use the nfsPreferredMethod as the // default, otherwise use preferredMethod as the default, if the methods are // the same we can skip the mountpoint check // This allows to configure a different method for NFS mounts, since inotify // cannot detect changes made by other machines. However as a default inotify // is fine, since the most common case is a NFS-mounted home, where all changes // are made locally. #177892. KDirWatch::Method preferredMethod = m_preferredMethod; if (m_nfsPreferredMethod != m_preferredMethod) { if (KFileSystemType::fileSystemType(e->path) == KFileSystemType::Nfs) { preferredMethod = m_nfsPreferredMethod; } } // Try the appropriate preferred method from the config first bool entryAdded = false; switch (preferredMethod) { #if HAVE_FAM case KDirWatch::FAM: entryAdded = useFAM(e); break; +#else + case KDirWatch::FAM: Q_UNREACHABLE(); break; #endif #if HAVE_SYS_INOTIFY_H case KDirWatch::INotify: entryAdded = useINotify(e); break; #endif #if HAVE_QFILESYSTEMWATCHER case KDirWatch::QFSWatch: entryAdded = useQFSWatch(e); break; #endif case KDirWatch::Stat: entryAdded = useStat(e); break; } // Failing that try in order INotify, FAM, QFSWatch, Stat if (!entryAdded) { #if HAVE_SYS_INOTIFY_H if (useINotify(e)) { return; } #endif #if HAVE_FAM if (useFAM(e)) { return; } #endif #if HAVE_QFILESYSTEMWATCHER if (useQFSWatch(e)) { return; } #endif useStat(e); } } void KDirWatchPrivate::removeWatch(Entry *e) { #if HAVE_FAM if (e->m_mode == FAMMode) { FAMCancelMonitor(&fc, &(e->fr)); qCDebug(KDIRWATCH).nospace() << "Cancelled FAM (Req " << FAMREQUEST_GETREQNUM(&(e->fr)) << ") for " << e->path; } #endif #if HAVE_SYS_INOTIFY_H if (e->m_mode == INotifyMode) { m_inotify_wd_to_entry.remove(e->wd); (void) inotify_rm_watch(m_inotify_fd, e->wd); if (s_verboseDebug) { qCDebug(KDIRWATCH).nospace() << "Cancelled INotify (fd " << m_inotify_fd << ", " << e->wd << ") for " << e->path; } } #endif #if HAVE_QFILESYSTEMWATCHER if (e->m_mode == QFSWatchMode && fsWatcher) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << "fsWatcher->removePath" << e->path; } fsWatcher->removePath(e->path); } #endif } void KDirWatchPrivate::removeEntry(KDirWatch *instance, const QString &_path, Entry *sub_entry) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << "path=" << _path << "sub_entry:" << sub_entry; } Entry *e = entry(_path); if (e) { removeEntry(instance, e, sub_entry); } } void KDirWatchPrivate::removeEntry(KDirWatch *instance, Entry *e, Entry *sub_entry) { removeList.remove(e); if (sub_entry) { e->m_entries.removeAll(sub_entry); } else { e->removeClient(instance); } if (!e->m_clients.empty() || !e->m_entries.empty()) { return; } if (delayRemove) { removeList.insert(e); // now e->isValid() is false return; } if (e->m_status == Normal) { removeWatch(e); } else { // Removed a NonExistent entry - we just remove it from the parent if (e->isDir) { removeEntry(nullptr, e->parentDirectory(), e); } else { removeEntry(nullptr, QFileInfo(e->path).absolutePath(), e); } } if (e->m_mode == StatMode) { statEntries--; if (statEntries == 0) { timer.stop(); // stop timer if lists are empty qCDebug(KDIRWATCH) << " Stopped Polling Timer"; } } if (s_verboseDebug) { qCDebug(KDIRWATCH).nospace() << "Removed " << (e->isDir ? "Dir " : "File ") << e->path << " for " << (sub_entry ? sub_entry->path : QString()) << " [" << (instance ? instance->objectName() : QString()) << "]"; } QString p = e->path; // take a copy, QMap::remove takes a reference and deletes, since e points into the map #if HAVE_SYS_INOTIFY_H m_inotify_wd_to_entry.remove(e->wd); #endif m_mapEntries.remove(p); // not valid any more } /* Called from KDirWatch destructor: * remove as client from all entries */ void KDirWatchPrivate::removeEntries(KDirWatch *instance) { int minfreq = 3600000; QStringList pathList; // put all entries where instance is a client in list EntryMap::Iterator it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) { Client *c = nullptr; for (Client &client : (*it).m_clients) { if (client.instance == instance) { c = &client; break; } } if (c) { c->count = 1; // forces deletion of instance as client pathList.append((*it).path); } else if ((*it).m_mode == StatMode && (*it).freq < minfreq) { minfreq = (*it).freq; } } for (const QString &path : qAsConst(pathList)) { removeEntry(instance, path, nullptr); } if (minfreq > freq) { // we can decrease the global polling frequency freq = minfreq; if (timer.isActive()) { timer.start(freq); } qCDebug(KDIRWATCH) << "Poll Freq now" << freq << "msec"; } } // instance ==0: stop scanning for all instances bool KDirWatchPrivate::stopEntryScan(KDirWatch *instance, Entry *e) { int stillWatching = 0; for (Client &client : e->m_clients) { if (!instance || instance == client.instance) { client.watchingStopped = true; } else if (!client.watchingStopped) { stillWatching += client.count; } } qCDebug(KDIRWATCH) << (instance ? instance->objectName() : QStringLiteral("all")) << "stopped scanning" << e->path << "(now" << stillWatching << "watchers)"; if (stillWatching == 0) { // if nobody is interested, we don't watch, and we don't report // changes that happened while not watching e->m_ctime = invalid_ctime; // invalid // Changing m_status like this would create wrong "created" events in stat mode. // To really "stop watching" we would need to determine 'stillWatching==0' in scanEntry... //e->m_status = NonExistent; } return true; } // instance ==0: start scanning for all instances bool KDirWatchPrivate::restartEntryScan(KDirWatch *instance, Entry *e, bool notify) { int wasWatching = 0, newWatching = 0; for (Client &client : e->m_clients) { if (!client.watchingStopped) { wasWatching += client.count; } else if (!instance || instance == client.instance) { client.watchingStopped = false; newWatching += client.count; } } if (newWatching == 0) { return false; } qCDebug(KDIRWATCH) << (instance ? instance->objectName() : QStringLiteral("all")) << "restarted scanning" << e->path << "(now" << wasWatching + newWatching << "watchers)"; // restart watching and emit pending events int ev = NoChange; if (wasWatching == 0) { if (!notify) { QT_STATBUF stat_buf; bool exists = (QT_STAT(QFile::encodeName(e->path).constData(), &stat_buf) == 0); if (exists) { // ctime is the 'creation time' on windows, but with qMax // we get the latest change of any kind, on any platform. e->m_ctime = qMax(stat_buf.st_ctime, stat_buf.st_mtime); e->m_status = Normal; if (s_verboseDebug) { qCDebug(KDIRWATCH) << "Setting status to Normal for" << e << e->path; } e->m_nlink = stat_buf.st_nlink; e->m_ino = stat_buf.st_ino; // Same as in scanEntry: ensure no subentry in parent dir removeEntry(nullptr, e->parentDirectory(), e); } else { e->m_ctime = invalid_ctime; e->m_status = NonExistent; e->m_nlink = 0; if (s_verboseDebug) { qCDebug(KDIRWATCH) << "Setting status to NonExistent for" << e << e->path; } } } e->msecLeft = 0; ev = scanEntry(e); } emitEvent(e, ev); return true; } // instance ==0: stop scanning for all instances void KDirWatchPrivate::stopScan(KDirWatch *instance) { EntryMap::Iterator it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) { stopEntryScan(instance, &(*it)); } } void KDirWatchPrivate::startScan(KDirWatch *instance, bool notify, bool skippedToo) { if (!notify) { resetList(instance, skippedToo); } EntryMap::Iterator it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) { restartEntryScan(instance, &(*it), notify); } // timer should still be running when in polling mode } // clear all pending events, also from stopped void KDirWatchPrivate::resetList(KDirWatch *instance, bool skippedToo) { Q_UNUSED(instance); EntryMap::Iterator it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) { for (Client &client : (*it).m_clients) { if (!client.watchingStopped || skippedToo) { client.pending = NoChange; } } } } // Return event happened on // int KDirWatchPrivate::scanEntry(Entry *e) { // Shouldn't happen: Ignore "unknown" notification method if (e->m_mode == UnknownMode) { return NoChange; } if (e->m_mode == FAMMode || e->m_mode == INotifyMode) { // we know nothing has changed, no need to stat if (!e->dirty) { return NoChange; } e->dirty = false; } if (e->m_mode == StatMode) { // only scan if timeout on entry timer happens; // e.g. when using 500msec global timer, a entry // with freq=5000 is only watched every 10th time e->msecLeft -= freq; if (e->msecLeft > 0) { return NoChange; } e->msecLeft += e->freq; } QT_STATBUF stat_buf; const bool exists = (QT_STAT(QFile::encodeName(e->path).constData(), &stat_buf) == 0); if (exists) { if (e->m_status == NonExistent) { // ctime is the 'creation time' on windows, but with qMax // we get the latest change of any kind, on any platform. e->m_ctime = qMax(stat_buf.st_ctime, stat_buf.st_mtime); e->m_status = Normal; e->m_ino = stat_buf.st_ino; if (s_verboseDebug) { qCDebug(KDIRWATCH) << "Setting status to Normal for just created" << e << e->path; } // We need to make sure the entry isn't listed in its parent's subentries... (#222974, testMoveTo) removeEntry(nullptr, e->parentDirectory(), e); return Created; } #if 1 // for debugging the if() below if (s_verboseDebug) { struct tm *tmp = localtime(&e->m_ctime); char outstr[200]; strftime(outstr, sizeof(outstr), "%H:%M:%S", tmp); qCDebug(KDIRWATCH) << e->path << "e->m_ctime=" << e->m_ctime << outstr << "stat_buf.st_ctime=" << stat_buf.st_ctime << "stat_buf.st_mtime=" << stat_buf.st_mtime << "e->m_nlink=" << e->m_nlink << "stat_buf.st_nlink=" << stat_buf.st_nlink << "e->m_ino=" << e->m_ino << "stat_buf.st_ino=" << stat_buf.st_ino; } #endif if ((e->m_ctime != invalid_ctime) && (qMax(stat_buf.st_ctime, stat_buf.st_mtime) != e->m_ctime || stat_buf.st_ino != e->m_ino || int(stat_buf.st_nlink) != int(e->m_nlink) #ifdef Q_OS_WIN // on Windows, we trust QFSW to get it right, the ctime comparisons above // fail for example when adding files to directories on Windows // which doesn't change the mtime of the directory || e->m_mode == QFSWatchMode #endif )) { e->m_ctime = qMax(stat_buf.st_ctime, stat_buf.st_mtime); e->m_nlink = stat_buf.st_nlink; if (e->m_ino != stat_buf.st_ino) { // The file got deleted and recreated. We need to watch it again. removeWatch(e); addWatch(e); e->m_ino = stat_buf.st_ino; return (Deleted|Created); } else { return Changed; } } return NoChange; } // dir/file doesn't exist e->m_nlink = 0; e->m_ino = 0; e->m_status = NonExistent; if (e->m_ctime == invalid_ctime) { return NoChange; } e->m_ctime = invalid_ctime; return Deleted; } /* Notify all interested KDirWatch instances about a given event on an entry * and stored pending events. When watching is stopped, the event is * added to the pending events. */ void KDirWatchPrivate::emitEvent(Entry *e, int event, const QString &fileName) { QString path(e->path); if (!fileName.isEmpty()) { if (!QDir::isRelativePath(fileName)) { path = fileName; } else { #ifdef Q_OS_UNIX path += QLatin1Char('/') + fileName; #elif defined(Q_OS_WIN) //current drive is passed instead of / path += QDir::currentPath().leftRef(2) + QLatin1Char('/') + fileName; #endif } } if (s_verboseDebug) { qCDebug(KDIRWATCH) << event << path << e->m_clients.size() << "clients"; } for (Client &c : e->m_clients) { if (c.instance == nullptr || c.count == 0) { continue; } if (c.watchingStopped) { // Do not add event to a list of pending events, the docs say restartDirScan won't emit! #if 0 if (event == Changed) { c.pending |= event; } else if (event == Created || event == Deleted) { c.pending = event; } #endif continue; } // not stopped if (event == NoChange || event == Changed) { event |= c.pending; } c.pending = NoChange; if (event == NoChange) { continue; } // Emit the signals delayed, to avoid unexpected re-entrance from the slots (#220153) if (event & Deleted) { QMetaObject::invokeMethod(c.instance, [c, path]() { c.instance->setDeleted(path); }, Qt::QueuedConnection); } if (event & Created) { QMetaObject::invokeMethod(c.instance, [c, path]() { c.instance->setCreated(path); }, Qt::QueuedConnection); // possible emit Change event after creation } if (event & Changed) { QMetaObject::invokeMethod(c.instance, [c, path]() { c.instance->setDirty(path); }, Qt::QueuedConnection); } } } // Remove entries which were marked to be removed void KDirWatchPrivate::slotRemoveDelayed() { delayRemove = false; // Removing an entry could also take care of removing its parent // (e.g. in FAM or inotify mode), which would remove other entries in removeList, // so don't use Q_FOREACH or iterators here... while (!removeList.isEmpty()) { Entry *entry = *removeList.begin(); removeEntry(nullptr, entry, nullptr); // this will remove entry from removeList } } /* Scan all entries to be watched for changes. This is done regularly * when polling. FAM and inotify use a single-shot timer to call this slot delayed. */ void KDirWatchPrivate::slotRescan() { if (s_verboseDebug) { qCDebug(KDIRWATCH); } EntryMap::Iterator it; // People can do very long things in the slot connected to dirty(), // like showing a message box. We don't want to keep polling during // that time, otherwise the value of 'delayRemove' will be reset. // ### TODO: now the emitEvent delays emission, this can be cleaned up bool timerRunning = timer.isActive(); if (timerRunning) { timer.stop(); } // We delay deletions of entries this way. // removeDir(), when called in slotDirty(), can cause a crash otherwise // ### TODO: now the emitEvent delays emission, this can be cleaned up delayRemove = true; if (rescan_all) { // mark all as dirty it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) { (*it).dirty = true; } rescan_all = false; } else { // propagate dirty flag to dependent entries (e.g. file watches) it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) if (((*it).m_mode == INotifyMode || (*it).m_mode == QFSWatchMode) && (*it).dirty) { (*it).propagate_dirty(); } } #if HAVE_SYS_INOTIFY_H QList cList; #endif it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) { // we don't check invalid entries (i.e. remove delayed) Entry *entry = &(*it); if (!entry->isValid()) { continue; } const int ev = scanEntry(entry); if (s_verboseDebug) { qCDebug(KDIRWATCH) << "scanEntry for" << entry->path << "says" << ev; } switch (entry->m_mode) { #if HAVE_SYS_INOTIFY_H case INotifyMode: if (ev == Deleted) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << "scanEntry says" << entry->path << "was deleted"; } addEntry(nullptr, entry->parentDirectory(), entry, true); } else if (ev == Created) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << "scanEntry says" << entry->path << "was created. wd=" << entry->wd; } if (entry->wd < 0) { cList.append(entry); addWatch(entry); } } break; #endif case FAMMode: case QFSWatchMode: if (ev == Created) { addWatch(entry); } break; default: // dunno about StatMode... break; } #if HAVE_SYS_INOTIFY_H if (entry->isDir) { // Report and clear the list of files that have changed in this directory. // Remove duplicates by changing to set and back again: // we don't really care about preserving the order of the // original changes. QStringList pendingFileChanges = entry->m_pendingFileChanges; pendingFileChanges.removeDuplicates(); for (const QString &changedFilename : qAsConst(pendingFileChanges)) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << "processing pending file change for" << changedFilename; } emitEvent(entry, Changed, changedFilename); } entry->m_pendingFileChanges.clear(); } #endif if (ev != NoChange) { emitEvent(entry, ev); } } if (timerRunning) { timer.start(freq); } #if HAVE_SYS_INOTIFY_H // Remove watch of parent of new created directories for (Entry *e : qAsConst(cList)) { removeEntry(nullptr, e->parentDirectory(), e); } #endif QTimer::singleShot(0, this, SLOT(slotRemoveDelayed())); } bool KDirWatchPrivate::isNoisyFile(const char *filename) { // $HOME/.X.err grows with debug output, so don't notify change if (*filename == '.') { if (strncmp(filename, ".X.err", 6) == 0) { return true; } if (strncmp(filename, ".xsession-errors", 16) == 0) { return true; } // fontconfig updates the cache on every KDE app start // (inclusive kio_thumbnail slaves) if (strncmp(filename, ".fonts.cache", 12) == 0) { return true; } } return false; } #if HAVE_FAM void KDirWatchPrivate::famEventReceived() { static FAMEvent fe; delayRemove = true; //qCDebug(KDIRWATCH) << "Fam event received"; while (use_fam && FAMPending(&fc)) { if (FAMNextEvent(&fc, &fe) == -1) { qCWarning(KCOREADDONS_DEBUG) << "FAM connection problem, switching to polling."; use_fam = false; delete sn; sn = nullptr; // Replace all FAMMode entries with INotify/Stat EntryMap::Iterator it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) if ((*it).m_mode == FAMMode && !(*it).m_clients.empty()) { Entry *e = &(*it); addWatch(e); } } else { checkFAMEvent(&fe); } } QTimer::singleShot(0, this, SLOT(slotRemoveDelayed())); } void KDirWatchPrivate::checkFAMEvent(FAMEvent *fe) { //qCDebug(KDIRWATCH); Entry *e = nullptr; EntryMap::Iterator it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) if (FAMREQUEST_GETREQNUM(&((*it).fr)) == FAMREQUEST_GETREQNUM(&(fe->fr))) { e = &(*it); break; } // Don't be too verbose ;-) if ((fe->code == FAMExists) || (fe->code == FAMEndExist) || (fe->code == FAMAcknowledge)) { if (e) { e->m_famReportedSeen = true; } return; } if (isNoisyFile(fe->filename)) { return; } // Entry *e = static_cast(fe->userdata); if (s_verboseDebug) { // don't enable this except when debugging, see #88538 qCDebug(KDIRWATCH) << "Processing FAM event (" << ((fe->code == FAMChanged) ? "FAMChanged" : (fe->code == FAMDeleted) ? "FAMDeleted" : (fe->code == FAMStartExecuting) ? "FAMStartExecuting" : (fe->code == FAMStopExecuting) ? "FAMStopExecuting" : (fe->code == FAMCreated) ? "FAMCreated" : (fe->code == FAMMoved) ? "FAMMoved" : (fe->code == FAMAcknowledge) ? "FAMAcknowledge" : (fe->code == FAMExists) ? "FAMExists" : (fe->code == FAMEndExist) ? "FAMEndExist" : "Unknown Code") << ", " << fe->filename << ", Req " << FAMREQUEST_GETREQNUM(&(fe->fr)) << ") e=" << e; } if (!e) { // this happens e.g. for FAMAcknowledge after deleting a dir... // qCDebug(KDIRWATCH) << "No entry for FAM event ?!"; return; } if (e->m_status == NonExistent) { qCDebug(KDIRWATCH) << "FAM event for nonExistent entry " << e->path; return; } // Delayed handling. This rechecks changes with own stat calls. e->dirty = true; if (!rescan_timer.isActive()) { rescan_timer.start(m_PollInterval); // singleshot } // needed FAM control actions on FAM events switch (fe->code) { case FAMDeleted: // fe->filename is an absolute path when a watched file-or-dir is deleted if (!QDir::isRelativePath(QFile::decodeName(fe->filename))) { FAMCancelMonitor(&fc, &(e->fr)); // needed ? qCDebug(KDIRWATCH) << "Cancelled FAMReq" << FAMREQUEST_GETREQNUM(&(e->fr)) << "for" << e->path; e->m_status = NonExistent; e->m_ctime = invalid_ctime; emitEvent(e, Deleted, e->path); // If the parent dir was already watched, tell it something changed Entry *parentEntry = entry(e->parentDirectory()); if (parentEntry) { parentEntry->dirty = true; } // Add entry to parent dir to notice if the entry gets recreated addEntry(nullptr, e->parentDirectory(), e, true /*isDir*/); } else { // A file in this directory has been removed, and wasn't explicitly watched. // We could still inform clients, like inotify does? But stat can't. // For now we just marked e dirty and slotRescan will emit the dir as dirty. //qCDebug(KDIRWATCH) << "Got FAMDeleted for" << QFile::decodeName(fe->filename) << "in" << e->path << ". Absolute path -> NOOP!"; } break; case FAMCreated: { // check for creation of a directory we have to watch QString tpath(e->path + QLatin1Char('/') + QFile::decodeName(fe->filename)); // This code is very similar to the one in inotifyEventReceived... Entry *sub_entry = e->findSubEntry(tpath); if (sub_entry /*&& sub_entry->isDir*/) { // We were waiting for this new file/dir to be created. We don't actually // emit an event here, as the rescan_timer will re-detect the creation and // do the signal emission there. sub_entry->dirty = true; rescan_timer.start(0); // process this asap, to start watching that dir } else if (e->isDir && !e->m_clients.empty()) { bool isDir = false; const QList clients = e->clientsForFileOrDir(tpath, &isDir); for (const Client *client : clients) { addEntry(client->instance, tpath, nullptr, isDir, isDir ? client->m_watchModes : KDirWatch::WatchDirOnly); } if (!clients.isEmpty()) { emitEvent(e, Created, tpath); qCDebug(KDIRWATCH).nospace() << clients.count() << " instance(s) monitoring the new " << (isDir ? "dir " : "file ") << tpath; } } } break; default: break; } } #else void KDirWatchPrivate::famEventReceived() { qCWarning(KCOREADDONS_DEBUG) << "Fam event received but FAM is not supported"; } #endif void KDirWatchPrivate::statistics() { EntryMap::Iterator it; qCDebug(KDIRWATCH) << "Entries watched:"; if (m_mapEntries.count() == 0) { qCDebug(KDIRWATCH) << " None."; } else { it = m_mapEntries.begin(); for (; it != m_mapEntries.end(); ++it) { Entry *e = &(*it); qCDebug(KDIRWATCH) << " " << *e; for (const Client &c : e->m_clients) { QByteArray pending; if (c.watchingStopped) { if (c.pending & Deleted) { pending += "deleted "; } if (c.pending & Created) { pending += "created "; } if (c.pending & Changed) { pending += "changed "; } if (!pending.isEmpty()) { pending = " (pending: " + pending + ')'; } pending = ", stopped" + pending; } qCDebug(KDIRWATCH) << " by " << c.instance->objectName() << " (" << c.count << " times)" << pending; } if (!e->m_entries.isEmpty()) { qCDebug(KDIRWATCH) << " dependent entries:"; for (Entry *d : qAsConst(e->m_entries)) { qCDebug(KDIRWATCH) << " " << d << d->path << (d->m_status == NonExistent ? "NonExistent" : "EXISTS!!! ERROR!"); if (s_verboseDebug) { Q_ASSERT(d->m_status == NonExistent); // it doesn't belong here otherwise } } } } } } #if HAVE_QFILESYSTEMWATCHER // Slot for QFileSystemWatcher void KDirWatchPrivate::fswEventReceived(const QString &path) { if (s_verboseDebug) { qCDebug(KDIRWATCH) << path; } EntryMap::Iterator it = m_mapEntries.find(path); if (it != m_mapEntries.end()) { Entry *e = &(*it); e->dirty = true; const int ev = scanEntry(e); if (s_verboseDebug) { qCDebug(KDIRWATCH) << "scanEntry for" << e->path << "says" << ev; } if (ev != NoChange) { emitEvent(e, ev); } if (ev == Deleted) { if (e->isDir) { addEntry(nullptr, e->parentDirectory(), e, true); } else { addEntry(nullptr, QFileInfo(e->path).absolutePath(), e, true); } } else if (ev == Created) { // We were waiting for it to appear; now watch it addWatch(e); } else if (e->isDir) { // Check if any file or dir was created under this directory, that we were waiting for for (Entry *sub_entry : qAsConst(e->m_entries)) { fswEventReceived(sub_entry->path); // recurse, to call scanEntry and see if something changed } } else { /* Even though QFileSystemWatcher only reported the file as modified, it is possible that the file * was in fact just deleted and then immediately recreated. If the file was deleted, QFileSystemWatcher * will delete the watch, and will ignore the file, even after it is recreated. Since it is impossible * to reliably detect this case, always re-request the watch on a dirty signal, to avoid losing the * underlying OS monitor. */ fsWatcher->addPath(e->path); } } } #else void KDirWatchPrivate::fswEventReceived(const QString &path) { Q_UNUSED(path); qCWarning(KCOREADDONS_DEBUG) << "QFileSystemWatcher event received but QFileSystemWatcher is not supported"; } #endif // HAVE_QFILESYSTEMWATCHER // // Class KDirWatch // Q_GLOBAL_STATIC(KDirWatch, s_pKDirWatchSelf) KDirWatch *KDirWatch::self() { return s_pKDirWatchSelf(); } // is this used anywhere? // yes, see kio/src/core/kcoredirlister_p.h:328 bool KDirWatch::exists() { return s_pKDirWatchSelf.exists() && dwp_self.hasLocalData(); } static void postRoutine_KDirWatch() { if (s_pKDirWatchSelf.exists()) { s_pKDirWatchSelf()->deleteQFSWatcher(); } } KDirWatch::KDirWatch(QObject *parent) : QObject(parent), d(createPrivate()) { static QBasicAtomicInt nameCounter = Q_BASIC_ATOMIC_INITIALIZER(1); const int counter = nameCounter.fetchAndAddRelaxed(1); // returns the old value setObjectName(QStringLiteral("KDirWatch-%1").arg(counter)); if (counter == 1) { // very first KDirWatch instance // Must delete QFileSystemWatcher before qApp is gone - bug 261541 qAddPostRoutine(postRoutine_KDirWatch); } } KDirWatch::~KDirWatch() { if (d && dwp_self.hasLocalData()) { // skip this after app destruction d->removeEntries(this); } } void KDirWatch::addDir(const QString &_path, WatchModes watchModes) { if (d) { d->addEntry(this, _path, nullptr, true, watchModes); } } void KDirWatch::addFile(const QString &_path) { if (!d) { return; } d->addEntry(this, _path, nullptr, false); } QDateTime KDirWatch::ctime(const QString &_path) const { KDirWatchPrivate::Entry *e = d->entry(_path); if (!e) { return QDateTime(); } return QDateTime::fromSecsSinceEpoch(e->m_ctime); } void KDirWatch::removeDir(const QString &_path) { if (d) { d->removeEntry(this, _path, nullptr); } } void KDirWatch::removeFile(const QString &_path) { if (d) { d->removeEntry(this, _path, nullptr); } } bool KDirWatch::stopDirScan(const QString &_path) { if (d) { KDirWatchPrivate::Entry *e = d->entry(_path); if (e && e->isDir) { return d->stopEntryScan(this, e); } } return false; } bool KDirWatch::restartDirScan(const QString &_path) { if (d) { KDirWatchPrivate::Entry *e = d->entry(_path); if (e && e->isDir) // restart without notifying pending events { return d->restartEntryScan(this, e, false); } } return false; } void KDirWatch::stopScan() { if (d) { d->stopScan(this); d->_isStopped = true; } } bool KDirWatch::isStopped() { return d->_isStopped; } void KDirWatch::startScan(bool notify, bool skippedToo) { if (d) { d->_isStopped = false; d->startScan(this, notify, skippedToo); } } bool KDirWatch::contains(const QString &_path) const { KDirWatchPrivate::Entry *e = d->entry(_path); if (!e) { return false; } for (const KDirWatchPrivate::Client &client : e->m_clients) { if (client.instance == this) { return true; } } return false; } void KDirWatch::deleteQFSWatcher() { delete d->fsWatcher; d->fsWatcher = nullptr; d = nullptr; } void KDirWatch::statistics() { if (!dwp_self.hasLocalData()) { qCDebug(KDIRWATCH) << "KDirWatch not used"; return; } dwp_self.localData()->statistics(); } void KDirWatch::setCreated(const QString &_file) { qCDebug(KDIRWATCH) << objectName() << "emitting created" << _file; emit created(_file); } void KDirWatch::setDirty(const QString &_file) { //qCDebug(KDIRWATCH) << objectName() << "emitting dirty" << _file; emit dirty(_file); } void KDirWatch::setDeleted(const QString &_file) { qCDebug(KDIRWATCH) << objectName() << "emitting deleted" << _file; emit deleted(_file); } KDirWatch::Method KDirWatch::internalMethod() const { // This reproduces the logic in KDirWatchPrivate::addWatch switch (d->m_preferredMethod) { #if HAVE_FAM case KDirWatch::FAM: if (d->use_fam) { return KDirWatch::FAM; } break; #endif #if HAVE_SYS_INOTIFY_H case KDirWatch::INotify: if (d->supports_inotify) { return KDirWatch::INotify; } break; #endif #if HAVE_QFILESYSTEMWATCHER case KDirWatch::QFSWatch: return KDirWatch::QFSWatch; #endif case KDirWatch::Stat: return KDirWatch::Stat; } #if HAVE_SYS_INOTIFY_H if (d->supports_inotify) { return KDirWatch::INotify; } #endif #if HAVE_FAM if (d->use_fam) { return KDirWatch::FAM; } #endif #if HAVE_QFILESYSTEMWATCHER return KDirWatch::QFSWatch; #else return KDirWatch::Stat; #endif } #include "moc_kdirwatch.cpp" #include "moc_kdirwatch_p.cpp" //sven