diff --git a/autotests/unit/file/filewatchtest.cpp b/autotests/unit/file/filewatchtest.cpp
--- a/autotests/unit/file/filewatchtest.cpp
+++ b/autotests/unit/file/filewatchtest.cpp
@@ -209,7 +209,6 @@
     for (const QList<QVariant>& event : qAsConst(spyIndexNew)) {
 	result.append(event.at(0).toString());
     }
-    QEXPECT_FAIL("", "Removal of included folders not deteced", Continue);
     QCOMPARE(result, {d2 + "/tx2a"});
     spyIndexNew.clear();
     result.clear();
diff --git a/src/file/filewatch.h b/src/file/filewatch.h
--- a/src/file/filewatch.h
+++ b/src/file/filewatch.h
@@ -86,6 +86,9 @@
     /// queue used to "compress" multiple file events like downloads
     PendingFileQueue* m_pendingFileQueue;
 
+    QStringList m_includedFolders;
+    QStringList m_excludedFolders;
+
     friend class FileWatchTest;
 };
 }
diff --git a/src/file/filewatch.cpp b/src/file/filewatch.cpp
--- a/src/file/filewatch.cpp
+++ b/src/file/filewatch.cpp
@@ -85,7 +85,6 @@
 // FIXME: listen to Create for folders!
 void FileWatch::watchFolder(const QString& path)
 {
-    qCDebug(BALOO) << path;
     if (m_dirWatch && !m_dirWatch->watchingPath(path)) {
         KInotify::WatchEvents flags(KInotify::EventMove | KInotify::EventDelete | KInotify::EventDeleteSelf
                                     | KInotify::EventCloseWrite | KInotify::EventCreate
@@ -197,11 +196,36 @@
 void FileWatch::updateIndexedFoldersWatches()
 {
     if (m_dirWatch) {
-        const QStringList folders = m_config->includeFolders();
-        for (const QString& folder : folders) {
-            m_dirWatch->removeWatch(folder);
-            watchFolder(folder);
+        const QStringList excludedFolders = m_config->excludeFolders();
+        const QStringList includedFolders = m_config->includeFolders();
+
+        for (const QString& folder : excludedFolders) {
+            // Remove watch for new excluded folders
+            if (!m_excludedFolders.contains(folder)) {
+                m_dirWatch->removeWatch(folder);
+            }
         }
+        for (const QString& folder : m_excludedFolders) {
+            // Add no longer excluded folders
+            if (!excludedFolders.contains(folder)) {
+                watchFolder(folder);
+            }
+        }
+
+        for (const QString& folder : m_includedFolders) {
+            // Remove no longer included folders
+            if (!includedFolders.contains(folder)) {
+                m_dirWatch->removeWatch(folder);
+            }
+        }
+        for (const QString& folder : includedFolders) {
+            if (!m_includedFolders.contains(folder)) {
+                watchFolder(folder);
+            }
+        }
+
+        m_excludedFolders = excludedFolders;
+        m_includedFolders = includedFolders;
     }
 }