diff --git a/autotests/unit/file/filewatchtest.cpp b/autotests/unit/file/filewatchtest.cpp
index e288741d..513760bb 100644
--- a/autotests/unit/file/filewatchtest.cpp
+++ b/autotests/unit/file/filewatchtest.cpp
@@ -1,232 +1,231 @@
 /*
  * <one line to give the library's name and an idea of what it does.>
  * Copyright (C) 2014  Vishesh Handa <me@vhanda.in>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  *
  */
 
 #include "filewatch.h"
 #include "fileindexerconfigutils.h"
 #include "database.h"
 #include "fileindexerconfig.h"
 #include "pendingfilequeue.h"
 
 #include <QTest>
 #include <QSignalSpy>
 #include <QTemporaryDir>
 #include <KFileMetaData/UserMetaData>
 
 namespace Baloo {
 
 class FileWatchTest : public QObject
 {
     Q_OBJECT
 private Q_SLOTS:
 
     void testFileCreation();
     void testConfigChange();
 };
 
 }
 
 using namespace Baloo;
 
 namespace {
     bool createFile(const QString& fileUrl) {
         QFile f1(fileUrl);
         f1.open(QIODevice::WriteOnly);
         f1.close();
         return QFile::exists(fileUrl);
     }
 
     void modifyFile(const QString& fileUrl) {
         QFile f1(fileUrl);
         f1.open(QIODevice::Append | QIODevice::Text);
 
         QTextStream stream(&f1);
         stream << "1";
     }
 }
 
 void FileWatchTest::testFileCreation()
 {
     QTemporaryDir includeDir;
 
     QStringList includeFolders;
     includeFolders << includeDir.path();
 
     QStringList excludeFolders;
     Test::writeIndexerConfig(includeFolders, excludeFolders);
 
     QTemporaryDir dbDir;
     Database db(dbDir.path());
     db.open(Baloo::Database::CreateDatabase);
 
     FileIndexerConfig config;
 
     FileWatch fileWatch(&db, &config);
     fileWatch.m_pendingFileQueue->setMaximumTimeout(0);
     fileWatch.m_pendingFileQueue->setMinimumTimeout(0);
     fileWatch.m_pendingFileQueue->setTrackingTime(0);
 
     QSignalSpy spy(&fileWatch, SIGNAL(installedWatches()));
     QVERIFY(spy.isValid());
 
     fileWatch.watchIndexedFolders();
     QVERIFY(spy.count() || spy.wait());
 
     QSignalSpy spyIndexNew(&fileWatch, SIGNAL(indexNewFile(QString)));
     QSignalSpy spyIndexModified(&fileWatch, SIGNAL(indexModifiedFile(QString)));
     QSignalSpy spyIndexXattr(&fileWatch, SIGNAL(indexXAttr(QString)));
 
     QVERIFY(spyIndexNew.isValid());
     QVERIFY(spyIndexModified.isValid());
     QVERIFY(spyIndexXattr.isValid());
 
     // Create a file and see if it is indexed
     QString fileUrl(includeDir.path() + "/t1");
     QVERIFY(createFile(fileUrl));
 
     QVERIFY(spyIndexNew.wait());
     QCOMPARE(spyIndexNew.count(), 1);
     QCOMPARE(spyIndexModified.count(), 0);
     QCOMPARE(spyIndexXattr.count(), 0);
 
     spyIndexNew.clear();
     spyIndexModified.clear();
     spyIndexXattr.clear();
 
     //
     // Modify the file
     //
     modifyFile(fileUrl);
 
     QVERIFY(spyIndexModified.wait());
     QCOMPARE(spyIndexNew.count(), 0);
     QCOMPARE(spyIndexModified.count(), 1);
     QCOMPARE(spyIndexXattr.count(), 0);
 
     spyIndexNew.clear();
     spyIndexModified.clear();
     spyIndexXattr.clear();
 
     //
     // Set an Xattr
     //
     KFileMetaData::UserMetaData umd(fileUrl);
     if (!umd.isSupported()) {
         qWarning() << "Xattr not supported on this filesystem:" << fileUrl;
         return;
     }
 
     const QString userComment(QStringLiteral("UserComment"));
     QVERIFY(umd.setUserComment(userComment) == KFileMetaData::UserMetaData::NoError);
 
     QVERIFY(spyIndexXattr.wait());
     QCOMPARE(spyIndexNew.count(), 0);
     QCOMPARE(spyIndexModified.count(), 0);
     QCOMPARE(spyIndexXattr.count(), 1);
 
     spyIndexNew.clear();
     spyIndexModified.clear();
     spyIndexXattr.clear();
 }
 
 void FileWatchTest::testConfigChange()
 {
     QTemporaryDir tmpDir;
 
     Database db(tmpDir.path());
     db.open(Baloo::Database::CreateDatabase);
 
     QString d1 = tmpDir.path() + QStringLiteral("/d1");
     QString d2 = tmpDir.path() + QStringLiteral("/d2");
     QString d11 = tmpDir.path() + QStringLiteral("/d1/d11");
     QString d21 = tmpDir.path() + QStringLiteral("/d2/d21");
     QString d22 = tmpDir.path() + QStringLiteral("/d2/d22");
 
     QDir().mkpath(d11);
     QDir().mkpath(d21);
     QDir().mkpath(d22);
 
     // parameters: includeFolders list, excludeFolders list
     Test::writeIndexerConfig({d1, d2}, {d11, d21});
     FileIndexerConfig config;
 
     FileWatch fileWatch(&db, &config);
     fileWatch.m_pendingFileQueue->setMaximumTimeout(0);
     fileWatch.m_pendingFileQueue->setMinimumTimeout(0);
     fileWatch.m_pendingFileQueue->setTrackingTime(0);
 
     QSignalSpy spy(&fileWatch, SIGNAL(installedWatches()));
     QVERIFY(spy.isValid());
 
     fileWatch.updateIndexedFoldersWatches();
     QVERIFY(spy.count() || spy.wait());
 
     QSignalSpy spyIndexNew(&fileWatch, SIGNAL(indexNewFile(QString)));
     QVERIFY(spyIndexNew.isValid());
     QVERIFY(createFile(d1 + "/t1"));
     QVERIFY(createFile(d2 + "/t2"));
 
     QVERIFY(spyIndexNew.wait());
     QCOMPARE(spyIndexNew.count(), 2);
     spyIndexNew.clear();
 
     // dir d22 is not yet excluded, so one event is expected
     QVERIFY(createFile(d11 + "/tx1"));
     QVERIFY(createFile(d21 + "/tx2"));
     QVERIFY(createFile(d22 + "/tx3"));
 
     QVERIFY(spyIndexNew.wait());
     QCOMPARE(spyIndexNew.count(), 1);
     QList<QVariant> event = spyIndexNew.at(0);
     QCOMPARE(event.at(0).toString(), d22 + "/tx3");
     spyIndexNew.clear();
 
     Test::writeIndexerConfig({d2}, {d22});
     config.forceConfigUpdate();
     fileWatch.updateIndexedFoldersWatches();
 
     // dir d1 is no longer included
     QVERIFY(createFile(d1 + "/tx1a"));
     QVERIFY(createFile(d2 + "/tx2a"));
     QVERIFY(spyIndexNew.wait());
     QList<QString> result;
     for (const QList<QVariant>& event : qAsConst(spyIndexNew)) {
 	result.append(event.at(0).toString());
     }
-    QEXPECT_FAIL("", "Removal of included folders not deteced", Continue);
     QCOMPARE(result, {d2 + "/tx2a"});
     spyIndexNew.clear();
     result.clear();
 
     // d11 is implicitly excluded, as d1 is no longer included
     // d22 is explicitly excluded now, d21 is included
     QVERIFY(createFile(d11 + "/tx1b"));
     QVERIFY(createFile(d21 + "/tx2b"));
     QVERIFY(createFile(d22 + "/tx3b"));
 
     QVERIFY(spyIndexNew.wait(500));
     for (const QList<QVariant>& event : qAsConst(spyIndexNew)) {
 	result.append(event.at(0).toString());
     }
     QCOMPARE(result, {d21 + "/tx2b"});
 }
 
 QTEST_MAIN(FileWatchTest)
 
 #include "filewatchtest.moc"
diff --git a/src/file/filewatch.cpp b/src/file/filewatch.cpp
index f5ea30bd..4c03fcf1 100644
--- a/src/file/filewatch.cpp
+++ b/src/file/filewatch.cpp
@@ -1,207 +1,231 @@
 /* This file is part of the KDE Project
    Copyright (c) 2007-2011 Sebastian Trueg <trueg@kde.org>
    Copyright (c) 2012-2014 Vishesh Handa <me@vhanda.in>
 
    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
    License version 2 as published by the Free Software Foundation.
 
    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Library General Public License for more details.
 
    You should have received a copy of the GNU Library General Public License
    along with this library; see the file COPYING.LIB.  If not, write to
    the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
    Boston, MA 02110-1301, USA.
 */
 
 #include "filewatch.h"
 #include "metadatamover.h"
 #include "fileindexerconfig.h"
 #include "pendingfilequeue.h"
 #include "regexpcache.h"
 #include "database.h"
 #include "pendingfile.h"
 #include "baloodebug.h"
 
 #include "kinotify.h"
 
 #include <QDateTime>
 #include <QFileInfo>
 #include <QDir>
 
 #include <syslog.h>
 
 using namespace Baloo;
 
 FileWatch::FileWatch(Database* db, FileIndexerConfig* config, QObject* parent)
     : QObject(parent)
     , m_db(db)
     , m_config(config)
     , m_dirWatch(nullptr)
 {
     Q_ASSERT(db);
     Q_ASSERT(config);
 
     m_metadataMover = new MetadataMover(m_db, this);
     connect(m_metadataMover, &MetadataMover::movedWithoutData, this, &FileWatch::indexNewFile);
     connect(m_metadataMover, &MetadataMover::fileRemoved, this, &FileWatch::fileRemoved);
 
     m_pendingFileQueue = new PendingFileQueue(this);
     connect(m_pendingFileQueue, &PendingFileQueue::indexNewFile, this, &FileWatch::indexNewFile);
     connect(m_pendingFileQueue, &PendingFileQueue::indexModifiedFile, this, &FileWatch::indexModifiedFile);
     connect(m_pendingFileQueue, &PendingFileQueue::indexXAttr, this, &FileWatch::indexXAttr);
     connect(m_pendingFileQueue, &PendingFileQueue::removeFileIndex, m_metadataMover, &MetadataMover::removeFileMetadata);
 
     // monitor the file system for changes (restricted by the inotify limit)
     m_dirWatch = new KInotify(m_config, this);
 
     connect(m_dirWatch, &KInotify::moved, this, &FileWatch::slotFileMoved);
     connect(m_dirWatch, &KInotify::deleted, this, &FileWatch::slotFileDeleted);
     connect(m_dirWatch, &KInotify::created, this, &FileWatch::slotFileCreated);
     connect(m_dirWatch, &KInotify::modified, this, &FileWatch::slotFileModified);
     connect(m_dirWatch, &KInotify::closedWrite, this, &FileWatch::slotFileClosedAfterWrite);
     connect(m_dirWatch, &KInotify::attributeChanged, this, &FileWatch::slotAttributeChanged);
     connect(m_dirWatch, &KInotify::watchUserLimitReached, this, &FileWatch::slotInotifyWatchUserLimitReached);
     connect(m_dirWatch, &KInotify::installedWatches, this, &FileWatch::installedWatches);
 }
 
 
 FileWatch::~FileWatch()
 {
 }
 
 void FileWatch::watchIndexedFolders()
 {
     // Watch all indexed folders
     const QStringList folders = m_config->includeFolders();
     for (const QString& folder : folders) {
         watchFolder(folder);
     }
 }
 
 // FIXME: listen to Create for folders!
 void FileWatch::watchFolder(const QString& path)
 {
-    qCDebug(BALOO) << path;
     if (m_dirWatch && !m_dirWatch->watchingPath(path)) {
         KInotify::WatchEvents flags(KInotify::EventMove | KInotify::EventDelete | KInotify::EventDeleteSelf
                                     | KInotify::EventCloseWrite | KInotify::EventCreate
                                     | KInotify::EventAttributeChange | KInotify::EventModify);
 
         m_dirWatch->addWatch(path, flags, KInotify::FlagOnlyDir);
     }
 }
 
 void FileWatch::slotFileMoved(const QString& urlFrom, const QString& urlTo)
 {
     if (m_config->shouldBeIndexed(urlTo)) {
         m_metadataMover->moveFileMetadata(urlFrom, urlTo);
     } else {
         QFileInfo src(urlFrom);
         QString url = urlFrom;
 
         if (src.isDir()) {
             Q_ASSERT(!url.endsWith('/'));
             url.append(QLatin1Char('/'));
         }
 
         PendingFile file(url);
         file.setDeleted();
 
         m_pendingFileQueue->enqueue(file);
     }
 }
 
 
 void FileWatch::slotFileDeleted(const QString& urlString, bool isDir)
 {
     // Directories must always end with a trailing slash '/'
     QString url = urlString;
     if (isDir) {
         Q_ASSERT(!url.endsWith('/'));
         url.append(QLatin1Char('/'));
     }
 
     PendingFile file(url);
     file.setDeleted();
 
     m_pendingFileQueue->enqueue(file);
 }
 
 
 void FileWatch::slotFileCreated(const QString& path, bool isDir)
 {
     Q_UNUSED(isDir);
     PendingFile file(path);
     file.setCreated();
 
     m_pendingFileQueue->enqueue(file);
 }
 
 void FileWatch::slotFileModified(const QString& path)
 {
     PendingFile file(path);
     file.setModified();
 
     //qCDebug(BALOO) << "MOD" << path;
     m_pendingFileQueue->enqueue(file);
 }
 
 void FileWatch::slotFileClosedAfterWrite(const QString& path)
 {
     QDateTime current = QDateTime::currentDateTime();
     QDateTime fileModification = QFileInfo(path).lastModified();
     QDateTime dirModification = QFileInfo(QFileInfo(path).absoluteDir().absolutePath()).lastModified();
 
     // If we have received a FileClosedAfterWrite event, then the file must have been
     // closed within the last minute.
     // This is being done cause many applications open the file under write mode, do not
     // make any modifications and then close the file. This results in us getting
     // the FileClosedAfterWrite event
     if (fileModification.secsTo(current) <= 1000 * 60 || dirModification.secsTo(current) <= 1000 * 60) {
         PendingFile file(path);
         file.setClosedOnWrite();
         //qCDebug(BALOO) << "CLOSE" << path;
         m_pendingFileQueue->enqueue(file);
     }
 }
 
 void FileWatch::slotAttributeChanged(const QString& path)
 {
     PendingFile file(path);
     file.setAttributeChanged();
 
     m_pendingFileQueue->enqueue(file);
 }
 
 // This slot is connected to a signal emitted in KInotify when
 // inotify_add_watch fails with ENOSPC.
 void FileWatch::slotInotifyWatchUserLimitReached(const QString&)
 {
     //If we got here, we hit the limit and couldn't authenticate to raise it,
     // so put something in the syslog so someone notices.
     syslog(LOG_USER | LOG_WARNING, "KDE Baloo File Indexer has reached the inotify folder watch limit. File changes will be ignored.");
     // we do it the brutal way for now hoping with new kernels and defaults this will never happen
     // Delete the KInotify
     // FIXME: Maybe we should be aborting?
     if (m_dirWatch) {
         m_dirWatch->deleteLater();
         m_dirWatch = nullptr;
     }
     Q_EMIT installedWatches();
 }
 
 void FileWatch::updateIndexedFoldersWatches()
 {
     if (m_dirWatch) {
-        const QStringList folders = m_config->includeFolders();
-        for (const QString& folder : folders) {
-            m_dirWatch->removeWatch(folder);
-            watchFolder(folder);
+        const QStringList excludedFolders = m_config->excludeFolders();
+        const QStringList includedFolders = m_config->includeFolders();
+
+        for (const QString& folder : excludedFolders) {
+            // Remove watch for new excluded folders
+            if (!m_excludedFolders.contains(folder)) {
+                m_dirWatch->removeWatch(folder);
+            }
         }
+        for (const QString& folder : m_excludedFolders) {
+            // Add no longer excluded folders
+            if (!excludedFolders.contains(folder)) {
+                watchFolder(folder);
+            }
+        }
+
+        for (const QString& folder : m_includedFolders) {
+            // Remove no longer included folders
+            if (!includedFolders.contains(folder)) {
+                m_dirWatch->removeWatch(folder);
+            }
+        }
+        for (const QString& folder : includedFolders) {
+            if (!m_includedFolders.contains(folder)) {
+                watchFolder(folder);
+            }
+        }
+
+        m_excludedFolders = excludedFolders;
+        m_includedFolders = includedFolders;
     }
 }
 
diff --git a/src/file/filewatch.h b/src/file/filewatch.h
index 568e892c..3d596968 100644
--- a/src/file/filewatch.h
+++ b/src/file/filewatch.h
@@ -1,93 +1,96 @@
 /* This file is part of the KDE Project
    Copyright (c) 2007-2011 Sebastian Trueg <trueg@kde.org>
 
    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
    License version 2 as published by the Free Software Foundation.
 
    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Library General Public License for more details.
 
    You should have received a copy of the GNU Library General Public License
    along with this library; see the file COPYING.LIB.  If not, write to
    the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
    Boston, MA 02110-1301, USA.
 */
 
 #ifndef BALOO_FILE_WATCH_H_
 #define BALOO_FILE_WATCH_H_
 
 #include <QObject>
 #include "pendingfile.h"
 
 class KInotify;
 
 namespace Baloo
 {
 class Database;
 class MetadataMover;
 class FileIndexerConfig;
 class PendingFileQueue;
 class FileWatchTest;
 
 class FileWatch : public QObject
 {
     Q_OBJECT
 
 public:
     FileWatch(Database* db, FileIndexerConfig* config, QObject* parent = nullptr);
     ~FileWatch();
 
 public Q_SLOTS:
     /**
      * To be called whenever the list of indexed folders changes. This is done because
      * the indexed folders are watched with the 'KInotify::EventCreate' event, and the
      * non-indexed folders are not.
      */
     void updateIndexedFoldersWatches();
 
     void watchIndexedFolders();
 
 Q_SIGNALS:
     void indexNewFile(const QString& string);
     void indexModifiedFile(const QString& string);
     void indexXAttr(const QString& path);
 
     /**
      * This signal is emitted when a file has been removed, and everyone else
      * should update their caches
      */
     void fileRemoved(const QString& path);
 
     void installedWatches();
 
 private Q_SLOTS:
     void slotFileMoved(const QString& from, const QString& to);
     void slotFileDeleted(const QString& urlString, bool isDir);
     void slotFileCreated(const QString& path, bool isDir);
     void slotFileClosedAfterWrite(const QString&);
     void slotAttributeChanged(const QString& path);
     void slotFileModified(const QString& path);
     void slotInotifyWatchUserLimitReached(const QString&);
 
 private:
     /** Watch a folder, provided it is not already watched*/
     void watchFolder(const QString& path);
 
     Database* m_db;
 
     MetadataMover* m_metadataMover;
     FileIndexerConfig* m_config;
 
     KInotify* m_dirWatch;
 
     /// queue used to "compress" multiple file events like downloads
     PendingFileQueue* m_pendingFileQueue;
 
+    QStringList m_includedFolders;
+    QStringList m_excludedFolders;
+
     friend class FileWatchTest;
 };
 }
 
 #endif