diff --git a/runners/services/autotests/fixtures/kdesystemsettings.desktop b/runners/services/autotests/fixtures/kdesystemsettings.desktop
new file mode 100644
--- /dev/null
+++ b/runners/services/autotests/fixtures/kdesystemsettings.desktop
@@ -0,0 +1,13 @@
+[Desktop Entry]
+Exec=systemsettings5
+Icon=preferences-system
+Type=Application
+X-KDE-StartupNotify=true
+NotShowIn=KDE;
+
+GenericName=KDE System Settings ServiceRunnerTest
+
+Name=KDE System Settings ServiceRunnerTest
+
+X-DBUS-StartupType=Unique
+Categories=Qt;KDE;Settings;
diff --git a/runners/services/autotests/fixtures/systemsettings.desktop b/runners/services/autotests/fixtures/systemsettings.desktop
new file mode 100644
--- /dev/null
+++ b/runners/services/autotests/fixtures/systemsettings.desktop
@@ -0,0 +1,14 @@
+[Desktop Entry]
+Exec=systemsettings5
+Icon=preferences-system
+Type=Application
+X-DocPath=systemsettings/index.html
+X-KDE-StartupNotify=true
+OnlyShowIn=KDE;
+
+GenericName=System Settings ServiceRunnerTest
+
+Name=System Settings ServiceRunnerTest
+
+X-DBUS-StartupType=Unique
+Categories=Qt;KDE;Settings;
diff --git a/runners/services/autotests/servicerunnertest.cpp b/runners/services/autotests/servicerunnertest.cpp
--- a/runners/services/autotests/servicerunnertest.cpp
+++ b/runners/services/autotests/servicerunnertest.cpp
@@ -38,6 +38,7 @@
 
     void testChromeAppsRelevance();
     void testKonsoleVsYakuakeComment();
+    void testSystemSettings();
 };
 
 void ServiceRunnerTest::initTestCase()
@@ -58,6 +59,11 @@
     setlocale(LC_ALL, "C.utf8");
 
     KSycoca::self()->ensureCacheValid();
+
+    // Make sure noDisplay behaves consistently WRT OnlyShowIn etc.
+    QVERIFY(setenv("XDG_CURRENT_DESKTOP", "KDE", 1) == 0);
+    // NOTE: noDisplay also includes X-KDE-OnlyShowOnQtPlatforms which is a bit harder to fake
+    //       and not currently under testing anyway.
 }
 
 void ServiceRunnerTest::cleanupTestCase()
@@ -123,6 +129,34 @@
     QVERIFY(yakuakeFound);
 }
 
+void ServiceRunnerTest::testSystemSettings()
+{
+    // In 5.9.0 'System Settings' suddenly didn't come back as a match for 'settings' anymore.
+    // Sytem Settings has a noKDE version and a KDE version, if the noKDE version is encountered
+    // first it will be added to the seen cache, however disqualification of already seen items
+    // may then also disqualify the KDE version of system settings on account of having already
+    // seen it. This test makes sure we find the right version.
+    ServiceRunner runner(this, QVariantList());
+    Plasma::RunnerContext context;
+    context.setQuery("settings");
+
+    runner.match(context);
+
+    bool systemSettingsFound = false;
+    bool foreignSystemSettingsFound = false;
+    for (auto match : context.matches()) {
+        qDebug() << "matched" << match.text();
+        if (match.text() == "System Settings ServiceRunnerTest") {
+            systemSettingsFound = true;
+        }
+        if (match.text() == "KDE System Settings ServiceRunnerTest") {
+            foreignSystemSettingsFound = true;
+        }
+    }
+    QVERIFY(systemSettingsFound);
+    QVERIFY(!foreignSystemSettingsFound);
+}
+
 QTEST_MAIN(ServiceRunnerTest)
 
 #include "servicerunnertest.moc"
diff --git a/runners/services/servicerunner.cpp b/runners/services/servicerunner.cpp
--- a/runners/services/servicerunner.cpp
+++ b/runners/services/servicerunner.cpp
@@ -76,7 +76,7 @@
 
     bool hasSeen(const KService::Ptr &service)
     {
-        return m_seen.contains(service->storageId()) ||
+        return m_seen.contains(service->storageId()) &&
                m_seen.contains(service->exec());
     }
 
@@ -88,6 +88,7 @@
     bool disqualify(const KService::Ptr &service)
     {
         auto ret = hasSeen(service) || service->noDisplay();
+        qCDebug(RUNNER_SERVICES) << service->name() << "disqualified?" << ret;
         seen(service);
         return ret;
     }