diff --git a/autotests/libs/testrunner/setup.h b/autotests/libs/testrunner/setup.h
--- a/autotests/libs/testrunner/setup.h
+++ b/autotests/libs/testrunner/setup.h
@@ -78,9 +78,9 @@
     void copyDirectory(const QString &src, const QString &dst);
     void createTempEnvironment();
     void cleanTempEnvironment();
-    bool isSetupDone() const;
     void setupFailed();
     void writeAkonadiserverrc(const QString &path);
+    void checkSetupDone();
 
 private:
     KProcess *mAkonadiDaemonProcess = nullptr;
diff --git a/autotests/libs/testrunner/setup.cpp b/autotests/libs/testrunner/setup.cpp
--- a/autotests/libs/testrunner/setup.cpp
+++ b/autotests/libs/testrunner/setup.cpp
@@ -87,47 +87,43 @@
         job->start();
     }
 
-    if (isSetupDone()) {
-        Q_EMIT setupDone();
-    }
+    checkSetupDone();
 }
 
 void SetupTest::agentCreationResult(KJob *job)
 {
-    qDebug() << "Agent created";
     --mSetupJobCount;
     if (job->error()) {
-        qCritical() << job->errorString();
+        qCritical() << "Failed to create agent:" << job->errorString();
         setupFailed();
-        return;
-    }
-    const bool needsSync = job->property("sync").toBool();
-    if (needsSync) {
-        ++mSetupJobCount;
-        qDebug() << "Scheduling Agent sync";
-        Akonadi::ResourceSynchronizationJob *sync = new Akonadi::ResourceSynchronizationJob(
-            qobject_cast<Akonadi::AgentInstanceCreateJob *>(job)->instance(), this);
-        connect(sync, &Akonadi::ResourceSynchronizationJob::result, this, &SetupTest::synchronizationResult);
-        sync->start();
+    } else {
+        const bool needsSync = job->property("sync").toBool();
+        const auto instance = qobject_cast<Akonadi::AgentInstanceCreateJob *>(job)->instance();
+        qDebug() << "Agent" << instance.identifier() << "created";
+        if (needsSync) {
+            ++mSetupJobCount;
+            qDebug() << "Scheduling Agent sync of" << instance.identifier();
+            Akonadi::ResourceSynchronizationJob *sync = new Akonadi::ResourceSynchronizationJob(instance, this);
+            connect(sync, &Akonadi::ResourceSynchronizationJob::result, this, &SetupTest::synchronizationResult);
+            sync->start();
+        }
     }
 
-    if (isSetupDone()) {
-        Q_EMIT setupDone();
-    }
+    checkSetupDone();
 }
 
 void SetupTest::synchronizationResult(KJob *job)
 {
-    qDebug() << "Sync done";
+    auto instance = qobject_cast<Akonadi::ResourceSynchronizationJob*>(job)->resource();
+    qDebug() << "Sync of" << instance.identifier() << "done";
+
     --mSetupJobCount;
     if (job->error()) {
         qCritical() << job->errorString();
         setupFailed();
     }
 
-    if (isSetupDone()) {
-        Q_EMIT setupDone();
-    }
+    checkSetupDone();
 }
 
 void SetupTest::serverStateChanged(Akonadi::ServerManager::State state)
@@ -344,7 +340,7 @@
     case Akonadi::ServerManager::NotRunning:
     case Akonadi::ServerManager::Broken:
         shutdownHarder();
-        Q_FALLTHROUGH();
+        break;
     case Akonadi::ServerManager::Stopping:
         // safety timeout
         QTimer::singleShot(30 * 1000, this, &SetupTest::shutdownHarder);
@@ -429,16 +425,24 @@
     setEnvironmentVariable("AKONADI_INSTANCE", instanceId());
 }
 
-bool SetupTest::isSetupDone() const
+void SetupTest::checkSetupDone()
 {
-    qDebug() << "isSetupDone:" << mSetupJobCount << mExitCode;
-    return mSetupJobCount == 0 && mExitCode == 0;
+    qDebug() << "checkSetupDone: pendingJobs =" << mSetupJobCount << ", exitCode =" << mExitCode;
+    if (mSetupJobCount == 0) {
+        if (mExitCode != 0) {
+            qInfo() << "Setup has failed, aborting test.";
+            shutdown();
+        } else {
+            qInfo() << "Setup successful";
+            Q_EMIT setupDone();
+        }
+    }
+
 }
 
 void SetupTest::setupFailed()
 {
     mExitCode = 1;
-    shutdown();
 }
 
 void SetupTest::setEnvironmentVariable(const QByteArray &name, const QString &value)
diff --git a/autotests/libs/testrunner/testrunner.cpp b/autotests/libs/testrunner/testrunner.cpp
--- a/autotests/libs/testrunner/testrunner.cpp
+++ b/autotests/libs/testrunner/testrunner.cpp
@@ -36,7 +36,7 @@
 
 void TestRunner::run()
 {
-    qDebug() << mArguments;
+    qDebug() << "Starting test" << mArguments;
     mProcess = new KProcess(this);
     mProcess->setProgram(mArguments);
     connect(mProcess, QOverload<int>::of(&KProcess::finished), this, &TestRunner::processFinished);
diff --git a/src/akonadicontrol/agentmanager.cpp b/src/akonadicontrol/agentmanager.cpp
--- a/src/akonadicontrol/agentmanager.cpp
+++ b/src/akonadicontrol/agentmanager.cpp
@@ -617,7 +617,7 @@
     // This is called by the D-Bus server when a service comes up, goes down or changes ownership for some reason
     // and this is where we "hook up" our different Agent interfaces.
 
-    //qCDebug(AKONADICONTROL_LOG) << "Service " << name << " owner changed from " << oldOwner << " to " << newOwner;
+    qCDebug(AKONADICONTROL_LOG) << "Service" << name << "owner changed from" << oldOwner << "to" << newOwner;
 
     if ((name == Akonadi::DBus::serviceName(Akonadi::DBus::Server) || name == Akonadi::DBus::serviceName(Akonadi::DBus::AgentServer)) && !newOwner.isEmpty()) {
         if (QDBusConnection::sessionBus().interface()->isServiceRegistered(Akonadi::DBus::serviceName(Akonadi::DBus::Server))
@@ -648,7 +648,10 @@
             return;
         }
 
-        if (!restarting) {
+        Q_ASSERT(mAgents.contains(instance->agentType()));
+        const bool isResource = mAgents.value(instance->agentType()).capabilities.contains(AgentType::CapabilityResource);
+
+        if (!restarting && (!isResource || instance->hasResourceInterface())) {
             Q_EMIT agentInstanceAdded(service->serviceName);
         }
 
@@ -664,7 +667,15 @@
             return;
         }
 
-        mAgentInstances.value(service->serviceName)->obtainResourceInterface();
+        const AgentInstance::Ptr instance = mAgentInstances.value(service->serviceName);
+        const bool restarting = instance->hasResourceInterface();
+        if (!instance->obtainResourceInterface()) {
+            return;
+        }
+
+        if (!restarting && instance->hasAgentInterface()) {
+            Q_EMIT agentInstanceAdded(service->serviceName);
+        }
 
         break;
     }