diff --git a/kcms/ksplash/kcm.h b/kcms/ksplash/kcm.h
--- a/kcms/ksplash/kcm.h
+++ b/kcms/ksplash/kcm.h
@@ -34,6 +34,7 @@
     Q_PROPERTY(QStandardItemModel *splashModel READ splashModel CONSTANT)
     Q_PROPERTY(QString selectedPlugin READ selectedPlugin WRITE setSelectedPlugin NOTIFY selectedPluginChanged)
     Q_PROPERTY(int selectedPluginIndex READ selectedPluginIndex NOTIFY selectedPluginIndexChanged)
+    Q_PROPERTY(bool testing READ testing NOTIFY testingChanged)
 
 public:
     enum Roles {
@@ -52,6 +53,8 @@
 
     int selectedPluginIndex() const;
 
+    bool testing() const;
+
     void loadModel();
 
 public Q_SLOTS:
@@ -65,11 +68,16 @@
     void selectedPluginChanged();
     void selectedPluginIndexChanged();
 
+    void testingChanged();
+    void testingFailed();
+
 private:
     QStandardItemModel *m_model;
     Plasma::Package m_package;
     QString m_selectedPlugin;
 
+    QProcess *m_testProcess = nullptr;
+
     KConfig m_config;
     KConfigGroup m_configGroup;
 };
diff --git a/kcms/ksplash/kcm.cpp b/kcms/ksplash/kcm.cpp
--- a/kcms/ksplash/kcm.cpp
+++ b/kcms/ksplash/kcm.cpp
@@ -29,7 +29,6 @@
 
 #include <QVBoxLayout>
 #include <QPushButton>
-#include <QMessageBox>
 #include <QStandardItemModel>
 #include <QQmlContext>
 #include <QDir>
@@ -195,18 +194,34 @@
     return -1;
 }
 
+bool KCMSplashScreen::testing() const
+{
+    return m_testProcess;
+}
+
 void KCMSplashScreen::test(const QString &plugin)
 {
-    if (plugin.isEmpty() || plugin == QLatin1String("None")) {
+    if (plugin.isEmpty() || plugin == QLatin1String("None") || m_testProcess) {
         return;
     }
 
-    QProcess proc;
-    QStringList arguments;
-    arguments << plugin << QStringLiteral("--test");
-    if (proc.execute(QStringLiteral("ksplashqml"), arguments)) {
-        QMessageBox::critical(nullptr, i18n("Error"), i18n("Failed to successfully test the splash screen."));
-    }
+    m_testProcess = new QProcess(this);
+    connect(m_testProcess, &QProcess::errorOccurred, this, [this](QProcess::ProcessError error) {
+        Q_UNUSED(error);
+        emit testingFailed();
+    });
+    connect(m_testProcess, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished), this,
+        [this](int exitCode, QProcess::ExitStatus exitStatus) {
+        Q_UNUSED(exitCode);
+        Q_UNUSED(exitStatus);
+
+        m_testProcess->deleteLater();
+        m_testProcess = nullptr;
+        emit testingChanged();
+    });
+
+    emit testingChanged();
+    m_testProcess->start(QStringLiteral("ksplashqml"), {plugin, QStringLiteral("--test")});
 }
 
 #include "kcm.moc"
diff --git a/kcms/ksplash/package/contents/ui/main.qml b/kcms/ksplash/package/contents/ui/main.qml
--- a/kcms/ksplash/package/contents/ui/main.qml
+++ b/kcms/ksplash/package/contents/ui/main.qml
@@ -26,9 +26,28 @@
 KCM.GridViewKCM {
     KCM.ConfigModule.quickHelp: i18n("This module lets you choose the splash screen theme.")
 
+    enabled: !kcm.testing
+
     view.model: kcm.splashModel
     //NOTE: pay attention to never break this binding
     view.currentIndex: kcm.selectedPluginIndex
+
+    // putting the InlineMessage as header item causes it to show up initially despite visible false
+    header: ColumnLayout {
+        Kirigami.InlineMessage {
+            id: testingFailedLabel
+            Layout.fillWidth: true
+            showCloseButton: true
+            type: Kirigami.MessageType.Error
+            text: i18n("Failed to test the splash screen.")
+
+            Connections {
+                target: kcm
+                onTestingFailed: testingFailedLabel.visible = true
+            }
+        }
+    }
+
     view.delegate: KCM.GridDelegate {
         id: delegate