diff --git a/applets/systemtray/package/contents/ui/main.qml b/applets/systemtray/package/contents/ui/main.qml
--- a/applets/systemtray/package/contents/ui/main.qml
+++ b/applets/systemtray/package/contents/ui/main.qml
@@ -96,6 +96,15 @@
     Containment.onAppletRemoved: {
     }
 
+    Connections {
+        target: plasmoid
+        onUserConfiguringChanged: {
+            if (plasmoid.userConfiguring) {
+                dialog.visible = false
+            }
+        }
+    }
+
      Connections {
         target: plasmoid.configuration