diff --git a/shell/panelconfigview.cpp b/shell/panelconfigview.cpp
--- a/shell/panelconfigview.cpp
+++ b/shell/panelconfigview.cpp
@@ -92,7 +92,6 @@
     if (m_panelView) {
         m_panelView->setVisibilityMode(m_visibilityMode);
     }
-    PanelShadows::self()->removeWindow(this);
 }
 
 void PanelConfigView::init()
@@ -265,6 +264,7 @@
             case QPlatformSurfaceEvent::SurfaceAboutToBeDestroyed:
                 delete m_shellSurface;
                 m_shellSurface = nullptr;
+                PanelShadows::self()->removeWindow(this);
                 break;
             }
         }
diff --git a/shell/panelshadows.cpp b/shell/panelshadows.cpp
--- a/shell/panelshadows.cpp
+++ b/shell/panelshadows.cpp
@@ -627,6 +627,10 @@
 
 void PanelShadows::Private::clearShadow(const QWindow *window)
 {
+    if (!static_cast<const QSurface*>(window)->surfaceHandle()) {
+        qWarning() << "Cannot clear shadow from window without native surface!";
+        return;
+    }
 #if HAVE_X11
     if (m_isX11) {
         clearShadowX11(window);
diff --git a/shell/panelview.cpp b/shell/panelview.cpp
--- a/shell/panelview.cpp
+++ b/shell/panelview.cpp
@@ -116,7 +116,6 @@
     if (containment()) {
         m_corona->requestApplicationConfigSync();
     }
-    PanelShadows::self()->removeWindow(this);
 }
 
 KConfigGroup PanelView::panelConfig(ShellCorona *corona, Plasma::Containment *containment, QScreen *screen)
@@ -807,6 +806,7 @@
                 case QPlatformSurfaceEvent::SurfaceAboutToBeDestroyed:
                     delete m_shellSurface;
                     m_shellSurface = nullptr;
+                    PanelShadows::self()->removeWindow(this);
                     break;
                 }
             }