diff --git a/autotests/integration/pointer_input.cpp b/autotests/integration/pointer_input.cpp
--- a/autotests/integration/pointer_input.cpp
+++ b/autotests/integration/pointer_input.cpp
@@ -829,7 +829,8 @@
     Cursor::setPos(800, 800);
     auto p = input()->pointer();
     // at the moment it should be the fallback cursor
-    QVERIFY(!p->cursorImage().isNull());
+    const QImage fallbackCursor = p->cursorImage();
+    QVERIFY(!fallbackCursor.isNull());
 
     // create a window
     QSignalSpy clientAddedSpy(waylandServer(), &WaylandServer::shellClientAdded);
@@ -843,10 +844,10 @@
     AbstractClient *window = workspace()->activeClient();
     QVERIFY(window);
 
-    // move cursor to center of window, this should first set a null pointer
+    // move cursor to center of window, this should first set a null pointer, so we still show old cursor
     Cursor::setPos(window->geometry().center());
     QCOMPARE(p->window().data(), window);
-    QVERIFY(p->cursorImage().isNull());
+    QCOMPARE(p->cursorImage(), fallbackCursor);
     QVERIFY(enteredSpy.wait());
 
     // create a cursor on the pointer
@@ -889,6 +890,7 @@
     Cursor::setPos(window->geometry().bottomLeft() + QPoint(20, 20));
     QVERIFY(p->window().isNull());
     QVERIFY(!p->cursorImage().isNull());
+    QCOMPARE(p->cursorImage(), fallbackCursor);
 }
 
 class HelperEffect : public Effect
@@ -934,8 +936,8 @@
     QVERIFY(!window->geometry().contains(QPoint(800, 800)));
     Cursor::setPos(window->geometry().center());
     QVERIFY(enteredSpy.wait());
-    // cursor image should be null
-    QVERIFY(p->cursorImage().isNull());
+    // cursor image should still be fallback
+    QCOMPARE(p->cursorImage(), fallback);
 
     // now create an effect and set an override cursor
     QScopedPointer<HelperEffect> effect(new HelperEffect);
diff --git a/autotests/integration/scene_qpainter_test.cpp b/autotests/integration/scene_qpainter_test.cpp
--- a/autotests/integration/scene_qpainter_test.cpp
+++ b/autotests/integration/scene_qpainter_test.cpp
@@ -182,12 +182,11 @@
     if (frameRenderedSpy.isEmpty()) {
         QVERIFY(frameRenderedSpy.wait());
     }
-    // we didn't set a cursor image on the surface yet, so it should be just black + window
+    // we didn't set a cursor image on the surface yet, so it should be just black + window and previous cursor
     QImage referenceImage(QSize(1280, 1024), QImage::Format_RGB32);
     referenceImage.fill(Qt::black);
     QPainter painter(&referenceImage);
     painter.fillRect(0, 0, 200, 300, Qt::blue);
-    QCOMPARE(referenceImage, *scene->qpainterRenderBuffer());
 
     // now let's set a cursor image
     QScopedPointer<Surface> cs(Test::createSurface());
@@ -215,6 +214,8 @@
     QScopedPointer<Surface> s(Test::createSurface());
     QScopedPointer<ShellSurface> ss(Test::createShellSurface(s.data()));
     QScopedPointer<Pointer> p(Test::waylandSeat()->createPointer());
+    QSignalSpy pointerEnteredSpy(p.data(), &Pointer::entered);
+    QVERIFY(pointerEnteredSpy.isValid());
 
     auto scene = KWin::Compositor::self()->scene();
     QVERIFY(scene);
@@ -225,7 +226,6 @@
     QScopedPointer<Surface> cs(Test::createSurface());
     QVERIFY(!cs.isNull());
     Test::render(cs.data(), QSize(10, 10), Qt::red);
-    p->setCursor(cs.data(), QPoint(5, 5));
 
     // now let's map the window
     s->setScale(2);
@@ -239,12 +239,11 @@
 
     //add buffer
     Test::render(s.data(), img);
-    Test::waitForWaylandWindowShown();
+    QVERIFY(pointerEnteredSpy.wait());
+    p->setCursor(cs.data(), QPoint(5, 5));
 
     // which should trigger a frame
-    if (frameRenderedSpy.isEmpty()) {
-        QVERIFY(frameRenderedSpy.wait());
-    }
+    QVERIFY(frameRenderedSpy.wait());
     QImage referenceImage(QSize(1280, 1024), QImage::Format_RGB32);
     referenceImage.fill(Qt::black);
     QPainter painter(&referenceImage);
diff --git a/pointer_input.cpp b/pointer_input.cpp
--- a/pointer_input.cpp
+++ b/pointer_input.cpp
@@ -455,6 +455,8 @@
     return false;
 }
 
+static bool s_cursorUpdateBlocking = false;
+
 void PointerInputRedirection::update()
 {
     if (!m_inited) {
@@ -517,7 +519,9 @@
         m_window = QPointer<Toplevel>(t);
         // TODO: add convenient API to update global pos together with updating focused surface
         warpXcbOnSurfaceLeft(t->surface());
+        s_cursorUpdateBlocking = true;
         seat->setFocusedPointerSurface(nullptr);
+        s_cursorUpdateBlocking = false;
         seat->setPointerPos(m_pos.toPoint());
         seat->setFocusedPointerSurface(t->surface(), t->inputTransformation());
         m_windowGeometryConnection = connect(t, &Toplevel::geometryChanged, this,
@@ -964,15 +968,18 @@
 
 void CursorImage::update()
 {
+    if (s_cursorUpdateBlocking) {
+        return;
+    }
     using namespace KWayland::Server;
     disconnect(m_serverCursor.connection);
     auto p = waylandServer()->seat()->focusedPointer();
     if (p) {
         m_serverCursor.connection = connect(p, &PointerInterface::cursorChanged, this, &CursorImage::updateServerCursor);
     } else {
         m_serverCursor.connection = QMetaObject::Connection();
+        reevaluteSource();
     }
-    updateServerCursor();
 }
 
 void CursorImage::updateDecoration()
@@ -1238,7 +1245,7 @@
         setSource(CursorSource::Decoration);
         return;
     }
-    if (!m_pointer->window().isNull()) {
+    if (!m_pointer->window().isNull() && waylandServer()->seat()->focusedPointer()) {
         setSource(CursorSource::PointerSurface);
         return;
     }