diff --git a/input.h b/input.h
--- a/input.h
+++ b/input.h
@@ -424,6 +424,13 @@
 
     virtual void focusUpdate(Toplevel *old, Toplevel *now) = 0;
 
+    /* Certain input devices can be in a state of having no valid
+     * position. An example are touch screens when no finger/pen
+     * is resting on the surface (no touch point).
+     */
+    virtual bool positionValid() const {
+        return false;
+    }
     virtual bool focusUpdatesBlocked() {
         return false;
     }
diff --git a/input.cpp b/input.cpp
--- a/input.cpp
+++ b/input.cpp
@@ -2265,16 +2265,19 @@
     if (!m_inited) {
         return;
     }
-    const auto pos = position().toPoint();
-    auto internalWindow = findInternalWindow(pos);
 
-    Toplevel *toplevel;
-    if (internalWindow) {
-        toplevel = waylandServer()->findClient(internalWindow);
-    } else {
-        toplevel = input()->findToplevel(pos);
-    }
+    Toplevel *toplevel = nullptr;
+    QWindow *internalWindow = nullptr;
 
+    if (!positionValid()) {
+        const auto pos = position().toPoint();
+        internalWindow = findInternalWindow(pos);
+        if (internalWindow) {
+            toplevel = waylandServer()->findClient(internalWindow);
+        } else {
+            toplevel = input()->findToplevel(pos);
+        }
+    }
     // Always set the toplevel at the position of the input device.
     setAt(toplevel);
 
diff --git a/touch_input.h b/touch_input.h
--- a/touch_input.h
+++ b/touch_input.h
@@ -50,6 +50,7 @@
     explicit TouchInputRedirection(InputRedirection *parent);
     virtual ~TouchInputRedirection();
 
+    bool positionValid() const override;
     bool focusUpdatesBlocked() override;
     void init();
 
diff --git a/touch_input.cpp b/touch_input.cpp
--- a/touch_input.cpp
+++ b/touch_input.cpp
@@ -78,13 +78,20 @@
     if (waylandServer()->seat()->isDragTouch()) {
         return true;
     }
-    if (m_touches > 0) {
+    if (m_touches > 1) {
         // first touch defines focus
         return true;
     }
     return false;
 }
 
+bool TouchInputRedirection::positionValid() const
+{
+    Q_ASSERT(m_touches >= 0);
+    // we can only determine a position with atleast one touch point
+    return m_touches == 0;
+}
+
 void TouchInputRedirection::focusUpdate(Toplevel *focusOld, Toplevel *focusNow)
 {
     // TODO: handle pointer grab aka popups
@@ -171,10 +178,10 @@
     }
     m_lastPosition = pos;
     m_windowUpdatedInCycle = false;
-    if (m_touches == 0) {
+    m_touches++;
+    if (m_touches == 1) {
         update();
     }
-    m_touches++;
     input()->processSpies(std::bind(&InputEventSpy::touchDown, std::placeholders::_1, id, pos, time));
     input()->processFilters(std::bind(&InputEventFilter::touchDown, std::placeholders::_1, id, pos, time));
     m_windowUpdatedInCycle = false;