diff --git a/autotests/integration/pointer_constraints_test.cpp b/autotests/integration/pointer_constraints_test.cpp
--- a/autotests/integration/pointer_constraints_test.cpp
+++ b/autotests/integration/pointer_constraints_test.cpp
@@ -224,6 +224,16 @@
     QCOMPARE(input()->pointer()->isConstrained(), true);
     QVERIFY(confinedSpy2.wait());
 
+    // deactivate the client, this should unconfine
+    workspace()->activateClient(nullptr);
+    QVERIFY(unconfinedSpy2.wait());
+    QCOMPARE(input()->pointer()->isConstrained(), false);
+
+    // activate it again, this confines again
+    workspace()->activateClient(static_cast<AbstractClient*>(input()->pointer()->window().data()));
+    QVERIFY(confinedSpy2.wait());
+    QCOMPARE(input()->pointer()->isConstrained(), true);
+
     // create a second window and move it above our constrained window
     QScopedPointer<Surface> surface2(Test::createSurface());
     QScopedPointer<QObject> shellSurface2(Test::createShellSurface(type, surface2.data()));
diff --git a/pointer_input.h b/pointer_input.h
--- a/pointer_input.h
+++ b/pointer_input.h
@@ -165,6 +165,7 @@
     QMetaObject::Connection m_windowGeometryConnection;
     QMetaObject::Connection m_internalWindowConnection;
     QMetaObject::Connection m_constraintsConnection;
+    QMetaObject::Connection m_constraintsActivatedConnection;
     QMetaObject::Connection m_confinedPointerRegionConnection;
     QMetaObject::Connection m_decorationGeometryConnection;
     bool m_confined = false;
diff --git a/pointer_input.cpp b/pointer_input.cpp
--- a/pointer_input.cpp
+++ b/pointer_input.cpp
@@ -544,6 +544,8 @@
         );
         m_constraintsConnection = connect(m_window->surface(), &KWayland::Server::SurfaceInterface::pointerConstraintsChanged,
                                           this, &PointerInputRedirection::updatePointerConstraints);
+        m_constraintsActivatedConnection = connect(workspace(), &Workspace::clientActivated,
+                                                   this, &PointerInputRedirection::updatePointerConstraints);
         // check whether a pointer confinement/lock fires
         m_blockConstraint = false;
         updatePointerConstraints();
@@ -588,6 +590,9 @@
 {
     disconnect(m_constraintsConnection);
     m_constraintsConnection = QMetaObject::Connection();
+
+    disconnect(m_constraintsActivatedConnection);
+    m_constraintsActivatedConnection = QMetaObject::Connection();
 }
 
 template <typename T>
@@ -617,13 +622,19 @@
     if (m_blockConstraint) {
         return;
     }
+    const bool windowIsActive = m_window == workspace()->activeClient();
     const auto cf = s->confinedPointer();
     if (cf) {
         if (cf->isConfined()) {
+            if (!windowIsActive) {
+                cf->setConfined(false);
+                m_confined = false;
+                disconnectConfinedPointerRegionConnection();
+            }
             return;
         }
         const QRegion r = getConstraintRegion(m_window.data(), cf.data());
-        if (r.contains(m_pos.toPoint())) {
+        if (windowIsActive && r.contains(m_pos.toPoint())) {
             cf->setConfined(true);
             m_confined = true;
             m_confinedPointerRegionConnection = connect(cf.data(), &KWayland::Server::ConfinedPointerInterface::regionChanged, this,
@@ -660,10 +671,14 @@
     const auto lock = s->lockedPointer();
     if (lock) {
         if (lock->isLocked()) {
+            if (!windowIsActive) {
+                lock->setLocked(false);
+                m_locked = false;
+            }
             return;
         }
         const QRegion r = getConstraintRegion(m_window.data(), lock.data());
-        if (r.contains(m_pos.toPoint())) {
+        if (windowIsActive && r.contains(m_pos.toPoint())) {
             lock->setLocked(true);
             m_locked = true;
             OSD::show(i18nc("notification about mouse pointer locked",