diff --git a/effects/backgroundcontrast/contrast.h b/effects/backgroundcontrast/contrast.h
--- a/effects/backgroundcontrast/contrast.h
+++ b/effects/backgroundcontrast/contrast.h
@@ -73,16 +73,17 @@
 private:
     QRegion contrastRegion(const EffectWindow *w) const;
     bool shouldContrast(const EffectWindow *w, int mask, const WindowPaintData &data) const;
-    void updateContrastRegion(EffectWindow *w) const;
-    void doContrast(const QRegion &shape, const QRect &screen, const float opacity, const QMatrix4x4 &screenProjection);
+    void updateContrastRegion(EffectWindow *w);
+    void doContrast(EffectWindow *w, const QRegion &shape, const QRect &screen, const float opacity, const QMatrix4x4 &screenProjection);
     void uploadRegion(QVector2D *&map, const QRegion &region);
     void uploadGeometry(GLVertexBuffer *vbo, const QRegion &region);
 
 private:
     ContrastShader *shader;
     long net_wm_contrast_region;
     QRegion m_paintedArea; // actually painted area which is greater than m_damagedArea
     QRegion m_currentContrast; // keeps track of the currently contrasted area of non-caching windows(from bottom to top)
+    QHash< const EffectWindow*, QMatrix4x4> m_colorMatrices;
     QHash< const EffectWindow*, QMetaObject::Connection > m_contrastChangedConnections; // used only in Wayland to keep track of effect changed
     KWayland::Server::ContrastManagerInterface *m_contrastManager = nullptr;
 };
diff --git a/effects/backgroundcontrast/contrast.cpp b/effects/backgroundcontrast/contrast.cpp
--- a/effects/backgroundcontrast/contrast.cpp
+++ b/effects/backgroundcontrast/contrast.cpp
@@ -88,7 +88,7 @@
     }
 }
 
-void ContrastEffect::updateContrastRegion(EffectWindow *w) const
+void ContrastEffect::updateContrastRegion(EffectWindow *w)
 {
     QRegion region;
     float colorTransform[16];
@@ -112,14 +112,14 @@
         }
 
         QMatrix4x4 colorMatrix(colorTransform);
-        shader->setColorMatrix(colorMatrix);
+        m_colorMatrices[w] = colorMatrix;
     }
 
     KWayland::Server::SurfaceInterface *surf = w->surface();
 
     if (surf && surf->contrast()) {
         region = surf->contrast()->region();
-        shader->setColorMatrix(colorMatrix(surf->contrast()->contrast(), surf->contrast()->intensity(), surf->contrast()->saturation()));
+        m_colorMatrices[w] = colorMatrix(surf->contrast()->contrast(), surf->contrast()->intensity(), surf->contrast()->saturation());
     }
 
     //!value.isNull() full window in X11 case, surf->contrast()
@@ -153,6 +153,7 @@
     if (m_contrastChangedConnections.contains(w)) {
         disconnect(m_contrastChangedConnections[w]);
         m_contrastChangedConnections.remove(w);
+        m_colorMatrices.remove(w);
     }
 }
 
@@ -397,7 +398,7 @@
         }
 
         if (!shape.isEmpty()) {
-            doContrast(shape, screen, data.opacity(), data.screenProjectionMatrix());
+            doContrast(w, shape, screen, data.opacity(), data.screenProjectionMatrix());
         }
     }
 
@@ -411,7 +412,7 @@
     effects->paintEffectFrame(frame, region, opacity, frameOpacity);
 }
 
-void ContrastEffect::doContrast(const QRegion& shape, const QRect& screen, const float opacity, const QMatrix4x4 &screenProjection)
+void ContrastEffect::doContrast(EffectWindow *w, const QRegion& shape, const QRect& screen, const float opacity, const QMatrix4x4 &screenProjection)
 {
     const QRegion actualShape = shape & screen;
     const QRect r = actualShape.boundingRect();
@@ -434,6 +435,7 @@
 
     // Draw the texture on the offscreen framebuffer object, while blurring it horizontally
 
+    shader->setColorMatrix(m_colorMatrices.value(w));
     shader->bind();