diff --git a/plugins/platforms/drm/CMakeLists.txt b/plugins/platforms/drm/CMakeLists.txt
--- a/plugins/platforms/drm/CMakeLists.txt
+++ b/plugins/platforms/drm/CMakeLists.txt
@@ -13,7 +13,7 @@
 )
 
 if(HAVE_GBM)
-    set(DRM_SOURCES ${DRM_SOURCES} egl_gbm_backend.cpp)
+    set(DRM_SOURCES ${DRM_SOURCES} egl_gbm_backend.cpp drm_buffer_gbm.cpp)
 endif()
 
 add_library(KWinWaylandDrmBackend MODULE ${DRM_SOURCES})
diff --git a/plugins/platforms/drm/drm_backend.h b/plugins/platforms/drm/drm_backend.h
--- a/plugins/platforms/drm/drm_backend.h
+++ b/plugins/platforms/drm/drm_backend.h
@@ -23,6 +23,9 @@
 #include "input.h"
 
 #include "drm_buffer.h"
+#if HAVE_GBM
+#include "drm_buffer_gbm.h"
+#endif
 #include "drm_inputeventfilter.h"
 #include "drm_pointer.h"
 
@@ -74,27 +77,27 @@
     OpenGLBackend* createOpenGLBackend() override;
 
     void init() override;
-    DrmBuffer *createBuffer(const QSize &size);
-    DrmBuffer *createBuffer(gbm_surface *surface);
+    DrmDumbBuffer *createBuffer(const QSize &size);
+    DrmSurfaceBuffer *createBuffer(gbm_surface *surface);
     void present(DrmBuffer *buffer, DrmOutput *output);
 
     int fd() const {
         return m_fd;
     }
     QVector<DrmOutput*> outputs() const {
         return m_outputs;
     }
-    QVector<DrmBuffer*> buffers() const {
-        return m_buffers;
-    }
     QVector<DrmPlane*> planes() const {
         return m_planes;
     }
-    void bufferDestroyed(DrmBuffer *b);
 
     void outputWentOff();
     void checkOutputsAreOn();
 
+    // QPainter reuses buffers
+    bool deleteBufferAfterPageFlip() const {
+        return m_deleteBufferAfterPageFlip;
+    }
     // returns use of AMS, default is not/legacy
     bool atomicModeSetting() const {
         return m_atomicModeSetting;
@@ -145,13 +148,13 @@
     QVector<DrmConnector*> m_connectors;
     // currently active output pipelines (planes + crtc + encoder + connector)
     QVector<DrmOutput*> m_outputs;
-    DrmBuffer *m_cursor[2];
+    DrmDumbBuffer *m_cursor[2];
+    bool m_deleteBufferAfterPageFlip;
     bool m_atomicModeSetting = false;
     bool m_cursorEnabled = false;
     int m_cursorIndex = 0;
     int m_pageFlipsPending = 0;
     bool m_active = false;
-    QVector<DrmBuffer*> m_buffers;
     // all available planes: primarys, cursors and overlays
     QVector<DrmPlane*> m_planes;
     QScopedPointer<DpmsInputEventFilter> m_dpmsFilter;
diff --git a/plugins/platforms/drm/drm_backend.cpp b/plugins/platforms/drm/drm_backend.cpp
--- a/plugins/platforms/drm/drm_backend.cpp
+++ b/plugins/platforms/drm/drm_backend.cpp
@@ -165,7 +165,7 @@
     }
     m_active = true;
     if (!usesSoftwareCursor()) {
-        DrmBuffer *c = m_cursor[(m_cursorIndex + 1) % 2];
+        DrmDumbBuffer *c = m_cursor[(m_cursorIndex + 1) % 2];
         const QPoint cp = Cursor::pos() - softwareCursorHotspot();
         for (auto it = m_outputs.constBegin(); it != m_outputs.constEnd(); ++it) {
             DrmOutput *o = *it;
@@ -604,7 +604,7 @@
 
 void DrmBackend::setCursor()
 {
-    DrmBuffer *c = m_cursor[m_cursorIndex];
+    DrmDumbBuffer *c = m_cursor[m_cursorIndex];
     m_cursorIndex = (m_cursorIndex + 1) % 2;
     if (m_cursorEnabled) {
         for (auto it = m_outputs.constBegin(); it != m_outputs.constEnd(); ++it) {
@@ -671,42 +671,36 @@
 
 QPainterBackend *DrmBackend::createQPainterBackend()
 {
+    m_deleteBufferAfterPageFlip = false;
     return new DrmQPainterBackend(this);
 }
 
 OpenGLBackend *DrmBackend::createOpenGLBackend()
 {
 #if HAVE_GBM
+    m_deleteBufferAfterPageFlip = true;
     return new EglGbmBackend(this);
 #else
     return Platform::createOpenGLBackend();
 #endif
 }
 
-DrmBuffer *DrmBackend::createBuffer(const QSize &size)
+DrmDumbBuffer *DrmBackend::createBuffer(const QSize &size)
 {
-    DrmBuffer *b = new DrmBuffer(this, size);
-    m_buffers << b;
+    DrmDumbBuffer *b = new DrmDumbBuffer(this, size);
     return b;
 }
 
-DrmBuffer *DrmBackend::createBuffer(gbm_surface *surface)
+DrmSurfaceBuffer *DrmBackend::createBuffer(gbm_surface *surface)
 {
 #if HAVE_GBM
-    DrmBuffer *b = new DrmBuffer(this, surface);
-    b->m_deleteAfterPageFlip = true;
-    m_buffers << b;
+    DrmSurfaceBuffer *b = new DrmSurfaceBuffer(this, surface);
     return b;
 #else
     return nullptr;
 #endif
 }
 
-void DrmBackend::bufferDestroyed(DrmBuffer *b)
-{
-    m_buffers.removeAll(b);
-}
-
 void DrmBackend::outputDpmsChanged()
 {
     if (m_outputs.isEmpty()) {
diff --git a/plugins/platforms/drm/drm_buffer.h b/plugins/platforms/drm/drm_buffer.h
--- a/plugins/platforms/drm/drm_buffer.h
+++ b/plugins/platforms/drm/drm_buffer.h
@@ -23,61 +23,61 @@
 #include <QImage>
 #include <QSize>
 
-struct gbm_bo;
-struct gbm_surface;
-
 namespace KWin
 {
 
 class DrmBackend;
 
 class DrmBuffer
 {
 public:
-    ~DrmBuffer();
+    DrmBuffer(DrmBackend *backend);
+    virtual ~DrmBuffer() = default;
 
-    bool map(QImage::Format format = QImage::Format_RGB32);
-    QImage *image() const {
-        return m_image;
-    }
-    quint32 handle() const {
-        return m_handle;
+    virtual bool needsModeChange(DrmBuffer *b) const {Q_UNUSED(b) return false;}
+
+    quint32 bufferId() const {
+        return m_bufferId;
     }
+
     const QSize &size() const {
         return m_size;
     }
-    quint32 bufferId() const {
-        return m_bufferId;
+
+    virtual void releaseGbm() {}
+
+protected:
+    DrmBackend *m_backend;
+    quint32 m_bufferId = 0;
+    QSize m_size;
+};
+
+class DrmDumbBuffer : public DrmBuffer
+{
+public:
+    DrmDumbBuffer(DrmBackend *backend, const QSize &size);
+    ~DrmDumbBuffer();
+
+    bool needsModeChange(DrmBuffer *b) const override;
+
+    bool map(QImage::Format format = QImage::Format_RGB32);
+    quint32 handle() const {
+        return m_handle;
+    }
+    QImage *image() const {
+        return m_image;
     }
+
     quint32 stride() const {
         return m_stride;
     }
-    gbm_bo *gbm() const {
-        return m_bo;
-    }
-    bool isGbm() const {
-        return m_bo != nullptr;
-    }
-    bool deleteAfterPageFlip() const {
-        return m_deleteAfterPageFlip;
-    }
-    void releaseGbm();
 
 private:
-    friend class DrmBackend;
-    DrmBuffer(DrmBackend *backend, const QSize &size);
-    DrmBuffer(DrmBackend *backend, gbm_surface *surface);
-    DrmBackend *m_backend;
-    gbm_surface *m_surface = nullptr;
-    gbm_bo *m_bo = nullptr;
-    QSize m_size;
     quint32 m_handle = 0;
-    quint32 m_bufferId = 0;
-    quint32 m_stride = 0;
     quint64 m_bufferSize = 0;
     void *m_memory = nullptr;
     QImage *m_image = nullptr;
-    bool m_deleteAfterPageFlip = false;
+    quint32 m_stride = 0;
 };
 
 }
diff --git a/plugins/platforms/drm/drm_buffer.cpp b/plugins/platforms/drm/drm_buffer.cpp
--- a/plugins/platforms/drm/drm_buffer.cpp
+++ b/plugins/platforms/drm/drm_buffer.cpp
@@ -27,18 +27,20 @@
 #include <errno.h>
 // drm
 #include <xf86drm.h>
-#if HAVE_GBM
-#include <gbm.h>
-#endif
 
 namespace KWin
 {
 
-
-DrmBuffer::DrmBuffer(DrmBackend *backend, const QSize &size)
+DrmBuffer:: DrmBuffer(DrmBackend *backend)
     : m_backend(backend)
-    , m_size(size)
 {
+}
+
+// DrmDumbBuffer
+DrmDumbBuffer::DrmDumbBuffer(DrmBackend *backend, const QSize &size)
+    : DrmBuffer(backend)
+{
+    m_size = size;
     drm_mode_create_dumb createArgs;
     memset(&createArgs, 0, sizeof createArgs);
     createArgs.bpp = 32;
@@ -57,46 +59,12 @@
     }
 }
 
-
-#if HAVE_GBM
-static void gbmCallback(gbm_bo *bo, void *data)
-{
-    DrmBackend *backend = reinterpret_cast<DrmBackend*>(data);
-    const auto &buffers = backend->buffers();
-    for (auto buffer: buffers) {
-        if (buffer->gbm() == bo) {
-            delete buffer;
-            return;
-        }
-    }
-}
-#endif
-
-DrmBuffer::DrmBuffer(DrmBackend *backend, gbm_surface *surface)
-    : m_backend(backend)
-    , m_surface(surface)
-{
-#if HAVE_GBM
-    m_bo = gbm_surface_lock_front_buffer(surface);
-    if (!m_bo) {
-        qCWarning(KWIN_DRM) << "Locking front buffer failed";
-        return;
-    }
-    m_size = QSize(gbm_bo_get_width(m_bo), gbm_bo_get_height(m_bo));
-    m_stride = gbm_bo_get_stride(m_bo);
-    if (drmModeAddFB(m_backend->fd(), m_size.width(), m_size.height(), 24, 32, m_stride, gbm_bo_get_handle(m_bo).u32, &m_bufferId) != 0) {
-        qCWarning(KWIN_DRM) << "drmModeAddFB failed";
-    }
-    gbm_bo_set_user_data(m_bo, m_backend, gbmCallback);
-#endif
-}
-
-DrmBuffer::~DrmBuffer()
+DrmDumbBuffer::~DrmDumbBuffer()
 {
     if (m_bufferId) {
         drmModeRmFB(m_backend->fd(), m_bufferId);
     }
-    m_backend->bufferDestroyed(this);
+
     delete m_image;
     if (m_memory) {
         munmap(m_memory, m_bufferSize);
@@ -106,10 +74,17 @@
         destroyArgs.handle = m_handle;
         drmIoctl(m_backend->fd(), DRM_IOCTL_MODE_DESTROY_DUMB, &destroyArgs);
     }
-    releaseGbm();
 }
 
-bool DrmBuffer::map(QImage::Format format)
+bool DrmDumbBuffer::needsModeChange(DrmBuffer *b) const {
+    if (DrmDumbBuffer *db = dynamic_cast<DrmDumbBuffer*>(b)) {
+        return m_stride != db->stride();
+    } else {
+        return true;
+    }
+}
+
+bool DrmDumbBuffer::map(QImage::Format format)
 {
     if (!m_handle || !m_bufferId) {
         return false;
@@ -129,14 +104,4 @@
     return !m_image->isNull();
 }
 
-void DrmBuffer::releaseGbm()
-{
-#if HAVE_GBM
-    if (m_bo) {
-        gbm_surface_release_buffer(m_surface, m_bo);
-        m_bo = nullptr;
-    }
-#endif
-}
-
 }
diff --git a/plugins/platforms/drm/drm_buffer_gbm.h b/plugins/platforms/drm/drm_buffer_gbm.h
new file mode 100644
--- /dev/null
+++ b/plugins/platforms/drm/drm_buffer_gbm.h
@@ -0,0 +1,61 @@
+/********************************************************************
+ KWin - the KDE window manager
+ This file is part of the KDE project.
+
+Copyright 2017 Roman Gilg <subdiff@gmail.com>
+Copyright 2015 Martin Gräßlin <mgraesslin@kde.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*********************************************************************/
+#ifndef KWIN_DRM_BUFFER_GBM_H
+#define KWIN_DRM_BUFFER_GBM_H
+
+#include "drm_buffer.h"
+
+struct gbm_bo;
+struct gbm_surface;
+
+namespace KWin
+{
+
+class DrmBackend;
+
+class DrmSurfaceBuffer : public DrmBuffer
+{
+public:
+    DrmSurfaceBuffer(DrmBackend *backend, gbm_surface *surface);
+    ~DrmSurfaceBuffer();
+
+    bool needsModeChange(DrmBuffer *b) const override {
+        if (DrmSurfaceBuffer *sb = dynamic_cast<DrmSurfaceBuffer*>(b)) {
+            return hasBo() != sb->hasBo();
+        } else {
+            return true;
+        }
+    }
+
+    bool hasBo() const {
+        return m_bo != nullptr;
+    }
+    void releaseGbm() override;
+
+private:
+    gbm_surface *m_surface = nullptr;
+    gbm_bo *m_bo = nullptr;
+};
+
+}
+
+#endif
+
diff --git a/plugins/platforms/drm/drm_buffer_gbm.cpp b/plugins/platforms/drm/drm_buffer_gbm.cpp
new file mode 100644
--- /dev/null
+++ b/plugins/platforms/drm/drm_buffer_gbm.cpp
@@ -0,0 +1,69 @@
+/********************************************************************
+ KWin - the KDE window manager
+ This file is part of the KDE project.
+
+Copyright 2017 Roman Gilg <subdiff@gmail.com>
+Copyright 2015 Martin Gräßlin <mgraesslin@kde.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*********************************************************************/
+#include "drm_backend.h"
+#include "drm_buffer_gbm.h"
+
+#include "logging.h"
+
+// system
+#include <sys/mman.h>
+#include <errno.h>
+// drm
+#include <xf86drm.h>
+#include <gbm.h>
+
+namespace KWin
+{
+
+// DrmSurfaceBuffer
+DrmSurfaceBuffer::DrmSurfaceBuffer(DrmBackend *backend, gbm_surface *surface)
+    : DrmBuffer(backend)
+    , m_surface(surface)
+{
+    m_bo = gbm_surface_lock_front_buffer(surface);
+    if (!m_bo) {
+        qCWarning(KWIN_DRM) << "Locking front buffer failed";
+        return;
+    }
+    m_size = QSize(gbm_bo_get_width(m_bo), gbm_bo_get_height(m_bo));
+    if (drmModeAddFB(m_backend->fd(), m_size.width(), m_size.height(), 24, 32, gbm_bo_get_stride(m_bo), gbm_bo_get_handle(m_bo).u32, &m_bufferId) != 0) {
+        qCWarning(KWIN_DRM) << "drmModeAddFB failed";
+    }
+    gbm_bo_set_user_data(m_bo, this, nullptr);
+}
+
+DrmSurfaceBuffer::~DrmSurfaceBuffer()
+{
+    if (m_bufferId) {
+        drmModeRmFB(m_backend->fd(), m_bufferId);
+    }
+    releaseGbm();
+}
+
+void DrmSurfaceBuffer::releaseGbm()
+{
+    if (m_bo) {
+        gbm_surface_release_buffer(m_surface, m_bo);
+        m_bo = nullptr;
+    }
+}
+
+}
diff --git a/plugins/platforms/drm/drm_object_crtc.h b/plugins/platforms/drm/drm_object_crtc.h
--- a/plugins/platforms/drm/drm_object_crtc.h
+++ b/plugins/platforms/drm/drm_object_crtc.h
@@ -27,6 +27,7 @@
 
 class DrmBackend;
 class DrmBuffer;
+class DrmDumbBuffer;
 
 class DrmCrtc : public DrmObject
 {
@@ -68,7 +69,7 @@
 
     DrmBuffer *m_currentBuffer = nullptr;
     DrmBuffer *m_nextBuffer = nullptr;
-    DrmBuffer *m_blackBuffer = nullptr;
+    DrmDumbBuffer *m_blackBuffer = nullptr;
 };
 
 }
diff --git a/plugins/platforms/drm/drm_object_crtc.cpp b/plugins/platforms/drm/drm_object_crtc.cpp
--- a/plugins/platforms/drm/drm_object_crtc.cpp
+++ b/plugins/platforms/drm/drm_object_crtc.cpp
@@ -69,7 +69,7 @@
 
 void DrmCrtc::flipBuffer()
 {
-    if (m_currentBuffer && m_currentBuffer->deleteAfterPageFlip() && m_currentBuffer != m_nextBuffer) {
+    if (m_currentBuffer && m_output->m_backend->deleteBufferAfterPageFlip() && m_currentBuffer != m_nextBuffer) {
         delete m_currentBuffer;
     }
     m_currentBuffer = m_nextBuffer;
@@ -82,7 +82,7 @@
 bool DrmCrtc::blank()
 {
     if (!m_blackBuffer) {
-        DrmBuffer *blackBuffer = m_output->m_backend->createBuffer(m_output->size());
+        DrmDumbBuffer *blackBuffer = m_output->m_backend->createBuffer(m_output->size());
         if (!blackBuffer->map()) {
             delete blackBuffer;
             return false;
@@ -93,7 +93,7 @@
 
     // TODO: Do this atomically
     if (m_output->setModeLegacy(m_blackBuffer)) {
-        if (m_currentBuffer && m_currentBuffer->deleteAfterPageFlip()) {
+        if (m_currentBuffer && m_output->m_backend->deleteBufferAfterPageFlip()) {
             delete m_currentBuffer;
             delete m_nextBuffer;
         }
diff --git a/plugins/platforms/drm/drm_output.h b/plugins/platforms/drm/drm_output.h
--- a/plugins/platforms/drm/drm_output.h
+++ b/plugins/platforms/drm/drm_output.h
@@ -46,6 +46,7 @@
 
 class DrmBackend;
 class DrmBuffer;
+class DrmDumbBuffer;
 class DrmPlane;
 class DrmConnector;
 class DrmCrtc;
@@ -62,7 +63,7 @@
     };
     virtual ~DrmOutput();
     void releaseGbm();
-    void showCursor(DrmBuffer *buffer);
+    void showCursor(DrmDumbBuffer *buffer);
     void hideCursor();
     void moveCursor(const QPoint &globalPos);
     bool init(drmModeConnector *connector);
@@ -121,7 +122,6 @@
     DrmConnector *m_conn = nullptr;
     DrmCrtc *m_crtc = nullptr;
     QPoint m_globalPos;
-    quint32 m_lastStride = 0;
     bool m_lastGbm = false;
     drmModeModeInfo m_mode;
     Edid m_edid;
diff --git a/plugins/platforms/drm/drm_output.cpp b/plugins/platforms/drm/drm_output.cpp
--- a/plugins/platforms/drm/drm_output.cpp
+++ b/plugins/platforms/drm/drm_output.cpp
@@ -84,7 +84,7 @@
     drmModeSetCursor(m_backend->fd(), m_crtc->id(), 0, 0, 0);
 }
 
-void DrmOutput::showCursor(DrmBuffer *c)
+void DrmOutput::showCursor(DrmDumbBuffer *c)
 {
     const QSize &s = c->size();
     drmModeSetCursor(m_backend->fd(), m_crtc->id(), c->handle(), s.width(), s.height());
@@ -690,7 +690,7 @@
 
 void DrmOutput::pageFlippedBufferRemover(DrmBuffer *oldbuffer, DrmBuffer *newbuffer)
 {
-    if (oldbuffer && oldbuffer->deleteAfterPageFlip() && oldbuffer != newbuffer) {
+    if (oldbuffer && m_backend->deleteBufferAfterPageFlip() && oldbuffer != newbuffer) {
         delete oldbuffer;
     }
 }
@@ -825,9 +825,10 @@
     }
 
     // Do we need to set a new mode first?
-    if (m_lastStride != buffer->stride() || m_lastGbm != buffer->isGbm()){
-        if (!setModeLegacy(buffer))
+    if (!m_crtc->current() || m_crtc->current()->needsModeChange(buffer)) {
+        if (!setModeLegacy(buffer)) {
             return false;
+        }
     }
     int errno_save = 0;
     const bool ok = drmModePageFlip(m_backend->fd(), m_crtc->id(), buffer->bufferId(), DRM_MODE_PAGE_FLIP_EVENT, this) == 0;
@@ -845,8 +846,6 @@
 {
     uint32_t connId = m_conn->id();
     if (drmModeSetCrtc(m_backend->fd(), m_crtc->id(), buffer->bufferId(), 0, 0, &connId, 1, &m_mode) == 0) {
-        m_lastStride = buffer->stride();
-        m_lastGbm = buffer->isGbm();
         return true;
     } else {
         qCWarning(KWIN_DRM) << "Mode setting failed";
diff --git a/plugins/platforms/drm/scene_qpainter_drm_backend.h b/plugins/platforms/drm/scene_qpainter_drm_backend.h
--- a/plugins/platforms/drm/scene_qpainter_drm_backend.h
+++ b/plugins/platforms/drm/scene_qpainter_drm_backend.h
@@ -26,7 +26,7 @@
 {
 
 class DrmBackend;
-class DrmBuffer;
+class DrmDumbBuffer;
 class DrmOutput;
 
 class DrmQPainterBackend : public QObject, public QPainterBackend
@@ -47,7 +47,7 @@
 private:
     void initOutput(DrmOutput *output);
     struct Output {
-        DrmBuffer *buffer[2];
+        DrmDumbBuffer *buffer[2];
         DrmOutput *output;
         int index = 0;
     };