diff --git a/backends/kwayland/waylandconfig.h b/backends/kwayland/waylandconfig.h
--- a/backends/kwayland/waylandconfig.h
+++ b/backends/kwayland/waylandconfig.h
@@ -69,9 +69,6 @@
     KScreen::ConfigPtr currentConfig();
     QMap<int, WaylandOutput*> outputMap() const;
 
-    void addOutput(quint32 name, quint32 version);
-    void removeOutput(quint32 name);
-
     void applyConfig(const KScreen::ConfigPtr &newConfig);
 
 Q_SIGNALS:
@@ -84,6 +81,10 @@
     void disconnected();
 
     void initConnection();
+
+    void addOutput(quint32 name, quint32 version);
+    void removeOutput(WaylandOutput *output);
+
     void blockSignals();
     void unblockSignals();
     void tryPendingConfig();
@@ -95,16 +96,16 @@
     KWayland::Client::Registry *m_registry;
     KWayland::Client::OutputManagement *m_outputManagement;
 
+    // KWayland names as keys
     QMap<int, WaylandOutput*> m_outputMap;
-    // Map between kwayland's outputdevice names and kscreen output ids
-    // key: wayland's name, value: kscreen id
-    QMap<int, int> m_outputIds;
+
+    // KWayland names
     QList<int> m_initializingOutputs;
-    bool m_registryInitialized;
     int m_lastOutputId = -1;
+
+    bool m_registryInitialized;
     bool m_blockSignals;
     QEventLoop m_syncLoop;
-    int m_newOutputId;
     KScreen::ConfigPtr m_kscreenConfig;
     KScreen::ConfigPtr m_kscreenPendingConfig;
     WaylandScreen *m_screen;
diff --git a/backends/kwayland/waylandconfig.cpp b/backends/kwayland/waylandconfig.cpp
--- a/backends/kwayland/waylandconfig.cpp
+++ b/backends/kwayland/waylandconfig.cpp
@@ -41,7 +41,6 @@
     , m_outputManagement(nullptr)
     , m_registryInitialized(false)
     , m_blockSignals(true)
-    , m_newOutputId(0)
     , m_kscreenConfig(new Config)
     , m_kscreenPendingConfig(nullptr)
     , m_screen(new WaylandScreen(this))
@@ -143,8 +142,6 @@
 
     connect(m_registry, &KWayland::Client::Registry::outputDeviceAnnounced,
             this, &WaylandConfig::addOutput);
-    connect(m_registry, &KWayland::Client::Registry::outputDeviceRemoved,
-            this, &WaylandConfig::removeOutput);
 
     connect(m_registry, &KWayland::Client::Registry::outputManagementAnnounced,
             this, [this](quint32 name, quint32 version) {
@@ -166,24 +163,19 @@
     m_registry->setup();
 }
 
+int s_outputId = 0;
+
 void WaylandConfig::addOutput(quint32 name, quint32 version)
 {
-    ++m_newOutputId;
-    quint32 new_id = m_newOutputId;
-
-    m_outputIds[name] = new_id;
-    if (m_outputMap.contains(new_id)) {
-        return;
-    }
-
-    if (!m_initializingOutputs.contains(name)) {
-        m_initializingOutputs << name;
-    }
-
-    auto op = new KWayland::Client::OutputDevice(this);
+    // KWayland should only add this output once before removing again
+    Q_ASSERT(!m_initializingOutputs.contains(name));
+    m_initializingOutputs << name;
 
-    WaylandOutput *waylandoutput = new WaylandOutput(new_id, this);
-    waylandoutput->bindOutputDevice(m_registry, op, name, version);
+    WaylandOutput *waylandoutput = new WaylandOutput(++s_outputId, name, this);
+    connect(waylandoutput, &WaylandOutput::deviceRemoved, this, [this, waylandoutput]() {
+        removeOutput(waylandoutput);
+    });
+    waylandoutput->createOutputDevice(m_registry, version);
 
     // finalize: when the output is done, we put it in the known outputs map,
     // remove if from the list of initializing outputs, and emit configChanged()
@@ -205,11 +197,16 @@
     });
 }
 
-void WaylandConfig::removeOutput(quint32 name)
+void WaylandConfig::removeOutput(WaylandOutput *output)
 {
-    const int kscreen_id = m_outputIds[name];
+    if (m_initializingOutputs.removeOne(output->wlName())) {
+        // output was not yet fully initialized, just remove here and return
+        delete output;
+        return;
+    }
 
-    auto output = m_outputMap.take(kscreen_id);
+    // remove the output from output mapping
+    Q_ASSERT(m_outputMap.take(output->id()) == output);
     m_screen->setOutputs(m_outputMap.values());
     delete output;
 
diff --git a/backends/kwayland/waylandoutput.h b/backends/kwayland/waylandoutput.h
--- a/backends/kwayland/waylandoutput.h
+++ b/backends/kwayland/waylandoutput.h
@@ -43,12 +43,12 @@
     void updateKScreenOutput(KScreen::OutputPtr &output);
 
     quint32 id() const;
+    quint32 wlName() const;
+
     QString name() const;
     bool enabled() const;
 
     KWayland::Client::OutputDevice* outputDevice() const;
-    void bindOutputDevice(KWayland::Client::Registry *registry,
-                          KWayland::Client::OutputDevice *op, quint32 name, quint32 version);
 
     // translation methods
     KScreen::Output::Rotation toKScreenRotation(
@@ -61,18 +61,21 @@
 
 Q_SIGNALS:
     void complete();
+    void deviceRemoved();
 
     // only emitted after complete()
     void changed();
 
 private:
     friend WaylandConfig;
-    explicit WaylandOutput(quint32 id, WaylandConfig *parent = nullptr);
+    explicit WaylandOutput(quint32 id, quint32 wlName, WaylandConfig *parent = nullptr);
 
+    void createOutputDevice(KWayland::Client::Registry *registry, quint32 version);
     void showOutput();
     QString modeName(const KWayland::Client::OutputDevice::Mode &m) const;
 
     quint32 m_id;
+    quint32 m_wlName;
     KWayland::Client::OutputDevice *m_output;
     KWayland::Client::Registry *m_registry;
 
diff --git a/backends/kwayland/waylandoutput.cpp b/backends/kwayland/waylandoutput.cpp
--- a/backends/kwayland/waylandoutput.cpp
+++ b/backends/kwayland/waylandoutput.cpp
@@ -38,9 +38,10 @@
     {Wl::OutputDevice::Transform::Flipped270, Output::Left}
 };
 
-WaylandOutput::WaylandOutput(quint32 id, WaylandConfig *parent)
+WaylandOutput::WaylandOutput(quint32 id, quint32 wlName, WaylandConfig *parent)
     : QObject(parent)
     , m_id(id)
+    , m_wlName(wlName)
     , m_output(nullptr)
 {
 }
@@ -82,6 +83,11 @@
     return m_id;
 }
 
+quint32 WaylandOutput::wlName() const
+{
+    return m_wlName;
+}
+
 bool WaylandOutput::enabled() const
 {
     return m_output != nullptr;
@@ -92,20 +98,16 @@
     return m_output;
 }
 
-void WaylandOutput::bindOutputDevice(Wl::Registry* registry, Wl::OutputDevice* op,
-                                     quint32 name, quint32 version)
+void WaylandOutput::createOutputDevice(Wl::Registry *registry, quint32 version)
 {
-    if (m_output == op) {
-        return;
-    }
-    m_output = op;
+    Q_ASSERT(!m_output);
+    m_output = registry->createOutputDevice(m_wlName, version);
 
+    connect(m_output, &Wl::OutputDevice::removed, this, &WaylandOutput::deviceRemoved);
     connect(m_output, &Wl::OutputDevice::done, this, [this]() {
                 Q_EMIT complete();
                 connect(m_output, &Wl::OutputDevice::changed, this, &WaylandOutput::changed);
     });
-
-    m_output->setup(registry->bindOutputDevice(name, version));
 }
 
 OutputPtr WaylandOutput::toKScreenOutput()