diff --git a/vnc/vncsshtunnelthread.h b/vnc/vncsshtunnelthread.h
--- a/vnc/vncsshtunnelthread.h
+++ b/vnc/vncsshtunnelthread.h
@@ -54,6 +54,7 @@
 
     QString password() const;
     void setPassword(const QString &password, PasswordOrigin origin);
+    void userCanceledPasswordRequest();
 
     void run() override;
 
@@ -71,6 +72,7 @@
     bool m_loopback;
     QString m_password;
     PasswordOrigin m_passwordOrigin;
+    bool m_passwordRequestCanceledByUser;
 
     std::atomic_bool m_stop_thread;
 };
diff --git a/vnc/vncsshtunnelthread.cpp b/vnc/vncsshtunnelthread.cpp
--- a/vnc/vncsshtunnelthread.cpp
+++ b/vnc/vncsshtunnelthread.cpp
@@ -66,6 +66,13 @@
     m_passwordOrigin = origin;
 }
 
+// This is called by the main thread, but from a slot connected to our signal via BlockingQueuedConnection
+// so this is safe even without a mutex, the semaphore in BlockingQueuedConnection takes care of the synchronization.
+void VncSshTunnelThread::userCanceledPasswordRequest()
+{
+    m_passwordRequestCanceledByUser = true;
+}
+
 void VncSshTunnelThread::run()
 {
     struct CleanupHelper
@@ -112,18 +119,23 @@
     // First try authenticating via ssh agent
     res = ssh_userauth_agent(session, NULL);
 
+    m_passwordRequestCanceledByUser = false;
     if (res != SSH_AUTH_SUCCESS) {
         // If ssh agent didn't work, try with password
         emit passwordRequest(NoFlags); // This calls blockingly to the main thread which will call setPassword
         res = ssh_userauth_password(session, NULL, m_password.toUtf8().constData());
 
         // If password didn't work but came from the wallet, ask the user for the password
-        if (res != SSH_AUTH_SUCCESS && m_passwordOrigin == PasswordFromWallet) {
+        if (!m_passwordRequestCanceledByUser && res != SSH_AUTH_SUCCESS && m_passwordOrigin == PasswordFromWallet) {
             emit passwordRequest(IgnoreWallet); // This calls blockingly to the main thread which will call setPassword
             res = ssh_userauth_password(session, NULL, m_password.toUtf8().constData());
         }
     }
 
+    if (m_passwordRequestCanceledByUser) {
+        return;
+    }
+
     if (res != SSH_AUTH_SUCCESS) {
         emit errorMessage(i18n("Error authenticating with password: %1", QString::fromLocal8Bit(ssh_get_error(session))));
         return;
diff --git a/vnc/vncview.cpp b/vnc/vncview.cpp
--- a/vnc/vncview.cpp
+++ b/vnc/vncview.cpp
@@ -28,6 +28,7 @@
 #include <QImage>
 #include <QPainter>
 #include <QMouseEvent>
+#include <QTimer>
 
 #ifdef QTONLY
     #include <QMessageBox>
@@ -371,7 +372,10 @@
         m_sshTunnelThread->setPassword(dialog.password(), VncSshTunnelThread::PasswordFromDialog);
     } else {
         qCDebug(KRDC) << "ssh password dialog not accepted";
-        startQuitting();
+        m_sshTunnelThread->userCanceledPasswordRequest();
+        // We need to use a single shot because otherwise startQuitting deletes the thread
+        // but we're here from a blocked queued connection and thus we deadlock
+        QTimer::singleShot(0, this, &VncView::startQuitting);
     }
 }
 #endif