diff --git a/applets/taskmanager/package/contents/ui/Task.qml b/applets/taskmanager/package/contents/ui/Task.qml
--- a/applets/taskmanager/package/contents/ui/Task.qml
+++ b/applets/taskmanager/package/contents/ui/Task.qml
@@ -140,7 +140,8 @@
     }
 
     onPositionChanged: {
-        if (pressX != -1 && dragHelper.isDrag(pressX, pressY, mouse.x, mouse.y)) {
+        // mouse.button is always 0 here, hence checking with mouse.buttons
+        if (pressX != -1 && mouse.buttons == Qt.LeftButton && dragHelper.isDrag(pressX, pressY, mouse.x, mouse.y)) {
             tasks.dragSource = task;
             dragHelper.startDrag(task, model.MimeType, model.MimeData,
                 model.LauncherUrl, model.DecorationRole);