diff --git a/libtaskmanager/tasksmodel.cpp b/libtaskmanager/tasksmodel.cpp
--- a/libtaskmanager/tasksmodel.cpp
+++ b/libtaskmanager/tasksmodel.cpp
@@ -887,45 +887,46 @@
             if (leftAll && !rightAll) {
                 return true;
             } else if (rightAll && !leftAll) {
-                return true;
+                return false;
             }
 
-            const QVariantList &leftDesktops = left.data(AbstractTasksModel::VirtualDesktops).toList();
-            QVariant leftDesktop;
-            int leftDesktopPos = virtualDesktopInfo->numberOfDesktops();
+            if (!(leftAll && rightAll)) {
+                const QVariantList &leftDesktops = left.data(AbstractTasksModel::VirtualDesktops).toList();
+                QVariant leftDesktop;
+                int leftDesktopPos = virtualDesktopInfo->numberOfDesktops();
 
-            for (const QVariant &desktop : leftDesktops) {
-                const int desktopPos = virtualDesktopInfo->position(desktop);
+                for (const QVariant &desktop : leftDesktops) {
+                    const int desktopPos = virtualDesktopInfo->position(desktop);
 
-                if (desktopPos < leftDesktopPos) {
-                    leftDesktop = desktop;
-                    leftDesktopPos = desktopPos;
+                    if (desktopPos <= leftDesktopPos) {
+                        leftDesktop = desktop;
+                        leftDesktopPos = desktopPos;
+                    }
                 }
-            }
 
-            const QVariantList &rightDesktops = right.data(AbstractTasksModel::VirtualDesktops).toList();
-            QVariant rightDesktop;
-            int rightDesktopPos = virtualDesktopInfo->numberOfDesktops();
+                const QVariantList &rightDesktops = right.data(AbstractTasksModel::VirtualDesktops).toList();
+                QVariant rightDesktop;
+                int rightDesktopPos = virtualDesktopInfo->numberOfDesktops();
 
-            for (const QVariant &desktop : rightDesktops) {
-                const int desktopPos = virtualDesktopInfo->position(desktop);
+                for (const QVariant &desktop : rightDesktops) {
+                    const int desktopPos = virtualDesktopInfo->position(desktop);
 
-                if (desktopPos < rightDesktopPos) {
-                    rightDesktop = desktop;
-                    rightDesktopPos = desktopPos;
+                    if (desktopPos <= rightDesktopPos) {
+                        rightDesktop = desktop;
+                        rightDesktopPos = desktopPos;
+                    }
                 }
-            }
 
-            if (!leftDesktop.isNull() && !rightDesktop.isNull() && (leftDesktop != rightDesktop)) {
-                return (virtualDesktopInfo->position(leftDesktop) < virtualDesktopInfo->position(rightDesktop));
-            } else if (!leftDesktop.isNull() && rightDesktop.isNull()) {
-                return false;
-            } else if (leftDesktop.isNull() && !rightDesktop.isNull()) {
-                return true;
+                if (!leftDesktop.isNull() && !rightDesktop.isNull() && (leftDesktop != rightDesktop)) {
+                    return (virtualDesktopInfo->position(leftDesktop) < virtualDesktopInfo->position(rightDesktop));
+                } else if (!leftDesktop.isNull() && rightDesktop.isNull()) {
+                    return false;
+                } else if (leftDesktop.isNull() && !rightDesktop.isNull()) {
+                    return true;
+                }
             }
-
-            return false;
         }
+        // fall through
         case SortActivity: {
             // updateActivityTaskCounts() counts the number of window tasks on each
             // activity. This will sort tasks by comparing a cumulative score made