diff --git a/libtaskmanager/taskfilterproxymodel.h b/libtaskmanager/taskfilterproxymodel.h
--- a/libtaskmanager/taskfilterproxymodel.h
+++ b/libtaskmanager/taskfilterproxymodel.h
@@ -271,6 +271,13 @@
      **/
     void setDemandingAttentionSkipsFilters(bool skip);
 
+    /**
+     * Returns whether the filter model accepts this source row.
+     *
+     * @param int A row in the source model.
+     */
+    bool acceptsRow(int sourceRow) const;
+
 Q_SIGNALS:
     void virtualDesktopChanged() const;
     void screenGeometryChanged() const;
diff --git a/libtaskmanager/taskfilterproxymodel.cpp b/libtaskmanager/taskfilterproxymodel.cpp
--- a/libtaskmanager/taskfilterproxymodel.cpp
+++ b/libtaskmanager/taskfilterproxymodel.cpp
@@ -239,12 +239,14 @@
     return mapToSource(index);
 }
 
-bool TaskFilterProxyModel::filterAcceptsRow(int sourceRow, const QModelIndex &sourceParent) const
+bool TaskFilterProxyModel::acceptsRow(int sourceRow) const
 {
-    Q_UNUSED(sourceParent)
-
     const QModelIndex &sourceIdx = sourceModel()->index(sourceRow, 0);
 
+    if (!sourceIdx.isValid()) {
+        return false;
+    }
+
     // Filter tasks that are not to be shown on the task bar.
     if (d->filterSkipTaskbar && sourceIdx.data(AbstractTasksModel::SkipTaskbar).toBool()) {
         return false;
@@ -308,4 +310,11 @@
     return true;
 }
 
+bool TaskFilterProxyModel::filterAcceptsRow(int sourceRow, const QModelIndex &sourceParent) const
+{
+    Q_UNUSED(sourceParent)
+
+    return acceptsRow(sourceRow);
+}
+
 }
diff --git a/libtaskmanager/tasksmodel.cpp b/libtaskmanager/tasksmodel.cpp
--- a/libtaskmanager/tasksmodel.cpp
+++ b/libtaskmanager/tasksmodel.cpp
@@ -626,16 +626,9 @@
                     for (int i = (row - 1); i >= 0; --i) {
                         const QModelIndex &concatProxyIndex = concatProxyModel->index(sortedPreFilterRows.at(i), 0);
 
-                        if (appsMatch(concatProxyIndex, idx)) {
-                            // Our sort map contains row indices prior to any filtering, but we don't
-                            // want to sort new tasks in next to siblings we're filtering out higher up
-                            // in the proxy chain, so check in with the filter model.
-                            const QModelIndex &filterProxyIndex = filterProxyModel->mapFromSource(concatProxyIndex);
-
-                            if (filterProxyIndex.isValid()) {
+                        if (appsMatch(concatProxyIndex, idx) && filterProxyModel->acceptsRow(concatProxyIndex.row())) {
                             sortedPreFilterRows.move(row, i + 1);
                             moved = true;
-                            }
 
                             break;
                         }