diff --git a/applets/taskmanager/package/contents/ui/Task.qml b/applets/taskmanager/package/contents/ui/Task.qml
--- a/applets/taskmanager/package/contents/ui/Task.qml
+++ b/applets/taskmanager/package/contents/ui/Task.qml
@@ -153,6 +153,7 @@
                     tasksModel.requestNewInstance(modelIndex());
                 } else if (plasmoid.configuration.middleClickAction == TaskManagerApplet.Backend.Close) {
                     tasksModel.requestClose(modelIndex());
+                    tasks.taskClosedWithMouseMiddleButton = true
                 } else if (plasmoid.configuration.middleClickAction == TaskManagerApplet.Backend.ToggleMinimized) {
                     tasksModel.requestToggleMinimized(modelIndex());
                 } else if (plasmoid.configuration.middleClickAction == TaskManagerApplet.Backend.ToggleGrouping) {
diff --git a/applets/taskmanager/package/contents/ui/main.qml b/applets/taskmanager/package/contents/ui/main.qml
--- a/applets/taskmanager/package/contents/ui/main.qml
+++ b/applets/taskmanager/package/contents/ui/main.qml
@@ -29,16 +29,20 @@
 import "code/layout.js" as LayoutManager
 import "code/tools.js" as TaskTools
 
-Item {
+MouseArea {
     id: tasks
 
     anchors.fill: parent
+    hoverEnabled: true
 
     property bool vertical: (plasmoid.formFactor == PlasmaCore.Types.Vertical)
     property bool iconsOnly: (plasmoid.pluginName == "org.kde.plasma.icontasks")
 
     property QtObject contextMenuComponent: Qt.createComponent("ContextMenu.qml");
 
+    property bool needLayoutRefresh: false;
+    property bool taskClosedWithMouseMiddleButton: false
+
     Plasmoid.preferredRepresentation: Plasmoid.fullRepresentation
 
     Plasmoid.onUserConfiguringChanged: {
@@ -85,6 +89,13 @@
         }
     }
 
+    onExited: {
+        if (needLayoutRefresh) {
+            LayoutManager.layout(taskRepeater)
+            needLayoutRefresh = false;
+        }
+    }
+
     TaskManager.TasksModel {
         id: tasksModel
 
@@ -439,7 +450,14 @@
 
             delegate: Task {}
             onItemAdded: taskList.layout()
-            onItemRemoved: taskList.layout()
+            onItemRemoved: {
+                if (!tasks.containsMouse || !taskClosedWithMouseMiddleButton) {
+                    taskList.layout();
+                } else {
+                    needLayoutRefresh = true;
+                    taskClosedWithMouseMiddleButton = false;
+                }
+            }
         }
     }