diff --git a/applets/taskmanager/package/contents/ui/Task.qml b/applets/taskmanager/package/contents/ui/Task.qml
--- a/applets/taskmanager/package/contents/ui/Task.qml
+++ b/applets/taskmanager/package/contents/ui/Task.qml
@@ -43,6 +43,7 @@
 
     readonly property int pid: model.AppPid !== undefined ? model.AppPid : 0
     readonly property string appName: model.AppName
+    readonly property variant winIdList: model.WinIdList
     property int itemIndex: index
     property bool inPopup: false
     property bool isWindow: model.IsWindow === true
@@ -152,6 +153,7 @@
                 if (plasmoid.configuration.middleClickAction === TaskManagerApplet.Backend.NewInstance) {
                     tasksModel.requestNewInstance(modelIndex());
                 } else if (plasmoid.configuration.middleClickAction === TaskManagerApplet.Backend.Close) {
+                    tasks.taskClosedWithMouseMiddleButton = winIdList.slice()
                     tasksModel.requestClose(modelIndex());
                 } else if (plasmoid.configuration.middleClickAction === TaskManagerApplet.Backend.ToggleMinimized) {
                     tasksModel.requestToggleMinimized(modelIndex());
diff --git a/applets/taskmanager/package/contents/ui/main.qml b/applets/taskmanager/package/contents/ui/main.qml
--- a/applets/taskmanager/package/contents/ui/main.qml
+++ b/applets/taskmanager/package/contents/ui/main.qml
@@ -29,16 +29,20 @@
 import "code/layout.js" as LayoutManager
 import "code/tools.js" as TaskTools
 
-Item {
+MouseArea {
     id: tasks
 
     anchors.fill: parent
+    hoverEnabled: true
 
     property bool vertical: (plasmoid.formFactor === PlasmaCore.Types.Vertical)
     property bool iconsOnly: (plasmoid.pluginName === "org.kde.plasma.icontasks")
 
     property QtObject contextMenuComponent: Qt.createComponent("ContextMenu.qml");
 
+    property bool needLayoutRefresh: false;
+    property variant taskClosedWithMouseMiddleButton: []
+
     Plasmoid.preferredRepresentation: Plasmoid.fullRepresentation
 
     Plasmoid.onUserConfiguringChanged: {
@@ -85,6 +89,13 @@
         }
     }
 
+    onExited: {
+        if (needLayoutRefresh) {
+            LayoutManager.layout(taskRepeater)
+            needLayoutRefresh = false;
+        }
+    }
+
     TaskManager.TasksModel {
         id: tasksModel
 
@@ -439,7 +450,15 @@
 
             delegate: Task {}
             onItemAdded: taskList.layout()
-            onItemRemoved: taskList.layout()
+            onItemRemoved: {
+                if (tasks.containsMouse && index != taskRepeater.count &&
+                    item.winIdList.length > 0 && taskClosedWithMouseMiddleButton.indexOf(item.winIdList[0]) > -1) {
+                    needLayoutRefresh = true;
+                } else {
+                    taskList.layout();
+                }
+                taskClosedWithMouseMiddleButton = [];
+            }
         }
     }