diff --git a/applets/taskmanager/package/contents/ui/AudioStream.qml b/applets/taskmanager/package/contents/ui/AudioStream.qml
--- a/applets/taskmanager/package/contents/ui/AudioStream.qml
+++ b/applets/taskmanager/package/contents/ui/AudioStream.qml
@@ -60,7 +60,7 @@
              SequentialAnimation {
                  // Delay showing the play indicator so we don't flash it for brief sounds.
                  PauseAnimation {
-                     duration: inPopup ? 0 : 2000
+                     duration: !task.delayAudioStreamIndicator || inPopup ? 0 : 2000
                  }
                  NumberAnimation {
                      property: "opacity"
diff --git a/applets/taskmanager/package/contents/ui/Task.qml b/applets/taskmanager/package/contents/ui/Task.qml
--- a/applets/taskmanager/package/contents/ui/Task.qml
+++ b/applets/taskmanager/package/contents/ui/Task.qml
@@ -60,6 +60,7 @@
 
     property Item audioStreamOverlay
     property var audioStreams: []
+    property bool delayAudioStreamIndicator: false
     readonly property bool hasAudioStream: plasmoid.configuration.indicateAudioStreams && audioStreams.length > 0
     readonly property bool playingAudio: hasAudioStream && audioStreams.some(function (item) {
         return !item.corked
@@ -78,8 +79,8 @@
 
     acceptedButtons: Qt.LeftButton | Qt.RightButton | Qt.MidButton | Qt.BackButton | Qt.ForwardButton
 
-    onPidChanged: updateAudioStreams()
-    onAppNameChanged: updateAudioStreams()
+    onPidChanged: updateAudioStreams({delay: false})
+    onAppNameChanged: updateAudioStreams({delay: false})
 
     onIsWindowChanged: {
         if (isWindow) {
@@ -235,7 +236,14 @@
         contextMenu.show();
     }
 
-    function updateAudioStreams() {
+    function updateAudioStreams(args) {
+        if (args) {
+            // When the task just appeared (e.g. virtual desktop switch), show the audio indicator
+            // right away. Only when audio streams change during the lifetime of this task, delay
+            // showing that to avoid distraction.
+            delayAudioStreamIndicator = !!args.delay;
+        }
+
         var pa = pulseAudio.item;
         if (!pa) {
             task.audioStreams = [];
@@ -269,7 +277,7 @@
     Connections {
         target: pulseAudio.item
         ignoreUnknownSignals: true // Plasma-PA might not be available
-        onStreamsChanged: task.updateAudioStreams()
+        onStreamsChanged: task.updateAudioStreams({delay: true})
     }
 
     Component {
@@ -572,6 +580,6 @@
             taskInitComponent.createObject(task);
         }
 
-        updateAudioStreams()
+        updateAudioStreams({delay: false})
     }
 }