diff --git a/autotests/src/inlinenote_test.cpp b/autotests/src/inlinenote_test.cpp
--- a/autotests/src/inlinenote_test.cpp
+++ b/autotests/src/inlinenote_test.cpp
@@ -88,37 +88,40 @@
 
     void inlineNoteActivated(const InlineNote &note, Qt::MouseButtons buttons, const QPoint &globalPos) override
     {
-        Q_UNUSED(note)
         Q_UNUSED(buttons)
         Q_UNUSED(globalPos)
         ++noteActivatedCount;
+        lastUnderMouse = note.underMouse();
     }
 
     void inlineNoteFocusInEvent(const InlineNote &note, const QPoint &globalPos) override
     {
-        Q_UNUSED(note)
         Q_UNUSED(globalPos)
         ++focusInCount;
+        lastUnderMouse = note.underMouse();
+
     }
 
     void inlineNoteFocusOutEvent(const InlineNote &note) override
     {
-        Q_UNUSED(note)
         ++focusOutCount;
+        // Here it should not be under the mosu
+        lastUnderMouse = note.underMouse();
     }
 
     void inlineNoteMouseMoveEvent(const InlineNote &note, const QPoint &globalPos) override
     {
-        Q_UNUSED(note)
         Q_UNUSED(globalPos)
         ++mouseMoveCount;
+        lastUnderMouse = note.underMouse();
     }
 
 public:
     int noteActivatedCount = 0;
     int focusInCount = 0;
     int focusOutCount = 0;
     int mouseMoveCount = 0;
+    bool lastUnderMouse = false;
 };
 }
 
@@ -180,6 +183,7 @@
     QCOMPARE(noteProvider.focusInCount, 0);
     QCOMPARE(noteProvider.focusOutCount, 0);
     QCOMPARE(noteProvider.mouseMoveCount, 0);
+    QVERIFY(noteProvider.lastUnderMouse == false);
 
     // move mouse onto first note
     auto internalView = findViewInternal(&view);
@@ -192,14 +196,16 @@
     QCOMPARE(noteProvider.focusOutCount, 0);
     QCOMPARE(noteProvider.mouseMoveCount, 0);
     QCOMPARE(noteProvider.noteActivatedCount, 0);
+    QVERIFY(noteProvider.lastUnderMouse);
 
     // move one pixel
     QTest::mouseMove(&view, coordCol05 + QPoint(xWidth / 2 + 1, 1));
     QTest::qWait(100);
     QCOMPARE(noteProvider.focusInCount, 1);
     QCOMPARE(noteProvider.focusOutCount, 0);
     QCOMPARE(noteProvider.mouseMoveCount, 1);
     QCOMPARE(noteProvider.noteActivatedCount, 0);
+    QVERIFY(noteProvider.lastUnderMouse);
 
     // activate
     QTest::mousePress(internalView, Qt::LeftButton, Qt::NoModifier, internalView->mapFromGlobal(view.mapToGlobal(coordCol05 + QPoint(xWidth / 2 + 1, 1))));
@@ -209,6 +215,7 @@
     QCOMPARE(noteProvider.focusOutCount, 0);
     QCOMPARE(noteProvider.mouseMoveCount, 1);
     QCOMPARE(noteProvider.noteActivatedCount, 1);
+    QVERIFY(noteProvider.lastUnderMouse);
 
     // focus out
     QTest::mouseMove(&view, coordCol04 + QPoint(0, 1));
@@ -218,6 +225,7 @@
     QCOMPARE(noteProvider.focusOutCount, 1);
     QCOMPARE(noteProvider.mouseMoveCount, 1);
     QCOMPARE(noteProvider.noteActivatedCount, 1);
+    QVERIFY(noteProvider.lastUnderMouse == false);
 
     iface->unregisterInlineNoteProvider(&noteProvider);
 }
diff --git a/src/view/kateview.cpp b/src/view/kateview.cpp
--- a/src/view/kateview.cpp
+++ b/src/view/kateview.cpp
@@ -3875,7 +3875,8 @@
     for (KTextEditor::InlineNoteProvider *provider : m_inlineNoteProviders) {
         int index = 0;
         for (auto column : provider->inlineNotes(line)) {
-            KateInlineNoteData note = {provider, this, {line, column}, index, m_viewInternal->m_activeInlineNote.m_underMouse, m_viewInternal->renderer()->currentFont(), m_viewInternal->renderer()->lineHeight()};
+            const bool underMouse = Cursor(line, column) == m_viewInternal->m_activeInlineNote.m_position;
+            KateInlineNoteData note = {provider, this, {line, column}, index, underMouse, m_viewInternal->renderer()->currentFont(), m_viewInternal->renderer()->lineHeight()};
             allInlineNotes.append(note);
             index++;
         }
diff --git a/src/view/kateviewinternal.cpp b/src/view/kateviewinternal.cpp
--- a/src/view/kateviewinternal.cpp
+++ b/src/view/kateviewinternal.cpp
@@ -2847,22 +2847,22 @@
     }
 
     if (e->buttons() == Qt::NoButton) {
-        const auto noteData = inlineNoteAt(e->globalPos());
-        const KTextEditor::InlineNote note(noteData);
-        const KTextEditor::InlineNote activeNote(m_activeInlineNote);
-        if (note.position().isValid()) {
-            if (!activeNote.position().isValid()) {
+        auto noteData = inlineNoteAt(e->globalPos());
+        if (noteData.m_position.isValid()) {
+            if (!m_activeInlineNote.m_position.isValid()) {
                 // no active note -- focus in
-                note.provider()->inlineNoteFocusInEvent(note, e->globalPos());
+                noteData.m_underMouse = true;
+                noteData.m_provider->inlineNoteFocusInEvent(KTextEditor::InlineNote(noteData), e->globalPos());
                 m_activeInlineNote = noteData;
             } else {
-                note.provider()->inlineNoteMouseMoveEvent(note, e->globalPos());
+                noteData.m_provider->inlineNoteMouseMoveEvent(KTextEditor::InlineNote(noteData), e->globalPos());
             }
             // the note might change its appearance in result to the event
-            tagLines(note.position(), note.position(), true);
-        } else if (activeNote.position().isValid()) {
-            activeNote.provider()->inlineNoteFocusOutEvent(activeNote);
-            tagLines(activeNote.position(), activeNote.position(), true);
+            tagLines(noteData.m_position, noteData.m_position, true);
+        } else if (m_activeInlineNote.m_position.isValid()) {
+            m_activeInlineNote.m_underMouse = false;
+            m_activeInlineNote.m_provider->inlineNoteFocusOutEvent(KTextEditor::InlineNote(m_activeInlineNote));
+            tagLines(m_activeInlineNote.m_position, m_activeInlineNote.m_position, true);
             m_activeInlineNote = {};
         }
     }