diff --git a/src/view/kateviewhelpers.cpp b/src/view/kateviewhelpers.cpp
--- a/src/view/kateviewhelpers.cpp
+++ b/src/view/kateviewhelpers.cpp
@@ -2015,9 +2015,14 @@
     const KateTextLayout &t = m_viewInternal->yToKateTextLayout(e->y());
     if (t.isValid()) {
         m_lastClickedLine = t.line();
-        if (positionToArea(e->pos()) != IconBorder && positionToArea(e->pos()) != AnnotationBorder) {
+        auto area = positionToArea(e->pos());
+        if (area != IconBorder && area != AnnotationBorder) {
+            auto pos = QPoint(0, e->y());
+            if (area == LineNumbers && e->button() == Qt::LeftButton && !(e->modifiers() & Qt::ShiftModifier)) {
+                m_viewInternal->beginSelectLine(pos);
+            }
             QMouseEvent forward(QEvent::MouseButtonPress,
-                                QPoint(0, e->y()), e->button(), e->buttons(), e->modifiers());
+                                pos, e->button(), e->buttons(), e->modifiers());
             m_viewInternal->mousePressEvent(&forward);
         }
         return e->accept();
@@ -2175,13 +2180,14 @@
 void KateIconBorder::mouseMoveEvent(QMouseEvent *e)
 {
     const KateTextLayout &t = m_viewInternal->yToKateTextLayout(e->y());
+    auto area = positionToArea(e->pos());
     if (t.isValid()) {
-        if (positionToArea(e->pos()) == FoldingMarkers) {
+        if (area == FoldingMarkers) {
             showDelayedBlock(t.line());
         } else {
             hideBlock();
         }
-        if (positionToArea(e->pos()) == AnnotationBorder) {
+        if (area == AnnotationBorder) {
             KTextEditor::AnnotationModel *model = m_view->annotationModel() ?
                                                   m_view->annotationModel() : m_doc->annotationModel();
             if (model) {
@@ -2191,23 +2197,24 @@
                 QTimer::singleShot(0, this, SLOT(update()));
             }
         } else {
-            if (positionToArea(e->pos()) == IconBorder) {
+            if (area == IconBorder) {
                 m_doc->requestMarkTooltip(t.line(), e->globalPos());
             }
 
             m_hoveredAnnotationGroupIdentifier.clear();
             hideAnnotationTooltip();
             QTimer::singleShot(0, this, SLOT(update()));
         }
-        if (positionToArea(e->pos()) != IconBorder) {
-            QPoint p = m_viewInternal->mapFromGlobal(e->globalPos());
-            QMouseEvent forward(QEvent::MouseMove, p, e->button(), e->buttons(), e->modifiers());
-            m_viewInternal->mouseMoveEvent(&forward);
-        }
     } else {
         // remove hovering if it's still there
         removeAnnotationHovering();
     }
+    // always forward event so dragging on border to select lines will scroll down when at top
+    if (area != IconBorder) {
+        QPoint p = m_viewInternal->mapFromGlobal(e->globalPos());
+        QMouseEvent forward(QEvent::MouseMove, p, e->button(), e->buttons(), e->modifiers());
+        m_viewInternal->mouseMoveEvent(&forward);
+    }
 
     QWidget::mouseMoveEvent(e);
 }
diff --git a/src/view/kateviewinternal.h b/src/view/kateviewinternal.h
--- a/src/view/kateviewinternal.h
+++ b/src/view/kateviewinternal.h
@@ -269,6 +269,7 @@
     void moveCursorToSelectionEdge();
     void updateCursor(const KTextEditor::Cursor &newCursor, bool force = false, bool center = false, bool calledExternally = false);
     void updateBracketMarks();
+    void beginSelectLine(const QPoint &pos);
 
     void placeCursor(const QPoint &p, bool keepSelection = false, bool updateSelection = true);
     bool isTargetSelected(const QPoint &p);
diff --git a/src/view/kateviewinternal.cpp b/src/view/kateviewinternal.cpp
--- a/src/view/kateviewinternal.cpp
+++ b/src/view/kateviewinternal.cpp
@@ -2781,6 +2781,14 @@
     m_possibleTripleClick = false;
 }
 
+void KateViewInternal::beginSelectLine(const QPoint &pos)
+{
+    placeCursor(pos);
+    m_possibleTripleClick = true;
+    m_scrollX = 0;
+    m_scrollY = 0;
+}
+
 void KateViewInternal::mouseReleaseEvent(QMouseEvent *e)
 {
     switch (e->button()) {