diff --git a/src/Screen.cpp b/src/Screen.cpp
--- a/src/Screen.cpp
+++ b/src/Screen.cpp
@@ -537,7 +537,8 @@
 QVector<LineProperty> Screen::getLineProperties(int startLine , int endLine) const
 {
     Q_ASSERT(startLine >= 0);
-    Q_ASSERT(endLine >= startLine && endLine < _history->getLines() + _lines);
+    Q_ASSERT(endLine >= startLine);
+    Q_ASSERT(endLine < _history->getLines() + _lines);
 
     const int mergedLines = endLine - startLine + 1;
     const int linesInHistory = qBound(0, _history->getLines() - startLine, mergedLines);
diff --git a/src/TerminalDisplay.h b/src/TerminalDisplay.h
--- a/src/TerminalDisplay.h
+++ b/src/TerminalDisplay.h
@@ -746,8 +746,9 @@
     bool _alternateScrolling;
     bool _bracketedPasteMode;
 
-    QPoint _iPntSel;  // initial selection point
-    QPoint _pntSel;  // current selection point
+    QPoint _initialSelectionPoint;  // initial selection point
+    QPoint _initialSelectionEnd;
+    QPoint _currentSelectionPoint;  // current selection point
     QPoint _tripleSelBegin;  // help avoid flicker
     int _actSel;     // selection state
     bool _wordSelectionMode;
diff --git a/src/TerminalDisplay.cpp b/src/TerminalDisplay.cpp
--- a/src/TerminalDisplay.cpp
+++ b/src/TerminalDisplay.cpp
@@ -59,6 +59,8 @@
 #include <KMessageWidget>
 #include <KIO/StatJob>
 
+#include <memory>
+
 // Konsole
 #include "Filter.h"
 #include "konsoledebug.h"
@@ -425,8 +427,8 @@
     , _usesMouseTracking(false)
     , _alternateScrolling(true)
     , _bracketedPasteMode(false)
-    , _iPntSel(QPoint())
-    , _pntSel(QPoint())
+    , _initialSelectionPoint(QPoint())
+    , _currentSelectionPoint(QPoint())
     , _tripleSelBegin(QPoint())
     , _actSel(0)
     , _wordSelectionMode(false)
@@ -2136,7 +2138,7 @@
                 _screenWindow->clearSelection();
 
                 pos.ry() += _scrollBar->value();
-                _iPntSel = _pntSel = pos;
+                _initialSelectionPoint = _currentSelectionPoint = pos;
                 _actSel = 1; // left mouse button pressed but nothing selected yet.
             } else if (_usesMouseTracking && !_readOnly) {
                     emit mouseSignal(0, charColumn + 1, charLine + 1 + _scrollBar->value() - _scrollBar->maximum() , 0);
@@ -2315,11 +2317,13 @@
 
     int linesBeyondWidget = 0;
 
-    QRect textBounds(tLx + _contentRect.left(),
+    const QRect textBounds(tLx + _contentRect.left(),
                      tLy + _contentRect.top(),
                      _usedColumns * _fontWidth - 1,
                      _usedLines * _fontHeight - 1);
 
+    const QRect characterBounds(0, 0, _columns, _lines);
+
     QPoint pos = position;
 
     // Adjust position within text area bounds.
@@ -2339,83 +2343,72 @@
 
     int charColumn = 0;
     int charLine = 0;
-    getCharacterPosition(pos, charLine, charColumn, true);
+    getCharacterPosition(pos, charLine, charColumn, !_wordSelectionMode && !_lineSelectionMode);
 
-    QPoint here = QPoint(charColumn, charLine);
-    QPoint ohere;
-    QPoint _iPntSelCorr = _iPntSel;
-    _iPntSelCorr.ry() -= _scrollBar->value();
-    QPoint _pntSelCorr = _pntSel;
-    _pntSelCorr.ry() -= _scrollBar->value();
+    const QPoint cursorPosition(charColumn, charLine);
+    QPoint selectionEnd = QPoint(charColumn, charLine);
+    QPoint selectionStart;
+    QPoint initialSelectionBegin = _initialSelectionPoint;
+    initialSelectionBegin.ry() -= _scrollBar->value();
+    QPoint initialSelectionEnd = _initialSelectionEnd;
+    initialSelectionEnd.ry() -= _scrollBar->value();
+    QPoint previousSelectionPoint = _currentSelectionPoint;
+    previousSelectionPoint.ry() -= _scrollBar->value();
     bool swapping = false;
 
+    int offset = 0;
+
     if (_wordSelectionMode) {
         // Extend to word boundaries
-        const bool left_not_right = (here.y() < _iPntSelCorr.y() ||
-                                     (here.y() == _iPntSelCorr.y() && here.x() < _iPntSelCorr.x()));
-        const bool old_left_not_right = (_pntSelCorr.y() < _iPntSelCorr.y() ||
-                                         (_pntSelCorr.y() == _iPntSelCorr.y() && _pntSelCorr.x() < _iPntSelCorr.x()));
-        swapping = left_not_right != old_left_not_right;
+        const bool extendingLeft = (selectionEnd.y() < initialSelectionBegin.y() ||
+                                     (selectionEnd.y() == initialSelectionBegin.y() && selectionEnd.x() < initialSelectionBegin.x()));
+        const bool wasExtendingLeft = (previousSelectionPoint.y() < initialSelectionBegin.y() ||
+                                         (previousSelectionPoint.y() == initialSelectionBegin.y() && previousSelectionPoint.x() < initialSelectionBegin.x()));
+        swapping = extendingLeft != wasExtendingLeft;
 
-        // Find left (left_not_right ? from here : from start of word)
-        QPoint left = left_not_right ? here : _iPntSelCorr;
-        // Find left (left_not_right ? from end of word : from here)
-        QPoint right = left_not_right ? _iPntSelCorr : here;
-
-        if (left.y() < 0 || left.y() >= _lines || left.x() < 0 || left.x() >= _columns) {
-            left = _pntSelCorr;
+        if (extendingLeft) {
+            selectionEnd = findWordStart(cursorPosition);
+            selectionStart = initialSelectionEnd;
         } else {
-            left = findWordStart(left);
-        }
-        if (right.y() < 0 || right.y() >= _lines || right.x() < 0 || right.x() >= _columns) {
-            right = _pntSelCorr;
-        } else {
-            right = findWordEnd(right);
+            selectionEnd = findWordEnd(cursorPosition);
+            selectionStart = initialSelectionBegin;
         }
 
-        // Pick which is start (ohere) and which is extension (here)
-        if (left_not_right) {
-            here = left;
-            ohere = right;
-        } else {
-            here = right;
-            ohere = left;
-        }
-        ohere.rx()++;
+        selectionStart.rx()++;
     }
 
     if (_lineSelectionMode) {
         // Extend to complete line
-        const bool above_not_below = (here.y() < _iPntSelCorr.y());
-        if (above_not_below) {
-            ohere = findLineEnd(_iPntSelCorr);
-            here = findLineStart(here);
+        const bool extendingUpwards = (selectionEnd.y() < initialSelectionBegin.y());
+        const bool wasExtendingUpwards = (previousSelectionPoint.y() < initialSelectionBegin.y());
+        swapping = extendingUpwards != wasExtendingUpwards;
+        if (extendingUpwards) {
+            selectionEnd = findLineStart(selectionEnd);
+            selectionStart = initialSelectionEnd;
         } else {
-            ohere = findLineStart(_iPntSelCorr);
-            here = findLineEnd(here);
+            selectionStart = initialSelectionBegin;
+            selectionEnd = findLineEnd(selectionEnd);
         }
 
-        swapping = !(_tripleSelBegin == ohere);
-        _tripleSelBegin = ohere;
+        _tripleSelBegin = selectionStart;
 
-        ohere.rx()++;
+        selectionStart.rx()++;
     }
 
-    int offset = 0;
     if (!_wordSelectionMode && !_lineSelectionMode) {
         QChar selClass;
 
-        const bool left_not_right = (here.y() < _iPntSelCorr.y() ||
-                                     (here.y() == _iPntSelCorr.y() && here.x() < _iPntSelCorr.x()));
-        const bool old_left_not_right = (_pntSelCorr.y() < _iPntSelCorr.y() ||
-                                         (_pntSelCorr.y() == _iPntSelCorr.y() && _pntSelCorr.x() < _iPntSelCorr.x()));
-        swapping = left_not_right != old_left_not_right;
+        const bool extendingLeft = (selectionEnd.y() < initialSelectionBegin.y() ||
+                                     (selectionEnd.y() == initialSelectionBegin.y() && selectionEnd.x() < initialSelectionBegin.x()));
+        const bool wasExtendingLeft = (previousSelectionPoint.y() < initialSelectionBegin.y() ||
+                                         (previousSelectionPoint.y() == initialSelectionBegin.y() && previousSelectionPoint.x() < initialSelectionBegin.x()));
+        swapping = extendingLeft != wasExtendingLeft;
 
         // Find left (left_not_right ? from here : from start)
-        const QPoint left = left_not_right ? here : _iPntSelCorr;
+        const QPoint left = extendingLeft ? selectionEnd : initialSelectionBegin;
 
         // Find left (left_not_right ? from start : from here)
-        QPoint right = left_not_right ? _iPntSelCorr : here;
+        QPoint right = extendingLeft ? initialSelectionBegin : selectionEnd;
         if (right.x() > 0 && !_columnSelectionMode) {
             if (right.x() - 1 < _columns && right.y() < _lines) {
                 selClass = charClass(_image[loc(right.x() - 1, right.y())]);
@@ -2423,41 +2416,41 @@
         }
 
         // Pick which is start (ohere) and which is extension (here)
-        if (left_not_right) {
-            here = left;
-            ohere = right;
+        if (extendingLeft) {
+            selectionEnd = left;
+            selectionStart = right;
             offset = 0;
         } else {
-            here = right;
-            ohere = left;
+            selectionEnd = right;
+            selectionStart = left;
             offset = -1;
         }
     }
 
-    if ((here == _pntSelCorr) && (scroll == _scrollBar->value())) {
+    if ((selectionEnd == previousSelectionPoint) && (scroll == _scrollBar->value())) {
         return; // not moved
     }
 
-    if (here == ohere) {
+    if (selectionEnd == selectionStart) {
         return; // It's not left, it's not right.
     }
 
     if (_actSel < 2 || swapping) {
         if (_columnSelectionMode && !_lineSelectionMode && !_wordSelectionMode) {
-            _screenWindow->setSelectionStart(ohere.x() , ohere.y() , true);
+            _screenWindow->setSelectionStart(selectionStart.x() , selectionStart.y() , true);
         } else {
-            _screenWindow->setSelectionStart(ohere.x() - 1 - offset , ohere.y() , false);
+            _screenWindow->setSelectionStart(selectionStart.x() - 1 - offset , selectionStart.y() , false);
         }
     }
 
     _actSel = 2; // within selection
-    _pntSel = here;
-    _pntSel.ry() += _scrollBar->value();
+    _currentSelectionPoint = selectionEnd;
+    _currentSelectionPoint.ry() += _scrollBar->value();
 
     if (_columnSelectionMode && !_lineSelectionMode && !_wordSelectionMode) {
-        _screenWindow->setSelectionEnd(here.x() , here.y());
+        _screenWindow->setSelectionEnd(selectionEnd.x() , selectionEnd.y());
     } else {
-        _screenWindow->setSelectionEnd(here.x() + offset , here.y());
+        _screenWindow->setSelectionEnd(selectionEnd.x() + offset , selectionEnd.y());
     }
 }
 
@@ -2585,8 +2578,8 @@
     }
 
     _screenWindow->clearSelection();
-    _iPntSel = pos;
-    _iPntSel.ry() += _scrollBar->value();
+    _currentSelectionPoint = pos;
+    _currentSelectionPoint.ry() += _scrollBar->value();
 
     _wordSelectionMode = true;
     _actSel = 2; // within selection
@@ -2602,6 +2595,12 @@
         _screenWindow->setSelectionStart(bgnSel.x() , bgnSel.y() , false);
         _screenWindow->setSelectionEnd(endSel.x() , endSel.y());
 
+        _initialSelectionPoint = bgnSel;
+        _initialSelectionPoint.ry() += _scrollBar->value();
+
+        _initialSelectionEnd = endSel;
+        _initialSelectionEnd.ry() += _scrollBar->value();
+
         copyToX11Selection();
     }
 
@@ -2764,9 +2763,14 @@
 
     Screen *screen = _screenWindow->screen();
     Character *image = _image;
-    Character *tmp_image = nullptr;
+    std::unique_ptr<Character[]> tempImage;
 
     int imgLine = pnt.y();
+
+    if (imgLine < 0 || imgLine >= _lines) {
+        return pnt;
+    }
+
     int x = pnt.x();
     int y = imgLine + firstVisibleLine;
     int imgLoc = loc(x, imgLine);
@@ -2774,58 +2778,47 @@
     const QChar selClass = charClass(image[imgLoc]);
     const int imageSize = regSize * _columns;
 
-    while (true) {
-        for (;;imgLoc--, x--) {
-            if (imgLoc < 1) {
-                // no more chars in this region
-                break;
+    while (imgLoc >= 0) {
+        for (; imgLoc > 0 && imgLine >= 0; imgLoc--, x--) {
+            const QChar &curClass = charClass(image[imgLoc - 1]);
+            if (curClass != selClass) {
+                return {x, y - firstVisibleLine};
             }
+
+            // has previous char on this line
             if (x > 0) {
-                // has previous char on this line
-                if (charClass(image[imgLoc - 1]) == selClass) {
-                    continue;
-                }
-                goto out;
-            } else if (imgLine > 0) {
-                // not the first line in the session
-                if ((lineProperties[imgLine - 1] & LINE_WRAPPED) != 0) {
-                    // have continuation on prev line
-                    if (charClass(image[imgLoc - 1]) == selClass) {
-                        x = _columns;
-                        imgLine--;
-                        y--;
-                        continue;
-                    }
-                }
-                goto out;
-            } else if (y > 0) {
-                // want more data, but need to fetch new region
-                break;
-            } else {
-                goto out;
+                continue;
             }
+
+            // not the first line in the session
+            if ((lineProperties[imgLine - 1] & LINE_WRAPPED) == 0) {
+                return {x, y - firstVisibleLine};
+            }
+
+            // have continuation on prev line
+            x = _columns;
+            imgLine--;
+            y--;
         }
+
         if (y <= 0) {
-            // No more data
-            goto out;
+            return {x, y - firstVisibleLine};
         }
-        int newRegStart = qMax(0, y - regSize + 1);
+
+        // Fetch new region
+        const int newRegStart = qMax(y - regSize + 1, 0);
         lineProperties = screen->getLineProperties(newRegStart, y - 1);
         imgLine = y - newRegStart;
 
-        delete[] tmp_image;
-        tmp_image = new Character[imageSize];
-        image = tmp_image;
-
-        screen->getImage(tmp_image, imageSize, newRegStart, y - 1);
-        imgLoc = loc(x, imgLine);
-        if (imgLoc < 1) {
-            // Reached the start of the session
-            break;
+        if (!tempImage) {
+            tempImage.reset(new Character[imageSize]);
+            image = tempImage.get();
         }
+
+        screen->getImage(image, imageSize, newRegStart, y - 1);
+        imgLoc = loc(x, imgLine);
     }
-out:
-    delete[] tmp_image;
+
     return {x, y - firstVisibleLine};
 }
 
@@ -2833,65 +2826,81 @@
 {
     const int regSize = qMax(_screenWindow->windowLines(), 10);
     const int curLine = _screenWindow->currentLine();
-    int i = pnt.y();
+    int line = pnt.y();
+
+    // The selection is already scrolled out of view, so assume it is already at a boundary
+    if (line < 0 || line >= _lines) {
+        return pnt;
+    }
+
     int x = pnt.x();
-    int y = i + curLine;
-    int j = loc(x, i);
+    int y = line + curLine;
     QVector<LineProperty> lineProperties = _lineProperties;
     Screen *screen = _screenWindow->screen();
     Character *image = _image;
-    Character *tmp_image = nullptr;
-    const QChar selClass = charClass(image[j]);
+    std::unique_ptr<Character[]> tempImage;
+
+    int imgPos = loc(x, line);
+    const QChar selClass = charClass(image[imgPos]);
+    QChar curClass;
+    QChar nextClass;
+
     const int imageSize = regSize * _columns;
-    const int maxY = _screenWindow->lineCount() - 1;
+    const int maxY = _screenWindow->lineCount();
     const int maxX = _columns - 1;
 
-    while (true) {
-        const int lineCount = lineProperties.count();
-        for (;;j++, x++) {
-            if (x < maxX) {
-                if (charClass(image[j + 1]) == selClass &&
-                    // A colon right before whitespace is never part of a word
-                    ! (image[j + 1].character == ':' && charClass(image[j + 2]) == QLatin1Char(' '))) {
-                    continue;
-                }
-                goto out;
-            } else if (i < lineCount - 1) {
-                if (((lineProperties[i] & LINE_WRAPPED) != 0) &&
-                    charClass(image[j + 1]) == selClass &&
-                    // A colon right before whitespace is never part of a word
-                    ! (image[j + 1].character == ':' && charClass(image[j + 2]) == QLatin1Char(' '))) {
-                    x = -1;
-                    i++;
-                    y++;
-                    continue;
-                }
-                goto out;
-            } else if (y < maxY) {
-                if (i < lineCount && ((lineProperties[i] & LINE_WRAPPED) == 0)) {
-                    goto out;
-                }
+    while (x >= 0 && line >= 0) {
+        imgPos = loc(x, line);
+
+        const int visibleLinesCount = lineProperties.count();
+        bool changedClass = false;
+
+        for (;y < maxY && line < visibleLinesCount; imgPos++, x++) {
+            curClass = charClass(image[imgPos + 1]);
+            nextClass = charClass(image[imgPos + 2]);
+
+            changedClass = curClass != selClass &&
+                // A colon right before whitespace is never part of a word
+                !(image[imgPos + 1].character == ':' && nextClass == QLatin1Char(' '));
+
+            if (changedClass) {
                 break;
-            } else {
-                goto out;
+            }
+
+            if (x >= maxX) {
+                if ((lineProperties[line] & LINE_WRAPPED) == 0) {
+                    break;
+                }
+
+                line++;
+                y++;
+                x = -1;
             }
         }
-        int newRegEnd = qMin(y + regSize - 1, maxY);
-        lineProperties = screen->getLineProperties(y, newRegEnd);
-        i = 0;
-        if (tmp_image == nullptr) {
-            tmp_image = new Character[imageSize];
-            image = tmp_image;
+
+        if (changedClass) {
+            break;
         }
-        screen->getImage(tmp_image, imageSize, y, newRegEnd);
-        x--;
-        j = loc(x, i);
+
+        if (line < visibleLinesCount && ((lineProperties[line] & LINE_WRAPPED) == 0)) {
+            break;
+        }
+
+        const int newRegEnd = qMin(y + regSize - 1, maxY - 1);
+        lineProperties = screen->getLineProperties(y, newRegEnd);
+        if (!tempImage) {
+            tempImage.reset(new Character[imageSize]);
+            image = tempImage.get();
+        }
+        screen->getImage(tempImage.get(), imageSize, y, newRegEnd);
+
+        line = 0;
     }
-out:
+
     y -= curLine;
     // In word selection mode don't select @ (64) if at end of word.
-    if (((image[j].rendition & RE_EXTENDED_CHAR) == 0) &&
-        (QChar(image[j].character) == QLatin1Char('@')) &&
+    if (((image[imgPos].rendition & RE_EXTENDED_CHAR) == 0) &&
+        (QChar(image[imgPos].character) == QLatin1Char('@')) &&
         (y > pnt.y() || x > pnt.x())) {
         if (x > 0) {
             x--;
@@ -2899,7 +2908,6 @@
             y--;
         }
     }
-    delete[] tmp_image;
 
     return {x, y};
 }
@@ -2935,7 +2943,7 @@
 
 void TerminalDisplay::selectLine(QPoint pos, bool entireLine)
 {
-    _iPntSel = pos;
+    _initialSelectionPoint = pos;
 
     _screenWindow->clearSelection();
 
@@ -2945,20 +2953,23 @@
     _actSel = 2; // within selection
 
     if (!entireLine) { // Select from cursor to end of line
-        _tripleSelBegin = findWordStart(_iPntSel);
+        _tripleSelBegin = findWordStart(pos);
         _screenWindow->setSelectionStart(_tripleSelBegin.x(),
                                          _tripleSelBegin.y() , false);
     } else {
-        _tripleSelBegin = findLineStart(_iPntSel);
+        _tripleSelBegin = findLineStart(pos);
         _screenWindow->setSelectionStart(0 , _tripleSelBegin.y() , false);
     }
 
-    _iPntSel = findLineEnd(_iPntSel);
-    _screenWindow->setSelectionEnd(_iPntSel.x() , _iPntSel.y());
+    _initialSelectionPoint = findLineStart(_initialSelectionPoint);
+    _initialSelectionEnd = findLineEnd(pos);
+
+    _screenWindow->setSelectionEnd(_initialSelectionEnd.x() , _initialSelectionEnd.y());
 
     copyToX11Selection();
 
-    _iPntSel.ry() += _scrollBar->value();
+    _initialSelectionPoint.ry() += _scrollBar->value();
+    _initialSelectionEnd.ry() += _scrollBar->value();
 }
 
 void TerminalDisplay::selectCurrentLine()