diff --git a/lib/documentview/rasterimageview.cpp b/lib/documentview/rasterimageview.cpp
--- a/lib/documentview/rasterimageview.cpp
+++ b/lib/documentview/rasterimageview.cpp
@@ -295,6 +295,8 @@
     } else if (zoomToFill()) {
         setZoom(computeZoomToFill(), QPointF(-1, -1), ForceUpdate);
     } else {
+        // Not only call updateBuffer, but also ensure the initial transformation mode
+        // of the image scaler is set correctly when zoom is unchanged (see Bug 396736).
         onZoomChanged();
     }
 
@@ -365,13 +367,7 @@
 
 void RasterImageView::onZoomChanged()
 {
-    // If we zoom to 400% or more, then assume the user wants to see the real
-    // pixels, for example to fine tune a crop operation
-    if (zoom() < 4.) {
-        d->mScaler->setTransformationMode(Qt::SmoothTransformation);
-    } else {
-        d->mScaler->setTransformationMode(Qt::FastTransformation);
-    }
+    d->mScaler->setZoom(zoom());
     if (!d->mUpdateTimer->isActive()) {
         updateBuffer();
     }
@@ -460,7 +456,6 @@
 void RasterImageView::updateBuffer(const QRegion& region)
 {
     d->mUpdateTimer->stop();
-    d->mScaler->setZoom(zoom());
     if (region.isEmpty()) {
         d->setScalerRegionToVisibleRect();
     } else {
diff --git a/lib/imagescaler.h b/lib/imagescaler.h
--- a/lib/imagescaler.h
+++ b/lib/imagescaler.h
@@ -49,8 +49,6 @@
     void setZoom(qreal);
     void setDestinationRegion(const QRegion&);
 
-    void setTransformationMode(Qt::TransformationMode);
-
 Q_SIGNALS:
     void scaledRect(int left, int top, const QImage&);
 
diff --git a/lib/imagescaler.cpp b/lib/imagescaler.cpp
--- a/lib/imagescaler.cpp
+++ b/lib/imagescaler.cpp
@@ -82,12 +82,12 @@
 
 void ImageScaler::setZoom(qreal zoom)
 {
-    d->mZoom = zoom;
-}
+    // If we zoom to 400% or more, then assume the user wants to see the real
+    // pixels, for example to fine tune a crop operation
+    d->mTransformationMode = zoom < 4. ? Qt::SmoothTransformation
+                                       : Qt::FastTransformation;
 
-void ImageScaler::setTransformationMode(Qt::TransformationMode mode)
-{
-    d->mTransformationMode = mode;
+    d->mZoom = zoom;
 }
 
 void ImageScaler::setDestinationRegion(const QRegion& region)
diff --git a/tests/auto/imagescalertest.cpp b/tests/auto/imagescalertest.cpp
--- a/tests/auto/imagescalertest.cpp
+++ b/tests/auto/imagescalertest.cpp
@@ -61,7 +61,8 @@
 
     QImage scaledImage = client.createFullImage();
 
-    QImage expectedImage = doc->image().scaled(doc->size() * zoom);
+    QImage expectedImage = doc->image().scaled(doc->size() * zoom,
+                                               Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
     QVERIFY(TestUtils::imageCompare(scaledImage, expectedImage));
 }