diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -26,14 +26,14 @@
 target_link_libraries(KF5IconThemes
   PUBLIC
     Qt5::Widgets
-    KF5::ConfigWidgets # for KColorScheme
   PRIVATE
     Qt5::DBus
     Qt5::Svg
     KF5::Archive       # for KCompressionDevice
     KF5::I18n          # for i18n in KIconDialog
     KF5::WidgetsAddons # for KPixmapSequence family
     KF5::ItemViews      # for KListWidgetSearchLine
+    KF5::ConfigWidgets # for KColorScheme
     KF5::CoreAddons    # for kshareddatacache.h
 )
 
diff --git a/src/kiconloader.h b/src/kiconloader.h
--- a/src/kiconloader.h
+++ b/src/kiconloader.h
@@ -28,8 +28,6 @@
 
 #include <kiconthemes_export.h>
 
-#include <kcolorscheme.h>
-
 class QIcon;
 class QMovie;
 class QPixmap;
@@ -463,20 +461,18 @@
     bool hasIcon(const QString &iconName) const;
 
     /**
-     * The color set that will be used for the svg stylesheet in case the
-     * loaded icons are svg-based, icons can be colored in different ways in
-     * different areas of the application
-     * @return one of KColorScheme ColorSet
-     * @since 5.38
+     * Sets the colors for this KIconLoader
      */
-    KColorScheme::ColorSet colorSet() const;
+    void setCustomPalette(const QPalette &palette);
 
     /**
-     * Sets the ColorSet for this KIconLoader
-     * @param colorSet one of KColorScheme ColorSet values
+     * The colors that will be used for the svg stylesheet in case the
+     * loaded icons are svg-based, icons can be colored in different ways in
+     * different areas of the application
+     * @return the palette, if any, an invalid one if the user didn't set it
      * @since 5.38
      */
-    void setColorSet(KColorScheme::ColorSet colorSet);
+    QPalette customPalette() const;
 
 public Q_SLOTS:
     /**
diff --git a/src/kiconloader.cpp b/src/kiconloader.cpp
--- a/src/kiconloader.cpp
+++ b/src/kiconloader.cpp
@@ -134,15 +134,14 @@
     }
 }
 
-static QString paletteId(const QPalette &pal, KColorScheme::ColorSet colorSet)
+static QString paletteId(const QPalette &pal)
 {
     // 8 per color. We want 3 colors thus 8*3=24.
     QString buffer(24, Qt::Uninitialized);
 
-    uintToHex(pal.text().color().rgba(), buffer.data());
+    uintToHex(pal.windowText().color().rgba(), buffer.data());
     uintToHex(pal.highlight().color().rgba(), buffer.data() + 8);
     uintToHex(pal.highlightedText().color().rgba(), buffer.data() + 16);
-    buffer += QString::number(colorSet);
 
     return buffer;
 }
@@ -397,8 +396,8 @@
 
     QHash<QString, bool> mIconAvailability; // icon name -> true (known to be available) or false (known to be unavailable)
     QElapsedTimer mLastUnknownIconCheck; // recheck for unknown icons after kiconloader_ms_between_checks
-    //the kcolorscheme's colorset used to recolor svg icons stylesheets
-    KColorScheme::ColorSet mColorSet = KColorScheme::Window;
+    //the palette used to recolor svg icons stylesheets
+    QPalette mPalette;
 };
 
 class KIconLoaderGlobalData : public QObject
@@ -805,16 +804,6 @@
     d->drawOverlays(this, group, state, pixmap, overlays);
 }
 
-KColorScheme::ColorSet KIconLoader::colorSet() const
-{
-    return d->mColorSet;
-}
-
-void KIconLoader::setColorSet(KColorScheme::ColorSet colorSet)
-{
-    d->mColorSet = colorSet;
-}
-
 QString KIconLoaderPrivate::removeIconExtension(const QString &name) const
 {
     if (name.endsWith(QLatin1String(".png"))
@@ -877,7 +866,7 @@
            % (group >= 0 ? mpEffect.fingerprint(group, state)
               : NULL_EFFECT_FINGERPRINT())
            % QLatin1Char('_')
-           % paletteId(qApp->palette(), mColorSet)
+           % paletteId(mPalette)
            % (q->theme() && q->theme()->followsColorScheme() && state == KIconLoader::SelectedState ? QStringLiteral("_selected") : QString());
 }
 
@@ -895,11 +884,11 @@
         return QByteArray();
     }
 
-    KColorScheme scheme(QPalette::Active, state == KIconLoader::SelectedState ? KColorScheme::Selection : mColorSet);
+    KColorScheme scheme(QPalette::Active, KColorScheme::Window);
     QString styleSheet = STYLESHEET_TEMPLATE().arg(
-        scheme.foreground(KColorScheme::NormalText).color().name(),
-        scheme.background(KColorScheme::NormalBackground).color().name(),
-        scheme.decoration(KColorScheme::FocusColor).color().name(),
+        state == KIconLoader::SelectedState ? mPalette.highlightedText().color().name() : mPalette.windowText().color().name(),
+        state == KIconLoader::SelectedState ? mPalette.highlight().color().name() : mPalette.window().color().name(),
+        state == KIconLoader::SelectedState ? mPalette.highlightedText().color().name() : mPalette.highlight().color().name(),
         scheme.foreground(KColorScheme::PositiveText).color().name(),
         scheme.foreground(KColorScheme::NeutralText).color().name(),
         scheme.foreground(KColorScheme::NegativeText).color().name());
@@ -1793,6 +1782,16 @@
     return found;
 }
 
+void KIconLoader::setCustomPalette(const QPalette &palette)
+{
+    d->mPalette = palette;
+}
+
+QPalette KIconLoader::customPalette() const
+{
+    return d->mPalette;
+}
+
 /*** the global icon loader ***/
 Q_GLOBAL_STATIC(KIconLoader, globalIconLoader)