diff --git a/autotests/CMakeLists.txt b/autotests/CMakeLists.txt
--- a/autotests/CMakeLists.txt
+++ b/autotests/CMakeLists.txt
@@ -27,15 +27,21 @@
  mkpathjobtest.cpp
  threadtest.cpp
  udsentrytest.cpp
- udsentry_benchmark.cpp
- kcoredirlister_benchmark.cpp
  deletejobtest.cpp
  urlutiltest.cpp
  batchrenamejobtest.cpp
  ksambasharetest.cpp
  NAME_PREFIX "kiocore-"
  LINK_LIBRARIES KF5::KIOCore KF5::I18n Qt5::Test Qt5::Network
 )
+set (BUILD_BENCHMARK OFF)
+if (BUILD_BENCHMARK)
+    ecm_add_tests(
+        kcoredirlister_benchmark.cpp
+        udsentry_benchmark.cpp
+        NAME_PREFIX "kiocore-benchmark-"
+        LINK_LIBRARIES KF5::KIOCore KF5::I18n Qt5::Test Qt5::Network)
+endif()
 
 target_link_libraries(threadtest Qt5::Concurrent)