diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 885c7d859f4..7290a979452 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,56 +1,57 @@
 
 find_program(BASH_PROGRAM bash)
 find_program(XGETTEXT_PROGRAM xgettext)
 find_program(MSGCAT_PROGRAM msgcat)
 
 if (BASH_PROGRAM AND XGETTEXT_PROGRAM AND MSGCAT_PROGRAM)
     macro(CALLIGRA_ADD_TEST_COMMAND _TEST_NAME)
         add_test(${_TEST_NAME} ${BASH_PROGRAM} "${CMAKE_CURRENT_SOURCE_DIR}/${_TEST_NAME}.sh"
                                 "${CMAKE_SOURCE_DIR}/kundo2_aware_xgettext.sh"
                                 "${XGETTEXT_PROGRAM}"
                                 "${MSGCAT_PROGRAM}"
                                 "${CMAKE_CURRENT_BINARY_DIR}/po"
         )
     endmacro()
     macro(CALLIGRA_ADD_UNIT_TEST _TEST_NAME)
         add_custom_target(${_TEST_NAME} ALL
             COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/${_TEST_NAME}.cpp" "${CMAKE_CURRENT_BINARY_DIR}/${_TEST_NAME}.cpp"
             COMMENT "Copy test files" VERBATIM
         )
         CALLIGRA_ADD_TEST_COMMAND(${_TEST_NAME})
     endmacro()
 
     # create the podir
     add_custom_target(podir ALL
         COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/po
         COMMENT "Create podir" VERBATIM)
     # copy parameters script
     add_custom_target(parameters ALL
         COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/parameters.sh ${CMAKE_CURRENT_BINARY_DIR}/parameters.sh
         COMMENT "Copy parameters script" VERBATIM)
 
     # add tests here
     CALLIGRA_ADD_UNIT_TEST(test_i18n)
     CALLIGRA_ADD_UNIT_TEST(test_i18nc)
     CALLIGRA_ADD_UNIT_TEST(test_i18nc_long)
 
     CALLIGRA_ADD_UNIT_TEST(test_i18np)
     CALLIGRA_ADD_UNIT_TEST(test_i18n_noop)
     CALLIGRA_ADD_UNIT_TEST(test_i18n_noop2)
 
     CALLIGRA_ADD_UNIT_TEST(test_xi18n)
     CALLIGRA_ADD_UNIT_TEST(test_xi18nc)
     CALLIGRA_ADD_UNIT_TEST(test_xi18nc_long)
     CALLIGRA_ADD_UNIT_TEST(test_xi18np)
     CALLIGRA_ADD_UNIT_TEST(test_xi18ncp)
 
     CALLIGRA_ADD_UNIT_TEST(test_kundo2_i18n)
     CALLIGRA_ADD_UNIT_TEST(test_kundo2_i18nc)
     CALLIGRA_ADD_UNIT_TEST(test_kundo2_i18nc_long)
 
     # this uses cpp files from earlier tests
     CALLIGRA_ADD_TEST_COMMAND(test_i18n_mixed)
 
     # add test of Messages.sh
     add_test(test_Messages ${BASH_PROGRAM} "${CMAKE_CURRENT_SOURCE_DIR}/test_messages.sh" "${CMAKE_SOURCE_DIR}")
+
 endif()
diff --git a/tests/test_installed_application.sh b/tests/test_installed_application.sh
new file mode 100644
index 00000000000..4e54b09c0e1
--- /dev/null
+++ b/tests/test_installed_application.sh
@@ -0,0 +1,51 @@
+#! /bin/sh
+# test_messages:
+# $1: program
+# $2: version
+# $3: input file
+# $4: data output directory
+
+# check input parameters
+
+prog=$(command -v "$1")
+
+if ! echo "x_$prog" | grep -Fxq "x_$1"; then
+    echo "FAIL  Cannot find correct program: $1, found: $prog"
+    echo "Note: $1 must be installed to run this test."
+    echo "      E.g: 'make install && ctest'"
+    exit 1;
+fi
+
+if test -z "$3"; then
+    echo "FAIL  Called without input file"
+    exit 1
+fi
+
+if ! test -f "$3"; then
+    echo "FAIL  $3 is not a file"
+    exit 1
+fi
+
+if test -z "$4"; then
+    echo "FAIL Called without data output directory"
+    exit 1
+fi
+if ! test -d "$4"; then
+    echo "FAIL $4 is not a directory"
+    exit 1
+fi
+
+
+version=$("$prog" --version | cut -d ' ' -f 2)
+echo "Testing: $prog version: $version"
+if echo "$version" | grep -Fxq "$2"; then
+    if ! $("$prog" --benchmark-loading '$3'); then
+        echo "FAIL $prog failed"
+        exit 1
+    fi
+else
+    echo "FAIL Not correct version: $2"
+    exit 1
+fi
+
+exit 0