diff --git a/autotests/data/testpackage/metadata.desktop b/autotests/data/testpackage/metadata.desktop
--- a/autotests/data/testpackage/metadata.desktop
+++ b/autotests/data/testpackage/metadata.desktop
@@ -44,6 +44,7 @@
 Name[x-test]=xxTest Packagexx
 Name[zh_CN]=测试包
 Name[zh_TW]=測試套件
+Comment=fancy shmancy summary
 Type=Service
 
 X-KDE-ServiceTypes=KPackage/Generic
diff --git a/autotests/data/testpackage/testpackage.testappdataxml b/autotests/data/testpackage/testpackage.testappdataxml
--- a/autotests/data/testpackage/testpackage.testappdataxml
+++ b/autotests/data/testpackage/testpackage.testappdataxml
@@ -44,6 +44,7 @@
     <name xml:lang="x-test">xxTest Packagexx</name>
     <name xml:lang="zh_CN">测试包</name>
     <name xml:lang="zh_TW">測試套件</name>
+    <summary>fancy shmancy summary</summary>
     <developer_name>Joe Blow &lt;jblow@kde.org&gt;</developer_name>
     <icon type="stock">plasma</icon>
     <project_license>GPL-2.0+</project_license>
diff --git a/autotests/kpackagetoolappstreamtest.cmake b/autotests/kpackagetoolappstreamtest.cmake
--- a/autotests/kpackagetoolappstreamtest.cmake
+++ b/autotests/kpackagetoolappstreamtest.cmake
@@ -7,3 +7,21 @@
 if (error_compare)
     message(FATAL_ERROR "error on compare: ${error_compare}")
 endif()
+
+# Make sure the standard test passes appstream validation.
+get_filename_component(generated_name ${generated} NAME)
+if(${generated_name} STREQUAL "testpackage.appdata.xml")
+    find_program(APPSTREAMCLI appstreamcli)
+    if(APPSTREAMCLI)
+        execute_process(COMMAND ${APPSTREAMCLI} validate ${generated}
+            ERROR_VARIABLE appstream_stderr
+            OUTPUT_VARIABLE appstream_stdout
+            RESULT_VARIABLE result
+        )
+        if(NOT result EQUAL 0)
+            message(FATAL_ERROR "appstream data seems to be imperfect: ${appstream_stderr} ${appstream_stdout}")
+        endif()
+    else()
+        message(WARNING "skipping appstream validation as no appstreamcli binary was found")
+    endif()
+endif()