Index: autotests/jobtest.h =================================================================== --- autotests/jobtest.h +++ autotests/jobtest.h @@ -130,6 +130,10 @@ void enterLoop(); enum { AlreadyExists = 1 }; void copyLocalFile(const QString &src, const QString &dest); + bool checkXattrFsSupport(const QString &writeTest); + bool setXattr(const QString &src); + QList readXattr(const QString &src); + void compareXattr(const QString &src, const QString &dest); void copyLocalDirectory(const QString &src, const QString &dest, int flags = 0); void moveLocalFile(const QString &src, const QString &dest); void moveLocalDirectory(const QString &src, const QString &dest); @@ -144,6 +148,9 @@ QStringList m_names; int m_dataReqCount; QString m_mimetype; + QString m_setXattrCmd; + QString m_getXattrCmd; + std::function m_setXattrFormatArgs; }; #endif Index: autotests/jobtest.cpp =================================================================== --- autotests/jobtest.cpp +++ autotests/jobtest.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -82,6 +83,39 @@ } } + /***** + * Set platform xattr related commands. + * Linux commands: setfattr, getfattr + * BSD commands: setextattr, getextattr + * MacOS commands: xattr -w, xattr -p + ****/ + m_getXattrCmd = QStandardPaths::findExecutable("getfattr"); + if (m_getXattrCmd.endsWith("getfattr")) { + m_setXattrCmd = QStandardPaths::findExecutable("setfattr"); + m_setXattrFormatArgs = [](const QString& attrName, const QString& value, const QString& fileName) { + return QStringList{QLatin1String("-n"), attrName, QLatin1String("-v"), value, fileName}; + }; + } else { + // On BSD there is lsextattr to list all xattrs and getextattr to get a value + // for specific xattr. For test purposes lsextattr is more suitable to be used + // as m_getXattrCmd, so search for it instead of getextattr. + m_getXattrCmd = QStandardPaths::findExecutable("lsextattr"); + if (m_getXattrCmd.endsWith("lsextattr")) { + m_setXattrCmd = QStandardPaths::findExecutable("setextattr"); + m_setXattrFormatArgs = [](const QString& attrName, const QString& value, const QString& fileName) { + return QStringList{QLatin1String("user"), attrName, value, fileName}; + }; + } else { + m_getXattrCmd = QStandardPaths::findExecutable("xattr"); + m_setXattrFormatArgs = [](const QString& attrName, const QString& value, const QString& fileName) { + return QStringList{QLatin1String("-w"), attrName, value, fileName}; + }; + if (!m_getXattrCmd.endsWith("xattr")) { + qWarning() << "Neither getfattr, getextattr nor xattr was found."; + } + } + } + qRegisterMetaType("KJob*"); qRegisterMetaType("KIO::Job*"); qRegisterMetaType("QDateTime"); @@ -414,7 +448,98 @@ QTRY_VERIFY(jobFinished); } -//// +static QHash getSampleXattrs() +{ + QHash attrs; + attrs["user.name with space"] = "value with spaces"; + attrs["user.baloo.rating"] = "1"; + attrs["user.fnewLine"] = "line1\\nline2"; + attrs["user.flistNull"] = "item1\\0item2"; + attrs["user.fattr.with.a.lot.of.namespaces"] = "true"; + attrs["user.fempty"] = ""; + return attrs; +} + +bool JobTest::checkXattrFsSupport(const QString &dir) +{ + const QString writeTest = dir + "/fsXattrTestFile"; + createTestFile(writeTest); + bool ret = setXattr(writeTest); + QFile::remove(writeTest); + return ret; +} + +bool JobTest::setXattr(const QString &dest) +{ + QProcess xattrWriter; + xattrWriter.setProcessChannelMode(QProcess::MergedChannels); + + QHash attrs = getSampleXattrs(); + QHashIterator i(attrs); + while (i.hasNext()) { + i.next(); + QStringList arguments = m_setXattrFormatArgs(i.key(), i.value(), dest); + xattrWriter.start(m_setXattrCmd, arguments); + xattrWriter.waitForStarted(); + xattrWriter.waitForFinished(-1); + if(xattrWriter.exitStatus() != QProcess::NormalExit) { + return false; + } + QList resultdest = xattrWriter.readAllStandardOutput().split('\n'); + if (!resultdest[0].isEmpty()) { + QWARN("Error writing user xattr. Xattr copy tests will be disabled."); + qDebug() << resultdest; + return false; + } + } + + return true; +} + +QList JobTest::readXattr(const QString &src) +{ + QProcess xattrReader; + xattrReader.setProcessChannelMode(QProcess::MergedChannels); + + QStringList arguments; + char outputSeparator = '\n'; + // Linux + if (m_getXattrCmd.endsWith("getfattr")) { + arguments = QStringList {"-d", src}; + } + // BSD + else if (m_getXattrCmd.endsWith("lsextattr")) { + arguments = QStringList {"-q", "user", src}; + outputSeparator = '\t'; + } + // MacOS + else { + arguments = QStringList {"-l", src }; + } + + xattrReader.start(m_getXattrCmd, arguments); + xattrReader.waitForFinished(); + QList result = xattrReader.readAllStandardOutput().split(outputSeparator); + if (m_getXattrCmd.endsWith("getfattr")) { + // Line 1 is the file name + result.removeAt(1); + } + else if (m_getXattrCmd.endsWith("lsextattr")) { + // cut off trailing \n + result.last().chop(1); + // lsextattr does not sort its output + std::sort(result.begin(), result.end()); + } + + return result; +} + +void JobTest::compareXattr(const QString &src, const QString &dest) +{ + auto srcAttrs = readXattr(src); + auto dstAttrs = readXattr(dest); + QCOMPARE(dstAttrs, srcAttrs); +} void JobTest::copyLocalFile(const QString &src, const QString &dest) { @@ -429,6 +554,7 @@ QVERIFY(QFile::exists(dest)); QVERIFY(QFile::exists(src)); // still there QCOMPARE(int(QFileInfo(dest).permissions()), int(0x6666)); + compareXattr(src, dest); { // check that the timestamp is the same (#24443) @@ -454,6 +580,7 @@ QVERIFY2(job->exec(), qPrintable(job->errorString())); QVERIFY(QFile::exists(dest)); QVERIFY(QFile::exists(src)); // still there + compareXattr(src, dest); { // check that the timestamp is the same (#24443) QFileInfo srcInfo(src); @@ -476,22 +603,25 @@ QVERIFY2(job->exec(), qPrintable(job->errorString())); QVERIFY(QFile::exists(dest)); QVERIFY(QFile::exists(src)); // still there + compareXattr(src, dest); // Do it again, with Overwrite. job = KIO::copyAs(u, d, KIO::Overwrite | KIO::HideProgressInfo); job->setUiDelegate(nullptr); job->setUiDelegateExtension(nullptr); QVERIFY2(job->exec(), qPrintable(job->errorString())); QVERIFY(QFile::exists(dest)); QVERIFY(QFile::exists(src)); // still there + compareXattr(src, dest); // Do it again, without Overwrite (should fail). job = KIO::copyAs(u, d, KIO::HideProgressInfo); job->setUiDelegate(nullptr); job->setUiDelegateExtension(nullptr); QVERIFY(!job->exec()); // Clean up + QFile::remove(src); QFile::remove(dest); } @@ -580,9 +710,13 @@ void JobTest::copyFileToSamePartition() { - const QString filePath = homeTmpDir() + "fileFromHome"; - const QString dest = homeTmpDir() + "fileFromHome_copied"; + const QString homeDir = homeTmpDir(); + const QString filePath = homeDir + "fileFromHome"; + const QString dest = homeDir + "fileFromHome_copied"; createTestFile(filePath); + if (checkXattrFsSupport(homeDir)) { + setXattr(filePath); + } copyLocalFile(filePath, dest); } @@ -642,9 +776,16 @@ void JobTest::copyFileToOtherPartition() { // qDebug(); - const QString filePath = homeTmpDir() + "fileFromHome"; - const QString dest = otherTmpDir() + "fileFromHome_copied"; + const QString homeDir = homeTmpDir(); + const QString otherHomeDir = otherTmpDir(); + const QString filePath = homeDir + "fileFromHome"; + const QString dest = otherHomeDir + "fileFromHome_copied"; + bool canRead = checkXattrFsSupport(homeDir); + bool canWrite = checkXattrFsSupport(otherHomeDir); createTestFile(filePath); + if (canRead && canWrite) { + setXattr(filePath); + } copyLocalFile(filePath, dest); } Index: src/ioslaves/file/ConfigureChecks.cmake =================================================================== --- src/ioslaves/file/ConfigureChecks.cmake +++ src/ioslaves/file/ConfigureChecks.cmake @@ -9,6 +9,9 @@ check_include_files(sys/time.h HAVE_SYS_TIME_H) check_include_files(string.h HAVE_STRING_H) check_include_files(limits.h HAVE_LIMITS_H) +check_include_files(sys/xattr.h HAVE_SYS_XATTR_H) +check_include_files("sys/types.h;sys/extattr.h" HAVE_SYS_EXTATTR_H) + check_function_exists(sendfile HAVE_SENDFILE) check_function_exists(posix_fadvise HAVE_FADVISE) # kioslave Index: src/ioslaves/file/config-kioslave-file.h.cmake =================================================================== --- src/ioslaves/file/config-kioslave-file.h.cmake +++ src/ioslaves/file/config-kioslave-file.h.cmake @@ -10,9 +10,12 @@ /* Defined to if you have a d_type member in struct dirent */ #cmakedefine01 HAVE_DIRENT_D_TYPE -/* Defined if system has extended file attributes support. */ +/* Defined if system has header file. */ #cmakedefine01 HAVE_SYS_XATTR_H +/* Defined if system has header file. */ +#cmakedefine01 HAVE_SYS_EXTATTR_H + /* Defined if system has the sendfile function. */ #cmakedefine01 HAVE_SENDFILE Index: src/ioslaves/file/file.h =================================================================== --- src/ioslaves/file/file.h +++ src/ioslaves/file/file.h @@ -59,6 +59,7 @@ void write(const QByteArray &data) override; void seek(KIO::filesize_t offset) override; void truncate(KIO::filesize_t length); + bool copyXattrs(const int src_fd, const int dest_fd); void close() override; /** Index: src/ioslaves/file/file_unix.cpp =================================================================== --- src/ioslaves/file/file_unix.cpp +++ src/ioslaves/file/file_unix.cpp @@ -25,10 +25,7 @@ #include #include -#if HAVE_SYS_XATTR_H -#include #include -#endif #include #include @@ -53,6 +50,13 @@ #include #endif +#if HAVE_SYS_XATTR_H +#include +//BSD uses a different include +#elif HAVE_SYS_EXTATTR_H +#include +#endif + using namespace KIO; /* 512 kB */ @@ -527,6 +531,118 @@ return PrivilegeOperationReturnValue::failure(errcode); } + +#if HAVE_SYS_XATTR_H || HAVE_SYS_EXTATTR_H +bool FileProtocol::copyXattrs(const int src_fd, const int dest_fd) +{ + // Get the list of keys + ssize_t listlen = 0; + QByteArray keylist; + while (true) { + keylist.resize(listlen); +#if HAVE_SYS_XATTR_H && !defined(__stub_getxattr) && !defined(Q_OS_MAC) + listlen = flistxattr(src_fd, keylist.data(), listlen); +#elif defined(Q_OS_MAC) + listlen = flistxattr(src_fd, keylist.data(), listlen, 0); +#elif HAVE_SYS_EXTATTR_H + listlen = extattr_list_fd(src_fd, EXTATTR_NAMESPACE_USER, + listlen == 0 ? nullptr : keylist.data(), + listlen); +#endif + if (listlen > 0 && keylist.size() == 0) { + continue; + } + if (listlen > 0 && keylist.size() > 0) { + break; + } + if (listlen == -1 && errno == ERANGE) { + listlen = 0; + continue; + } + if (listlen == -1) { + if (errno == ENOTSUP) { + qCDebug(KIO_FILE) << "source filesystem does not support xattrs"; + } + return false; + } + if (listlen == 0) { + qCDebug(KIO_FILE) << "the file don't have any xattr"; + return true; + } + } + + keylist.resize(listlen); + + // Linux and MacOS return a list of null terminated strings, each string = [data,'\0'] + // BSDs return a list of items, each item consisting of the size byte + // prepended to the key = [size, data] + QByteArray::const_iterator keyPtr = keylist.cbegin(); + size_t keyLen; + QByteArray value; + + // For each key + while (keyPtr != keylist.cend()) { + // Get size of the key +#if HAVE_SYS_XATTR_H + keyLen = strlen(keyPtr); +#elif HAVE_SYS_EXTATTR_H + keyLen = static_cast(*keyPtr); + keyPtr++; +#endif + QByteArray key(keyPtr, keyLen); + + // Get the value for key + ssize_t valuelen = 0; + do { + value.resize(valuelen); +#if HAVE_SYS_XATTR_H && !defined(__stub_getxattr) && !defined(Q_OS_MAC) + valuelen = fgetxattr(src_fd, key.constData(), value.data(), valuelen); +#elif defined(Q_OS_MAC) + valuelen = fgetxattr(src_fd, key.constData(), value.data(), valuelen, 0, 0); +#elif HAVE_SYS_EXTATTR_H + valuelen = extattr_get_fd(src_fd, EXTATTR_NAMESPACE_USER, key.constData(), + valuelen == 0 ? nullptr : value.data(), + valuelen); +#endif + if (valuelen > 0 && value.size() == 0) { + continue; + } + if (valuelen > 0 && value.size() > 0) { + break; + } + if (valuelen == -1 && errno == ERANGE) { + valuelen = 0; + continue; + } + // happens when attr value is an empty string + if (valuelen == 0) { + break; + } + } while (true); + + // Write key:value pair on destination +#if HAVE_SYS_XATTR_H && !defined(__stub_getxattr) && !defined(Q_OS_MAC) + ssize_t destlen = fsetxattr(dest_fd, key.constData(), value.constData(), valuelen, 0); +#elif defined(Q_OS_MAC) + ssize_t destlen = fsetxattr(dest_fd, key.constData(), value.constData(), valuelen, 0, 0); +#elif HAVE_SYS_EXTATTR_H + ssize_t destlen = extattr_set_fd(dest_fd, EXTATTR_NAMESPACE_USER, key.constData(), value.constData(), valuelen); +#endif + if (destlen == -1 && errno == ENOTSUP) { + qCDebug(KIO_FILE) << "Destination filesystem does not support xattrs"; + return false; + } + +#if HAVE_SYS_XATTR_H + keyPtr += keyLen + 1; +#elif HAVE_SYS_EXTATTR_H + keyPtr += keyLen; +#endif + } + return true; +} +#endif // HAVE_SYS_XATTR_H || HAVE_SYS_EXTATTR_H + void FileProtocol::copy(const QUrl &srcUrl, const QUrl &destUrl, int _mode, JobFlags _flags) { @@ -758,6 +874,13 @@ } #endif + // Copy Extended attributes +#if HAVE_SYS_XATTR_H || HAVE_SYS_EXTATTR_H + if (!copyXattrs(src_file.handle(), dest_file.handle())) { + qCDebug(KIO_FILE) << "cant copy Extended attributes"; + } +#endif + src_file.close(); dest_file.flush(); // so the write() happens before futimes()