diff --git a/src/Archiver.cxx b/src/Archiver.cxx index dcd7f29..9b9d437 100644 --- a/src/Archiver.cxx +++ b/src/Archiver.cxx @@ -1,1457 +1,1468 @@ //************************************************************************** -// Copyright 2006 - 2017 Martin Koller, kollix@aon.at +// Copyright 2006 - 2018 Martin Koller, kollix@aon.at // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, version 2 of the License // //************************************************************************** #include +#include #include -#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // For INT64_MAX: // The ISO C99 standard specifies that in C++ implementations these // macros (stdint.h,inttypes.h) should only be defined if explicitly requested. // ISO C99: 7.18 Integer types #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS #endif #include #include //-------------------------------------------------------------------------------- QString Archiver::sliceScript; Archiver *Archiver::instance; const KIO::filesize_t MAX_SLICE = INT64_MAX; // 64bit max value //-------------------------------------------------------------------------------- Archiver::Archiver(QWidget *parent) : QObject(parent), archive(nullptr), totalBytes(0), totalFiles(0), filteredFiles(0), sliceNum(0), mediaNeedsChange(false), fullBackupInterval(1), incrementalBackup(false), forceFullBackup(false), - sliceCapacity(MAX_SLICE), interactive(parent != nullptr), + sliceCapacity(MAX_SLICE), compressionType(KCompressionDevice::None), interactive(parent != nullptr), cancelled(false), runs(false), skippedFiles(false), verbose(false), jobResult(0) { instance = this; maxSliceMBs = Archiver::UNLIMITED; numKeptBackups = Archiver::UNLIMITED; setCompressFiles(false); if ( !interactive ) { connect(this, SIGNAL(logging(const QString &)), this, SLOT(loggingSlot(const QString &))); connect(this, SIGNAL(warning(const QString &)), this, SLOT(warningSlot(const QString &))); } } //-------------------------------------------------------------------------------- void Archiver::setCompressFiles(bool b) { if ( b ) { - ext = QStringLiteral(".bz2"); - KFilterBase *base = KCompressionDevice::filterForCompressionType(KCompressionDevice::BZip2); + ext = QStringLiteral(".xz"); + compressionType = KCompressionDevice::Xz; + KFilterBase *base = KCompressionDevice::filterForCompressionType(compressionType); if ( !base ) - ext = QStringLiteral(".gz"); + { + ext = QStringLiteral(".bz2"); + compressionType = KCompressionDevice::BZip2; + base = KCompressionDevice::filterForCompressionType(compressionType); + if ( !base ) + { + ext = QStringLiteral(".gz"); + compressionType = KCompressionDevice::GZip; + } + } delete base; } else { ext = QString(); } } //-------------------------------------------------------------------------------- void Archiver::setTarget(const QUrl &target) { targetURL = target; calculateCapacity(); } //-------------------------------------------------------------------------------- void Archiver::setMaxSliceMBs(int mbs) { maxSliceMBs = mbs; calculateCapacity(); } //-------------------------------------------------------------------------------- void Archiver::setKeptBackups(int num) { numKeptBackups = num; } //-------------------------------------------------------------------------------- void Archiver::setFilter(const QString &filter) { filters.clear(); QStringList list = filter.split(QLatin1Char(' '), QString::SkipEmptyParts); foreach (const QString &str, list) filters.append(QRegExp(str, Qt::CaseSensitive, QRegExp::Wildcard)); } //-------------------------------------------------------------------------------- QString Archiver::getFilter() const { QString filter; foreach (const QRegExp ®, filters) { filter += reg.pattern(); filter += QLatin1Char(' '); } return filter; } //-------------------------------------------------------------------------------- void Archiver::setDirFilter(const QString &filter) { dirFilters.clear(); QStringList list = filter.split(QLatin1Char('\n'), QString::SkipEmptyParts); foreach (const QString &str, list) { QString expr = str.trimmed(); if ( !expr.isEmpty() ) dirFilters.append(QRegExp(expr, Qt::CaseSensitive, QRegExp::Wildcard)); } } //-------------------------------------------------------------------------------- QString Archiver::getDirFilter() const { QString filter; foreach (const QRegExp ®, dirFilters) { filter += reg.pattern(); filter += QLatin1Char('\n'); } return filter; } //-------------------------------------------------------------------------------- void Archiver::setFullBackupInterval(int days) { fullBackupInterval = days; if ( fullBackupInterval == 1 ) { setIncrementalBackup(false); lastFullBackup = QDateTime(); lastBackup = QDateTime(); } } //-------------------------------------------------------------------------------- void Archiver::setForceFullBackup(bool force) { forceFullBackup = force; emit backupTypeChanged(isIncrementalBackup()); } //-------------------------------------------------------------------------------- void Archiver::setIncrementalBackup(bool inc) { incrementalBackup = inc; emit backupTypeChanged(isIncrementalBackup()); } //-------------------------------------------------------------------------------- void Archiver::setFilePrefix(const QString &prefix) { filePrefix = prefix; } //-------------------------------------------------------------------------------- void Archiver::calculateCapacity() { if ( targetURL.isEmpty() ) return; // calculate how large a slice can actually be // - limited by the target directory (when we store directly into a local dir) // - limited by the "tmp" dir when we create a tmp file for later upload via KIO // - limited by Qt (64bit int) // - limited by user defined maxSliceMBs KIO::filesize_t totalBytes = 0; if ( targetURL.isLocalFile() ) { if ( ! getDiskFree(targetURL.path(), totalBytes, sliceCapacity) ) return; } else { getDiskFree(QDir::tempPath() + QLatin1Char('/'), totalBytes, sliceCapacity); // as "tmp" is also used by others and by us when compressing a file, // don't eat it up completely. Reserve 10% sliceCapacity = sliceCapacity * 9 / 10; } // limit to what Qt can handle sliceCapacity = qMin(sliceCapacity, MAX_SLICE); if ( maxSliceMBs != UNLIMITED ) { KIO::filesize_t max = static_cast(maxSliceMBs) * 1024 * 1024; sliceCapacity = qMin(sliceCapacity, max); } sliceBytes = 0; // if the disk is full (capacity == 0), don't tell the user "unlimited" // sliceCapacity == 0 has a special meaning as "unlimited"; see MainWidget.cxx if ( sliceCapacity == 0 ) sliceCapacity = 1; emit targetCapacity(sliceCapacity); } //-------------------------------------------------------------------------------- bool Archiver::loadProfile(const QString &fileName, QStringList &includes, QStringList &excludes, QString &error) { QFile file(fileName); if ( ! file.open(QIODevice::ReadOnly) ) { error = file.errorString(); return false; } loadedProfile = fileName; QString target; QChar type, blank; QTextStream stream(&file); // back to default (in case old profile read which does not include these) setFilePrefix(QString()); setMaxSliceMBs(Archiver::UNLIMITED); setFullBackupInterval(1); // default as in previous versions filters.clear(); dirFilters.clear(); while ( ! stream.atEnd() ) { stream.skipWhiteSpace(); stream >> type; // read a QChar without skipping whitespace stream >> blank; // read a QChar without skipping whitespace if ( type == QLatin1Char('M') ) { target = stream.readLine(); // include white space } else if ( type == QLatin1Char('P') ) { QString prefix = stream.readLine(); // include white space setFilePrefix(prefix); } else if ( type == QLatin1Char('R') ) { int max; stream >> max; setKeptBackups(max); } else if ( type == QLatin1Char('F') ) { int days; stream >> days; setFullBackupInterval(days); } else if ( type == QLatin1Char('B') ) // last dateTime for backup { QString dateTime; stream >> dateTime; lastBackup = QDateTime::fromString(dateTime, Qt::ISODate); } else if ( type == QLatin1Char('L') ) // last dateTime for full backup { QString dateTime; stream >> dateTime; lastFullBackup = QDateTime::fromString(dateTime, Qt::ISODate); } else if ( type == QLatin1Char('S') ) { int max; stream >> max; setMaxSliceMBs(max); } else if ( type == QLatin1Char('C') ) { int change; stream >> change; setMediaNeedsChange(change); } else if ( type == QLatin1Char('X') ) { setFilter(stream.readLine()); // include white space } else if ( type == QLatin1Char('x') ) { dirFilters.append(QRegExp(stream.readLine(), Qt::CaseSensitive, QRegExp::Wildcard)); } else if ( type == QLatin1Char('Z') ) { int compress; stream >> compress; setCompressFiles(compress); } else if ( type == QLatin1Char('I') ) { includes.append(stream.readLine()); } else if ( type == QLatin1Char('E') ) { excludes.append(stream.readLine()); } else stream.readLine(); // skip unknown key and rest of line } file.close(); setTarget(QUrl::fromUserInput(target)); setIncrementalBackup( (fullBackupInterval > 1) && lastFullBackup.isValid() && (lastFullBackup.daysTo(QDateTime::currentDateTime()) < fullBackupInterval)); return true; } //-------------------------------------------------------------------------------- bool Archiver::saveProfile(const QString &fileName, const QStringList &includes, const QStringList &excludes, QString &error) { QFile file(fileName); if ( ! file.open(QIODevice::WriteOnly) ) { error = file.errorString(); return false; } QTextStream stream(&file); stream << "M " << targetURL.toString(QUrl::PreferLocalFile) << endl; stream << "P " << getFilePrefix() << endl; stream << "S " << getMaxSliceMBs() << endl; stream << "R " << getKeptBackups() << endl; stream << "F " << getFullBackupInterval() << endl; if ( getLastFullBackup().isValid() ) stream << "L " << getLastFullBackup().toString(Qt::ISODate) << endl; if ( getLastBackup().isValid() ) stream << "B " << getLastBackup().toString(Qt::ISODate) << endl; stream << "C " << static_cast(getMediaNeedsChange()) << endl; stream << "Z " << static_cast(getCompressFiles()) << endl; if ( !filters.isEmpty() ) stream << "X " << getFilter() << endl; foreach (const QRegExp &exp, dirFilters) stream << "x " << exp.pattern() << endl; foreach (const QString &str, includes) stream << "I " << str << endl; foreach (const QString &str, excludes) stream << "E " << str << endl; file.close(); return true; } //-------------------------------------------------------------------------------- bool Archiver::createArchive(const QStringList &includes, const QStringList &excludes) { if ( includes.isEmpty() ) { emit warning(i18n("Nothing selected for backup")); return false; } if ( !targetURL.isValid() ) { emit warning(i18n("The target dir '%1' is not valid", targetURL.toString())); return false; } // non-interactive mode only allows local targets as KIO needs $DISPLAY if ( !interactive && !targetURL.isLocalFile() ) { emit warning(i18n("The target dir '%1' must be a local file system dir and no remote URL", targetURL.toString())); return false; } // check if the target dir exists and optionally create it if ( targetURL.isLocalFile() ) { QDir dir(targetURL.path()); if ( !dir.exists() ) { if ( !interactive || (KMessageBox::warningYesNo(static_cast(parent()), i18n("The target directory '%1' does not exist.\n\n" "Shall I create it?", dir.absolutePath())) == KMessageBox::Yes) ) { if ( !dir.mkpath(QStringLiteral(".")) ) { emit warning(i18n("Could not create the target directory '%1'.\n" "The operating system reports: %2", dir.absolutePath(), QString::fromLatin1(strerror(errno)))); return false; } } else { emit warning(i18n("The target dir does not exist")); return false; } } } excludeDirs.clear(); excludeFiles.clear(); // build map for directories and files to be excluded for fast lookup foreach (const QString &name, excludes) { QFileInfo info(name); if ( !info.isSymLink() && info.isDir() ) excludeDirs.insert(name); else excludeFiles.insert(name); } baseName = QString(); sliceNum = 0; totalBytes = 0; totalFiles = 0; filteredFiles = 0; cancelled = false; skippedFiles = false; sliceList.clear(); QDateTime startTime = QDateTime::currentDateTime(); runs = true; emit inProgress(true); QTimer runTimer; if ( interactive ) // else we do not need to be interrupted during the backup { connect(&runTimer, SIGNAL(timeout()), this, SLOT(updateElapsed())); runTimer.start(1000); } elapsed.start(); if ( ! getNextSlice() ) { runs = false; emit inProgress(false); return false; } for (QStringList::const_iterator it = includes.constBegin(); !cancelled && (it != includes.constEnd()); ++it) { QString entry = *it; if ( (entry.length() > 1) && entry.endsWith(QLatin1Char('/')) ) entry.truncate(entry.length() - 1); QFileInfo info(entry); if ( !info.isSymLink() && info.isDir() ) { QDir dir(info.absoluteFilePath()); addDirFiles(dir); } else addFile(info.absoluteFilePath()); } finishSlice(); // reduce the number of old backups to the defined number if ( !cancelled && (numKeptBackups != UNLIMITED) ) { emit logging(i18n("...reducing number of kept archives to max. %1", numKeptBackups)); if ( !targetURL.isLocalFile() ) // KIO needs $DISPLAY; non-interactive only allowed for local targets { QPointer listJob; listJob = KIO::listDir(targetURL, KIO::DefaultFlags, false); connect(listJob, SIGNAL(entries(KIO::Job *, const KIO::UDSEntryList &)), this, SLOT(slotListResult(KIO::Job *, const KIO::UDSEntryList &))); while ( listJob ) qApp->processEvents(QEventLoop::WaitForMoreEvents); } else // non-intercative. create UDSEntryList on our own { QDir dir(targetURL.path()); targetDirList.clear(); - foreach (QString fileName, dir.entryList()) + foreach (const QString &fileName, dir.entryList()) { KIO::UDSEntry entry; +#if (KIO_VERSION >= QT_VERSION_CHECK(5, 48, 0)) + entry.fastInsert(KIO::UDSEntry::UDS_NAME, fileName); +#else entry.insert(KIO::UDSEntry::UDS_NAME, fileName); +#endif targetDirList.append(entry); } jobResult = 0; } if ( jobResult == 0 ) { std::sort(targetDirList.begin(), targetDirList.end(), Archiver::UDSlessThan); QString prefix = filePrefix.isEmpty() ? QString::fromLatin1("backup_") : (filePrefix + QLatin1String("_")); QString sliceName; int num = 0; - foreach(KIO::UDSEntry entry, targetDirList) + foreach (const KIO::UDSEntry &entry, targetDirList) { QString entryName = entry.stringValue(KIO::UDSEntry::UDS_NAME); if ( entryName.startsWith(prefix) && // only matching current profile entryName.endsWith(QLatin1String(".tar")) ) // just to be sure { if ( (num < numKeptBackups) && (sliceName.isEmpty() || !entryName.startsWith(sliceName)) ) // whenever a new backup set (different time) is found { sliceName = entryName.left(prefix.length() + strlen("yyyy.MM.dd-hh.mm.ss_")); if ( !entryName.endsWith(QLatin1String("_inc.tar")) ) // do not count partial (differential) backup files num++; if ( num == numKeptBackups ) num++; // from here on delete all others } if ( (num > numKeptBackups) && // delete all other files !entryName.startsWith(sliceName) ) // keep complete last matching archive set { QUrl url = targetURL; url = url.adjusted(QUrl::StripTrailingSlash); url.setPath(url.path() + QLatin1Char('/') + entryName); emit logging(i18n("...deleting %1", entryName)); // delete the file using KIO if ( !targetURL.isLocalFile() ) // KIO needs $DISPLAY; non-interactive only allowed for local targets { QPointer delJob; delJob = KIO::file_delete(url, KIO::DefaultFlags); connect(delJob, SIGNAL(result(KJob *)), this, SLOT(slotResult(KJob *))); while ( delJob ) qApp->processEvents(QEventLoop::WaitForMoreEvents); } else { QDir dir(targetURL.path()); dir.remove(entryName); } } } } } else { emit warning(i18n("fetching directory listing of target failed. Can not reduce kept archives.")); } } runs = false; emit inProgress(false); runTimer.stop(); updateElapsed(); // to catch the last partly second if ( !cancelled ) { lastBackup = startTime; if ( !isIncrementalBackup() ) { lastFullBackup = lastBackup; setIncrementalBackup(fullBackupInterval > 1); // after a full backup, the next will be incremental } if ( (fullBackupInterval > 1) && !loadedProfile.isEmpty() ) { QString error; if ( !saveProfile(loadedProfile, includes, excludes, error) ) { emit warning(i18n("Could not write backup timestamps into profile %1: %2", loadedProfile, error)); } } emit logging(i18n("-- Filtered Files: %1", filteredFiles)); if ( skippedFiles ) emit logging(i18n("!! Backup finished but files were skipped !!")); else emit logging(i18n("-- Backup successfully finished --")); if ( interactive ) { int ret = KMessageBox::questionYesNoList(static_cast(parent()), skippedFiles ? i18n("The backup has finished but files were skipped.\n" "What do you want to do now?") : i18n("The backup has finished successfully.\n" "What do you want to do now?"), sliceList, QString(), KStandardGuiItem::cont(), KStandardGuiItem::quit(), QLatin1String("showDoneInfo")); if ( ret == KMessageBox::No ) // quit qApp->quit(); } else { std::cerr << "-------" << std::endl; foreach (QString slice, sliceList) std::cerr << slice.toUtf8().constData() << std::endl; std::cerr << "-------" << std::endl; std::cerr << i18n("Totals: Files: %1, Size: %2, Duration: %3", totalFiles, KIO::convertSize(totalBytes), QTime(0, 0).addMSecs(elapsed.elapsed()).toString(QStringLiteral("HH:mm:ss"))) .toUtf8().constData() << std::endl; } return true; } else { emit logging(i18n("...Backup aborted!")); return false; } } //-------------------------------------------------------------------------------- void Archiver::cancel() { if ( !runs ) return; if ( job ) { job->kill(); job = nullptr; } if ( !cancelled ) { cancelled = true; if ( archive ) { archive->close(); // else I can not remove the file - don't know why delete archive; archive = nullptr; } QFile(archiveName).remove(); // remove the unfinished tar file (which is now corrupted) emit warning(i18n("Backup cancelled")); } } //-------------------------------------------------------------------------------- void Archiver::finishSlice() { if ( archive ) archive->close(); if ( ! cancelled ) { runScript(QStringLiteral("slice_closed")); if ( targetURL.isLocalFile() ) { emit logging(i18n("...finished slice %1", archiveName)); sliceList << archiveName; // store name for display at the end } else { QUrl source = QUrl::fromLocalFile(archiveName); QUrl target = targetURL; while ( true ) { // copy to have the archive for the script later down job = KIO::copy(source, target, KIO::DefaultFlags); connect(job, SIGNAL(result(KJob *)), this, SLOT(slotResult(KJob *))); emit logging(i18n("...uploading archive %1 to %2", source.fileName(), target.toString())); while ( job ) qApp->processEvents(QEventLoop::WaitForMoreEvents); if ( jobResult == 0 ) { target = target.adjusted(QUrl::StripTrailingSlash); target.setPath(target.path() + QLatin1Char('/') + source.fileName()); sliceList << target.toLocalFile(); // store name for display at the end break; } else { enum { ASK, CANCEL, RETRY } action = ASK; while ( action == ASK ) { int ret = KMessageBox::warningYesNoCancel(static_cast(parent()), i18n("How shall we proceed with the upload?"), i18n("Upload Failed"), KGuiItem(i18n("Retry")), KGuiItem(i18n("Change Target"))); if ( ret == KMessageBox::Cancel ) { action = CANCEL; break; } else if ( ret == KMessageBox::No ) // change target { target = QFileDialog::getExistingDirectoryUrl(static_cast(parent())); if ( target.isEmpty() ) action = ASK; else action = RETRY; } else action = RETRY; } if ( action == CANCEL ) break; } } if ( jobResult != 0 ) cancel(); } } if ( ! cancelled ) runScript(QStringLiteral("slice_finished")); if ( !targetURL.isLocalFile() ) QFile(archiveName).remove(); // remove the tmp file delete archive; archive = nullptr; } //-------------------------------------------------------------------------------- void Archiver::slotResult(KJob *theJob) { if ( (jobResult = theJob->error()) ) { theJob->uiDelegate()->showErrorMessage(); emit warning(theJob->errorString()); } } //-------------------------------------------------------------------------------- void Archiver::slotListResult(KIO::Job *theJob, const KIO::UDSEntryList &entries) { if ( (jobResult = theJob->error()) ) { theJob->uiDelegate()->showErrorMessage(); emit warning(theJob->errorString()); } targetDirList = entries; } //-------------------------------------------------------------------------------- void Archiver::runScript(const QString &mode) { // do some extra action via external script (program) if ( sliceScript.length() ) { QString mountPoint; if ( targetURL.isLocalFile() ) { KMountPoint::Ptr ptr = KMountPoint::currentMountPoints().findByPath(targetURL.path()); if ( ptr ) mountPoint = ptr->mountPoint(); } KProcess proc; proc << sliceScript << mode << archiveName << targetURL.toString(QUrl::PreferLocalFile) << mountPoint; connect(&proc, &KProcess::readyReadStandardOutput, this, &Archiver::receivedOutput); proc.setOutputChannelMode(KProcess::MergedChannels); if ( proc.execute() == -2 ) { QString message = i18n("The script '%1' could not be started.", sliceScript); if ( interactive ) KMessageBox::error(static_cast(parent()), message); else emit warning(message); } } } //-------------------------------------------------------------------------------- void Archiver::receivedOutput() { KProcess *proc = static_cast(sender()); QByteArray buffer = proc->readAllStandardOutput(); QString msg = QString::fromUtf8(buffer); if ( msg.endsWith(QLatin1String("\n")) ) msg.truncate(msg.length() - 1); emit warning(msg); } //-------------------------------------------------------------------------------- bool Archiver::getNextSlice() { sliceNum++; if ( archive ) { emit sliceProgress(100); finishSlice(); if ( cancelled ) return false; if ( interactive && mediaNeedsChange && KMessageBox::warningContinueCancel(static_cast(parent()), i18n("The medium is full. Please insert medium Nr. %1", sliceNum)) == KMessageBox::Cancel ) { cancel(); return false; } } emit newSlice(sliceNum); if ( baseName.isEmpty() ) { QString prefix = filePrefix.isEmpty() ? QString::fromLatin1("backup") : filePrefix; if ( targetURL.isLocalFile() ) baseName = targetURL.path() + QStringLiteral("/") + prefix + QDateTime::currentDateTime().toString(QStringLiteral("_yyyy.MM.dd-hh.mm.ss")); else baseName = QDir::tempPath() + QLatin1Char('/') + prefix + QDateTime::currentDateTime().toString(QStringLiteral("_yyyy.MM.dd-hh.mm.ss")); } archiveName = baseName + QStringLiteral("_%1").arg(sliceNum); if ( isIncrementalBackup() ) archiveName += QStringLiteral("_inc.tar"); // mark the file as being not a full backup else archiveName += QStringLiteral(".tar"); runScript(QStringLiteral("slice_init")); calculateCapacity(); // don't create a bz2 compressed file as we compress each file on its own archive = new KTar(archiveName, QStringLiteral("application/x-tar")); while ( (sliceCapacity < 1024) || !archive->open(QIODevice::WriteOnly) ) // disk full ? { if ( !interactive ) emit warning(i18n("The file '%1' can not be opened for writing.", archiveName)); if ( !interactive || (KMessageBox::warningYesNo(static_cast(parent()), i18n("The file '%1' can not be opened for writing.\n\n" "Do you want to retry?", archiveName)) == KMessageBox::No) ) { delete archive; archive = nullptr; cancel(); return false; } calculateCapacity(); // try again; maybe the user freed up some space } return true; } //-------------------------------------------------------------------------------- void Archiver::emitArchiveError() const { QString err; if ( archive->device() ) err = archive->device()->errorString(); if ( err.isEmpty() ) { emit warning(i18n("Could not write to archive. Maybe the medium is full.")); } else { emit warning(i18n("Could not write to archive.\n" "The operating system reports: %1", err)); } } //-------------------------------------------------------------------------------- void Archiver::addDirFiles(QDir &dir) { QString absolutePath = dir.absolutePath(); if ( excludeDirs.contains(absolutePath) ) return; foreach (const QRegExp &exp, dirFilters) { if ( exp.exactMatch(absolutePath) ) { if ( interactive || verbose ) emit logging(i18n("...skipping filtered directory %1", absolutePath)); return; } } // add the dir itself struct stat status; memset(&status, 0, sizeof(status)); if ( ::stat(QFile::encodeName(absolutePath), &status) == -1 ) { emit warning(i18n("Could not get information of directory: %1\n" "The operating system reports: %2", absolutePath, QString::fromLatin1(strerror(errno)))); return; } QFileInfo dirInfo(absolutePath); if ( ! dirInfo.isReadable() ) { emit warning(i18n("Directory '%1' is not readable. Skipping.", absolutePath)); skippedFiles = true; return; } totalFiles++; emit totalFilesChanged(totalFiles); if ( interactive || verbose ) emit logging(absolutePath); qApp->processEvents(QEventLoop::AllEvents, 5); if ( cancelled ) return; if ( ! archive->writeDir(QStringLiteral(".") + absolutePath, dirInfo.owner(), dirInfo.group(), status.st_mode, dirInfo.lastRead(), dirInfo.lastModified(), dirInfo.created()) ) { emit warning(i18n("Could not write directory '%1' to archive.\n" "Maybe the medium is full.", absolutePath)); return; } dir.setFilter(QDir::AllEntries | QDir::Hidden | QDir::System | QDir::NoDotAndDotDot); const QFileInfoList list = dir.entryInfoList(); for (int i = 0; !cancelled && (i < list.count()); i++) { if ( !list[i].isSymLink() && list[i].isDir() ) { QDir dir(list[i].absoluteFilePath()); addDirFiles(dir); } else addFile(list[i].absoluteFilePath()); } } //-------------------------------------------------------------------------------- bool Archiver::fileIsFiltered(const QString &fileName) const { foreach (const QRegExp &exp, filters) if ( exp.exactMatch(fileName) ) return true; return false; } //-------------------------------------------------------------------------------- void Archiver::addFile(const QFileInfo &info) { if ( (isIncrementalBackup() && (info.lastModified() < lastBackup)) || fileIsFiltered(info.fileName()) ) { filteredFiles++; return; } if ( excludeFiles.contains(info.absoluteFilePath()) ) return; // avoid including my own archive file // (QFileInfo to have correct path comparison even in case archiveName contains // etc.) // startsWith() is needed as KDE4 KTar does not create directly the .tar file but until it's closed // the file is named "...tarXXXX.new" if ( info.absoluteFilePath().startsWith(QFileInfo(archiveName).absoluteFilePath()) ) return; if ( cancelled ) return; /* don't skip. We probably do not need to read it anyway, since it might be empty if ( ! info.isReadable() ) { emit warning(i18n("File '%1' is not readable. Skipping.").arg(info.absoluteFilePath())); skippedFiles = true; return; } */ // emit before we do the compression, so that the receiver can already show // with which file we work // show filename + size if ( interactive || verbose ) emit logging(info.absoluteFilePath() + QStringLiteral(" (%1)").arg(KIO::convertSize(info.size()))); qApp->processEvents(QEventLoop::AllEvents, 5); if ( cancelled ) return; if ( info.isSymLink() ) { archive->addLocalFile(info.absoluteFilePath(), QStringLiteral(".") + info.absoluteFilePath()); totalFiles++; emit totalFilesChanged(totalFiles); return; } if ( !getCompressFiles() ) { AddFileStatus ret = addLocalFile(info); // this also increases totalBytes if ( ret == Error ) { cancel(); // we must cancel as the tar-file is now corrupt (file was only partly written) return; } else if ( ret == Skipped ) { skippedFiles = true; return; } } else // add the file compressed { // as we can't know which size the file will have after compression, // we create a compressed file and put this into the archive QTemporaryFile tmpFile; if ( ! compressFile(info.absoluteFilePath(), tmpFile) || cancelled ) return; // here we have the compressed file in tmpFile tmpFile.open(); // size() only works if open if ( (sliceBytes + tmpFile.size()) > sliceCapacity ) if ( ! getNextSlice() ) return; // to be able to create the exact same metadata (permission, date, owner) we need // to fill the file into the archive with the following: { struct stat status; memset(&status, 0, sizeof(status)); if ( ::stat(QFile::encodeName(info.absoluteFilePath()), &status) == -1 ) { emit warning(i18n("Could not get information of file: %1\n" "The operating system reports: %2", info.absoluteFilePath(), QString::fromLatin1(strerror(errno)))); skippedFiles = true; return; } if ( ! archive->prepareWriting(QStringLiteral(".") + info.absoluteFilePath() + ext, info.owner(), info.group(), tmpFile.size(), status.st_mode, info.lastRead(), info.lastModified(), info.created()) ) { emitArchiveError(); cancel(); return; } const int BUFFER_SIZE = 8*1024; static char buffer[BUFFER_SIZE]; qint64 len; int count = 0; while ( ! tmpFile.atEnd() ) { len = tmpFile.read(buffer, BUFFER_SIZE); if ( len < 0 ) // error in reading { emit warning(i18n("Could not read from file '%1'\n" "The operating system reports: %2", info.absoluteFilePath(), tmpFile.errorString())); cancel(); return; } if ( ! archive->writeData(buffer, len) ) { emitArchiveError(); cancel(); return; } count = (count + 1) % 50; if ( count == 0 ) { qApp->processEvents(QEventLoop::AllEvents, 5); if ( cancelled ) return; } } if ( ! archive->finishWriting(tmpFile.size()) ) { emitArchiveError(); cancel(); return; } } // get filesize sliceBytes = archive->device()->pos(); // account for tar overhead totalBytes += tmpFile.size(); emit sliceProgress(static_cast(sliceBytes * 100 / sliceCapacity)); } totalFiles++; emit totalFilesChanged(totalFiles); emit totalBytesChanged(totalBytes); qApp->processEvents(QEventLoop::AllEvents, 5); } //-------------------------------------------------------------------------------- Archiver::AddFileStatus Archiver::addLocalFile(const QFileInfo &info) { struct stat sourceStat; memset(&sourceStat, 0, sizeof(sourceStat)); if ( ::stat(QFile::encodeName(info.absoluteFilePath()), &sourceStat) == -1 ) { emit warning(i18n("Could not get information of file: %1\n" "The operating system reports: %2", info.absoluteFilePath(), QString::fromLatin1(strerror(errno)))); return Skipped; } QFile sourceFile(info.absoluteFilePath()); // if the size is 0 (e.g. a pipe), don't open it since we will not read any content // and Qt hangs when opening a pipe if ( (info.size() > 0) && !sourceFile.open(QIODevice::ReadOnly) ) { emit warning(i18n("Could not open file '%1' for reading.", info.absoluteFilePath())); return Skipped; } if ( (sliceBytes + info.size()) > sliceCapacity ) if ( ! getNextSlice() ) return Error; if ( ! archive->prepareWriting(QStringLiteral(".") + info.absoluteFilePath(), info.owner(), info.group(), info.size(), sourceStat.st_mode, info.lastRead(), info.lastModified(), info.created()) ) { emitArchiveError(); return Error; } const int BUFFER_SIZE = 8*1024; static char buffer[BUFFER_SIZE]; qint64 len; int count = 0, progress; QTime timer; timer.start(); bool msgShown = false; qint64 written = 0; while ( info.size() && !sourceFile.atEnd() && !cancelled ) { len = sourceFile.read(buffer, BUFFER_SIZE); if ( len < 0 ) // error in reading { emit warning(i18n("Could not read from file '%1'\n" "The operating system reports: %2", info.absoluteFilePath(), sourceFile.errorString())); return Error; } if ( ! archive->writeData(buffer, len) ) { emitArchiveError(); return Error; } totalBytes += len; written += len; progress = static_cast(written * 100 / info.size()); // stay responsive count = (count + 1) % 50; if ( count == 0 ) { if ( msgShown ) emit fileProgress(progress); emit totalBytesChanged(totalBytes); qApp->processEvents(QEventLoop::AllEvents, 5); } if ( !msgShown && (timer.elapsed() > 3000) && (progress < 50) ) { emit fileProgress(progress); if ( interactive || verbose ) emit logging(i18n("...archiving file %1", info.absoluteFilePath())); if ( interactive ) QApplication::setOverrideCursor(QCursor(Qt::BusyCursor)); qApp->processEvents(QEventLoop::AllEvents, 5); msgShown = true; } } emit fileProgress(100); sourceFile.close(); if ( !cancelled ) { // get filesize sliceBytes = archive->device()->pos(); // account for tar overhead emit sliceProgress(static_cast(sliceBytes * 100 / sliceCapacity)); } if ( msgShown && interactive ) QApplication::restoreOverrideCursor(); if ( !cancelled && !archive->finishWriting(info.size()) ) { emitArchiveError(); return Error; } return cancelled ? Error : Added; } //-------------------------------------------------------------------------------- bool Archiver::compressFile(const QString &origName, QFile &comprFile) { QFile origFile(origName); if ( ! origFile.open(QIODevice::ReadOnly) ) { emit warning(i18n("Could not read file: %1\n" "The operating system reports: %2", origName, origFile.errorString())); skippedFiles = true; return false; } else { - KCompressionDevice::CompressionType type = - KFilterDev::compressionTypeForMimeType(ext == QStringLiteral(".bz2") ? QStringLiteral("application/x-bzip2") : QStringLiteral("application/x-gzip")); - - KCompressionDevice filter(&comprFile, false, type); + KCompressionDevice filter(&comprFile, false, compressionType); if ( !filter.open(QIODevice::WriteOnly) ) { emit warning(i18n("Could not create temporary file for compressing: %1\n" "The operating system reports: %2", origName, filter.errorString())); return false; } const int BUFFER_SIZE = 8*1024; static char buffer[BUFFER_SIZE]; qint64 len; int count = 0, progress; QTime timer; timer.start(); bool msgShown = false; KIO::filesize_t fileSize = origFile.size(); KIO::filesize_t written = 0; while ( fileSize && !origFile.atEnd() && !cancelled ) { len = origFile.read(buffer, BUFFER_SIZE); qint64 wrote = filter.write(buffer, len); if ( len != wrote ) { emit warning(i18n("Could not write to temporary file")); return false; } written += len; progress = static_cast(written * 100 / fileSize); // keep the ui responsive count = (count + 1) % 50; if ( count == 0 ) { if ( msgShown ) emit fileProgress(progress); qApp->processEvents(QEventLoop::AllEvents, 5); } if ( !msgShown && (timer.elapsed() > 3000) && (progress < 50) ) { emit fileProgress(progress); emit logging(i18n("...compressing file %1", origName)); if ( interactive ) QApplication::setOverrideCursor(QCursor(Qt::BusyCursor)); qApp->processEvents(QEventLoop::AllEvents, 5); msgShown = true; } } emit fileProgress(100); origFile.close(); if ( msgShown && interactive ) QApplication::restoreOverrideCursor(); } return true; } //-------------------------------------------------------------------------------- bool Archiver::getDiskFree(const QString &path, KIO::filesize_t &capacityB, KIO::filesize_t &freeB) { struct statvfs vfs; memset(&vfs, 0, sizeof(vfs)); if ( ::statvfs(QFile::encodeName(path), &vfs) == -1 ) return false; capacityB = static_cast(vfs.f_blocks) * static_cast(vfs.f_frsize); freeB = static_cast(vfs.f_bavail) * static_cast(vfs.f_frsize); return true; } //-------------------------------------------------------------------------------- void Archiver::loggingSlot(const QString &message) { std::cerr << message.toUtf8().constData() << std::endl; } //-------------------------------------------------------------------------------- void Archiver::warningSlot(const QString &message) { std::cerr << i18n("WARNING:").toUtf8().constData() << message.toUtf8().constData() << std::endl; } //-------------------------------------------------------------------------------- void Archiver::updateElapsed() { emit elapsedChanged(QTime(0, 0).addMSecs(elapsed.elapsed())); } //-------------------------------------------------------------------------------- // sort by name of entries in descending order (younger names are first) -bool Archiver::UDSlessThan(KIO::UDSEntry &left, KIO::UDSEntry &right) +bool Archiver::UDSlessThan(const KIO::UDSEntry &left, const KIO::UDSEntry &right) { return left.stringValue(KIO::UDSEntry::UDS_NAME) > right.stringValue(KIO::UDSEntry::UDS_NAME); } //-------------------------------------------------------------------------------- diff --git a/src/Archiver.hxx b/src/Archiver.hxx index 5f15b0c..887472a 100644 --- a/src/Archiver.hxx +++ b/src/Archiver.hxx @@ -1,205 +1,207 @@ //************************************************************************** // Copyright 2006 - 2017 Martin Koller, kollix@aon.at // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, version 2 of the License // //************************************************************************** #ifndef _ARCHIVER_H_ #define _ARCHIVER_H_ // the class which does the archiving #include #include #include #include #include #include #include #include #include #include #include #include +#include class KTar; class QDir; class QFileInfo; class QFile; class Archiver : public QObject { Q_OBJECT public: explicit Archiver(QWidget *parent); static Archiver *instance; // always call after you have already set maxSliceMBs, as the sliceCapacity // might be limited with it void setTarget(const QUrl &target); const QUrl &getTarget() const { return targetURL; } enum { UNLIMITED = 0 }; void setMaxSliceMBs(int mbs); int getMaxSliceMBs() const { return maxSliceMBs; } void setFilePrefix(const QString &prefix); const QString &getFilePrefix() const { return filePrefix; } void setMediaNeedsChange(bool b) { mediaNeedsChange = b; } bool getMediaNeedsChange() const { return mediaNeedsChange; } void setCompressFiles(bool b); bool getCompressFiles() const { return !ext.isEmpty(); } // number of backups to keep before older ones will be deleted (UNLIMITED or 1..n) void setKeptBackups(int num); int getKeptBackups() const { return numKeptBackups; } // define a filename filter in wildcard format each separated with a space // e.g. "*.png *.ogg" void setFilter(const QString &filter); QString getFilter() const; // define a list of path wildcards to filter complete dirs, separated by newline void setDirFilter(const QString &filter); QString getDirFilter() const; // interval for a full backup instead differential backup; when 1 given == full backup void setFullBackupInterval(int days); int getFullBackupInterval() const { return fullBackupInterval; } const QDateTime &getLastFullBackup() const { return lastFullBackup; } const QDateTime &getLastBackup() const { return lastBackup; } // print every single file/dir in non-interactive mode void setVerbose(bool b) { verbose = b; } // loads the profile into the Archiver and returns includes/excludes lists // return true if loaded, false on file open error bool loadProfile(const QString &fileName, QStringList &includes, QStringList &excludes, QString &error); void setLoadedProfile(const QString &fileName) { loadedProfile = fileName; } bool saveProfile(const QString &fileName, const QStringList &includes, const QStringList &excludes, QString &error); // return true if the backup completed successfully, else false bool createArchive(const QStringList &includes, const QStringList &excludes); KIO::filesize_t getTotalBytes() const { return totalBytes; } int getTotalFiles() const { return totalFiles; } bool isInProgress() const { return runs; } static bool getDiskFree(const QString &path, KIO::filesize_t &capacityBytes, KIO::filesize_t &freeBytes); // TODO: put probably in some global settings object static QString sliceScript; public slots: void cancel(); // cancel a running creation void setForceFullBackup(bool force = true); signals: void inProgress(bool runs) const; void logging(const QString &) const; void warning(const QString &) const; void targetCapacity(KIO::filesize_t bytes) const; void sliceProgress(int percent) const; void fileProgress(int percent) const; void newSlice(int) const; void totalFilesChanged(int) const; void totalBytesChanged(KIO::filesize_t) const; void elapsedChanged(const QTime &) const; void backupTypeChanged(bool incremental) const; private slots: void slotResult(KJob *); void slotListResult(KIO::Job *, const KIO::UDSEntryList &); void receivedOutput(); void loggingSlot(const QString &message); // for non-interactive output void warningSlot(const QString &message); // for non-interactive output void updateElapsed(); private: void calculateCapacity(); // also emits signals void addDirFiles(QDir &dir); void addFile(const QFileInfo &info); enum AddFileStatus { Error, Added, Skipped }; AddFileStatus addLocalFile(const QFileInfo &info); bool compressFile(const QString &origName, QFile &comprFile); void finishSlice(); bool getNextSlice(); void runScript(const QString &mode); void setIncrementalBackup(bool inc); // returns true if the next backup will be an incremental one, false for a full backup bool isIncrementalBackup() const { return !forceFullBackup && incrementalBackup; } // return true if given fileName matches any of the defined filters bool fileIsFiltered(const QString &fileName) const; void emitArchiveError() const; - static bool UDSlessThan(KIO::UDSEntry &left, KIO::UDSEntry &right); + static bool UDSlessThan(const KIO::UDSEntry &left, const KIO::UDSEntry &right); private: QSet excludeFiles; QSet excludeDirs; QString archiveName; QString filePrefix; // default = "backup" QStringList sliceList; QString loadedProfile; KTar *archive; KIO::filesize_t totalBytes; int totalFiles; int filteredFiles; // filter or time filter (incremental backup) QElapsedTimer elapsed; QList filters; QList dirFilters; QUrl targetURL; QString baseName; int sliceNum; int maxSliceMBs; bool mediaNeedsChange; bool compressFiles; int numKeptBackups; KIO::UDSEntryList targetDirList; QDateTime lastFullBackup; QDateTime lastBackup; int fullBackupInterval; bool incrementalBackup; bool forceFullBackup; KIO::filesize_t sliceBytes; KIO::filesize_t sliceCapacity; QString ext; + KCompressionDevice::CompressionType compressionType; bool interactive; bool cancelled; bool runs; bool skippedFiles; // did we skip files during backup ? bool verbose; QPointer job; int jobResult; }; #endif