diff --git a/autotests/kcompressiondevicetest.cpp b/autotests/kcompressiondevicetest.cpp index 12fd095..794de26 100644 --- a/autotests/kcompressiondevicetest.cpp +++ b/autotests/kcompressiondevicetest.cpp @@ -1,192 +1,219 @@ /* This file is part of the KDE project Copyright (C) 2015 Luiz Romário Santana Rios This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "kcompressiondevicetest.h" +#include "kcompressiondevice_p.h" #include #include #include #include #include #include #include #include #include QTEST_MAIN(KCompressionDeviceTest) static QString archiveFileName(const QString &extension) { return QFINDTESTDATA(QString("kcompressiondevice_test.%1").arg(extension)); } QNetworkReply *KCompressionDeviceTest::getArchive(const QString &extension) { const QString kcompressionTest = archiveFileName(extension); QNetworkReply *r = qnam.get(QNetworkRequest(QUrl::fromLocalFile(kcompressionTest))); QEventLoop l; connect(&qnam, &QNetworkAccessManager::finished, &l, &QEventLoop::quit); l.exec(); return r; } QString KCompressionDeviceTest::formatExtension(KCompressionDevice::CompressionType type) const { switch (type) { case KCompressionDevice::GZip: return "tar.gz"; case KCompressionDevice::BZip2: return "tar.bz2"; case KCompressionDevice::Xz: return "tar.xz"; case KCompressionDevice::None: return QString(); } return QString(); // silence compiler warning } void KCompressionDeviceTest::setDeviceToArchive( QIODevice *d, KCompressionDevice::CompressionType type) { KCompressionDevice *devRawPtr = new KCompressionDevice(d, true, type); archive.reset(new KTar(devRawPtr)); device.reset(devRawPtr); } void KCompressionDeviceTest::testBufferedDevice(KCompressionDevice::CompressionType type) { QNetworkReply *r = getArchive(formatExtension(type)); const QByteArray data = r->readAll(); QVERIFY(!data.isEmpty()); const int expectedSize = QFileInfo(archiveFileName(formatExtension(type))).size(); QVERIFY(expectedSize > 0); QCOMPARE(data.size(), expectedSize); QBuffer *b = new QBuffer; b->setData(data); setDeviceToArchive(b, type); testExtraction(); } void KCompressionDeviceTest::testExtraction() { QTemporaryDir temp; QString oldCurrentDir = QDir::currentPath(); QDir::setCurrent(temp.path()); QVERIFY(archive->open(QIODevice::ReadOnly)); QVERIFY(archive->directory()->copyTo(".")); QVERIFY(QDir("examples").exists()); QVERIFY(QDir("examples/bzip2gzip").exists()); QVERIFY(QDir("examples/helloworld").exists()); QVERIFY(QDir("examples/tarlocalfiles").exists()); QVERIFY(QDir("examples/unzipper").exists()); QVector fileList; fileList << QLatin1String("examples/bzip2gzip/CMakeLists.txt") << QLatin1String("examples/bzip2gzip/main.cpp") << QLatin1String("examples/helloworld/CMakeLists.txt") << QLatin1String("examples/helloworld/helloworld.pro") << QLatin1String("examples/helloworld/main.cpp") << QLatin1String("examples/tarlocalfiles/CMakeLists.txt") << QLatin1String("examples/tarlocalfiles/main.cpp") << QLatin1String("examples/unzipper/CMakeLists.txt") << QLatin1String("examples/unzipper/main.cpp"); for (const QString& s : qAsConst(fileList)) { QFileInfo extractedFile(s); QFileInfo sourceFile(QFINDTESTDATA("../" + s)); QVERIFY(extractedFile.exists()); QCOMPARE(extractedFile.size(), sourceFile.size()); } QDir::setCurrent(oldCurrentDir); } void KCompressionDeviceTest::regularKTarUsage() { archive.reset(new KTar(QFINDTESTDATA("kcompressiondevice_test.tar.gz"))); device.reset(); testExtraction(); } void KCompressionDeviceTest::testGZipBufferedDevice() { testBufferedDevice(KCompressionDevice::GZip); } void KCompressionDeviceTest::testBZip2BufferedDevice() { #if HAVE_BZIP2_SUPPORT testBufferedDevice(KCompressionDevice::BZip2); #else QSKIP("This test needs bzip2 support"); #endif } void KCompressionDeviceTest::testXzBufferedDevice() { #if HAVE_XZ_SUPPORT testBufferedDevice(KCompressionDevice::Xz); #else QSKIP("This test needs xz support"); #endif } void KCompressionDeviceTest::testWriteErrorOnOpen() { // GIVEN QString fileName("/I/dont/exist/kcompressiondevicetest-write.gz"); KCompressionDevice dev(fileName, KCompressionDevice::GZip); // WHEN QVERIFY(!dev.open(QIODevice::WriteOnly)); // THEN QCOMPARE(dev.error(), QFileDevice::OpenError); #ifdef Q_OS_WIN QCOMPARE(dev.errorString(), QStringLiteral("The system cannot find the path specified.")); #else QCOMPARE(dev.errorString(), QStringLiteral("No such file or directory")); #endif } void KCompressionDeviceTest::testWriteErrorOnClose() { // GIVEN QFile file("kcompressiondevicetest-write.gz"); KCompressionDevice dev(&file, false, KCompressionDevice::GZip); QVERIFY(dev.open(QIODevice::WriteOnly)); const QByteArray data = "Hello world"; QCOMPARE(dev.write(data), data.size()); // This is nasty, it's just a way to try and trigger an error on flush, without filling up a partition first ;) file.close(); QVERIFY(file.open(QIODevice::ReadOnly)); QTest::ignoreMessage(QtWarningMsg, "QIODevice::write (QFile, \"kcompressiondevicetest-write.gz\"): ReadOnly device"); // WHEN dev.close(); // I want a QVERIFY here... https://bugreports.qt.io/browse/QTBUG-70033 // THEN QCOMPARE(int(dev.error()), int(QFileDevice::WriteError)); } + +void KCompressionDeviceTest::testSeekReadUncompressedBuffer() +{ + const int dataSize = BUFFER_SIZE + BUFFER_SIZE / 2; + QByteArray ba(dataSize, 0); + + // all data is zero except after BUFFER_SIZE that it's 0 to 9 + for (int i = 0; i < 10; ++i) { + ba[BUFFER_SIZE + i] = i; + } + + QBuffer b; + b.setData(ba); + QVERIFY(b.open(QIODevice::ReadOnly)); + + KCompressionDevice kcd(&b, false, KCompressionDevice::GZip); + QVERIFY(kcd.open(QIODevice::ReadOnly)); + QVERIFY(kcd.seek(BUFFER_SIZE)); + + // the 10 bytes after BUFFER_SIZE should be 0 to 9 + const QByteArray kcdData = kcd.read(10); + QCOMPARE(kcdData.size(), 10); + for (int i = 0; i < kcdData.size(); ++i) { + QCOMPARE(kcdData[i], i); + } +} diff --git a/autotests/kcompressiondevicetest.h b/autotests/kcompressiondevicetest.h index 879a41e..64f9f6b 100644 --- a/autotests/kcompressiondevicetest.h +++ b/autotests/kcompressiondevicetest.h @@ -1,62 +1,64 @@ /* This file is part of the KDE project Copyright (C) 2015 Luiz Romário Santana Rios This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef KCOMPRESSIONDEVICETEST_H #define KCOMPRESSIONDEVICETEST_H #include #include #include #include #include class QNetworkReply; class KCompressionDeviceTest : public QObject { Q_OBJECT private: QNetworkReply *getArchive(const QString &extension); QString formatExtension(KCompressionDevice::CompressionType type) const; void setDeviceToArchive( QIODevice *d, KCompressionDevice::CompressionType type); void testBufferedDevice(KCompressionDevice::CompressionType type); void testExtraction(); QNetworkAccessManager qnam; QScopedPointer device; QScopedPointer archive; private Q_SLOTS: void regularKTarUsage(); void testGZipBufferedDevice(); void testBZip2BufferedDevice(); void testXzBufferedDevice(); void testWriteErrorOnOpen(); void testWriteErrorOnClose(); + + void testSeekReadUncompressedBuffer(); }; #endif diff --git a/src/kcompressiondevice.cpp b/src/kcompressiondevice.cpp index 784c150..2138fb8 100644 --- a/src/kcompressiondevice.cpp +++ b/src/kcompressiondevice.cpp @@ -1,424 +1,423 @@ /* This file is part of the KDE libraries Copyright (C) 2000 David Faure Copyright (C) 2011 Mario Bensi This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "kcompressiondevice.h" +#include "kcompressiondevice_p.h" #include "loggingcategory.h" #include #include "kfilterbase.h" #include #include // for EOF #include #include #include "kgzipfilter.h" #include "knonefilter.h" #if HAVE_BZIP2_SUPPORT #include "kbzip2filter.h" #endif #if HAVE_XZ_SUPPORT #include "kxzfilter.h" #endif -#define BUFFER_SIZE 8*1024 - #include class KCompressionDevicePrivate { public: KCompressionDevicePrivate(KCompressionDevice *q) : bNeedHeader(true) , bSkipHeaders(false) , bOpenedUnderlyingDevice(false) , type(KCompressionDevice::None) , errorCode(QFileDevice::NoError) , deviceReadPos(0) , q(q) { } void propagateErrorCode(); bool bNeedHeader; bool bSkipHeaders; bool bOpenedUnderlyingDevice; QByteArray buffer; // Used as 'input buffer' when reading, as 'output buffer' when writing QByteArray origFileName; KFilterBase::Result result; KFilterBase *filter; KCompressionDevice::CompressionType type; QFileDevice::FileError errorCode; qint64 deviceReadPos; KCompressionDevice *q; }; void KCompressionDevicePrivate::propagateErrorCode() { QIODevice *dev = filter->device(); if (QFileDevice *fileDev = qobject_cast(dev)) { if (fileDev->error() != QFileDevice::NoError) { errorCode = fileDev->error(); q->setErrorString(dev->errorString()); } } // ... we have no generic way to propagate errors from other kinds of iodevices. Sucks, heh? :( } KFilterBase *KCompressionDevice::filterForCompressionType(KCompressionDevice::CompressionType type) { switch (type) { case KCompressionDevice::GZip: return new KGzipFilter; case KCompressionDevice::BZip2: #if HAVE_BZIP2_SUPPORT return new KBzip2Filter; #else return nullptr; #endif case KCompressionDevice::Xz: #if HAVE_XZ_SUPPORT return new KXzFilter; #else return nullptr; #endif case KCompressionDevice::None: return new KNoneFilter; } return nullptr; } KCompressionDevice::KCompressionDevice(QIODevice *inputDevice, bool autoDeleteInputDevice, CompressionType type) : d(new KCompressionDevicePrivate(this)) { assert(inputDevice); d->filter = filterForCompressionType(type); if (d->filter) { d->type = type; d->filter->setDevice(inputDevice, autoDeleteInputDevice); } } KCompressionDevice::KCompressionDevice(const QString &fileName, CompressionType type) : d(new KCompressionDevicePrivate(this)) { QFile *f = new QFile(fileName); d->filter = filterForCompressionType(type); if (d->filter) { d->type = type; d->filter->setDevice(f, true); } else { delete f; } } KCompressionDevice::~KCompressionDevice() { if (isOpen()) { close(); } delete d->filter; delete d; } KCompressionDevice::CompressionType KCompressionDevice::compressionType() const { return d->type; } bool KCompressionDevice::open(QIODevice::OpenMode mode) { if (isOpen()) { //qCWarning(KArchiveLog) << "KCompressionDevice::open: device is already open"; return true; // QFile returns false, but well, the device -is- open... } if (!d->filter) { return false; } d->bOpenedUnderlyingDevice = false; //qCDebug(KArchiveLog) << mode; if (mode == QIODevice::ReadOnly) { d->buffer.resize(0); } else { d->buffer.resize(BUFFER_SIZE); d->filter->setOutBuffer(d->buffer.data(), d->buffer.size()); } if (!d->filter->device()->isOpen()) { if (!d->filter->device()->open(mode)) { //qCWarning(KArchiveLog) << "KCompressionDevice::open: Couldn't open underlying device"; d->propagateErrorCode(); return false; } d->bOpenedUnderlyingDevice = true; } d->bNeedHeader = !d->bSkipHeaders; d->filter->setFilterFlags(d->bSkipHeaders ? KFilterBase::NoHeaders : KFilterBase::WithHeaders); if (!d->filter->init(mode)) { return false; } d->result = KFilterBase::Ok; setOpenMode(mode); return true; } void KCompressionDevice::close() { if (!isOpen()) { return; } if (d->filter->mode() == QIODevice::WriteOnly && d->errorCode == QFileDevice::NoError) { write(nullptr, 0); // finish writing } //qCDebug(KArchiveLog) << "Calling terminate()."; if (!d->filter->terminate()) { //qCWarning(KArchiveLog) << "KCompressionDevice::close: terminate returned an error"; d->errorCode = QFileDevice::UnspecifiedError; } if (d->bOpenedUnderlyingDevice) { QIODevice *dev = d->filter->device(); dev->close(); d->propagateErrorCode(); } setOpenMode(QIODevice::NotOpen); } QFileDevice::FileError KCompressionDevice::error() const { return d->errorCode; } bool KCompressionDevice::seek(qint64 pos) { if (d->deviceReadPos == pos) { return QIODevice::seek(pos); } //qCDebug(KArchiveLog) << "seek(" << pos << ") called, current pos=" << QIODevice::pos(); Q_ASSERT(d->filter->mode() == QIODevice::ReadOnly); if (pos == 0) { if (!QIODevice::seek(pos)) return false; // We can forget about the cached data d->bNeedHeader = !d->bSkipHeaders; d->result = KFilterBase::Ok; d->filter->setInBuffer(nullptr, 0); d->filter->reset(); d->deviceReadPos = 0; return d->filter->device()->reset(); } qint64 bytesToRead; if (d->deviceReadPos < pos) { // we can start from here bytesToRead = pos - d->deviceReadPos; // Since we're going to do a read() below // we need to reset the internal QIODevice pos to the real position we are // so that after read() we are indeed pointing to the pos seek // asked us to be in if (!QIODevice::seek(d->deviceReadPos)) { return false; } } else { // we have to start from 0 ! Ugly and slow, but better than the previous // solution (KTarGz was allocating everything into memory) if (!seek(0)) { // recursive return false; } bytesToRead = pos; } //qCDebug(KArchiveLog) << "reading " << bytesToRead << " dummy bytes"; QByteArray dummy(qMin(bytesToRead, qint64(3 * BUFFER_SIZE)), 0); const bool result = (read(dummy.data(), bytesToRead) == bytesToRead); return result; } bool KCompressionDevice::atEnd() const { return (d->type == KCompressionDevice::None || d->result == KFilterBase::End) && QIODevice::atEnd() // take QIODevice's internal buffer into account && d->filter->device()->atEnd(); } qint64 KCompressionDevice::readData(char *data, qint64 maxlen) { Q_ASSERT(d->filter->mode() == QIODevice::ReadOnly); //qCDebug(KArchiveLog) << "maxlen=" << maxlen; KFilterBase *filter = d->filter; uint dataReceived = 0; // We came to the end of the stream if (d->result == KFilterBase::End) { return dataReceived; } // If we had an error, return -1. if (d->result != KFilterBase::Ok) { return -1; } qint64 availOut = maxlen; filter->setOutBuffer(data, maxlen); while (dataReceived < maxlen) { if (filter->inBufferEmpty()) { // Not sure about the best size to set there. // For sure, it should be bigger than the header size (see comment in readHeader) d->buffer.resize(BUFFER_SIZE); // Request data from underlying device int size = filter->device()->read(d->buffer.data(), d->buffer.size()); //qCDebug(KArchiveLog) << "got" << size << "bytes from device"; if (size) { filter->setInBuffer(d->buffer.data(), size); } else { // Not enough data available in underlying device for now break; } } if (d->bNeedHeader) { (void) filter->readHeader(); d->bNeedHeader = false; } d->result = filter->uncompress(); if (d->result == KFilterBase::Error) { //qCWarning(KArchiveLog) << "KCompressionDevice: Error when uncompressing data"; break; } // We got that much data since the last time we went here uint outReceived = availOut - filter->outBufferAvailable(); //qCDebug(KArchiveLog) << "avail_out = " << filter->outBufferAvailable() << " result=" << d->result << " outReceived=" << outReceived; if (availOut < uint(filter->outBufferAvailable())) { //qCWarning(KArchiveLog) << " last availOut " << availOut << " smaller than new avail_out=" << filter->outBufferAvailable() << " !"; } dataReceived += outReceived; data += outReceived; availOut = maxlen - dataReceived; if (d->result == KFilterBase::End) { // We're actually at the end, no more data to check if (filter->device()->atEnd()) { break; } // Still not done, re-init and try again filter->init(filter->mode()); } filter->setOutBuffer(data, availOut); } d->deviceReadPos += dataReceived; return dataReceived; } qint64 KCompressionDevice::writeData(const char *data /*0 to finish*/, qint64 len) { KFilterBase *filter = d->filter; Q_ASSERT(filter->mode() == QIODevice::WriteOnly); // If we had an error, return 0. if (d->result != KFilterBase::Ok) { return 0; } bool finish = (data == nullptr); if (!finish) { filter->setInBuffer(data, len); if (d->bNeedHeader) { (void)filter->writeHeader(d->origFileName); d->bNeedHeader = false; } } uint dataWritten = 0; uint availIn = len; while (dataWritten < len || finish) { d->result = filter->compress(finish); if (d->result == KFilterBase::Error) { //qCWarning(KArchiveLog) << "KCompressionDevice: Error when compressing data"; // What to do ? break; } // Wrote everything ? if (filter->inBufferEmpty() || (d->result == KFilterBase::End)) { // We got that much data since the last time we went here uint wrote = availIn - filter->inBufferAvailable(); //qCDebug(KArchiveLog) << " Wrote everything for now. avail_in=" << filter->inBufferAvailable() << "result=" << d->result << "wrote=" << wrote; // Move on in the input buffer data += wrote; dataWritten += wrote; availIn = len - dataWritten; //qCDebug(KArchiveLog) << " availIn=" << availIn << "dataWritten=" << dataWritten << "pos=" << pos(); if (availIn > 0) { filter->setInBuffer(data, availIn); } } if (filter->outBufferFull() || (d->result == KFilterBase::End) || finish) { //qCDebug(KArchiveLog) << " writing to underlying. avail_out=" << filter->outBufferAvailable(); int towrite = d->buffer.size() - filter->outBufferAvailable(); if (towrite > 0) { // Write compressed data to underlying device int size = filter->device()->write(d->buffer.data(), towrite); if (size != towrite) { //qCWarning(KArchiveLog) << "KCompressionDevice::write. Could only write " << size << " out of " << towrite << " bytes"; d->errorCode = QFileDevice::WriteError; setErrorString(tr("Could not write. Partition full?")); return 0; // indicate an error } //qCDebug(KArchiveLog) << " wrote " << size << " bytes"; } if (d->result == KFilterBase::End) { Q_ASSERT(finish); // hopefully we don't get end before finishing break; } d->buffer.resize(BUFFER_SIZE); filter->setOutBuffer(d->buffer.data(), d->buffer.size()); } } return dataWritten; } void KCompressionDevice::setOrigFileName(const QByteArray &fileName) { d->origFileName = fileName; } void KCompressionDevice::setSkipHeaders() { d->bSkipHeaders = true; } KFilterBase *KCompressionDevice::filterBase() { return d->filter; } diff --git a/src/kcompressiondevice_p.h b/src/kcompressiondevice_p.h new file mode 100644 index 0000000..3048480 --- /dev/null +++ b/src/kcompressiondevice_p.h @@ -0,0 +1,24 @@ +/* This file is part of the KDE libraries + Copyright (C) 2000 David Faure + Copyright (C) 2011 Mario Bensi + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License version 2 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public License + along with this library; see the file COPYING.LIB. If not, write to + the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +#ifndef __kcompressiondevice_p_h +#define __kcompressiondevice_p_h + +#define BUFFER_SIZE 8*1024 + +#endif