diff --git a/src/backend/datasources/filters/AbstractFileFilter.h b/src/backend/datasources/filters/AbstractFileFilter.h index a1c74c12c..f69f29ae5 100644 --- a/src/backend/datasources/filters/AbstractFileFilter.h +++ b/src/backend/datasources/filters/AbstractFileFilter.h @@ -1,72 +1,72 @@ /*************************************************************************** File : AbstractFileFilter.h Project : LabPlot Description : file I/O-filter related interface -------------------------------------------------------------------- Copyright : (C) 2009-2018 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ABSTRACTFILEFILTER_H #define ABSTRACTFILEFILTER_H #include "backend/core/AbstractColumn.h" #include #include #include // smart pointer class AbstractDataSource; class XmlStreamReader; class QXmlStreamWriter; class AbstractFileFilter : public QObject { Q_OBJECT Q_ENUMS(ImportMode) public: enum FileType {Ascii, Binary, Image, HDF5, NETCDF, FITS, Json, ROOT, NgspiceRawAscii}; enum ImportMode {Append, Prepend, Replace}; AbstractFileFilter() {} ~AbstractFileFilter() override {} static bool isNan(QString); static AbstractColumn::ColumnMode columnMode(const QString& valueString, const QString& dateTimeFormat, QLocale::Language); static QStringList numberFormats(); static AbstractFileFilter::FileType fileType(const QString&); static QStringList fileTypes(); virtual QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - ImportMode = AbstractFileFilter::Replace, int lines = -1) = 0; + ImportMode = AbstractFileFilter::Replace) = 0; virtual void write(const QString& fileName, AbstractDataSource*) = 0; virtual void loadFilterSettings(const QString& filterName) = 0; virtual void saveFilterSettings(const QString& filterName) const = 0; virtual void save(QXmlStreamWriter*) const = 0; virtual bool load(XmlStreamReader*) = 0; signals: void completed(int) const; //!< int ranging from 0 to 100 notifies about the status of a read/write process }; #endif diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index f5d0d8a7e..3ba59e461 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,1655 +1,1655 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2018 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include "backend/core/Project.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" #include "backend/lib/macros.h" #include "backend/lib/trace.h" #include #include #include #include #include /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() {} /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource* dataSource) { d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { return d->readFromLiveDevice(device, dataSource, from); } /*! reads the content of the file \c fileName. */ -QVector AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - d->readDataFromFile(fileName, dataSource, importMode, lines); +QVector AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { + d->readDataFromFile(fileName, dataSource, importMode); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } QVector AsciiFilter::preview(QIODevice &device) { return d->preview(device); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE" << "2xSPACE" << "3xSPACE" << "4xSPACE" << "2xTAB"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } QString AsciiFilter::fileInfoString(const QString& fileName) { QString info(i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName))); info += QLatin1String("
"); info += i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); return info; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " to determine number of lines"); return 0; } if (!device.canReadLine()) return -1; size_t lineCount = 0; while (!device.atEnd()) { device.readLine(); lineCount++; } //TODO: wc is much faster but not portable /* QElapsedTimer myTimer; myTimer.start(); QProcess wc; wc.start(QString("wc"), QStringList() << "-l" << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) lineCount = wc.readLine().split(' ')[0].toInt(); lineCount++; // last line not counted DEBUG(" Elapsed time counting lines : " << myTimer.elapsed() << " ms"); */ return lineCount; } /*! returns the number of lines in the device \c device and 0 if sequential. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice &device) { if (device.isSequential()) return 0; if (!device.canReadLine()) DEBUG("WARNING in AsciiFilter::lineNumber(): device cannot 'readLine()' but using it anyway."); size_t lineCount = 0; device.seek(0); while (!device.atEnd()) { device.readLine(); lineCount++; } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool AsciiFilter::NaNValueToZeroEnabled() const { if (d->nanValue == 0) return true; return false; } void AsciiFilter::setRemoveQuotesEnabled(bool b) { d->removeQuotesEnabled = b; } bool AsciiFilter::removeQuotesEnabled() const { return d->removeQuotesEnabled; } void AsciiFilter::setVectorNames(const QString& s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner), commentCharacter("#"), separatingCharacter("auto"), numberFormat(QLocale::C), autoModeEnabled(true), headerEnabled(true), skipEmptyParts(false), simplifyWhitespacesEnabled(true), nanValue(NAN), removeQuotesEnabled(false), createIndexEnabled(false), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_actualStartRow(1), m_actualRows(0), m_actualCols(0), m_prepared(false), m_columnOffset(0) { } /*! * get a single line from device */ QStringList AsciiFilterPrivate::getLineString(QIODevice& device) { QString line; do { // skip comment lines in data lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::getLineString(): device cannot 'readLine()' but using it anyway."); // line = device.readAll(); line = device.readLine(); } while (line.startsWith(commentCharacter)); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); DEBUG("data line : \'" << line.toStdString() << '\''); QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); //TODO: remove quotes here? QDEBUG("data line, parsed: " << lineStringList); return lineStringList; } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("AsciiFilterPrivate::prepareDeviceToRead(): is sequential = " << device.isSequential() << ", can readLine = " << device.canReadLine()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; ///////////////////////////////////////////////////////////////// // Find first data line (ignoring comment lines) DEBUG(" Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { QString line; if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); line = device.readLine(); DEBUG(" line = " << line.toStdString()); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } //TOOD: this logic seems to be wrong. If the user asks to read from line startRow, we should start here independent of any comments if (line.startsWith(commentCharacter)) // ignore commented lines before startRow i--; } // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); firstLine = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive); m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); QDEBUG("first line: " << firstLineStringList); DEBUG("headerEnabled = " << headerEnabled); //optionally, remove potential spaces in the first line if (simplifyWhitespacesEnabled) { for (int i = 0; i < firstLineStringList.size(); ++i) firstLineStringList[i] = firstLineStringList[i].simplified(); } if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); m_actualStartRow = startRow + 1; } else m_actualStartRow = startRow; // set range to read if (endColumn == -1) { if (headerEnabled || vectorNames.size() == 0) endColumn = firstLineStringList.size(); // last column else //number of vector names provided in the import dialog (not more than the maximal number of columns in the file) endColumn = qMin(vectorNames.size(), firstLineStringList.size()); } if (createIndexEnabled) { vectorNames.prepend(i18n("Index")); endColumn++; } m_actualCols = endColumn - startColumn + 1; //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ ///////////////////////////////////////////////////////////////// // parse first data line to determine data type for each column if (!device.isSequential()) firstLineStringList = getLineString(device); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Integer; col = 1; } for (auto& valueString: firstLineStringList) { // parse columns available in first data line if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } // parsing more lines to better determine data types for (unsigned int i = 0; i < m_dataTypeLines; ++i) { firstLineStringList = getLineString(device); if (createIndexEnabled) col = 1; else col = 0; for (auto& valueString: firstLineStringList) { if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (col == m_actualCols) break; AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); // numeric: integer -> numeric if (mode == AbstractColumn::Numeric && columnModes[col] == AbstractColumn::Integer) columnModes[col] = mode; // text: non text -> text if (mode == AbstractColumn::Text && columnModes[col] != AbstractColumn::Text) columnModes[col] = mode; col++; } } QDEBUG("column modes = " << columnModes); // ATTENTION: This resets the position in the device to 0 m_actualRows = (int)AsciiFilter::lineNumber(device); // reset to start of file //TODO: seems to be redundant since it's already done in the lineNumber() call above if (!device.isSequential()) device.seek(0); ///////////////////////////////////////////////////////////////// int actualEndRow = endRow; DEBUG("endRow(actualEndRow) = " << endRow << ", m_actualRows = " << m_actualRows); if (endRow == -1 || endRow > m_actualRows) actualEndRow = m_actualRows; if (m_actualRows > actualEndRow) m_actualRows = actualEndRow; DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << m_actualStartRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header incl. start rows): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0 && !device.isSequential()) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ -void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { +void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " - << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); + << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode)); KFilterDev device(fileName); - readDataFromDevice(device, dataSource, importMode, lines); + readDataFromDevice(device, dataSource, importMode, -1); } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { DEBUG("AsciiFilterPrivate::readFromLiveDevice(): bytes available = " << device.bytesAvailable() << ", from = " << from); if (!(device.bytesAvailable() > 0)) { DEBUG(" No new data available"); return 0; } LiveDataSource* spreadsheet = dynamic_cast(dataSource); if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe) if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return 0; if (!m_prepared) { DEBUG("Preparing .."); switch (spreadsheet->sourceType()) { case LiveDataSource::SourceType::FileOrPipe: { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return 0; } break; } case LiveDataSource::SourceType::NetworkTcpSocket: case LiveDataSource::SourceType::NetworkUdpSocket: case LiveDataSource::SourceType::LocalSocket: case LiveDataSource::SourceType::SerialPort: m_actualRows = 1; if (createIndexEnabled) { m_actualCols = 2; columnModes << AbstractColumn::Integer << AbstractColumn::Numeric; vectorNames << i18n("Index") << i18n("Value"); } else { m_actualCols = 1; columnModes << AbstractColumn::Numeric; vectorNames << i18n("Value"); } QDEBUG(" vector names = " << vectorNames); } // prepare import for spreadsheet spreadsheet->setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); DEBUG(" data source resized to col: " << m_actualCols); DEBUG(" data source rowCount: " << spreadsheet->rowCount()); //columns in a file data source don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } int keepNValues = spreadsheet->keepNValues(); if (keepNValues == 0) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(keepNValues); m_actualRows = keepNValues; } m_dataContainer.resize(m_actualCols); DEBUG(" Setting data .."); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } DEBUG("Prepared!"); } qint64 bytesread = 0; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif LiveDataSource::ReadingType readingType; if (!m_prepared) { readingType = LiveDataSource::ReadingType::TillEnd; } else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = LiveDataSource::ReadingType::TillEnd; //if we read the whole file we just start from the beginning of it //and read till end else if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) readingType = LiveDataSource::ReadingType::TillEnd; else readingType = spreadsheet->readingType(); } DEBUG(" reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType)); //move to the last read position, from == total bytes read //since the other source types are sequencial we cannot seek on them if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); DEBUG(" bytes available = " << device.bytesAvailable()); //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleSizeNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != LiveDataSource::ReadingType::TillEnd) newData.resize(spreadsheet->sampleSize()); int newDataIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif while (!device.atEnd()) { DEBUG(" reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType)); DEBUG(" source type = " << ENUM_TO_STRING(LiveDataSource, SourceType, spreadsheet->sourceType())); if (readingType != LiveDataSource::ReadingType::TillEnd) { switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData[newDataIdx++] = device.readAll(); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData[newDataIdx++] = device.read(device.bytesAvailable()); break; case LiveDataSource::SourceType::FileOrPipe: case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::readFromLiveDevice(): device cannot 'readLine()' but using it anyway."); newData[newDataIdx++] = device.readLine(); } } else { // ReadingType::TillEnd switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData.push_back(device.readAll()); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData.push_back(device.read(device.bytesAvailable())); break; case LiveDataSource::SourceType::FileOrPipe: case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::readFromLiveDevice(): device cannot 'readLine()' but using it anyway."); newData.push_back(device.readLine()); } } newLinesTillEnd++; if (readingType != LiveDataSource::ReadingType::TillEnd) { newLinesForSampleSizeNotTillEnd++; //for Continuous reading and FromEnd we read sample rate number of lines if possible //here TillEnd and Whole file behave the same if (newLinesForSampleSizeNotTillEnd == spreadsheet->sampleSize()) break; } } QDEBUG(" data read: " << newData); } //now we reset the readingType if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = spreadsheet->readingType(); //we had less new lines than the sample size specified if (readingType != LiveDataSource::ReadingType::TillEnd) QDEBUG("Removed empty lines: " << newData.removeAll("")); //back to the last read position before counting when reading from files if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); int currentRow = 0; // indexes the position in the vector(column) int linesToRead = 0; int keepNValues = spreadsheet->keepNValues(); DEBUG("Increase row count"); if (m_prepared) { //increase row count if we don't have a fixed size //but only after the preparation step if (keepNValues == 0) { if (readingType != LiveDataSource::ReadingType::TillEnd) m_actualRows += qMin(newData.size(), spreadsheet->sampleSize()); else { //we don't increase it if we reread the whole file, we reset it if (!(spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)) m_actualRows += newData.size(); else m_actualRows = newData.size(); } //appending if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) linesToRead = m_actualRows; else linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; } else { // fixed size if (readingType == LiveDataSource::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; //TODO after reading we should skip the next data lines //because it's TillEnd actually } else linesToRead = newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinuouslyFixed or FromEnd, WholeFile is disabled linesToRead = qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } DEBUG(" actual row = " << m_actualRows); if (linesToRead == 0) return 0; } else { linesToRead = newLinesTillEnd; if (headerEnabled) --m_actualRows; } DEBUG(" lines to read = " << linesToRead); //TODO: check other source types if (spreadsheet->sourceType() == LiveDataSource::SourceType::NetworkUdpSocket) { if (m_actualRows < linesToRead) { DEBUG(" SET actual rows to " << linesToRead); m_actualRows = linesToRead; } } //new rows/resize columns if we don't have a fixed size //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize if (keepNValues == 0) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { //when we have a fixed size we have to pop sampleSize number of lines if specified //here popping, setting currentRow if (!m_prepared) { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); } else { if (readingType == LiveDataSource::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) { currentRow = 0; } else { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } } else { //we read max sample rate number of lines when the reading mode //is ContinuouslyFixed or FromEnd currentRow = m_actualRows - qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet qDebug() << "reading from line" << currentRow << " till end" << newLinesTillEnd; qDebug() << "Lines to read:" << linesToRead <<", actual rows:" << m_actualRows << ", actual cols:" << m_actualCols; newDataIdx = 0; if (readingType == LiveDataSource::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->sampleSize()) newDataIdx = newData.size() - spreadsheet->sampleSize(); //since we skip a couple of lines, we need to count those bytes too for (int i = 0; i < newDataIdx; ++i) bytesread += newData.at(i).size(); } } qDebug() << "newDataIdx: " << newDataIdx; //TODO static int indexColumnIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif int row = 0; if (readingType == LiveDataSource::ReadingType::TillEnd || (readingType == LiveDataSource::ReadingType::ContinuousFixed)) { if (headerEnabled) { if (!m_prepared) { row = 1; bytesread += newData.at(0).size(); } } } if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { if (readingType == LiveDataSource::ReadingType::WholeFile) { if (headerEnabled) { row = 1; bytesread += newData.at(0).size(); } } } for (; row < linesToRead; ++row) { DEBUG(" row = " << row); QString line; if (readingType == LiveDataSource::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(row); //when we read the whole file we don't care about the previous position //so we don't have to count those bytes if (readingType != LiveDataSource::ReadingType::WholeFile) { if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { bytesread += line.size(); } } //qDebug() << "line bytes: " << line.size() << " line: " << line; if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList; // only FileOrPipe support multiple columns if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); else lineStringList << line; QDEBUG(" line = " << lineStringList << ", separator = \'" << m_separator << "\'"); if (createIndexEnabled) { if (spreadsheet->keepNValues() == 0) lineStringList.prepend(QString::number(currentRow)); else lineStringList.prepend(QString::number(indexColumnIdx++)); } QDEBUG(" column modes = " << columnModes); for (int n = 0; n < m_actualCols; ++n) { DEBUG(" actual col = " << n); if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); DEBUG(" value string = " << valueString.toStdString()); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { DEBUG(" Numeric"); bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::Integer: { DEBUG(" Integer"); bool isNumber; const int value = locale.toInt(valueString, &isNumber); DEBUG(" container size = " << m_dataContainer.size() << ", current row = " << currentRow); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { DEBUG(" missing columns in this line"); switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); const Project* project = spreadsheet->project(); QVector curves = project->children(AbstractAspect::Recursive); QVector plots; for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); //determine the plots where the column is consumed for (const auto* curve: curves) { if (curve->xColumn() == column || curve->yColumn() == column) { CartesianPlot* plot = dynamic_cast(curve->parentAspect()); if (plots.indexOf(plot) == -1) { plots << plot; plot->setSuppressDataChangedSignal(true); } } } column->setChanged(); } //loop over all affected plots and retransform them for (auto* plot: plots) { plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } m_prepared = true; return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return; } // matrix data has only one column mode (which is not text) if (dynamic_cast(dataSource)) { auto mode = columnModes[0]; if (mode == AbstractColumn::Text) mode = AbstractColumn::Numeric; for (auto& c: columnModes) if (c != mode) c = mode; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows - m_actualStartRow + 1, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); ++i) { QString line = device.readLine(); // skip start lines if (m_actualStartRow > 1) { m_actualStartRow--; continue; } line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); //prepend the index if required //TODO: come up maybe with a solution with adding the index inside of the loop below, //without conversion to string, prepending to the list and then conversion back to integer. if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); // remove left white spaces if (skipEmptyParts) { for (int n = 0; n < lineStringList.size(); ++n) { QString valueString = lineStringList.at(n); if (!QString::compare(valueString, " ")) { lineStringList.removeAt(n); n--; } } } for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, dateTimeFormat, importMode); } /*! * preview for special devices (local/UDP/TCP socket or serial port) */ QVector AsciiFilterPrivate::preview(QIODevice &device) { DEBUG("AsciiFilterPrivate::preview(): bytesAvailable = " << device.bytesAvailable() << ", isSequential = " << device.isSequential()); QVector dataStrings; if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return dataStrings; } if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return dataStrings; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif int linesToRead = 0; QVector newData; while (!device.atEnd()) { if (device.canReadLine()) newData.push_back(device.readLine()); else // UDP fails otherwise newData.push_back(device.readAll()); linesToRead++; } QDEBUG(" data = " << newData); if (linesToRead == 0) return dataStrings; int col = 0; int colMax = newData.at(0).size(); if (createIndexEnabled) colMax++; columnModes.resize(colMax); if (createIndexEnabled) { columnModes[0] = AbstractColumn::ColumnMode::Integer; col = 1; vectorNames.prepend(i18n("Index")); } vectorNames.append(i18n("Value")); QDEBUG(" vector names = " << vectorNames); for (const auto& valueString: newData.at(0).split(' ', QString::SkipEmptyParts)) { if (col == colMax) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } for (int i = 0; i < linesToRead; ++i) { QString line = newData.at(i); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(' ', QString::SkipEmptyParts); if (createIndexEnabled) lineStringList.prepend(QString::number(i)); QStringList lineString; for (int n = 0; n < lineStringList.size(); ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 16); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; // set column names for preview if (!headerEnabled) { int start = 0; if (createIndexEnabled) start = 1; for (int i=start;i 1) { m_actualStartRow--; continue; } line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); QDEBUG(" line = " << lineStringList); //prepend index if required if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); QStringList lineString; for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); //DEBUG(" valueString = " << valueString.toStdString()); if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces continue; // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 15); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to ascii file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement( "asciiFilter"); writer->writeAttribute( "commentCharacter", d->commentCharacter); writer->writeAttribute( "separatingCharacter", d->separatingCharacter); writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); writer->writeAttribute( "nanValue", QString::number(d->nanValue)); writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled)); writer->writeAttribute( "startRow", QString::number(d->startRow)); writer->writeAttribute( "endRow", QString::number(d->endRow)); writer->writeAttribute( "startColumn", QString::number(d->startColumn)); writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("commentCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("commentCharacter").toString()); else d->commentCharacter = str; str = attribs.value("separatingCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("separatingCharacter").toString()); else d->separatingCharacter = str; str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("createIndex").toString()); else d->createIndexEnabled = str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("autoMode").toString()); else d->autoModeEnabled = str.toInt(); str = attribs.value("header").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("header").toString()); else d->headerEnabled = str.toInt(); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty str = attribs.value("simplifyWhitespaces").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("simplifyWhitespaces").toString()); else d->simplifyWhitespacesEnabled = str.toInt(); str = attribs.value("nanValue").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("nanValue").toString()); else d->nanValue = str.toDouble(); str = attribs.value("removeQuotes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("removeQuotes").toString()); else d->removeQuotesEnabled = str.toInt(); str = attribs.value("skipEmptyParts").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipEmptyParts").toString()); else d->skipEmptyParts = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startRow").toString()); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endRow").toString()); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startColumn").toString()); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endColumn").toString()); else d->endColumn = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/AsciiFilter.h b/src/backend/datasources/filters/AsciiFilter.h index 7fd2b2d11..df45e8d65 100644 --- a/src/backend/datasources/filters/AsciiFilter.h +++ b/src/backend/datasources/filters/AsciiFilter.h @@ -1,116 +1,115 @@ /*************************************************************************** File : AsciiFilter.h Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTER_H #define ASCIIFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" class QStringList; class QIODevice; class AsciiFilterPrivate; class QAbstractSocket; class AsciiFilter : public AbstractFileFilter { Q_OBJECT public: AsciiFilter(); ~AsciiFilter() override; static QStringList separatorCharacters(); static QStringList commentCharacters(); static QStringList dataTypes(); static QStringList predefinedFilters(); static QString fileInfoString(const QString&); static int columnNumber(const QString& fileName, const QString& separator = QString()); static size_t lineNumber(const QString& fileName); static size_t lineNumber(QIODevice&); // calculate number of lines if device supports it // read data from any device void readDataFromDevice(QIODevice& device, AbstractDataSource*, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*dataSource); - qint64 readFromLiveDevice(QIODevice& device, AbstractDataSource*, - qint64 from = -1); + qint64 readFromLiveDevice(QIODevice& device, AbstractDataSource*, qint64 from = -1); // overloaded function to read from file QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace) override; void write(const QString& fileName, AbstractDataSource*) override; QVector preview(const QString& fileName, int lines); QVector preview(QIODevice& device); void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setCommentCharacter(const QString&); QString commentCharacter() const; void setSeparatingCharacter(const QString&); QString separatingCharacter() const; void setDateTimeFormat(const QString&); QString dateTimeFormat() const; void setNumberFormat(QLocale::Language); QLocale::Language numberFormat() const; void setAutoModeEnabled(const bool); bool isAutoModeEnabled() const; void setHeaderEnabled(const bool); bool isHeaderEnabled() const; void setSkipEmptyParts(const bool); bool skipEmptyParts() const; void setSimplifyWhitespacesEnabled(const bool); bool simplifyWhitespacesEnabled() const; void setNaNValueToZero(const bool); bool NaNValueToZeroEnabled() const; void setRemoveQuotesEnabled(const bool); bool removeQuotesEnabled() const; void setCreateIndexEnabled(const bool); void setVectorNames(const QString&); QStringList vectorNames() const; QVector columnModes(); void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class AsciiFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/AsciiFilterPrivate.h b/src/backend/datasources/filters/AsciiFilterPrivate.h index ddd94ab86..71d8656ac 100644 --- a/src/backend/datasources/filters/AsciiFilterPrivate.h +++ b/src/backend/datasources/filters/AsciiFilterPrivate.h @@ -1,86 +1,86 @@ /*************************************************************************** File : AsciiFilterPrivate.h Project : LabPlot Description : Private implementation class for AsciiFilter. -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTERPRIVATE_H #define ASCIIFILTERPRIVATE_H class KFilterDev; class AbstractDataSource; class AbstractColumn; class AsciiFilterPrivate { public: explicit AsciiFilterPrivate(AsciiFilter*); QStringList getLineString(QIODevice&); int prepareDeviceToRead(QIODevice&); void readDataFromDevice(QIODevice&, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); qint64 readFromLiveDevice(QIODevice&, AbstractDataSource*, qint64 from = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); QVector preview(QIODevice& device); const AsciiFilter* q; QString commentCharacter; QString separatingCharacter; QString dateTimeFormat; QLocale::Language numberFormat; bool autoModeEnabled; bool headerEnabled; bool skipEmptyParts; bool simplifyWhitespacesEnabled; double nanValue; bool removeQuotesEnabled; bool createIndexEnabled; QStringList vectorNames; QVector columnModes; int startRow; int endRow; int startColumn; int endColumn; private: static const unsigned int m_dataTypeLines = 10; // lines to read for determining data types QString m_separator; int m_actualStartRow; int m_actualRows; int m_actualCols; int m_prepared; int m_columnOffset; // indexes the "start column" in the datasource. Data will be imported starting from this column. QVector m_dataContainer; // pointers to the actual data containers }; #endif diff --git a/src/backend/datasources/filters/BinaryFilter.cpp b/src/backend/datasources/filters/BinaryFilter.cpp index d5993b7c5..7481309cb 100644 --- a/src/backend/datasources/filters/BinaryFilter.cpp +++ b/src/backend/datasources/filters/BinaryFilter.cpp @@ -1,617 +1,617 @@ /*************************************************************************** File : BinaryFilter.cpp Project : LabPlot Description : Binary I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/BinaryFilter.h" #include "backend/datasources/filters/BinaryFilterPrivate.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include /*! \class BinaryFilter \brief Manages the import/export of data organized as columns (vectors) from/to a binary file. \ingroup datasources */ BinaryFilter::BinaryFilter():AbstractFileFilter(), d(new BinaryFilterPrivate(this)) {} BinaryFilter::~BinaryFilter() {} /*! reads the content of the file \c fileName. */ -QVector BinaryFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - d->readDataFromFile(fileName, dataSource, importMode, lines); +QVector BinaryFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { + d->readDataFromFile(fileName, dataSource, importMode); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } /*! reads the content of the device \c device. */ void BinaryFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } QVector BinaryFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void BinaryFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! returns the list of all predefined data formats. */ QStringList BinaryFilter::dataTypes() { return (QStringList()<<"int8 (8 bit signed integer)"<<"int16 (16 bit signed integer)"<<"int32 (32 bit signed integer)"<<"int64 (64 bit signed integer)" <<"uint8 (8 bit unsigned integer)"<<"uint16 (16 bit unsigned integer)"<<"uint32 (32 bit unsigned integer)"<<"uint64 (64 bit unsigned integer)" <<"real32 (single precision floats)"<<"real64 (double precision floats)"); } /*! returns the list of all predefined byte order. */ QStringList BinaryFilter::byteOrders() { return (QStringList() << "Little endian" << "Big endian"); } /*! returns the size of the predefined data types */ int BinaryFilter::dataSize(BinaryFilter::DataType type) { int sizes[] = {1,2,4,8,1,2,4,8,4,8}; return sizes[(int)type]; } /*! returns the number of rows (length of vectors) in the file \c fileName. */ size_t BinaryFilter::rowNumber(const QString& fileName, const size_t vectors, const BinaryFilter::DataType type) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) return 0; size_t rows = 0; while (!device.atEnd()) { // one row for (size_t i = 0; i < vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(type); ++j) device.read(1); } rows++; } return rows; } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void BinaryFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void BinaryFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void BinaryFilter::setVectors(const size_t v) { d->vectors = v; } size_t BinaryFilter::vectors() const { return d->vectors; } void BinaryFilter::setDataType(const BinaryFilter::DataType t) { d->dataType = t; } BinaryFilter::DataType BinaryFilter::dataType() const { return d->dataType; } void BinaryFilter::setByteOrder(const BinaryFilter::ByteOrder b) { d->byteOrder = b; } BinaryFilter::ByteOrder BinaryFilter::byteOrder() const { return d->byteOrder; } void BinaryFilter::setSkipStartBytes(const size_t s) { d->skipStartBytes = s; } size_t BinaryFilter::skipStartBytes() const { return d->skipStartBytes; } void BinaryFilter::setStartRow(const int s) { d->startRow = s; } int BinaryFilter::startRow() const { return d->startRow; } void BinaryFilter::setEndRow(const int e) { d->endRow = e; } int BinaryFilter::endRow() const { return d->endRow; } void BinaryFilter::setSkipBytes(const size_t s) { d->skipBytes = s; } size_t BinaryFilter::skipBytes() const { return d->skipBytes; } void BinaryFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void BinaryFilter::setAutoModeEnabled(bool b) { d->autoModeEnabled = b; } bool BinaryFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } //##################################################################### //################### Private implementation ########################## //##################################################################### BinaryFilterPrivate::BinaryFilterPrivate(BinaryFilter* owner) : q(owner), vectors(2), dataType(BinaryFilter::INT8), byteOrder(BinaryFilter::LittleEndian), startRow(1), endRow(-1), numRows(0), skipStartBytes(0), skipBytes(0), createIndexEnabled(false), autoModeEnabled(true), m_actualRows(0), m_actualCols(0) { } /*! reads the content of the device \c device to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ -void BinaryFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { +void BinaryFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("readDataFromFile()"); KFilterDev device(fileName); numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); if (! device.open(QIODevice::ReadOnly)) { DEBUG(" could not open file " << fileName.toStdString()); return; } - readDataFromDevice(device, dataSource, importMode, lines); + readDataFromDevice(device, dataSource, importMode, -1); } /*! * returns 1 if the current read position in the device is at the end and 0 otherwise. */ int BinaryFilterPrivate::prepareStreamToRead(QDataStream& in) { DEBUG("prepareStreamToRead()"); if (byteOrder == BinaryFilter::BigEndian) in.setByteOrder(QDataStream::BigEndian); else if (byteOrder == BinaryFilter::LittleEndian) in.setByteOrder(QDataStream::LittleEndian); // catch case that skipStartBytes or startRow is bigger than file if (skipStartBytes >= BinaryFilter::dataSize(dataType) * vectors * numRows || startRow > (int)numRows) return 1; // skip bytes at start for (size_t i = 0; i < skipStartBytes; ++i) { qint8 tmp; in >> tmp; } // skip until start row for (size_t i = 0; i < (startRow-1) * vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(dataType); ++j) { qint8 tmp; in >> tmp; } } // set range of rows if (endRow == -1) m_actualRows = (int)numRows - startRow + 1; else if (endRow > (int)numRows - startRow + 1) m_actualRows = (int)numRows; else m_actualRows = endRow - startRow + 1; m_actualCols = (int)vectors; DEBUG("numRows = " << numRows); DEBUG("endRow = " << endRow); DEBUG("actual rows = " << m_actualRows); DEBUG("actual cols = " << m_actualCols); return 0; } /*! reads \c lines lines of the device \c device and return as string for preview. */ QVector BinaryFilterPrivate::preview(const QString& fileName, int lines) { DEBUG("BinaryFilterPrivate::preview( " << fileName.toStdString() << ", " << lines << ")"); QVector dataStrings; KFilterDev device(fileName); if (! device.open(QIODevice::ReadOnly)) return dataStrings << (QStringList() << i18n("could not open device")); numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); QDataStream in(&device); const int deviceError = prepareStreamToRead(in); if(deviceError) return dataStrings << (QStringList() << i18n("data selection empty")); //TODO: support other modes columnModes.resize(m_actualCols); //TODO: use given names QStringList vectorNames; if (createIndexEnabled) vectorNames.prepend(i18n("Index")); if (lines == -1) lines = m_actualRows; // read data //TODO: use ColumnMode ? DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(m_actualRows, lines); ++i) { QStringList lineString; //prepend the index if required if (createIndexEnabled) lineString << QString::number(i+1); for (int n = 0; n < m_actualCols; ++n) { switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT16: { qint16 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT32: { qint32 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT64: { qint64 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT8: { quint8 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT16: { quint16 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT32: { quint32 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT64: { quint64 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::REAL32: { float value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::REAL64: { double value; in >> value; lineString << QString::number(value); break; } } } dataStrings << lineString; emit q->completed(100*i/m_actualRows); } return dataStrings; } /*! reads the content of the file \c fileName to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ void BinaryFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("BinaryFilterPrivate::readDataFromDevice()"); QDataStream in(&device); const int deviceError = prepareStreamToRead(in); if (deviceError) { dataSource->clear(); DEBUG("device error"); return; } if (createIndexEnabled) m_actualCols++; QVector dataContainer; int columnOffset = 0; //TODO: support other modes columnModes.resize(m_actualCols); //TODO: use given names QStringList vectorNames; if (createIndexEnabled) { vectorNames.prepend(i18n("Index")); columnModes[0] = AbstractColumn::Integer; } columnOffset = dataSource->prepareImport(dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); if (lines == -1) lines = m_actualRows; // start column int startColumn = 0; if (createIndexEnabled) startColumn++; // read data //TODO: use ColumnMode ? DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(m_actualRows, lines); ++i) { DEBUG("reading row " << i); //prepend the index if required if (createIndexEnabled) static_cast*>(dataContainer[0])->operator[](i) = i+1; for (int n = startColumn; n < m_actualCols; ++n) { DEBUG("reading column " << n); switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT16: { qint16 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT32: { qint32 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT64: { qint64 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT8: { quint8 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT16: { quint16 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT32: { quint32 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT64: { quint64 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::REAL32: { float value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::REAL64: { double value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } } } if (m_actualRows > 0) emit q->completed(100*i/m_actualRows); } dataSource->finalizeImport(columnOffset, 1, m_actualCols, "", importMode); } /*! writes the content of \c dataSource to the file \c fileName. */ void BinaryFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void BinaryFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("binaryFilter"); writer->writeAttribute("vectors", QString::number(d->vectors) ); writer->writeAttribute("dataType", QString::number(d->dataType) ); writer->writeAttribute("byteOrder", QString::number(d->byteOrder) ); writer->writeAttribute("autoMode", QString::number(d->autoModeEnabled) ); writer->writeAttribute("startRow", QString::number(d->startRow) ); writer->writeAttribute("endRow", QString::number(d->endRow) ); writer->writeAttribute("skipStartBytes", QString::number(d->skipStartBytes) ); writer->writeAttribute("skipBytes", QString::number(d->skipBytes) ); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled) ); writer->writeEndElement(); } /*! Loads from XML. */ bool BinaryFilter::load(XmlStreamReader* reader) { KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); // read attributes QString str = attribs.value("vectors").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("vectors").toString()); else d->vectors = (size_t)str.toULong(); str = attribs.value("dataType").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("dataType").toString()); else d->dataType = (BinaryFilter::DataType) str.toInt(); str = attribs.value("byteOrder").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("byteOrder").toString()); else d->byteOrder = (BinaryFilter::ByteOrder) str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("autoMode").toString()); else d->autoModeEnabled = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startRow").toString()); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endRow").toString()); else d->endRow = str.toInt(); str = attribs.value("skipStartBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipStartBytes").toString()); else d->skipStartBytes = (size_t)str.toULong(); str = attribs.value("skipBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipBytes").toString()); else d->skipBytes = (size_t)str.toULong(); str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("createIndex").toString()); else d->createIndexEnabled = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/BinaryFilter.h b/src/backend/datasources/filters/BinaryFilter.h index 0be8b1855..1ab677033 100644 --- a/src/backend/datasources/filters/BinaryFilter.h +++ b/src/backend/datasources/filters/BinaryFilter.h @@ -1,93 +1,93 @@ /*************************************************************************** File : BinaryFilter.h Project : LabPlot Description : Binary I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef BINARYFILTER_H #define BINARYFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" class BinaryFilterPrivate; class QStringList; class QIODevice; class BinaryFilter : public AbstractFileFilter { Q_OBJECT Q_ENUMS(DataType) Q_ENUMS(ByteOrder) public: //TODO; use ColumnMode? enum DataType {INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, REAL32, REAL64}; enum ByteOrder {LittleEndian, BigEndian}; BinaryFilter(); ~BinaryFilter() override; static QStringList dataTypes(); static QStringList byteOrders(); static int dataSize(BinaryFilter::DataType); static size_t rowNumber(const QString& fileName, const size_t vectors, const BinaryFilter::DataType); // read data from any device void readDataFromDevice(QIODevice&, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); QVector readDataFromFile(const QString& fileName, AbstractDataSource*, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace) override; void write(const QString& fileName, AbstractDataSource*) override; QVector preview(const QString& fileName, int lines); void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setVectors(const size_t); size_t vectors() const; void setDataType(const BinaryFilter::DataType); BinaryFilter::DataType dataType() const; void setByteOrder(const BinaryFilter::ByteOrder); BinaryFilter::ByteOrder byteOrder() const; void setSkipStartBytes(const size_t); size_t skipStartBytes() const; void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setSkipBytes(const size_t); size_t skipBytes() const; void setCreateIndexEnabled(const bool); void setAutoModeEnabled(const bool); bool isAutoModeEnabled() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class BinaryFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/BinaryFilterPrivate.h b/src/backend/datasources/filters/BinaryFilterPrivate.h index fccb921ef..29f35ece9 100644 --- a/src/backend/datasources/filters/BinaryFilterPrivate.h +++ b/src/backend/datasources/filters/BinaryFilterPrivate.h @@ -1,69 +1,69 @@ /*************************************************************************** File : BinaryFilterPrivate.h Project : LabPlot Description : Private implementation class for BinaryFilter. -------------------------------------------------------------------- Copyright : (C) 2015-2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef BINARYFILTERPRIVATE_H #define BINARYFILTERPRIVATE_H #include class AbstractDataSource; class AbstractColumn; class BinaryFilterPrivate { public: explicit BinaryFilterPrivate(BinaryFilter*); int prepareStreamToRead(QDataStream&); void readDataFromDevice(QIODevice& device, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); const BinaryFilter* q; size_t vectors; BinaryFilter::DataType dataType; BinaryFilter::ByteOrder byteOrder; QVector columnModes; int startRow; // start row (value*vectors) to read (can be -1) int endRow; // end row to (value*vectors) read (can be -1) size_t numRows; // number of rows size_t skipStartBytes; // bytes to skip at start size_t skipBytes; // bytes to skip after each value bool createIndexEnabled; // if create index column bool autoModeEnabled; private: int m_actualRows; int m_actualCols; }; #endif diff --git a/src/backend/datasources/filters/FITSFilter.cpp b/src/backend/datasources/filters/FITSFilter.cpp index 6a799f5fc..dd4bdc262 100644 --- a/src/backend/datasources/filters/FITSFilter.cpp +++ b/src/backend/datasources/filters/FITSFilter.cpp @@ -1,1638 +1,1637 @@ /*************************************************************************** File : FITSFilter.cpp Project : LabPlot Description : FITS I/O-filter -------------------------------------------------------------------- Copyright : (C) 2016 by Fabian Kristof (fkristofszabolcs@gmail.com) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "FITSFilter.h" #include "FITSFilterPrivate.h" #include "backend/core/column/Column.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/datatypes/Double2StringFilter.h" #include "backend/matrix/MatrixModel.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/matrix/Matrix.h" #include "commonfrontend/matrix/MatrixView.h" #include #include #include /*! \class FITSFilter * \brief Manages the import/export of data from/to a FITS file. * \since 2.2.0 * \ingroup datasources */ FITSFilter::FITSFilter():AbstractFileFilter(), d(new FITSFilterPrivate(this)) {} FITSFilter::~FITSFilter() {} -QVector FITSFilter::readDataFromFile(const QString &fileName, AbstractDataSource *dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - Q_UNUSED(lines); +QVector FITSFilter::readDataFromFile(const QString &fileName, AbstractDataSource *dataSource, AbstractFileFilter::ImportMode importMode) { return d->readCHDU(fileName, dataSource, importMode); } QVector FITSFilter::readChdu(const QString &fileName, bool* okToMatrix, int lines) { return d->readCHDU(fileName, NULL, AbstractFileFilter::Replace, okToMatrix, lines); } void FITSFilter::write(const QString &fileName, AbstractDataSource *dataSource) { d->writeCHDU(fileName, dataSource); } void FITSFilter::addNewKeyword(const QString &filename, const QList &keywords) { d->addNewKeyword(filename, keywords); } void FITSFilter::updateKeywords(const QString &fileName, const QList& originals, const QVector& updates) { d->updateKeywords(fileName, originals, updates); } void FITSFilter::deleteKeyword(const QString &fileName, const QList& keywords) { d->deleteKeyword(fileName, keywords); } void FITSFilter::addKeywordUnit(const QString &fileName, const QList &keywords) { d->addKeywordUnit(fileName, keywords); } void FITSFilter::removeExtensions(const QStringList &extensions) { d->removeExtensions(extensions); } void FITSFilter::parseHeader(const QString &fileName, QTableWidget *headerEditTable, bool readKeys, const QList &keys) { d->parseHeader(fileName, headerEditTable, readKeys, keys); } void FITSFilter::parseExtensions(const QString &fileName, QTreeWidget *tw, bool checkPrimary) { d->parseExtensions(fileName, tw, checkPrimary); } QList FITSFilter::chduKeywords(const QString &fileName) { return d->chduKeywords(fileName); } void FITSFilter::loadFilterSettings(const QString& fileName) { Q_UNUSED(fileName) } void FITSFilter::saveFilterSettings(const QString& fileName) const { Q_UNUSED(fileName) } /*! * \brief contains the {StandardKeywords \ MandatoryKeywords} keywords * \return A list of keywords */ QStringList FITSFilter::standardKeywords() { return QStringList() << QLatin1String("(blank)") << QLatin1String("CROTA") << QLatin1String("EQUINOX") << QLatin1String("NAXIS") << QLatin1String("TBCOL") << QLatin1String("TUNIT") << QLatin1String("AUTHOR") << QLatin1String("CRPIX") << QLatin1String("EXTEND") << QLatin1String("OBJECT") << QLatin1String("TDIM") << QLatin1String("TZERO") << QLatin1String("BITPIX") << QLatin1String("CRVAL") << QLatin1String("EXTLEVEL") << QLatin1String("OBSERVER") << QLatin1String("TDISP") << QLatin1String("XTENSION") << QLatin1String("BLANK") << QLatin1String("CTYPE") << QLatin1String("EXTNAME") << QLatin1String("ORIGIN") << QLatin1String("TELESCOP") << QLatin1String("BLOCKED") << QLatin1String("DATAMAX") << QLatin1String("EXTVER") << QLatin1String("BSCALE") << QLatin1String("DATAMIN") << QLatin1String("PSCAL") << QLatin1String("TFORM") << QLatin1String("BUNIT") << QLatin1String("DATE") << QLatin1String("GROUPS") << QLatin1String("PTYPE") << QLatin1String("THEAP") << QLatin1String("BZERO") << QLatin1String("DATE-OBS") << QLatin1String("HISTORY") << QLatin1String("PZERO") << QLatin1String("TNULL") << QLatin1String("CDELT") << QLatin1String("INSTRUME") << QLatin1String("REFERENC") << QLatin1String("TSCAL") << QLatin1String("COMMENT") << QLatin1String("EPOCH") << QLatin1String("NAXIS") << QLatin1String("SIMPLE") << QLatin1String("TTYPE"); } /*! * \brief Returns a list of keywords, that are mandatory for an image extension of a FITS file * see: * https://archive.stsci.edu/fits/fits_standard/node64.html * \return A list of keywords */ QStringList FITSFilter::mandatoryImageExtensionKeywords() { return QStringList() << QLatin1String("XTENSION") << QLatin1String("BITPIX") << QLatin1String("NAXIS") << QLatin1String("PCOUNT") << QLatin1String("GCOUNT") << QLatin1String("END"); } /*! * \brief Returns a list of keywords, that are mandatory for a table extension (ascii or bintable) * of a FITS file * see: * https://archive.stsci.edu/fits/fits_standard/node58.html * https://archive.stsci.edu/fits/fits_standard/node68.html * \return A list of keywords */ QStringList FITSFilter::mandatoryTableExtensionKeywords() { return QStringList() << QLatin1String("XTENSION") << QLatin1String("BITPIX") << QLatin1String("NAXIS") << QLatin1String("NAXIS1") << QLatin1String("NAXIS2") << QLatin1String("PCOUNT") << QLatin1String("GCOUNT") << QLatin1String("TFIELDS") << QLatin1String("END"); } /*! * \brief Returns a list of strings that represent units which are used for autocompletion when adding * keyword units to keywords * \return A list of strings that represent units */ QStringList FITSFilter::units() { return QStringList() << QLatin1String("m (Metre)") << QLatin1String("kg (Kilogram)") << QLatin1String("s (Second)") << QString("M☉ (Solar mass)") << QLatin1String("AU (Astronomical unit") << QLatin1String("l.y (Light year)") << QLatin1String("km (Kilometres") << QLatin1String("pc (Parsec)") << QLatin1String("K (Kelvin)") << QLatin1String("mol (Mole)") << QLatin1String("cd (Candela)"); } /*! * \brief Sets the startColumn to \a column * \param column the column to be set */ void FITSFilter::setStartColumn(const int column) { d->startColumn = column; } /*! * \brief Returns startColumn * \return The startColumn */ int FITSFilter::startColumn() const { return d->startColumn; } /*! * \brief Sets the endColumn to \a column * \param column the column to be set */ void FITSFilter::setEndColumn(const int column) { d->endColumn = column; } /*! * \brief Returns endColumn * \return The endColumn */ int FITSFilter::endColumn() const { return d->endColumn; } /*! * \brief Sets the startRow to \a row * \param row the row to be set */ void FITSFilter::setStartRow(const int row) { d->startRow = row; } /*! * \brief Returns startRow * \return The startRow */ int FITSFilter::startRow() const { return d->startRow; } /*! * \brief Sets the endRow to \a row * \param row the row to be set */ void FITSFilter::setEndRow(const int row) { d->endRow = row; } /*! * \brief Returns endRow * \return The endRow */ int FITSFilter::endRow() const { return d->endRow; } /*! * \brief Sets commentsAsUnits to \a commentsAsUnits * * This is used when spreadsheets are exported to FITS table extensions and comments are used as the * units of the table's columns. * \param commentsAsUnits */ void FITSFilter::setCommentsAsUnits(const bool commentsAsUnits) { d->commentsAsUnits = commentsAsUnits; } /*! * \brief Sets exportTo to \a exportTo * * This is used to decide whether the container should be exported to a FITS image or a FITS table * For an image \a exportTo should be 0, for a table 1 * \param exportTo */ void FITSFilter::setExportTo(const int exportTo) { d->exportTo = exportTo; } QString FITSFilter::fileInfoString(const QString& fileName) { const int imagesCount = FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("IMAGES")).size(); QString info(i18n("Images: %1", QString::number(imagesCount ))); info += QLatin1String("
"); const int tablesCount = FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("TABLES")).size(); info += i18n("Tables: %1", QString::number(tablesCount)); return info; } //##################################################################### //################### Private implementation ########################## //##################################################################### FITSFilterPrivate::FITSFilterPrivate(FITSFilter* owner) : q(owner), startRow(-1), endRow(-1), startColumn(-1), endColumn(-1), commentsAsUnits(false), exportTo(0) { #ifdef HAVE_FITS m_fitsFile = 0; #endif } /*! * \brief Read the current header data unit from file \a filename in data source \a dataSource in \a importMode import mode * \param fileName the name of the file to be read * \param dataSource the data source to be filled * \param importMode */ QVector FITSFilterPrivate::readCHDU(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, bool* okToMatrix, int lines) { DEBUG("FITSFilterPrivate::readCHDU() file name = " << fileName.toStdString()); QVector dataStrings; #ifdef HAVE_FITS int status = 0; if(fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status)) { DEBUG(" ERROR opening file " << fileName.toStdString()); printError(status); return dataStrings; } int chduType; if (fits_get_hdu_type(m_fitsFile, &chduType, &status)) { printError(status); return dataStrings; } long actualRows; int actualCols; int columnOffset = 0; bool noDataSource = (dataSource == NULL); if(chduType == IMAGE_HDU) { DEBUG("IMAGE_HDU"); int maxdim = 2; int bitpix; int naxis; long naxes[2]; if (fits_get_img_param(m_fitsFile, maxdim, &bitpix, &naxis, naxes, &status)) { printError(status); return dataStrings; } if (naxis == 0) return dataStrings; actualRows = naxes[1]; actualCols = naxes[0]; if (lines == -1) lines = actualRows; else { if (lines > actualRows) lines = actualRows; } if (endRow != -1) { if (!noDataSource) lines = endRow; } if (endColumn != -1) actualCols = endColumn; if (noDataSource) dataStrings.reserve(lines); int i = 0; int j = 0; if (startRow != 1) i = startRow; if (startColumn != 1) j = startColumn; const int jstart = j; //TODO: support other modes QVector columnModes; columnModes.resize(actualCols - j); QStringList vectorNames; QVector dataContainer; if (!noDataSource) { dataContainer.reserve(actualCols - j); columnOffset = dataSource->prepareImport(dataContainer, importMode, lines - i, actualCols - j, vectorNames, columnModes); } long pixelCount = lines * actualCols; double* data = new double[pixelCount]; if (!data) { qDebug() << "Not enough memory for data"; return dataStrings; } if (fits_read_img(m_fitsFile, TDOUBLE, 1, pixelCount, NULL, data, NULL, &status)) { printError(status); return dataStrings << (QStringList() << QString("Error")); } int ii = 0; DEBUG(" Import " << lines << " lines"); for (; i < lines; ++i) { int jj = 0; QStringList line; line.reserve(actualCols - j); for (; j < actualCols; ++j) { if (noDataSource) line << QString::number(data[i*naxes[0] +j]); else static_cast*>(dataContainer[jj++])->operator[](ii) = data[i* naxes[0] + j]; } dataStrings << line; j = jstart; ii++; } delete[] data; if (dataSource) dataSource->finalizeImport(columnOffset, 1, actualCols, "", importMode); fits_close_file(m_fitsFile, &status); return dataStrings; } else if ((chduType == ASCII_TBL) || (chduType == BINARY_TBL)) { DEBUG("ASCII_TBL or BINARY_TBL"); if (endColumn != -1) actualCols = endColumn; else fits_get_num_cols(m_fitsFile, &actualCols, &status); if (endRow != -1) actualRows = endRow; else fits_get_num_rows(m_fitsFile, &actualRows, &status); QStringList columnNames; QList columnsWidth; QStringList columnUnits; columnUnits.reserve(actualCols); columnsWidth.reserve(actualCols); columnNames.reserve(actualCols); int colWidth; char keyword[FLEN_KEYWORD]; char value[FLEN_VALUE]; int col = 1; if (startColumn != 1) { if (startColumn != 0) col = startColumn; } for (; col <=actualCols; ++col) { status = 0; fits_make_keyn("TTYPE", col, keyword, &status); fits_read_key(m_fitsFile, TSTRING, keyword, value, NULL, &status); columnNames.append(QLatin1String(value)); fits_make_keyn("TUNIT", col, keyword, &status); fits_read_key(m_fitsFile, TSTRING, keyword, value, NULL, &status); columnUnits.append(QLatin1String(value)); fits_get_col_display_width(m_fitsFile, col, &colWidth, &status); columnsWidth.append(colWidth); } status = 0; if (lines == -1) lines = actualRows; else if (lines > actualRows) lines = actualRows; if (endRow != -1) lines = endRow; QVector stringDataPointers; QVector numericDataPointers; QList columnNumericTypes; int startCol = 0; if (startColumn != 1) startCol = startColumn; int startRrow = 0; if (startRow != 1) startRrow = startRow; columnNumericTypes.reserve(actualCols); int datatype; int c = 1; if (startColumn != 1) { if (startColumn != 0) c = startColumn; } QList matrixNumericColumnIndices; for (; c <= actualCols; ++c) { fits_get_coltype(m_fitsFile, c, &datatype, NULL, NULL, &status); switch (datatype) { case TSTRING: columnNumericTypes.append(false); break; case TSHORT: columnNumericTypes.append(true); break; case TLONG: columnNumericTypes.append(true); break; case TFLOAT: columnNumericTypes.append(true); break; case TDOUBLE: columnNumericTypes.append(true); break; case TLOGICAL: columnNumericTypes.append(false); break; case TBIT: columnNumericTypes.append(true); break; case TBYTE: columnNumericTypes.append(true); break; case TCOMPLEX: columnNumericTypes.append(true); break; default: columnNumericTypes.append(false); break; } if ((datatype != TSTRING) && (datatype != TLOGICAL)) matrixNumericColumnIndices.append(c); } if (noDataSource) *okToMatrix = matrixNumericColumnIndices.isEmpty() ? false : true; if (!noDataSource) { DEBUG("HAS DataSource"); Spreadsheet* spreadsheet = dynamic_cast(dataSource); if(spreadsheet) { numericDataPointers.reserve(actualCols - startCol); stringDataPointers.reserve(actualCols - startCol); spreadsheet->setUndoAware(false); columnOffset = spreadsheet->resize(importMode, columnNames, actualCols - startCol); if (importMode == AbstractFileFilter::Replace) { spreadsheet->clear(); spreadsheet->setRowCount(lines - startRrow); } else { if (spreadsheet->rowCount() < (lines - startRrow)) spreadsheet->setRowCount(lines - startRrow); } DEBUG("Reading columns ..."); for (int n = 0; n < actualCols - startCol; ++n) { if (columnNumericTypes.at(n)) { spreadsheet->column(columnOffset+ n)->setColumnMode(AbstractColumn::Numeric); QVector* datap = static_cast* >(spreadsheet->column(columnOffset+n)->data()); numericDataPointers.push_back(datap); if (importMode == AbstractFileFilter::Replace) datap->clear(); } else { spreadsheet->column(columnOffset+ n)->setColumnMode(AbstractColumn::Text); QStringList* list = static_cast(spreadsheet->column(columnOffset+n)->data()); stringDataPointers.push_back(list); if (importMode == AbstractFileFilter::Replace) list->clear(); } } DEBUG(" ... DONE"); stringDataPointers.squeeze(); } else { numericDataPointers.reserve(matrixNumericColumnIndices.size()); columnOffset = dataSource->prepareImport(numericDataPointers, importMode, lines - startRrow, matrixNumericColumnIndices.size()); } numericDataPointers.squeeze(); } char* array = new char[1000]; //TODO: why 1000? int row = 1; if (startRow != 1) { if (startRow != 0) row = startRow; } int coll = 1; if (startColumn != 1) { if (startColumn != 0) coll = startColumn; } bool isMatrix = false; if (dynamic_cast(dataSource)) { isMatrix = true; coll = matrixNumericColumnIndices.first(); actualCols = matrixNumericColumnIndices.last(); if (importMode == AbstractFileFilter::Replace) { for (auto* col: numericDataPointers) static_cast*>(col)->clear(); } } for (; row <= lines; ++row) { int numericixd = 0; int stringidx = 0; QStringList line; line.reserve(actualCols-coll); for (int col = coll; col <= actualCols; ++col) { if (isMatrix) { if (!matrixNumericColumnIndices.contains(col)) continue; } if(fits_read_col_str(m_fitsFile, col, row, 1, 1, NULL, &array, NULL, &status)) printError(status); if (!noDataSource) { const QString& str = QString::fromLatin1(array); if (str.isEmpty()) { if (columnNumericTypes.at(col - 1)) static_cast*>(numericDataPointers[numericixd++])->push_back(0); else stringDataPointers[stringidx++]->append(QLatin1String("NULL")); } else { if (columnNumericTypes.at(col - 1)) static_cast*>(numericDataPointers[numericixd++])->push_back(str.toDouble()); else { if (!stringDataPointers.isEmpty()) stringDataPointers[stringidx++]->operator<<(str.simplified()); } } } else { QString tmpColstr = QString::fromLatin1(array); tmpColstr = tmpColstr.simplified(); if (tmpColstr.isEmpty()) line << QLatin1String("NULL"); else line << tmpColstr; } } dataStrings << line; } delete[] array; if (!noDataSource) dataSource->finalizeImport(columnOffset, 1, actualCols, "", importMode); fits_close_file(m_fitsFile, &status); return dataStrings; } else qDebug() << "Incorrect header type"; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(importMode) Q_UNUSED(okToMatrix) Q_UNUSED(lines) #endif return dataStrings; } /*! * \brief Export from data source \a dataSource to file \a fileName * \param fileName the name of the file to be exported to * \param dataSource the datasource whose data is exported */ void FITSFilterPrivate::writeCHDU(const QString &fileName, AbstractDataSource *dataSource) { #ifdef HAVE_FITS if (!fileName.endsWith(QLatin1String(".fits"))) return; int status = 0; bool existed = false; if (!QFile::exists(fileName)) { if (fits_create_file(&m_fitsFile, fileName.toLatin1(), &status)) { printError(status); qDebug() << fileName; return; } } else { if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } else existed = true; } Matrix* const matrix = dynamic_cast(dataSource); if (matrix) { //FITS image if (exportTo == 0) { long naxes[2] = { matrix->columnCount(), matrix->rowCount() }; if (fits_create_img(m_fitsFile, FLOAT_IMG, 2, naxes, &status)) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); return; } const long nelem = naxes[0] * naxes[1]; double* const array = new double[nelem]; const QVector >* const data = static_cast>*>(matrix->data()); for (int col = 0; col < naxes[0]; ++col) for (int row = 0; row < naxes[1]; ++row) array[row * naxes[0] + col] = data->at(row).at(col); if (fits_write_img(m_fitsFile, TDOUBLE, 1, nelem, array, &status )) { printError(status); status = 0; } fits_close_file(m_fitsFile, &status); delete[] array; //FITS table } else { const int nrows = matrix->rowCount(); const int tfields = matrix->columnCount(); QVector columnNames; columnNames.resize(tfields); columnNames.reserve(tfields); QVector tform; tform.resize(tfields); tform.reserve(tfields); QVector tunit; tunit.resize(tfields); tunit.reserve(tfields); //TODO: mode const QVector>* const matrixData = static_cast>*>(matrix->data()); QVector column; const MatrixModel* matrixModel = static_cast(matrix->view())->model(); const int precision = matrix->precision(); for (int i = 0; i < tfields; ++i) { column = matrixData->at(i); const QString& columnName = matrixModel->headerData(i, Qt::Horizontal).toString(); columnNames[i] = new char[columnName.size()]; strcpy(columnNames[i], columnName.toLatin1().data()); tunit[i] = new char[1]; strcpy(tunit[i], ""); int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (matrix->text(row, i).size() > maxSize) maxSize = matrix->text(row, i).size(); } QString tformn; if (precision > 0) { tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".") + QString::number(precision); } else tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".0"); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); } //TODO extension name containing[] ? if (fits_create_tbl(m_fitsFile, ASCII_TBL, nrows, tfields, columnNames.data(), tform.data(), tunit.data(), matrix->name().toLatin1().data(),&status )) { printError(status); qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); double* columnNumeric = new double[nrows]; for (int col = 1; col <= tfields; ++col) { column = matrixData->at(col-1); for (int r = 0; r < column.size(); ++r) columnNumeric[r] = column.at(r); fits_write_col(m_fitsFile, TDOUBLE, col, 1, 1, nrows, columnNumeric, &status); if (status) { printError(status); delete[] columnNumeric; status = 0; if (!existed) { QFile file(fileName); file.remove(); } fits_close_file(m_fitsFile, &status); return; } } delete[] columnNumeric; fits_close_file(m_fitsFile, &status); } return; } Spreadsheet* const spreadsheet = dynamic_cast(dataSource); if (spreadsheet) { //FITS image if (exportTo == 0) { int maxRowIdx = -1; //don't export lots of empty lines if all of those contain nans // TODO: option? for (int c = 0; c < spreadsheet->columnCount(); ++c) { const Column* const col = spreadsheet->column(c); int currMaxRoxIdx = -1; for (int r = col->rowCount(); r >= 0; --r) { if (col->isValid(r)) { currMaxRoxIdx = r; break; } } if (currMaxRoxIdx > maxRowIdx) { maxRowIdx = currMaxRoxIdx; } } long naxes[2] = { spreadsheet->columnCount(), maxRowIdx + 1}; if (fits_create_img(m_fitsFile, FLOAT_IMG, 2, naxes, &status)) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } const long nelem = naxes[0] * naxes[1]; double* array = new double[nelem]; for (int row = 0; row < naxes[1]; ++row) { for (int col = 0; col < naxes[0]; ++col) array[row * naxes[0] + col] = spreadsheet->column(col)->valueAt(row); } if (fits_write_img(m_fitsFile, TDOUBLE, 1, nelem, array, &status )) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } fits_close_file(m_fitsFile, &status); delete[] array; } else { const int nrows = spreadsheet->rowCount(); const int tfields = spreadsheet->columnCount(); QVector columnNames; columnNames.resize(tfields); columnNames.reserve(tfields); QVector tform; tform.resize(tfields); tform.reserve(tfields); QVector tunit; tunit.resize(tfields); tunit.reserve(tfields); for (int i = 0; i < tfields; ++i) { const Column* const column = spreadsheet->column(i); columnNames[i] = new char[column->name().size()]; strcpy(columnNames[i], column->name().toLatin1().data()); if (commentsAsUnits) { tunit[i] = new char[column->comment().size()]; strcpy(tunit[i], column->comment().toLatin1().constData()); } else { tunit[i] = new char[2]; strcpy(tunit[i], ""); } switch (column->columnMode()) { case AbstractColumn::Numeric: { int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (QString::number(column->valueAt(row)).size() > maxSize) maxSize = QString::number(column->valueAt(row)).size(); } const Double2StringFilter* const filter = static_cast(column->outputFilter()); bool decimals = false; for (int ii = 0; ii < nrows; ++ii) { bool ok; QString cell = column->asStringColumn()->textAt(ii); double val = cell.toDouble(&ok); if (cell.size() > QString::number(val).size() + 1) { decimals = true; break; } } QString tformn; if (decimals) { int maxStringSize = -1; for (int row = 0; row < nrows; ++row) { if (column->asStringColumn()->textAt(row).size() > maxStringSize) maxStringSize = column->asStringColumn()->textAt(row).size(); } const int diff = abs(maxSize - maxStringSize); maxSize+= diff; tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".") + QString::number(filter->numDigits()); } else tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".0"); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); break; } case AbstractColumn::Text: { int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (column->textAt(row).size() > maxSize) maxSize = column->textAt(row).size(); } const QString& tformn = QLatin1String("A") + QString::number(maxSize); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); break; } case AbstractColumn::Integer: //TODO case AbstractColumn::DateTime: case AbstractColumn::Day: case AbstractColumn::Month: break; } } //TODO extension name containing[] ? if (fits_create_tbl(m_fitsFile, ASCII_TBL, nrows, tfields, columnNames.data(), tform.data(), tunit.data(), spreadsheet->name().toLatin1().data(),&status )) { printError(status); qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); QVector column; column.resize(nrows); column.reserve(nrows); double* columnNumeric = new double[nrows]; bool hadTextColumn = false; for (int col = 1; col <= tfields; ++col) { const Column* c = spreadsheet->column(col-1); AbstractColumn::ColumnMode columnMode = c->columnMode(); if (columnMode == AbstractColumn::Numeric) { for (int row = 0; row < nrows; ++row) columnNumeric[row] = c->valueAt(row); fits_write_col(m_fitsFile, TDOUBLE, col, 1, 1, nrows, columnNumeric, &status); if (status) { printError(status); delete[] columnNumeric; status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } } else { hadTextColumn = true; for (int row = 0; row < nrows; ++row) { column[row] = new char[c->textAt(row).size()]; strcpy(column[row], c->textAt(row).toLatin1().data()); } fits_write_col(m_fitsFile, TSTRING, col, 1, 1, nrows, column.data(), &status); if (status) { printError(status); qDeleteAll(column); status = 0; fits_close_file(m_fitsFile, &status); return; } } } delete[] columnNumeric; if (hadTextColumn) qDeleteAll(column); status = 0; fits_close_file(m_fitsFile, &status); } } #else Q_UNUSED(fileName) Q_UNUSED(dataSource) #endif } /*! * \brief Return a map of the available extensions names in file \a filename * The keys of the map are the extension types, the values are the names * \param fileName the name of the FITS file to be analyzed */ QMultiMap FITSFilterPrivate::extensionNames(const QString& fileName) { DEBUG("FITSFilterPrivate::extensionNames() file name = " << fileName.toStdString()); #ifdef HAVE_FITS QMultiMap extensions; int status = 0; fitsfile* fitsFile = 0; if (fits_open_file(&fitsFile, fileName.toLatin1(), READONLY, &status )) return QMultiMap(); int hduCount; if (fits_get_num_hdus(fitsFile, &hduCount, &status)) return QMultiMap(); int imageCount = 0; int asciiTableCount = 0; int binaryTableCount = 0; for (int currentHDU = 1; (currentHDU <= hduCount) && !status; ++currentHDU) { int hduType; status = 0; fits_get_hdu_type(fitsFile, &hduType, &status); switch (hduType) { case IMAGE_HDU: imageCount++; break; case ASCII_TBL: asciiTableCount++; break; case BINARY_TBL: binaryTableCount++; break; } char* keyVal = new char[FLEN_VALUE]; QString extName; if (!fits_read_keyword(fitsFile,"EXTNAME", keyVal, NULL, &status)) { extName = QLatin1String(keyVal); extName = extName.mid(1, extName.length() -2).simplified(); } else { status = 0; if (!fits_read_keyword(fitsFile,"HDUNAME", keyVal, NULL, &status)) { extName = QLatin1String(keyVal); extName = extName.mid(1, extName.length() -2).simplified(); } else { status = 0; switch (hduType) { case IMAGE_HDU: if (imageCount == 1) extName = i18n("Primary header"); else extName = i18n("IMAGE #%1", imageCount); break; case ASCII_TBL: extName = i18n("ASCII_TBL #%1", asciiTableCount); break; case BINARY_TBL: extName = i18n("BINARY_TBL #%1", binaryTableCount); break; } } } delete[] keyVal; status = 0; extName = extName.trimmed(); switch (hduType) { case IMAGE_HDU: extensions.insert(QLatin1String("IMAGES"), extName); break; case ASCII_TBL: extensions.insert(QLatin1String("TABLES"), extName); break; case BINARY_TBL: extensions.insert(QLatin1String("TABLES"), extName); break; } fits_movrel_hdu(fitsFile, 1, NULL, &status); } if (status == END_OF_FILE) status = 0; fits_close_file(fitsFile, &status); return extensions; #else Q_UNUSED(fileName) return QMultiMap(); #endif } /*! * \brief Prints the error text corresponding to the status code \a status * \param status the status code of the error */ void FITSFilterPrivate::printError(int status) const { #ifdef HAVE_FITS if (status) { char errorText[FLEN_ERRMSG]; fits_get_errstatus(status, errorText ); qDebug() << QLatin1String(errorText); } #else Q_UNUSED(status) #endif } /*! * \brief Add the keywords \a keywords to the current header unit * \param keywords the keywords to be added * \param fileName the name of the FITS file (extension) in which the keywords are added */ void FITSFilterPrivate::addNewKeyword(const QString& fileName, const QList& keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const FITSFilter::Keyword& keyword: keywords) { status = 0; if (!keyword.key.compare(QLatin1String("COMMENT"))) { if (fits_write_comment(m_fitsFile, keyword.value.toLatin1(), &status)) printError(status); } else if (!keyword.key.compare(QLatin1String("HISTORY"))) { if (fits_write_history(m_fitsFile, keyword.value.toLatin1(), &status)) printError(status); } else if (!keyword.key.compare(QLatin1String("DATE"))) { if (fits_write_date(m_fitsFile, &status)) printError(status); } else { int ok = 0; if (keyword.key.length() <= FLEN_KEYWORD) { ok++; if (keyword.value.length() <= FLEN_VALUE) { ok++; if(keyword.comment.length() <= FLEN_COMMENT) ok++; } } if (ok == 3) { bool ok; double val = keyword.value.toDouble(&ok); if (ok) { if (fits_write_key(m_fitsFile, TDOUBLE, keyword.key.toLatin1().data(), &val, keyword.comment.toLatin1().data(), &status)) printError(status); } else { if (fits_write_key(m_fitsFile, TSTRING, keyword.key.toLatin1().data(), keyword.value.toLatin1().data(), keyword.comment.toLatin1().data(), &status)) printError(status); } } else if ( ok == 2) { //comment too long } else if ( ok == 1) { //value too long } else { //keyword too long } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(keywords) Q_UNUSED(fileName) #endif } /*! * \brief Update keywords in the current header unit * \param fileName The name of the FITS file (extension) in which the keywords will be updated * \param originals The original keywords of the FITS file (extension) * \param updates The keywords that contain the updated values */ void FITSFilterPrivate::updateKeywords(const QString& fileName, const QList& originals, const QVector& updates) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } FITSFilter::Keyword updatedKeyword; FITSFilter::Keyword originalKeyword; FITSFilter::KeywordUpdate keywordUpdate; for (int i = 0; i < updates.size(); ++i) { updatedKeyword = updates.at(i); originalKeyword = originals.at(i); keywordUpdate = originals.at(i).updates; if (keywordUpdate.keyUpdated && keywordUpdate.valueUpdated && keywordUpdate.commentUpdated) { if (updatedKeyword.isEmpty()) { if (fits_delete_key(m_fitsFile, originalKeyword.key.toLatin1(), &status)) { printError(status); status = 0; } continue; } } if (!updatedKeyword.key.isEmpty()) { if (fits_modify_name(m_fitsFile, originalKeyword.key.toLatin1(), updatedKeyword.key.toLatin1(), &status )) { printError(status); status = 0; } } if (!updatedKeyword.value.isEmpty()) { bool ok; int intValue; double doubleValue; bool updated = false; doubleValue = updatedKeyword.value.toDouble(&ok); if (ok) { if (fits_update_key(m_fitsFile,TDOUBLE, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), &doubleValue, NULL, &status)) printError(status); else updated = true; } if (!updated) { intValue = updatedKeyword.value.toInt(&ok); if (ok) { if (fits_update_key(m_fitsFile,TINT, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), &intValue, NULL, &status)) printError(status); else updated = true; } } if (!updated) { if (fits_update_key(m_fitsFile,TSTRING, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), updatedKeyword.value.toLatin1().data(), NULL, &status)) printError(status); } } else { if (keywordUpdate.valueUpdated) { if (fits_update_key_null(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), NULL, &status)) { printError(status); status = 0; } } } if (!updatedKeyword.comment.isEmpty()) { if (fits_modify_comment(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), updatedKeyword.comment.toLatin1().data(), &status)) { printError(status); status = 0; } } else { if (keywordUpdate.commentUpdated) { if (fits_modify_comment(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), QString("").toLatin1().data(), &status)) { printError(status); status = 0; } } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(originals) Q_UNUSED(updates) #endif } /*! * \brief Delete the keywords \a keywords from the current header unit * \param fileName the name of the FITS file (extension) in which the keywords will be deleted. * \param keywords the keywords to deleted */ void FITSFilterPrivate::deleteKeyword(const QString& fileName, const QList &keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const auto& keyword : keywords) { if (!keyword.key.isEmpty()) { status = 0; if (fits_delete_key(m_fitsFile, keyword.key.toLatin1(), &status)) printError(status); } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(keywords) Q_UNUSED(fileName) #endif } /*! * \brief FITSFilterPrivate::addKeywordUnit * \param fileName the FITS file (extension) in which the keyword units are updated/added * \param keywords the keywords whose units were modified/added */ void FITSFilterPrivate::addKeywordUnit(const QString &fileName, const QList &keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for(const FITSFilter::Keyword& keyword : keywords) { if (keyword.updates.unitUpdated) { if (fits_write_key_unit(m_fitsFile, keyword.key.toLatin1(), keyword.unit.toLatin1().data(), &status)) { printError(status); status = 0; } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(keywords) #endif } /*! * \brief Remove extensions from a FITS file * \param extensions The extensions to be removed from the FITS file */ void FITSFilterPrivate::removeExtensions(const QStringList &extensions) { #ifdef HAVE_FITS int status = 0; for (const auto& ext : extensions) { status = 0; if (fits_open_file(&m_fitsFile, ext.toLatin1(), READWRITE, &status )) { printError(status); continue; } if (fits_delete_hdu(m_fitsFile, NULL, &status)) printError(status); status = 0; fits_close_file(m_fitsFile, &status); } #else Q_UNUSED(extensions) #endif } /*! * \brief Returns a list of keywords in the current header of \a fileName * \param fileName the name of the FITS file (extension) to be opened * \return A list of keywords */ QList FITSFilterPrivate::chduKeywords(const QString& fileName) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status )) { printError(status); return QList (); } int numberOfKeys; if (fits_get_hdrspace(m_fitsFile, &numberOfKeys, NULL, &status)) { printError(status); return QList (); } QList keywords; keywords.reserve(numberOfKeys); char* key = new char[FLEN_KEYWORD]; char* value = new char[FLEN_VALUE]; char* comment = new char[FLEN_COMMENT]; char* unit = new char[FLEN_VALUE]; for (int i = 1; i <= numberOfKeys; ++i) { QStringList recordValues; FITSFilter::Keyword keyword; if (fits_read_keyn(m_fitsFile, i, key, value, comment, &status)) { printError(status); status = 0; continue; } fits_read_key_unit(m_fitsFile, key, unit, &status); recordValues << QLatin1String(key) << QLatin1String(value) << QLatin1String(comment) << QLatin1String(unit); keyword.key = recordValues[0].simplified(); keyword.value = recordValues[1].simplified(); keyword.comment = recordValues[2].simplified(); keyword.unit = recordValues[3].simplified(); keywords.append(keyword); } delete[] key; delete[] value; delete[] comment; delete[] unit; fits_close_file(m_fitsFile, &status); return keywords; #else Q_UNUSED(fileName) return QList(); #endif } /*! * \brief Builds the table \a headerEditTable from the keywords \a keys * \param fileName The name of the FITS file from which the keys are read if \a readKeys is \c true * \param headerEditTable The table to be built * \param readKeys It's used to determine whether the keywords are provided or they should be read from * file \a fileName * \param keys The keywords that are provided if the keywords were read already. */ void FITSFilterPrivate::parseHeader(const QString &fileName, QTableWidget *headerEditTable, bool readKeys, const QList& keys) { #ifdef HAVE_FITS QList keywords; if (readKeys) keywords = chduKeywords(fileName); else keywords = keys; headerEditTable->setRowCount(keywords.size()); QTableWidgetItem* item; for (int i = 0; i < keywords.size(); ++i) { const FITSFilter::Keyword& keyword = keywords.at(i); const bool mandatory = FITSFilter::mandatoryImageExtensionKeywords().contains(keyword.key) || FITSFilter::mandatoryTableExtensionKeywords().contains(keyword.key); item = new QTableWidgetItem(keyword.key); const QString& itemText = item->text(); const bool notEditableKey = mandatory || itemText.contains(QLatin1String("TFORM")) || itemText.contains(QLatin1String("TTYPE")) || itemText.contains(QLatin1String("TUNIT")) || itemText.contains(QLatin1String("TDISP")) || itemText.contains(QLatin1String("TBCOL")) || itemText.contains(QLatin1String("TZERO")); const bool notEditableValue= mandatory || itemText.contains(QLatin1String("TFORM")) || itemText.contains(QLatin1String("TDISP")) || itemText.contains(QLatin1String("TBCOL")) || itemText.contains(QLatin1String("TZERO")); if (notEditableKey) item->setFlags(item->flags() & ~Qt::ItemIsEditable); else item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 0, item ); item = new QTableWidgetItem(keyword.value); if (notEditableValue) item->setFlags(item->flags() & ~Qt::ItemIsEditable); else item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 1, item ); QString commentFieldText; if (!keyword.unit.isEmpty()) { if (keyword.updates.unitUpdated) { const QString& comment = keyword.comment.right( keyword.comment.size() - keyword.comment.indexOf(QChar(']'))-1); commentFieldText = QLatin1String("[") + keyword.unit + QLatin1String("] ") + comment; } else { if (keyword.comment.at(0) == QLatin1Char('[')) commentFieldText = keyword.comment; else commentFieldText = QLatin1String("[") + keyword.unit + QLatin1String("] ") + keyword.comment; } } else commentFieldText = keyword.comment; item = new QTableWidgetItem(commentFieldText); item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 2, item ); } headerEditTable->resizeColumnsToContents(); #else Q_UNUSED(fileName) Q_UNUSED(headerEditTable) Q_UNUSED(readKeys) Q_UNUSED(keys) #endif } /*! * \brief Helper function to return the value of the key \a key * \param fileName The name of the FITS file (extension) in which the keyword with key \a key should exist * \param key The key of the keyword whose value it's returned * \return The value of the keyword as a string */ const QString FITSFilterPrivate::valueOf(const QString& fileName, const char *key) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status )) { printError(status); return QString (); } char* keyVal = new char[FLEN_VALUE]; QString keyValue; if (!fits_read_keyword(m_fitsFile, key, keyVal, NULL, &status)) { keyValue = QLatin1String(keyVal); keyValue = keyValue.simplified(); } else { printError(status); delete[] keyVal; fits_close_file(m_fitsFile, &status); return QString(); } delete[] keyVal; status = 0; fits_close_file(m_fitsFile, &status); return keyValue; #else Q_UNUSED(fileName) Q_UNUSED(key) return QString(); #endif } /*! * \brief Build the extensions tree from FITS file (extension) \a fileName * \param fileName The name of the FITS file to be opened * \param tw The QTreeWidget to be built * \param checkPrimary Used to determine whether the tree will be used for import or the header edit, * if it's \c true and if the primary array it's empty, then the item won't be added to the tree */ void FITSFilterPrivate::parseExtensions(const QString &fileName, QTreeWidget *tw, bool checkPrimary) { DEBUG("FITSFilterPrivate::parseExtensions()"); #ifdef HAVE_FITS const QMultiMap& extensions = extensionNames(fileName); const QStringList& imageExtensions = extensions.values(QLatin1String("IMAGES")); const QStringList& tableExtensions = extensions.values(QLatin1String("TABLES")); QTreeWidgetItem* root = tw->invisibleRootItem(); //TODO: fileName may contain any data type: check if it's a FITS file QTreeWidgetItem* treeNameItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << fileName); root->addChild(treeNameItem); treeNameItem->setExpanded(true); QTreeWidgetItem* imageExtensionItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << i18n("Images")); imageExtensionItem->setFlags(imageExtensionItem->flags() & ~Qt::ItemIsSelectable ); QString primaryHeaderNaxis = valueOf(fileName, "NAXIS"); const int naxis = primaryHeaderNaxis.toInt(); bool noImage = false; for (const QString& ext : imageExtensions) { QTreeWidgetItem* treeItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << ext); if (ext == i18n("Primary header")) { if (checkPrimary && naxis == 0) continue; } imageExtensionItem->addChild(treeItem); } if (imageExtensionItem->childCount() > 0) { treeNameItem->addChild(imageExtensionItem); imageExtensionItem->setIcon(0,QIcon::fromTheme("view-preview")); imageExtensionItem->setExpanded(true); imageExtensionItem->child(0)->setSelected(true); tw->setCurrentItem(imageExtensionItem->child(0)); } else noImage = true; if (tableExtensions.size() > 0) { QTreeWidgetItem* tableExtensionItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << i18n("Tables")); tableExtensionItem->setFlags(tableExtensionItem->flags() & ~Qt::ItemIsSelectable ); for (const QString& ext : tableExtensions) { QTreeWidgetItem* treeItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << ext); tableExtensionItem->addChild(treeItem); } if (tableExtensionItem->childCount() > 0) { treeNameItem->addChild(tableExtensionItem); tableExtensionItem->setIcon(0,QIcon::fromTheme("x-office-spreadsheet")); tableExtensionItem->setExpanded(true); if (noImage) { tableExtensionItem->child(0)->setSelected(true); tw->setCurrentItem(tableExtensionItem->child(0)); } } } #else Q_UNUSED(fileName) Q_UNUSED(tw) Q_UNUSED(checkPrimary) #endif DEBUG("FITSFilterPrivate::parseExtensions() DONE"); } /*! * \brief FITSFilterPrivate::~FITSFilterPrivate */ FITSFilterPrivate::~FITSFilterPrivate() { } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void FITSFilter::save(QXmlStreamWriter * writer) const { Q_UNUSED(writer) } /*! Loads from XML. */ bool FITSFilter::load(XmlStreamReader * loader) { Q_UNUSED(loader) return false; } diff --git a/src/backend/datasources/filters/FITSFilter.h b/src/backend/datasources/filters/FITSFilter.h index b0c70ff41..8b9bbfa59 100644 --- a/src/backend/datasources/filters/FITSFilter.h +++ b/src/backend/datasources/filters/FITSFilter.h @@ -1,118 +1,118 @@ /*************************************************************************** File : FITSFilter.h Project : LabPlot Description : FITS I/O-filter -------------------------------------------------------------------- Copyright : (C) 2016 by Fabian Kristof (fkristofszabolcs@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef FITSFILTER_H #define FITSFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include #include #include #include #include class FITSFilterPrivate; class FITSHeaderEditWidget; class FITSFilter : public AbstractFileFilter { Q_OBJECT public: FITSFilter(); ~FITSFilter() override; QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace) override; void write(const QString& fileName, AbstractDataSource*) override; QVector readChdu(const QString& fileName, bool *okToMatrix = nullptr, int lines = -1); void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; struct KeywordUpdate { KeywordUpdate() : keyUpdated(false), valueUpdated(false), commentUpdated(false), unitUpdated(false) {} bool keyUpdated; bool valueUpdated; bool commentUpdated; bool unitUpdated; }; struct Keyword { Keyword(const QString& key, const QString& value, const QString& comment): key(key), value(value), comment(comment) {} Keyword() {} QString key; QString value; QString comment; QString unit; bool operator==(const Keyword& other) const { return other.key == key && other.value == value && other.comment == comment; } bool isEmpty() const { return key.isEmpty() && value.isEmpty() && comment.isEmpty(); } KeywordUpdate updates; }; static QString fileInfoString(const QString&); void updateKeywords(const QString& fileName, const QList& originals, const QVector& updates); void addNewKeyword(const QString& filename, const QList& keywords); void addKeywordUnit(const QString& fileName, const QList& keywords); void deleteKeyword(const QString& fileName, const QList& keywords); void removeExtensions(const QStringList& extensions); void parseHeader(const QString &fileName, QTableWidget* headerEditTable, bool readKeys = true, const QList &keys = QList()); void parseExtensions(const QString& fileName, QTreeWidget* tw, bool checkPrimary = false); QList chduKeywords(const QString &fileName); static QStringList standardKeywords(); static QStringList mandatoryImageExtensionKeywords(); static QStringList mandatoryTableExtensionKeywords(); static QStringList units(); void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void setCommentsAsUnits(const bool); void setExportTo(const int); private: std::unique_ptr const d; friend class FITSFilterPrivate; }; #endif // FITSFILTER_H diff --git a/src/backend/datasources/filters/HDF5Filter.cpp b/src/backend/datasources/filters/HDF5Filter.cpp index 826a8582d..6d56fc859 100644 --- a/src/backend/datasources/filters/HDF5Filter.cpp +++ b/src/backend/datasources/filters/HDF5Filter.cpp @@ -1,1717 +1,1716 @@ /*************************************************************************** File : HDF5Filter.cpp Project : LabPlot Description : HDF5 I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ /* TODO: * Feature: implement missing data types and ranks * Performance: only fill dataPointer or dataStrings (not both) */ #include "backend/datasources/filters/HDF5Filter.h" #include "backend/datasources/filters/HDF5FilterPrivate.h" #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include /*! \class HDF5Filter \brief Manages the import/export of data from/to a HDF5 file. \ingroup datasources */ HDF5Filter::HDF5Filter():AbstractFileFilter(), d(new HDF5FilterPrivate(this)) {} HDF5Filter::~HDF5Filter() {} /*! parses the content of the file \c fileName. */ void HDF5Filter::parse(const QString & fileName, QTreeWidgetItem* rootItem) { d->parse(fileName, rootItem); } /*! reads the content of the data set \c dataSet from file \c fileName. */ QVector HDF5Filter::readCurrentDataSet(const QString& fileName, AbstractDataSource* dataSource, bool &ok, AbstractFileFilter::ImportMode importMode, int lines) { return d->readCurrentDataSet(fileName, dataSource, ok, importMode, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ -QVector HDF5Filter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode, int lines) { - return d->readDataFromFile(fileName, dataSource, mode, lines); +QVector HDF5Filter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { + return d->readDataFromFile(fileName, dataSource, mode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void HDF5Filter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void HDF5Filter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void HDF5Filter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void HDF5Filter::setCurrentDataSetName(const QString& ds) { d->currentDataSetName = ds; } const QString HDF5Filter::currentDataSetName() const { return d->currentDataSetName; } void HDF5Filter::setStartRow(const int s) { d->startRow = s; } int HDF5Filter::startRow() const { return d->startRow; } void HDF5Filter::setEndRow(const int e) { d->endRow = e; } int HDF5Filter::endRow() const { return d->endRow; } void HDF5Filter::setStartColumn(const int c) { d->startColumn = c; } int HDF5Filter::startColumn() const { return d->startColumn; } void HDF5Filter::setEndColumn(const int c) { d->endColumn = c; } int HDF5Filter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### HDF5FilterPrivate::HDF5FilterPrivate(HDF5Filter* owner) : q(owner),currentDataSetName(""),startRow(1), endRow(-1), startColumn(1), endColumn(-1) { #ifdef HAVE_HDF5 m_status = 0; #endif } #ifdef HAVE_HDF5 void HDF5FilterPrivate::handleError(int err, const QString& function, const QString& arg) { #ifdef NDEBUG Q_UNUSED(err) Q_UNUSED(function) Q_UNUSED(arg) #else if (err < 0) DEBUG("ERROR " << err << ":" << function.toStdString() << "() - " << arg.toStdString()); #endif } QString HDF5FilterPrivate::translateHDF5Order(H5T_order_t o) { QString order; switch (o) { case H5T_ORDER_LE: order = "LE"; break; case H5T_ORDER_BE: order = "BE"; break; case H5T_ORDER_VAX: order = "VAX"; break; case H5T_ORDER_MIXED: order = "MIXED"; break; case H5T_ORDER_NONE: order = "NONE"; break; case H5T_ORDER_ERROR: order = "ERROR"; break; } return order; } QString HDF5FilterPrivate::translateHDF5Type(hid_t t) { QString type; if (H5Tequal(t, H5T_STD_I8LE) || H5Tequal(t, H5T_STD_I8BE)) type = "CHAR"; else if (H5Tequal(t, H5T_STD_U8LE) || H5Tequal(t, H5T_STD_U8BE)) type = "UCHAR"; else if (H5Tequal(t, H5T_STD_I16LE) || H5Tequal(t, H5T_STD_I16BE)) type = "SHORT"; else if (H5Tequal(t, H5T_STD_U16LE) || H5Tequal(t, H5T_STD_U16BE)) type = "USHORT"; else if (H5Tequal(t, H5T_STD_I32LE) || H5Tequal(t, H5T_STD_I32BE)) type = "INT"; else if (H5Tequal(t, H5T_STD_U32LE) || H5Tequal(t, H5T_STD_U32BE)) type = "UINT"; else if (H5Tequal(t, H5T_NATIVE_LONG)) type = "LONG"; else if (H5Tequal(t, H5T_NATIVE_ULONG)) type = "ULONG"; else if (H5Tequal(t, H5T_STD_I64LE) || H5Tequal(t, H5T_STD_I64BE)) type = "LLONG"; else if (H5Tequal(t, H5T_STD_U64LE) || H5Tequal(t, H5T_STD_U64BE)) type = "ULLONG"; else if (H5Tequal(t, H5T_IEEE_F32LE) || H5Tequal(t, H5T_IEEE_F32BE)) type = "FLOAT"; else if (H5Tequal(t, H5T_IEEE_F64LE) || H5Tequal(t, H5T_IEEE_F64BE)) type = "DOUBLE"; else if (H5Tequal(t, H5T_NATIVE_LDOUBLE)) type = "LDOUBLE"; else type = "UNKNOWN"; return type; } QString HDF5FilterPrivate::translateHDF5Class(H5T_class_t c) { QString dclass; switch (c) { case H5T_INTEGER: dclass = "INTEGER"; break; case H5T_FLOAT: dclass = "FLOAT"; break; case H5T_STRING: dclass = "STRING"; break; case H5T_BITFIELD: dclass = "BITFIELD"; break; case H5T_OPAQUE: dclass = "OPAQUE"; break; case H5T_COMPOUND: dclass = "COMPOUND"; break; case H5T_ARRAY: dclass = "ARRAY"; break; case H5T_ENUM: dclass = "ENUM"; break; case H5T_REFERENCE: dclass = "REFERENCE"; break; case H5T_VLEN: dclass = "VLEN"; break; case H5T_TIME: dclass = "TIME"; break; case H5T_NCLASSES: dclass = "NCLASSES"; break; case H5T_NO_CLASS: dclass = "NOCLASS"; break; } return dclass; } QStringList HDF5FilterPrivate::readHDF5Compound(hid_t tid) { size_t typeSize = H5Tget_size(tid); QString line; line += QLatin1String("COMPOUND(") + QString::number(typeSize) + QLatin1String(") : ("); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); for (int i=0; i < members; ++i) { H5T_class_t mclass = H5Tget_member_class(tid, i); handleError((int)mclass, "H5Tget_member_class"); hid_t mtype = H5Tget_member_type(tid, i); handleError((int)mtype, "H5Tget_member_type"); size_t size = H5Tget_size(mtype); handleError((int)size, "H5Tget_size"); QString typeString = translateHDF5Class(mclass); if (mclass == H5T_INTEGER || mclass == H5T_FLOAT) typeString = translateHDF5Type(mtype); line += H5Tget_member_name(tid, i) + QLatin1String("[") + typeString + QLatin1String("(") + QString::number(size) + QLatin1String(")]"); if (i == members-1) line += QLatin1String(")"); else line += QLatin1String(","); m_status = H5Tclose(mtype); handleError(m_status, "H5Tclose"); } QStringList dataString; dataString << line; return dataString; } template QStringList HDF5FilterPrivate::readHDF5Data1D(hid_t dataset, hid_t type, int rows, int lines, void* dataContainer) { DEBUG("readHDF5Data1D() rows =" << rows << "lines =" << lines); QStringList dataString; // we read all rows of data T* data = new T[rows]; m_status = H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Dread"); DEBUG(" startRow =" << startRow << "endRow =" << endRow); // DEBUG("dataContainer =" << dataContainer); for (int i = startRow-1; i < qMin(endRow, lines+startRow-1); ++i) { if (dataContainer) // read to data source static_cast*>(dataContainer)->operator[](i-startRow+1) = data[i]; else // for preview dataString << QString::number(static_cast(data[i])); } delete[] data; return dataString; } QStringList HDF5FilterPrivate::readHDF5CompoundData1D(hid_t dataset, hid_t tid, int rows, int lines, QVector& dataContainer) { DEBUG("HDF5FilterPrivate::readHDF5CompoundData1D()"); DEBUG(" dataContainer size = " << dataContainer.size()); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); DEBUG(" # members = " << members); QStringList dataString; if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) dataString << QLatin1String("("); dataContainer.resize(members); // avoid "index out of range" for preview } for (int m = 0; m < members; ++m) { hid_t mtype = H5Tget_member_type(tid, m); handleError((int)mtype, "H5Tget_member_type"); size_t msize = H5Tget_size(mtype); handleError((int)msize, "H5Tget_size"); hid_t ctype = H5Tcreate(H5T_COMPOUND, msize); handleError((int)ctype, "H5Tcreate"); m_status = H5Tinsert(ctype, H5Tget_member_name(tid, m), 0, mtype); handleError(m_status, "H5Tinsert"); QStringList mdataString; if (H5Tequal(mtype, H5T_STD_I8LE) || H5Tequal(mtype, H5T_STD_I8BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 2: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 4: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 8: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; } } else if (H5Tequal(mtype, H5T_STD_U8LE) || H5Tequal(mtype, H5T_STD_U8BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 2: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 4: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 8: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; } } else if (H5Tequal(mtype, H5T_STD_I16LE) || H5Tequal(mtype, H5T_STD_I16BE) || H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U16LE) || H5Tequal(mtype, H5T_STD_U16BE) || H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_I32LE) || H5Tequal(mtype, H5T_STD_I32BE) || H5Tequal(mtype, H5T_NATIVE_INT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U32LE) || H5Tequal(mtype, H5T_STD_U32BE) || H5Tequal(mtype, H5T_NATIVE_UINT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_LONG)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_ULONG)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_I64LE) || H5Tequal(mtype, H5T_STD_I64BE) || H5Tequal(mtype, H5T_NATIVE_LLONG)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U64LE) || H5Tequal(mtype, H5T_STD_U64BE) || H5Tequal(mtype, H5T_NATIVE_ULLONG)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_IEEE_F32LE) || H5Tequal(mtype, H5T_IEEE_F32BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_IEEE_F64LE) || H5Tequal(mtype, H5T_IEEE_F64BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_LDOUBLE)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else { if (dataContainer[m]) { for (int i = startRow-1; i < qMin(endRow, lines+startRow-1); ++i) static_cast*>(dataContainer[m])->operator[](i-startRow+1) = 0; } else { for (int i = 0; i < qMin(rows, lines); ++i) mdataString << QLatin1String("_"); } H5T_class_t mclass = H5Tget_member_class(tid, m); handleError((int)mclass, "H5Tget_member_class"); DEBUG("unsupported type of class " << translateHDF5Class(mclass).toStdString()); } if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) { dataString[i] += mdataString[i]; if (m < members-1) dataString[i] += QLatin1String(","); } } H5Tclose(ctype); } if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) dataString[i] += QLatin1String(")"); } return dataString; } template QVector HDF5FilterPrivate::readHDF5Data2D(hid_t dataset, hid_t type, int rows, int cols, int lines, QVector& dataPointer) { DEBUG("readHDF5Data2D() rows = " << rows << ", cols =" << cols << ", lines =" << lines); QVector dataStrings; T** data = (T**) malloc(rows*sizeof(T*)); data[0] = (T*) malloc(cols*rows*sizeof(T)); for (int i = 1; i < rows; ++i) data[i] = data[0]+i*cols; m_status = H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0][0]); handleError(m_status,"H5Dread"); for (int i = 0; i < qMin(rows, lines); ++i) { QStringList line; line.reserve(cols); for (int j = 0; j < cols; ++j) { if (dataPointer[0]) static_cast*>(dataPointer[j-startColumn+1])->operator[](i-startRow+1) = data[i][j]; else line << QString::number(static_cast(data[i][j])); } dataStrings << line; } free(data[0]); free(data); QDEBUG(dataStrings); return dataStrings; } QVector HDF5FilterPrivate::readHDF5CompoundData2D(hid_t dataset, hid_t tid, int rows, int cols, int lines) { DEBUG("readHDF5CompoundData2D() rows =" << rows << "cols =" << cols << "lines =" << lines); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); DEBUG(" # members =" << members); QVector dataStrings; for (int i = 0; i < qMin(rows, lines); ++i) { QStringList lineStrings; for (int j = 0; j < cols; ++j) lineStrings << QLatin1String("("); dataStrings << lineStrings; } //QStringList* data = new QStringList[members]; for (int m = 0; m < members; ++m) { hid_t mtype = H5Tget_member_type(tid, m); handleError((int)mtype, "H5Tget_member_type"); size_t msize = H5Tget_size(mtype); handleError((int)msize, "H5Tget_size"); hid_t ctype = H5Tcreate(H5T_COMPOUND, msize); handleError((int)ctype, "H5Tcreate"); m_status = H5Tinsert(ctype, H5Tget_member_name(tid, m), 0, mtype); handleError(m_status, "H5Tinsert"); // dummy container for all data columns // initially contains one pointer set to NULL QVector dummy(1, nullptr); QVector mdataStrings; if (H5Tequal(mtype, H5T_STD_I8LE) || H5Tequal(mtype, H5T_STD_I8BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 2: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 4: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 8: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; } } else if (H5Tequal(mtype, H5T_STD_U8LE) || H5Tequal(mtype, H5T_STD_U8BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 2: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 4: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 8: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; } } else if (H5Tequal(mtype, H5T_STD_I16LE) || H5Tequal(mtype, H5T_STD_I16BE)|| H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U16LE) || H5Tequal(mtype, H5T_STD_U16BE) || H5Tequal(mtype, H5T_NATIVE_USHORT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_I32LE) || H5Tequal(mtype, H5T_STD_I32BE) || H5Tequal(mtype, H5T_NATIVE_INT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U32LE) || H5Tequal(mtype, H5T_STD_U32BE) || H5Tequal(mtype, H5T_NATIVE_UINT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_LONG)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_ULONG)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_I64LE) || H5Tequal(mtype, H5T_STD_I64BE) || H5Tequal(mtype, H5T_NATIVE_LLONG)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U64LE) || H5Tequal(mtype, H5T_STD_U64BE) || H5Tequal(mtype, H5T_NATIVE_ULLONG)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_IEEE_F32LE) || H5Tequal(mtype, H5T_IEEE_F32BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_IEEE_F64LE) || H5Tequal(mtype, H5T_IEEE_F64BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_LDOUBLE)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else { for (int i = 0; i < qMin(rows, lines); ++i) { QStringList lineString; for (int j = 0; j < cols; ++j) lineString << QLatin1String("_"); mdataStrings << lineString; } #ifndef NDEBUG H5T_class_t mclass = H5Tget_member_class(tid, m); #endif DEBUG("unsupported class " << translateHDF5Class(mclass).toStdString()); } m_status = H5Tclose(ctype); handleError(m_status, "H5Tclose"); for (int i = 0; i < qMin(rows, lines); i++) { for (int j = 0; j < cols; j++) { dataStrings[i][j] += mdataStrings[i][j]; if (m < members-1) dataStrings[i][j] += QLatin1String(","); } } } for (int i = 0; i < qMin(rows, lines); ++i) { for (int j = 0; j < cols; ++j) dataStrings[i][j] += QLatin1String(")"); } QDEBUG("dataStrings =" << dataStrings); return dataStrings; } QStringList HDF5FilterPrivate::readHDF5Attr(hid_t aid) { QStringList attr; char name[MAXNAMELENGTH]; m_status = H5Aget_name(aid, MAXNAMELENGTH, name); handleError(m_status, "H5Aget_name"); attr << QString(name); // DEBUG(" name =" << QString(name)); hid_t aspace = H5Aget_space(aid); // the dimensions of the attribute data handleError((int)aspace, "H5Aget_space"); hid_t atype = H5Aget_type(aid); handleError((int)atype, "H5Aget_type"); hid_t aclass = H5Tget_class(atype); handleError((int)aclass, "H5Aget_class"); if (aclass == H5T_STRING) { char buf[MAXSTRINGLENGTH]; // buffer to read attr value hid_t amem = H5Tget_native_type(atype, H5T_DIR_ASCEND); handleError((int)amem, "H5Tget_native_type"); m_status = H5Aread(aid, amem, buf); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString(buf); m_status = H5Tclose(amem); handleError(m_status, "H5Tclose"); } else if (aclass == H5T_INTEGER) { if (H5Tequal(atype, H5T_STD_I8LE)) { qint8 value; m_status = H5Aread(aid, H5T_STD_I8LE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I8BE)) { qint8 value; m_status = H5Aread(aid, H5T_STD_I8BE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: { qint8 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 2: { qint16 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 4: { qint32 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 8: { qint64 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } default: DEBUG("unknown size " << sizeof(H5T_NATIVE_CHAR) << " of H5T_NATIVE_CHAR"); return QStringList(""); } } else if (H5Tequal(atype, H5T_STD_U8LE)) { uint8_t value; m_status = H5Aread(aid, H5T_STD_U8LE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U8BE)) { uint8_t value; m_status = H5Aread(aid, H5T_STD_U8BE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: { uint8_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 2: { uint16_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 4: { uint32_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 8: { uint64_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } default: DEBUG("unknown size " << sizeof(H5T_NATIVE_UCHAR) << " of H5T_NATIVE_UCHAR"); return QStringList(""); } } else if (H5Tequal(atype, H5T_STD_I16LE) || H5Tequal(atype, H5T_STD_I16BE) || H5Tequal(atype, H5T_NATIVE_SHORT)) { short value; m_status = H5Aread(aid, H5T_NATIVE_SHORT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U16LE) || H5Tequal(atype, H5T_STD_U16BE) || H5Tequal(atype, H5T_NATIVE_USHORT)) { unsigned short value; m_status = H5Aread(aid, H5T_NATIVE_USHORT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I32LE) || H5Tequal(atype, H5T_STD_I32BE) || H5Tequal(atype, H5T_NATIVE_INT)) { int value; m_status = H5Aread(aid, H5T_NATIVE_INT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U32LE) || H5Tequal(atype, H5T_STD_U32BE) || H5Tequal(atype, H5T_NATIVE_UINT)) { unsigned int value; m_status = H5Aread(aid, H5T_NATIVE_UINT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_LONG)) { long value; m_status = H5Aread(aid, H5T_NATIVE_LONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_ULONG)) { unsigned long value; m_status = H5Aread(aid, H5T_NATIVE_ULONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I64LE) || H5Tequal(atype, H5T_STD_I64BE) || H5Tequal(atype, H5T_NATIVE_LLONG)) { long long value; m_status = H5Aread(aid, H5T_NATIVE_LLONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U64LE) || H5Tequal(atype, H5T_STD_U64BE) || H5Tequal(atype, H5T_NATIVE_ULLONG)) { unsigned long long value; m_status = H5Aread(aid, H5T_NATIVE_ULLONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else attr<<" (unknown integer)"; } else if (aclass == H5T_FLOAT) { if (H5Tequal(atype, H5T_IEEE_F32LE) || H5Tequal(atype, H5T_IEEE_F32BE)) { float value; m_status = H5Aread(aid, H5T_NATIVE_FLOAT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_IEEE_F64LE) || H5Tequal(atype, H5T_IEEE_F64BE)) { double value; m_status = H5Aread(aid, H5T_NATIVE_DOUBLE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_LDOUBLE)) { long double value; m_status = H5Aread(aid, H5T_NATIVE_LDOUBLE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number((double)value); } else attr<<" (unknown float)"; } m_status = H5Tclose(atype); handleError(m_status, "H5Tclose"); m_status = H5Sclose(aspace); handleError(m_status, "H5Sclose"); return attr; } QStringList HDF5FilterPrivate::scanHDF5Attrs(hid_t oid) { QStringList attrList; int numAttr = H5Aget_num_attrs(oid); handleError(numAttr, "H5Aget_num_attrs"); DEBUG("number of attr =" << numAttr); for (int i = 0; i < numAttr; ++i) { hid_t aid = H5Aopen_idx(oid, i); handleError((int)aid, "H5Aopen_idx"); attrList << readHDF5Attr(aid); if (i < numAttr-1) attrList << QLatin1String(", "); m_status = H5Aclose(aid); handleError(m_status, "H5Aclose"); } return attrList; } QStringList HDF5FilterPrivate::readHDF5DataType(hid_t tid) { H5T_class_t typeClass = H5Tget_class(tid); handleError((int)typeClass, "H5Tget_class"); QStringList typeProps; QString typeString = translateHDF5Class(typeClass); if (typeClass == H5T_INTEGER || typeClass == H5T_FLOAT) typeString = translateHDF5Type(tid); typeProps<setIcon(0, QIcon::fromTheme("accessories-calculator")); dataTypeItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(dataTypeItem); } void HDF5FilterPrivate::scanHDF5DataSet(hid_t did, char *dataSetName, QTreeWidgetItem* parentItem) { QString attr = scanHDF5Attrs(did).join(""); char link[MAXNAMELENGTH]; m_status = H5Iget_name(did, link, MAXNAMELENGTH); handleError(m_status, "H5Iget_name"); QStringList dataSetProps; hsize_t size = H5Dget_storage_size(did); handleError((int)size, "H5Dget_storage_size"); hid_t datatype = H5Dget_type(did); handleError((int)datatype, "H5Dget_type"); size_t typeSize = H5Tget_size(datatype); handleError((int)typeSize, "H5Dget_size"); dataSetProps << readHDF5DataType(datatype); hid_t dataspace = H5Dget_space(did); int rank = H5Sget_simple_extent_ndims(dataspace); handleError(rank, "H5Sget_simple_extent_ndims"); unsigned int rows = 1, cols = 1, regs = 1; if (rank == 1) { hsize_t dims_out[1]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else if (rank == 2) { hsize_t dims_out[2]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; cols = dims_out[1]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String("x") << QString::number(cols) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else if (rank == 3) { hsize_t dims_out[3]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; cols = dims_out[1]; regs = dims_out[2]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String("x") << QString::number(cols) << QLatin1String("x") << QString::number(regs) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else dataSetProps << QLatin1String(", ") << i18n("rank %1 not supported yet", rank); hid_t pid = H5Dget_create_plist(did); handleError((int)pid, "H5Dget_create_plist"); dataSetProps << ", " << readHDF5PropertyList(pid).join(""); QTreeWidgetItem* dataSetItem = new QTreeWidgetItem(QStringList()<setIcon(0, QIcon::fromTheme("x-office-spreadsheet")); for (int i = 0; i < dataSetItem->columnCount(); ++i) { if (rows > 0 && cols > 0 && regs > 0) { dataSetItem->setBackground(i, QColor(192,255,192)); dataSetItem->setForeground(i, Qt::black); dataSetItem->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable); } else dataSetItem->setFlags(Qt::NoItemFlags); } parentItem->addChild(dataSetItem); } void HDF5FilterPrivate::scanHDF5Link(hid_t gid, char *linkName, QTreeWidgetItem* parentItem) { char target[MAXNAMELENGTH]; m_status = H5Gget_linkval(gid, linkName, MAXNAMELENGTH, target) ; handleError(m_status, "H5Gget_linkval"); QTreeWidgetItem* linkItem = new QTreeWidgetItem(QStringList() << QString(linkName) << i18n("symbolic link") << i18n("link to %1", QFile::decodeName(target))); linkItem->setIcon(0, QIcon::fromTheme("emblem-symbolic-link")); linkItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(linkItem); } void HDF5FilterPrivate::scanHDF5Group(hid_t gid, char *groupName, QTreeWidgetItem* parentItem) { DEBUG("HDF5FilterPrivate::scanHDF5Group()"); //check for hard link H5G_stat_t statbuf; m_status = H5Gget_objinfo(gid, ".", true, &statbuf); handleError(m_status, "H5Gget_objinfo"); if (statbuf.nlink > 1) { if (m_multiLinkList.contains(statbuf.objno[0])) { QTreeWidgetItem* objectItem = new QTreeWidgetItem(QStringList()<setIcon(0, QIcon::fromTheme("link")); objectItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(objectItem); return; } else { m_multiLinkList.append(statbuf.objno[0]); DEBUG(" group multiple links: "<setIcon(0, QIcon::fromTheme("folder")); groupItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(groupItem); hsize_t numObj; m_status = H5Gget_num_objs(gid, &numObj); handleError(m_status, "H5Gget_num_objs"); for (unsigned int i = 0; i < numObj; ++i) { char memberName[MAXNAMELENGTH]; m_status = H5Gget_objname_by_idx(gid, (hsize_t)i, memberName, (size_t)MAXNAMELENGTH ); handleError(m_status, "H5Gget_objname_by_idx"); int otype = H5Gget_objtype_by_idx(gid, (size_t)i ); handleError(otype, "H5Gget_objtype_by_idx"); switch (otype) { case H5G_LINK: { scanHDF5Link(gid, memberName, groupItem); break; } case H5G_GROUP: { hid_t grpid = H5Gopen(gid, memberName, H5P_DEFAULT); handleError((int)grpid, "H5Gopen"); scanHDF5Group(grpid, memberName, groupItem); m_status = H5Gclose(grpid); handleError(m_status, "H5Gclose"); break; } case H5G_DATASET: { hid_t dsid = H5Dopen(gid, memberName, H5P_DEFAULT); handleError((int)dsid, "H5Dopen"); scanHDF5DataSet(dsid, memberName, groupItem); m_status = H5Dclose(dsid); handleError(m_status, "H5Dclose"); break; } case H5G_TYPE: { hid_t tid = H5Topen(gid, memberName, H5P_DEFAULT); handleError((int)tid, "H5Topen"); scanHDF5DataType(tid, memberName, groupItem); m_status = H5Tclose(tid); handleError(m_status, "H5Tclose"); break; } default: QTreeWidgetItem* objectItem = new QTreeWidgetItem(QStringList() << QString(memberName) << i18n("unknown")); objectItem->setFlags(Qt::ItemIsEnabled); groupItem->addChild(objectItem); break; } } } #endif /*! parses the content of the file \c fileName and fill the tree using rootItem. */ void HDF5FilterPrivate::parse(const QString& fileName, QTreeWidgetItem* rootItem) { DEBUG("HDF5FilterPrivate::parse()"); #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); DEBUG("fileName = " << bafileName.data()); // check file type first htri_t isHdf5 = H5Fis_hdf5(bafileName.data()); if (isHdf5 == 0) { DEBUG(bafileName.data() << " is not a HDF5 file! Giving up."); return; } if (isHdf5 < 0) { DEBUG("H5Fis_hdf5() failed on " << bafileName.data() << "! Giving up."); return; } // open file hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); handleError((int)file, "H5Fopen", fileName); if (file < 0) { DEBUG("Opening file " << bafileName.data() << " failed! Giving up."); return; } char rootName[] = "/"; hid_t group = H5Gopen(file, rootName, H5P_DEFAULT); handleError((int)group, "H5Gopen", rootName); // CRASHES multiLinkList.clear(); scanHDF5Group(group, rootName, rootItem); m_status = H5Gclose(group); handleError(m_status, "H5Gclose", ""); m_status = H5Fclose(file); handleError(m_status, "H5Fclose", ""); #else DEBUG("HDF5 not available"); Q_UNUSED(fileName) Q_UNUSED(rootItem) #endif } /*! reads the content of the date set in the file \c fileName to a string (for preview) or to the data source. */ QVector HDF5FilterPrivate::readCurrentDataSet(const QString& fileName, AbstractDataSource* dataSource, bool &ok, AbstractFileFilter::ImportMode mode, int lines) { DEBUG("HDF5Filter::readCurrentDataSet()"); QVector dataStrings; if (currentDataSetName.isEmpty()) { //return QString("No data set selected").replace(' ',QChar::Nbsp); ok = false; return dataStrings << (QStringList() << i18n("No data set selected")); } DEBUG(" current data set =" << currentDataSetName.toStdString()); #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); handleError((int)file, "H5Fopen", fileName); QByteArray badataSet = currentDataSetName.toLatin1(); hid_t dataset = H5Dopen2(file, badataSet.data(), H5P_DEFAULT); handleError((int)file, "H5Dopen2", currentDataSetName); // Get datatype and dataspace hid_t dtype = H5Dget_type(dataset); handleError((int)dtype, "H5Dget_type"); H5T_class_t dclass = H5Tget_class(dtype); handleError((int)dclass, "H5Dget_class"); size_t typeSize = H5Tget_size(dtype); handleError((int)(typeSize-1), "H5Dget_size"); hid_t dataspace = H5Dget_space(dataset); handleError((int)dataspace, "H5Dget_space"); int rank = H5Sget_simple_extent_ndims(dataspace); handleError(rank, "H5Dget_simple_extent_ndims"); DEBUG(" rank =" << rank); int columnOffset = 0; // offset to import data int actualRows = 0, actualCols = 0; // rows and cols to read // dataContainer is used to store the data read from the dataSource // it contains the pointers of all columns // initially there is one pointer set to nullptr // check for dataContainer[0] != nullptr to decide if dataSource can be used QVector dataContainer(1, nullptr); // rank= 0: single value, 1: vector, 2: matrix, 3: 3D data, ... switch (rank) { case 0: { // single value actualCols = 1; switch (dclass) { case H5T_STRING: { char* data = (char *) malloc(typeSize * sizeof(char)); hid_t memtype = H5Tcopy(H5T_C_S1); handleError((int)memtype, "H5Tcopy"); m_status = H5Tset_size(memtype, typeSize); handleError(m_status, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Tread"); dataStrings << (QStringList() << data); free(data); break; } case H5T_INTEGER: case H5T_FLOAT: case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_COMPOUND: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataStrings << (QStringList() << i18n("rank 0 not implemented yet for type %1", translateHDF5Class(dclass))); qDebug() << dataStrings; } default: break; } break; } case 1: { // vector hsize_t size, maxSize; m_status = H5Sget_simple_extent_dims(dataspace, &size, &maxSize); handleError(m_status, "H5Sget_simple_extent_dims"); int rows = size; if (endRow == -1) endRow = rows; if (lines == -1) lines = endRow; actualRows = endRow-startRow+1; actualCols = 1; #ifndef NDEBUG H5T_order_t order = H5Tget_order(dtype); handleError((int)order, "H5Sget_order"); #endif QDEBUG(translateHDF5Class(dclass) << '(' << typeSize << ')' << translateHDF5Order(order) << ", rows:" << rows << " max:" << maxSize); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); QStringList dataString; // data saved in a list switch (dclass) { case H5T_STRING: { DEBUG("rank 1 H5T_STRING"); hid_t memtype = H5Tcopy(H5T_C_S1); handleError((int)memtype, "H5Tcopy"); char** data = (char **) malloc(rows * sizeof (char *)); if (H5Tis_variable_str(dtype)) { m_status = H5Tset_size(memtype, H5T_VARIABLE); handleError((int)memtype, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Dread"); } else { data[0] = (char *) malloc(rows * typeSize * sizeof (char)); for (int i = 1; i < rows; ++i) data[i] = data[0] + i * typeSize; m_status = H5Tset_size(memtype, typeSize); handleError((int)memtype, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data[0]); handleError(m_status, "H5Dread"); } for (int i = startRow-1; i < qMin(endRow, lines+startRow-1); ++i) dataString << data[i]; free(data); break; } case H5T_INTEGER: { if (H5Tequal(dtype, H5T_STD_I8LE)) dataString = readHDF5Data1D(dataset, H5T_STD_I8LE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I8BE)) dataString = readHDF5Data1D(dataset, H5T_STD_I8BE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 2: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 4: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 8: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; } } else if (H5Tequal(dtype, H5T_STD_U8LE)) dataString = readHDF5Data1D(dataset, H5T_STD_U8LE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U8BE)) dataString = readHDF5Data1D(dataset, H5T_STD_U8BE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 2: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 4: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 8: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; } } else if (H5Tequal(dtype, H5T_STD_I16LE) || H5Tequal(dtype, H5T_STD_I16BE) || H5Tequal(dtype, H5T_NATIVE_SHORT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_SHORT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U16LE) || H5Tequal(dtype, H5T_STD_U16BE) || H5Tequal(dtype, H5T_NATIVE_USHORT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_USHORT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I32LE) || H5Tequal(dtype, H5T_STD_I32BE) || H5Tequal(dtype, H5T_NATIVE_INT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_INT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U32LE) || H5Tequal(dtype, H5T_STD_U32BE) || H5Tequal(dtype, H5T_NATIVE_UINT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_UINT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_LONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_ULONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_ULONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I64LE) || H5Tequal(dtype, H5T_STD_I64BE) || H5Tequal(dtype, H5T_NATIVE_LLONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LLONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U64LE) || H5Tequal(dtype, H5T_STD_U64BE) || H5Tequal(dtype, H5T_NATIVE_ULLONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_ULLONG, rows, lines, dataContainer[0]); else { ok = false; dataString = (QStringList() << i18n("unsupported integer type for rank 1")); qDebug() << dataString; } break; } case H5T_FLOAT: { if (H5Tequal(dtype, H5T_IEEE_F32LE) || H5Tequal(dtype, H5T_IEEE_F32BE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_FLOAT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_IEEE_F64LE) || H5Tequal(dtype, H5T_IEEE_F64BE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_DOUBLE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LDOUBLE, rows, lines, dataContainer[0]); else { ok = false; dataString = (QStringList() << i18n("unsupported float type for rank 1")); qDebug() << dataString; } break; } case H5T_COMPOUND: { int members = H5Tget_nmembers(dtype); handleError(members, "H5Tget_nmembers"); if (dataSource) { // re-create data pointer dataContainer.clear(); dataSource->prepareImport(dataContainer, mode, actualRows, members, vectorNames, columnModes); } else dataStrings << readHDF5Compound(dtype); dataString = readHDF5CompoundData1D(dataset, dtype, rows, lines, dataContainer); break; } case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataString = (QStringList() << i18n("rank 1 not implemented yet for type %1", translateHDF5Class(dclass))); qDebug() << dataString; } default: break; } if (dataSource == NULL) { QDEBUG("dataString =" << dataString); for (int i = 0; i < qMin(rows, lines); ++i) dataStrings << (QStringList() << dataString[i]); } break; } case 2: { // matrix hsize_t dims_out[2]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); handleError(m_status, "H5Sget_simple_extent_dims"); int rows = dims_out[0]; int cols = dims_out[1]; if (endRow == -1) endRow=rows; if (lines == -1) lines = endRow; if (endColumn == -1) endColumn = cols; actualRows = endRow-startRow+1; actualCols = endColumn-startColumn+1; #ifndef NDEBUG H5T_order_t order = H5Tget_order(dtype); handleError((int)order, "H5Tget_order"); #endif QDEBUG(translateHDF5Class(dclass) << '(' << typeSize << ')' << translateHDF5Order(order) << "," << rows << "x" << cols); DEBUG("startRow/endRow" << startRow << endRow); DEBUG("startColumn/endColumn" << startColumn << endColumn); DEBUG("actual rows/cols" << actualRows << actualCols); DEBUG("lines" << lines); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); // read data switch (dclass) { case H5T_INTEGER: { if (H5Tequal(dtype, H5T_STD_I8LE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_I8LE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I8BE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_I8BE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 2: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 4: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 8: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; } } else if (H5Tequal(dtype, H5T_STD_U8LE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_U8LE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U8BE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_U8BE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 2: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 4: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 8: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; } } else if (H5Tequal(dtype, H5T_STD_I16LE) || H5Tequal(dtype, H5T_STD_I16BE) || H5Tequal(dtype, H5T_NATIVE_SHORT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_SHORT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U16LE) || H5Tequal(dtype, H5T_STD_U16BE) || H5Tequal(dtype, H5T_NATIVE_USHORT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_USHORT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I32LE) || H5Tequal(dtype, H5T_STD_I32BE) || H5Tequal(dtype, H5T_NATIVE_INT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_INT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U32LE) || H5Tequal(dtype, H5T_STD_U32BE) || H5Tequal(dtype, H5T_NATIVE_UINT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UINT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_LONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_ULONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_ULONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I64LE) || H5Tequal(dtype, H5T_STD_I64BE) || H5Tequal(dtype, H5T_NATIVE_LLONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LLONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U64LE) || H5Tequal(dtype, H5T_STD_U64BE) || H5Tequal(dtype, H5T_NATIVE_ULLONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_ULLONG, rows, cols, lines, dataContainer); else { ok=false; dataStrings << (QStringList() << i18n("unsupported integer type for rank 2")); qDebug() << dataStrings; } break; } case H5T_FLOAT: { if (H5Tequal(dtype, H5T_IEEE_F32LE) || H5Tequal(dtype, H5T_IEEE_F32BE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_FLOAT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_IEEE_F64LE) || H5Tequal(dtype, H5T_IEEE_F64BE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_DOUBLE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LDOUBLE, rows, cols, lines, dataContainer); else { ok = false; dataStrings << (QStringList() << i18n("unsupported float type for rank 2")); qDebug() << dataStrings; } break; } case H5T_COMPOUND: { dataStrings << readHDF5Compound(dtype); qDebug() << dataStrings; dataStrings << readHDF5CompoundData2D(dataset,dtype,rows,cols,lines); break; } case H5T_STRING: { // TODO: implement this ok = false; dataStrings << (QStringList() << i18n("rank 2 not implemented yet for type %1, size = %2", translateHDF5Class(dclass), typeSize)); qDebug() << dataStrings; break; } case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataStrings << (QStringList() << i18n("rank 2 not implemented yet for type %1", translateHDF5Class(dclass))); qDebug() << dataStrings; } default: break; } break; } default: { // 3D or more data ok = false; dataStrings << (QStringList() << i18n("rank %1 not implemented yet for type %2", rank, translateHDF5Class(dclass))); qDebug() << dataStrings; } } m_status = H5Sclose(dataspace); handleError(m_status, "H5Sclose"); m_status = H5Tclose(dtype); handleError(m_status, "H5Tclose"); m_status = H5Dclose(dataset); handleError(m_status, "H5Dclose"); m_status = H5Fclose(file); handleError(m_status, "H5Fclose"); if (!dataSource) return dataStrings; dataSource->finalizeImport(columnOffset, 1, actualCols, "", mode); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(mode) Q_UNUSED(lines) #endif return dataStrings; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ -QVector HDF5FilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode, int lines) { - Q_UNUSED(lines); +QVector HDF5FilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { DEBUG("HDF5Filter::read()"); QVector dataStrings; if (currentDataSetName.isEmpty()) { DEBUG("No data set selected"); return dataStrings; } bool ok = true; return readCurrentDataSet(fileName, dataSource, ok, mode); } /*! writes the content of \c dataSource to the file \c fileName. */ void HDF5FilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: writing HDF5 not implemented yet } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void HDF5Filter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("hdfFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool HDF5Filter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/datasources/filters/HDF5Filter.h b/src/backend/datasources/filters/HDF5Filter.h index 5b5a54eeb..8cb9612e3 100644 --- a/src/backend/datasources/filters/HDF5Filter.h +++ b/src/backend/datasources/filters/HDF5Filter.h @@ -1,73 +1,73 @@ /*************************************************************************** File : HDF5Filter.h Project : LabPlot Description : HDF5 I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef HDF5FILTER_H #define HDF5FILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include class QTreeWidgetItem; class HDF5FilterPrivate; class HDF5Filter : public AbstractFileFilter { Q_OBJECT public: HDF5Filter(); ~HDF5Filter() override; void parse(const QString& fileName, QTreeWidgetItem* rootItem); QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace) override; QVector readCurrentDataSet(const QString& fileName, AbstractDataSource*, bool& ok, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*) override; void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setCurrentDataSetName(const QString&); const QString currentDataSetName() const; void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class HDF5FilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/HDF5FilterPrivate.h b/src/backend/datasources/filters/HDF5FilterPrivate.h index a71ee4538..f611d0908 100644 --- a/src/backend/datasources/filters/HDF5FilterPrivate.h +++ b/src/backend/datasources/filters/HDF5FilterPrivate.h @@ -1,88 +1,88 @@ /*************************************************************************** File : HDF5FilterPrivate.h Project : LabPlot Description : Private implementation class for HDF5Filter. -------------------------------------------------------------------- Copyright : (C) 2015-2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef HDF5FILTERPRIVATE_H #define HDF5FILTERPRIVATE_H #include #ifdef HAVE_HDF5 #include #endif class AbstractDataSource; class HDF5FilterPrivate { public: explicit HDF5FilterPrivate(HDF5Filter*); void parse(const QString & fileName, QTreeWidgetItem* rootItem); QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); QVector readCurrentDataSet(const QString& fileName, AbstractDataSource*, bool &ok, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); const HDF5Filter* q; QString currentDataSetName; int startRow; int endRow; int startColumn; int endColumn; private: #ifdef HAVE_HDF5 int m_status; #endif const static int MAXNAMELENGTH = 1024; const static int MAXSTRINGLENGTH = 1024*1024; QList m_multiLinkList; // used to find hard links #ifdef HAVE_HDF5 void handleError(int err, const QString& function, const QString& arg = QString()); QString translateHDF5Order(H5T_order_t); QString translateHDF5Type(hid_t); QString translateHDF5Class(H5T_class_t); QStringList readHDF5Compound(hid_t tid); template QStringList readHDF5Data1D(hid_t dataset, hid_t type, int rows, int lines, void* dataPointer = nullptr); QStringList readHDF5CompoundData1D(hid_t dataset, hid_t tid, int rows, int lines, QVector& dataPointer); template QVector readHDF5Data2D(hid_t dataset, hid_t ctype, int rows, int cols, int lines, QVector& dataPointer); QVector readHDF5CompoundData2D(hid_t dataset, hid_t tid, int rows, int cols, int lines); QStringList readHDF5Attr(hid_t aid); QStringList scanHDF5Attrs(hid_t oid); QStringList readHDF5DataType(hid_t tid); QStringList readHDF5PropertyList(hid_t pid); void scanHDF5DataType(hid_t tid, char* dataTypeName, QTreeWidgetItem* parentItem); void scanHDF5Link(hid_t gid, char* linkName, QTreeWidgetItem* parentItem); void scanHDF5DataSet(hid_t dsid, char* dataSetName, QTreeWidgetItem* parentItem); void scanHDF5Group(hid_t gid, char* groupName, QTreeWidgetItem* parentItem); #endif }; #endif diff --git a/src/backend/datasources/filters/ImageFilter.cpp b/src/backend/datasources/filters/ImageFilter.cpp index eea39fc47..d0762efdc 100644 --- a/src/backend/datasources/filters/ImageFilter.cpp +++ b/src/backend/datasources/filters/ImageFilter.cpp @@ -1,291 +1,290 @@ /*************************************************************************** File : ImageFilter.cpp Project : LabPlot Description : Image I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015 by Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/ImageFilter.h" #include "backend/datasources/filters/ImageFilterPrivate.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/core/column/Column.h" #include #include /*! \class ImageFilter \brief Manages the import/export of data from/to an image file. \ingroup datasources */ ImageFilter::ImageFilter():AbstractFileFilter(), d(new ImageFilterPrivate(this)) {} ImageFilter::~ImageFilter() {} /*! returns the list of all predefined import formats. */ QStringList ImageFilter::importFormats() { return (QStringList() << i18n("Matrix (grayscale)") << i18n("XYZ (grayscale)") << i18n("XYRGB")); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ -QVector ImageFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - return d->readDataFromFile(fileName, dataSource, importMode, lines); +QVector ImageFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { + return d->readDataFromFile(fileName, dataSource, importMode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void ImageFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void ImageFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void ImageFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void ImageFilter::setImportFormat(const ImageFilter::ImportFormat f) { d->importFormat = f; } ImageFilter::ImportFormat ImageFilter::importFormat() const { return d->importFormat; } void ImageFilter::setStartRow(const int s) { d->startRow = s; } int ImageFilter::startRow() const { return d->startRow; } void ImageFilter::setEndRow(const int e) { d->endRow = e; } int ImageFilter::endRow() const { return d->endRow; } void ImageFilter::setStartColumn(const int s) { d->startColumn = s; } int ImageFilter::startColumn() const { return d->startColumn; } void ImageFilter::setEndColumn(const int e) { d->endColumn = e; } int ImageFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### ImageFilterPrivate::ImageFilterPrivate(ImageFilter* owner) : q(owner), importFormat(ImageFilter::MATRIX), startRow(1), endRow(-1), startColumn(1), endColumn(-1) { } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ -QVector ImageFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode, int lines) { - Q_UNUSED(lines); +QVector ImageFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { QVector dataStrings; QImage image = QImage(fileName); if (image.isNull() || image.format() == QImage::Format_Invalid) { #ifdef QT_DEBUG qDebug()<<"failed to read image"<*>(dataContainer[j])->operator[](i) = value; } emit q->completed(100*i/actualRows); } break; } case ImageFilter::XYZ: { int currentRow = 0; for (int i = startRow-1; i < endRow; ++i) { for (int j = startColumn-1; j < endColumn; ++j) { QRgb color=image.pixel(j, i); static_cast*>(dataContainer[0])->operator[](currentRow) = i+1; static_cast*>(dataContainer[1])->operator[](currentRow) = j+1; static_cast*>(dataContainer[2])->operator[](currentRow) = qGray(color); currentRow++; } emit q->completed(100*i/actualRows); } break; } case ImageFilter::XYRGB: { int currentRow = 0; for (int i = startRow-1; i < endRow; ++i) { for ( int j = startColumn-1; j < endColumn; ++j) { QRgb color = image.pixel(j, i); static_cast*>(dataContainer[0])->operator[](currentRow) = i+1; static_cast*>(dataContainer[1])->operator[](currentRow) = j+1; static_cast*>(dataContainer[2])->operator[](currentRow) = qRed(color); static_cast*>(dataContainer[3])->operator[](currentRow) = qGreen(color); static_cast*>(dataContainer[4])->operator[](currentRow) = qBlue(color); currentRow++; } emit q->completed(100*i/actualRows); } break; } } Spreadsheet* spreadsheet = dynamic_cast(dataSource); if (spreadsheet) { QString comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); for ( int n = 0; n < actualCols; ++n) { Column* column = spreadsheet->column(columnOffset+n); column->setComment(comment); column->setUndoAware(true); if (mode == AbstractFileFilter::Replace) { column->setSuppressDataChangedSignal(false); column->setChanged(); } } } dataSource->finalizeImport(); return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void ImageFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void ImageFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("imageFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool ImageFilter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/datasources/filters/ImageFilter.h b/src/backend/datasources/filters/ImageFilter.h index 54cde00e5..71fe26e17 100644 --- a/src/backend/datasources/filters/ImageFilter.h +++ b/src/backend/datasources/filters/ImageFilter.h @@ -1,74 +1,74 @@ /*************************************************************************** File : ImageFilter.h Project : LabPlot Description : Image I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef IMAGEFILTER_H #define IMAGEFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" class ImageFilterPrivate; class QStringList; class ImageFilter : public AbstractFileFilter { Q_OBJECT Q_ENUMS(ImportFormat) public: enum ImportFormat {MATRIX,XYZ,XYRGB}; ImageFilter(); ~ImageFilter() override; static QStringList importFormats(); QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace) override; void write(const QString& fileName, AbstractDataSource*) override; void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setImportFormat(const ImageFilter::ImportFormat); ImageFilter::ImportFormat importFormat() const; void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class ImageFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/ImageFilterPrivate.h b/src/backend/datasources/filters/ImageFilterPrivate.h index 4577f9074..6c03e83f3 100644 --- a/src/backend/datasources/filters/ImageFilterPrivate.h +++ b/src/backend/datasources/filters/ImageFilterPrivate.h @@ -1,50 +1,50 @@ /*************************************************************************** File : ImageFilterPrivate.h Project : LabPlot Description : Private implementation class for ImageFilter. -------------------------------------------------------------------- Copyright : (C) 2015 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef IMAGEFILTERPRIVATE_H #define IMAGEFILTERPRIVATE_H class AbstractDataSource; class ImageFilterPrivate { public: explicit ImageFilterPrivate(ImageFilter*); QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void write(const QString& fileName, AbstractDataSource*); const ImageFilter* q; ImageFilter::ImportFormat importFormat; // how to import the image int startRow; // start row int endRow; // end row int startColumn; // start column int endColumn; // end column }; #endif diff --git a/src/backend/datasources/filters/JsonFilter.cpp b/src/backend/datasources/filters/JsonFilter.cpp index a8ed54e19..82e12bb93 100644 --- a/src/backend/datasources/filters/JsonFilter.cpp +++ b/src/backend/datasources/filters/JsonFilter.cpp @@ -1,836 +1,836 @@ /*************************************************************************** File : JsonFilter.cpp Project : LabPlot Description : JSON I/O-filter. -------------------------------------------------------------------- -------------------------------------------------------------------- Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/JsonFilter.h" #include "backend/datasources/filters/JsonFilterPrivate.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include #include #include #include /*! \class JsonFilter \brief Manages the import/export of data from/to a file formatted using JSON. \ingroup datasources */ JsonFilter::JsonFilter() : AbstractFileFilter(), d(new JsonFilterPrivate(this)) {} JsonFilter::~JsonFilter() {} /*! reads the content of the device \c device. */ void JsonFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } /*! reads the content of the file \c fileName. */ -QVector JsonFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - d->readDataFromFile(fileName, dataSource, importMode, lines); +QVector JsonFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { + d->readDataFromFile(fileName, dataSource, importMode); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector JsonFilter::preview(const QString& fileName) { return d->preview(fileName); } QVector JsonFilter::preview(QIODevice& device) { return d->preview(device); } QVector JsonFilter::preview(QJsonDocument& doc) { return d->preview(doc); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void JsonFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void JsonFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void JsonFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list of all predefined data types. */ QStringList JsonFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } /*! returns the list of all predefined data row types. */ QStringList JsonFilter::dataRowTypes() { return (QStringList() << "Array" << "Object"); } void JsonFilter::setDataRowType(QJsonValue::Type type) { d->rowType = type; } QJsonValue::Type JsonFilter::dataRowType() const { return d->rowType; } void JsonFilter::setModelRows(QVector rows){ d->modelRows = rows; } QVector JsonFilter::modelRows() const { return d->modelRows; } void JsonFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString JsonFilter::dateTimeFormat() const { return d->dateTimeFormat; } void JsonFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language JsonFilter::numberFormat() const { return d->numberFormat; } void JsonFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool JsonFilter::NaNValueToZeroEnabled() const { if (d->nanValue == 0) return true; return false; } void JsonFilter::setCreateIndexEnabled(bool b){ d->createIndexEnabled = b; } void JsonFilter::setParseRowsName(bool b) { d->parseRowsName = b; } void JsonFilter::setVectorNames(const QString& s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList JsonFilter::vectorNames() const { return d->vectorNames; } QVector JsonFilter::columnModes() { return d->columnModes; } void JsonFilter::setStartRow(const int r) { d->startRow = r; } int JsonFilter::startRow() const { return d->startRow; } void JsonFilter::setEndRow(const int r) { d->endRow = r; } int JsonFilter::endRow() const { return d->endRow; } void JsonFilter::setStartColumn(const int c) { d->startColumn = c; } int JsonFilter::startColumn() const { return d->startColumn; } void JsonFilter::setEndColumn(const int c) { d->endColumn = c; } int JsonFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### JsonFilterPrivate::JsonFilterPrivate(JsonFilter* owner) : q(owner), model(new QJsonModel()), containerType(JsonFilter::Object), rowType(QJsonValue::Object), numberFormat(QLocale::C), createIndexEnabled(false), parseRowsName(false), vectorNames(), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_actualRows(0), m_actualCols(0), m_prepared(false), m_columnOffset(0) { } //TODO: delete model from memory /*! returns 1 if row is invalid and 0 otherwise. */ int JsonFilterPrivate::checkRow(QJsonValueRef value, int& countCols) { switch(rowType){ //TODO: implement other value types case QJsonValue::Array: { QJsonArray row = value.toArray(); if(row.isEmpty()) return 1; countCols = (countCols == -1 || countCols > row.count()) ? row.count() : countCols; break; } case QJsonValue::Object: { QJsonObject row = value.toObject(); if(row.isEmpty()) return 1; countCols = (countCols == -1 || countCols > row.count()) ? row.count() : countCols; break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return 1; } return 0; } /*! returns -1 if a parse error has occurred, 1 if the current row type not supported and 0 otherwise. */ int JsonFilterPrivate::parseColumnModes(QJsonValue row, QString rowName) { columnModes.resize(m_actualCols); int colIndexInContainer = startColumn - 1; for(int i = 0; i < m_actualCols; ++i){ if((createIndexEnabled || parseRowsName) && i == 0){ if(createIndexEnabled) columnModes[i] = AbstractColumn::Integer; if(parseRowsName) columnModes[i + createIndexEnabled] = AbstractFileFilter::columnMode(rowName, dateTimeFormat, numberFormat); i = i + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue columnValue; switch (rowType) { //TODO: implement other value types case QJsonValue::Array: { QJsonArray arr = row.toArray(); if(arr.count() < colIndexInContainer + 1) return -1; columnValue = *(row.toArray().begin() + colIndexInContainer); break; } case QJsonValue::Object: { QJsonObject obj = row.toObject(); if(obj.count() < colIndexInContainer + 1) return -1; if(vectorNames.count() == 0) vectorNames = row.toObject().keys(); columnValue = *(row.toObject().begin() + colIndexInContainer); break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return 1; } switch (columnValue.type()) { case QJsonValue::Double: columnModes[i] = AbstractColumn::Numeric; break; case QJsonValue::String: columnModes[i] = AbstractFileFilter::columnMode(columnValue.toString(), dateTimeFormat, numberFormat); break; case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return -1; } colIndexInContainer++; } if(parseRowsName) vectorNames.prepend("row name"); if(createIndexEnabled) vectorNames.prepend("index"); return 0; } void JsonFilterPrivate::setEmptyValue(int column, int row){ switch (columnModes[column]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[column])->operator[](row) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[column])->operator[](row) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[column])->operator[](row) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[column])->operator[](row) = ""; break; case AbstractColumn::Month: case AbstractColumn::Day: break; } } void JsonFilterPrivate::setValueFromString(int column, int row, QString valueString){ QLocale locale(numberFormat); switch (columnModes[column]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[column])->operator[](row) = isNumber ? value : nanValue; break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[column])->operator[](row) = isNumber ? value : 0; break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[column])->operator[](row) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[column])->operator[](row) = valueString; break; case AbstractColumn::Month: case AbstractColumn::Day: break; } } /*! returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end */ int JsonFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("device is sequential = " << device.isSequential()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; QJsonParseError err; QJsonDocument doc = QJsonDocument::fromJson(device.readAll(), &err); if(err.error != QJsonParseError::NoError || doc.isEmpty()) return 1; if(prepareDocumentToRead(doc) != 0) return 2; // reset to start of file if (!device.isSequential()) device.seek(0); return 0; } /*! returns 2 if a parse error has occurred and 0 otherwise. */ int JsonFilterPrivate::prepareDocumentToRead(const QJsonDocument& doc) { model->loadJson(doc); if(modelRows.isEmpty()) m_preparedDoc = doc; else { QModelIndex index; for(auto it = modelRows.begin(); it != modelRows.end(); ++it){ index = model->index(*it, 0, index); } m_preparedDoc = model->genJsonByIndex(index); } if(!m_preparedDoc.isEmpty()){ if(m_preparedDoc.isArray()) containerType = JsonFilter::Array; else if(m_preparedDoc.isObject()) containerType = JsonFilter::Object; else return 2; } else return 2; int countRows = 0; int countCols = -1; QJsonValue firstRow; QString firstRowName = ""; parseRowsName = parseRowsName && rowType == QJsonValue::Object; switch(containerType) { case JsonFilter::Array: { QJsonArray arr = m_preparedDoc.array(); if(arr.count() < startRow) return 2; int endRowOffset = (endRow == -1 || endRow > arr.count()) ? arr.count() : endRow; firstRow = *(arr.begin() + (startRow - 1)); for(QJsonArray::iterator it = arr.begin() + (startRow - 1); it != arr.begin() + endRowOffset; ++it) { if(checkRow(*it, countCols) != 0) return 2; countRows++; } break; } case JsonFilter::Object: { QJsonObject obj = m_preparedDoc.object(); if(obj.count() < startRow) return 2; int startRowOffset = startRow - 1; int endRowOffset = (endRow == -1 || endRow > obj.count()) ? obj.count() : endRow; firstRow = *(obj.begin() + startRowOffset); firstRowName = (obj.begin() + startRowOffset).key(); for(QJsonObject::iterator it = obj.begin() + startRowOffset; it != obj.begin() + endRowOffset; ++it) { if(checkRow(*it, countCols) != 0) return 2; countRows++; } break; } } if(endColumn == -1 || endColumn > countCols) endColumn = countCols; m_actualRows = countRows; m_actualCols = endColumn - startColumn + 1 + createIndexEnabled + parseRowsName; if(parseColumnModes(firstRow, firstRowName) != 0) return 2; DEBUG("start/end column: = " << startColumn << ' ' << endColumn); DEBUG("start/end rows = " << startRow << ' ' << endRow); DEBUG("actual cols/rows = " << m_actualCols << ' ' << m_actualRows); return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ -void JsonFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { +void JsonFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { KFilterDev device(fileName); - readDataFromDevice(device, dataSource, importMode, lines); + readDataFromDevice(device, dataSource, importMode, -1); } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { if(!m_prepared) { const int deviceError = prepareDeviceToRead(device); if(deviceError != 0){ DEBUG("Device error = " << deviceError); return; } //TODO: support other modes and vector names m_prepared = true; } importData(dataSource, importMode, lines); } /*! reads the content of document \c doc to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::readDataFromDocument(const QJsonDocument& doc, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { if(!m_prepared) { const int docError = prepareDocumentToRead(doc); if(docError != 0){ DEBUG("Document parse error = " << docError); return; } //TODO: support other modes and vector names m_prepared = true; } importData(dataSource, importMode, lines); } /*! import the content of document \c m_preparedDoc to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::importData(AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { Q_UNUSED(lines) m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); int rowOffset = startRow - 1; DEBUG("reading " << m_actualRows << " lines"); for(int i = 0; i < m_actualRows; ++i) { QString rowName; QJsonValue row; switch (containerType) { case JsonFilter::Array: row = *(m_preparedDoc.array().begin() + rowOffset + i); break; case JsonFilter::Object: rowName = (m_preparedDoc.object().begin() + rowOffset + i).key(); row = *(m_preparedDoc.object().begin() + rowOffset + i); break; } int colIndex = 0; for(int n = 0; n < m_actualCols; ++n) { if((createIndexEnabled || parseRowsName) && n == 0) { if(createIndexEnabled) static_cast*>(m_dataContainer[n])->operator[](i) = i + 1; if(parseRowsName) setValueFromString(n + createIndexEnabled, i, rowName); n = n + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue value; switch(rowType){ //TODO: implement other value types case QJsonValue::Array: { value = *(row.toArray().begin() + colIndex); break; } case QJsonValue::Object: { value = *(row.toObject().begin() + colIndex); break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: break; } switch(value.type()) { case QJsonValue::Double: if(columnModes[n] == AbstractColumn::Numeric) static_cast*>(m_dataContainer[n])->operator[](i) = value.toDouble(); else setEmptyValue(n, i + startRow - 1); break; case QJsonValue::String: setValueFromString(n, i, value.toString()); break; case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: setEmptyValue(n, i + startRow - 1); break; } colIndex++; } emit q->completed(100 * i/m_actualRows); } //TODO: fix (startColumn + m_actualCols - 1) dataSource->finalizeImport(m_columnOffset, startColumn, startColumn + m_actualCols - 1, dateTimeFormat, importMode); } /*! generates the preview for the file \c fileName. */ QVector JsonFilterPrivate::preview(const QString& fileName) { KFilterDev device(fileName); return preview(device); } /*! generates the preview for device \c device. */ QVector JsonFilterPrivate::preview(QIODevice &device) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return QVector(); } return preview(); } /*! generates the preview for document \c doc. */ QVector JsonFilterPrivate::preview(QJsonDocument &doc) { if(prepareDocumentToRead(doc) != 0) return QVector(); return preview(); } /*! generates the preview for document \c m_preparedDoc. */ QVector JsonFilterPrivate::preview() { QVector dataStrings; int rowOffset = startRow - 1; DEBUG("reading " << m_actualRows << " lines"); for(int i = 0; i < m_actualRows; ++i) { QString rowName; QJsonValue row; switch (containerType) { case JsonFilter::Object: rowName = (m_preparedDoc.object().begin() + rowOffset + i).key(); row = *(m_preparedDoc.object().begin() + rowOffset + i); break; case JsonFilter::Array: row = *(m_preparedDoc.array().begin() + rowOffset + i); break; } QStringList lineString; int colIndex = 0; for(int n = 0; n < m_actualCols; ++n) { if((createIndexEnabled || parseRowsName) && n == 0) { if(createIndexEnabled) lineString += QString::number(i + 1); if(parseRowsName) lineString += rowName; n = n + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue value; switch(rowType){ case QJsonValue::Object: { value = *(row.toObject().begin() + colIndex); break; } case QJsonValue::Array: { value = *(row.toArray().begin() + colIndex); break; } //TODO: implement other value types case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: break; } switch(value.type()) { case QJsonValue::Double: if(columnModes[n] == AbstractColumn::Numeric) lineString += QString::number(value.toDouble(), 'g', 16); else lineString += lineString += QLatin1String(""); break; case QJsonValue::String: { //TODO: add parsing string before appending lineString += value.toString(); break; } case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: lineString += QLatin1String(""); break; } colIndex++; } dataStrings << lineString; emit q->completed(100 * i/m_actualRows); } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void JsonFilterPrivate::write(const QString& fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to json file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void JsonFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("jsonFilter"); writer->writeAttribute("rowType", QString::number(d->rowType)); writer->writeAttribute("dateTimeFormat", d->dateTimeFormat); writer->writeAttribute("numberFormat", QString::number(d->numberFormat)); writer->writeAttribute("createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute("parseRowsName", QString::number(d->parseRowsName)); writer->writeAttribute("nanValue", QString::number(d->nanValue)); writer->writeAttribute("startRow", QString::number(d->startRow)); writer->writeAttribute("endRow", QString::number(d->endRow)); writer->writeAttribute("startColumn", QString::number(d->startColumn)); writer->writeAttribute("endColumn", QString::number(d->endColumn)); QStringList list; for(auto it = modelRows().begin(); it != modelRows().end(); ++it){ list.append(QString::number(*it)); } writer->writeAttribute("modelRows", list.join(';')); writer->writeEndElement(); DEBUG("JsonFilter save params"); } /*! Loads from XML. */ bool JsonFilter::load(XmlStreamReader* reader) { QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("rowType").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'rowType'")); else d->rowType = static_cast(str.toInt()); str = attribs.value("dateTimeFormat").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'dateTimeFormat'")); else d->dateTimeFormat = str; str = attribs.value("numberFormat").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'numberFormat'")); else d->numberFormat = static_cast(str.toInt()); str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); str = attribs.value("parseRowsName").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'parseRowsName'")); else d->parseRowsName = str.toInt(); str = attribs.value("nanValue").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'nanValue'")); else d->nanValue = str.toDouble(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startColumn'")); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endColumn'")); else d->endColumn = str.toInt(); QStringList list = attribs.value("modelRows").toString().split(';'); if (list.isEmpty()) reader->raiseWarning(attributeWarning.arg("'modelRows'")); else{ d->modelRows = QVector(); for(auto it = list.begin(); it !=list.end(); ++it) d->modelRows.append((*it).toInt()); } DEBUG("JsonFilter load params"); return true; } diff --git a/src/backend/datasources/filters/JsonFilter.h b/src/backend/datasources/filters/JsonFilter.h index 322862858..1879b2118 100644 --- a/src/backend/datasources/filters/JsonFilter.h +++ b/src/backend/datasources/filters/JsonFilter.h @@ -1,106 +1,106 @@ /*************************************************************************** File : JsonFilter.h Project : LabPlot Description : JSON I/O-filter. -------------------------------------------------------------------- -------------------------------------------------------------------- Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef JSONFILTER_H #define JSONFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" #include class QStringList; class QIODevice; class QJsonDocument; class QJsonModel; class JsonFilterPrivate; class JsonFilter : public AbstractFileFilter { Q_OBJECT public: enum DataContainerType {Array, Object}; JsonFilter(); ~JsonFilter() override; static QStringList dataTypes(); static QStringList dataRowTypes(); // read data from any device void readDataFromDevice(QIODevice& device, AbstractDataSource*, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); // overloaded function to read from file QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace) override; void write(const QString& fileName, AbstractDataSource*) override; QVector preview(const QString& fileName); QVector preview(QIODevice& device); QVector preview(QJsonDocument& doc); void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setDataRowType(const QJsonValue::Type); QJsonValue::Type dataRowType() const; void setModelRows(const QVector); QVector modelRows() const; void setDateTimeFormat(const QString&); QString dateTimeFormat() const; void setNumberFormat(QLocale::Language); QLocale::Language numberFormat() const; void setNaNValueToZero(const bool); bool NaNValueToZeroEnabled() const; void setCreateIndexEnabled(const bool); void setParseRowsName(const bool); void setVectorNames(const QString&); QStringList vectorNames() const; QVector columnModes(); void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class JsonFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/JsonFilterPrivate.h b/src/backend/datasources/filters/JsonFilterPrivate.h index fc48061d9..62f753ffb 100644 --- a/src/backend/datasources/filters/JsonFilterPrivate.h +++ b/src/backend/datasources/filters/JsonFilterPrivate.h @@ -1,97 +1,97 @@ /*************************************************************************** File : JsonFilterPrivate.h Project : LabPlot Description : Private implementation class for JsonFilter. -------------------------------------------------------------------- -------------------------------------------------------------------- Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef JSONFILTERPRIVATE_H #define JSONFILTERPRIVATE_H #include #include "QJsonModel.h" class KFilterDev; class AbstractDataSource; class AbstractColumn; class JsonFilterPrivate { public: JsonFilterPrivate (JsonFilter* owner); int checkRow(QJsonValueRef value, int &countCols); int parseColumnModes(QJsonValue row, QString rowName = ""); void setEmptyValue(int column, int row); void setValueFromString(int column, int row, QString value); int prepareDeviceToRead(QIODevice&); int prepareDocumentToRead(const QJsonDocument&); void readDataFromDevice(QIODevice& device, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void readDataFromDocument(const QJsonDocument& doc, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void importData(AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName); QVector preview(QIODevice& device); QVector preview(QJsonDocument& doc); QVector preview(); const JsonFilter* q; QJsonModel* model; JsonFilter::DataContainerType containerType; QJsonValue::Type rowType; QVector modelRows; QString dateTimeFormat; QLocale::Language numberFormat; double nanValue; bool createIndexEnabled; bool parseRowsName; QStringList vectorNames; QVector columnModes; int startRow; // start row int endRow; // end row int startColumn; // start column int endColumn; // end column private: int m_actualRows; int m_actualCols; int m_prepared; int m_columnOffset; // indexes the "start column" in the datasource. Data will be imported starting from this column. QVector m_dataContainer; // pointers to the actual data containers (columns). QJsonDocument m_preparedDoc; // parsed Json document }; #endif diff --git a/src/backend/datasources/filters/NetCDFFilter.cpp b/src/backend/datasources/filters/NetCDFFilter.cpp index 6faf32d12..45ad30b18 100644 --- a/src/backend/datasources/filters/NetCDFFilter.cpp +++ b/src/backend/datasources/filters/NetCDFFilter.cpp @@ -1,691 +1,690 @@ /*************************************************************************** File : NetCDFFilter.cpp Project : LabPlot Description : NetCDF I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/NetCDFFilter.h" #include "backend/datasources/filters/NetCDFFilterPrivate.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/core/column/Column.h" #include #include /*! \class NetCDFFilter \brief Manages the import/export of data from/to a NetCDF file. \ingroup datasources */ NetCDFFilter::NetCDFFilter():AbstractFileFilter(), d(new NetCDFFilterPrivate(this)) {} NetCDFFilter::~NetCDFFilter() {} /*! parses the content of the file \c ileName. */ void NetCDFFilter::parse(const QString & fileName, QTreeWidgetItem* rootItem) { d->parse(fileName, rootItem); } /*! reads the content of the selected attribute from file \c fileName. */ QString NetCDFFilter::readAttribute(const QString & fileName, const QString & name, const QString & varName) { return d->readAttribute(fileName, name, varName); } /*! reads the content of the current variable from file \c fileName. */ QVector NetCDFFilter::readCurrentVar(const QString& fileName, AbstractDataSource* dataSource, - AbstractFileFilter::ImportMode importMode, int lines) { + AbstractFileFilter::ImportMode importMode, int lines) { return d->readCurrentVar(fileName, dataSource, importMode, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ -QVector NetCDFFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode, int lines) { - return d->readDataFromFile(fileName, dataSource, mode, lines); +QVector NetCDFFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { + return d->readDataFromFile(fileName, dataSource, mode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void NetCDFFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void NetCDFFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void NetCDFFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void NetCDFFilter::setCurrentVarName(const QString& ds) { d->currentVarName = ds; } const QString NetCDFFilter::currentVarName() const { return d->currentVarName; } void NetCDFFilter::setStartRow(const int s) { d->startRow = s; } int NetCDFFilter::startRow() const { return d->startRow; } void NetCDFFilter::setEndRow(const int e) { d->endRow = e; } int NetCDFFilter::endRow() const { return d->endRow; } void NetCDFFilter::setStartColumn(const int c) { d->startColumn=c; } int NetCDFFilter::startColumn() const { return d->startColumn; } void NetCDFFilter::setEndColumn(const int c) { d->endColumn = c; } int NetCDFFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### NetCDFFilterPrivate::NetCDFFilterPrivate(NetCDFFilter* owner) : q(owner), startRow(1), endRow(-1), startColumn(1), endColumn(-1) { #ifdef HAVE_NETCDF m_status = 0; #endif } #ifdef HAVE_NETCDF void NetCDFFilterPrivate::handleError(int err, const QString& function) { if (err != NC_NOERR) qDebug() << "NETCDF ERROR:" << function << "() - " << nc_strerror(m_status); } QString NetCDFFilterPrivate::translateDataType(nc_type type) { QString typeString; switch (type) { case NC_BYTE: typeString = "BYTE"; break; case NC_UBYTE: typeString = "UBYTE"; break; case NC_CHAR: typeString = "CHAR"; break; case NC_SHORT: typeString = "SHORT"; break; case NC_USHORT: typeString = "USHORT"; break; case NC_INT: typeString = "INT"; break; case NC_UINT: typeString = "UINT"; break; case NC_INT64: typeString = "INT64"; break; case NC_UINT64: typeString = "UINT64"; break; case NC_FLOAT: typeString = "FLOAT"; break; case NC_DOUBLE: typeString = "DOUBLE"; break; case NC_STRING: typeString = "STRING"; break; default: typeString = "UNKNOWN"; } return typeString; } QString NetCDFFilterPrivate::scanAttrs(int ncid, int varid, int attid, QTreeWidgetItem* parentItem) { char name[NC_MAX_NAME + 1]; int nattr, nstart = 0; if (attid == -1) { m_status = nc_inq_varnatts(ncid, varid, &nattr); handleError(m_status, "nc_inq_varnatts"); } else { nstart = attid; nattr = attid+1; } nc_type type; size_t len; QStringList valueString; for (int i = nstart; i < nattr; i++) { valueString.clear(); m_status = nc_inq_attname(ncid, varid, i, name); handleError(m_status, "nc_inq_attname"); m_status = nc_inq_att(ncid, varid, name, &type, &len); handleError(m_status, "nc_inq_att"); QDEBUG(" attr" << i+1 << "name/type/len =" << name << translateDataType(type) << len); //read attribute switch (type) { case NC_BYTE: { signed char *value = (signed char *)malloc(len*sizeof(signed char)); m_status = nc_get_att_schar(ncid, varid, name, value); handleError(m_status, "nc_get_att_schar"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_UBYTE: { unsigned char *value = (unsigned char *)malloc(len*sizeof(unsigned char)); m_status = nc_get_att_uchar(ncid, varid, name, value); handleError(m_status, "nc_get_att_uchar"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_CHAR: { char *value = (char *)malloc((len+1)*sizeof(char)); m_status = nc_get_att_text(ncid, varid, name, value); handleError(m_status, "nc_get_att_text"); value[len] = 0; valueString << value; free(value); break; } case NC_SHORT: { short *value = (short *)malloc(len*sizeof(short)); m_status = nc_get_att_short(ncid, varid, name, value); handleError(m_status, "nc_get_att_short"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_USHORT: { unsigned short *value = (unsigned short *)malloc(len*sizeof(unsigned short)); m_status = nc_get_att_ushort(ncid, varid, name, value); handleError(m_status, "nc_get_att_ushort"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_INT: { int *value = (int *)malloc(len*sizeof(int)); m_status = nc_get_att_int(ncid, varid, name, value); handleError(m_status, "nc_get_att_int"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_UINT: { unsigned int *value = (unsigned int *)malloc(len*sizeof(unsigned int)); m_status = nc_get_att_uint(ncid, varid, name, value); handleError(m_status, "nc_get_att_uint"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_INT64: { long long *value = (long long *)malloc(len*sizeof(long long)); m_status = nc_get_att_longlong(ncid, varid, name, value); handleError(m_status, "nc_get_att_longlong"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_UINT64: { unsigned long long *value = (unsigned long long *)malloc(len*sizeof(unsigned long long)); m_status = nc_get_att_ulonglong(ncid, varid, name, value); handleError(m_status, "nc_get_att_ulonglong"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_FLOAT: { float *value = (float *)malloc(len*sizeof(float)); m_status = nc_get_att_float(ncid, varid, name, value); handleError(m_status, "nc_get_att_float"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_DOUBLE: { double *value = (double *)malloc(len*sizeof(double)); m_status = nc_get_att_double(ncid, varid, name, value); handleError(m_status, "nc_get_att_double"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } default: valueString << "not supported"; } if (parentItem != NULL) { QString typeName; if (varid == NC_GLOBAL) typeName = i18n("global attribute"); else { char varName[NC_MAX_NAME + 1]; m_status = nc_inq_varname(ncid, varid, varName); typeName = i18n("%1 attribute", QString(varName)); } QStringList props; props << translateDataType(type) << " (" << QString::number(len) << ")"; QTreeWidgetItem *attrItem = new QTreeWidgetItem(QStringList() << QString(name) << typeName << props.join("") << valueString.join(", ")); attrItem->setIcon(0, QIcon::fromTheme("accessories-calculator")); attrItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(attrItem); } } return valueString.join("\n"); } void NetCDFFilterPrivate::scanDims(int ncid, int ndims, QTreeWidgetItem* parentItem) { int ulid; m_status = nc_inq_unlimdim(ncid, &ulid); handleError(m_status, "nc_inq_att"); char name[NC_MAX_NAME + 1]; size_t len; for (int i = 0; i < ndims; i++) { m_status = nc_inq_dim(ncid, i, name, &len); handleError(m_status, "nc_inq_att"); DEBUG(" dim" << i+1 << ": name/len =" << name << len); QStringList props; props<setIcon(0, QIcon::fromTheme("accessories-calculator")); attrItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(attrItem); } } void NetCDFFilterPrivate::scanVars(int ncid, int nvars, QTreeWidgetItem* parentItem) { char name[NC_MAX_NAME + 1]; nc_type type; int ndims, nattrs; int dimids[NC_MAX_VAR_DIMS]; for (int i = 0; i < nvars; i++) { m_status = nc_inq_var(ncid, i, name, &type, &ndims, dimids, &nattrs); handleError(m_status, "nc_inq_att"); QDEBUG(" var" << i+1 << ": name/type=" << name << translateDataType(type)); DEBUG(" ndims/nattr" << ndims << nattrs); QStringList props; props << translateDataType(type); char dname[NC_MAX_NAME + 1]; size_t dlen; props << "("; for (int j = 0; j < ndims; j++) { m_status = nc_inq_dim(ncid, dimids[j], dname, &dlen); if (j != 0) props << "x"; props << QString::number(dlen); } props << ")"; QTreeWidgetItem *varItem = new QTreeWidgetItem(QStringList() << QString(name) << i18n("variable") << props.join("")<<""); varItem->setIcon(0, QIcon::fromTheme("x-office-spreadsheet")); varItem->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable); // highlight item for (int c = 0; c < varItem->columnCount(); c++) { varItem->setBackground(c, QColor(192, 255, 192)); varItem->setForeground(c, Qt::black); } parentItem->addChild(varItem); scanAttrs(ncid, i, -1, varItem); } } #endif /*! parses the content of the file \c fileName and fill the tree using rootItem. */ void NetCDFFilterPrivate::parse(const QString & fileName, QTreeWidgetItem* rootItem) { #ifdef HAVE_NETCDF QByteArray bafileName = fileName.toLatin1(); int ncid; m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); int ndims, nvars, nattr, uldid; m_status = nc_inq(ncid, &ndims, &nvars, &nattr, &uldid); handleError(m_status, "nc_inq"); DEBUG(" nattr/ndims/nvars =" << nattr << ndims << nvars); QTreeWidgetItem *attrItem = new QTreeWidgetItem(QStringList() << QString(i18n("Attributes"))); attrItem->setIcon(0,QIcon::fromTheme("folder")); attrItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(attrItem); scanAttrs(ncid, NC_GLOBAL, -1, attrItem); QTreeWidgetItem *dimItem = new QTreeWidgetItem(QStringList() << QString(i18n("Dimensions"))); dimItem->setIcon(0, QIcon::fromTheme("folder")); dimItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(dimItem); scanDims(ncid, ndims, dimItem); QTreeWidgetItem *varItem = new QTreeWidgetItem(QStringList() << QString(i18n("Variables"))); varItem->setIcon(0, QIcon::fromTheme("folder")); varItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(varItem); scanVars(ncid, nvars, varItem); #else Q_UNUSED(fileName) Q_UNUSED(rootItem) #endif } QString NetCDFFilterPrivate::readAttribute(const QString & fileName, const QString & name, const QString & varName) { #ifdef HAVE_NETCDF int ncid; QByteArray bafileName = fileName.toLatin1(); m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); // get varid int varid; if (varName == "global") varid = NC_GLOBAL; else { QByteArray bavarName = varName.toLatin1(); m_status = nc_inq_varid(ncid, bavarName.data(), &varid); handleError(m_status, "nc_inq_varid"); } // attribute 'name' int attid; QByteArray baName = name.toLatin1(); m_status = nc_inq_attid(ncid, varid, baName.data(), &attid); handleError(m_status, "nc_inq_attid"); return scanAttrs(ncid, varid, attid); #else Q_UNUSED(fileName) Q_UNUSED(name) Q_UNUSED(varName) return QString(); #endif } /*! reads the content of the variable in the file \c fileName to a string (for preview) or to the data source. */ QVector NetCDFFilterPrivate::readCurrentVar(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode, int lines) { QVector dataStrings; if (currentVarName.isEmpty()) return dataStrings << (QStringList() << i18n("No variable selected")); QDEBUG(" current variable =" << currentVarName); #ifdef HAVE_NETCDF int ncid; QByteArray bafileName = fileName.toLatin1(); m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); int varid; QByteArray baVarName = currentVarName.toLatin1(); m_status = nc_inq_varid(ncid, baVarName.data(), &varid); handleError(m_status, "nc_inq_varid"); int ndims; nc_type type; m_status = nc_inq_varndims(ncid, varid, &ndims); handleError(m_status, "nc_inq_varndims"); m_status = nc_inq_vartype(ncid, varid, &type); handleError(m_status, "nc_inq_type"); int* dimids = (int *) malloc(ndims * sizeof(int)); m_status = nc_inq_vardimid(ncid, varid, dimids); handleError(m_status, "nc_inq_vardimid"); int actualRows = 0, actualCols = 0; int columnOffset = 0; QVector dataContainer; switch (ndims) { case 0: dataStrings << (QStringList() << i18n("zero dimensions")); qDebug() << dataStrings; break; case 1: { size_t size; m_status = nc_inq_dimlen(ncid, dimids[0], &size); handleError(m_status, "nc_inq_dimlen"); if (endRow == -1) endRow = (int)size; if (lines == -1) lines = endRow; actualRows = endRow-startRow+1; actualCols = 1; DEBUG("start/end row" << startRow << endRow); DEBUG("act rows/cols" << actualRows << actualCols); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); double* data = nullptr; if (dataSource) data = static_cast*>(dataContainer[0])->data(); else data = new double[(unsigned int)actualRows]; size_t start = (size_t)(startRow-1), count = (size_t)actualRows; m_status = nc_get_vara_double(ncid, varid, &start, &count, data); handleError(m_status, "nc_get_vara_double"); if (!dataSource) { for (int i = 0; i < qMin(actualRows, lines); i++) dataStrings << (QStringList() << QString::number(data[i])); delete[] data; } break; } case 2: { size_t rows, cols; m_status = nc_inq_dimlen(ncid, dimids[0], &rows); handleError(m_status, "nc_inq_dimlen"); m_status = nc_inq_dimlen(ncid, dimids[1], &cols); handleError(m_status, "nc_inq_dimlen"); if (endRow == -1) endRow = (int)rows; if (lines == -1) lines = endRow; if (endColumn == -1) endColumn = (int)cols; actualRows = endRow-startRow+1; actualCols = endColumn-startColumn+1; DEBUG("dim =" << rows << "x" << cols); DEBUG("startRow/endRow:" << startRow << endRow); DEBUG("startColumn/endColumn:" << startColumn << endColumn); DEBUG("actual rows/cols:" << actualRows << actualCols); DEBUG("lines:" << lines); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); double** data = (double**) malloc(rows * sizeof(double*)); data[0] = (double*)malloc( cols * rows * sizeof(double) ); for (unsigned int i = 1; i < rows; i++) data[i] = data[0] + i*cols; m_status = nc_get_var_double(ncid, varid, &data[0][0]); handleError(m_status, "nc_get_var_double"); for (int i = 0; i < qMin((int)rows, lines); i++) { QStringList line; for (size_t j = 0; j < cols; j++) { if (dataContainer[0]) static_cast*>(dataContainer[(int)(j-(size_t)startColumn+1)])->operator[](i-startRow+1) = data[i][(int)j]; else line << QString::number(data[i][j]); } dataStrings << line; emit q->completed(100*i/actualRows); } free(data[0]); free(data); break; } default: dataStrings << (QStringList() << i18n("%1 dimensional data of type %2 not supported yet", ndims, translateDataType(type))); qDebug() << dataStrings; } free(dimids); if (dataSource) dataSource->finalizeImport(columnOffset, 1, actualCols, "", mode); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(mode) Q_UNUSED(lines) #endif return dataStrings; } /*! reads the content of the current selected variable from file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ -QVector NetCDFFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode, int lines) { - Q_UNUSED(lines); +QVector NetCDFFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { QVector dataStrings; if (currentVarName.isEmpty()) { DEBUG(" No variable selected"); return dataStrings; } DEBUG(" current variable =" << currentVarName.toStdString()); return readCurrentVar(fileName, dataSource, mode); } /*! writes the content of \c dataSource to the file \c fileName. */ void NetCDFFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: not implemented yet } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void NetCDFFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("netcdfFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool NetCDFFilter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/datasources/filters/NetCDFFilter.h b/src/backend/datasources/filters/NetCDFFilter.h index ea3977411..6ae9c5636 100644 --- a/src/backend/datasources/filters/NetCDFFilter.h +++ b/src/backend/datasources/filters/NetCDFFilter.h @@ -1,73 +1,73 @@ /*************************************************************************** File : NetCDFFilter.h Project : LabPlot Description : NetCDF I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef NETCDFFILTER_H #define NETCDFFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include #include class NetCDFFilterPrivate; class NetCDFFilter : public AbstractFileFilter { Q_OBJECT public: NetCDFFilter(); ~NetCDFFilter() override; void parse(const QString& fileName, QTreeWidgetItem* rootItem); QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace) override; QString readAttribute(const QString & fileName, const QString & name, const QString & varName); QVector readCurrentVar(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*) override; void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setCurrentVarName(const QString&); const QString currentVarName() const; void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class NetCDFFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/NetCDFFilterPrivate.h b/src/backend/datasources/filters/NetCDFFilterPrivate.h index 1ec03378c..7532e0b41 100644 --- a/src/backend/datasources/filters/NetCDFFilterPrivate.h +++ b/src/backend/datasources/filters/NetCDFFilterPrivate.h @@ -1,69 +1,69 @@ /*************************************************************************** File : NetCDFFilterPrivate.h Project : LabPlot Description : Private implementation class for NetCDFFilter. -------------------------------------------------------------------- Copyright : (C) 2015 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef NETCDFFILTERPRIVATE_H #define NETCDFFILTERPRIVATE_H #ifdef HAVE_NETCDF #include #endif class AbstractDataSource; class NetCDFFilterPrivate { public: explicit NetCDFFilterPrivate(NetCDFFilter*); void parse(const QString & fileName, QTreeWidgetItem* rootItem); QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); QString readAttribute(const QString& fileName, const QString& name, const QString& varName); QVector readCurrentVar(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); const NetCDFFilter* q; QString currentVarName; int startRow; int endRow; int startColumn; int endColumn; private: #ifdef HAVE_NETCDF int m_status; void handleError(int status, const QString& function); QString translateDataType(nc_type type); QString scanAttrs(int ncid, int varid, int attid, QTreeWidgetItem* parentItem = nullptr); void scanDims(int ncid, int ndims, QTreeWidgetItem* parentItem); void scanVars(int ncid, int nvars, QTreeWidgetItem* parentItem); #endif }; #endif diff --git a/src/backend/datasources/filters/NgspiceRawAsciiFilter.cpp b/src/backend/datasources/filters/NgspiceRawAsciiFilter.cpp index d34d5e49b..0bd4ddb52 100644 --- a/src/backend/datasources/filters/NgspiceRawAsciiFilter.cpp +++ b/src/backend/datasources/filters/NgspiceRawAsciiFilter.cpp @@ -1,396 +1,395 @@ /*************************************************************************** File : NgspiceRawAsciiFilter.cpp Project : LabPlot Description : Ngspice RAW ASCII filter -------------------------------------------------------------------- Copyright : (C) 2018 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/datasources/filters/NgspiceRawAsciiFilter.h" #include "backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h" #include "backend/lib/trace.h" #include /*! \class NgspiceRawAsciiFilter \brief Import of data stored in Ngspice's raw formant, ASCCI version of it. \ingroup datasources */ NgspiceRawAsciiFilter::NgspiceRawAsciiFilter() : AbstractFileFilter(), d(new NgspiceRawAsciiFilterPrivate(this)) {} NgspiceRawAsciiFilter::~NgspiceRawAsciiFilter() {} bool NgspiceRawAsciiFilter::isNgspiceAsciiFile(const QString& fileName) { QFile file(fileName); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { DEBUG("Failed to open the file " << fileName.toStdString()); return false; } QString line = file.readLine(); if (!line.startsWith(QLatin1String("Title:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("Date:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("Plotname:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("Flags:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("No. Variables:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("No. Points:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("Variables:"))) return false; return true; } QString NgspiceRawAsciiFilter::fileInfoString(const QString& fileName) { QFile file(fileName); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) return QString(); QString info; while (!file.atEnd()) { QString line = file.readLine(); if (line.simplified() == QLatin1String("Values:")) break; if (!info.isEmpty()) info += QLatin1String("
"); info += line; } return info; } /*! reads the content of the file \c fileName. */ -QVector NgspiceRawAsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - d->readDataFromFile(fileName, dataSource, importMode, lines); +QVector NgspiceRawAsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { + d->readDataFromFile(fileName, dataSource, importMode); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector NgspiceRawAsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void NgspiceRawAsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /*! loads the predefined filter settings for \c filterName */ void NgspiceRawAsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void NgspiceRawAsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } void NgspiceRawAsciiFilter::setStartRow(const int r) { d->startRow = r; } int NgspiceRawAsciiFilter::startRow() const { return d->startRow; } void NgspiceRawAsciiFilter::setEndRow(const int r) { d->endRow = r; } int NgspiceRawAsciiFilter::endRow() const { return d->endRow; } void NgspiceRawAsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int NgspiceRawAsciiFilter::startColumn() const { return d->startColumn; } void NgspiceRawAsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int NgspiceRawAsciiFilter::endColumn() const { return d->endColumn; } QStringList NgspiceRawAsciiFilter::vectorNames() const { return d->vectorNames; } QVector NgspiceRawAsciiFilter::columnModes() { return d->columnModes; } //##################################################################### //################### Private implementation ########################## //##################################################################### NgspiceRawAsciiFilterPrivate::NgspiceRawAsciiFilterPrivate(NgspiceRawAsciiFilter* owner) : q(owner), startRow(1), endRow(-1), startColumn(1), endColumn(-1) { } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ -void NgspiceRawAsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - Q_UNUSED(lines); +void NgspiceRawAsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("NgspiceRawAsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " - << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); + << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode)); QFile file(fileName); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { DEBUG("Failed to open the file " << fileName.toStdString()); return; } //skip the first four lines in the header file.readLine(); file.readLine(); file.readLine(); file.readLine(); //number of variables QString line = file.readLine(); const int vars = line.right(line.length() - 15).toInt(); //remove the "No. Variables: " sub-string //number of points line = file.readLine(); const int points = line.right(line.length() - 12).toInt(); //remove the "No. Points: " sub-string //add names of the variables vectorNames.clear(); columnModes.clear(); file.readLine(); for (int i = 0; i points) ? points : endRow; const int actualRows = actualEndRow - startRow + 1; const int actualCols = hasComplexValues ? vars*2 : vars; const int columnOffset = dataSource->prepareImport(m_dataContainer, importMode, actualRows, actualCols, vectorNames, columnModes); //skip data lines, if required DEBUG(" Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { for (int j = 0; j < vars; ++j) file.readLine(); file.readLine(); //skip the empty line after each value block } //read the data points QStringList lineString; int currentRow = 0; // indexes the position in the vector(column) QLocale locale(QLocale::C); bool isNumber(false); for (int i = 0; i < actualEndRow; ++i) { lineString.clear(); for (int j = 0; j < vars; ++j) { line = file.readLine(); QStringList tokens = line.split(QLatin1Char('\t')); QString valueString = tokens.at(1).simplified(); //string containing the value(s) if (hasComplexValues) { QStringList realImgTokens = valueString.split(QLatin1Char(',')); if (realImgTokens.size() == 2) { //sanity check to make sure we really have both parts //real part double value = locale.toDouble(realImgTokens.at(0), &isNumber); static_cast*>(m_dataContainer[2*j])->operator[](currentRow) = (isNumber ? value : NAN); //imaginary part value = locale.toDouble(realImgTokens.at(1), &isNumber); static_cast*>(m_dataContainer[2*j+1])->operator[](currentRow) = (isNumber ? value : NAN); } } else { const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[j])->operator[](currentRow) = (isNumber ? value : NAN); } } file.readLine(); //skip the empty line after each value block currentRow++; emit q->completed(100 * currentRow/actualRows); } dataSource->finalizeImport(columnOffset, startColumn, endColumn, "", importMode); } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector NgspiceRawAsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; QFile file(fileName); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { DEBUG("Failed to open the file " << fileName.toStdString()); return dataStrings; } //skip the first four lines in the header file.readLine(); file.readLine(); file.readLine(); file.readLine(); //number of variables QString line = file.readLine(); const int vars = line.right(line.length() - 15).toInt(); //remove the "No. Variables: " sub-string //number of points line = file.readLine(); const int points = line.right(line.length() - 12).toInt(); //remove the "No. Points: " sub-string //add names of the variables vectorNames.clear(); columnModes.clear(); file.readLine(); for (int i = 0; i readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace) override; void write(const QString& fileName, AbstractDataSource*) override; QVector preview(const QString& fileName, int lines); void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; QStringList vectorNames() const; QVector columnModes(); void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class NgspiceRawAsciiFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h b/src/backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h index 001236eae..c2d4d0d31 100644 --- a/src/backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h +++ b/src/backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h @@ -1,56 +1,56 @@ /*************************************************************************** File : NgspiceRawNgspiceRawAsciiFilterPrivate.h Project : LabPlot Description : Ngspice RAW ASCII filter -------------------------------------------------------------------- Copyright : (C) 2018 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef NGSPICERAWASCIIFILTERPRIVATE_H #define NGSPICERAWASCIIFILTERPRIVATE_H class AbstractDataSource; class NgspiceRawAsciiFilterPrivate { public: explicit NgspiceRawAsciiFilterPrivate(NgspiceRawAsciiFilter*); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); const NgspiceRawAsciiFilter* q; QStringList vectorNames; QVector columnModes; int startRow; int endRow; int startColumn; int endColumn; private: QVector m_dataContainer; // pointers to the actual data containers }; #endif diff --git a/src/backend/datasources/filters/ROOTFilter.cpp b/src/backend/datasources/filters/ROOTFilter.cpp index c87b40125..d454ae69f 100644 --- a/src/backend/datasources/filters/ROOTFilter.cpp +++ b/src/backend/datasources/filters/ROOTFilter.cpp @@ -1,648 +1,646 @@ /*************************************************************************** File : ROOTFilter.cpp Project : LabPlot Description : ROOT(CERN) I/O-filter -------------------------------------------------------------------- Copyright : (C) 2018 by Christoph Roick (chrisito@gmx.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/ROOTFilter.h" #include "backend/datasources/filters/ROOTFilterPrivate.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/Column.h" #include #ifdef HAVE_ZIP #include #include #endif #include #include #include #include #include #include ROOTFilter::ROOTFilter():AbstractFileFilter(), d(new ROOTFilterPrivate) {} ROOTFilter::~ROOTFilter() {} QVector ROOTFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, - AbstractFileFilter::ImportMode importMode, int) { + AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } void ROOTFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } void ROOTFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } void ROOTFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } void ROOTFilter::setCurrentHistogram(const QString& histogram) { d->currentHistogram = histogram; } const QString ROOTFilter::currentHistogram() const { return d->currentHistogram; } QStringList ROOTFilter::listHistograms(const QString& fileName) { return d->listHistograms(fileName); } QVector ROOTFilter::previewCurrentHistogram(const QString& fileName, int first, int last) { return d->previewCurrentHistogram(fileName, first, last); } int ROOTFilter::binsInCurrentHistogram(const QString& fileName) { return d->binsInCurrentHistogram(fileName); } void ROOTFilter::setStartBin(const int s) { d->startBin = s; } int ROOTFilter::startBin() const { return d->startBin; } void ROOTFilter::setEndBin(const int e) { d->endBin = e; } int ROOTFilter::endBin() const { return d->endBin; } void ROOTFilter::setColumns(const int columns) { d->columns = columns; } int ROOTFilter::columns() const { return d->columns; } void ROOTFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("rootFilter"); writer->writeAttribute("startBin", QString::number(d->startBin) ); writer->writeAttribute("endBin", QString::number(d->endBin) ); writer->writeAttribute("columns", QString::number(d->columns) ); writer->writeEndElement(); } bool ROOTFilter::load(XmlStreamReader* reader) { QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); // read attributes QString str = attribs.value("startBin").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startBin'")); else d->startBin = str.toInt(); str = attribs.value("endBin").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endBin'")); else d->endBin = str.toInt(); str = attribs.value("columns").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'columns'")); else d->columns = str.toInt(); return true; } /**************** ROOTFilterPrivate implementation *******************/ ROOTFilterPrivate::ROOTFilterPrivate() {} void ROOTFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, - AbstractFileFilter::ImportMode importMode) { - DEBUG("readDataFromFile()"); + AbstractFileFilter::ImportMode importMode) { + DEBUG("ROOTFilterPrivate::readDataFromFile()"); setFile(fileName); auto bins = readHistogram(); const int nbins = (int)bins.size(); // skip underflow and overflow bins by default int first = qMax(startBin, 0); int last = qMin(endBin, nbins - 1); QStringList colNames = createHeaders(); QVector dataContainer; const int columnOffset = dataSource->prepareImport(dataContainer, importMode, last - first + 1, colNames.size(), - colNames, QVector( - colNames.size(), AbstractColumn::Numeric)); + colNames, QVector(colNames.size(), AbstractColumn::Numeric)); // read data DEBUG("reading " << first - last + 1 << " lines"); int c = 0; if (columns & ROOTFilter::Center) { QVector& container = *static_cast*>(dataContainer[c++]); for (int i = first; i <= last; ++i) container[i - first] = (i > 0 && i < nbins - 1) ? 0.5 * (bins[i].lowedge + bins[i + 1].lowedge) : i == 0 ? bins.front().lowedge // -infinity : -bins.front().lowedge; // +infinity } if (columns & ROOTFilter::Low) { QVector& container = *static_cast*>(dataContainer[c++]); for (int i = first; i <= last; ++i) container[i - first] = bins[i].lowedge; } if (columns & ROOTFilter::Content) { QVector& container = *static_cast*>(dataContainer[c++]); for (int i = first; i <= last; ++i) container[i - first] = bins[i].content; } if (columns & ROOTFilter::Error) { QVector& container = *static_cast*>(dataContainer[c++]); for (int i = first; i <= last; ++i) container[i - first] = std::sqrt(bins[i].sumw2); } dataSource->finalizeImport(columnOffset, 0, colNames.size() - 1, QString(), importMode); } void ROOTFilterPrivate::write(const QString& fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } QStringList ROOTFilterPrivate::listHistograms(const QString& fileName) { setFile(fileName); QStringList histList; for (const auto& hist : currentROOTHist->listHistograms()) { histList << QString::fromStdString(hist); } return histList; } -QVector ROOTFilterPrivate::previewCurrentHistogram(const QString& fileName, - int first, int last) { +QVector ROOTFilterPrivate::previewCurrentHistogram(const QString& fileName, int first, int last) { DEBUG("previewCurrentHistogram()"); setFile(fileName); auto bins = readHistogram(); const int nbins = (int)bins.size(); last = qMin(nbins - 1, last); QVector preview(qMax(last - first + 2, 1)); preview.last() = createHeaders(); // read data DEBUG("reading " << preview.size() << " lines"); if (columns & ROOTFilter::Center) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number( (i > 0 && i < nbins - 1) ? 0.5 * (bins[i].lowedge + bins[i + 1].lowedge) : i == 0 ? bins.front().lowedge // -infinity : -bins.front().lowedge); // +infinity } if (columns & ROOTFilter::Low) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(bins[i].lowedge); } if (columns & ROOTFilter::Content) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(bins[i].content); } if (columns & ROOTFilter::Error) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(std::sqrt(bins[i].sumw2)); } return preview; } int ROOTFilterPrivate::binsInCurrentHistogram(const QString& fileName) { setFile(fileName); QStringList nameindex = currentHistogram.split(';'); bool ok = nameindex.size() > 1; int cycle = ok ? nameindex.last().toInt(&ok) : 1; if (ok) { nameindex.removeLast(); } else { cycle = 1; } return currentROOTHist->histogramBins(nameindex.join(';').toStdString(), cycle); } QStringList ROOTFilterPrivate::createHeaders() { QStringList colNames; if (columns & ROOTFilter::Center) colNames << i18n("Bin Center"); if (columns & ROOTFilter::Low) colNames << i18n("Low Edge"); if (columns & ROOTFilter::Content) colNames << i18n("Content"); if (columns & ROOTFilter::Error) colNames << i18n("Error"); return colNames; } void ROOTFilterPrivate::setFile(const QString& fileName) { if (!currentROOTHist || fileName != currentFile) { currentFile = fileName; currentROOTHist.reset(new ROOTHist(fileName.toStdString())); } } std::vector ROOTFilterPrivate::readHistogram() { QStringList nameindex = currentHistogram.split(';'); bool ok = nameindex.size() > 1; int cycle = ok ? nameindex.last().toInt(&ok) : 1; if (ok) { nameindex.removeLast(); } else { cycle = 1; } return currentROOTHist->readHistogram(nameindex.join(';').toStdString(), cycle); } /******************** ROOTHist implementation ************************/ namespace ROOTHistHelpers { /// Read value from stream template T read(std::ifstream& is) { union { T val; char buf[sizeof(T)]; } r; for (size_t i = 0; i < sizeof(T); ++i) { r.buf[sizeof(T) - i - 1] = is.get(); } return r.val; } /// Read value from buffer template T read(char*& s) { union { T val; char buf[sizeof(T)]; } r; for (size_t i = 0; i < sizeof(T); ++i) { r.buf[sizeof(T) - i - 1] = *(s++); } return r.val; } /// Get version of ROOT object, obtain number of bytes in object short Version(char*& buffer, size_t& count) { count = read(buffer); short version = (count & 0x40000000) ? read(buffer) : read(buffer -= 4); count &= (count & 0x40000000) ? (~0x40000000) : 0; return version; } /// Get version of ROOT object short Version(char*& buffer) { size_t c; return Version(buffer, c); } /// Skip ROOT object void Skip(char*& buffer, const size_t& n) { for (size_t i = 0; i < n; ++i) { char* nbuf = buffer + 4; size_t count; Version(buffer, count); buffer = nbuf + count; } } /// Skip TObject header void SkipObject(char*& buffer) { Version(buffer); buffer += 8; } /// Get TString std::string String(char*& buffer) { size_t s = *(buffer++); if (s == 0) return std::string(); else { if (s == 0xFF) s = read(buffer); buffer += s; return std::string(buffer - s, buffer); } } } using namespace ROOTHistHelpers; ROOTHist::ROOTHist(const std::string& filename) : filename(filename) { // The file structure is described in root/io/io/src/TFile.cxx std::ifstream is(filename, std::ifstream::binary); std::string root(4, 0); is.read(const_cast(root.data()), 4); if (root != "root") return; is.seekg(8); // skip version int pos = read(is); is.seekg(16); int lastpos = read(is); is.seekg(33); compression = read(is); compression = compression > 0 ? compression : 0; while (is.good() && pos < lastpos) { is.seekg(pos); size_t lcdata = read(is); is.seekg(2, is.cur); // short version = read(is); size_t ldata = read(is); is.seekg(4, is.cur); size_t lkey = read(is); short cycle = read(is); is.seekg(8, is.cur); std::string cname(read(is), 0); is.read(&cname[0], cname.size()); if (cname.size() == 4 && cname.substr(0, 3) == "TH1") { KeyBuffer::ContentType type; switch (cname[3]) { case 'D': type = KeyBuffer::ContentType::Double; break; case 'F': type = KeyBuffer::ContentType::Float; break; case 'I': type = KeyBuffer::ContentType::Int; break; case 'S': type = KeyBuffer::ContentType::Short; break; case 'C': type = KeyBuffer::ContentType::Byte; break; default: type = KeyBuffer::ContentType::Invalid; break; } if (type) { std::string name(read(is), 0); is.read(&name[0], name.size()); std::string title(read(is), 0); is.read(&title[0], title.size()); auto it = histkeys.end(); // see root/io/io/src/TKey.cxx for reference if (compression && ldata > 256) { # ifdef HAVE_ZIP // Default: compression level // ZLIB: 100 + compression level // LZ4: 400 + compression level if (compression / 100 <= 1 || compression / 100 == 4) { // see root/core/zip/src/RZip.cxx -> R__unzip std::string lib(2, 0); is.read(&lib[0], 2); char method = is.get(); size_t chcdata = is.get(); chcdata |= (is.get() << 8); chcdata |= (is.get() << 16); size_t chdata = is.get(); chdata |= (is.get() << 8); chdata |= (is.get() << 16); if (chcdata == lcdata - lkey - 9 && chdata == ldata) { if (lib == "ZL" && method == Z_DEFLATED) { it = histkeys.emplace(name + ";" + std::to_string(cycle), KeyBuffer{name, title, cycle, type, KeyBuffer::zlib, pos + lkey + 9, chcdata, chdata, 0} ).first; } else if (lib == "L4" && method == LZ4_versionNumber() / (100 * 100)) { it = histkeys.emplace(name + ";" + std::to_string(cycle), KeyBuffer{name, title, cycle, type, KeyBuffer::lz4, pos + lkey + 9 + 8, chcdata - 8, chdata, 0} ).first; } } } # endif } else { it = histkeys.emplace(name + ";" + std::to_string(cycle), KeyBuffer{name, title, cycle, type, KeyBuffer::none, pos + lkey, ldata, ldata, 0} ).first; } if (it != histkeys.end()) readNBins(it->second); } } pos += (int)lcdata; } } std::vector ROOTHist::listHistograms() const { std::vector l; for (auto& n : histkeys) { l.emplace_back(n.first); } return l; } std::vector ROOTHist::readHistogram(const std::string& name, int cycle) { auto it = histkeys.find(name + ";" + std::to_string(cycle)); if (it == histkeys.end()) return std::vector(); std::string buffer = data(it->second); if (!buffer.empty()) { // The object structure can be retrieved from TFile::GetStreamerInfoList(). // Every ROOT object contains a version number which may include the byte count // for the object. The latter is currently assumed to be present to skip unused // objects. No checks are performed. The corresponding ROOT code is quite nested // but the actual readout is straight forward. // Files for reference: // root/io/io/src/TBufferFile.cxx -> ReadVersion, ReadTString, ReadObjectAny // root/core/cont/src/TArrayD.cxx -> Streamer // root/hist/hist/src/TH1.cxx -> Streamer char *buf = &buffer[0]; size_t count; Version(buf, count); // TH1(D/F/I/S/C) char* dbuf = buf + 4; Version(buf, count); // TH1 dbuf += count; Skip(buf, 4); // skip TNamed, TAttLine, TAttFill, TAttMarker std::vector r(read(buf)); // fNcells if (r.size() < 3) return std::vector(); r.front().lowedge = -std::numeric_limits::infinity(); // x-Axis char* nbuf = buf + 4; Version(buf, count); // TAxis nbuf += count; Skip(buf, 2); // skip TNamed, TAttAxis int nbins = read(buf); double xmin = read(buf); double xmax = read(buf); const size_t nborders = read(buf); if (nborders == r.size() - 1) { for (size_t i = 0; i < nborders; ++i) { r[i + 1].lowedge = read(buf); } } else { buf += sizeof(double) * nbins; const double scale = (xmax - xmin) / static_cast(nbins); for (size_t i = 0; i < r.size() - 1; ++i) { r[i + 1].lowedge = static_cast(i) * scale + xmin; } } buf = nbuf; Skip(buf, 2); // skip y-Axis and z-Axis; buf += 68; // skip 2 shorts and 8 doubles buf += read(buf) * 8; // skip fContour array if (static_cast(read(buf)) == r.size()) { for (auto& b : r) b.sumw2 = read(buf); // always double } buf = dbuf; // skip to contents of TH1(D/F/I/S/C) if (static_cast(read(buf)) == r.size()) { switch (it->second.type) { case KeyBuffer::ContentType::Double: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Float: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Int: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Short: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Byte: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Invalid: // never reached default: break; } } return r; } else return std::vector(); } void ROOTHist::readNBins(ROOTHist::KeyBuffer& kbuffer) { std::string buffer = data(kbuffer); if (!buffer.empty()) { // The object structure can be retrieved from TFile::GetStreamerInfoList(). // Every ROOT object contains a version number which may include the byte count // for the object. The latter is currently assumed to be present to skip unused // objects. No checks are performed. The corresponding ROOT code is quite nested // but the actual readout is straight forward. // Files for reference: // root/io/io/src/TBufferFile.cxx -> ReadVersion, ReadTString, ReadObjectAny // root/core/cont/src/TArrayD.cxx -> Streamer // root/hist/hist/src/TH1.cxx -> Streamer char *buf = &buffer[0]; size_t count; Version(buf, count); // TH1(D/F/I/S/C) char* dbuf = buf + 4; Version(buf, count); // TH1 dbuf += count; Skip(buf, 4); // skip TNamed, TAttLine, TAttFill, TAttMarker kbuffer.nbins = read(buf); // fNcells } } std::string ROOTHist::data(const ROOTHist::KeyBuffer& buffer) const { std::ifstream is(filename, std::ifstream::binary); return data(buffer, is); } std::string ROOTHist::data(const ROOTHist::KeyBuffer& buffer, std::ifstream& is) const { std::string data(buffer.count, 0); is.seekg(buffer.start); if (buffer.compression == KeyBuffer::none) { is.read(&data[0], buffer.count); return data; #ifdef HAVE_ZIP } else if (buffer.compression == KeyBuffer::zlib) { std::string cdata(buffer.compressed_count, 0); is.read(&cdata[0], buffer.compressed_count); size_t luncomp = buffer.count; if (uncompress((Bytef *)data.data(), &luncomp, (Bytef *)cdata.data(), cdata.size()) == Z_OK && data.size() == luncomp) return data; } else { std::string cdata(buffer.compressed_count, 0); is.read(&cdata[0], buffer.compressed_count); if (LZ4_decompress_safe(cdata.data(), const_cast(data.data()), buffer.compressed_count, buffer.count) == static_cast(buffer.count)) return data; #endif } return std::string(); } diff --git a/src/backend/datasources/filters/ROOTFilter.h b/src/backend/datasources/filters/ROOTFilter.h index f4cd085f1..a0f54767b 100644 --- a/src/backend/datasources/filters/ROOTFilter.h +++ b/src/backend/datasources/filters/ROOTFilter.h @@ -1,112 +1,112 @@ /*************************************************************************** File : ROOTFilter.h Project : LabPlot Description : ROOT(CERN) I/O-filter -------------------------------------------------------------------- Copyright : (C) 2018 Christoph Roick (chrisito@gmx.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ROOTFILTER_H #define ROOTFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" class ROOTFilterPrivate; class QStringList; class QIODevice; /// Manages the importing of histograms from ROOT files class ROOTFilter : public AbstractFileFilter { Q_OBJECT public: ROOTFilter(); ~ROOTFilter() override; enum ColumnTypes {Center = 1, Low = 2, Content = 4, Error = 8}; /** * @brief Read data from the currently selected histogram * * The ROOT file is kept open until the file name is changed */ QVector readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, - AbstractFileFilter::ImportMode importMode, int) override; + AbstractFileFilter::ImportMode importMode) override; /// Currently writing to ROOT files is not supported void write(const QString& fileName, AbstractDataSource*) override; void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; /// List names of histograms contained in ROOT file QStringList listHistograms(const QString& fileName); /// Set the current histograms, which is one out of listHistograms void setCurrentHistogram(const QString&); /// Get the name of the currently set histogram const QString currentHistogram() const; /// Get preview data of the currently set histogram QVector previewCurrentHistogram(const QString& fileName, int first, int last); /// Get the number of bins in the current histogram int binsInCurrentHistogram(const QString& fileName); /** * @brief Set the first bin of the histogram to be read * * The default of -1 skips the underflow bin with index 0 */ void setStartBin(const int bin); /// Get the index of the first bin to be read int startBin() const; /** * @brief Set the last bin of the histogram to be read * * The default of -1 skips the overflow bin */ void setEndBin(const int bin); /// Get the index of the last bin to be read int endBin() const; /** * @brief Set the first column of the histogram to be read * * The following columns are available: Bin Center, Content, Error */ void setColumns(const int columns); /// Get the index of the first column to be read int columns() const; /// Save bin limitation settings void save(QXmlStreamWriter*) const override; /// Load bin limitation settings bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class ROOTFilterPrivate; }; #endif