diff --git a/src/backend/datasources/AbstractDataSource.h b/src/backend/datasources/AbstractDataSource.h index e4fd5fee2..74a6a68a4 100644 --- a/src/backend/datasources/AbstractDataSource.h +++ b/src/backend/datasources/AbstractDataSource.h @@ -1,52 +1,52 @@ /*************************************************************************** File : AbstractDataSource.h Project : LabPlot Description : Interface for data sources -------------------------------------------------------------------- Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2015 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ABSTRACTDATASOURCE_H #define ABSTRACTDATASOURCE_H #include "backend/core/AbstractPart.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" #include class QStringList; class AbstractDataSource : public AbstractPart { Q_OBJECT public: AbstractDataSource(const QString& name, AspectType type); ~AbstractDataSource() override = default; void clear(); - virtual int prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode, int actualRows, int actualCols, + virtual int prepareImport(std::vector& dataContainer, AbstractFileFilter::ImportMode, int actualRows, int actualCols, QStringList colNameList = QStringList(), QVector = QVector()) = 0; virtual void finalizeImport(int columnOffset = 0, int startColumn = 0, int endColumn = 0, const QString& dateTimeFormat = QString(), AbstractFileFilter::ImportMode importMode = AbstractFileFilter::Replace) = 0; }; #endif // ABSTRACTDATASOURCE_H diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index b6744ef83..60f07d5ff 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,2651 +1,2651 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2018 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2019 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include "backend/core/Project.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" #include "backend/lib/macros.h" #include "backend/lib/trace.h" #ifdef HAVE_MQTT #include "backend/datasources/MQTTClient.h" #include "backend/datasources/MQTTTopic.h" #endif #include #include #include #if defined(Q_OS_LINUX) || defined(Q_OS_BSD4) #include #include #endif /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(Ascii), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() = default; /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource* dataSource) { d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { return d->readFromLiveDevice(device, dataSource, from); } #ifdef HAVE_MQTT QVector AsciiFilter::preview(const QString& message) { return d->preview(message); } /*! reads the content of a message received by the topic. */ void AsciiFilter::readMQTTTopic(const QString& message, AbstractDataSource* dataSource) { d->readMQTTTopic(message, dataSource); } /*! Returns the statistical data, that the MQTTTopic needs for the will message. */ QString AsciiFilter::MQTTColumnStatistics(const MQTTTopic* topic) const { return d->MQTTColumnStatistics(topic); } /*! Returns the column mode of the last column (the value column of the MQTTTopic). */ AbstractColumn::ColumnMode AsciiFilter::MQTTColumnMode() const { return d->MQTTColumnMode(); } /*! After the MQTTTopic is loaded, prepares the filter for reading. */ void AsciiFilter::setPreparedForMQTT(bool prepared, MQTTTopic* topic, const QString& separator) { d->setPreparedForMQTT(prepared, topic, separator); } #endif /*! returns the separator used by the filter. */ QString AsciiFilter::separator() const { return d->separator(); } /*! returns the separator used by the filter. */ int AsciiFilter::isPrepared() { return d->isPrepared(); } /*! reads the content of the file \c fileName. */ void AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } QVector AsciiFilter::preview(QIODevice& device) { return d->preview(device); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE" << "2xSPACE" << "3xSPACE" << "4xSPACE" << "2xTAB"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } QString AsciiFilter::fileInfoString(const QString& fileName) { QString info(i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName))); info += QLatin1String("
"); info += i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); return info; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " to determine number of lines"); return 0; } // if (!device.canReadLine()) // return -1; size_t lineCount = 0; #if defined(Q_OS_LINUX) || defined(Q_OS_BSD4) //on linux and BSD use wc, if available, which is much faster than counting lines in the file if (device.compressionType() == KCompressionDevice::None && !QStandardPaths::findExecutable(QLatin1String("wc")).isEmpty()) { QProcess wc; wc.start(QLatin1String("wc"), QStringList() << QLatin1String("-l") << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) { QString line(wc.readLine()); // wc on macOS has leading spaces: use SkipEmptyParts lineCount = line.split(' ', QString::SkipEmptyParts)[0].toInt(); } return lineCount; } #endif while (!device.atEnd()) { device.readLine(); lineCount++; } return lineCount; } /*! returns the number of lines in the device \c device and 0 if sequential. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice& device) const { if (device.isSequential()) return 0; // if (!device.canReadLine()) // DEBUG("WARNING in AsciiFilter::lineNumber(): device cannot 'readLine()' but using it anyway."); size_t lineCount = 0; device.seek(0); if (d->readingFile) lineCount = lineNumber(d->readingFileName); else { while (!device.atEnd()) { device.readLine(); lineCount++; } } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } bool AsciiFilter::createIndexEnabled() const { return d->createIndexEnabled; } void AsciiFilter::setCreateTimestampEnabled(bool b) { d->createTimestampEnabled = b; } bool AsciiFilter::createTimestampEnabled() const { return d->createTimestampEnabled; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = std::numeric_limits::quiet_NaN(); } bool AsciiFilter::NaNValueToZeroEnabled() const { return (d->nanValue == 0); } void AsciiFilter::setRemoveQuotesEnabled(bool b) { d->removeQuotesEnabled = b; } bool AsciiFilter::removeQuotesEnabled() const { return d->removeQuotesEnabled; } void AsciiFilter::setVectorNames(const QString& s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner) { } /*! * get a single line from device */ QStringList AsciiFilterPrivate::getLineString(QIODevice& device) { QString line; do { // skip comment lines in data lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::getLineString(): device cannot 'readLine()' but using it anyway."); // line = device.readAll(); line = device.readLine(); } while (!commentCharacter.isEmpty() && line.startsWith(commentCharacter)); line.remove(QRegExp("[\\n\\r]")); // remove any newline DEBUG("data line : \'" << line.toStdString() << '\''); QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); //TODO: remove quotes here? if (simplifyWhitespacesEnabled) { for (int i = 0; i < lineStringList.size(); ++i) lineStringList[i] = lineStringList[i].simplified(); } QDEBUG("data line, parsed: " << lineStringList); return lineStringList; } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("AsciiFilterPrivate::prepareDeviceToRead(): is sequential = " << device.isSequential() << ", can readLine = " << device.canReadLine()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; ///////////////////////////////////////////////////////////////// // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; // skip the comment lines first if (!commentCharacter.isEmpty()) { do { if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); if (device.atEnd()) { DEBUG("device at end! Giving up."); if (device.isSequential()) break; else return 1; } firstLine = device.readLine(); } while (firstLine.startsWith(commentCharacter) || firstLine.simplified().isEmpty()); } // navigate to the line where we asked to start reading from DEBUG(" Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); if (device.atEnd()) { DEBUG("device at end! Giving up."); if (device.isSequential()) break; else return 1; } firstLine = device.readLine(); DEBUG(" line = " << firstLine.toStdString()); } DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (removeQuotesEnabled) firstLine = firstLine.remove(QLatin1Char('"')); //TODO: this doesn't work, the split below introduces whitespaces again // if (simplifyWhitespacesEnabled) // firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) m_separator = firstLine.mid(length1, 1); else m_separator = ' '; } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive); m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); QDEBUG("first line: " << firstLineStringList); DEBUG("headerEnabled: " << headerEnabled); //optionally, remove potential spaces in the first line //TODO: this part should be obsolete actually if we do firstLine = firstLine.simplified(); above... if (simplifyWhitespacesEnabled) { for (int i = 0; i < firstLineStringList.size(); ++i) firstLineStringList[i] = firstLineStringList[i].simplified(); } //in GUI in AsciiOptionsWidget we start counting from 1, subtract 1 here to start from zero m_actualStartRow = startRow - 1; if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); ++m_actualStartRow; } // set range to read if (endColumn == -1) { if (headerEnabled || vectorNames.size() == 0) endColumn = firstLineStringList.size(); // last column else //number of vector names provided in the import dialog (not more than the maximal number of columns in the file) endColumn = qMin(vectorNames.size(), firstLineStringList.size()); } if (endColumn < startColumn) m_actualCols = 0; else m_actualCols = endColumn - startColumn + 1; if (createIndexEnabled) { vectorNames.prepend(i18n("Index")); m_actualCols++; } //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ ///////////////////////////////////////////////////////////////// // parse first data line to determine data type for each column // if the first line was already parsed as the header, read the next line if (headerEnabled && !device.isSequential()) firstLineStringList = getLineString(device); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Integer; col = 1; } for (auto& valueString : firstLineStringList) { // parse columns available in first data line if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } // parsing more lines to better determine data types for (unsigned int i = 0; i < m_dataTypeLines; ++i) { if (device.atEnd()) // EOF reached break; firstLineStringList = getLineString(device); createIndexEnabled ? col = 1 : col = 0; for (auto& valueString : firstLineStringList) { if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); if (col == m_actualCols) break; AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); // numeric: integer -> numeric if (mode == AbstractColumn::Numeric && columnModes[col] == AbstractColumn::Integer) columnModes[col] = mode; // text: non text -> text if (mode == AbstractColumn::Text && columnModes[col] != AbstractColumn::Text) columnModes[col] = mode; col++; } } QDEBUG("column modes = " << columnModes); // ATTENTION: This resets the position in the device to 0 m_actualRows = (int)q->lineNumber(device); const int actualEndRow = (endRow == -1 || endRow > m_actualRows) ? m_actualRows : endRow; if (actualEndRow > m_actualStartRow) m_actualRows = actualEndRow - m_actualStartRow; else m_actualRows = 0; DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << m_actualStartRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0 && !device.isSequential()) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode)); //dirty hack: set readingFile and readingFileName in order to know in lineNumber(QIODevice) //that we're reading from a file and to benefit from much faster wc on linux //TODO: redesign the APIs and remove this later readingFile = true; readingFileName = fileName; KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode); readingFile = false; } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { DEBUG("AsciiFilterPrivate::readFromLiveDevice(): bytes available = " << device.bytesAvailable() << ", from = " << from); if (device.bytesAvailable() <= 0) { DEBUG(" No new data available"); return 0; } //TODO: may be also a matrix? auto* spreadsheet = dynamic_cast(dataSource); if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe) if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return 0; if (!m_prepared) { DEBUG(" Preparing .."); switch (spreadsheet->sourceType()) { case LiveDataSource::SourceType::FileOrPipe: { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG(" Device error = " << deviceError); return 0; } break; } case LiveDataSource::SourceType::NetworkTcpSocket: case LiveDataSource::SourceType::NetworkUdpSocket: case LiveDataSource::SourceType::LocalSocket: case LiveDataSource::SourceType::SerialPort: m_actualRows = 1; if (createIndexEnabled) { m_actualCols = 2; columnModes << AbstractColumn::Integer << AbstractColumn::Numeric; vectorNames << i18n("Index") << i18n("Value"); } else { m_actualCols = 1; columnModes << AbstractColumn::Numeric; vectorNames << i18n("Value"); } QDEBUG(" vector names = " << vectorNames); break; case LiveDataSource::SourceType::MQTT: break; } // prepare import for spreadsheet spreadsheet->setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); //columns in a file data source don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } int keepNValues = spreadsheet->keepNValues(); if (keepNValues == 0) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(keepNValues); m_actualRows = keepNValues; } m_dataContainer.resize(m_actualCols); DEBUG(" data source resized to col: " << m_actualCols); DEBUG(" data source rowCount: " << spreadsheet->rowCount()); DEBUG(" Setting data .."); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } DEBUG(" Prepared!"); } qint64 bytesread = 0; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif LiveDataSource::ReadingType readingType; if (!m_prepared) { readingType = LiveDataSource::ReadingType::TillEnd; } else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = LiveDataSource::ReadingType::TillEnd; //if we read the whole file we just start from the beginning of it //and read till end else if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) readingType = LiveDataSource::ReadingType::TillEnd; else readingType = spreadsheet->readingType(); } DEBUG(" Reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType)); //move to the last read position, from == total bytes read //since the other source types are sequential we cannot seek on them if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleSizeNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != LiveDataSource::ReadingType::TillEnd) newData.resize(spreadsheet->sampleSize()); int newDataIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif DEBUG(" source type = " << ENUM_TO_STRING(LiveDataSource, SourceType, spreadsheet->sourceType())); while (!device.atEnd()) { if (readingType != LiveDataSource::ReadingType::TillEnd) { switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData[newDataIdx++] = device.readAll(); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData[newDataIdx++] = device.read(device.bytesAvailable()); break; case LiveDataSource::SourceType::FileOrPipe: newData.push_back(device.readLine()); break; case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: newData[newDataIdx++] = device.read(device.bytesAvailable()); break; case LiveDataSource::SourceType::MQTT: break; } } else { // ReadingType::TillEnd switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData.push_back(device.readAll()); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData.push_back(device.read(device.bytesAvailable())); break; case LiveDataSource::SourceType::FileOrPipe: newData.push_back(device.readLine()); break; case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: newData.push_back(device.read(device.bytesAvailable())); break; case LiveDataSource::SourceType::MQTT: break; } } newLinesTillEnd++; if (readingType != LiveDataSource::ReadingType::TillEnd) { newLinesForSampleSizeNotTillEnd++; //for Continuous reading and FromEnd we read sample rate number of lines if possible //here TillEnd and Whole file behave the same if (newLinesForSampleSizeNotTillEnd == spreadsheet->sampleSize()) break; } } QDEBUG(" data read: " << newData); } //now we reset the readingType if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = spreadsheet->readingType(); //we had less new lines than the sample size specified if (readingType != LiveDataSource::ReadingType::TillEnd) QDEBUG(" Removed empty lines: " << newData.removeAll(QString())); //back to the last read position before counting when reading from files if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); int currentRow = 0; // indexes the position in the vector(column) int linesToRead = 0; int keepNValues = spreadsheet->keepNValues(); DEBUG(" Increase row count. keepNValues = " << keepNValues); if (m_prepared) { //increase row count if we don't have a fixed size //but only after the preparation step if (keepNValues == 0) { DEBUG(" keep All values"); if (readingType != LiveDataSource::ReadingType::TillEnd) m_actualRows += qMin(newData.size(), spreadsheet->sampleSize()); else { //we don't increase it if we reread the whole file, we reset it if (!(spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)) m_actualRows += newData.size(); else m_actualRows = newData.size(); } //appending if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) linesToRead = m_actualRows; else linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; } else { // fixed size DEBUG(" keep " << keepNValues << " values"); if (readingType == LiveDataSource::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; //TODO after reading we should skip the next data lines //because it's TillEnd actually } else linesToRead = newLinesTillEnd; } else { //we read max sample size number of lines when the reading mode //is ContinuouslyFixed or FromEnd, WholeFile is disabled linesToRead = qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } if (linesToRead == 0) return 0; } else // not prepared linesToRead = newLinesTillEnd; DEBUG(" lines to read = " << linesToRead); DEBUG(" actual rows (w/o header) = " << m_actualRows); //TODO // if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe || spreadsheet->sourceType() == LiveDataSource::SourceType::NetworkUdpSocket) { // if (m_actualRows < linesToRead) { // DEBUG(" SET lines to read to " << m_actualRows); // linesToRead = m_actualRows; // } // } //new rows/resize columns if we don't have a fixed size //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize if (keepNValues == 0) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { // fixed size //when we have a fixed size we have to pop sampleSize number of lines if specified //here popping, setting currentRow if (!m_prepared) { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); } else { if (readingType == LiveDataSource::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) { currentRow = 0; } else { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } } else { //we read max sample size number of lines when the reading mode //is ContinuouslyFixed or FromEnd currentRow = m_actualRows - qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif // enable data change signal for (int col = 0; col < m_actualCols; ++col) spreadsheet->child(col)->setSuppressDataChangedSignal(false); for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet DEBUG(" Reading from line " << currentRow << " till end line " << newLinesTillEnd); DEBUG(" Lines to read:" << linesToRead <<", actual rows:" << m_actualRows << ", actual cols:" << m_actualCols); newDataIdx = 0; if (readingType == LiveDataSource::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->sampleSize()) newDataIdx = newData.size() - spreadsheet->sampleSize(); //since we skip a couple of lines, we need to count those bytes too for (int i = 0; i < newDataIdx; ++i) bytesread += newData.at(i).size(); } } DEBUG(" newDataIdx: " << newDataIdx); static int indexColumnIdx = 1; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif int row = 0; if (readingType == LiveDataSource::ReadingType::TillEnd || (readingType == LiveDataSource::ReadingType::ContinuousFixed)) { if (headerEnabled) { if (!m_prepared) { row = 1; bytesread += newData.at(0).size(); } } } if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { if (readingType == LiveDataSource::ReadingType::WholeFile) { if (headerEnabled) { row = 1; bytesread += newData.at(0).size(); } } } QLocale locale(numberFormat); for (; row < linesToRead; ++row) { DEBUG("\n Reading row " << row + 1 << " of " << linesToRead); QString line; if (readingType == LiveDataSource::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(row); //when we read the whole file we don't care about the previous position //so we don't have to count those bytes if (readingType != LiveDataSource::ReadingType::WholeFile) { if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { bytesread += line.size(); } } if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines continue; QStringList lineStringList; // only FileOrPipe support multiple columns if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); else lineStringList << line; QDEBUG(" line = " << lineStringList << ", separator = \'" << m_separator << "\'"); DEBUG(" Line bytes: " << line.size() << " line: " << line.toStdString()); if (simplifyWhitespacesEnabled) { for (int i = 0; i < lineStringList.size(); ++i) lineStringList[i] = lineStringList[i].simplified(); } if (createIndexEnabled) { if (spreadsheet->keepNValues() == 0) lineStringList.prepend(QString::number(currentRow + 1)); else lineStringList.prepend(QString::number(indexColumnIdx++)); } QDEBUG(" column modes = " << columnModes); for (int n = 0; n < m_actualCols; ++n) { DEBUG(" actual col = " << n); if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); DEBUG(" value string = " << valueString.toStdString()); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { DEBUG(" Numeric"); bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); // qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::Integer: { DEBUG(" Integer"); bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); // qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { DEBUG(" missing columns in this line"); switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow).clear(); break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); //determine the dependent plots QVector plots; for (int n = 0; n < m_actualCols; ++n) spreadsheet->column(n)->addUsedInPlots(plots); //suppress retransform in the dependent plots for (auto* plot : plots) plot->setSuppressDataChangedSignal(true); for (int n = 0; n < m_actualCols; ++n) spreadsheet->column(n)->setChanged(); //retransform the dependent plots for (auto* plot : plots) { plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } else m_prepared = true; DEBUG("AsciiFilterPrivate::readFromLiveDevice() DONE"); return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return; } // matrix data has only one column mode if (dynamic_cast(dataSource)) { auto mode = columnModes[0]; //TODO: remove this when Matrix supports text type if (mode == AbstractColumn::Text) mode = AbstractColumn::Numeric; for (auto& c : columnModes) if (c != mode) c = mode; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; //skip data lines, if required DEBUG(" Skipping " << m_actualStartRow << " lines"); for (int i = 0; i < m_actualStartRow; ++i) device.readLine(); DEBUG(" Reading " << qMin(lines, m_actualRows) << " lines, " << m_actualCols << " columns"); if (qMin(lines, m_actualRows) == 0 || m_actualCols == 0) return; QString line; QString valueString; //Don't put the definition QStringList lineStringList outside of the for-loop, //the compile doesn't seem to omptimize the desctructor of QList well enough in this case. for (int i = 0; i < qMin(lines, m_actualRows); ++i) { line = device.readLine(); // remove any newline line.remove(QLatin1Char('\n')); line.remove(QLatin1Char('\r')); if (removeQuotesEnabled) line.remove(QLatin1Char('"')); if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines continue; QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); // DEBUG(" Line bytes: " << line.size() << " line: " << line.toStdString()); if (simplifyWhitespacesEnabled) { for (int i = 0; i < lineStringList.size(); ++i) lineStringList[i] = lineStringList[i].simplified(); } // remove left white spaces if (skipEmptyParts) { for (int n = 0; n < lineStringList.size(); ++n) { valueString = lineStringList.at(n); if (!QString::compare(valueString, " ")) { lineStringList.removeAt(n); n--; } } } for (int n = 0; n < m_actualCols; ++n) { // index column if required if (n == 0 && createIndexEnabled) { static_cast*>(m_dataContainer[0])->operator[](currentRow) = i + 1; continue; } //column counting starts with 1, subtract 1 as well as another 1 for the index column if required int col = createIndexEnabled ? n + startColumn - 2: n + startColumn - 1; if (col < lineStringList.size()) { valueString = lineStringList.at(col); // set value depending on data type - switch (columnModes[n]) { + switch (columnModes.at(n)) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: { auto* colData = static_cast*>(m_dataContainer[n]); colData->operator[](currentRow) = valueString; break; } case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else { // missing columns in this line - switch (columnModes[n]) { + switch (columnModes.at(n)) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow).clear(); break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } DEBUG(" Read " << currentRow << " lines"); //we might have skipped empty lines above. shrink the spreadsheet if the number of read lines (=currentRow) //is smaller than the initial size of the spreadsheet (=m_actualRows). //TODO: should also be relevant for Matrix auto* s = dynamic_cast(dataSource); if (s && currentRow != m_actualRows && importMode == AbstractFileFilter::Replace) s->setRowCount(currentRow); dataSource->finalizeImport(m_columnOffset, startColumn, startColumn + m_actualCols - 1, dateTimeFormat, importMode); } /*! * preview for special devices (local/UDP/TCP socket or serial port) */ QVector AsciiFilterPrivate::preview(QIODevice &device) { DEBUG("AsciiFilterPrivate::preview(): bytesAvailable = " << device.bytesAvailable() << ", isSequential = " << device.isSequential()); QVector dataStrings; if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return dataStrings; } if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return dataStrings; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif int linesToRead = 0; QVector newData; //TODO: serial port "read(nBytes)"? while (!device.atEnd()) { if (device.canReadLine()) newData.push_back(device.readLine()); else // UDP fails otherwise newData.push_back(device.readAll()); linesToRead++; } QDEBUG(" data = " << newData); if (linesToRead == 0) return dataStrings; int col = 0; int colMax = newData.at(0).size(); if (createIndexEnabled) colMax++; columnModes.resize(colMax); if (createIndexEnabled) { columnModes[0] = AbstractColumn::ColumnMode::Integer; col = 1; vectorNames.prepend(i18n("Index")); } vectorNames.append(i18n("Value")); QDEBUG(" vector names = " << vectorNames); for (const auto& valueString : newData.at(0).split(' ', QString::SkipEmptyParts)) { if (col == colMax) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } QString line; QLocale locale(numberFormat); QStringList lineString; for (int i = 0; i < linesToRead; ++i) { line = newData.at(i); // remove any newline line = line.remove('\n'); line = line.remove('\r'); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines continue; QStringList lineStringList = line.split(' ', QString::SkipEmptyParts); if (createIndexEnabled) lineStringList.prepend(QString::number(i + 1)); for (int n = 0; n < lineStringList.size(); ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 16); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QString(); } dataStrings << lineString; } return dataStrings; } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; //dirty hack: set readingFile and readingFileName in order to know in lineNumber(QIODevice) //that we're reading from a file and to benefit from much faster wc on linux //TODO: redesign the APIs and remove this later readingFile = true; readingFileName = fileName; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); readingFile = false; if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; // set column names for preview if (!headerEnabled) { int start = 0; if (createIndexEnabled) start = 1; for (int i = start; i < m_actualCols; i++) vectorNames << "Column " + QString::number(i + 1); } QDEBUG(" column names = " << vectorNames); //skip data lines, if required DEBUG(" Skipping " << m_actualStartRow << " lines"); for (int i = 0; i < m_actualStartRow; ++i) device.readLine(); DEBUG(" Generating preview for " << qMin(lines, m_actualRows) << " lines"); QString line; for (int i = 0; i < qMin(lines, m_actualRows); ++i) { line = device.readLine(); // remove any newline line = line.remove('\n'); line = line.remove('\r'); if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines continue; QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); QDEBUG(" line = " << lineStringList); DEBUG(" Line bytes: " << line.size() << " line: " << line.toStdString()); if (simplifyWhitespacesEnabled) { for (int i = 0; i < lineStringList.size(); ++i) lineStringList[i] = lineStringList[i].simplified(); } QStringList lineString; for (int n = 0; n < m_actualCols; ++n) { // index column if required if (n == 0 && createIndexEnabled) { lineString += QString::number(i + 1); continue; } //column counting starts with 1, subtract 1 as well as another 1 for the index column if required int col = createIndexEnabled ? n + startColumn - 2: n + startColumn - 1; if (col < lineStringList.size()) { QString valueString = lineStringList.at(col); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); //DEBUG(" valueString = " << valueString.toStdString()); if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces continue; // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 15); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QString(); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to ascii file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement( "asciiFilter"); writer->writeAttribute( "commentCharacter", d->commentCharacter); writer->writeAttribute( "separatingCharacter", d->separatingCharacter); writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute( "createTimestamp", QString::number(d->createTimestampEnabled)); writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); writer->writeAttribute( "nanValue", QString::number(d->nanValue)); writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled)); writer->writeAttribute( "startRow", QString::number(d->startRow)); writer->writeAttribute( "endRow", QString::number(d->endRow)); writer->writeAttribute( "startColumn", QString::number(d->startColumn)); writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str; READ_STRING_VALUE("commentCharacter", commentCharacter); READ_STRING_VALUE("separatingCharacter", separatingCharacter); READ_INT_VALUE("createIndex", createIndexEnabled, bool); READ_INT_VALUE("createTimestamp", createTimestampEnabled, bool); READ_INT_VALUE("autoMode", autoModeEnabled, bool); READ_INT_VALUE("header", headerEnabled, bool); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty READ_INT_VALUE("simplifyWhitespaces", simplifyWhitespacesEnabled, bool); READ_DOUBLE_VALUE("nanValue", nanValue); READ_INT_VALUE("removeQuotes", removeQuotesEnabled, bool); READ_INT_VALUE("skipEmptyParts", skipEmptyParts, bool); READ_INT_VALUE("startRow", startRow, int); READ_INT_VALUE("endRow", endRow, int); READ_INT_VALUE("startColumn", startColumn, int); READ_INT_VALUE("endColumn", endColumn, int); return true; } int AsciiFilterPrivate::isPrepared() { return m_prepared; } #ifdef HAVE_MQTT int AsciiFilterPrivate::prepareToRead(const QString& message) { QStringList lines = message.split('\n'); if (lines.isEmpty()) return 1; // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine = lines.at(0); if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts); } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive); m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); QDEBUG("first line: " << firstLineStringList); //all columns are read plus the optional column for the index and for the timestamp m_actualCols = firstLineStringList.size() + int(createIndexEnabled) + int(createTimestampEnabled); //column names: //when reading the message strings for different topics, it's not possible to specify vector names //since the different topics can have different content and different number of columns/vectors //->we always set the vector names here to fixed values vectorNames.clear(); columnModes.clear(); //add index column if (createIndexEnabled) { vectorNames << i18n("index"); columnModes << AbstractColumn::Integer; } //add timestamp column if (createTimestampEnabled) { vectorNames << i18n("timestamp"); columnModes << AbstractColumn::DateTime; } //parse the first data line to determine data type for each column int i = 1; for (auto& valueString : firstLineStringList) { if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); vectorNames << i18n("value %1", i); columnModes << AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); ++i; } m_actualStartRow = startRow; m_actualRows = lines.size(); QDEBUG("column modes = " << columnModes); DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows); return 0; } /*! * generates the preview for the string \s message. */ QVector AsciiFilterPrivate::preview(const QString& message) { QVector dataStrings; prepareToRead(message); //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data QStringList lines = message.split('\n'); int i = 0; for (auto line : lines) { if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) // skip empty or commented lines continue; const QStringList& lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); QDEBUG(" line = " << lineStringList); QStringList lineString; // index column if required if (createIndexEnabled) lineString += QString::number(i + 1); // timestamp column if required if (createTimestampEnabled) lineString += QDateTime::currentDateTime().toString(); int offset = int(createIndexEnabled) + int(createTimestampEnabled); for (int n = 0; n < m_actualCols - offset; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); //DEBUG(" valueString = " << valueString.toStdString()); if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces continue; // set value depending on data type switch (columnModes[n+offset]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 15); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QString(); } ++i; dataStrings << lineString; } return dataStrings; } /*! * \brief Returns the statistical data that is needed by the topic for its MQTTClient's will message * \param topic */ QString AsciiFilterPrivate::MQTTColumnStatistics(const MQTTTopic* topic) const { Column* const tempColumn = topic->child(m_actualCols - 1); QString statistics; QVector willStatistics = topic->mqttClient()->willStatistics(); //Add every statistical data to the string, the flag of which is set true for (int i = 0; i <= willStatistics.size(); i++) { if (willStatistics[i]) { switch (static_cast(i) ) { case MQTTClient::WillStatisticsType::ArithmeticMean: statistics += QLatin1String("Arithmetic mean: ") + QString::number(tempColumn->statistics().arithmeticMean) + "\n"; break; case MQTTClient::WillStatisticsType::ContraharmonicMean: statistics += QLatin1String("Contraharmonic mean: ") + QString::number(tempColumn->statistics().contraharmonicMean) + "\n"; break; case MQTTClient::WillStatisticsType::Entropy: statistics += QLatin1String("Entropy: ") + QString::number(tempColumn->statistics().entropy) + "\n"; break; case MQTTClient::WillStatisticsType::GeometricMean: statistics += QLatin1String("Geometric mean: ") + QString::number(tempColumn->statistics().geometricMean) + "\n"; break; case MQTTClient::WillStatisticsType::HarmonicMean: statistics += QLatin1String("Harmonic mean: ") + QString::number(tempColumn->statistics().harmonicMean) + "\n"; break; case MQTTClient::WillStatisticsType::Kurtosis: statistics += QLatin1String("Kurtosis: ") + QString::number(tempColumn->statistics().kurtosis) + "\n"; break; case MQTTClient::WillStatisticsType::Maximum: statistics += QLatin1String("Maximum: ") + QString::number(tempColumn->statistics().maximum) + "\n"; break; case MQTTClient::WillStatisticsType::MeanDeviation: statistics += QLatin1String("Mean deviation: ") + QString::number(tempColumn->statistics().meanDeviation) + "\n"; break; case MQTTClient::WillStatisticsType::MeanDeviationAroundMedian: statistics += QLatin1String("Mean deviation around median: ") + QString::number(tempColumn->statistics().meanDeviationAroundMedian) + "\n"; break; case MQTTClient::WillStatisticsType::Median: statistics += QLatin1String("Median: ") + QString::number(tempColumn->statistics().median) + "\n"; break; case MQTTClient::WillStatisticsType::MedianDeviation: statistics += QLatin1String("Median deviation: ") + QString::number(tempColumn->statistics().medianDeviation) + "\n"; break; case MQTTClient::WillStatisticsType::Minimum: statistics += QLatin1String("Minimum: ") + QString::number(tempColumn->statistics().minimum) + "\n"; break; case MQTTClient::WillStatisticsType::Skewness: statistics += QLatin1String("Skewness: ") + QString::number(tempColumn->statistics().skewness) + "\n"; break; case MQTTClient::WillStatisticsType::StandardDeviation: statistics += QLatin1String("Standard deviation: ") + QString::number(tempColumn->statistics().standardDeviation) + "\n"; break; case MQTTClient::WillStatisticsType::Variance: statistics += QLatin1String("Variance: ") + QString::number(tempColumn->statistics().variance) + "\n"; break; case MQTTClient::WillStatisticsType::NoStatistics: default: break; } } } return statistics; } AbstractColumn::ColumnMode AsciiFilterPrivate::MQTTColumnMode() const { return columnModes[m_actualCols - 1]; } /*! * \brief reads the content of a message received by the topic. * Uses the settings defined in the MQTTTopic's MQTTClient * \param message * \param topic * \param dataSource */ void AsciiFilterPrivate::readMQTTTopic(const QString& message, AbstractDataSource* dataSource) { //If the message is empty, there is nothing to do if (message.isEmpty()) { DEBUG("No new data available"); return; } MQTTTopic* spreadsheet = dynamic_cast(dataSource); const int keepNValues = spreadsheet->mqttClient()->keepNValues(); if (!m_prepared) { qDebug()<<"Start preparing filter for: " << spreadsheet->topicName(); //Prepare the filter const int mqttPrepareError = prepareToRead(message); if (mqttPrepareError != 0) { DEBUG("Mqtt Prepare Error = " << mqttPrepareError); qDebug()<setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); qDebug() << "fds resized to col: " << m_actualCols; qDebug() << "fds rowCount: " << spreadsheet->rowCount(); //columns in a MQTTTopic don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } if (keepNValues == 0) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues()); m_actualRows = spreadsheet->mqttClient()->keepNValues(); } m_dataContainer.resize(m_actualCols); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif MQTTClient::ReadingType readingType; if (!m_prepared) { //if filter is not prepared we read till the end readingType = MQTTClient::ReadingType::TillEnd; } else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (static_cast (spreadsheet->mqttClient()->readingType()) == MQTTClient::ReadingType::FromEnd) readingType = MQTTClient::ReadingType::TillEnd; else readingType = spreadsheet->mqttClient()->readingType(); } //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleSizeNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != MQTTClient::ReadingType::TillEnd) { newData.reserve(spreadsheet->mqttClient()->sampleSize()); newData.resize(spreadsheet->mqttClient()->sampleSize()); } int newDataIdx = 0; //TODO: bool sampleSizeReached = false; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif QStringList newDataList = message.split(QRegExp("\n|\r\n|\r"), QString::SkipEmptyParts); for (auto& line : newDataList) { newData.push_back(line); newLinesTillEnd++; if (readingType != MQTTClient::ReadingType::TillEnd) { newLinesForSampleSizeNotTillEnd++; //for Continuous reading and FromEnd we read sample rate number of lines if possible if (newLinesForSampleSizeNotTillEnd == spreadsheet->mqttClient()->sampleSize()) { //TODO: sampleSizeReached = true; break; } } } } qDebug()<<"Processing message done"; //now we reset the readingType if (spreadsheet->mqttClient()->readingType() == MQTTClient::ReadingType::FromEnd) readingType = static_cast(spreadsheet->mqttClient()->readingType()); //we had less new lines than the sample rate specified if (readingType != MQTTClient::ReadingType::TillEnd) qDebug() << "Removed empty lines: " << newData.removeAll(QString()); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); if (m_prepared ) { if (keepNValues == 0) m_actualRows = spreadsheetRowCountBeforeResize; else { //if the keepNValues changed since the last read we have to manage the columns accordingly if (m_actualRows != spreadsheet->mqttClient()->keepNValues()) { if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues()); qDebug()<<"rowcount set to: " << spreadsheet->mqttClient()->keepNValues(); } //Calculate the difference between the old and new keepNValues int rowDiff = 0; if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) rowDiff = m_actualRows - spreadsheet->mqttClient()->keepNValues(); if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) rowDiff = spreadsheet->mqttClient()->keepNValues() - m_actualRows; for (int n = 0; n < columnModes.size(); ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); m_dataContainer[n] = static_cast(vector); //if the keepNValues got smaller then we move the last keepNValues count of data //in the first keepNValues places if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) { for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) { static_cast*>(m_dataContainer[n])->operator[] (i) = static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); } } //if the keepNValues got bigger we move the existing values to the last m_actualRows positions //then fill the remaining lines with NaN if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { vector->reserve( spreadsheet->mqttClient()->keepNValues()); vector->resize( spreadsheet->mqttClient()->keepNValues()); for (int i = 1; i <= m_actualRows; i++) { static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); } for (int i = 0; i < rowDiff; i++) static_cast*>(m_dataContainer[n])->operator[](i) = nanValue; } break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); m_dataContainer[n] = static_cast(vector); //if the keepNValues got smaller then we move the last keepNValues count of data //in the first keepNValues places if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) { for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) { static_cast*>(m_dataContainer[n])->operator[] (i) = static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); } } //if the keepNValues got bigger we move the existing values to the last m_actualRows positions //then fill the remaining lines with 0 if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { vector->reserve( spreadsheet->mqttClient()->keepNValues()); vector->resize( spreadsheet->mqttClient()->keepNValues()); for (int i = 1; i <= m_actualRows; i++) { static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); } for (int i = 0; i < rowDiff; i++) static_cast*>(m_dataContainer[n])->operator[](i) = 0; } break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); m_dataContainer[n] = static_cast(vector); //if the keepNValues got smaller then we move the last keepNValues count of data //in the first keepNValues places if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) { for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) { static_cast*>(m_dataContainer[n])->operator[] (i) = static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); } } //if the keepNValues got bigger we move the existing values to the last m_actualRows positions //then fill the remaining lines with empty lines if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { vector->reserve( spreadsheet->mqttClient()->keepNValues()); vector->resize( spreadsheet->mqttClient()->keepNValues()); for (int i = 1; i <= m_actualRows; i++) { static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); } for (int i = 0; i < rowDiff; i++) static_cast*>(m_dataContainer[n])->operator[](i).clear(); } break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); m_dataContainer[n] = static_cast(vector); //if the keepNValues got smaller then we move the last keepNValues count of data //in the first keepNValues places if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) { for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) { static_cast*>(m_dataContainer[n])->operator[] (i) = static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); } } //if the keepNValues got bigger we move the existing values to the last m_actualRows positions //then fill the remaining lines with null datetime if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { vector->reserve( spreadsheet->mqttClient()->keepNValues()); vector->resize( spreadsheet->mqttClient()->keepNValues()); for (int i = 1; i <= m_actualRows; i++) { static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); } for (int i = 0; i < rowDiff; i++) static_cast*>(m_dataContainer[n])->operator[](i) = QDateTime(); } break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } //if the keepNValues got smaller resize the spreadsheet if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues()); //set the new row count m_actualRows = spreadsheet->mqttClient()->keepNValues(); qDebug()<<"actual rows: "<mqttClient()->sampleSize()); else { m_actualRows += newData.size(); } } //fixed size if (keepNValues != 0) { if (readingType == MQTTClient::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; } else linesToRead = newLinesTillEnd; } else { //we read max sample size number of lines when the reading mode //is ContinuouslyFixed or FromEnd if (spreadsheet->mqttClient()->sampleSize() <= spreadsheet->mqttClient()->keepNValues()) linesToRead = qMin(spreadsheet->mqttClient()->sampleSize(), newLinesTillEnd); else linesToRead = qMin(spreadsheet->mqttClient()->keepNValues(), newLinesTillEnd); } } else linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; if (linesToRead == 0) return; } else { if (keepNValues != 0) linesToRead = newLinesTillEnd > m_actualRows ? m_actualRows : newLinesTillEnd; else linesToRead = newLinesTillEnd; } qDebug()<<"linestoread = " << linesToRead; //new rows/resize columns if we don't have a fixed size if (keepNValues == 0) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { //when we have a fixed size we have to pop sampleSize number of lines if specified //here popping, setting currentRow if (!m_prepared) currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); else { if (readingType == MQTTClient::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinuouslyFixed or FromEnd currentRow = m_actualRows - linesToRead; } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet qDebug() << "reading from line: " << currentRow << " lines till end: " << newLinesTillEnd; qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows; newDataIdx = 0; //From end means that we read the last sample size amount of data if (readingType == MQTTClient::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->mqttClient()->sampleSize()) newDataIdx = newData.size() - spreadsheet->mqttClient()->sampleSize(); } } qDebug() << "newDataIdx: " << newDataIdx; //read the data static int indexColumnIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif int row = 0; QLocale locale(numberFormat); for (; row < linesToRead; ++row) { QString line; if (readingType == MQTTClient::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(row); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || (!commentCharacter.isEmpty() && line.startsWith(commentCharacter))) continue; //add index if required int offset = 0; if (createIndexEnabled) { int index = (keepNValues != 0) ? indexColumnIdx++ : currentRow; static_cast*>(m_dataContainer[0])->operator[](currentRow) = index; ++offset; } //add current timestamp if required if (createTimestampEnabled) { static_cast*>(m_dataContainer[offset])->operator[](currentRow) = QDateTime::currentDateTime(); ++offset; } //parse the columns QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); qDebug()<<"########################################################################"; qDebug()<*>(m_dataContainer[col])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[col])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[col])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); static_cast*>(m_dataContainer[col])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { DEBUG(" missing columns in this line"); switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[col])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[col])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[col])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[col])->operator[](currentRow).clear(); break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); const Project* project = spreadsheet->project(); QVector curves = project->children(AbstractAspect::Recursive); QVector plots; for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); //determine the plots where the column is consumed for (const auto* curve : curves) { if (curve->xColumn() == column || curve->yColumn() == column) { CartesianPlot* plot = dynamic_cast(curve->parentAspect()); if (plots.indexOf(plot) == -1) { plots << plot; plot->setSuppressDataChangedSignal(true); } } } column->setChanged(); } //loop over all affected plots and retransform them for (auto* const plot : plots) { //TODO setting this back to true triggers again a lot of retransforms in the plot (one for each curve). // plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } else m_prepared = true; DEBUG("AsciiFilterPrivate::readFromMQTTTopic() DONE"); } /*! * \brief After the MQTTTopic was loaded, the filter is prepared for reading * \param prepared * \param topic * \param separator */ void AsciiFilterPrivate::setPreparedForMQTT(bool prepared, MQTTTopic* topic, const QString& separator) { m_prepared = prepared; //If originally it was prepared we have to restore the settings if (prepared) { m_separator = separator; m_actualCols = endColumn - startColumn + 1; m_actualRows = topic->rowCount(); //set the column modes columnModes.resize(topic->columnCount()); for (int i = 0; i < topic->columnCount(); ++i) { columnModes[i] = topic->column(i)->columnMode(); } //set the data containers m_dataContainer.resize(m_actualCols); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) topic->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(topic->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(topic->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(topic->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(topic->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } #endif /*! * \brief Returns the separator used by the filter * \return */ QString AsciiFilterPrivate::separator() const { return m_separator; } diff --git a/src/backend/datasources/filters/AsciiFilterPrivate.h b/src/backend/datasources/filters/AsciiFilterPrivate.h index 5afa8dcc9..3b4023915 100644 --- a/src/backend/datasources/filters/AsciiFilterPrivate.h +++ b/src/backend/datasources/filters/AsciiFilterPrivate.h @@ -1,109 +1,109 @@ /*************************************************************************** File : AsciiFilterPrivate.h Project : LabPlot Description : Private implementation class for AsciiFilter. -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTERPRIVATE_H #define ASCIIFILTERPRIVATE_H class KFilterDev; class AbstractDataSource; class AbstractColumn; class AbstractAspect; class Spreadsheet; class MQTTTopic; class AsciiFilterPrivate { public: explicit AsciiFilterPrivate(AsciiFilter*); QStringList getLineString(QIODevice&); int prepareDeviceToRead(QIODevice&); void readDataFromDevice(QIODevice&, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); qint64 readFromLiveDevice(QIODevice&, AbstractDataSource*, qint64 from = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); QVector preview(QIODevice& device); QString separator() const; #ifdef HAVE_MQTT int prepareToRead(const QString&); QVector preview(const QString& message); AbstractColumn::ColumnMode MQTTColumnMode() const; QString MQTTColumnStatistics(const MQTTTopic*) const; void readMQTTTopic(const QString& message, AbstractDataSource*); void setPreparedForMQTT(bool, MQTTTopic*, const QString&); #endif const AsciiFilter* q; QString commentCharacter{'#'}; QString separatingCharacter{QStringLiteral("auto")}; QString dateTimeFormat; QLocale::Language numberFormat{QLocale::C}; bool autoModeEnabled{true}; bool headerEnabled{true}; bool skipEmptyParts{false}; bool simplifyWhitespacesEnabled{false}; double nanValue{NAN}; bool removeQuotesEnabled{false}; bool createIndexEnabled{false}; bool createTimestampEnabled{true}; QStringList vectorNames; QVector columnModes; int startRow{1}; int endRow{-1}; int startColumn{1}; int endColumn{-1}; int mqttPreviewFirstEmptyColCount{0}; int isPrepared(); //TODO: redesign and remove this later bool readingFile{false}; QString readingFileName; private: static const unsigned int m_dataTypeLines = 10; // maximum lines to read for determining data types QString m_separator; int m_actualStartRow{1}; int m_actualRows{0}; int m_actualCols{0}; int m_prepared{false}; int m_columnOffset{0}; // indexes the "start column" in the datasource. Data will be imported starting from this column. - QVector m_dataContainer; // pointers to the actual data containers + std::vector m_dataContainer; // pointers to the actual data containers }; #endif diff --git a/src/backend/datasources/filters/BinaryFilter.cpp b/src/backend/datasources/filters/BinaryFilter.cpp index 9ec725747..f81f94071 100644 --- a/src/backend/datasources/filters/BinaryFilter.cpp +++ b/src/backend/datasources/filters/BinaryFilter.cpp @@ -1,613 +1,613 @@ /*************************************************************************** File : BinaryFilter.cpp Project : LabPlot Description : Binary I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2018 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/BinaryFilter.h" #include "backend/datasources/filters/BinaryFilterPrivate.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include #include /*! \class BinaryFilter \brief Manages the import/export of data organized as columns (vectors) from/to a binary file. \ingroup datasources */ BinaryFilter::BinaryFilter():AbstractFileFilter(Binary), d(new BinaryFilterPrivate(this)) {} BinaryFilter::~BinaryFilter() = default; /*! reads the content of the file \c fileName. */ void BinaryFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } /*! reads the content of the device \c device. */ void BinaryFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } QVector BinaryFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void BinaryFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! returns the list of all predefined data formats. */ QStringList BinaryFilter::dataTypes() { return (QStringList() <<"int8 (8 bit signed integer)" <<"int16 (16 bit signed integer)" <<"int32 (32 bit signed integer)" <<"int64 (64 bit signed integer)" <<"uint8 (8 bit unsigned integer)" <<"uint16 (16 bit unsigned integer)" <<"uint32 (32 bit unsigned integer)" <<"uint64 (64 bit unsigned integer)" <<"real32 (single precision floats)" <<"real64 (double precision floats)" ); } /*! returns the size of the predefined data types */ int BinaryFilter::dataSize(BinaryFilter::DataType type) { std::array sizes = {1,2,4,8,1,2,4,8,4,8}; return sizes[(int)type]; } /*! returns the number of rows (length of vectors) in the file \c fileName. */ size_t BinaryFilter::rowNumber(const QString& fileName, const size_t vectors, const BinaryFilter::DataType type) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) return 0; size_t rows = 0; while (!device.atEnd()) { // one row for (size_t i = 0; i < vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(type); ++j) device.read(1); } rows++; } return rows; } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void BinaryFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void BinaryFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void BinaryFilter::setVectors(const size_t v) { d->vectors = v; } size_t BinaryFilter::vectors() const { return d->vectors; } void BinaryFilter::setDataType(const BinaryFilter::DataType t) { d->dataType = t; } BinaryFilter::DataType BinaryFilter::dataType() const { return d->dataType; } void BinaryFilter::setByteOrder(const QDataStream::ByteOrder b) { d->byteOrder = b; } QDataStream::ByteOrder BinaryFilter::byteOrder() const { return d->byteOrder; } void BinaryFilter::setSkipStartBytes(const size_t s) { d->skipStartBytes = s; } size_t BinaryFilter::skipStartBytes() const { return d->skipStartBytes; } void BinaryFilter::setStartRow(const int s) { d->startRow = s; } int BinaryFilter::startRow() const { return d->startRow; } void BinaryFilter::setEndRow(const int e) { d->endRow = e; } int BinaryFilter::endRow() const { return d->endRow; } void BinaryFilter::setSkipBytes(const size_t s) { d->skipBytes = s; } size_t BinaryFilter::skipBytes() const { return d->skipBytes; } void BinaryFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void BinaryFilter::setAutoModeEnabled(bool b) { d->autoModeEnabled = b; } bool BinaryFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } QString BinaryFilter::fileInfoString(const QString& fileName) { DEBUG("BinaryFilter::fileInfoString()"); QString info; //TODO Q_UNUSED(fileName); return info; } //##################################################################### //################### Private implementation ########################## //##################################################################### BinaryFilterPrivate::BinaryFilterPrivate(BinaryFilter* owner) : q(owner) {} /*! reads the content of the device \c device to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ void BinaryFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("readDataFromFile()"); KFilterDev device(fileName); numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); if (! device.open(QIODevice::ReadOnly)) { DEBUG(" could not open file " << fileName.toStdString()); return; } readDataFromDevice(device, dataSource, importMode); } /*! * returns 1 if the current read position in the device is at the end and 0 otherwise. */ int BinaryFilterPrivate::prepareStreamToRead(QDataStream& in) { DEBUG("prepareStreamToRead()"); in.setByteOrder(byteOrder); // catch case that skipStartBytes or startRow is bigger than file if (skipStartBytes >= BinaryFilter::dataSize(dataType) * vectors * numRows || startRow > (int)numRows) return 1; // skip bytes at start for (size_t i = 0; i < skipStartBytes; ++i) { qint8 tmp; in >> tmp; } // skip until start row for (size_t i = 0; i < (startRow-1) * vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(dataType); ++j) { qint8 tmp; in >> tmp; } } // set range of rows if (endRow == -1) m_actualRows = (int)numRows - startRow + 1; else if (endRow > (int)numRows - startRow + 1) m_actualRows = (int)numRows; else m_actualRows = endRow - startRow + 1; m_actualCols = (int)vectors; DEBUG("numRows = " << numRows); DEBUG("endRow = " << endRow); DEBUG("actual rows = " << m_actualRows); DEBUG("actual cols = " << m_actualCols); return 0; } /*! reads \c lines lines of the device \c device and return as string for preview. */ QVector BinaryFilterPrivate::preview(const QString& fileName, int lines) { DEBUG("BinaryFilterPrivate::preview( " << fileName.toStdString() << ", " << lines << ")"); QVector dataStrings; KFilterDev device(fileName); if (! device.open(QIODevice::ReadOnly)) return dataStrings << (QStringList() << i18n("could not open device")); numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); QDataStream in(&device); const int deviceError = prepareStreamToRead(in); if (deviceError) return dataStrings << (QStringList() << i18n("data selection empty")); //TODO: support other modes columnModes.resize(m_actualCols); //TODO: use given names QStringList vectorNames; if (createIndexEnabled) vectorNames.prepend(i18n("Index")); if (lines == -1) lines = m_actualRows; // read data DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(m_actualRows, lines); ++i) { QStringList lineString; //prepend the index if required if (createIndexEnabled) lineString << QString::number(i+1); for (int n = 0; n < m_actualCols; ++n) { //TODO: use ColumnMode when it supports all types switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT16: { qint16 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT32: { qint32 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT64: { qint64 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT8: { quint8 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT16: { quint16 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT32: { quint32 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT64: { quint64 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::REAL32: { float value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::REAL64: { double value; in >> value; lineString << QString::number(value); break; } } } dataStrings << lineString; emit q->completed(100*i/m_actualRows); } return dataStrings; } /*! reads the content of the file \c fileName to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ void BinaryFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("BinaryFilterPrivate::readDataFromDevice()"); QDataStream in(&device); const int deviceError = prepareStreamToRead(in); if (deviceError) { dataSource->clear(); DEBUG("device error"); return; } if (createIndexEnabled) m_actualCols++; - QVector dataContainer; + std::vector dataContainer; int columnOffset = 0; //TODO: support other modes columnModes.resize(m_actualCols); //TODO: use given names QStringList vectorNames; if (createIndexEnabled) { vectorNames.prepend(i18n("Index")); columnModes[0] = AbstractColumn::Integer; } columnOffset = dataSource->prepareImport(dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); if (lines == -1) lines = m_actualRows; // start column int startColumn = 0; if (createIndexEnabled) startColumn++; // read data DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(m_actualRows, lines); ++i) { DEBUG("reading row " << i); //prepend the index if required if (createIndexEnabled) static_cast*>(dataContainer[0])->operator[](i) = i+1; for (int n = startColumn; n < m_actualCols; ++n) { DEBUG("reading column " << n); //TODO: use ColumnMode when it supports all types switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT16: { qint16 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT32: { qint32 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT64: { qint64 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT8: { quint8 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT16: { quint16 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT32: { quint32 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT64: { quint64 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::REAL32: { float value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::REAL64: { double value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } } } if (m_actualRows > 0) emit q->completed(100*i/m_actualRows); } dataSource->finalizeImport(columnOffset, 1, m_actualCols, QString(), importMode); } /*! writes the content of \c dataSource to the file \c fileName. */ void BinaryFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: writing binary files not supported yet } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void BinaryFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("binaryFilter"); writer->writeAttribute("vectors", QString::number(d->vectors) ); writer->writeAttribute("dataType", QString::number(d->dataType) ); writer->writeAttribute("byteOrder", QString::number(d->byteOrder) ); writer->writeAttribute("autoMode", QString::number(d->autoModeEnabled) ); writer->writeAttribute("startRow", QString::number(d->startRow) ); writer->writeAttribute("endRow", QString::number(d->endRow) ); writer->writeAttribute("skipStartBytes", QString::number(d->skipStartBytes) ); writer->writeAttribute("skipBytes", QString::number(d->skipBytes) ); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled) ); writer->writeEndElement(); } /*! Loads from XML. */ bool BinaryFilter::load(XmlStreamReader* reader) { KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); // read attributes QString str = attribs.value("vectors").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("vectors").toString()); else d->vectors = (size_t)str.toULong(); str = attribs.value("dataType").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("dataType").toString()); else d->dataType = (BinaryFilter::DataType) str.toInt(); str = attribs.value("byteOrder").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("byteOrder").toString()); else d->byteOrder = (QDataStream::ByteOrder) str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("autoMode").toString()); else d->autoModeEnabled = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startRow").toString()); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endRow").toString()); else d->endRow = str.toInt(); str = attribs.value("skipStartBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipStartBytes").toString()); else d->skipStartBytes = (size_t)str.toULong(); str = attribs.value("skipBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipBytes").toString()); else d->skipBytes = (size_t)str.toULong(); str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("createIndex").toString()); else d->createIndexEnabled = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/FITSFilter.cpp b/src/backend/datasources/filters/FITSFilter.cpp index e07a69c53..47cf4c577 100644 --- a/src/backend/datasources/filters/FITSFilter.cpp +++ b/src/backend/datasources/filters/FITSFilter.cpp @@ -1,1604 +1,1603 @@ /*************************************************************************** File : FITSFilter.cpp Project : LabPlot Description : FITS I/O-filter -------------------------------------------------------------------- Copyright : (C) 2016 by Fabian Kristof (fkristofszabolcs@gmail.com) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "FITSFilter.h" #include "FITSFilterPrivate.h" #include "backend/core/column/Column.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/datatypes/Double2StringFilter.h" #include "backend/matrix/MatrixModel.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/matrix/Matrix.h" #include "commonfrontend/matrix/MatrixView.h" #include #include #include /*! \class FITSFilter * \brief Manages the import/export of data from/to a FITS file. * \since 2.2.0 * \ingroup datasources */ FITSFilter::FITSFilter():AbstractFileFilter(FITS), d(new FITSFilterPrivate(this)) {} FITSFilter::~FITSFilter() = default; void FITSFilter::readDataFromFile(const QString &fileName, AbstractDataSource *dataSource, AbstractFileFilter::ImportMode importMode) { d->readCHDU(fileName, dataSource, importMode); } QVector FITSFilter::readChdu(const QString &fileName, bool* okToMatrix, int lines) { return d->readCHDU(fileName, nullptr, AbstractFileFilter::Replace, okToMatrix, lines); } void FITSFilter::write(const QString &fileName, AbstractDataSource *dataSource) { d->writeCHDU(fileName, dataSource); } void FITSFilter::addNewKeyword(const QString &filename, const QList &keywords) { d->addNewKeyword(filename, keywords); } void FITSFilter::updateKeywords(const QString &fileName, const QList& originals, const QVector& updates) { d->updateKeywords(fileName, originals, updates); } void FITSFilter::deleteKeyword(const QString &fileName, const QList& keywords) { d->deleteKeyword(fileName, keywords); } void FITSFilter::addKeywordUnit(const QString &fileName, const QList &keywords) { d->addKeywordUnit(fileName, keywords); } void FITSFilter::removeExtensions(const QStringList &extensions) { d->removeExtensions(extensions); } void FITSFilter::parseHeader(const QString &fileName, QTableWidget *headerEditTable, bool readKeys, const QList &keys) { d->parseHeader(fileName, headerEditTable, readKeys, keys); } void FITSFilter::parseExtensions(const QString &fileName, QTreeWidget *tw, bool checkPrimary) { d->parseExtensions(fileName, tw, checkPrimary); } QList FITSFilter::chduKeywords(const QString &fileName) { return d->chduKeywords(fileName); } void FITSFilter::loadFilterSettings(const QString& fileName) { Q_UNUSED(fileName) } void FITSFilter::saveFilterSettings(const QString& fileName) const { Q_UNUSED(fileName) } /*! * \brief contains the {StandardKeywords \ MandatoryKeywords} keywords * \return A list of keywords */ QStringList FITSFilter::standardKeywords() { return QStringList() << QLatin1String("(blank)") << QLatin1String("CROTA") << QLatin1String("EQUINOX") << QLatin1String("NAXIS") << QLatin1String("TBCOL") << QLatin1String("TUNIT") << QLatin1String("AUTHOR") << QLatin1String("CRPIX") << QLatin1String("EXTEND") << QLatin1String("OBJECT") << QLatin1String("TDIM") << QLatin1String("TZERO") << QLatin1String("BITPIX") << QLatin1String("CRVAL") << QLatin1String("EXTLEVEL") << QLatin1String("OBSERVER") << QLatin1String("TDISP") << QLatin1String("XTENSION") << QLatin1String("BLANK") << QLatin1String("CTYPE") << QLatin1String("EXTNAME") << QLatin1String("ORIGIN") << QLatin1String("TELESCOP") << QLatin1String("BLOCKED") << QLatin1String("DATAMAX") << QLatin1String("EXTVER") << QLatin1String("BSCALE") << QLatin1String("DATAMIN") << QLatin1String("PSCAL") << QLatin1String("TFORM") << QLatin1String("BUNIT") << QLatin1String("DATE") << QLatin1String("GROUPS") << QLatin1String("PTYPE") << QLatin1String("THEAP") << QLatin1String("BZERO") << QLatin1String("DATE-OBS") << QLatin1String("HISTORY") << QLatin1String("PZERO") << QLatin1String("TNULL") << QLatin1String("CDELT") << QLatin1String("INSTRUME") << QLatin1String("REFERENC") << QLatin1String("TSCAL") << QLatin1String("COMMENT") << QLatin1String("EPOCH") << QLatin1String("NAXIS") << QLatin1String("SIMPLE") << QLatin1String("TTYPE"); } /*! * \brief Returns a list of keywords, that are mandatory for an image extension of a FITS file * see: * https://archive.stsci.edu/fits/fits_standard/node64.html * \return A list of keywords */ QStringList FITSFilter::mandatoryImageExtensionKeywords() { return QStringList() << QLatin1String("XTENSION") << QLatin1String("BITPIX") << QLatin1String("NAXIS") << QLatin1String("PCOUNT") << QLatin1String("GCOUNT") << QLatin1String("END"); } /*! * \brief Returns a list of keywords, that are mandatory for a table extension (ascii or bintable) * of a FITS file * see: * https://archive.stsci.edu/fits/fits_standard/node58.html * https://archive.stsci.edu/fits/fits_standard/node68.html * \return A list of keywords */ QStringList FITSFilter::mandatoryTableExtensionKeywords() { return QStringList() << QLatin1String("XTENSION") << QLatin1String("BITPIX") << QLatin1String("NAXIS") << QLatin1String("NAXIS1") << QLatin1String("NAXIS2") << QLatin1String("PCOUNT") << QLatin1String("GCOUNT") << QLatin1String("TFIELDS") << QLatin1String("END"); } /*! * \brief Returns a list of strings that represent units which are used for autocompletion when adding * keyword units to keywords * \return A list of strings that represent units */ QStringList FITSFilter::units() { return QStringList() << QLatin1String("m (Metre)") << QLatin1String("kg (Kilogram)") << QLatin1String("s (Second)") << QString("M☉ (Solar mass)") << QLatin1String("AU (Astronomical unit") << QLatin1String("l.y (Light year)") << QLatin1String("km (Kilometres") << QLatin1String("pc (Parsec)") << QLatin1String("K (Kelvin)") << QLatin1String("mol (Mole)") << QLatin1String("cd (Candela)"); } /*! * \brief Sets the startColumn to \a column * \param column the column to be set */ void FITSFilter::setStartColumn(const int column) { d->startColumn = column; } /*! * \brief Returns startColumn * \return The startColumn */ int FITSFilter::startColumn() const { return d->startColumn; } /*! * \brief Sets the endColumn to \a column * \param column the column to be set */ void FITSFilter::setEndColumn(const int column) { d->endColumn = column; } /*! * \brief Returns endColumn * \return The endColumn */ int FITSFilter::endColumn() const { return d->endColumn; } /*! * \brief Sets the startRow to \a row * \param row the row to be set */ void FITSFilter::setStartRow(const int row) { d->startRow = row; } /*! * \brief Returns startRow * \return The startRow */ int FITSFilter::startRow() const { return d->startRow; } /*! * \brief Sets the endRow to \a row * \param row the row to be set */ void FITSFilter::setEndRow(const int row) { d->endRow = row; } /*! * \brief Returns endRow * \return The endRow */ int FITSFilter::endRow() const { return d->endRow; } /*! * \brief Sets commentsAsUnits to \a commentsAsUnits * * This is used when spreadsheets are exported to FITS table extensions and comments are used as the * units of the table's columns. * \param commentsAsUnits */ void FITSFilter::setCommentsAsUnits(const bool commentsAsUnits) { d->commentsAsUnits = commentsAsUnits; } /*! * \brief Sets exportTo to \a exportTo * * This is used to decide whether the container should be exported to a FITS image or a FITS table * For an image \a exportTo should be 0, for a table 1 * \param exportTo */ void FITSFilter::setExportTo(const int exportTo) { d->exportTo = exportTo; } QString FITSFilter::fileInfoString(const QString& fileName) { const int imagesCount = FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("IMAGES")).size(); QString info(i18n("Images: %1", QString::number(imagesCount))); info += QLatin1String("
"); const int tablesCount = FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("TABLES")).size(); info += i18n("Tables: %1", QString::number(tablesCount)); return info; } //##################################################################### //################### Private implementation ########################## //##################################################################### FITSFilterPrivate::FITSFilterPrivate(FITSFilter* owner) : q(owner) {} /*! * \brief Read the current header data unit from file \a filename in data source \a dataSource in \a importMode import mode * \param fileName the name of the file to be read * \param dataSource the data source to be filled * \param importMode */ QVector FITSFilterPrivate::readCHDU(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, bool* okToMatrix, int lines) { DEBUG("FITSFilterPrivate::readCHDU() file name = " << fileName.toStdString()); QVector dataStrings; #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status)) { DEBUG(" ERROR opening file " << fileName.toStdString()); printError(status); return dataStrings; } int chduType; if (fits_get_hdu_type(m_fitsFile, &chduType, &status)) { printError(status); return dataStrings; } long actualRows; int actualCols; int columnOffset = 0; bool noDataSource = (dataSource == nullptr); if (chduType == IMAGE_HDU) { DEBUG("IMAGE_HDU"); int maxdim = 2; int bitpix; int naxis; long naxes[2]; if (fits_get_img_param(m_fitsFile, maxdim, &bitpix, &naxis, naxes, &status)) { printError(status); return dataStrings; } if (naxis == 0) return dataStrings; actualRows = naxes[1]; actualCols = naxes[0]; if (lines == -1) lines = actualRows; else { if (lines > actualRows) lines = actualRows; } if (endRow != -1) { if (!noDataSource) lines = endRow; } if (endColumn != -1) actualCols = endColumn; if (noDataSource) dataStrings.reserve(lines); int i = 0; int j = 0; if (startRow != 1) i = startRow; if (startColumn != 1) j = startColumn; const int jstart = j; //TODO: support other modes QVector columnModes; columnModes.resize(actualCols - j); QStringList vectorNames; - QVector dataContainer; + std::vector dataContainer; if (!noDataSource) { dataContainer.reserve(actualCols - j); columnOffset = dataSource->prepareImport(dataContainer, importMode, lines - i, actualCols - j, vectorNames, columnModes); } long pixelCount = lines * actualCols; double* data = new double[pixelCount]; if (!data) { qDebug() << "Not enough memory for data"; return dataStrings; } if (fits_read_img(m_fitsFile, TDOUBLE, 1, pixelCount, nullptr, data, nullptr, &status)) { printError(status); return dataStrings << (QStringList() << QString("Error")); } int ii = 0; DEBUG(" Import " << lines << " lines"); for (; i < lines; ++i) { int jj = 0; QStringList line; line.reserve(actualCols - j); for (; j < actualCols; ++j) { if (noDataSource) line << QString::number(data[i*naxes[0] +j]); else static_cast*>(dataContainer[jj++])->operator[](ii) = data[i* naxes[0] + j]; } dataStrings << line; j = jstart; ii++; } delete[] data; if (dataSource) dataSource->finalizeImport(columnOffset, 1, actualCols, QString(), importMode); fits_close_file(m_fitsFile, &status); return dataStrings; } else if ((chduType == ASCII_TBL) || (chduType == BINARY_TBL)) { DEBUG("ASCII_TBL or BINARY_TBL"); if (endColumn != -1) actualCols = endColumn; else fits_get_num_cols(m_fitsFile, &actualCols, &status); if (endRow != -1) actualRows = endRow; else fits_get_num_rows(m_fitsFile, &actualRows, &status); QStringList columnNames; QList columnsWidth; QStringList columnUnits; columnUnits.reserve(actualCols); columnsWidth.reserve(actualCols); columnNames.reserve(actualCols); int colWidth; char keyword[FLEN_KEYWORD]; char value[FLEN_VALUE]; int col = 1; if (startColumn != 1) { if (startColumn != 0) col = startColumn; } for (; col <= actualCols; ++col) { status = 0; fits_make_keyn("TTYPE", col, keyword, &status); fits_read_key(m_fitsFile, TSTRING, keyword, value, nullptr, &status); columnNames.append(QLatin1String(value)); fits_make_keyn("TUNIT", col, keyword, &status); fits_read_key(m_fitsFile, TSTRING, keyword, value, nullptr, &status); columnUnits.append(QLatin1String(value)); fits_get_col_display_width(m_fitsFile, col, &colWidth, &status); columnsWidth.append(colWidth); } status = 0; if (lines == -1) lines = actualRows; else if (lines > actualRows) lines = actualRows; if (endRow != -1) lines = endRow; QVector stringDataPointers; - QVector numericDataPointers; + std::vector numericDataPointers; QList columnNumericTypes; int startCol = 0; if (startColumn != 1) startCol = startColumn; int startRrow = 0; if (startRow != 1) startRrow = startRow; columnNumericTypes.reserve(actualCols); int datatype; int c = 1; if (startColumn != 1) { if (startColumn != 0) c = startColumn; } QList matrixNumericColumnIndices; for (; c <= actualCols; ++c) { fits_get_coltype(m_fitsFile, c, &datatype, nullptr, nullptr, &status); switch (datatype) { case TSTRING: columnNumericTypes.append(false); break; case TSHORT: columnNumericTypes.append(true); break; case TLONG: columnNumericTypes.append(true); break; case TFLOAT: columnNumericTypes.append(true); break; case TDOUBLE: columnNumericTypes.append(true); break; case TLOGICAL: columnNumericTypes.append(false); break; case TBIT: columnNumericTypes.append(true); break; case TBYTE: columnNumericTypes.append(true); break; case TCOMPLEX: columnNumericTypes.append(true); break; default: columnNumericTypes.append(false); break; } if ((datatype != TSTRING) && (datatype != TLOGICAL)) matrixNumericColumnIndices.append(c); } if (noDataSource) *okToMatrix = matrixNumericColumnIndices.isEmpty() ? false : true; if (!noDataSource) { DEBUG("HAS DataSource"); auto* spreadsheet = dynamic_cast(dataSource); if (spreadsheet) { numericDataPointers.reserve(actualCols - startCol); stringDataPointers.reserve(actualCols - startCol); spreadsheet->setUndoAware(false); columnOffset = spreadsheet->resize(importMode, columnNames, actualCols - startCol); if (importMode == AbstractFileFilter::Replace) { spreadsheet->clear(); spreadsheet->setRowCount(lines - startRrow); } else { if (spreadsheet->rowCount() < (lines - startRrow)) spreadsheet->setRowCount(lines - startRrow); } DEBUG("Reading columns ..."); for (int n = 0; n < actualCols - startCol; ++n) { if (columnNumericTypes.at(n)) { spreadsheet->column(columnOffset+ n)->setColumnMode(AbstractColumn::Numeric); auto* datap = static_cast* >(spreadsheet->column(columnOffset+n)->data()); numericDataPointers.push_back(datap); if (importMode == AbstractFileFilter::Replace) datap->clear(); } else { spreadsheet->column(columnOffset+ n)->setColumnMode(AbstractColumn::Text); auto* list = static_cast(spreadsheet->column(columnOffset+n)->data()); stringDataPointers.push_back(list); if (importMode == AbstractFileFilter::Replace) list->clear(); } } DEBUG(" ... DONE"); stringDataPointers.squeeze(); } else { numericDataPointers.reserve(matrixNumericColumnIndices.size()); columnOffset = dataSource->prepareImport(numericDataPointers, importMode, lines - startRrow, matrixNumericColumnIndices.size()); } - numericDataPointers.squeeze(); } int row = 1; if (startRow != 1) { if (startRow != 0) row = startRow; } int coll = 1; if (startColumn != 1) { if (startColumn != 0) coll = startColumn; } bool isMatrix = false; if (dynamic_cast(dataSource)) { isMatrix = true; coll = matrixNumericColumnIndices.first(); actualCols = matrixNumericColumnIndices.last(); if (importMode == AbstractFileFilter::Replace) { for (auto* col : numericDataPointers) static_cast*>(col)->clear(); } } char array[FLEN_VALUE]; char* tmpArr[1] = {array}; for (; row <= lines; ++row) { int numericixd = 0; int stringidx = 0; QStringList line; line.reserve(actualCols-coll); for (int col = coll; col <= actualCols; ++col) { if (isMatrix) { if (!matrixNumericColumnIndices.contains(col)) continue; } if (fits_read_col_str(m_fitsFile, col, row, 1, 1, nullptr, tmpArr, nullptr, &status)) printError(status); if (!noDataSource) { QString str = QString::fromLatin1(array); if (str.isEmpty()) { if (columnNumericTypes.at(col - 1)) static_cast*>(numericDataPointers[numericixd++])->push_back(0); else stringDataPointers[stringidx++]->append(QLatin1String("NULL")); } else { if (columnNumericTypes.at(col - 1)) static_cast*>(numericDataPointers[numericixd++])->push_back(str.toDouble()); else { if (!stringDataPointers.isEmpty()) stringDataPointers[stringidx++]->operator<<(str.simplified()); } } } else { QString tmpColstr = QString::fromLatin1(array); tmpColstr = tmpColstr.simplified(); if (tmpColstr.isEmpty()) line << QLatin1String("NULL"); else line << tmpColstr; } } dataStrings << line; } if (!noDataSource) dataSource->finalizeImport(columnOffset, 1, actualCols, QString(), importMode); fits_close_file(m_fitsFile, &status); return dataStrings; } else qDebug() << "Incorrect header type"; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(importMode) Q_UNUSED(okToMatrix) Q_UNUSED(lines) #endif return dataStrings; } /*! * \brief Export from data source \a dataSource to file \a fileName * \param fileName the name of the file to be exported to * \param dataSource the datasource whose data is exported */ void FITSFilterPrivate::writeCHDU(const QString &fileName, AbstractDataSource *dataSource) { #ifdef HAVE_FITS if (!fileName.endsWith(QLatin1String(".fits"))) return; int status = 0; bool existed = false; if (!QFile::exists(fileName)) { if (fits_create_file(&m_fitsFile, fileName.toLatin1(), &status)) { printError(status); qDebug() << fileName; return; } } else { if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } else existed = true; } Matrix* const matrix = dynamic_cast(dataSource); if (matrix) { //FITS image if (exportTo == 0) { long naxes[2] = { matrix->columnCount(), matrix->rowCount() }; if (fits_create_img(m_fitsFile, FLOAT_IMG, 2, naxes, &status)) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); return; } const long nelem = naxes[0] * naxes[1]; double* const array = new double[nelem]; const QVector >* const data = static_cast>*>(matrix->data()); for (int col = 0; col < naxes[0]; ++col) for (int row = 0; row < naxes[1]; ++row) array[row * naxes[0] + col] = data->at(row).at(col); if (fits_write_img(m_fitsFile, TDOUBLE, 1, nelem, array, &status )) { printError(status); status = 0; } fits_close_file(m_fitsFile, &status); delete[] array; //FITS table } else { const int nrows = matrix->rowCount(); const int tfields = matrix->columnCount(); QVector columnNames; columnNames.resize(tfields); columnNames.squeeze(); QVector tform; tform.resize(tfields); tform.squeeze(); //TODO: mode const QVector>* const matrixData = static_cast>*>(matrix->data()); const MatrixModel* matrixModel = static_cast(matrix->view())->model(); const int precision = matrix->precision(); for (int i = 0; i < tfields; ++i) { const QString& columnName = matrixModel->headerData(i, Qt::Horizontal).toString(); columnNames[i] = new char[columnName.size() + 1]; strcpy(columnNames[i], columnName.toLatin1().constData()); int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (matrix->text(row, i).size() > maxSize) maxSize = matrix->text(row, i).size(); } QString tformn; if (precision > 0) { tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".") + QString::number(precision); } else tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".0"); tform[i] = new char[tformn.size() + 1]; strcpy(tform[i], tformn.toLatin1().constData()); } //TODO extension name containing[] ? int r = fits_create_tbl(m_fitsFile, ASCII_TBL, nrows, tfields, columnNames.data(), tform.data(), nullptr, matrix->name().toLatin1().constData(), &status); for (int i = 0; i < tfields; ++i) { delete[] tform[i]; delete[] columnNames[i]; } if (r) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } double* columnNumeric = new double[nrows]; for (int col = 1; col <= tfields; ++col) { const QVector& column = matrixData->at(col-1); for (int r = 0; r < column.size(); ++r) columnNumeric[r] = column.at(r); fits_write_col(m_fitsFile, TDOUBLE, col, 1, 1, nrows, columnNumeric, &status); if (status) { printError(status); delete[] columnNumeric; status = 0; if (!existed) { QFile file(fileName); file.remove(); } fits_close_file(m_fitsFile, &status); return; } } delete[] columnNumeric; fits_close_file(m_fitsFile, &status); } return; } auto* const spreadsheet = dynamic_cast(dataSource); if (spreadsheet) { //FITS image if (exportTo == 0) { int maxRowIdx = -1; //don't export lots of empty lines if all of those contain nans // TODO: option? for (int c = 0; c < spreadsheet->columnCount(); ++c) { const Column* const col = spreadsheet->column(c); int currMaxRoxIdx = -1; for (int r = col->rowCount(); r >= 0; --r) { if (col->isValid(r)) { currMaxRoxIdx = r; break; } } if (currMaxRoxIdx > maxRowIdx) { maxRowIdx = currMaxRoxIdx; } } long naxes[2] = { spreadsheet->columnCount(), maxRowIdx + 1}; if (fits_create_img(m_fitsFile, FLOAT_IMG, 2, naxes, &status)) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } const long nelem = naxes[0] * naxes[1]; double* array = new double[nelem]; for (int row = 0; row < naxes[1]; ++row) { for (int col = 0; col < naxes[0]; ++col) array[row * naxes[0] + col] = spreadsheet->column(col)->valueAt(row); } if (fits_write_img(m_fitsFile, TDOUBLE, 1, nelem, array, &status )) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } fits_close_file(m_fitsFile, &status); delete[] array; } else { const int nrows = spreadsheet->rowCount(); const int tfields = spreadsheet->columnCount(); QVector columnNames; columnNames.resize(tfields); columnNames.squeeze(); QVector tform; tform.resize(tfields); tform.squeeze(); QVector tunit; tunit.resize(tfields); tunit.squeeze(); for (int i = 0; i < tfields; ++i) { const Column* const column = spreadsheet->column(i); columnNames[i] = new char[column->name().size() + 1]; strcpy(columnNames[i], column->name().toLatin1().constData()); if (commentsAsUnits) { tunit[i] = new char[column->comment().size() + 1]; strcpy(tunit[i], column->comment().toLatin1().constData()); } else { tunit[i] = new char[1]; tunit[i][0] = '\0'; } switch (column->columnMode()) { case AbstractColumn::Numeric: { int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (QString::number(column->valueAt(row)).size() > maxSize) maxSize = QString::number(column->valueAt(row)).size(); } const Double2StringFilter* const filter = static_cast(column->outputFilter()); bool decimals = false; for (int ii = 0; ii < nrows; ++ii) { bool ok; QString cell = column->asStringColumn()->textAt(ii); double val = cell.toDouble(&ok); if (cell.size() > QString::number(val).size() + 1) { decimals = true; break; } } QString tformn; if (decimals) { int maxStringSize = -1; for (int row = 0; row < nrows; ++row) { if (column->asStringColumn()->textAt(row).size() > maxStringSize) maxStringSize = column->asStringColumn()->textAt(row).size(); } const int diff = abs(maxSize - maxStringSize); maxSize+= diff; tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".") + QString::number(filter->numDigits()); } else tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".0"); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); break; } case AbstractColumn::Text: { int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (column->textAt(row).size() > maxSize) maxSize = column->textAt(row).size(); } const QString& tformn = QLatin1String("A") + QString::number(maxSize); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); break; } case AbstractColumn::Integer: //TODO case AbstractColumn::DateTime: case AbstractColumn::Day: case AbstractColumn::Month: break; } } //TODO extension name containing[] ? int r = fits_create_tbl(m_fitsFile, ASCII_TBL, nrows, tfields, columnNames.data(), tform.data(), tunit.data(), spreadsheet->name().toLatin1().constData(), &status); for (int i = 0; i < tfields; ++i) { delete[] tform[i]; delete[] tunit[i]; delete[] columnNames[i]; } if (r) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } QVector column; column.resize(nrows); column.squeeze(); double* columnNumeric = new double[nrows]; for (int col = 1; col <= tfields; ++col) { const Column* c = spreadsheet->column(col-1); AbstractColumn::ColumnMode columnMode = c->columnMode(); if (columnMode == AbstractColumn::Numeric) { for (int row = 0; row < nrows; ++row) columnNumeric[row] = c->valueAt(row); fits_write_col(m_fitsFile, TDOUBLE, col, 1, 1, nrows, columnNumeric, &status); if (status) { printError(status); delete[] columnNumeric; status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } } else { for (int row = 0; row < nrows; ++row) { column[row] = new char[c->textAt(row).size() + 1]; strcpy(column[row], c->textAt(row).toLatin1().constData()); } fits_write_col(m_fitsFile, TSTRING, col, 1, 1, nrows, column.data(), &status); for (int row = 0; row < nrows; ++row) delete[] column[row]; if (status) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); delete[] columnNumeric; return; } } } delete[] columnNumeric; status = 0; fits_close_file(m_fitsFile, &status); } } #else Q_UNUSED(fileName) Q_UNUSED(dataSource) #endif } /*! * \brief Return a map of the available extensions names in file \a filename * The keys of the map are the extension types, the values are the names * \param fileName the name of the FITS file to be analyzed */ QMultiMap FITSFilterPrivate::extensionNames(const QString& fileName) { DEBUG("FITSFilterPrivate::extensionNames() file name = " << fileName.toStdString()); #ifdef HAVE_FITS QMultiMap extensions; int status = 0; fitsfile* fitsFile = nullptr; if (fits_open_file(&fitsFile, fileName.toLatin1(), READONLY, &status )) return QMultiMap(); int hduCount; if (fits_get_num_hdus(fitsFile, &hduCount, &status)) return QMultiMap(); int imageCount = 0; int asciiTableCount = 0; int binaryTableCount = 0; for (int currentHDU = 1; (currentHDU <= hduCount) && !status; ++currentHDU) { int hduType; status = 0; fits_get_hdu_type(fitsFile, &hduType, &status); switch (hduType) { case IMAGE_HDU: imageCount++; break; case ASCII_TBL: asciiTableCount++; break; case BINARY_TBL: binaryTableCount++; break; } char* keyVal = new char[FLEN_VALUE]; QString extName; if (!fits_read_keyword(fitsFile,"EXTNAME", keyVal, nullptr, &status)) { extName = QLatin1String(keyVal); extName = extName.mid(1, extName.length() -2).simplified(); } else { status = 0; if (!fits_read_keyword(fitsFile,"HDUNAME", keyVal, nullptr, &status)) { extName = QLatin1String(keyVal); extName = extName.mid(1, extName.length() -2).simplified(); } else { status = 0; switch (hduType) { case IMAGE_HDU: if (imageCount == 1) extName = i18n("Primary header"); else extName = i18n("IMAGE #%1", imageCount); break; case ASCII_TBL: extName = i18n("ASCII_TBL #%1", asciiTableCount); break; case BINARY_TBL: extName = i18n("BINARY_TBL #%1", binaryTableCount); break; } } } delete[] keyVal; status = 0; extName = extName.trimmed(); switch (hduType) { case IMAGE_HDU: extensions.insert(QLatin1String("IMAGES"), extName); break; case ASCII_TBL: extensions.insert(QLatin1String("TABLES"), extName); break; case BINARY_TBL: extensions.insert(QLatin1String("TABLES"), extName); break; } fits_movrel_hdu(fitsFile, 1, nullptr, &status); } if (status == END_OF_FILE) status = 0; fits_close_file(fitsFile, &status); return extensions; #else Q_UNUSED(fileName) return QMultiMap(); #endif } /*! * \brief Prints the error text corresponding to the status code \a status * \param status the status code of the error */ void FITSFilterPrivate::printError(int status) const { #ifdef HAVE_FITS if (status) { char errorText[FLEN_ERRMSG]; fits_get_errstatus(status, errorText ); qDebug() << QLatin1String(errorText); } #else Q_UNUSED(status) #endif } /*! * \brief Add the keywords \a keywords to the current header unit * \param keywords the keywords to be added * \param fileName the name of the FITS file (extension) in which the keywords are added */ void FITSFilterPrivate::addNewKeyword(const QString& fileName, const QList& keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const FITSFilter::Keyword& keyword: keywords) { status = 0; if (!keyword.key.compare(QLatin1String("COMMENT"))) { if (fits_write_comment(m_fitsFile, keyword.value.toLatin1(), &status)) printError(status); } else if (!keyword.key.compare(QLatin1String("HISTORY"))) { if (fits_write_history(m_fitsFile, keyword.value.toLatin1(), &status)) printError(status); } else if (!keyword.key.compare(QLatin1String("DATE"))) { if (fits_write_date(m_fitsFile, &status)) printError(status); } else { int ok = 0; if (keyword.key.length() <= FLEN_KEYWORD) { ok++; if (keyword.value.length() <= FLEN_VALUE) { ok++; if (keyword.comment.length() <= FLEN_COMMENT) ok++; } } if (ok == 3) { bool ok; double val = keyword.value.toDouble(&ok); if (ok) { if (fits_write_key(m_fitsFile, TDOUBLE, keyword.key.toLatin1().data(), &val, keyword.comment.toLatin1().data(), &status)) printError(status); } else { if (fits_write_key(m_fitsFile, TSTRING, keyword.key.toLatin1().data(), keyword.value.toLatin1().data(), keyword.comment.toLatin1().data(), &status)) printError(status); } } else if ( ok == 2) { //comment too long } else if ( ok == 1) { //value too long } else { //keyword too long } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(keywords) Q_UNUSED(fileName) #endif } /*! * \brief Update keywords in the current header unit * \param fileName The name of the FITS file (extension) in which the keywords will be updated * \param originals The original keywords of the FITS file (extension) * \param updates The keywords that contain the updated values */ void FITSFilterPrivate::updateKeywords(const QString& fileName, const QList& originals, const QVector& updates) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } FITSFilter::Keyword updatedKeyword; FITSFilter::Keyword originalKeyword; FITSFilter::KeywordUpdate keywordUpdate; for (int i = 0; i < updates.size(); ++i) { updatedKeyword = updates.at(i); originalKeyword = originals.at(i); keywordUpdate = originals.at(i).updates; if (keywordUpdate.keyUpdated && keywordUpdate.valueUpdated && keywordUpdate.commentUpdated) { if (updatedKeyword.isEmpty()) { if (fits_delete_key(m_fitsFile, originalKeyword.key.toLatin1(), &status)) { printError(status); status = 0; } continue; } } if (!updatedKeyword.key.isEmpty()) { if (fits_modify_name(m_fitsFile, originalKeyword.key.toLatin1(), updatedKeyword.key.toLatin1(), &status )) { printError(status); status = 0; } } if (!updatedKeyword.value.isEmpty()) { bool ok; int intValue; double doubleValue; bool updated = false; doubleValue = updatedKeyword.value.toDouble(&ok); if (ok) { if (fits_update_key(m_fitsFile,TDOUBLE, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), &doubleValue, nullptr, &status)) printError(status); else updated = true; } if (!updated) { intValue = updatedKeyword.value.toInt(&ok); if (ok) { if (fits_update_key(m_fitsFile,TINT, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), &intValue, nullptr, &status)) printError(status); else updated = true; } } if (!updated) { if (fits_update_key(m_fitsFile,TSTRING, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), updatedKeyword.value.toLatin1().data(), nullptr, &status)) printError(status); } } else { if (keywordUpdate.valueUpdated) { if (fits_update_key_null(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), nullptr, &status)) { printError(status); status = 0; } } } if (!updatedKeyword.comment.isEmpty()) { if (fits_modify_comment(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), updatedKeyword.comment.toLatin1().data(), &status)) { printError(status); status = 0; } } else { if (keywordUpdate.commentUpdated) { if (fits_modify_comment(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), QByteArray().constData(), &status)) { printError(status); status = 0; } } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(originals) Q_UNUSED(updates) #endif } /*! * \brief Delete the keywords \a keywords from the current header unit * \param fileName the name of the FITS file (extension) in which the keywords will be deleted. * \param keywords the keywords to deleted */ void FITSFilterPrivate::deleteKeyword(const QString& fileName, const QList &keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const auto& keyword : keywords) { if (!keyword.key.isEmpty()) { status = 0; if (fits_delete_key(m_fitsFile, keyword.key.toLatin1(), &status)) printError(status); } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(keywords) Q_UNUSED(fileName) #endif } /*! * \brief FITSFilterPrivate::addKeywordUnit * \param fileName the FITS file (extension) in which the keyword units are updated/added * \param keywords the keywords whose units were modified/added */ void FITSFilterPrivate::addKeywordUnit(const QString &fileName, const QList &keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const FITSFilter::Keyword& keyword : keywords) { if (keyword.updates.unitUpdated) { if (fits_write_key_unit(m_fitsFile, keyword.key.toLatin1(), keyword.unit.toLatin1().data(), &status)) { printError(status); status = 0; } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(keywords) #endif } /*! * \brief Remove extensions from a FITS file * \param extensions The extensions to be removed from the FITS file */ void FITSFilterPrivate::removeExtensions(const QStringList &extensions) { #ifdef HAVE_FITS int status = 0; for (const auto& ext : extensions) { status = 0; if (fits_open_file(&m_fitsFile, ext.toLatin1(), READWRITE, &status )) { printError(status); continue; } if (fits_delete_hdu(m_fitsFile, nullptr, &status)) printError(status); status = 0; fits_close_file(m_fitsFile, &status); } #else Q_UNUSED(extensions) #endif } /*! * \brief Returns a list of keywords in the current header of \a fileName * \param fileName the name of the FITS file (extension) to be opened * \return A list of keywords */ QList FITSFilterPrivate::chduKeywords(const QString& fileName) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status )) { printError(status); return QList (); } int numberOfKeys; if (fits_get_hdrspace(m_fitsFile, &numberOfKeys, nullptr, &status)) { printError(status); return QList (); } QList keywords; keywords.reserve(numberOfKeys); char* key = new char[FLEN_KEYWORD]; char* value = new char[FLEN_VALUE]; char* comment = new char[FLEN_COMMENT]; char* unit = new char[FLEN_VALUE]; for (int i = 1; i <= numberOfKeys; ++i) { QStringList recordValues; FITSFilter::Keyword keyword; if (fits_read_keyn(m_fitsFile, i, key, value, comment, &status)) { printError(status); status = 0; continue; } fits_read_key_unit(m_fitsFile, key, unit, &status); recordValues << QLatin1String(key) << QLatin1String(value) << QLatin1String(comment) << QLatin1String(unit); keyword.key = recordValues[0].simplified(); keyword.value = recordValues[1].simplified(); keyword.comment = recordValues[2].simplified(); keyword.unit = recordValues[3].simplified(); keywords.append(keyword); } delete[] key; delete[] value; delete[] comment; delete[] unit; fits_close_file(m_fitsFile, &status); return keywords; #else Q_UNUSED(fileName) return QList(); #endif } /*! * \brief Builds the table \a headerEditTable from the keywords \a keys * \param fileName The name of the FITS file from which the keys are read if \a readKeys is \c true * \param headerEditTable The table to be built * \param readKeys It's used to determine whether the keywords are provided or they should be read from * file \a fileName * \param keys The keywords that are provided if the keywords were read already. */ void FITSFilterPrivate::parseHeader(const QString &fileName, QTableWidget *headerEditTable, bool readKeys, const QList& keys) { #ifdef HAVE_FITS QList keywords; if (readKeys) keywords = chduKeywords(fileName); else keywords = keys; headerEditTable->setRowCount(keywords.size()); QTableWidgetItem* item; for (int i = 0; i < keywords.size(); ++i) { const FITSFilter::Keyword& keyword = keywords.at(i); const bool mandatory = FITSFilter::mandatoryImageExtensionKeywords().contains(keyword.key) || FITSFilter::mandatoryTableExtensionKeywords().contains(keyword.key); item = new QTableWidgetItem(keyword.key); const QString& itemText = item->text(); const bool notEditableKey = mandatory || itemText.contains(QLatin1String("TFORM")) || itemText.contains(QLatin1String("TTYPE")) || itemText.contains(QLatin1String("TUNIT")) || itemText.contains(QLatin1String("TDISP")) || itemText.contains(QLatin1String("TBCOL")) || itemText.contains(QLatin1String("TZERO")); const bool notEditableValue = mandatory || itemText.contains(QLatin1String("TFORM")) || itemText.contains(QLatin1String("TDISP")) || itemText.contains(QLatin1String("TBCOL")) || itemText.contains(QLatin1String("TZERO")); if (notEditableKey) item->setFlags(item->flags() & ~Qt::ItemIsEditable); else item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 0, item ); item = new QTableWidgetItem(keyword.value); if (notEditableValue) item->setFlags(item->flags() & ~Qt::ItemIsEditable); else item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 1, item ); QString commentFieldText; if (!keyword.unit.isEmpty()) { if (keyword.updates.unitUpdated) { const QString& comment = keyword.comment.right(keyword.comment.size() - keyword.comment.indexOf(QChar(']'))-1); commentFieldText = QLatin1String("[") + keyword.unit + QLatin1String("] ") + comment; } else { if (keyword.comment.at(0) == QLatin1Char('[')) commentFieldText = keyword.comment; else commentFieldText = QLatin1String("[") + keyword.unit + QLatin1String("] ") + keyword.comment; } } else commentFieldText = keyword.comment; item = new QTableWidgetItem(commentFieldText); item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 2, item ); } headerEditTable->resizeColumnsToContents(); #else Q_UNUSED(fileName) Q_UNUSED(headerEditTable) Q_UNUSED(readKeys) Q_UNUSED(keys) #endif } /*! * \brief Helper function to return the value of the key \a key * \param fileName The name of the FITS file (extension) in which the keyword with key \a key should exist * \param key The key of the keyword whose value it's returned * \return The value of the keyword as a string */ const QString FITSFilterPrivate::valueOf(const QString& fileName, const char *key) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status )) { printError(status); return QString (); } char* keyVal = new char[FLEN_VALUE]; QString keyValue; if (!fits_read_keyword(m_fitsFile, key, keyVal, nullptr, &status)) { keyValue = QLatin1String(keyVal); keyValue = keyValue.simplified(); } else { printError(status); delete[] keyVal; fits_close_file(m_fitsFile, &status); return QString(); } delete[] keyVal; status = 0; fits_close_file(m_fitsFile, &status); return keyValue; #else Q_UNUSED(fileName) Q_UNUSED(key) return QString(); #endif } /*! * \brief Build the extensions tree from FITS file (extension) \a fileName * \param fileName The name of the FITS file to be opened * \param tw The QTreeWidget to be built * \param checkPrimary Used to determine whether the tree will be used for import or the header edit, * if it's \c true and if the primary array it's empty, then the item won't be added to the tree */ void FITSFilterPrivate::parseExtensions(const QString &fileName, QTreeWidget *tw, bool checkPrimary) { DEBUG("FITSFilterPrivate::parseExtensions()"); #ifdef HAVE_FITS const QMultiMap& extensions = extensionNames(fileName); const QStringList& imageExtensions = extensions.values(QLatin1String("IMAGES")); const QStringList& tableExtensions = extensions.values(QLatin1String("TABLES")); QTreeWidgetItem* root = tw->invisibleRootItem(); //TODO: fileName may contain any data type: check if it's a FITS file QTreeWidgetItem* treeNameItem = new QTreeWidgetItem((QTreeWidgetItem*)nullptr, QStringList() << fileName); root->addChild(treeNameItem); treeNameItem->setExpanded(true); QTreeWidgetItem* imageExtensionItem = new QTreeWidgetItem((QTreeWidgetItem*)nullptr, QStringList() << i18n("Images")); imageExtensionItem->setFlags(imageExtensionItem->flags() & ~Qt::ItemIsSelectable ); QString primaryHeaderNaxis = valueOf(fileName, "NAXIS"); const int naxis = primaryHeaderNaxis.toInt(); bool noImage = false; for (const QString& ext : imageExtensions) { QTreeWidgetItem* treeItem = new QTreeWidgetItem((QTreeWidgetItem*)nullptr, QStringList() << ext); if (ext == i18n("Primary header")) { if (checkPrimary && naxis == 0) { delete treeItem; continue; } } imageExtensionItem->addChild(treeItem); } if (imageExtensionItem->childCount() > 0) { treeNameItem->addChild(imageExtensionItem); imageExtensionItem->setIcon(0,QIcon::fromTheme("view-preview")); imageExtensionItem->setExpanded(true); imageExtensionItem->child(0)->setSelected(true); tw->setCurrentItem(imageExtensionItem->child(0)); } else noImage = true; if (tableExtensions.size() > 0) { QTreeWidgetItem* tableExtensionItem = new QTreeWidgetItem((QTreeWidgetItem*)nullptr, QStringList() << i18n("Tables")); tableExtensionItem->setFlags(tableExtensionItem->flags() & ~Qt::ItemIsSelectable ); for (const QString& ext : tableExtensions) { QTreeWidgetItem* treeItem = new QTreeWidgetItem((QTreeWidgetItem*)nullptr, QStringList() << ext); tableExtensionItem->addChild(treeItem); } if (tableExtensionItem->childCount() > 0) { treeNameItem->addChild(tableExtensionItem); tableExtensionItem->setIcon(0,QIcon::fromTheme("x-office-spreadsheet")); tableExtensionItem->setExpanded(true); if (noImage) { tableExtensionItem->child(0)->setSelected(true); tw->setCurrentItem(tableExtensionItem->child(0)); } } } #else Q_UNUSED(fileName) Q_UNUSED(tw) Q_UNUSED(checkPrimary) #endif DEBUG("FITSFilterPrivate::parseExtensions() DONE"); } /*! * \brief FITSFilterPrivate::~FITSFilterPrivate */ FITSFilterPrivate::~FITSFilterPrivate() = default; //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void FITSFilter::save(QXmlStreamWriter * writer) const { Q_UNUSED(writer) } /*! Loads from XML. */ bool FITSFilter::load(XmlStreamReader * loader) { Q_UNUSED(loader) return false; } diff --git a/src/backend/datasources/filters/HDF5Filter.cpp b/src/backend/datasources/filters/HDF5Filter.cpp index f5663e834..57986c6e5 100644 --- a/src/backend/datasources/filters/HDF5Filter.cpp +++ b/src/backend/datasources/filters/HDF5Filter.cpp @@ -1,1935 +1,1935 @@ /*************************************************************************** File : HDF5Filter.cpp Project : LabPlot Description : HDF5 I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2018 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ /* TODO: * Feature: implement missing data types and ranks * Performance: only fill dataPointer or dataStrings (not both) */ #include "backend/datasources/filters/HDF5Filter.h" #include "backend/datasources/filters/HDF5FilterPrivate.h" #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include /*! \class HDF5Filter \brief Manages the import/export of data from/to a HDF5 file. \ingroup datasources */ HDF5Filter::HDF5Filter():AbstractFileFilter(HDF5), d(new HDF5FilterPrivate(this)) {} HDF5Filter::~HDF5Filter() = default; /*! parses the content of the file \c fileName. */ void HDF5Filter::parse(const QString & fileName, QTreeWidgetItem* rootItem) { d->parse(fileName, rootItem); } /*! reads the content of the data set \c dataSet from file \c fileName. */ QVector HDF5Filter::readCurrentDataSet(const QString& fileName, AbstractDataSource* dataSource, bool &ok, AbstractFileFilter::ImportMode importMode, int lines) { return d->readCurrentDataSet(fileName, dataSource, ok, importMode, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ void HDF5Filter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { d->readDataFromFile(fileName, dataSource, mode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void HDF5Filter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void HDF5Filter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void HDF5Filter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void HDF5Filter::setCurrentDataSetName(const QString& ds) { d->currentDataSetName = ds; } const QString HDF5Filter::currentDataSetName() const { return d->currentDataSetName; } void HDF5Filter::setStartRow(const int s) { d->startRow = s; } int HDF5Filter::startRow() const { return d->startRow; } void HDF5Filter::setEndRow(const int e) { d->endRow = e; } int HDF5Filter::endRow() const { return d->endRow; } void HDF5Filter::setStartColumn(const int c) { d->startColumn = c; } int HDF5Filter::startColumn() const { return d->startColumn; } void HDF5Filter::setEndColumn(const int c) { d->endColumn = c; } int HDF5Filter::endColumn() const { return d->endColumn; } QString HDF5Filter::fileInfoString(const QString& fileName) { DEBUG("HDF5Filter::fileInfoString()"); QString info; #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); DEBUG("fileName = " << bafileName.data()); // check file type first htri_t isHdf5 = H5Fis_hdf5(bafileName.data()); if (isHdf5 == 0) { DEBUG(bafileName.data() << " is not a HDF5 file! isHdf5 = " << isHdf5 << " Giving up."); return i18n("Not a HDF5 file"); } if (isHdf5 < 0) { DEBUG("H5Fis_hdf5() failed on " << bafileName.data() << "! Giving up."); return i18n("Failed checking file"); } // open file hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); HDF5FilterPrivate::handleError((int)file, "H5Fopen", fileName); if (file < 0) { DEBUG("Opening file " << bafileName.data() << " failed! Giving up."); return i18n("Failed opening HDF5 file"); } hsize_t size; herr_t status = H5Fget_filesize(file, &size); if (status >= 0) { info += i18n("File size: %1 bytes", QString::number(size)); info += QLatin1String("
"); } hssize_t freesize = H5Fget_freespace(file); info += i18n("Free space: %1 bytes", QString::number(freesize)); info += QLatin1String("
"); info += QLatin1String("
"); ssize_t objectCount; objectCount = H5Fget_obj_count(file, H5F_OBJ_FILE); info += i18n("Number of files: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_DATASET); info += i18n("Number of data sets: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_GROUP); info += i18n("Number of groups: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_DATATYPE); info += i18n("Number of named datatypes: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_ATTR); info += i18n("Number of attributes: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_ALL); info += i18n("Number of all objects: %1", QString::number(objectCount)); info += QLatin1String("
"); #ifdef HAVE_AT_LEAST_HDF5_1_10_0 // using H5Fget_info2 struct (see H5Fpublic.h) H5F_info2_t file_info; status = H5Fget_info2(file, &file_info); if (status >= 0) { info += QLatin1String("
"); info += i18n("Version of superblock: %1", QString::number(file_info.super.version)); info += QLatin1String("
"); info += i18n("Size of superblock: %1 bytes", QString::number(file_info.super.super_size)); info += QLatin1String("
"); info += i18n("Size of superblock extension: %1 bytes", QString::number(file_info.super.super_ext_size)); info += QLatin1String("
"); info += i18n("Version of free-space manager: %1", QString::number(file_info.free.version)); info += QLatin1String("
"); info += i18n("Size of free-space manager metadata: %1 bytes", QString::number(file_info.free.meta_size)); info += QLatin1String("
"); info += i18n("Total size of free space: %1 bytes", QString::number(file_info.free.tot_space)); info += QLatin1String("
"); info += i18n("Version of shared object header: %1", QString::number(file_info.sohm.version)); info += QLatin1String("
"); info += i18n("Size of shared object header: %1 bytes", QString::number(file_info.sohm.hdr_size)); info += QLatin1String("
"); info += i18n("Size of all shared object header indexes: %1 bytes", QString::number(file_info.sohm.msgs_info.index_size)); info += QLatin1String("
"); info += i18n("Size of the heap: %1 bytes", QString::number(file_info.sohm.msgs_info.heap_size)); info += QLatin1String("
"); } #else // using H5Fget_info1 struct (named H5F_info_t in HDF5 1.8) H5F_info_t file_info; status = H5Fget_info(file, &file_info); if (status >= 0) { info += i18n("Size of superblock extension: %1 bytes", QString::number(file_info.super_ext_size)); info += QLatin1String("
"); info += i18n("Size of shared object header: %1 bytes", QString::number(file_info.sohm.hdr_size)); info += QLatin1String("
"); info += i18n("Size of all shared object header indexes: %1 bytes", QString::number(file_info.sohm.msgs_info.index_size)); info += QLatin1String("
"); info += i18n("Size of the heap: %1 bytes", QString::number(file_info.sohm.msgs_info.heap_size)); info += QLatin1String("
"); } #endif // cache information //see https://support.hdfgroup.org/HDF5/doc/RM/RM_H5F.html info += QLatin1String("
"); H5AC_cache_config_t config; config.version = H5AC__CURR_CACHE_CONFIG_VERSION; status = H5Fget_mdc_config(file, &config); if (status >= 0) { info += i18n("Cache config version: %1", QString::number(config.version)); info += QLatin1String("
"); info += i18n("Adaptive cache resize report function enabled: %1", config.rpt_fcn_enabled ? i18n("Yes") : i18n("No")); info += QLatin1String("
"); info += i18n("Cache initial maximum size: %1 bytes", QString::number(config.initial_size)); info += QLatin1String("
"); info += i18n("Adaptive cache maximum size: %1 bytes", QString::number(config.max_size)); info += QLatin1String("
"); info += i18n("Adaptive cache minimum size: %1 bytes", QString::number(config.min_size)); info += QLatin1String("
"); //TODO: more settings } double hit_rate; status = H5Fget_mdc_hit_rate(file, &hit_rate); Q_UNUSED(status); info += i18n("Metadata cache hit rate: %1", QString::number(hit_rate)); info += QLatin1String("
"); //TODO: herr_t H5Fget_mdc_image_info(hid_t file_id, haddr_t *image_addr, hsize_t *image_len) size_t max_size, min_clean_size, cur_size; int cur_num_entries; status = H5Fget_mdc_size(file, &max_size, &min_clean_size, &cur_size, &cur_num_entries); if (status >= 0) { info += i18n("Current cache maximum size: %1 bytes", QString::number(max_size)); info += QLatin1String("
"); info += i18n("Current cache minimum clean size: %1 bytes", QString::number(min_clean_size)); info += QLatin1String("
"); info += i18n("Current cache size: %1 bytes", QString::number(cur_size)); info += QLatin1String("
"); info += i18n("Current number of entries in the cache: %1", QString::number(cur_num_entries)); info += QLatin1String("
"); } //TODO: 1.10 herr_t H5Fget_metadata_read_retry_info( hid_t file_id, H5F_retry_info_t *info ) /* TODO: not available hbool_t atomicMode; status = H5Fget_mpi_atomicity(file, &atomicMode); if (status >= 0) { info += i18n("MPI file access atomic mode: %1", atomicMode ? i18n("Yes") : i18n("No")); info += QLatin1String("
"); }*/ #ifdef HAVE_AT_LEAST_HDF5_1_10_0 hbool_t is_enabled, is_currently_logging; status = H5Fget_mdc_logging_status(file, &is_enabled, &is_currently_logging); if (status >= 0) { info += i18n("Logging enabled: %1", is_enabled ? i18n("Yes") : i18n("No")); info += QLatin1String("
"); info += i18n("Events are currently logged: %1", is_currently_logging ? i18n("Yes") : i18n("No")); info += QLatin1String("
"); } #endif #ifdef HAVE_AT_LEAST_HDF5_1_10_1 unsigned int accesses[2], hits[2], misses[2], evictions[2], bypasses[2]; status = H5Fget_page_buffering_stats(file, accesses, hits, misses, evictions, bypasses); if (status >= 0) { info += i18n("Metadata/raw data page buffer accesses: %1 %2", QString::number(accesses[0]), QString::number(accesses[1])); info += QLatin1String("
"); info += i18n("Metadata/raw data page buffer hits: %1 %2", QString::number(hits[0]), QString::number(hits[1])); info += QLatin1String("
"); info += i18n("Metadata/raw data page buffer misses: %1 %2", QString::number(misses[0]), QString::number(misses[1])); info += QLatin1String("
"); info += i18n("Metadata/raw data page buffer evictions: %1 %2", QString::number(evictions[0]), QString::number(evictions[1])); info += QLatin1String("
"); info += i18n("Metadata/raw data accesses bypassing page buffer: %1 %2", QString::number(bypasses[0]), QString::number(bypasses[1])); info += QLatin1String("
"); } else { info += i18n("Page buffer disabled"); info += QLatin1String("
"); DEBUG("H5Fget_page_buffering_stats() status = " << status); } #endif #else Q_UNUSED(fileName); #endif return info; } /*! * Get file content in DDL (Data Description Language) format * uses "h5dump" */ QString HDF5Filter::fileDDLString(const QString& fileName) { DEBUG("HDF5Filter::fileDDLString()"); QString DDLString; #ifdef Q_OS_LINUX auto* proc = new QProcess(); QStringList args; args << "-H" << fileName; proc->start( "h5dump", args); if (proc->waitForReadyRead(1000) == false) DDLString += i18n("Reading from file %1 failed.", fileName); else { DDLString += proc->readAll(); DDLString.replace('\n', "
\n"); DDLString.replace("\t","    "); //DEBUG(" DDL string: " << DDLString.toStdString()); } #else //TODO: h5dump on Win, Mac Q_UNUSED(fileName) #endif return DDLString; } //##################################################################### //################### Private implementation ########################## //##################################################################### HDF5FilterPrivate::HDF5FilterPrivate(HDF5Filter* owner) : q(owner) { #ifdef HAVE_HDF5 m_status = 0; #endif } #ifdef HAVE_HDF5 void HDF5FilterPrivate::handleError(int err, const QString& function, const QString& arg) { #ifdef NDEBUG Q_UNUSED(err) Q_UNUSED(function) Q_UNUSED(arg) #else if (err < 0) { DEBUG("ERROR " << err << ": " << function.toStdString() << "() - " << arg.toStdString()); } #endif } QString HDF5FilterPrivate::translateHDF5Order(H5T_order_t o) { QString order; switch (o) { case H5T_ORDER_LE: order = "LE"; break; case H5T_ORDER_BE: order = "BE"; break; case H5T_ORDER_VAX: order = "VAX"; break; case H5T_ORDER_MIXED: order = "MIXED"; break; case H5T_ORDER_NONE: order = "NONE"; break; case H5T_ORDER_ERROR: order = "ERROR"; break; } return order; } QString HDF5FilterPrivate::translateHDF5Type(hid_t t) { QString type; if (H5Tequal(t, H5T_STD_I8LE) || H5Tequal(t, H5T_STD_I8BE)) type = "CHAR"; else if (H5Tequal(t, H5T_STD_U8LE) || H5Tequal(t, H5T_STD_U8BE)) type = "UCHAR"; else if (H5Tequal(t, H5T_STD_I16LE) || H5Tequal(t, H5T_STD_I16BE)) type = "SHORT"; else if (H5Tequal(t, H5T_STD_U16LE) || H5Tequal(t, H5T_STD_U16BE)) type = "USHORT"; else if (H5Tequal(t, H5T_STD_I32LE) || H5Tequal(t, H5T_STD_I32BE)) type = "INT"; else if (H5Tequal(t, H5T_STD_U32LE) || H5Tequal(t, H5T_STD_U32BE)) type = "UINT"; else if (H5Tequal(t, H5T_NATIVE_LONG)) type = "LONG"; else if (H5Tequal(t, H5T_NATIVE_ULONG)) type = "ULONG"; else if (H5Tequal(t, H5T_STD_I64LE) || H5Tequal(t, H5T_STD_I64BE)) type = "LLONG"; else if (H5Tequal(t, H5T_STD_U64LE) || H5Tequal(t, H5T_STD_U64BE)) type = "ULLONG"; else if (H5Tequal(t, H5T_IEEE_F32LE) || H5Tequal(t, H5T_IEEE_F32BE)) type = "FLOAT"; else if (H5Tequal(t, H5T_IEEE_F64LE) || H5Tequal(t, H5T_IEEE_F64BE)) type = "DOUBLE"; else if (H5Tequal(t, H5T_NATIVE_LDOUBLE)) type = "LDOUBLE"; else type = "UNKNOWN"; return type; } QString HDF5FilterPrivate::translateHDF5Class(H5T_class_t c) { QString dclass; switch (c) { case H5T_INTEGER: dclass = "INTEGER"; break; case H5T_FLOAT: dclass = "FLOAT"; break; case H5T_STRING: dclass = "STRING"; break; case H5T_BITFIELD: dclass = "BITFIELD"; break; case H5T_OPAQUE: dclass = "OPAQUE"; break; case H5T_COMPOUND: dclass = "COMPOUND"; break; case H5T_ARRAY: dclass = "ARRAY"; break; case H5T_ENUM: dclass = "ENUM"; break; case H5T_REFERENCE: dclass = "REFERENCE"; break; case H5T_VLEN: dclass = "VLEN"; break; case H5T_TIME: dclass = "TIME"; break; case H5T_NCLASSES: dclass = "NCLASSES"; break; case H5T_NO_CLASS: dclass = "NOCLASS"; break; } return dclass; } QStringList HDF5FilterPrivate::readHDF5Compound(hid_t tid) { size_t typeSize = H5Tget_size(tid); QString line; line += QLatin1String("COMPOUND(") + QString::number(typeSize) + QLatin1String(") : ("); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); for (int i = 0; i < members; ++i) { H5T_class_t mclass = H5Tget_member_class(tid, i); handleError((int)mclass, "H5Tget_member_class"); hid_t mtype = H5Tget_member_type(tid, i); handleError((int)mtype, "H5Tget_member_type"); size_t size = H5Tget_size(mtype); handleError((int)size, "H5Tget_size"); QString typeString = translateHDF5Class(mclass); if (mclass == H5T_INTEGER || mclass == H5T_FLOAT) typeString = translateHDF5Type(mtype); line += H5Tget_member_name(tid, i) + QLatin1String("[") + typeString + QLatin1String("(") + QString::number(size) + QLatin1String(")]"); if (i == members-1) line += QLatin1String(")"); else line += QLatin1String(","); m_status = H5Tclose(mtype); handleError(m_status, "H5Tclose"); } QStringList dataString; dataString << line; return dataString; } template QStringList HDF5FilterPrivate::readHDF5Data1D(hid_t dataset, hid_t type, int rows, int lines, void* dataContainer) { DEBUG("readHDF5Data1D() rows = " << rows << ", lines = " << lines); QStringList dataString; // we read all rows of data T* data = new T[rows]; m_status = H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Dread"); DEBUG(" startRow = " << startRow << ", endRow = " << endRow); DEBUG(" dataContainer = " << dataContainer); for (int i = startRow - 1; i < qMin(endRow, lines + startRow - 1); ++i) { if (dataContainer) // read to data source static_cast*>(dataContainer)->operator[](i-startRow+1) = data[i]; else // for preview dataString << QString::number(static_cast(data[i])); } delete[] data; return dataString; } -QStringList HDF5FilterPrivate::readHDF5CompoundData1D(hid_t dataset, hid_t tid, int rows, int lines, QVector& dataContainer) { +QStringList HDF5FilterPrivate::readHDF5CompoundData1D(hid_t dataset, hid_t tid, int rows, int lines, std::vector& dataContainer) { DEBUG("HDF5FilterPrivate::readHDF5CompoundData1D()"); DEBUG(" dataContainer size = " << dataContainer.size()); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); DEBUG(" # members = " << members); QStringList dataString; if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) dataString << QLatin1String("("); dataContainer.resize(members); // avoid "index out of range" for preview } for (int m = 0; m < members; ++m) { hid_t mtype = H5Tget_member_type(tid, m); handleError((int)mtype, "H5Tget_member_type"); size_t msize = H5Tget_size(mtype); handleError((int)msize, "H5Tget_size"); hid_t ctype = H5Tcreate(H5T_COMPOUND, msize); handleError((int)ctype, "H5Tcreate"); m_status = H5Tinsert(ctype, H5Tget_member_name(tid, m), 0, mtype); handleError(m_status, "H5Tinsert"); QStringList mdataString; if (H5Tequal(mtype, H5T_STD_I8LE) || H5Tequal(mtype, H5T_STD_I8BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 2: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 4: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 8: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; } } else if (H5Tequal(mtype, H5T_STD_U8LE) || H5Tequal(mtype, H5T_STD_U8BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 2: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 4: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 8: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; } } else if (H5Tequal(mtype, H5T_STD_I16LE) || H5Tequal(mtype, H5T_STD_I16BE) || H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U16LE) || H5Tequal(mtype, H5T_STD_U16BE) || H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_I32LE) || H5Tequal(mtype, H5T_STD_I32BE) || H5Tequal(mtype, H5T_NATIVE_INT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U32LE) || H5Tequal(mtype, H5T_STD_U32BE) || H5Tequal(mtype, H5T_NATIVE_UINT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_LONG)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_ULONG)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_I64LE) || H5Tequal(mtype, H5T_STD_I64BE) || H5Tequal(mtype, H5T_NATIVE_LLONG)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U64LE) || H5Tequal(mtype, H5T_STD_U64BE) || H5Tequal(mtype, H5T_NATIVE_ULLONG)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_IEEE_F32LE) || H5Tequal(mtype, H5T_IEEE_F32BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_IEEE_F64LE) || H5Tequal(mtype, H5T_IEEE_F64BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_LDOUBLE)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else { if (dataContainer[m]) { for (int i = startRow-1; i < qMin(endRow, lines + startRow - 1); ++i) static_cast*>(dataContainer[m])->operator[](i - startRow + 1) = 0; } else { for (int i = 0; i < qMin(rows, lines); ++i) mdataString << QLatin1String("_"); } H5T_class_t mclass = H5Tget_member_class(tid, m); handleError((int)mclass, "H5Tget_member_class"); DEBUG("unsupported type of class " << translateHDF5Class(mclass).toStdString()); } if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) { dataString[i] += mdataString[i]; if (m < members - 1) dataString[i] += QLatin1String(","); } } H5Tclose(ctype); } if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) dataString[i] += QLatin1String(")"); } return dataString; } template -QVector HDF5FilterPrivate::readHDF5Data2D(hid_t dataset, hid_t type, int rows, int cols, int lines, QVector& dataPointer) { +QVector HDF5FilterPrivate::readHDF5Data2D(hid_t dataset, hid_t type, int rows, int cols, int lines, std::vector& dataPointer) { DEBUG("readHDF5Data2D() rows = " << rows << ", cols =" << cols << ", lines =" << lines); QVector dataStrings; if (rows == 0 || cols == 0) return dataStrings; T** data = (T**) malloc(rows*sizeof(T*)); data[0] = (T*) malloc(cols*rows*sizeof(T)); for (int i = 1; i < rows; ++i) data[i] = data[0] + i*cols; m_status = H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0][0]); handleError(m_status,"H5Dread"); for (int i = 0; i < qMin(rows, lines); ++i) { QStringList line; line.reserve(cols); for (int j = 0; j < cols; ++j) { if (dataPointer[0]) static_cast*>(dataPointer[j-startColumn+1])->operator[](i-startRow+1) = data[i][j]; else line << QString::number(static_cast(data[i][j])); } dataStrings << line; } free(data[0]); free(data); QDEBUG(dataStrings); return dataStrings; } QVector HDF5FilterPrivate::readHDF5CompoundData2D(hid_t dataset, hid_t tid, int rows, int cols, int lines) { DEBUG("readHDF5CompoundData2D() rows =" << rows << "cols =" << cols << "lines =" << lines); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); DEBUG(" # members =" << members); QVector dataStrings; for (int i = 0; i < qMin(rows, lines); ++i) { QStringList lineStrings; for (int j = 0; j < cols; ++j) lineStrings << QLatin1String("("); dataStrings << lineStrings; } //QStringList* data = new QStringList[members]; for (int m = 0; m < members; ++m) { hid_t mtype = H5Tget_member_type(tid, m); handleError((int)mtype, "H5Tget_member_type"); size_t msize = H5Tget_size(mtype); handleError((int)msize, "H5Tget_size"); hid_t ctype = H5Tcreate(H5T_COMPOUND, msize); handleError((int)ctype, "H5Tcreate"); m_status = H5Tinsert(ctype, H5Tget_member_name(tid, m), 0, mtype); handleError(m_status, "H5Tinsert"); // dummy container for all data columns // initially contains one pointer set to NULL - QVector dummy(1, nullptr); + std::vector dummy(1, nullptr); QVector mdataStrings; if (H5Tequal(mtype, H5T_STD_I8LE) || H5Tequal(mtype, H5T_STD_I8BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 2: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 4: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 8: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; } } else if (H5Tequal(mtype, H5T_STD_U8LE) || H5Tequal(mtype, H5T_STD_U8BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 2: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 4: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 8: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; } } else if (H5Tequal(mtype, H5T_STD_I16LE) || H5Tequal(mtype, H5T_STD_I16BE)|| H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U16LE) || H5Tequal(mtype, H5T_STD_U16BE) || H5Tequal(mtype, H5T_NATIVE_USHORT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_I32LE) || H5Tequal(mtype, H5T_STD_I32BE) || H5Tequal(mtype, H5T_NATIVE_INT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U32LE) || H5Tequal(mtype, H5T_STD_U32BE) || H5Tequal(mtype, H5T_NATIVE_UINT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_LONG)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_ULONG)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_I64LE) || H5Tequal(mtype, H5T_STD_I64BE) || H5Tequal(mtype, H5T_NATIVE_LLONG)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U64LE) || H5Tequal(mtype, H5T_STD_U64BE) || H5Tequal(mtype, H5T_NATIVE_ULLONG)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_IEEE_F32LE) || H5Tequal(mtype, H5T_IEEE_F32BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_IEEE_F64LE) || H5Tequal(mtype, H5T_IEEE_F64BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_LDOUBLE)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else { for (int i = 0; i < qMin(rows, lines); ++i) { QStringList lineString; for (int j = 0; j < cols; ++j) lineString << QLatin1String("_"); mdataStrings << lineString; } #ifndef NDEBUG H5T_class_t mclass = H5Tget_member_class(tid, m); #endif DEBUG("unsupported class " << translateHDF5Class(mclass).toStdString()); } m_status = H5Tclose(ctype); handleError(m_status, "H5Tclose"); for (int i = 0; i < qMin(rows, lines); i++) { for (int j = 0; j < cols; j++) { dataStrings[i][j] += mdataStrings[i][j]; if (m < members-1) dataStrings[i][j] += QLatin1String(","); } } } for (int i = 0; i < qMin(rows, lines); ++i) { for (int j = 0; j < cols; ++j) dataStrings[i][j] += QLatin1String(")"); } QDEBUG("dataStrings =" << dataStrings); return dataStrings; } QStringList HDF5FilterPrivate::readHDF5Attr(hid_t aid) { QStringList attr; char name[MAXNAMELENGTH]; m_status = H5Aget_name(aid, MAXNAMELENGTH, name); handleError(m_status, "H5Aget_name"); attr << QString(name); // DEBUG(" name =" << QString(name)); hid_t aspace = H5Aget_space(aid); // the dimensions of the attribute data handleError((int)aspace, "H5Aget_space"); hid_t atype = H5Aget_type(aid); handleError((int)atype, "H5Aget_type"); hid_t aclass = H5Tget_class(atype); handleError((int)aclass, "H5Aget_class"); if (aclass == H5T_STRING) { char buf[MAXSTRINGLENGTH]; // buffer to read attr value hid_t amem = H5Tget_native_type(atype, H5T_DIR_ASCEND); handleError((int)amem, "H5Tget_native_type"); m_status = H5Aread(aid, amem, buf); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString(buf); m_status = H5Tclose(amem); handleError(m_status, "H5Tclose"); } else if (aclass == H5T_INTEGER) { if (H5Tequal(atype, H5T_STD_I8LE)) { qint8 value; m_status = H5Aread(aid, H5T_STD_I8LE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I8BE)) { qint8 value; m_status = H5Aread(aid, H5T_STD_I8BE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: { qint8 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 2: { qint16 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 4: { qint32 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 8: { qint64 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } default: DEBUG("unknown size " << sizeof(H5T_NATIVE_CHAR) << " of H5T_NATIVE_CHAR"); return QStringList(QString()); } } else if (H5Tequal(atype, H5T_STD_U8LE)) { uint8_t value; m_status = H5Aread(aid, H5T_STD_U8LE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U8BE)) { uint8_t value; m_status = H5Aread(aid, H5T_STD_U8BE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: { uint8_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 2: { uint16_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 4: { uint32_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 8: { uint64_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } default: DEBUG("unknown size " << sizeof(H5T_NATIVE_UCHAR) << " of H5T_NATIVE_UCHAR"); return QStringList(QString()); } } else if (H5Tequal(atype, H5T_STD_I16LE) || H5Tequal(atype, H5T_STD_I16BE) || H5Tequal(atype, H5T_NATIVE_SHORT)) { short value; m_status = H5Aread(aid, H5T_NATIVE_SHORT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U16LE) || H5Tequal(atype, H5T_STD_U16BE) || H5Tequal(atype, H5T_NATIVE_USHORT)) { unsigned short value; m_status = H5Aread(aid, H5T_NATIVE_USHORT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I32LE) || H5Tequal(atype, H5T_STD_I32BE) || H5Tequal(atype, H5T_NATIVE_INT)) { int value; m_status = H5Aread(aid, H5T_NATIVE_INT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U32LE) || H5Tequal(atype, H5T_STD_U32BE) || H5Tequal(atype, H5T_NATIVE_UINT)) { unsigned int value; m_status = H5Aread(aid, H5T_NATIVE_UINT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_LONG)) { long value; m_status = H5Aread(aid, H5T_NATIVE_LONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_ULONG)) { unsigned long value; m_status = H5Aread(aid, H5T_NATIVE_ULONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I64LE) || H5Tequal(atype, H5T_STD_I64BE) || H5Tequal(atype, H5T_NATIVE_LLONG)) { long long value; m_status = H5Aread(aid, H5T_NATIVE_LLONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U64LE) || H5Tequal(atype, H5T_STD_U64BE) || H5Tequal(atype, H5T_NATIVE_ULLONG)) { unsigned long long value; m_status = H5Aread(aid, H5T_NATIVE_ULLONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else attr<<" (unknown integer)"; } else if (aclass == H5T_FLOAT) { if (H5Tequal(atype, H5T_IEEE_F32LE) || H5Tequal(atype, H5T_IEEE_F32BE)) { float value; m_status = H5Aread(aid, H5T_NATIVE_FLOAT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_IEEE_F64LE) || H5Tequal(atype, H5T_IEEE_F64BE)) { double value; m_status = H5Aread(aid, H5T_NATIVE_DOUBLE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_LDOUBLE)) { long double value; m_status = H5Aread(aid, H5T_NATIVE_LDOUBLE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number((double)value); } else attr<<" (unknown float)"; } m_status = H5Tclose(atype); handleError(m_status, "H5Tclose"); m_status = H5Sclose(aspace); handleError(m_status, "H5Sclose"); return attr; } QStringList HDF5FilterPrivate::scanHDF5Attrs(hid_t oid) { QStringList attrList; int numAttr = H5Aget_num_attrs(oid); handleError(numAttr, "H5Aget_num_attrs"); DEBUG("number of attr = " << numAttr); for (int i = 0; i < numAttr; ++i) { hid_t aid = H5Aopen_idx(oid, i); handleError((int)aid, "H5Aopen_idx"); attrList << readHDF5Attr(aid); if (i < numAttr-1) attrList << QLatin1String(", "); m_status = H5Aclose(aid); handleError(m_status, "H5Aclose"); } return attrList; } QStringList HDF5FilterPrivate::readHDF5DataType(hid_t tid) { H5T_class_t typeClass = H5Tget_class(tid); handleError((int)typeClass, "H5Tget_class"); QStringList typeProps; QString typeString = translateHDF5Class(typeClass); if (typeClass == H5T_INTEGER || typeClass == H5T_FLOAT) typeString = translateHDF5Type(tid); typeProps<setIcon(0, QIcon::fromTheme("accessories-calculator")); dataTypeItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(dataTypeItem); } void HDF5FilterPrivate::scanHDF5DataSet(hid_t did, char *dataSetName, QTreeWidgetItem* parentItem) { QString attr = scanHDF5Attrs(did).join(QString()); char link[MAXNAMELENGTH]; m_status = H5Iget_name(did, link, MAXNAMELENGTH); handleError(m_status, "H5Iget_name"); QStringList dataSetProps; hsize_t size = H5Dget_storage_size(did); handleError((int)size, "H5Dget_storage_size"); hid_t datatype = H5Dget_type(did); handleError((int)datatype, "H5Dget_type"); size_t typeSize = H5Tget_size(datatype); handleError((int)typeSize, "H5Dget_size"); dataSetProps << readHDF5DataType(datatype); hid_t dataspace = H5Dget_space(did); int rank = H5Sget_simple_extent_ndims(dataspace); handleError(rank, "H5Sget_simple_extent_ndims"); unsigned int rows = 1, cols = 1, regs = 1; if (rank == 1) { hsize_t dims_out[1]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, nullptr); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else if (rank == 2) { hsize_t dims_out[2]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, nullptr); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; cols = dims_out[1]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String("x") << QString::number(cols) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else if (rank == 3) { hsize_t dims_out[3]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, nullptr); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; cols = dims_out[1]; regs = dims_out[2]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String("x") << QString::number(cols) << QLatin1String("x") << QString::number(regs) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else dataSetProps << QLatin1String(", ") << i18n("rank %1 not supported yet", rank); hid_t pid = H5Dget_create_plist(did); handleError((int)pid, "H5Dget_create_plist"); dataSetProps << ", " << readHDF5PropertyList(pid).join(QString()); QTreeWidgetItem* dataSetItem = new QTreeWidgetItem(QStringList()<setIcon(0, QIcon::fromTheme("x-office-spreadsheet")); for (int i = 0; i < dataSetItem->columnCount(); ++i) { if (rows > 0 && cols > 0 && regs > 0) { dataSetItem->setBackground(i, QColor(192,255,192)); dataSetItem->setForeground(i, Qt::black); dataSetItem->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable); } else dataSetItem->setFlags(Qt::NoItemFlags); } parentItem->addChild(dataSetItem); } void HDF5FilterPrivate::scanHDF5Link(hid_t gid, char *linkName, QTreeWidgetItem* parentItem) { char target[MAXNAMELENGTH]; m_status = H5Gget_linkval(gid, linkName, MAXNAMELENGTH, target) ; handleError(m_status, "H5Gget_linkval"); QTreeWidgetItem* linkItem = new QTreeWidgetItem(QStringList() << QString(linkName) << i18n("symbolic link") << i18n("link to %1", QFile::decodeName(target))); linkItem->setIcon(0, QIcon::fromTheme("emblem-symbolic-link")); linkItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(linkItem); } void HDF5FilterPrivate::scanHDF5Group(hid_t gid, char *groupName, QTreeWidgetItem* parentItem) { DEBUG("HDF5FilterPrivate::scanHDF5Group()"); //check for hard link H5G_stat_t statbuf; m_status = H5Gget_objinfo(gid, ".", true, &statbuf); handleError(m_status, "H5Gget_objinfo"); if (statbuf.nlink > 1) { if (m_multiLinkList.contains(statbuf.objno[0])) { QTreeWidgetItem* objectItem = new QTreeWidgetItem(QStringList()<setIcon(0, QIcon::fromTheme("link")); objectItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(objectItem); return; } else { m_multiLinkList.append(statbuf.objno[0]); DEBUG(" group multiple links: "<setIcon(0, QIcon::fromTheme("folder")); groupItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(groupItem); hsize_t numObj; m_status = H5Gget_num_objs(gid, &numObj); handleError(m_status, "H5Gget_num_objs"); for (unsigned int i = 0; i < numObj; ++i) { char memberName[MAXNAMELENGTH]; m_status = H5Gget_objname_by_idx(gid, (hsize_t)i, memberName, (size_t)MAXNAMELENGTH ); handleError(m_status, "H5Gget_objname_by_idx"); int otype = H5Gget_objtype_by_idx(gid, (size_t)i ); handleError(otype, "H5Gget_objtype_by_idx"); switch (otype) { case H5G_LINK: { scanHDF5Link(gid, memberName, groupItem); break; } case H5G_GROUP: { hid_t grpid = H5Gopen(gid, memberName, H5P_DEFAULT); handleError((int)grpid, "H5Gopen"); scanHDF5Group(grpid, memberName, groupItem); m_status = H5Gclose(grpid); handleError(m_status, "H5Gclose"); break; } case H5G_DATASET: { hid_t dsid = H5Dopen(gid, memberName, H5P_DEFAULT); handleError((int)dsid, "H5Dopen"); scanHDF5DataSet(dsid, memberName, groupItem); m_status = H5Dclose(dsid); handleError(m_status, "H5Dclose"); break; } case H5G_TYPE: { hid_t tid = H5Topen(gid, memberName, H5P_DEFAULT); handleError((int)tid, "H5Topen"); scanHDF5DataType(tid, memberName, groupItem); m_status = H5Tclose(tid); handleError(m_status, "H5Tclose"); break; } default: QTreeWidgetItem* objectItem = new QTreeWidgetItem(QStringList() << QString(memberName) << i18n("unknown")); objectItem->setFlags(Qt::ItemIsEnabled); groupItem->addChild(objectItem); break; } } } #endif /*! parses the content of the file \c fileName and fill the tree using rootItem. */ void HDF5FilterPrivate::parse(const QString& fileName, QTreeWidgetItem* rootItem) { DEBUG("HDF5FilterPrivate::parse()"); #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); DEBUG("fileName = " << bafileName.data()); // check file type first htri_t isHdf5 = H5Fis_hdf5(bafileName.data()); if (isHdf5 == 0) { DEBUG(bafileName.data() << " is not a HDF5 file! Giving up."); return; } if (isHdf5 < 0) { DEBUG("H5Fis_hdf5() failed on " << bafileName.data() << "! Giving up."); return; } // open file hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); handleError((int)file, "H5Fopen", fileName); if (file < 0) { DEBUG("Opening file " << bafileName.data() << " failed! Giving up."); return; } char rootName[] = "/"; hid_t group = H5Gopen(file, rootName, H5P_DEFAULT); handleError((int)group, "H5Gopen", rootName); // multiLinkList.clear(); crashes scanHDF5Group(group, rootName, rootItem); m_status = H5Gclose(group); handleError(m_status, "H5Gclose", QString()); m_status = H5Fclose(file); handleError(m_status, "H5Fclose", QString()); #else DEBUG("HDF5 not available"); Q_UNUSED(fileName) Q_UNUSED(rootItem) #endif } /*! reads the content of the date set in the file \c fileName to a string (for preview) or to the data source. */ QVector HDF5FilterPrivate::readCurrentDataSet(const QString& fileName, AbstractDataSource* dataSource, bool &ok, AbstractFileFilter::ImportMode mode, int lines) { DEBUG("HDF5Filter::readCurrentDataSet()"); QVector dataStrings; if (currentDataSetName.isEmpty()) { //return QString("No data set selected").replace(' ',QChar::Nbsp); ok = false; return dataStrings << (QStringList() << i18n("No data set selected")); } DEBUG(" current data set = " << currentDataSetName.toStdString()); #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); handleError((int)file, "H5Fopen", fileName); QByteArray badataSet = currentDataSetName.toLatin1(); hid_t dataset = H5Dopen2(file, badataSet.data(), H5P_DEFAULT); handleError((int)file, "H5Dopen2", currentDataSetName); // Get datatype and dataspace hid_t dtype = H5Dget_type(dataset); handleError((int)dtype, "H5Dget_type"); H5T_class_t dclass = H5Tget_class(dtype); handleError((int)dclass, "H5Dget_class"); size_t typeSize = H5Tget_size(dtype); handleError((int)(typeSize-1), "H5Dget_size"); hid_t dataspace = H5Dget_space(dataset); handleError((int)dataspace, "H5Dget_space"); int rank = H5Sget_simple_extent_ndims(dataspace); handleError(rank, "H5Dget_simple_extent_ndims"); DEBUG(" rank = " << rank); int columnOffset = 0; // offset to import data int actualRows = 0, actualCols = 0; // rows and cols to read // dataContainer is used to store the data read from the dataSource // it contains the pointers of all columns // initially there is one pointer set to nullptr // check for dataContainer[0] != nullptr to decide if dataSource can be used - QVector dataContainer(1, nullptr); + std::vector dataContainer(1, nullptr); // rank= 0: single value, 1: vector, 2: matrix, 3: 3D data, ... switch (rank) { case 0: { // single value actualCols = 1; switch (dclass) { case H5T_STRING: { char* data = (char *) malloc(typeSize * sizeof(char)); hid_t memtype = H5Tcopy(H5T_C_S1); handleError((int)memtype, "H5Tcopy"); m_status = H5Tset_size(memtype, typeSize); handleError(m_status, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Tread"); dataStrings << (QStringList() << data); free(data); break; } case H5T_INTEGER: case H5T_FLOAT: case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_COMPOUND: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataStrings << (QStringList() << i18n("rank 0 not implemented yet for type %1", translateHDF5Class(dclass))); QDEBUG(dataStrings); } default: break; } break; } case 1: { // vector hsize_t size, maxSize; m_status = H5Sget_simple_extent_dims(dataspace, &size, &maxSize); handleError(m_status, "H5Sget_simple_extent_dims"); int rows = size; if (endRow == -1) endRow = rows; if (lines == -1) lines = endRow; actualRows = endRow - startRow + 1; actualCols = 1; #ifndef NDEBUG H5T_order_t order = H5Tget_order(dtype); handleError((int)order, "H5Sget_order"); #endif QDEBUG(translateHDF5Class(dclass) << '(' << typeSize << ')' << translateHDF5Order(order) << ", rows:" << rows << " max:" << maxSize); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); // use current data set name (without path) for column name QStringList vectorNames = {currentDataSetName.mid(currentDataSetName.lastIndexOf("/") + 1)}; QDEBUG(" vector names = " << vectorNames) if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); QStringList dataString; // data saved in a list switch (dclass) { case H5T_STRING: { DEBUG("rank 1 H5T_STRING"); hid_t memtype = H5Tcopy(H5T_C_S1); handleError((int)memtype, "H5Tcopy"); char** data = (char **) malloc(rows * sizeof (char *)); if (H5Tis_variable_str(dtype)) { m_status = H5Tset_size(memtype, H5T_VARIABLE); handleError((int)memtype, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Dread"); } else { data[0] = (char *) malloc(rows * typeSize * sizeof (char)); for (int i = 1; i < rows; ++i) data[i] = data[0] + i * typeSize; m_status = H5Tset_size(memtype, typeSize); handleError((int)memtype, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data[0]); handleError(m_status, "H5Dread"); } for (int i = startRow-1; i < qMin(endRow, lines + startRow - 1); ++i) dataString << data[i]; free(data); break; } case H5T_INTEGER: { if (H5Tequal(dtype, H5T_STD_I8LE)) dataString = readHDF5Data1D(dataset, H5T_STD_I8LE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I8BE)) dataString = readHDF5Data1D(dataset, H5T_STD_I8BE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 2: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 4: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 8: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; } } else if (H5Tequal(dtype, H5T_STD_U8LE)) dataString = readHDF5Data1D(dataset, H5T_STD_U8LE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U8BE)) dataString = readHDF5Data1D(dataset, H5T_STD_U8BE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 2: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 4: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 8: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; } } else if (H5Tequal(dtype, H5T_STD_I16LE) || H5Tequal(dtype, H5T_STD_I16BE) || H5Tequal(dtype, H5T_NATIVE_SHORT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_SHORT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U16LE) || H5Tequal(dtype, H5T_STD_U16BE) || H5Tequal(dtype, H5T_NATIVE_USHORT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_USHORT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I32LE) || H5Tequal(dtype, H5T_STD_I32BE) || H5Tequal(dtype, H5T_NATIVE_INT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_INT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U32LE) || H5Tequal(dtype, H5T_STD_U32BE) || H5Tequal(dtype, H5T_NATIVE_UINT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_UINT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_LONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_ULONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_ULONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I64LE) || H5Tequal(dtype, H5T_STD_I64BE) || H5Tequal(dtype, H5T_NATIVE_LLONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LLONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U64LE) || H5Tequal(dtype, H5T_STD_U64BE) || H5Tequal(dtype, H5T_NATIVE_ULLONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_ULLONG, rows, lines, dataContainer[0]); else { ok = false; dataString = (QStringList() << i18n("unsupported integer type for rank 1")); QDEBUG(dataString); } break; } case H5T_FLOAT: { if (H5Tequal(dtype, H5T_IEEE_F32LE) || H5Tequal(dtype, H5T_IEEE_F32BE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_FLOAT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_IEEE_F64LE) || H5Tequal(dtype, H5T_IEEE_F64BE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_DOUBLE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LDOUBLE, rows, lines, dataContainer[0]); else { ok = false; dataString = (QStringList() << i18n("unsupported float type for rank 1")); QDEBUG(dataString); } break; } case H5T_COMPOUND: { int members = H5Tget_nmembers(dtype); handleError(members, "H5Tget_nmembers"); if (dataSource) { // re-create data pointer dataContainer.clear(); dataSource->prepareImport(dataContainer, mode, actualRows, members, vectorNames, columnModes); } else dataStrings << readHDF5Compound(dtype); dataString = readHDF5CompoundData1D(dataset, dtype, rows, lines, dataContainer); break; } case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataString = (QStringList() << i18n("rank 1 not implemented yet for type %1", translateHDF5Class(dclass))); QDEBUG(dataString); } default: break; } if (!dataSource) { // preview QDEBUG("dataString =" << dataString); DEBUG(" data string size = " << dataString.size()); DEBUG(" rows = " << rows << ", lines = " << lines << ", actual rows = " << actualRows); for (int i = 0; i < qMin(actualRows, lines); ++i) dataStrings << (QStringList() << dataString[i]); } break; } case 2: { // matrix hsize_t dims_out[2]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, nullptr); handleError(m_status, "H5Sget_simple_extent_dims"); int rows = dims_out[0]; int cols = dims_out[1]; if (endRow == -1) endRow = rows; if (lines == -1) lines = endRow; if (endColumn == -1) endColumn = cols; actualRows = endRow-startRow+1; actualCols = endColumn-startColumn+1; #ifndef NDEBUG H5T_order_t order = H5Tget_order(dtype); handleError((int)order, "H5Tget_order"); #endif QDEBUG(translateHDF5Class(dclass) << '(' << typeSize << ')' << translateHDF5Order(order) << "," << rows << "x" << cols); DEBUG("startRow/endRow" << startRow << endRow); DEBUG("startColumn/endColumn" << startColumn << endColumn); DEBUG("actual rows/cols" << actualRows << actualCols); DEBUG("lines" << lines); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); // use current data set name (without path) append by "_" and column number for column names QStringList vectorNames; QString colName = currentDataSetName.mid(currentDataSetName.lastIndexOf("/") + 1); for (int i = 0; i < actualCols; i++) vectorNames << colName + QLatin1String("_") + QString::number(i + 1); QDEBUG(" vector names = " << vectorNames) if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); // read data switch (dclass) { case H5T_INTEGER: { if (H5Tequal(dtype, H5T_STD_I8LE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_I8LE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I8BE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_I8BE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 2: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 4: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 8: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; } } else if (H5Tequal(dtype, H5T_STD_U8LE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_U8LE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U8BE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_U8BE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 2: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 4: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 8: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; } } else if (H5Tequal(dtype, H5T_STD_I16LE) || H5Tequal(dtype, H5T_STD_I16BE) || H5Tequal(dtype, H5T_NATIVE_SHORT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_SHORT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U16LE) || H5Tequal(dtype, H5T_STD_U16BE) || H5Tequal(dtype, H5T_NATIVE_USHORT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_USHORT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I32LE) || H5Tequal(dtype, H5T_STD_I32BE) || H5Tequal(dtype, H5T_NATIVE_INT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_INT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U32LE) || H5Tequal(dtype, H5T_STD_U32BE) || H5Tequal(dtype, H5T_NATIVE_UINT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UINT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_LONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_ULONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_ULONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I64LE) || H5Tequal(dtype, H5T_STD_I64BE) || H5Tequal(dtype, H5T_NATIVE_LLONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LLONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U64LE) || H5Tequal(dtype, H5T_STD_U64BE) || H5Tequal(dtype, H5T_NATIVE_ULLONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_ULLONG, rows, cols, lines, dataContainer); else { ok = false; dataStrings << (QStringList() << i18n("unsupported integer type for rank 2")); QDEBUG(dataStrings); } break; } case H5T_FLOAT: { if (H5Tequal(dtype, H5T_IEEE_F32LE) || H5Tequal(dtype, H5T_IEEE_F32BE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_FLOAT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_IEEE_F64LE) || H5Tequal(dtype, H5T_IEEE_F64BE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_DOUBLE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LDOUBLE, rows, cols, lines, dataContainer); else { ok = false; dataStrings << (QStringList() << i18n("unsupported float type for rank 2")); QDEBUG(dataStrings); } break; } case H5T_COMPOUND: { dataStrings << readHDF5Compound(dtype); QDEBUG(dataStrings); dataStrings << readHDF5CompoundData2D(dataset,dtype,rows,cols,lines); break; } case H5T_STRING: { // TODO: implement this ok = false; dataStrings << (QStringList() << i18n("rank 2 not implemented yet for type %1, size = %2", translateHDF5Class(dclass), typeSize)); QDEBUG(dataStrings); break; } case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataStrings << (QStringList() << i18n("rank 2 not implemented yet for type %1", translateHDF5Class(dclass))); QDEBUG(dataStrings); } default: break; } break; } default: { // 3D or more data ok = false; dataStrings << (QStringList() << i18n("rank %1 not implemented yet for type %2", rank, translateHDF5Class(dclass))); QDEBUG(dataStrings); } } m_status = H5Sclose(dataspace); handleError(m_status, "H5Sclose"); m_status = H5Tclose(dtype); handleError(m_status, "H5Tclose"); m_status = H5Dclose(dataset); handleError(m_status, "H5Dclose"); m_status = H5Fclose(file); handleError(m_status, "H5Fclose"); if (!dataSource) return dataStrings; dataSource->finalizeImport(columnOffset, 1, actualCols, QString(), mode); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(mode) Q_UNUSED(lines) #endif return dataStrings; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void HDF5FilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { DEBUG("HDF5Filter::readDataFromFile()"); if (currentDataSetName.isEmpty()) { DEBUG("WARNING: No data set selected"); return; } bool ok = true; readCurrentDataSet(fileName, dataSource, ok, mode); } /*! writes the content of \c dataSource to the file \c fileName. */ void HDF5FilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: writing HDF5 not implemented yet } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void HDF5Filter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("hdfFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool HDF5Filter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/datasources/filters/HDF5FilterPrivate.h b/src/backend/datasources/filters/HDF5FilterPrivate.h index f995b24c6..c493a7f78 100644 --- a/src/backend/datasources/filters/HDF5FilterPrivate.h +++ b/src/backend/datasources/filters/HDF5FilterPrivate.h @@ -1,90 +1,90 @@ /*************************************************************************** File : HDF5FilterPrivate.h Project : LabPlot Description : Private implementation class for HDF5Filter. -------------------------------------------------------------------- Copyright : (C) 2015-2018 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef HDF5FILTERPRIVATE_H #define HDF5FILTERPRIVATE_H #include #ifdef HAVE_HDF5 #include #endif class AbstractDataSource; class HDF5FilterPrivate { public: explicit HDF5FilterPrivate(HDF5Filter*); - + #ifdef HAVE_HDF5 static void handleError(int err, const QString& function, const QString& arg = QString()); #endif void parse(const QString & fileName, QTreeWidgetItem* rootItem); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); QVector readCurrentDataSet(const QString& fileName, AbstractDataSource*, bool &ok, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); const HDF5Filter* q; QString currentDataSetName; int startRow{1}; int endRow{-1}; int startColumn{1}; int endColumn{-1}; private: #ifdef HAVE_HDF5 int m_status; #endif const static int MAXNAMELENGTH = 1024; const static int MAXSTRINGLENGTH = 1024*1024; QList m_multiLinkList; // used to find hard links #ifdef HAVE_HDF5 QString translateHDF5Order(H5T_order_t); QString translateHDF5Type(hid_t); QString translateHDF5Class(H5T_class_t); QStringList readHDF5Compound(hid_t tid); template QStringList readHDF5Data1D(hid_t dataset, hid_t type, int rows, int lines, void* dataPointer = nullptr); - QStringList readHDF5CompoundData1D(hid_t dataset, hid_t tid, int rows, int lines, QVector& dataPointer); + QStringList readHDF5CompoundData1D(hid_t dataset, hid_t tid, int rows, int lines, std::vector& dataPointer); template QVector readHDF5Data2D(hid_t dataset, hid_t ctype, int rows, int cols, int lines, - QVector& dataPointer); + std::vector& dataPointer); QVector readHDF5CompoundData2D(hid_t dataset, hid_t tid, int rows, int cols, int lines); QStringList readHDF5Attr(hid_t aid); QStringList scanHDF5Attrs(hid_t oid); QStringList readHDF5DataType(hid_t tid); QStringList readHDF5PropertyList(hid_t pid); void scanHDF5DataType(hid_t tid, char* dataTypeName, QTreeWidgetItem* parentItem); void scanHDF5Link(hid_t gid, char* linkName, QTreeWidgetItem* parentItem); void scanHDF5DataSet(hid_t dsid, char* dataSetName, QTreeWidgetItem* parentItem); void scanHDF5Group(hid_t gid, char* groupName, QTreeWidgetItem* parentItem); #endif }; #endif diff --git a/src/backend/datasources/filters/ImageFilter.cpp b/src/backend/datasources/filters/ImageFilter.cpp index ce5edca37..2a93511b1 100644 --- a/src/backend/datasources/filters/ImageFilter.cpp +++ b/src/backend/datasources/filters/ImageFilter.cpp @@ -1,294 +1,294 @@ /*************************************************************************** File : ImageFilter.cpp Project : LabPlot Description : Image I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015 by Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/ImageFilter.h" #include "backend/datasources/filters/ImageFilterPrivate.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/core/column/Column.h" #include #include /*! \class ImageFilter \brief Manages the import/export of data from/to an image file. \ingroup datasources */ ImageFilter::ImageFilter():AbstractFileFilter(Image), d(new ImageFilterPrivate(this)) {} ImageFilter::~ImageFilter() = default; /*! returns the list of all predefined import formats. */ QStringList ImageFilter::importFormats() { return (QStringList() << i18n("Matrix (grayscale)") << i18n("XYZ (grayscale)") << i18n("XYRGB") ); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ void ImageFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void ImageFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void ImageFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void ImageFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void ImageFilter::setImportFormat(const ImageFilter::ImportFormat f) { d->importFormat = f; } ImageFilter::ImportFormat ImageFilter::importFormat() const { return d->importFormat; } void ImageFilter::setStartRow(const int s) { d->startRow = s; } int ImageFilter::startRow() const { return d->startRow; } void ImageFilter::setEndRow(const int e) { d->endRow = e; } int ImageFilter::endRow() const { return d->endRow; } void ImageFilter::setStartColumn(const int s) { d->startColumn = s; } int ImageFilter::startColumn() const { return d->startColumn; } void ImageFilter::setEndColumn(const int e) { d->endColumn = e; } int ImageFilter::endColumn() const { return d->endColumn; } QString ImageFilter::fileInfoString(const QString& fileName) { DEBUG("ImageFilter::fileInfoString()"); QString info; //TODO Q_UNUSED(fileName); return info; } //##################################################################### //################### Private implementation ########################## //##################################################################### ImageFilterPrivate::ImageFilterPrivate(ImageFilter* owner) : q(owner) {} /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void ImageFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { QImage image = QImage(fileName); if (image.isNull() || image.format() == QImage::Format_Invalid) { #ifdef QT_DEBUG qDebug()<<"failed to read image"<*>(dataContainer[j])->operator[](i) = value; } emit q->completed(100*i/actualRows); } break; } case ImageFilter::XYZ: { int currentRow = 0; for (int i = startRow-1; i < endRow; ++i) { for (int j = startColumn-1; j < endColumn; ++j) { QRgb color = image.pixel(j, i); static_cast*>(dataContainer[0])->operator[](currentRow) = i+1; static_cast*>(dataContainer[1])->operator[](currentRow) = j+1; static_cast*>(dataContainer[2])->operator[](currentRow) = qGray(color); currentRow++; } emit q->completed(100*i/actualRows); } break; } case ImageFilter::XYRGB: { int currentRow = 0; for (int i = startRow-1; i < endRow; ++i) { for ( int j = startColumn-1; j < endColumn; ++j) { QRgb color = image.pixel(j, i); static_cast*>(dataContainer[0])->operator[](currentRow) = i+1; static_cast*>(dataContainer[1])->operator[](currentRow) = j+1; static_cast*>(dataContainer[2])->operator[](currentRow) = qRed(color); static_cast*>(dataContainer[3])->operator[](currentRow) = qGreen(color); static_cast*>(dataContainer[4])->operator[](currentRow) = qBlue(color); currentRow++; } emit q->completed(100*i/actualRows); } break; } } auto* spreadsheet = dynamic_cast(dataSource); if (spreadsheet) { QString comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); for ( int n = 0; n < actualCols; ++n) { Column* column = spreadsheet->column(columnOffset+n); column->setComment(comment); column->setUndoAware(true); if (mode == AbstractFileFilter::Replace) { column->setSuppressDataChangedSignal(false); column->setChanged(); } } } dataSource->finalizeImport(); return; } /*! writes the content of \c dataSource to the file \c fileName. */ void ImageFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void ImageFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("imageFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool ImageFilter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/datasources/filters/JsonFilterPrivate.h b/src/backend/datasources/filters/JsonFilterPrivate.h index 899abd860..0b0637c04 100644 --- a/src/backend/datasources/filters/JsonFilterPrivate.h +++ b/src/backend/datasources/filters/JsonFilterPrivate.h @@ -1,97 +1,97 @@ /*************************************************************************** File : JsonFilterPrivate.h Project : LabPlot Description : Private implementation class for JsonFilter. -------------------------------------------------------------------- -------------------------------------------------------------------- Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef JSONFILTERPRIVATE_H #define JSONFILTERPRIVATE_H #include "QJsonModel.h" class QJsonDocument; class KFilterDev; class AbstractDataSource; class AbstractColumn; class JsonFilterPrivate { public: explicit JsonFilterPrivate (JsonFilter* owner); int checkRow(QJsonValueRef value, int &countCols); int parseColumnModes(QJsonValue row, QString rowName = QString()); void setEmptyValue(int column, int row); void setValueFromString(int column, int row, QString value); int prepareDeviceToRead(QIODevice&); int prepareDocumentToRead(const QJsonDocument&); void readDataFromDevice(QIODevice& device, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void readDataFromDocument(const QJsonDocument& doc, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void importData(AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName); QVector preview(QIODevice& device); QVector preview(QJsonDocument& doc); QVector preview(); const JsonFilter* q; QJsonModel* model; JsonFilter::DataContainerType containerType{JsonFilter::Object}; QJsonValue::Type rowType{QJsonValue::Object}; QVector modelRows; QString dateTimeFormat; QLocale::Language numberFormat{QLocale::C}; double nanValue{NAN}; bool createIndexEnabled{false}; bool importObjectNames{false}; QStringList vectorNames; QVector columnModes; int startRow{1}; // start row int endRow{-1}; // end row int startColumn{1}; // start column int endColumn{-1}; // end column private: int m_actualRows{0}; int m_actualCols{0}; int m_prepared{false}; int m_columnOffset{0}; // indexes the "start column" in the datasource. Data will be imported starting from this column. - QVector m_dataContainer; // pointers to the actual data containers (columns). + std::vector m_dataContainer; // pointers to the actual data containers (columns). QJsonDocument m_preparedDoc; // parsed Json document }; #endif diff --git a/src/backend/datasources/filters/NetCDFFilter.cpp b/src/backend/datasources/filters/NetCDFFilter.cpp index 123f74fd4..21e5e15f9 100644 --- a/src/backend/datasources/filters/NetCDFFilter.cpp +++ b/src/backend/datasources/filters/NetCDFFilter.cpp @@ -1,994 +1,994 @@ /*************************************************************************** File : NetCDFFilter.cpp Project : LabPlot Description : NetCDF I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2019 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/NetCDFFilter.h" #include "backend/datasources/filters/NetCDFFilterPrivate.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/core/column/Column.h" #include #include ///////////// macros /////////////////////////////////////////////// #define NC_GET_ATT(type, ftype) \ auto* value = (type*)malloc(len*sizeof(type)); \ m_status = nc_get_att_ ##ftype(ncid, varid, name, value); \ handleError(m_status, "nc_get_att_" #ftype); \ for (unsigned int l = 0; l < len; l++) \ valueString << QString::number(value[l]); \ free(value); #define NC_SCAN_VAR(type, ftype) \ type data; \ m_status = nc_get_var_ ##ftype(ncid, i, &data); \ handleError(m_status, "nc_get_var_" #ftype); \ rowStrings << QString::number(data); #define NC_READ_VAR(type, ftype, dtype) \ type data; \ m_status = nc_get_var_ ##ftype(ncid, varid, &data); \ handleError(m_status, "nc_get_var_" #ftype); \ \ if (dataSource) { \ dtype *sourceData = static_cast*>(dataContainer[0])->data(); \ sourceData[0] = (dtype)data; \ } else { /* preview */ \ dataStrings << (QStringList() << QString::number(data)); \ } #define NC_READ_AVAR(type, ftype, dtype) \ auto* data = new type[(unsigned int)actualRows]; \ \ size_t start = (size_t)(startRow - 1), count = (size_t)actualRows; \ m_status = nc_get_vara_ ##ftype(ncid, varid, &start, &count, data); \ handleError(m_status, "nc_get_vara_" #ftype); \ \ if (dataSource) { \ dtype *sourceData = static_cast*>(dataContainer[0])->data(); \ for (int i = 0; i < actualRows; i++) \ sourceData[i] = (dtype)data[i]; \ } else { /* preview */ \ for (int i = 0; i < qMin(actualRows, lines); i++) \ dataStrings << (QStringList() << QString::number(data[i])); \ } \ delete[] data; // for native types (atm: int, double) #define NC_READ_AVAR_NATIVE(type) \ type* data = nullptr; \ if (dataSource) \ data = static_cast*>(dataContainer[0])->data(); \ else \ data = new type[(unsigned int)actualRows]; \ \ size_t start = (size_t)(startRow - 1), count = (size_t)actualRows; \ m_status = nc_get_vara_ ##type(ncid, varid, &start, &count, data); \ handleError(m_status, "nc_get_vara_" #type); \ \ if (!dataSource) { /* preview */ \ for (int i = 0; i < qMin(actualRows, lines); i++) \ dataStrings << (QStringList() << QString::number(data[i])); \ delete[] data; \ } #define NC_READ_VAR2(type, ftype, dtype) \ auto** data = (type**) malloc(rows * sizeof(type*)); \ data[0] = (type*)malloc(cols * rows * sizeof(type)); \ for (unsigned int i = 1; i < rows; i++) \ data[i] = data[0] + i*cols; \ \ m_status = nc_get_var_ ##ftype(ncid, varid, &data[0][0]); \ handleError(m_status, "nc_get_var_" #ftype); \ \ if (m_status == NC_NOERR) { \ for (int i = 0; i < qMin((int)rows, lines); i++) { \ QStringList line; \ for (size_t j = 0; j < cols; j++) { \ if (dataSource && dataContainer[0]) \ static_cast*>(dataContainer[(int)(j-(size_t)startColumn+1)])->operator[](i-startRow+1) = data[i][(int)j]; \ else \ line << QString::number(data[i][j]); \ } \ dataStrings << line; \ emit q->completed(100*i/actualRows); \ } \ } \ free(data[0]); \ free(data); ////////////////////////////////////////////////////////////////////// /*! \class NetCDFFilter \brief Manages the import/export of data from/to a NetCDF file. \ingroup datasources */ NetCDFFilter::NetCDFFilter():AbstractFileFilter(NETCDF), d(new NetCDFFilterPrivate(this)) {} NetCDFFilter::~NetCDFFilter() = default; /*! parses the content of the file \c ileName. */ void NetCDFFilter::parse(const QString & fileName, QTreeWidgetItem* rootItem) { d->parse(fileName, rootItem); } /*! reads the content of the selected attribute from file \c fileName. */ QString NetCDFFilter::readAttribute(const QString & fileName, const QString & name, const QString & varName) { return d->readAttribute(fileName, name, varName); } /*! reads the content of the current variable from file \c fileName. */ QVector NetCDFFilter::readCurrentVar(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { return d->readCurrentVar(fileName, dataSource, importMode, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ void NetCDFFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { d->readDataFromFile(fileName, dataSource, mode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void NetCDFFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void NetCDFFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void NetCDFFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void NetCDFFilter::setCurrentVarName(const QString& ds) { d->currentVarName = ds; } const QString NetCDFFilter::currentVarName() const { return d->currentVarName; } void NetCDFFilter::setStartRow(const int s) { d->startRow = s; } int NetCDFFilter::startRow() const { return d->startRow; } void NetCDFFilter::setEndRow(const int e) { d->endRow = e; } int NetCDFFilter::endRow() const { return d->endRow; } void NetCDFFilter::setStartColumn(const int c) { d->startColumn = c; } int NetCDFFilter::startColumn() const { return d->startColumn; } void NetCDFFilter::setEndColumn(const int c) { d->endColumn = c; } int NetCDFFilter::endColumn() const { return d->endColumn; } QString NetCDFFilter::fileInfoString(const QString& fileName) { DEBUG("NetCDFFilter::fileInfoString()"); QByteArray bafileName = fileName.toLatin1(); DEBUG("fileName = " << bafileName.data()); QString info; #ifdef HAVE_NETCDF int ncid, status; status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); NetCDFFilterPrivate::handleError(status, "nc_open"); if (status != NC_NOERR) { DEBUG(" File error. Giving up"); return i18n("Error opening file"); } int ndims, nvars, nattr, uldid; status = nc_inq(ncid, &ndims, &nvars, &nattr, &uldid); NetCDFFilterPrivate::handleError(status, "nc_inq"); DEBUG(" nattr/ndims/nvars = " << nattr << ' ' << ndims << ' ' << nvars); if (status == NC_NOERR) { info += i18n("Number of global attributes: %1", QString::number(nattr)); info += QLatin1String("
"); info += i18n("Number of dimensions: %1", QString::number(ndims)); info += QLatin1String("
"); info += i18n("Number of variables: %1", QString::number(nvars)); info += QLatin1String("
"); int format; status = nc_inq_format(ncid, &format); if (status == NC_NOERR) info += i18n("Format version: %1", NetCDFFilterPrivate::translateFormat(format)); info += QLatin1String("
"); info += i18n("Using library version %1", QString(nc_inq_libvers())); } else { info += i18n("Error getting file info"); } status = ncclose(ncid); NetCDFFilterPrivate::handleError(status, "nc_close"); #endif return info; } /*! * Get file content in CDL (Common Data form Language) format * uses "ncdump" */ QString NetCDFFilter::fileCDLString(const QString& fileName) { DEBUG("NetCDFFilter::fileCDLString()"); QString CDLString; #ifdef Q_OS_LINUX auto* proc = new QProcess(); QStringList args; args << "-cs" << fileName; proc->start( "ncdump", args); if (proc->waitForReadyRead(1000) == false) CDLString += i18n("Reading from file %1 failed.", fileName); else { CDLString += proc->readAll(); CDLString.replace('\n', "
\n"); CDLString.replace("\t","    "); //DEBUG(" CDL string: " << CDLString.toStdString()); } #else //TODO: ncdump on Win, Mac Q_UNUSED(fileName) #endif return CDLString; } //##################################################################### //################### Private implementation ########################## //##################################################################### NetCDFFilterPrivate::NetCDFFilterPrivate(NetCDFFilter* owner) : q(owner) { #ifdef HAVE_NETCDF m_status = 0; #endif } #ifdef HAVE_NETCDF void NetCDFFilterPrivate::handleError(int err, const QString& function) { if (err != NC_NOERR) { DEBUG("NETCDF ERROR:" << function.toStdString() << "() - " << nc_strerror(err)); return; } Q_UNUSED(function); } QString NetCDFFilterPrivate::translateDataType(nc_type type) { QString typeString; switch (type) { case NC_BYTE: typeString = "BYTE"; break; case NC_UBYTE: typeString = "UBYTE"; break; case NC_CHAR: typeString = "CHAR"; break; case NC_SHORT: typeString = "SHORT"; break; case NC_USHORT: typeString = "USHORT"; break; case NC_INT: typeString = "INT"; break; case NC_UINT: typeString = "UINT"; break; case NC_INT64: typeString = "INT64"; break; case NC_UINT64: typeString = "UINT64"; break; case NC_FLOAT: typeString = "FLOAT"; break; case NC_DOUBLE: typeString = "DOUBLE"; break; case NC_STRING: typeString = "STRING"; break; default: typeString = "UNKNOWN"; } return typeString; } QString NetCDFFilterPrivate::translateFormat(int format) { QString formatString; switch (format) { case NC_FORMAT_CLASSIC: formatString = "NC_FORMAT_CLASSIC"; break; case NC_FORMAT_64BIT: formatString = "NC_FORMAT_64BIT"; break; case NC_FORMAT_NETCDF4: formatString = "NC_FORMAT_NETCDF4"; break; case NC_FORMAT_NETCDF4_CLASSIC: formatString = "NC_FORMAT_NETCDF4_CLASSIC"; break; } return formatString; } QString NetCDFFilterPrivate::scanAttrs(int ncid, int varid, int attid, QTreeWidgetItem* parentItem) { char name[NC_MAX_NAME + 1]; int nattr, nstart = 0; if (attid == -1) { m_status = nc_inq_varnatts(ncid, varid, &nattr); handleError(m_status, "nc_inq_varnatts"); } else { nstart = attid; nattr = attid + 1; } nc_type type; size_t len; QStringList valueString; for (int i = nstart; i < nattr; i++) { valueString.clear(); m_status = nc_inq_attname(ncid, varid, i, name); handleError(m_status, "nc_inq_attname"); m_status = nc_inq_att(ncid, varid, name, &type, &len); handleError(m_status, "nc_inq_att"); QDEBUG(" attr" << i+1 << "name/type/len =" << name << translateDataType(type) << len); //read attribute switch (type) { case NC_BYTE: { NC_GET_ATT(signed char, schar); break; } case NC_UBYTE: { NC_GET_ATT(unsigned char, uchar); break; } case NC_CHAR: { // not number char *value = (char *)malloc((len+1)*sizeof(char)); m_status = nc_get_att_text(ncid, varid, name, value); handleError(m_status, "nc_get_att_text"); value[len] = 0; valueString << value; free(value); break; } case NC_SHORT: { NC_GET_ATT(short, short); break; } case NC_USHORT: { NC_GET_ATT(unsigned short, ushort); break; } case NC_INT: { NC_GET_ATT(int, int); break; } case NC_UINT: { NC_GET_ATT(unsigned int, uint); break; } case NC_INT64: { NC_GET_ATT(long long, longlong); break; } case NC_UINT64: { NC_GET_ATT(unsigned long long, ulonglong); break; } case NC_FLOAT: { NC_GET_ATT(float, float); break; } case NC_DOUBLE: { NC_GET_ATT(double, double); break; } //TODO: NC_STRING default: valueString << "not supported"; } if (parentItem != nullptr) { QString typeName; if (varid == NC_GLOBAL) typeName = i18n("global attribute"); else { char varName[NC_MAX_NAME + 1]; m_status = nc_inq_varname(ncid, varid, varName); typeName = i18n("%1 attribute", QString(varName)); } QStringList props; props << translateDataType(type) << " (" << QString::number(len) << ")"; QTreeWidgetItem *attrItem = new QTreeWidgetItem(QStringList() << QString(name) << typeName << props.join(QString()) << valueString.join(", ")); attrItem->setIcon(0, QIcon::fromTheme("accessories-calculator")); attrItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(attrItem); } } return valueString.join("\n"); } void NetCDFFilterPrivate::scanDims(int ncid, int ndims, QTreeWidgetItem* parentItem) { int ulid; m_status = nc_inq_unlimdim(ncid, &ulid); handleError(m_status, "nc_inq_att"); char name[NC_MAX_NAME + 1]; size_t len; for (int i = 0; i < ndims; i++) { m_status = nc_inq_dim(ncid, i, name, &len); handleError(m_status, "nc_inq_att"); DEBUG(" dim" << i+1 << ": name/len =" << name << len); QStringList props; props<setIcon(0, QIcon::fromTheme("accessories-calculator")); attrItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(attrItem); } } void NetCDFFilterPrivate::scanVars(int ncid, int nvars, QTreeWidgetItem* parentItem) { char name[NC_MAX_NAME + 1]; nc_type type; int ndims, nattrs; int dimids[NC_MAX_VAR_DIMS]; for (int i = 0; i < nvars; i++) { m_status = nc_inq_var(ncid, i, name, &type, &ndims, dimids, &nattrs); handleError(m_status, "nc_inq_att"); QDEBUG(" var" << i+1 << ": name/type=" << name << translateDataType(type)); DEBUG(" ndims/nattr" << ndims << nattrs); QStringList props; // properties column props << translateDataType(type); char dname[NC_MAX_NAME + 1]; size_t dlen; props << "("; if (ndims == 0) props << QString::number(0); for (int j = 0; j < ndims; j++) { m_status = nc_inq_dim(ncid, dimids[j], dname, &dlen); if (j != 0) props << "x"; props << QString::number(dlen); } props << ")"; QStringList rowStrings; rowStrings << QString(name) << i18n("variable") << props.join(QString()); if (ndims == 0) {// get value of zero dim var switch (type) { case NC_BYTE: { NC_SCAN_VAR(signed char, schar); break; } case NC_UBYTE: { NC_SCAN_VAR(unsigned char, uchar); break; } case NC_CHAR: { // not number char data; m_status = nc_get_var_text(ncid, i, &data); handleError(m_status, "nc_get_var_text"); rowStrings << QString(data); break; } case NC_SHORT: { NC_SCAN_VAR(short, short); break; } case NC_USHORT: { NC_SCAN_VAR(unsigned short, ushort); break; } case NC_INT: { NC_SCAN_VAR(int, int); break; } case NC_UINT: { NC_SCAN_VAR(unsigned int, uint); break; } case NC_INT64: { NC_SCAN_VAR(long long, longlong); break; } case NC_UINT64: { NC_SCAN_VAR(unsigned long long, ulonglong); break; } case NC_DOUBLE: { NC_SCAN_VAR(double, double); break; } case NC_FLOAT: { NC_SCAN_VAR(float, float); break; } } } else { rowStrings << QString(); } auto* varItem = new QTreeWidgetItem(rowStrings); varItem->setIcon(0, QIcon::fromTheme("x-office-spreadsheet")); varItem->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable); // highlight item for (int c = 0; c < varItem->columnCount(); c++) { varItem->setBackground(c, QColor(192, 255, 192)); varItem->setForeground(c, Qt::black); } parentItem->addChild(varItem); scanAttrs(ncid, i, -1, varItem); } } #endif /*! parses the content of the file \c fileName and fill the tree using rootItem. */ void NetCDFFilterPrivate::parse(const QString & fileName, QTreeWidgetItem* rootItem) { DEBUG("NetCDFFilterPrivate::parse()"); #ifdef HAVE_NETCDF QByteArray bafileName = fileName.toLatin1(); DEBUG("fileName = " << bafileName.data()); int ncid; m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); if (m_status != NC_NOERR) { DEBUG(" Giving up"); return; } int ndims, nvars, nattr, uldid; m_status = nc_inq(ncid, &ndims, &nvars, &nattr, &uldid); handleError(m_status, "nc_inq"); DEBUG(" nattr/ndims/nvars = " << nattr << ' ' << ndims << ' ' << nvars); QTreeWidgetItem *attrItem = new QTreeWidgetItem(QStringList() << QString(i18n("Attributes"))); attrItem->setIcon(0,QIcon::fromTheme("folder")); attrItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(attrItem); scanAttrs(ncid, NC_GLOBAL, -1, attrItem); QTreeWidgetItem *dimItem = new QTreeWidgetItem(QStringList() << QString(i18n("Dimensions"))); dimItem->setIcon(0, QIcon::fromTheme("folder")); dimItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(dimItem); scanDims(ncid, ndims, dimItem); QTreeWidgetItem *varItem = new QTreeWidgetItem(QStringList() << QString(i18n("Variables"))); varItem->setIcon(0, QIcon::fromTheme("folder")); varItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(varItem); scanVars(ncid, nvars, varItem); m_status = ncclose(ncid); handleError(m_status, "nc_close"); #else Q_UNUSED(fileName) Q_UNUSED(rootItem) #endif } QString NetCDFFilterPrivate::readAttribute(const QString & fileName, const QString & name, const QString & varName) { #ifdef HAVE_NETCDF int ncid; QByteArray bafileName = fileName.toLatin1(); m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); if (m_status != NC_NOERR) { DEBUG(" Giving up"); return QString(); } // get varid int varid; if (varName == "global") varid = NC_GLOBAL; else { QByteArray bavarName = varName.toLatin1(); m_status = nc_inq_varid(ncid, bavarName.data(), &varid); handleError(m_status, "nc_inq_varid"); } // attribute 'name' int attid; QByteArray baName = name.toLatin1(); m_status = nc_inq_attid(ncid, varid, baName.data(), &attid); handleError(m_status, "nc_inq_attid"); QString nameString = scanAttrs(ncid, varid, attid); m_status = ncclose(ncid); handleError(m_status, "nc_close"); return nameString; #else Q_UNUSED(fileName) Q_UNUSED(name) Q_UNUSED(varName) return QString(); #endif } /*! reads the content of the variable in the file \c fileName to a string (for preview) or to the data source. */ QVector NetCDFFilterPrivate::readCurrentVar(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode, int lines) { QVector dataStrings; if (currentVarName.isEmpty()) return dataStrings << (QStringList() << i18n("No variable selected")); DEBUG(" current variable = " << currentVarName.toStdString()); #ifdef HAVE_NETCDF int ncid; QByteArray bafileName = fileName.toLatin1(); m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); if (m_status != NC_NOERR) { DEBUG(" Giving up"); return dataStrings; } int varid; QByteArray baVarName = currentVarName.toLatin1(); m_status = nc_inq_varid(ncid, baVarName.data(), &varid); handleError(m_status, "nc_inq_varid"); int ndims; nc_type type; m_status = nc_inq_varndims(ncid, varid, &ndims); handleError(m_status, "nc_inq_varndims"); m_status = nc_inq_vartype(ncid, varid, &type); handleError(m_status, "nc_inq_type"); int* dimids = (int *) malloc(ndims * sizeof(int)); m_status = nc_inq_vardimid(ncid, varid, dimids); handleError(m_status, "nc_inq_vardimid"); int actualRows = 0, actualCols = 0; int columnOffset = 0; - QVector dataContainer; + std::vector dataContainer; switch (ndims) { case 0: { DEBUG(" zero dimensions"); actualRows = actualCols = 1; // only one value QVector columnModes; columnModes.resize(actualCols); switch (type) { case NC_BYTE: case NC_UBYTE: case NC_SHORT: case NC_USHORT: case NC_INT: columnModes[0] = AbstractColumn::Integer; break; case NC_UINT: // converted to double (int is too small) case NC_INT64: // converted to double (int is too small) case NC_UINT64: // converted to double (int is too small) case NC_DOUBLE: case NC_FLOAT: columnModes[0] = AbstractColumn::Numeric; break; case NC_CHAR: columnModes[0] = AbstractColumn::Text; break; //TODO: NC_STRING } //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); DEBUG(" Reading data of type " << translateDataType(type).toStdString()); switch (type) { case NC_BYTE: { NC_READ_VAR(signed char, schar, int); break; } case NC_UBYTE: { NC_READ_VAR(unsigned char, uchar, int); break; } case NC_CHAR: { // no number char data; m_status = nc_get_var_text(ncid, varid, &data); handleError(m_status, "nc_get_var_text"); if (dataSource) { QString *sourceData = static_cast*>(dataContainer[0])->data(); sourceData[0] = QString(data); } else { // preview dataStrings << (QStringList() << QString(data)); } break; } case NC_SHORT: { NC_READ_VAR(short, short, int); break; } case NC_USHORT: { NC_READ_VAR(unsigned short, ushort, int); break; } case NC_INT: { NC_READ_VAR(int, int, int); break; } case NC_UINT: { NC_READ_VAR(unsigned int, uint, double); break; } // converted to double (int is too small) case NC_INT64: { NC_READ_VAR(long long, longlong, double); break; } // converted to double (int is too small) case NC_UINT64: { NC_READ_VAR(unsigned long long, ulonglong, double); break; } // converted to double (int is too small) case NC_DOUBLE: { NC_READ_VAR(double, double, double); break; } case NC_FLOAT: { NC_READ_VAR(float, float, double); break; } } break; } case 1: { size_t size; m_status = nc_inq_dimlen(ncid, dimids[0], &size); handleError(m_status, "nc_inq_dimlen"); if (endRow == -1) endRow = (int)size; if (lines == -1) lines = endRow; actualRows = endRow - startRow + 1; actualCols = 1; // only one column DEBUG("start/end row: " << startRow << ' ' << endRow); DEBUG("act rows/cols: " << actualRows << ' ' << actualCols); QVector columnModes; columnModes.resize(actualCols); switch (type) { case NC_BYTE: case NC_UBYTE: case NC_SHORT: case NC_USHORT: case NC_INT: columnModes[0] = AbstractColumn::Integer; break; case NC_UINT: // converted to double (int is too small) case NC_INT64: // converted to double (int is too small) case NC_UINT64: // converted to double (int is too small) case NC_DOUBLE: case NC_FLOAT: columnModes[0] = AbstractColumn::Numeric; break; case NC_CHAR: columnModes[0] = AbstractColumn::Text; break; //TODO: NC_STRING } //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); DEBUG(" Reading data of type " << translateDataType(type).toStdString()); switch (type) { case NC_BYTE: { NC_READ_AVAR(signed char, schar, int); break; } case NC_UBYTE: { NC_READ_AVAR(unsigned char, uchar, int); break; } case NC_CHAR: { // not number char* data = new char[(unsigned int)actualRows]; size_t start = (size_t)(startRow - 1), count = (size_t)actualRows; m_status = nc_get_vara_text(ncid, varid, &start, &count, data); handleError(m_status, "nc_get_vara_text"); if (dataSource) { QString *sourceData = static_cast*>(dataContainer[0])->data(); for (int i = 0; i < actualRows; i++) sourceData[i] = QString(data[i]); } else { // preview for (int i = 0; i < qMin(actualRows, lines); i++) dataStrings << (QStringList() << QString(data[i])); } delete[] data; break; } case NC_SHORT: { NC_READ_AVAR(short, short, int); break; } case NC_USHORT: { NC_READ_AVAR(unsigned short, ushort, int); break; } case NC_INT: { NC_READ_AVAR_NATIVE(int); break; } case NC_UINT: { NC_READ_AVAR(unsigned int, uint, double); break; } // converted to double (int is too small) case NC_INT64: { NC_READ_AVAR(long long, longlong, double); break; } // converted to double (int is too small) case NC_UINT64: { NC_READ_AVAR(unsigned long long, ulonglong, double); break; } // converted to double (int is too small) case NC_DOUBLE: { NC_READ_AVAR_NATIVE(double); break; } case NC_FLOAT: { NC_READ_AVAR(float, float, double); break; } //TODO: NC_STRING default: DEBUG(" data type not supported yet"); } break; } case 2: { size_t rows, cols; m_status = nc_inq_dimlen(ncid, dimids[0], &rows); handleError(m_status, "nc_inq_dimlen"); m_status = nc_inq_dimlen(ncid, dimids[1], &cols); handleError(m_status, "nc_inq_dimlen"); if (endRow == -1) endRow = (int)rows; if (lines == -1) lines = endRow; if (endColumn == -1) endColumn = (int)cols; actualRows = endRow-startRow+1; actualCols = endColumn-startColumn+1; DEBUG("dim = " << rows << "x" << cols); DEBUG("startRow/endRow: " << startRow << ' ' << endRow); DEBUG("startColumn/endColumn: " << startColumn << ' ' << endColumn); DEBUG("actual rows/cols: " << actualRows << ' ' << actualCols); DEBUG("lines: " << lines); QVector columnModes; columnModes.resize(actualCols); switch (type) { case NC_BYTE: case NC_UBYTE: case NC_SHORT: case NC_USHORT: case NC_INT: for (int i = 0; i < actualCols; i++) columnModes[i] = AbstractColumn::Integer; break; case NC_UINT: // converted to double (int is too small) case NC_INT64: // converted to double (int is too small) case NC_UINT64: // converted to double (int is too small) case NC_DOUBLE: case NC_FLOAT: for (int i = 0; i < actualCols; i++) columnModes[i] = AbstractColumn::Numeric; break; case NC_CHAR: for (int i = 0; i < actualCols; i++) columnModes[i] = AbstractColumn::Text; break; //TODO: NC_STRING } //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); switch (type) { case NC_BYTE: { NC_READ_VAR2(signed char, schar, int); break; } case NC_UBYTE: { NC_READ_VAR2(unsigned char, uchar, int); break; } case NC_CHAR: { // no number char** data = (char**) malloc(rows * sizeof(char*)); data[0] = (char*)malloc(cols * rows * sizeof(char)); for (unsigned int i = 1; i < rows; i++) data[i] = data[0] + i*cols; m_status = nc_get_var_text(ncid, varid, &data[0][0]); handleError(m_status, "nc_get_var_text"); if (m_status == NC_NOERR) { for (int i = 0; i < qMin((int)rows, lines); i++) { QStringList line; for (size_t j = 0; j < cols; j++) { if (dataSource && dataContainer[0]) static_cast*>(dataContainer[(int)(j-(size_t)startColumn+1)])->operator[](i-startRow+1) = data[i][(int)j]; else line << QString(data[i][j]); } dataStrings << line; emit q->completed(100*i/actualRows); } } free(data[0]); free(data); break; } case NC_SHORT: { NC_READ_VAR2(short, short, int); break; } case NC_USHORT: { NC_READ_VAR2(unsigned short, ushort, int); break; } case NC_INT: { NC_READ_VAR2(int, int, int); break; } case NC_UINT: { NC_READ_VAR2(unsigned int, uint, double); break; } // converted to double (int is too small) case NC_INT64: { NC_READ_VAR2(long long, longlong, double); break; } // converted to double (int is too small) case NC_UINT64: { NC_READ_VAR2(unsigned long long, ulonglong, double); break; } // converted to double (int is too small) case NC_FLOAT: { NC_READ_VAR2(float, float, double); break; } case NC_DOUBLE: { NC_READ_VAR2(double, double, double); break; } //TODO: NC_STRING default: DEBUG(" data type not supported yet"); } break; } default: dataStrings << (QStringList() << i18n("%1 dimensional data of type %2 not supported yet", ndims, translateDataType(type))); QDEBUG(dataStrings); } free(dimids); m_status = ncclose(ncid); handleError(m_status, "nc_close"); if (dataSource) dataSource->finalizeImport(columnOffset, 1, actualCols, QString(), mode); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(mode) Q_UNUSED(lines) #endif return dataStrings; } /*! reads the content of the current selected variable from file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ QVector NetCDFFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { QVector dataStrings; if (currentVarName.isEmpty()) { DEBUG(" No variable selected"); return dataStrings; } return readCurrentVar(fileName, dataSource, mode); } /*! writes the content of \c dataSource to the file \c fileName. */ void NetCDFFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: write NetCDF files not implemented yet } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void NetCDFFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("netcdfFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool NetCDFFilter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h b/src/backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h index aec5c29f8..966a3d195 100644 --- a/src/backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h +++ b/src/backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h @@ -1,54 +1,54 @@ /*************************************************************************** File : NgspiceRawNgspiceRawAsciiFilterPrivate.h Project : LabPlot Description : Ngspice RAW ASCII filter -------------------------------------------------------------------- Copyright : (C) 2018 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef NGSPICERAWASCIIFILTERPRIVATE_H #define NGSPICERAWASCIIFILTERPRIVATE_H class AbstractDataSource; class NgspiceRawAsciiFilterPrivate { public: explicit NgspiceRawAsciiFilterPrivate(NgspiceRawAsciiFilter*); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); const NgspiceRawAsciiFilter* q; QStringList vectorNames; QVector columnModes; int startRow{1}; int endRow{-1}; private: - QVector m_dataContainer; // pointers to the actual data containers + std::vector m_dataContainer; // pointers to the actual data containers }; #endif diff --git a/src/backend/datasources/filters/NgspiceRawBinaryFilterPrivate.h b/src/backend/datasources/filters/NgspiceRawBinaryFilterPrivate.h index 0daab0c53..9531f4fac 100644 --- a/src/backend/datasources/filters/NgspiceRawBinaryFilterPrivate.h +++ b/src/backend/datasources/filters/NgspiceRawBinaryFilterPrivate.h @@ -1,56 +1,56 @@ /*************************************************************************** File : NgspiceRawNgspiceRawBinaryFilterPrivate.h Project : LabPlot Description : Ngspice RAW Binary filter -------------------------------------------------------------------- Copyright : (C) 2018 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef NGSPICERAWBINARYFILTERPRIVATE_H #define NGSPICERAWBINARYFILTERPRIVATE_H class AbstractDataSource; class NgspiceRawBinaryFilterPrivate { public: explicit NgspiceRawBinaryFilterPrivate(NgspiceRawBinaryFilter*); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); const NgspiceRawBinaryFilter* q; QStringList vectorNames; QVector columnModes; int startRow{1}; int endRow{-1}; private: const static int BYTE_SIZE = 8; - QVector m_dataContainer; // pointers to the actual data containers + std::vector m_dataContainer; // pointers to the actual data containers }; #endif diff --git a/src/backend/datasources/filters/ROOTFilter.cpp b/src/backend/datasources/filters/ROOTFilter.cpp index 37867c16f..117ea7e99 100644 --- a/src/backend/datasources/filters/ROOTFilter.cpp +++ b/src/backend/datasources/filters/ROOTFilter.cpp @@ -1,1503 +1,1503 @@ /*************************************************************************** File : ROOTFilter.cpp Project : LabPlot Description : ROOT(CERN) I/O-filter -------------------------------------------------------------------- Copyright : (C) 2018 by Christoph Roick (chrisito@gmx.de) Copyright : (C) 2018 by Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/ROOTFilter.h" #include "backend/datasources/filters/ROOTFilterPrivate.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/core/column/Column.h" #include #include #include #include #include #ifdef HAVE_ZIP #include #include #endif #include #include #include #include #include #include ROOTFilter::ROOTFilter():AbstractFileFilter(ROOT), d(new ROOTFilterPrivate) {} ROOTFilter::~ROOTFilter() = default; void ROOTFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } void ROOTFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } void ROOTFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } void ROOTFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } void ROOTFilter::setCurrentObject(const QString& object) { d->currentObject = object; } const QString ROOTFilter::currentObject() const { return d->currentObject; } ROOTFilter::Directory ROOTFilter::listHistograms(const QString& fileName) const { return d->listHistograms(fileName); } ROOTFilter::Directory ROOTFilter::listTrees(const QString& fileName) const { return d->listTrees(fileName); } QVector ROOTFilter::listLeaves(const QString& fileName, qint64 pos) const { return d->listLeaves(fileName, pos); } QVector ROOTFilter::previewCurrentObject(const QString& fileName, int first, int last) const { return d->previewCurrentObject(fileName, first, last); } int ROOTFilter::rowsInCurrentObject(const QString& fileName) const { return d->rowsInCurrentObject(fileName); } void ROOTFilter::setStartRow(const int s) { d->startRow = s; } int ROOTFilter::startRow() const { return d->startRow; } void ROOTFilter::setEndRow(const int e) { d->endRow = e; } int ROOTFilter::endRow() const { return d->endRow; } void ROOTFilter::setColumns(const QVector& columns) { d->columns = columns; } QVector ROOTFilter::columns() const { return d->columns; } void ROOTFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("rootFilter"); writer->writeAttribute("object", d->currentObject); writer->writeAttribute("startRow", QString::number(d->startRow)); writer->writeAttribute("endRow", QString::number(d->endRow)); for (const auto & c : d->columns) { writer->writeStartElement("column"); for (const auto & s : c) writer->writeTextElement("id", s); writer->writeEndElement(); } writer->writeEndElement(); } bool ROOTFilter::load(XmlStreamReader* reader) { QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); // read attributes d->currentObject = attribs.value("object").toString(); if (d->currentObject.isEmpty()) reader->raiseWarning(attributeWarning.arg("object")); QString str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("startRow")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("endRow")); else d->endRow = str.toInt(); d->columns.clear(); while (reader->readNextStartElement()) { if (reader->name() == "column") { QStringList c; while (reader->readNextStartElement()) { if (reader->name() == "id") c << reader->readElementText(); else reader->skipCurrentElement(); } if (!c.empty()) d->columns << c; } else reader->skipCurrentElement(); } if (d->columns.empty()) reader->raiseWarning(i18n("No column available")); return true; } /**************** ROOTFilterPrivate implementation *******************/ ROOTFilterPrivate::ROOTFilterPrivate() = default; ROOTFilterPrivate::FileType ROOTFilterPrivate::currentObjectPosition(const QString& fileName, long int& pos) { QStringList typeobject = currentObject.split(':'); if (typeobject.size() < 2) return Invalid; FileType type; if (typeobject.first() == QStringLiteral("Hist")) type = Hist; else if (typeobject.first() == QStringLiteral("Tree")) type = Tree; else return Invalid; typeobject.removeFirst(); QStringList path = typeobject.join(':').split('/'); ROOTFilter::Directory dir = type == Hist ? listHistograms(fileName) : listTrees(fileName); const ROOTFilter::Directory* node = &dir; while (path.size() > 1) { bool next = false; for (const auto& child : node->children) { if (child.name == path.first()) { node = &child; path.pop_front(); next = true; break; } } if (!next) return Invalid; } for (const auto& child : node->content) { if (child.first == path.first()) { pos = child.second; break; } } return type; } void ROOTFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("ROOTFilterPrivate::readDataFromFile()"); long int pos = 0; auto type = currentObjectPosition(fileName, pos); if (pos == 0) return; if (type == Hist) { auto bins = readHistogram(pos); const int nbins = static_cast(bins.size()); // skip underflow and overflow bins by default int first = qMax(qAbs(startRow), 0); int last = endRow < 0 ? nbins - 1 : qMax(first - 1, qMin(endRow, nbins - 1)); QStringList headers; for (const auto& l : columns) { headers << l.last(); } - QVector dataContainer; + std::vector dataContainer; const int columnOffset = dataSource->prepareImport(dataContainer, importMode, last - first + 1, columns.size(), headers, QVector(columns.size(), AbstractColumn::Numeric)); // read data DEBUG(" reading " << first - last + 1 << " lines"); int c = 0; Spreadsheet* spreadsheet = dynamic_cast(dataSource); for (const auto& l : columns) { QVector& container = *static_cast*>(dataContainer[c]); if (l.first() == QStringLiteral("center")) { if (spreadsheet) spreadsheet->column(columnOffset + c)->setPlotDesignation(Column::X); for (int i = first; i <= last; ++i) container[i - first] = (i > 0 && i < nbins - 1) ? 0.5 * (bins[i].lowedge + bins[i + 1].lowedge) : i == 0 ? bins.front().lowedge // -infinity : -bins.front().lowedge; // +infinity } else if (l.first() == QStringLiteral("low")) { if (spreadsheet) spreadsheet->column(columnOffset + c)->setPlotDesignation(Column::X); for (int i = first; i <= last; ++i) container[i - first] = bins[i].lowedge; } else if (l.first() == QStringLiteral("content")) { if (spreadsheet) spreadsheet->column(columnOffset + c)->setPlotDesignation(Column::Y); for (int i = first; i <= last; ++i) container[i - first] = bins[i].content; } else if (l.first() == QStringLiteral("error")) { if (spreadsheet) spreadsheet->column(columnOffset + c)->setPlotDesignation(Column::YError); for (int i = first; i <= last; ++i) container[i - first] = std::sqrt(bins[i].sumw2); } ++c; } dataSource->finalizeImport(columnOffset, 0, columns.size() - 1, QString(), importMode); } else if (type == Tree) { const int nentries = static_cast(currentROOTData->treeEntries(pos)); int first = qMax(qAbs(startRow), 0); int last = qMax(first - 1, qMin(endRow, nentries - 1)); QStringList headers; for (const auto& l : columns) { QString lastelement = l.back(), leaf = l.front(); bool isArray = false; if (lastelement.at(0) == '[' && lastelement.at(lastelement.size() - 1) == ']') { lastelement.mid(1, lastelement.length() - 2).toUInt(&isArray); } if (!isArray || l.count() == 2) headers << l.join(isArray ? QString() : QString(':')); else headers << l.first() + QChar(':') + l.at(1) + l.back(); } - QVector dataContainer; + std::vector dataContainer; const int columnOffset = dataSource->prepareImport(dataContainer, importMode, last - first + 1, columns.size(), headers, QVector(columns.size(), AbstractColumn::Numeric)); int c = 0; for (const auto& l : columns) { unsigned int element = 0; QString lastelement = l.back(), leaf = l.front(); bool isArray = false; if (lastelement.at(0) == '[' && lastelement.at(lastelement.size() - 1) == ']') { element = lastelement.mid(1, lastelement.length() - 2).toUInt(&isArray); if (!isArray) element = 0; if (l.count() > 2) leaf = l.at(1); } else if (l.count() > 1) leaf = l.at(1); QVector& container = *static_cast*>(dataContainer[c++]); auto data = readTree(pos, l.first(), leaf, (int)element, last); for (int i = first; i <= last; ++i) container[i - first] = data[i]; } dataSource->finalizeImport(columnOffset, 0, columns.size() - 1, QString(), importMode); } } void ROOTFilterPrivate::write(const QString& fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); } ROOTFilter::Directory ROOTFilterPrivate::listContent(const std::map& dataContent, std::string (ROOTData::*nameFunc)(long int)) { ROOTFilter::Directory dirs; QHash::type::value_type*, ROOTFilter::Directory*> filledDirs; for (const auto& path : dataContent) { if (!path.second.content.empty()) { QStack addpath; auto pos = &path; ROOTFilter::Directory* currentdir = &dirs; while (true) { auto it = filledDirs.find(pos); if (it != filledDirs.end()) { currentdir = it.value(); break; } auto jt = dataContent.find(pos->second.parent); if (jt != dataContent.end()) { addpath.push(pos); pos = &(*jt); } else break; } while (!addpath.empty()) { auto pos = addpath.pop(); ROOTFilter::Directory dir; dir.name = QString::fromStdString(pos->second.name); currentdir->children << dir; currentdir = ¤tdir->children.last(); filledDirs[pos] = currentdir; } for (auto hist : path.second.content) { auto name = ((*currentROOTData).*nameFunc)(hist); if (!name.empty()) currentdir->content << qMakePair(QString::fromStdString(name), hist); } } } return dirs; } ROOTFilter::Directory ROOTFilterPrivate::listHistograms(const QString& fileName) { if (setFile(fileName)) return listContent(currentROOTData->listHistograms(), &ROOTData::histogramName); else return ROOTFilter::Directory{}; } ROOTFilter::Directory ROOTFilterPrivate::listTrees(const QString& fileName) { if (setFile(fileName)) return listContent(currentROOTData->listTrees(), &ROOTData::treeName); else return ROOTFilter::Directory{}; } QVector ROOTFilterPrivate::listLeaves(const QString& fileName, quint64 pos) { QVector leafList; if (setFile(fileName)) { for (const auto& leaf : currentROOTData->listLeaves(pos)) { leafList << QStringList(QString::fromStdString(leaf.branch)); if (leaf.branch != leaf.leaf) leafList.last() << QString::fromStdString(leaf.leaf); if (leaf.elements > 1) leafList.last() << QString("[%1]").arg(leaf.elements); } } return leafList; } QVector ROOTFilterPrivate::previewCurrentObject(const QString& fileName, int first, int last) { DEBUG("ROOTFilterPrivate::previewCurrentObject()"); long int pos = 0; auto type = currentObjectPosition(fileName, pos); if (pos == 0) return QVector(1, QStringList()); if (type == Hist) { auto bins = readHistogram(pos); const int nbins = static_cast(bins.size()); last = qMin(nbins - 1, last); QVector preview(qMax(last - first + 2, 1)); DEBUG(" reading " << preview.size() - 1 << " lines"); // set headers for (const auto& l : columns) { preview.last() << l.last(); } // read data for (const auto& l : columns) { if (l.first() == QStringLiteral("center")) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number( (i > 0 && i < nbins - 1) ? 0.5 * (bins[i].lowedge + bins[i + 1].lowedge) : i == 0 ? bins.front().lowedge // -infinity : -bins.front().lowedge); // +infinity } else if (l.first() == QStringLiteral("low")) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(bins[i].lowedge); } else if (l.first() == QStringLiteral("content")) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(bins[i].content); } else if (l.first() == QStringLiteral("error")) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(std::sqrt(bins[i].sumw2)); } } return preview; } else if (type == Tree) { last = qMin(last, currentROOTData->treeEntries(pos) - 1); QVector preview(qMax(last - first + 2, 1)); DEBUG(" reading " << preview.size() - 1 << " lines"); // read data leaf by leaf and set headers for (const auto& l : columns) { unsigned int element = 0; QString lastelement = l.back(), leaf = l.front(); bool isArray = false; if (lastelement.at(0) == '[' && lastelement.at(lastelement.size() - 1) == ']') { element = lastelement.mid(1, lastelement.length() - 2).toUInt(&isArray); if (!isArray) element = 0; if (l.count() > 2) leaf = l.at(1); } else if (l.count() > 1) leaf = l.at(1); auto data = readTree(pos, l.first(), leaf, (int)element, last); for (int i = first; i <= last; ++i) preview[i - first] << QString::number(data[i]); if (!isArray || l.count() == 2) preview.last() << l.join(isArray ? QString() : QString(':')); else preview.last() << l.first() + QChar(':') + l.at(1) + l.back(); } return preview; } else return QVector(1, QStringList()); } int ROOTFilterPrivate::rowsInCurrentObject(const QString& fileName) { long int pos = 0; auto type = currentObjectPosition(fileName, pos); if (pos == 0) return 0; switch (type) { case Hist: return currentROOTData->histogramBins(pos); case Tree: return currentROOTData->treeEntries(pos); case Invalid: default: return 0; } } bool ROOTFilterPrivate::setFile(const QString& fileName) { QFileInfo file(fileName); if (!file.exists()) { currentObject.clear(); columns.clear(); currentROOTData.reset(); return false; } QDateTime modified = file.lastModified(); qint64 size = file.size(); if (!currentROOTData || fileName != currentFile.name || modified != currentFile.modified || size != currentFile.size) { currentFile.name = fileName; currentFile.modified = modified; currentFile.size = size; currentROOTData.reset(new ROOTData(fileName.toStdString())); } return true; } std::vector ROOTFilterPrivate::readHistogram(quint64 pos) { return currentROOTData->readHistogram(pos); } std::vector ROOTFilterPrivate::readTree(quint64 pos, const QString& branchName, const QString& leafName, int element, int last) { return currentROOTData->listEntries(pos, branchName.toStdString(), leafName.toStdString(), element, last + 1); } /******************** ROOTData implementation ************************/ namespace ROOTDataHelpers { /// Read value from stream template T read(std::ifstream& is) { union { T val; char buf[sizeof(T)]; } r; for (size_t i = 0; i < sizeof(T); ++i) is.get(r.buf[sizeof(T) - i - 1]); return r.val; } /// Read value from buffer template T read(char*& s) { union { T val; char buf[sizeof(T)]; } r; for (size_t i = 0; i < sizeof(T); ++i) r.buf[sizeof(T) - i - 1] = *(s++); return r.val; } /// Read value from buffer and cast to U template U readcast(char*& s) { return static_cast(read(s)); } /// Get version of ROOT object, obtain number of bytes in object short Version(char*& buffer, size_t& count) { // root/io/io/src/TBufferFile.cxx -> ReadVersion count = read(buffer); short version = (count & 0x40000000) ? read(buffer) : read(buffer -= 4); count = (count & 0x40000000) ? (count & ~0x40000000) - 2 : 2; return version; } /// Get version of ROOT object short Version(char*& buffer) { size_t c; return Version(buffer, c); } /// Skip ROOT object void Skip(char*& buffer, const size_t& n) { for (size_t i = 0; i < n; ++i) { size_t count; Version(buffer, count); buffer += count; } } /// Skip TObject header void SkipObject(char*& buffer) { Version(buffer); buffer += 8; } /// Get TString std::string String(char*& buffer) { // root/io/io/src/TBufferFile.cxx -> ReadTString size_t s = *(buffer++); if (s == 0) return std::string(); else { if (s == 0xFF) s = read(buffer); buffer += s; return std::string(buffer - s, buffer); } } /// Get the header of an object in TObjArray std::string readObject(char*& buf, char* const buf0, std::map& tags) { // root/io/io/src/TBufferFile.cxx -> ReadObjectAny std::string clname; unsigned int tag = read(buf); if (tag & 0x40000000) { tag = read(buf); if (tag == 0xFFFFFFFF) { tags[buf - buf0 - 2] = clname = buf; buf += clname.size() + 1; } else { clname = tags[tag & ~0x80000000]; } } return clname; } } using namespace ROOTDataHelpers; ROOTData::ROOTData(const std::string& filename) : filename(filename) { // The file structure is described in root/io/io/src/TFile.cxx std::ifstream is(filename, std::ifstream::binary); std::string root(4, 0); is.read(const_cast(root.data()), 4); if (root != "root") return; int fileVersion = read(is); long int pos = read(is); histdirs.emplace(pos, Directory{}); treedirs.emplace(pos, Directory{}); long int endpos = fileVersion < 1000000 ? read(is) : read(is); is.seekg(33); int compression = read(is); compression = compression > 0 ? compression : 0; while (is.good() && pos < endpos) { is.seekg(pos); int lcdata = read(is); if (lcdata == 0) { break; } if (lcdata < 0) { pos -= lcdata; continue; } short version = read(is); size_t ldata = read(is); is.seekg(4, is.cur); // skip the date size_t lkey = read(is); short cycle = read(is); long int pseek; if (version > 1000) { is.seekg(8, is.cur); pseek = read(is); } else { is.seekg(4, is.cur); pseek = read(is); } std::string cname(read(is), 0); is.read(&cname[0], cname.size()); std::string name(read(is), 0); is.read(&name[0], name.size()); std::string title(read(is), 0); is.read(&title[0], title.size()); ContentType type = Invalid; if (cname.size() == 4 && cname.substr(0, 3) == "TH1") { type = histType(cname[3]); } else if (cname == "TTree") type = Tree; else if (cname.substr(0, 7) == "TNtuple") type = NTuple; else if (cname == "TBasket") type = Basket; else if (cname == "TList" && name == "StreamerInfo") type = Streamer; else if (cname == "TDirectory") { auto it = histdirs.find(pseek); if (it == histdirs.end()) it = histdirs.begin(); histdirs.emplace(pos, Directory{name, it->first}); it = treedirs.find(pseek); if (it == treedirs.end()) it = treedirs.begin(); treedirs.emplace(pos, Directory{name, it->first}); } if (type) { if (type == Basket) is.seekg(19, std::ifstream::cur); // TODO read info instead? KeyBuffer buffer; buffer.type = Invalid; // see root/io/io/src/TKey.cxx for reference int complib = 0; if (compression) { // Default: compression level // ZLIB: 100 + compression level // LZ4: 400 + compression level // do not rely on this, but read the header std::string lib(2, 0); is.read(&lib[0], 2); complib = lib == "ZL" ? 1 : lib == "XZ" ? 2 : lib == "CS" ? 3 : lib == "L4" ? 4 : 0; } if (complib > 0) { # ifdef HAVE_ZIP // see root/core/zip/src/RZip.cxx -> R__unzip const int method = is.get(); size_t chcdata = is.get(); chcdata |= (is.get() << 8); chcdata |= (is.get() << 16); size_t chdata = is.get(); chdata |= (is.get() << 8); chdata |= (is.get() << 16); if (chcdata == lcdata - lkey - 9 && chdata == ldata) { if (complib == 1 && method == Z_DEFLATED) { buffer = KeyBuffer{type, name, title, cycle, lkey, KeyBuffer::zlib, pos + lkey + 9, chcdata, chdata, 0}; } else if (complib == 4 && method == LZ4_versionNumber() / 10000) { buffer = KeyBuffer{type, name, title, cycle, lkey, KeyBuffer::lz4, pos + lkey + 9 + 8, chcdata - 8, chdata, 0}; } } # endif } else { buffer = KeyBuffer{type, name, title, cycle, lkey, KeyBuffer::none, pos + lkey, ldata, ldata, 0}; } switch (buffer.type) { case Basket: basketkeys.emplace(pos, buffer); break; case Tree: case NTuple: { auto it = treedirs.find(pseek); if (it == treedirs.end()) it = treedirs.begin(); bool keyreplaced = false; for (auto & tpos : it->second.content) { auto jt = treekeys.find(tpos); if (jt != treekeys.end() && jt->second.name == buffer.name && jt->second.cycle < buffer.cycle) { // override key with lower cylce number tpos = pos; treekeys.erase(jt); keyreplaced = true; break; } } if (!keyreplaced) it->second.content.push_back(pos); treekeys.emplace(pos, buffer); break; } case Streamer: readStreamerInfo(buffer); break; case Double: case Float: case Int: case Short: case Byte: { auto it = histdirs.find(pseek); if (it == histdirs.end()) it = histdirs.begin(); it->second.content.push_back(pos); histkeys.emplace(pos, buffer); break; } case Invalid: case Long: case Bool: case CString: break; } } pos += lcdata; } // Create default object structures if no StreamerInfo was found. // Obtained by running the following in ROOT with a file passed as an argument: // // _file0->GetStreamerInfoList()->Print() // // auto l = (TStreamerInfo*)_file0->GetStreamerInfoList()->At(ENTRYNUMBER); // l->Print(); // for (int i = 0; i < l->GetNelement(); ++i) { // auto e = l->GetElement(i); // e->Print(); // cout << e->GetFullName() << " " << e->GetTypeName() << " " << e->GetSize() << endl; // } static const StreamerInfo dummyobject{"Object", 0, std::string(), false, false}; if (!treekeys.empty()) { if (!streamerInfo.count("TTree")) { streamerInfo["TTree"] = {dummyobject, dummyobject, dummyobject, dummyobject, StreamerInfo{"fEntries", 8, std::string(), false, false}, StreamerInfo{std::string(), 5 * 8 + 4 * 4, std::string(), false, false}, StreamerInfo{"fNClusterRange", 4, std::string(), true, false}, StreamerInfo{std::string(), 6 * 8, std::string(), false, false}, StreamerInfo{"fNClusterRangeEnd", 8, "fNClusterRange", false, true}, StreamerInfo{"fNClusterSize", 8, "fNClusterRange", false, true}, StreamerInfo{"fBranches", 0, std::string(), false, false} }; } if (!streamerInfo.count("TBranch")) { streamerInfo["TBranch"] = {StreamerInfo{"TNamed", 0, std::string(), false, false}, dummyobject, StreamerInfo{std::string(), 3 * 4, std::string(), false, false}, StreamerInfo{"fWriteBasket", 4, std::string(), false, false}, StreamerInfo{std::string(), 8 + 4, std::string(), false, false}, StreamerInfo{"fMaxBaskets", 4, std::string(), true, false}, StreamerInfo{std::string(), 4 + 4 * 8, std::string(), false, false}, StreamerInfo{"fBranches", 0, std::string(), false, false}, StreamerInfo{"fLeaves", 0, std::string(), false, false}, StreamerInfo{"fBaskets", 0, std::string(), false, false}, StreamerInfo{"fBasketBytes", 4, "fMaxBaskets", false, true}, StreamerInfo{"fBasketEntry", 8, "fMaxBaskets", false, true}, StreamerInfo{"fBasketSeek", 8, "fMaxBaskets", false, true} }; } } if (!histkeys.empty()) { if (!streamerInfo.count("TH1")) { streamerInfo["TH1"] = {dummyobject, dummyobject, dummyobject, dummyobject, StreamerInfo{"fNcells", 4, std::string(), false, false}, StreamerInfo{"fXaxis", 0, std::string(), false, false}, StreamerInfo{"fYaxis", 0, std::string(), false, false}, StreamerInfo{"fZaxis", 0, std::string(), false, false}, StreamerInfo{std::string(), 2 * 2 + 8 * 8, std::string(), false, false}, dummyobject, StreamerInfo{"fSumw2", 0, std::string(), false, false}}; } if (!streamerInfo.count("TAxis")) { streamerInfo["TAxis"] = {dummyobject, dummyobject, StreamerInfo{"fNbins", 4, std::string(), false, false}, StreamerInfo{"fXmin", 8, std::string(), false, false}, StreamerInfo{"fXmax", 8, std::string(), false, false}, StreamerInfo{"fXbins", 0, std::string(), false, false}}; } } for (auto& tree : treekeys) readNEntries(tree.second); for (auto& hist : histkeys) readNBins(hist.second); } void ROOTData::readNBins(ROOTData::KeyBuffer& kbuffer) { std::string buffer = data(kbuffer); if (!buffer.empty()) { char* buf = &buffer[0]; std::map counts; Version(buf); // TH1(D/F/I/S/C) Version(buf); // TH1 advanceTo(buf, streamerInfo.find("TH1")->second, std::string(), "fNcells", counts); kbuffer.nrows = read(buf); // fNcells } } std::string ROOTData::histogramName(long int pos) { auto it = histkeys.find(pos); if (it != histkeys.end()) return it->second.name + ';' + std::to_string(it->second.cycle); return std::string(); } int ROOTData::histogramBins(long int pos) { auto it = histkeys.find(pos); if (it != histkeys.end()) return it->second.nrows; return 0; } std::vector ROOTData::readHistogram(long int pos) { auto it = histkeys.find(pos); if (it == histkeys.end()) return std::vector(); std::string buffer = data(it->second); if (!buffer.empty()) { char* buf = &buffer[0]; std::map counts; auto& streamerTH1 = streamerInfo.find("TH1")->second; auto& streamerTAxis = streamerInfo.find("TAxis")->second; size_t count; Version(buf); // TH1(D/F/I/S/C) Version(buf, count); // TH1 char* const dbuf = buf + count; advanceTo(buf, streamerTH1, std::string(), "fNcells", counts); std::vector r(read(buf)); // fNcells if (r.size() < 3) return std::vector(); r.front().lowedge = -std::numeric_limits::infinity(); advanceTo(buf, streamerTH1, "fNcells", "fXaxis", counts); // x-Axis Version(buf, count); // TAxis char* const nbuf = buf + count; advanceTo(buf, streamerTAxis, std::string(), "fNbins", counts); const int nbins = read(buf); advanceTo(buf, streamerTAxis, "fNbins", "fXmin", counts); const double xmin = read(buf); advanceTo(buf, streamerTAxis, "fXmin", "fXmax", counts); const double xmax = read(buf); advanceTo(buf, streamerTAxis, "fXmax", "fXbins", counts); const size_t nborders = read(buf); // TArrayD // root/core/cont/src/TArrayD.cxx -> Streamer if (nborders == r.size() - 1) { for (size_t i = 0; i < nborders; ++i) { r[i + 1].lowedge = read(buf); } } else { //UNUSED: buf += sizeof(double) * nbins; const double scale = (xmax - xmin) / static_cast(nbins); for (size_t i = 0; i < r.size() - 1; ++i) { r[i + 1].lowedge = static_cast(i) * scale + xmin; } } buf = nbuf; // go beyond x-Axis advanceTo(buf, streamerTH1, "fXaxis", "fSumw2", counts); if (static_cast(read(buf)) == r.size()) { // TArrayD for (auto& b : r) b.sumw2 = read(buf); // always double } buf = dbuf; // skip to contents of TH1(D/F/I/S/C) if (static_cast(read(buf)) == r.size()) { auto readf = readType(it->second.type); for (auto& b : r) b.content = readf(buf); } return r; } else return std::vector(); } void ROOTData::readNEntries(ROOTData::KeyBuffer& kbuffer) { std::string buffer = data(kbuffer); if (!buffer.empty()) { char* buf = &buffer[0]; std::map counts; if (kbuffer.type == NTuple) Version(buf); // TNtuple(D) Version(buf); // TTree advanceTo(buf, streamerInfo.find("TTree")->second, std::string(), "fEntries", counts); kbuffer.nrows = read(buf); // fEntries } } std::string ROOTData::treeName(long int pos) { auto it = treekeys.find(pos); if (it != treekeys.end()) return it->second.name; return std::string(); } int ROOTData::treeEntries(long int pos) { auto it = treekeys.find(pos); if (it != treekeys.end()) return it->second.nrows; else return 0; } std::vector ROOTData::listLeaves(long int pos) const { std::vector leaves; auto it = treekeys.find(pos); if (it == treekeys.end()) return leaves; std::ifstream is(filename, std::ifstream::binary); std::string datastring = data(it->second, is); if (datastring.empty()) return leaves; char* buf = &datastring[0]; char* const buf0 = buf - it->second.keylength; std::map counts; auto& streamerTBranch = streamerInfo.find("TBranch")->second; if (it->second.type == NTuple) Version(buf); // TNtuple(D) Version(buf); // TTree advanceTo(buf, streamerInfo.find("TTree")->second, std::string(), "fBranches", counts); // read the list of branches Version(buf); // TObjArray SkipObject(buf); String(buf); const size_t nbranches = read(buf); const size_t lowb = read(buf); std::map tags; for (size_t i = 0; i < nbranches; ++i) { std::string clname = readObject(buf, buf0, tags); size_t count; Version(buf, count); // TBranch or TBranchElement char* const nbuf = buf + count; if (i >= lowb) { if (clname == "TBranchElement") { Version(buf); // TBranch } advanceTo(buf, streamerTBranch, std::string(), "TNamed", counts); Version(buf); // TNamed SkipObject(buf); const std::string branch = String(buf); String(buf); // TODO add reading of nested branches (fBranches) advanceTo(buf, streamerTBranch, "TNamed", "fLeaves", counts); // fLeaves Version(buf); // TObjArray SkipObject(buf); String(buf); const size_t nleaves = read(buf); const size_t lowb = read(buf); for (size_t i = 0; i < nleaves; ++i) { std::string clname = readObject(buf, buf0, tags); Version(buf, count); // TLeaf(D/F/B/S/I/L/C/O) char* nbuf = buf + count; if (i >= lowb && clname.size() == 6 && clname.compare(0, 5, "TLeaf") == 0) { Version(buf); // TLeaf Version(buf); // TNamed SkipObject(buf); const std::string leafname = String(buf); String(buf); // title size_t elements = read(buf); int bytes = read(buf); if ((leafType(clname.back()) & 0xF) != bytes) qDebug() << "ROOTData: type " << clname.back() << " does not match its size!"; buf += 5; leaves.emplace_back(LeafInfo{branch, leafname, leafType(clname.back()), !read(buf), elements}); } buf = nbuf; } } buf = nbuf; } return leaves; } template std::vector ROOTData::listEntries(long int pos, const std::string& branchname, const std::string& leafname, const size_t element, const size_t nentries) const { std::vector entries; auto it = treekeys.find(pos); if (it == treekeys.end()) return entries; std::ifstream is(filename, std::ifstream::binary); std::string datastring = data(it->second, is); if (datastring.empty()) return entries; char* buf = &datastring[0]; char* const buf0 = buf - it->second.keylength; std::map counts; auto& streamerTTree = streamerInfo.find("TTree")->second; auto& streamerTBranch = streamerInfo.find("TBranch")->second; if (it->second.type == NTuple) Version(buf); // TNtuple(D) Version(buf); // TTree advanceTo(buf, streamerTTree, std::string(), "fEntries", counts); entries.reserve(std::min(static_cast(read(buf)), nentries)); // reserve space (maximum for number of entries) advanceTo(buf, streamerTTree, "fEntries", "fBranches", counts); // read the list of branches Version(buf); // TObjArray SkipObject(buf); String(buf); const size_t nbranches = read(buf); const size_t lowb = read(buf); std::map tags; for (size_t i = 0; i < nbranches; ++i) { std::string clname = readObject(buf, buf0, tags); size_t count; Version(buf, count); // TBranch or TBranchElement char* const nbuf = buf + count; if (i >= lowb) { if (clname == "TBranchElement") { Version(buf); } Version(buf); // TNamed SkipObject(buf); const std::string currentbranch = String(buf); String(buf); advanceTo(buf, streamerTBranch, "TNamed", "fWriteBasket", counts); int fWriteBasket = read(buf); // TODO add reading of nested branches (fBranches) advanceTo(buf, streamerTBranch, "fWriteBasket", "fLeaves", counts); // fLeaves Version(buf); // TObjArray SkipObject(buf); String(buf); const size_t nleaves = read(buf); const size_t lowb = read(buf); int leafoffset = 0, leafcount = 0, leafcontent = 0, leafsize = 0; bool leafsign = false; ContentType leaftype = Invalid; for (size_t i = 0; i < nleaves; ++i) { std::string clname = readObject(buf, buf0, tags); Version(buf, count); // TLeaf(D/F/L/I/S/B/O/C/Element) char* nbuf = buf + count; if (currentbranch == branchname) { if (i >= lowb && clname.size() >= 5 && clname.compare(0, 5, "TLeaf") == 0) { Version(buf); // TLeaf Version(buf); // TNamed SkipObject(buf); const bool istheleaf = (clname.size() == 6 && leafname == String(buf)); String(buf); const int len = read(buf); const int size = read(buf); if (istheleaf) { leafoffset = leafcount; leafsize = size; leaftype = leafType(clname.back()); } leafcount += len * size; if (istheleaf) { leafcontent = leafcount - leafoffset; buf += 1; leafsign = !read(buf); } } } buf = nbuf; } if (leafcontent == 0) { buf = nbuf; continue; } if (static_cast(element) * leafsize >= leafcontent) { qDebug() << "ROOTData: " << leafname.c_str() << " only contains " << leafcontent / leafsize << " elements."; break; } advanceTo(buf, streamerTBranch, "fLeaves", "fBaskets", counts); // fBaskets (probably empty) Version(buf, count); // TObjArray char* const basketsbuf = buf += count + 1; // TODO there is one byte to be skipped in fBaskets, why is that? advanceTo(buf, streamerTBranch, "fBaskets", "fBasketEntry", counts); for (int i = 0; i <= fWriteBasket; ++i) { if (static_cast(read(buf)) > nentries) { fWriteBasket = i; break; } } // rewind to the end of fBaskets and look for the fBasketSeek array advanceTo(buf = basketsbuf, streamerTBranch, "fBaskets", "fBasketSeek", counts); auto readf = readType(leaftype, leafsign); for (int i = 0; i < fWriteBasket; ++i) { long int pos = read(buf); auto it = basketkeys.find(pos); if (it != basketkeys.end()) { std::string basketbuffer = data(it->second); if (!basketbuffer.empty()) { char* bbuf = &basketbuffer[0]; char* const bufend = bbuf + basketbuffer.size(); while (bbuf + leafcount <= bufend && entries.size() < nentries) { bbuf += leafoffset + leafsize * element; entries.emplace_back(readf(bbuf)); bbuf += leafcount - leafsize * (element + 1) - leafoffset; } } } else { qDebug() << "ROOTData: fBasketSeek(" << i << "): " << pos << " (not available)"; } } } buf = nbuf; } return entries; } ROOTData::ContentType ROOTData::histType(const char type) { switch (type) { case 'D': return Double; case 'F': return Float; case 'I': return Int; case 'S': return Short; case 'C': return Byte; default: return Invalid; } } ROOTData::ContentType ROOTData::leafType(const char type) { switch (type) { case 'D': return Double; case 'F': return Float; case 'L': return Long; case 'I': return Int; case 'S': return Short; case 'B': return Byte; case 'O': return Bool; case 'C': return CString; default: return Invalid; } } template T (*ROOTData::readType(ROOTData::ContentType type, bool sign) const)(char*&) { switch (type) { case Double: return readcast; case Float: return readcast; case Long: return sign ? readcast : readcast; case Int: return sign ? readcast : readcast; case Short: return sign ? readcast : readcast; case Byte: return sign ? readcast : readcast; case Bool: return readcast; case CString: case Tree: case NTuple: case Basket: case Streamer: case Invalid: break; } return readcast; } std::string ROOTData::data(const ROOTData::KeyBuffer& buffer) const { std::ifstream is(filename, std::ifstream::binary); return data(buffer, is); } std::string ROOTData::data(const ROOTData::KeyBuffer& buffer, std::ifstream& is) const { std::string data(buffer.count, 0); is.seekg(buffer.start); if (buffer.compression == KeyBuffer::none) { is.read(&data[0], buffer.count); return data; #ifdef HAVE_ZIP } else if (buffer.compression == KeyBuffer::zlib) { std::string cdata(buffer.compressed_count, 0); is.read(&cdata[0], buffer.compressed_count); uLongf luncomp = buffer.count; if (uncompress((Bytef *)data.data(), &luncomp, (Bytef *)cdata.data(), cdata.size()) == Z_OK && data.size() == luncomp) return data; } else { std::string cdata(buffer.compressed_count, 0); is.read(&cdata[0], buffer.compressed_count); if (LZ4_decompress_safe(cdata.data(), const_cast(data.data()), buffer.compressed_count, buffer.count) == static_cast(buffer.count)) return data; #endif } return std::string(); } void ROOTData::readStreamerInfo(const ROOTData::KeyBuffer& buffer) { std::ifstream is(filename, std::ifstream::binary); std::string datastring = data(buffer, is); if (!datastring.empty()) { char* buf = &datastring[0]; char* const buf0 = buf - buffer.keylength; Version(buf); SkipObject(buf); // TCollection String(buf); const int nobj = read(buf); std::map tags; for (int i = 0; i < nobj; ++i) { std::string clname = readObject(buf, buf0, tags); size_t count; Version(buf, count); char* const nbuf = buf + count; if (clname == "TStreamerInfo") { Version(buf); SkipObject(buf); std::vector& sinfo = streamerInfo[String(buf)]; String(buf); buf += 8; // skip check sum and version clname = readObject(buf, buf0, tags); Version(buf, count); if (clname != "TObjArray") { buf += count; continue; } SkipObject(buf); // TObjArray String(buf); const int nobj = read(buf); const int lowb = read(buf); for (int i = 0; i < nobj; ++i) { std::string clname = readObject(buf, buf0, tags); Version(buf, count); char* const nbuf = buf + count; const bool isbasicpointer = clname == "TStreamerBasicPointer"; const bool ispointer = isbasicpointer || clname == "TStreamerObjectPointer"; if (i >= lowb) { if (ispointer || clname == "TStreamerBase" || clname == "TStreamerBasicType" || clname == "TStreamerObject" || clname == "TStreamerObjectAny" || clname == "TStreamerString" || clname == "TStreamerSTL") { Version(buf); // TStreamerXXX Version(buf); // TStreamerElement SkipObject(buf); const std::string name = String(buf); const std::string title = String(buf); int type = read(buf); size_t size = read(buf); if (clname.compare(0, 15, "TStreamerObject") == 0) size = 0; std::string counter; bool iscounter = false; if (ispointer) { if (!title.empty() && title.front() == '[') { const size_t endref = title.find(']', 1); if (endref != title.npos) { counter = title.substr(1, endref - 1); } } if (isbasicpointer) { // see root/io/io/inc/TStreamerInfo.h -> TStreamerInfo::EReadWrite switch (type - 40) { case 1: // char case 11: // unsigned char size = 1; break; case 2: // short case 12: // unsigned short case 19: // float16 size = 2; break; case 3: // int case 5: // float case 9: // double32 case 13: // unsigned int size = 4; break; case 4: // long case 8: // double case 14: // unsigned long case 16: // long case 17: // unsigned long size = 8; break; } } } else if (clname == "TStreamerBasicType") { iscounter = type == 6; // see root/io/io/inc/TStreamerInfo.h -> TStreamerInfo::EReadWrite } sinfo.emplace_back(StreamerInfo{name, size, counter, iscounter, ispointer}); } } buf = nbuf; } } else buf = nbuf; buf += 1; // trailing zero of TObjArray* } } else DEBUG("ROOTData: Inflation failed!") } bool ROOTData::advanceTo(char*& buf, const std::vector& objects, const std::string& current, const std::string& target, std::map& counts) { // The object structure can be retrieved from TFile::GetStreamerInfoList(). // Every ROOT object contains a version number which may include the byte count // for the object. The latter is currently assumed to be present to skip unused // objects. No checks are performed. The corresponding ROOT code is quite nested // but the actual readout is straight forward. auto it = objects.begin(); if (!current.empty()) { for (; it != objects.end(); ++it) { if (it->name == target) { return false; // target lies before current buffer position } else if (it->name == current) { ++it; break; } } } for (; it != objects.end(); ++it) { if (it->name == target) return true; if (it->size == 0) Skip(buf, 1); else if (it->iscounter) counts[it->name] = read(buf); else if (it->ispointer) { if (it->counter.empty()) buf += it->size + 1; else buf += it->size * counts[it->counter] + 1; } else buf += it->size; } return false; } // needs to be after ROOTDataHelpers namespace declaration QString ROOTFilter::fileInfoString(const QString& fileName) { DEBUG("ROOTFilter::fileInfoString()"); QString info; // The file structure is described in root/io/io/src/TFile.cxx std::ifstream is(fileName.toStdString(), std::ifstream::binary); std::string root(4, 0); is.read(const_cast(root.data()), 4); if (root != "root") { DEBUG(" Not a ROOT file. root = " << root); return i18n("Not a ROOT file"); } int version = read(is); info += i18n("File format version: %1", QString::number(version)); info += QLatin1String("
"); is.seekg(20); int freeBytes = read(is); int freeRecords = read(is); int namedBytes = read(is); char pointerBytes = read(is); info += i18n("FREE data record size: %1 bytes", QString::number(freeBytes)); info += QLatin1String("
"); info += i18n("Number of free data records: %1", QString::number(freeRecords)); info += QLatin1String("
"); info += i18n("TNamed size: %1 bytes", QString::number(namedBytes)); info += QLatin1String("
"); info += i18n("Size of file pointers: %1 bytes", QString::number(pointerBytes)); info += QLatin1String("
"); int compression = read(is); compression = compression > 0 ? compression : 0; info += i18n("Compression level and algorithm: %1", QString::number(compression)); info += QLatin1String("
"); is.seekg(41); int infoBytes = read(is); info += i18n("Size of TStreamerInfo record: %1 bytes", QString::number(infoBytes)); info += QLatin1String("
"); Q_UNUSED(fileName); return info; } diff --git a/src/backend/matrix/Matrix.cpp b/src/backend/matrix/Matrix.cpp index 4be93c72b..65b1ae060 100644 --- a/src/backend/matrix/Matrix.cpp +++ b/src/backend/matrix/Matrix.cpp @@ -1,1300 +1,1300 @@ /*************************************************************************** File : Matrix.cpp Project : Matrix Description : Spreadsheet with a MxN matrix data model -------------------------------------------------------------------- Copyright : (C) 2008-2009 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2015-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017-2018 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "Matrix.h" #include "MatrixPrivate.h" #include "matrixcommands.h" #include "backend/matrix/MatrixModel.h" #include "backend/core/Folder.h" #include "backend/lib/commandtemplates.h" #include "backend/lib/XmlStreamReader.h" #include "commonfrontend/matrix/MatrixView.h" #include "kdefrontend/spreadsheet/ExportSpreadsheetDialog.h" #include #include #include #include #include #include #include /*! This class manages matrix based data (i.e., mathematically a MxN matrix with M rows, N columns). This data is typically used to for 3D plots. The values of the matrix are stored as generic values. Each column of the matrix is stored in a QVector objects. \ingroup backend */ Matrix::Matrix(int rows, int cols, const QString& name, const AbstractColumn::ColumnMode mode) : AbstractDataSource(name, AspectType::Matrix), d(new MatrixPrivate(this, mode)) { //set initial number of rows and columns appendColumns(cols); appendRows(rows); init(); } Matrix::Matrix(const QString& name, bool loading, const AbstractColumn::ColumnMode mode) : AbstractDataSource(name, AspectType::Matrix), d(new MatrixPrivate(this, mode)) { if (!loading) init(); } Matrix::~Matrix() { delete d; } void Matrix::init() { KConfig config; KConfigGroup group = config.group("Matrix"); //matrix dimension int rows = group.readEntry("RowCount", 10); int cols = group.readEntry("ColumnCount", 10); appendRows(rows); appendColumns(cols); //mapping to logical x- and y-coordinates d->xStart = group.readEntry("XStart", 0.0); d->xEnd = group.readEntry("XEnd", 1.0); d->yStart = group.readEntry("YStart", 0.0); d->yEnd = group.readEntry("YEnd", 1.0); //format QByteArray formatba = group.readEntry("NumericFormat", "f").toLatin1(); d->numericFormat = *formatba.data(); d->precision = group.readEntry("Precision", 3); d->headerFormat = (Matrix::HeaderFormat)group.readEntry("HeaderFormat", (int)Matrix::HeaderRowsColumns); } /*! Returns an icon to be used for decorating my views. */ QIcon Matrix::icon() const { return QIcon::fromTheme("labplot-matrix"); } /*! Returns a new context menu. The caller takes ownership of the menu. */ QMenu* Matrix::createContextMenu() { QMenu* menu = AbstractPart::createContextMenu(); emit requestProjectContextMenu(menu); return menu; } QWidget* Matrix::view() const { if (!m_partView) { m_view = new MatrixView(const_cast(this)); m_partView = m_view; m_model = m_view->model(); } return m_partView; } bool Matrix::exportView() const { auto* dlg = new ExportSpreadsheetDialog(m_view); dlg->setFileName(name()); dlg->setMatrixMode(true); //TODO FITS filter to decide if it can be exported to both dlg->setExportTo(QStringList() << i18n("FITS image") << i18n("FITS table")); if (m_view->selectedColumnCount() == 0) { dlg->setExportSelection(false); } bool ret; if ( (ret = (dlg->exec() == QDialog::Accepted)) ) { const QString path = dlg->path(); WAIT_CURSOR; if (dlg->format() == ExportSpreadsheetDialog::LaTeX) { const bool verticalHeader = dlg->matrixVerticalHeader(); const bool horizontalHeader = dlg->matrixHorizontalHeader(); const bool latexHeader = dlg->exportHeader(); const bool gridLines = dlg->gridLines(); const bool entire = dlg->entireSpreadheet(); const bool captions = dlg->captions(); m_view->exportToLaTeX(path, verticalHeader, horizontalHeader, latexHeader, gridLines, entire, captions); } else if (dlg->format() == ExportSpreadsheetDialog::FITS) { const int exportTo = dlg->exportToFits(); m_view->exportToFits(path, exportTo ); } else { const QString separator = dlg->separator(); const QLocale::Language format = dlg->numberFormat(); m_view->exportToFile(path, separator, format); } RESET_CURSOR; } delete dlg; return ret; } bool Matrix::printView() { QPrinter printer; auto* dlg = new QPrintDialog(&printer, m_view); bool ret; dlg->setWindowTitle(i18nc("@title:window", "Print Matrix")); if ( (ret = (dlg->exec() == QDialog::Accepted)) ) m_view->print(&printer); delete dlg; return ret; } bool Matrix::printPreview() const { QPrintPreviewDialog* dlg = new QPrintPreviewDialog(m_view); connect(dlg, &QPrintPreviewDialog::paintRequested, m_view, &MatrixView::print); return dlg->exec(); } //############################################################################## //########################## getter methods ################################## //############################################################################## void* Matrix::data() const { return d->data; } BASIC_D_READER_IMPL(Matrix, AbstractColumn::ColumnMode, mode, mode) BASIC_D_READER_IMPL(Matrix, int, rowCount, rowCount) BASIC_D_READER_IMPL(Matrix, int, columnCount, columnCount) BASIC_D_READER_IMPL(Matrix, double, xStart, xStart) BASIC_D_READER_IMPL(Matrix, double, xEnd, xEnd) BASIC_D_READER_IMPL(Matrix, double, yStart, yStart) BASIC_D_READER_IMPL(Matrix, double, yEnd, yEnd) BASIC_D_READER_IMPL(Matrix, char, numericFormat, numericFormat) BASIC_D_READER_IMPL(Matrix, int, precision, precision) BASIC_D_READER_IMPL(Matrix, Matrix::HeaderFormat, headerFormat, headerFormat) CLASS_D_READER_IMPL(Matrix, QString, formula, formula) void Matrix::setSuppressDataChangedSignal(bool b) { if (m_model) m_model->setSuppressDataChangedSignal(b); } void Matrix::setChanged() { if (m_model) m_model->setChanged(); } //############################################################################## //################# setter methods and undo commands ########################## //############################################################################## void Matrix::setRowCount(int count) { if (count == d->rowCount) return; const int diff = count - d->rowCount; if (diff > 0) appendRows(diff); else if (diff < 0) removeRows(rowCount() + diff, -diff); } void Matrix::setColumnCount(int count) { if (count == d->columnCount) return; const int diff = count - columnCount(); if (diff > 0) appendColumns(diff); else if (diff < 0) removeColumns(columnCount() + diff, -diff); } STD_SETTER_CMD_IMPL_F_S(Matrix, SetXStart, double, xStart, updateViewHeader) void Matrix::setXStart(double xStart) { if (xStart != d->xStart) exec(new MatrixSetXStartCmd(d, xStart, ki18n("%1: x-start changed"))); } STD_SETTER_CMD_IMPL_F_S(Matrix, SetXEnd, double, xEnd, updateViewHeader) void Matrix::setXEnd(double xEnd) { if (xEnd != d->xEnd) exec(new MatrixSetXEndCmd(d, xEnd, ki18n("%1: x-end changed"))); } STD_SETTER_CMD_IMPL_F_S(Matrix, SetYStart, double, yStart, updateViewHeader) void Matrix::setYStart(double yStart) { if (yStart != d->yStart) exec(new MatrixSetYStartCmd(d, yStart, ki18n("%1: y-start changed"))); } STD_SETTER_CMD_IMPL_F_S(Matrix, SetYEnd, double, yEnd, updateViewHeader) void Matrix::setYEnd(double yEnd) { if (yEnd != d->yEnd) exec(new MatrixSetYEndCmd(d, yEnd, ki18n("%1: y-end changed"))); } STD_SETTER_CMD_IMPL_S(Matrix, SetNumericFormat, char, numericFormat) void Matrix::setNumericFormat(char format) { if (format != d->numericFormat) exec(new MatrixSetNumericFormatCmd(d, format, ki18n("%1: numeric format changed"))); } STD_SETTER_CMD_IMPL_S(Matrix, SetPrecision, int, precision) void Matrix::setPrecision(int precision) { if (precision != d->precision) exec(new MatrixSetPrecisionCmd(d, precision, ki18n("%1: precision changed"))); } //TODO: make this undoable? void Matrix::setHeaderFormat(Matrix::HeaderFormat format) { d->headerFormat = format; m_model->updateHeader(); if (m_view) m_view->resizeHeaders(); emit headerFormatChanged(format); } //columns void Matrix::insertColumns(int before, int count) { if (count < 1 || before < 0 || before > columnCount()) return; WAIT_CURSOR; exec(new MatrixInsertColumnsCmd(d, before, count)); RESET_CURSOR; } void Matrix::appendColumns(int count) { insertColumns(columnCount(), count); } void Matrix::removeColumns(int first, int count) { if (count < 1 || first < 0 || first+count > columnCount()) return; WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixRemoveColumnsCmd(d, first, count)); break; case AbstractColumn::Text: exec(new MatrixRemoveColumnsCmd(d, first, count)); break; case AbstractColumn::Integer: exec(new MatrixRemoveColumnsCmd(d, first, count)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixRemoveColumnsCmd(d, first, count)); break; } RESET_CURSOR; } void Matrix::clearColumn(int c) { WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixClearColumnCmd(d, c)); break; case AbstractColumn::Text: exec(new MatrixClearColumnCmd(d, c)); break; case AbstractColumn::Integer: exec(new MatrixClearColumnCmd(d, c)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixClearColumnCmd(d, c)); break; } RESET_CURSOR; } //rows void Matrix::insertRows(int before, int count) { if (count < 1 || before < 0 || before > rowCount()) return; WAIT_CURSOR; exec(new MatrixInsertRowsCmd(d, before, count)); RESET_CURSOR; } void Matrix::appendRows(int count) { insertRows(rowCount(), count); } void Matrix::removeRows(int first, int count) { if (count < 1 || first < 0 || first+count > rowCount()) return; WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixRemoveRowsCmd(d, first, count)); break; case AbstractColumn::Text: exec(new MatrixRemoveRowsCmd(d, first, count)); break; case AbstractColumn::Integer: exec(new MatrixRemoveRowsCmd(d, first, count)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixRemoveRowsCmd(d, first, count)); break; } RESET_CURSOR; } void Matrix::clearRow(int r) { switch (d->mode) { case AbstractColumn::Numeric: for (int c = 0; c < columnCount(); ++c) exec(new MatrixSetCellValueCmd(d, r, c, 0.0)); break; case AbstractColumn::Text: for (int c = 0; c < columnCount(); ++c) exec(new MatrixSetCellValueCmd(d, r, c, QString())); break; case AbstractColumn::Integer: for (int c = 0; c < columnCount(); ++c) exec(new MatrixSetCellValueCmd(d, r, c, 0)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int c = 0; c < columnCount(); ++c) exec(new MatrixSetCellValueCmd(d, r, c, QDateTime())); break; } } //! Return the value in the given cell (needs explicit instantiation) template T Matrix::cell(int row, int col) const { return d->cell(row, col); } template double Matrix::cell(int row, int col) const; template int Matrix::cell(int row, int col) const; template QDateTime Matrix::cell(int row, int col) const; template QString Matrix::cell(int row, int col) const; //! Return the text displayed in the given cell (needs explicit instantiation) template QString Matrix::text(int row, int col) { return QLocale().toString(cell(row,col)); } // special cases template <> QString Matrix::text(int row, int col) { return QLocale().toString(cell(row,col), d->numericFormat, d->precision); } template <> QString Matrix::text(int row, int col) { return cell(row,col); } template QString Matrix::text(int row, int col); template QString Matrix::text(int row, int col); //! Set the value of the cell (needs explicit instantiation) template void Matrix::setCell(int row, int col, T value) { if (row < 0 || row >= rowCount()) return; if (col < 0 || col >= columnCount()) return; exec(new MatrixSetCellValueCmd(d, row, col, value)); } template void Matrix::setCell(int row, int col, double value); template void Matrix::setCell(int row, int col, int value); template void Matrix::setCell(int row, int col, QDateTime value); template void Matrix::setCell(int row, int col, QString value); void Matrix::clearCell(int row, int col) { switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixSetCellValueCmd(d, row, col, 0.0)); break; case AbstractColumn::Text: exec(new MatrixSetCellValueCmd(d, row, col, QString())); break; case AbstractColumn::Integer: exec(new MatrixSetCellValueCmd(d, row, col, 0)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixSetCellValueCmd(d, row, col, QDateTime())); break; } } void Matrix::setDimensions(int rows, int cols) { if ( (rows < 0) || (cols < 0 ) || (rows == rowCount() && cols == columnCount()) ) return; WAIT_CURSOR; beginMacro(i18n("%1: set matrix size to %2x%3", name(), rows, cols)); int col_diff = cols - columnCount(); if (col_diff > 0) insertColumns(columnCount(), col_diff); else if (col_diff < 0) removeColumns(columnCount() + col_diff, -col_diff); int row_diff = rows - rowCount(); if (row_diff > 0) appendRows(row_diff); else if (row_diff < 0) removeRows(rowCount() + row_diff, -row_diff); endMacro(); RESET_CURSOR; } void Matrix::copy(Matrix* other) { WAIT_CURSOR; beginMacro(i18n("%1: copy %2", name(), other->name())); int rows = other->rowCount(); int columns = other->columnCount(); setDimensions(rows, columns); for (int i = 0; i < rows; i++) setRowHeight(i, other->rowHeight(i)); for (int i = 0; i < columns; i++) setColumnWidth(i, other->columnWidth(i)); d->suppressDataChange = true; switch (d->mode) { case AbstractColumn::Numeric: for (int i = 0; i < columns; i++) setColumnCells(i, 0, rows-1, other->columnCells(i, 0, rows-1)); break; case AbstractColumn::Text: for (int i = 0; i < columns; i++) setColumnCells(i, 0, rows-1, other->columnCells(i, 0, rows-1)); break; case AbstractColumn::Integer: for (int i = 0; i < columns; i++) setColumnCells(i, 0, rows-1, other->columnCells(i, 0, rows-1)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int i = 0; i < columns; i++) setColumnCells(i, 0, rows-1, other->columnCells(i, 0, rows-1)); break; } setCoordinates(other->xStart(), other->xEnd(), other->yStart(), other->yEnd()); setNumericFormat(other->numericFormat()); setPrecision(other->precision()); d->formula = other->formula(); d->suppressDataChange = false; emit dataChanged(0, 0, rows-1, columns-1); if (m_view) m_view->adjustHeaders(); endMacro(); RESET_CURSOR; } //! Duplicate the matrix inside its folder void Matrix::duplicate() { Matrix* matrix = new Matrix(rowCount(), columnCount(), name()); matrix->copy(this); if (folder()) folder()->addChild(matrix); } void Matrix::addRows() { if (!m_view) return; WAIT_CURSOR; int count = m_view->selectedRowCount(false); beginMacro(i18np("%1: add %2 rows", "%1: add %2 rows", name(), count)); exec(new MatrixInsertRowsCmd(d, rowCount(), count)); endMacro(); RESET_CURSOR; } void Matrix::addColumns() { if (!m_view) return; WAIT_CURSOR; int count = m_view->selectedRowCount(false); beginMacro(i18np("%1: add %2 column", "%1: add %2 columns", name(), count)); exec(new MatrixInsertColumnsCmd(d, columnCount(), count)); endMacro(); RESET_CURSOR; } void Matrix::setCoordinates(double x1, double x2, double y1, double y2) { exec(new MatrixSetCoordinatesCmd(d, x1, x2, y1, y2)); } void Matrix::setFormula(const QString& formula) { exec(new MatrixSetFormulaCmd(d, formula)); } //! This method should only be called by the view. /** This method does not change the view, it only changes the * values that are saved when the matrix is saved. The view * has to take care of reading and applying these values */ void Matrix::setRowHeight(int row, int height) { d->setRowHeight(row, height); } //! This method should only be called by the view. /** This method does not change the view, it only changes the * values that are saved when the matrix is saved. The view * has to take care of reading and applying these values */ void Matrix::setColumnWidth(int col, int width) { d->setColumnWidth(col, width); } int Matrix::rowHeight(int row) const { return d->rowHeight(row); } int Matrix::columnWidth(int col) const { return d->columnWidth(col); } //! Return the values in the given cells as vector template QVector Matrix::columnCells(int col, int first_row, int last_row) { return d->columnCells(col, first_row, last_row); } //! Set the values in the given cells from a type T vector template void Matrix::setColumnCells(int col, int first_row, int last_row, const QVector& values) { WAIT_CURSOR; exec(new MatrixSetColumnCellsCmd(d, col, first_row, last_row, values)); RESET_CURSOR; } //! Return the values in the given cells as vector (needs explicit instantiation) template QVector Matrix::rowCells(int row, int first_column, int last_column) { return d->rowCells(row, first_column, last_column); } template QVector Matrix::rowCells(int row, int first_column, int last_column); template QVector Matrix::rowCells(int row, int first_column, int last_column); template QVector Matrix::rowCells(int row, int first_column, int last_column); template QVector Matrix::rowCells(int row, int first_column, int last_column); //! Set the values in the given cells from a type T vector template void Matrix::setRowCells(int row, int first_column, int last_column, const QVector& values) { WAIT_CURSOR; exec(new MatrixSetRowCellsCmd(d, row, first_column, last_column, values)); RESET_CURSOR; } void Matrix::setData(void* data) { bool isEmpty = false; switch (d->mode) { case AbstractColumn::Numeric: if (static_cast>*>(data)->isEmpty()) isEmpty = true; break; case AbstractColumn::Text: if (static_cast>*>(data)->isEmpty()) isEmpty = true; break; case AbstractColumn::Integer: if (static_cast>*>(data)->isEmpty()) isEmpty = true; break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: if (static_cast>*>(data)->isEmpty()) isEmpty = true; break; } if (!isEmpty) exec(new MatrixReplaceValuesCmd(d, data)); } //############################################################################## //######################### Public slots ##################################### //############################################################################## //! Clear the whole matrix (i.e. reset all cells) void Matrix::clear() { WAIT_CURSOR; beginMacro(i18n("%1: clear", name())); switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixClearCmd(d)); break; case AbstractColumn::Text: exec(new MatrixClearCmd(d)); break; case AbstractColumn::Integer: exec(new MatrixClearCmd(d)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixClearCmd(d)); break; } endMacro(); RESET_CURSOR; } void Matrix::transpose() { WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixTransposeCmd(d)); break; case AbstractColumn::Text: exec(new MatrixTransposeCmd(d)); break; case AbstractColumn::Integer: exec(new MatrixTransposeCmd(d)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixTransposeCmd(d)); break; } RESET_CURSOR; } void Matrix::mirrorHorizontally() { WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixMirrorHorizontallyCmd(d)); break; case AbstractColumn::Text: exec(new MatrixMirrorHorizontallyCmd(d)); break; case AbstractColumn::Integer: exec(new MatrixMirrorHorizontallyCmd(d)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixMirrorHorizontallyCmd(d)); break; } RESET_CURSOR; } void Matrix::mirrorVertically() { WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixMirrorVerticallyCmd(d)); break; case AbstractColumn::Text: exec(new MatrixMirrorVerticallyCmd(d)); break; case AbstractColumn::Integer: exec(new MatrixMirrorVerticallyCmd(d)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixMirrorVerticallyCmd(d)); break; } RESET_CURSOR; } //############################################################################## //###################### Private implementation ############################### //############################################################################## MatrixPrivate::MatrixPrivate(Matrix* owner, const AbstractColumn::ColumnMode m) : q(owner), data(nullptr), mode(m), rowCount(0), columnCount(0), suppressDataChange(false) { switch (mode) { case AbstractColumn::Numeric: data = new QVector>(); break; case AbstractColumn::Text: data = new QVector>(); break; case AbstractColumn::Month: case AbstractColumn::Day: case AbstractColumn::DateTime: data = new QVector>(); break; case AbstractColumn::Integer: data = new QVector>(); break; } } MatrixPrivate::~MatrixPrivate() { if (data) { switch (mode) { case AbstractColumn::Numeric: delete static_cast>*>(data); break; case AbstractColumn::Text: delete static_cast>*>(data); break; case AbstractColumn::Integer: delete static_cast>*>(data); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: delete static_cast>*>(data); break; } } } void MatrixPrivate::updateViewHeader() { q->m_view->model()->updateHeader(); } /*! Insert \count columns before column number \c before */ void MatrixPrivate::insertColumns(int before, int count) { Q_ASSERT(before >= 0); Q_ASSERT(before <= columnCount); emit q->columnsAboutToBeInserted(before, count); switch (mode) { case AbstractColumn::Numeric: for (int i = 0; i < count; i++) { static_cast>*>(data)->insert(before+i, QVector(rowCount)); columnWidths.insert(before+i, 0); } break; case AbstractColumn::Text: for (int i = 0; i < count; i++) { static_cast>*>(data)->insert(before+i, QVector(rowCount)); columnWidths.insert(before+i, 0); } break; case AbstractColumn::Integer: for (int i = 0; i < count; i++) { static_cast>*>(data)->insert(before+i, QVector(rowCount)); columnWidths.insert(before+i, 0); } break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int i = 0; i < count; i++) { static_cast>*>(data)->insert(before+i, QVector(rowCount)); columnWidths.insert(before+i, 0); } break; } columnCount += count; emit q->columnsInserted(before, count); } /*! Remove \c count columns starting with column index \c first */ void MatrixPrivate::removeColumns(int first, int count) { emit q->columnsAboutToBeRemoved(first, count); Q_ASSERT(first >= 0); Q_ASSERT(first + count <= columnCount); switch (mode) { case AbstractColumn::Numeric: (static_cast>*>(data))->remove(first, count); break; case AbstractColumn::Text: (static_cast>*>(data))->remove(first, count); break; case AbstractColumn::Integer: (static_cast>*>(data))->remove(first, count); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: (static_cast>*>(data))->remove(first, count); break; } for (int i = 0; i < count; i++) columnWidths.remove(first); columnCount -= count; emit q->columnsRemoved(first, count); } /*! Insert \c count rows before row with the index \c before */ void MatrixPrivate::insertRows(int before, int count) { emit q->rowsAboutToBeInserted(before, count); Q_ASSERT(before >= 0); Q_ASSERT(before <= rowCount); switch (mode) { case AbstractColumn::Numeric: for (int col = 0; col < columnCount; col++) for (int i = 0; i < count; i++) (static_cast>*>(data))->operator[](col).insert(before+i, 0.0); break; case AbstractColumn::Text: for (int col = 0; col < columnCount; col++) for (int i = 0; i < count; i++) (static_cast>*>(data))->operator[](col).insert(before+i, QString()); break; case AbstractColumn::Integer: for (int col = 0; col < columnCount; col++) for (int i = 0; i < count; i++) (static_cast>*>(data))->operator[](col).insert(before+i, 0); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int col = 0; col < columnCount; col++) for (int i = 0; i < count; i++) (static_cast>*>(data))->operator[](col).insert(before+i, QDateTime()); } for (int i = 0; i < count; i++) rowHeights.insert(before+i, 0); rowCount += count; emit q->rowsInserted(before, count); } /*! Remove \c count columns starting from the column with index \c first */ void MatrixPrivate::removeRows(int first, int count) { emit q->rowsAboutToBeRemoved(first, count); Q_ASSERT(first >= 0); Q_ASSERT(first+count <= rowCount); switch (mode) { case AbstractColumn::Numeric: for (int col = 0; col < columnCount; col++) (static_cast>*>(data))->operator[](col).remove(first, count); break; case AbstractColumn::Text: for (int col = 0; col < columnCount; col++) (static_cast>*>(data))->operator[](col).remove(first, count); break; case AbstractColumn::Integer: for (int col = 0; col < columnCount; col++) (static_cast>*>(data))->operator[](col).remove(first, count); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int col = 0; col < columnCount; col++) (static_cast>*>(data))->operator[](col).remove(first, count); break; } for (int i = 0; i < count; i++) rowHeights.remove(first); rowCount -= count; emit q->rowsRemoved(first, count); } //! Fill column with zeroes void MatrixPrivate::clearColumn(int col) { switch (mode) { case AbstractColumn::Numeric: static_cast>*>(data)->operator[](col).fill(0.0); break; case AbstractColumn::Text: static_cast>*>(data)->operator[](col).fill(QString()); break; case AbstractColumn::Integer: static_cast>*>(data)->operator[](col).fill(0); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: static_cast>*>(data)->operator[](col).fill(QDateTime()); break; } if (!suppressDataChange) emit q->dataChanged(0, col, rowCount-1, col); } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## void Matrix::save(QXmlStreamWriter* writer) const { DEBUG("Matrix::save()"); writer->writeStartElement("matrix"); writeBasicAttributes(writer); writeCommentElement(writer); //formula writer->writeStartElement("formula"); writer->writeCharacters(d->formula); writer->writeEndElement(); //format writer->writeStartElement("format"); writer->writeAttribute("mode", QString::number(d->mode)); writer->writeAttribute("headerFormat", QString::number(d->headerFormat)); writer->writeAttribute("numericFormat", QString(QChar(d->numericFormat))); writer->writeAttribute("precision", QString::number(d->precision)); writer->writeEndElement(); //dimensions writer->writeStartElement("dimension"); writer->writeAttribute("columns", QString::number(d->columnCount)); writer->writeAttribute("rows", QString::number(d->rowCount)); writer->writeAttribute("x_start", QString::number(d->xStart)); writer->writeAttribute("x_end", QString::number(d->xEnd)); writer->writeAttribute("y_start", QString::number(d->yStart)); writer->writeAttribute("y_end", QString::number(d->yEnd)); writer->writeEndElement(); //vector with row heights writer->writeStartElement("row_heights"); const char* data = reinterpret_cast(d->rowHeights.constData()); int size = d->rowHeights.size() * sizeof(int); writer->writeCharacters(QByteArray::fromRawData(data,size).toBase64()); writer->writeEndElement(); //vector with column widths writer->writeStartElement("column_widths"); data = reinterpret_cast(d->columnWidths.constData()); size = d->columnWidths.size()*sizeof(int); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); //columns DEBUG(" mode = " << d->mode); switch (d->mode) { case AbstractColumn::Numeric: size = d->rowCount*sizeof(double); for (int i = 0; i < d->columnCount; ++i) { data = reinterpret_cast(static_cast>*>(d->data)->at(i).constData()); writer->writeStartElement("column"); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); } break; case AbstractColumn::Text: size = d->rowCount*sizeof(QString); for (int i = 0; i < d->columnCount; ++i) { QDEBUG(" string: " << static_cast>*>(d->data)->at(i)); data = reinterpret_cast(static_cast>*>(d->data)->at(i).constData()); writer->writeStartElement("column"); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); } break; case AbstractColumn::Integer: size = d->rowCount*sizeof(int); for (int i = 0; i < d->columnCount; ++i) { data = reinterpret_cast(static_cast>*>(d->data)->at(i).constData()); writer->writeStartElement("column"); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); } break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: size = d->rowCount*sizeof(QDateTime); for (int i = 0; i < d->columnCount; ++i) { data = reinterpret_cast(static_cast>*>(d->data)->at(i).constData()); writer->writeStartElement("column"); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); } break; } writer->writeEndElement(); // "matrix" } bool Matrix::load(XmlStreamReader* reader, bool preview) { DEBUG("Matrix::load()"); if (!readBasicAttributes(reader)) return false; KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs; QString str; // read child elements while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement() && reader->name() == "matrix") break; if (!reader->isStartElement()) continue; if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if (!preview && reader->name() == "formula") { d->formula = reader->text().toString().trimmed(); } else if (!preview && reader->name() == "format") { attribs = reader->attributes(); str = attribs.value("mode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("mode").toString()); else d->mode = AbstractColumn::ColumnMode(str.toInt()); str = attribs.value("headerFormat").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("headerFormat").toString()); else d->headerFormat = Matrix::HeaderFormat(str.toInt()); str = attribs.value("numericFormat").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("numericFormat").toString()); else { QByteArray formatba = str.toLatin1(); d->numericFormat = *formatba.data(); } str = attribs.value("precision").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("precision").toString()); else d->precision = str.toInt(); } else if (!preview && reader->name() == "dimension") { attribs = reader->attributes(); str = attribs.value("columns").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("columns").toString()); else d->columnCount = str.toInt(); str = attribs.value("rows").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("rows").toString()); else d->rowCount = str.toInt(); str = attribs.value("x_start").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("x_start").toString()); else d->xStart = str.toDouble(); str = attribs.value("x_end").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("x_end").toString()); else d->xEnd = str.toDouble(); str = attribs.value("y_start").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("y_start").toString()); else d->yStart = str.toDouble(); str = attribs.value("y_end").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("y_end").toString()); else d->yEnd = str.toDouble(); } else if (!preview && reader->name() == "row_heights") { reader->readNext(); QString content = reader->text().toString().trimmed(); QByteArray bytes = QByteArray::fromBase64(content.toLatin1()); int count = bytes.size()/sizeof(int); d->rowHeights.resize(count); memcpy(d->rowHeights.data(), bytes.data(), count*sizeof(int)); } else if (!preview && reader->name() == "column_widths") { reader->readNext(); QString content = reader->text().toString().trimmed(); QByteArray bytes = QByteArray::fromBase64(content.toLatin1()); int count = bytes.size()/sizeof(int); d->columnWidths.resize(count); memcpy(d->columnWidths.data(), bytes.data(), count*sizeof(int)); } else if (!preview && reader->name() == "column") { //TODO: parallelize reading of columns? reader->readNext(); QString content = reader->text().toString().trimmed(); QByteArray bytes = QByteArray::fromBase64(content.toLatin1()); switch (d->mode) { case AbstractColumn::Numeric: { int count = bytes.size()/sizeof(double); QVector column; column.resize(count); memcpy(column.data(), bytes.data(), count*sizeof(double)); static_cast>*>(d->data)->append(column); break; } case AbstractColumn::Text: { int count = bytes.size()/sizeof(char); QVector column; column.resize(count); //TODO: warning (GCC8): writing to an object of type 'class QString' with no trivial copy-assignment; use copy-assignment or copy-initialization instead //memcpy(column.data(), bytes.data(), count*sizeof(QString)); //QDEBUG(" string: " << column.data()); static_cast>*>(d->data)->append(column); break; } case AbstractColumn::Integer: { int count = bytes.size()/sizeof(int); QVector column; column.resize(count); memcpy(column.data(), bytes.data(), count*sizeof(int)); static_cast>*>(d->data)->append(column); break; } case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: { int count = bytes.size()/sizeof(QDateTime); QVector column; column.resize(count); //TODO: warning (GCC8): writing to an object of type 'class QDateTime' with no trivial copy-assignment; use copy-assignment or copy-initialization instead //memcpy(column.data(), bytes.data(), count*sizeof(QDateTime)); static_cast>*>(d->data)->append(column); break; } } } else { // unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } return true; } void Matrix::registerShortcuts() { m_view->registerShortcuts(); } void Matrix::unregisterShortcuts() { m_view->unregisterShortcuts(); } //############################################################################## //######################## Data Import ####################################### //############################################################################## -int Matrix::prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode mode, +int Matrix::prepareImport(std::vector& dataContainer, AbstractFileFilter::ImportMode mode, int actualRows, int actualCols, QStringList colNameList, QVector columnMode) { QDEBUG("prepareImport() rows =" << actualRows << " cols =" << actualCols); QDEBUG(" column modes = " << columnMode); Q_UNUSED(colNameList); int columnOffset = 0; setUndoAware(false); setSuppressDataChangedSignal(true); // resize the matrix if (mode == AbstractFileFilter::Replace) { clear(); setDimensions(actualRows, actualCols); } else { if (rowCount() < actualRows) setDimensions(actualRows, actualCols); else setDimensions(rowCount(), actualCols); } // data() returns a void* which is a pointer to a matrix of any data type (see ColumnPrivate.cpp) dataContainer.resize(actualCols); switch (columnMode[0]) { // only columnMode[0] is used case AbstractColumn::Numeric: for (int n = 0; n < actualCols; n++) { QVector* vector = &(static_cast>*>(data())->operator[](n)); vector->resize(actualRows); dataContainer[n] = static_cast(vector); } d->mode = AbstractColumn::Numeric; break; case AbstractColumn::Integer: for (int n = 0; n < actualCols; n++) { QVector* vector = &(static_cast>*>(data())->operator[](n)); vector->resize(actualRows); dataContainer[n] = static_cast(vector); } d->mode = AbstractColumn::Integer; break; case AbstractColumn::Text: for (int n = 0; n < actualCols; n++) { QVector* vector = &(static_cast>*>(data())->operator[](n)); vector->resize(actualRows); dataContainer[n] = static_cast(vector); } d->mode = AbstractColumn::Text; break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int n = 0; n < actualCols; n++) { QVector* vector = &(static_cast>*>(data())->operator[](n)); vector->resize(actualRows); dataContainer[n] = static_cast(vector); } d->mode = AbstractColumn::DateTime; break; } return columnOffset; } void Matrix::finalizeImport(int columnOffset, int startColumn, int endColumn, const QString& dateTimeFormat, AbstractFileFilter::ImportMode importMode) { DEBUG("Matrix::finalizeImport()"); Q_UNUSED(columnOffset); Q_UNUSED(startColumn); Q_UNUSED(endColumn); Q_UNUSED(dateTimeFormat); Q_UNUSED(importMode); setSuppressDataChangedSignal(false); setChanged(); setUndoAware(true); DEBUG("Matrix::finalizeImport() DONE"); } diff --git a/src/backend/matrix/Matrix.h b/src/backend/matrix/Matrix.h index 86f7c915d..708e5d902 100644 --- a/src/backend/matrix/Matrix.h +++ b/src/backend/matrix/Matrix.h @@ -1,168 +1,168 @@ /*************************************************************************** File : Matrix.h Project : Matrix Description : Spreadsheet with a MxN matrix data model -------------------------------------------------------------------- Copyright : (C) 2008-2009 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2015-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef MATRIX_H #define MATRIX_H #include "backend/datasources/AbstractDataSource.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/lib/macros.h" class MatrixPrivate; class MatrixModel; class MatrixView; class Matrix : public AbstractDataSource { Q_OBJECT Q_ENUMS(HeaderFormat) public: enum HeaderFormat {HeaderRowsColumns, HeaderValues, HeaderRowsColumnsValues}; explicit Matrix(const QString& name, bool loading = false, const AbstractColumn::ColumnMode = AbstractColumn::Numeric); Matrix(int rows, int cols, const QString& name, const AbstractColumn::ColumnMode = AbstractColumn::Numeric); ~Matrix() override; QIcon icon() const override; QMenu* createContextMenu() override; QWidget* view() const override; bool exportView() const override; bool printView() override; bool printPreview() const override; void* data() const; void setData(void*); BASIC_D_ACCESSOR_DECL(AbstractColumn::ColumnMode, mode, Mode) BASIC_D_ACCESSOR_DECL(int, rowCount, RowCount) BASIC_D_ACCESSOR_DECL(int, columnCount, ColumnCount) BASIC_D_ACCESSOR_DECL(char, numericFormat, NumericFormat) BASIC_D_ACCESSOR_DECL(int, precision, Precision) BASIC_D_ACCESSOR_DECL(HeaderFormat, headerFormat, HeaderFormat) BASIC_D_ACCESSOR_DECL(double, xStart, XStart) BASIC_D_ACCESSOR_DECL(double, xEnd, XEnd) BASIC_D_ACCESSOR_DECL(double, yStart, YStart) BASIC_D_ACCESSOR_DECL(double, yEnd, YEnd) CLASS_D_ACCESSOR_DECL(QString, formula, Formula) void setSuppressDataChangedSignal(bool); void setChanged(); int rowHeight(int row) const; void setRowHeight(int row, int height); int columnWidth(int col) const; void setColumnWidth(int col, int width); void setDimensions(int rows, int cols); void setCoordinates(double x1, double x2, double y1, double y2); void insertColumns(int before, int count); void appendColumns(int count); void removeColumns(int first, int count); void clearColumn(int); void insertRows(int before, int count); void appendRows(int count); void removeRows(int first, int count); void clearRow(int); template T cell(int row, int col) const; template QString text(int row, int col); template void setCell(int row, int col, T value); void clearCell(int row, int col); template QVector columnCells(int col, int first_row, int last_row); template void setColumnCells(int col, int first_row, int last_row, const QVector& values); template QVector rowCells(int row, int first_column, int last_column); template void setRowCells(int row, int first_column, int last_column, const QVector& values); void copy(Matrix* other); void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*, bool preview) override; void registerShortcuts() override; void unregisterShortcuts() override; - int prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode, + int prepareImport(std::vector& dataContainer, AbstractFileFilter::ImportMode, int rows, int cols, QStringList colNameList, QVector) override; void finalizeImport(int columnOffset, int startColumn, int endColumn, const QString& dateTimeFormat, AbstractFileFilter::ImportMode) override; typedef MatrixPrivate Private; public slots: void clear(); void transpose(); void mirrorVertically(); void mirrorHorizontally(); void addColumns(); void addRows(); void duplicate(); signals: void requestProjectContextMenu(QMenu*); void columnsAboutToBeInserted(int before, int count); void columnsInserted(int first, int count); void columnsAboutToBeRemoved(int first, int count); void columnsRemoved(int first, int count); void rowsAboutToBeInserted(int before, int count); void rowsInserted(int first, int count); void rowsAboutToBeRemoved(int first, int count); void rowsRemoved(int first, int count); void dataChanged(int top, int left, int bottom, int right); void coordinatesChanged(); void rowCountChanged(int); void columnCountChanged(int); void xStartChanged(double); void xEndChanged(double); void yStartChanged(double); void yEndChanged(double); void numericFormatChanged(char); void precisionChanged(int); void headerFormatChanged(Matrix::HeaderFormat); private: void init(); MatrixPrivate* const d; mutable MatrixModel* m_model{nullptr}; mutable MatrixView* m_view{nullptr}; friend class MatrixPrivate; }; #endif diff --git a/src/backend/spreadsheet/Spreadsheet.cpp b/src/backend/spreadsheet/Spreadsheet.cpp index 4c6cdfa3c..2647de3cf 100644 --- a/src/backend/spreadsheet/Spreadsheet.cpp +++ b/src/backend/spreadsheet/Spreadsheet.cpp @@ -1,998 +1,998 @@ /*************************************************************************** File : Spreadsheet.cpp Project : LabPlot Description : Aspect providing a spreadsheet table with column logic -------------------------------------------------------------------- Copyright : (C) 2006-2008 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2006-2009 Knut Franke (knut.franke@gmx.de) Copyright : (C) 2012-2019 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "Spreadsheet.h" #include "SpreadsheetModel.h" #include "backend/core/AspectPrivate.h" #include "backend/core/AbstractAspect.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/datatypes/DateTime2StringFilter.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include "commonfrontend/spreadsheet/SpreadsheetView.h" #include #include #include #include #include /*! \class Spreadsheet \brief Aspect providing a spreadsheet table with column logic. Spreadsheet is a container object for columns with no data of its own. By definition, it's columns are all of its children inheriting from class Column. Thus, the basic API is already defined by AbstractAspect (managing the list of columns, notification of column insertion/removal) and Column (changing and monitoring state of the actual data). Spreadsheet stores a pointer to its primary view of class SpreadsheetView. SpreadsheetView calls the Spreadsheet API but Spreadsheet only notifies SpreadsheetView by signals without calling its API directly. This ensures a maximum independence of UI and backend. SpreadsheetView can be easily replaced by a different class. User interaction is completely handled in SpreadsheetView and translated into Spreadsheet API calls (e.g., when a user edits a cell this will be handled by the delegate of SpreadsheetView and Spreadsheet will not know whether a script or a user changed the data.). All actions, menus etc. for the user interaction are handled SpreadsheetView, e.g., via a context menu. Selections are also handled by SpreadsheetView. The view itself is created by the first call to view(); \ingroup backend */ Spreadsheet::Spreadsheet(const QString& name, bool loading, AspectType type) : AbstractDataSource(name, type) { if (!loading) init(); } /*! initializes the spreadsheet with the default number of columns and rows */ void Spreadsheet::init() { KConfigGroup group = KSharedConfig::openConfig()->group(QLatin1String("Spreadsheet")); const int columns = group.readEntry(QLatin1String("ColumnCount"), 2); const int rows = group.readEntry(QLatin1String("RowCount"), 100); for (int i = 0; i < columns; i++) { Column* new_col = new Column(QString::number(i+1), AbstractColumn::Numeric); new_col->setPlotDesignation(i == 0 ? AbstractColumn::X : AbstractColumn::Y); addChild(new_col); } setRowCount(rows); } void Spreadsheet::setModel(SpreadsheetModel* model) { m_model = model; } SpreadsheetModel* Spreadsheet::model() { return m_model; } /*! Constructs a primary view on me. This method may be called multiple times during the life time of an Aspect, or it might not get called at all. Aspects must not depend on the existence of a view for their operation. */ QWidget* Spreadsheet::view() const { if (!m_partView) { bool readOnly = (this->parentAspect()->type() == AspectType::DatapickerCurve); m_view = new SpreadsheetView(const_cast(this), readOnly); m_partView = m_view; } return m_partView; } bool Spreadsheet::exportView() const { return m_view->exportView(); } bool Spreadsheet::printView() { return m_view->printView(); } bool Spreadsheet::printPreview() const { return m_view->printPreview(); } /*! Returns the maximum number of rows in the spreadsheet. */ int Spreadsheet::rowCount() const { int result = 0; for (auto* col : children()) { const int col_rows = col->rowCount(); if ( col_rows > result) result = col_rows; } return result; } void Spreadsheet::removeRows(int first, int count) { if ( count < 1 || first < 0 || first+count > rowCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: remove 1 row", "%1: remove %2 rows", name(), count) ); for (auto* col : children(IncludeHidden)) col->removeRows(first, count); endMacro(); RESET_CURSOR; } void Spreadsheet::insertRows(int before, int count) { if ( count < 1 || before < 0 || before > rowCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: insert 1 row", "%1: insert %2 rows", name(), count) ); for (auto* col : children(IncludeHidden)) col->insertRows(before, count); endMacro(); RESET_CURSOR; } void Spreadsheet::appendRows(int count) { insertRows(rowCount(), count); } void Spreadsheet::appendRow() { insertRows(rowCount(), 1); } void Spreadsheet::appendColumns(int count) { insertColumns(columnCount(), count); } void Spreadsheet::appendColumn() { insertColumns(columnCount(), 1); } void Spreadsheet::prependColumns(int count) { insertColumns(0, count); } /*! Sets the number of rows of the spreadsheet to \c new_size */ void Spreadsheet::setRowCount(int new_size) { int current_size = rowCount(); if (new_size > current_size) insertRows(current_size, new_size-current_size); if (new_size < current_size && new_size >= 0) removeRows(new_size, current_size-new_size); } /*! Returns the column with the number \c index. Shallow wrapper around \sa AbstractAspect::child() - see there for caveat. */ Column* Spreadsheet::column(int index) const { return child(index); } /*! Returns the column with the name \c name. */ Column* Spreadsheet::column(const QString &name) const { return child(name); } /*! Returns the total number of columns in the spreadsheet. */ int Spreadsheet::columnCount() const { return childCount(); } /*! Returns the number of columns matching the given designation. */ int Spreadsheet::columnCount(AbstractColumn::PlotDesignation pd) const { int count = 0; for (auto* col : children()) if (col->plotDesignation() == pd) count++; return count; } void Spreadsheet::removeColumns(int first, int count) { if ( count < 1 || first < 0 || first+count > columnCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: remove 1 column", "%1: remove %2 columns", name(), count) ); for (int i = 0; i < count; i++) child(first)->remove(); endMacro(); RESET_CURSOR; } void Spreadsheet::insertColumns(int before, int count) { WAIT_CURSOR; beginMacro( i18np("%1: insert 1 column", "%1: insert %2 columns", name(), count) ); Column * before_col = column(before); int rows = rowCount(); for (int i = 0; i < count; i++) { Column * new_col = new Column(QString::number(i+1), AbstractColumn::Numeric); new_col->setPlotDesignation(AbstractColumn::Y); new_col->insertRows(0, rows); insertChildBefore(new_col, before_col); } endMacro(); RESET_CURSOR; } /*! Sets the number of columns to \c new_size */ void Spreadsheet::setColumnCount(int new_size) { int old_size = columnCount(); if ( old_size == new_size || new_size < 0 ) return; if (new_size < old_size) removeColumns(new_size, old_size-new_size); else insertColumns(old_size, new_size-old_size); } /*! Clears the whole spreadsheet. */ void Spreadsheet::clear() { WAIT_CURSOR; beginMacro(i18n("%1: clear", name())); for (auto* col : children()) col->clear(); endMacro(); RESET_CURSOR; } /*! Clears all mask in the spreadsheet. */ void Spreadsheet::clearMasks() { WAIT_CURSOR; beginMacro(i18n("%1: clear all masks", name())); for (auto* col : children()) col->clearMasks(); endMacro(); RESET_CURSOR; } /*! Returns a new context menu. The caller takes ownership of the menu. */ QMenu* Spreadsheet::createContextMenu() { QMenu* menu = AbstractPart::createContextMenu(); Q_ASSERT(menu); emit requestProjectContextMenu(menu); return menu; } void Spreadsheet::moveColumn(int from, int to) { Column* col = child(from); beginMacro(i18n("%1: move column %2 from position %3 to %4.", name(), col->name(), from+1, to+1)); col->remove(); insertChildBefore(col, child(to)); endMacro(); } void Spreadsheet::copy(Spreadsheet* other) { WAIT_CURSOR; beginMacro(i18n("%1: copy %2", name(), other->name())); for (auto* col : children()) col->remove(); for (auto* src_col : other->children()) { Column * new_col = new Column(src_col->name(), src_col->columnMode()); new_col->copy(src_col); new_col->setPlotDesignation(src_col->plotDesignation()); QVector< Interval > masks = src_col->maskedIntervals(); for (const auto& iv : masks) new_col->setMasked(iv); QVector< Interval > formulas = src_col->formulaIntervals(); for (const auto& iv : formulas) new_col->setFormula(iv, src_col->formula(iv.start())); new_col->setWidth(src_col->width()); addChild(new_col); } setComment(other->comment()); endMacro(); RESET_CURSOR; } // FIXME: replace index-based API with Column*-based one /*! Determines the corresponding X column. */ int Spreadsheet::colX(int col) { for (int i = col-1; i >= 0; i--) { if (column(i)->plotDesignation() == AbstractColumn::X) return i; } int cols = columnCount(); for (int i = col+1; i < cols; i++) { if (column(i)->plotDesignation() == AbstractColumn::X) return i; } return -1; } /*! Determines the corresponding Y column. */ int Spreadsheet::colY(int col) { int cols = columnCount(); if (column(col)->plotDesignation() == AbstractColumn::XError || column(col)->plotDesignation() == AbstractColumn::YError) { // look to the left first for (int i = col-1; i >= 0; i--) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } for (int i = col+1; i < cols; i++) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } } else { // look to the right first for (int i = col+1; i < cols; i++) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } for (int i = col-1; i >= 0; i--) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } } return -1; } /*! Sorts the given list of column. If 'leading' is a null pointer, each column is sorted separately. */ void Spreadsheet::sortColumns(Column* leading, QVector cols, bool ascending) { if (cols.isEmpty()) return; // the normal QPair comparison does not work properly with descending sorting // therefore we use our own compare functions class CompareFunctions { public: static bool doubleLess(const QPair& a, const QPair& b) { return a.first < b.first; } static bool doubleGreater(const QPair& a, const QPair& b) { return a.first > b.first; } static bool integerLess(const QPair& a, const QPair& b) { return a.first < b.first; } static bool integerGreater(const QPair& a, const QPair& b) { return a.first > b.first; } static bool QStringLess(const QPair& a, const QPair& b) { return a < b; } static bool QStringGreater(const QPair& a, const QPair& b) { return a > b; } static bool QDateTimeLess(const QPair& a, const QPair& b) { return a < b; } static bool QDateTimeGreater(const QPair& a, const QPair& b) { return a > b; } }; WAIT_CURSOR; beginMacro(i18n("%1: sort columns", name())); if (leading == nullptr) { // sort separately for (auto* col : cols) { switch (col->columnMode()) { case AbstractColumn::Numeric: { int rows = col->rowCount(); QVector< QPair > map; for (int j = 0; j < rows; j++) map.append(QPair(col->valueAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::Integer: { int rows = col->rowCount(); QVector< QPair > map; for (int j = 0; j < rows; j++) map.append(QPair(col->valueAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator> it(map); Column* temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::Text: { int rows = col->rowCount(); QVector> map; for (int j = 0; j < rows; j++) map.append(QPair(col->textAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringGreater); QVectorIterator< QPair > it(map); Column* temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { int rows = col->rowCount(); QVector< QPair > map; for (int j = 0; j < rows; j++) map.append(QPair(col->dateTimeAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeGreater); QVectorIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } } } } else { // sort with leading column switch (leading->columnMode()) { case AbstractColumn::Numeric: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->valueAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator> it(map); for (auto* col : cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::Integer: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->valueAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::integerLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::integerGreater); QVectorIterator> it(map); for (auto* col : cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::Text: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->textAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringGreater); QVectorIterator> it(map); for (auto* col : cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->dateTimeAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeGreater); QVectorIterator> it(map); for (auto* col : cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } } } endMacro(); RESET_CURSOR; } // end of sortColumns() /*! Returns an icon to be used for decorating my views. */ QIcon Spreadsheet::icon() const { return QIcon::fromTheme("labplot-spreadsheet"); } /*! Returns the text displayed in the given cell. */ QString Spreadsheet::text(int row, int col) const { Column* c = column(col); if (!c) return QString(); return c->asStringColumn()->textAt(row); } /*! * This slot is, indirectly, called when a child of \c Spreadsheet (i.e. column) was selected in \c ProjectExplorer. * Emits the signal \c columnSelected that is handled in \c SpreadsheetView. */ void Spreadsheet::childSelected(const AbstractAspect* aspect) { const Column* column = qobject_cast(aspect); if (column) { int index = indexOfChild(column); emit columnSelected(index); } } /*! * This slot is, indirectly, called when a child of \c Spreadsheet (i.e. column) was deselected in \c ProjectExplorer. * Emits the signal \c columnDeselected that is handled in \c SpreadsheetView. */ void Spreadsheet::childDeselected(const AbstractAspect* aspect) { const Column* column = qobject_cast(aspect); if (column) { int index = indexOfChild(column); emit columnDeselected(index); } } /*! * Emits the signal to select or to deselect the column number \c index in the project explorer, * if \c selected=true or \c selected=false, respectively. * The signal is handled in \c AspectTreeModel and forwarded to the tree view in \c ProjectExplorer. * This function is called in \c SpreadsheetView upon selection changes. */ void Spreadsheet::setColumnSelectedInView(int index, bool selected) { if (selected) { emit childAspectSelectedInView(child(index)); //deselect the spreadsheet in the project explorer, if a child (column) was selected. //prevents unwanted multiple selection with spreadsheet (if it was selected before). emit childAspectDeselectedInView(this); } else emit childAspectDeselectedInView(child(index)); } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void Spreadsheet::save(QXmlStreamWriter* writer) const { writer->writeStartElement("spreadsheet"); writeBasicAttributes(writer); writeCommentElement(writer); //columns for (auto* col : children(IncludeHidden)) col->save(writer); writer->writeEndElement(); // "spreadsheet" } /*! Loads from XML. */ bool Spreadsheet::load(XmlStreamReader* reader, bool preview) { if (!readBasicAttributes(reader)) return false; // read child elements while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement()) break; if (reader->isStartElement()) { if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if (reader->name() == "column") { Column* column = new Column(QString()); if (!column->load(reader, preview)) { delete column; setColumnCount(0); return false; } addChildFast(column); } else { // unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } } return !reader->hasError(); } void Spreadsheet::registerShortcuts() { //TODO: when we create a live-data source we don't have the view here yet. why? if (m_view) m_view->registerShortcuts(); } void Spreadsheet::unregisterShortcuts() { if (m_view) m_view->unregisterShortcuts(); } //############################################################################## //######################## Data Import ####################################### //############################################################################## -int Spreadsheet::prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode importMode, +int Spreadsheet::prepareImport(std::vector& dataContainer, AbstractFileFilter::ImportMode importMode, int actualRows, int actualCols, QStringList colNameList, QVector columnMode) { DEBUG("Spreadsheet::prepareImport()") DEBUG(" resize spreadsheet to rows = " << actualRows << " and cols = " << actualCols) QDEBUG(" column name list = " << colNameList) int columnOffset = 0; setUndoAware(false); if (m_model != nullptr) m_model->suppressSignals(true); //make the available columns undo unaware before we resize and rename them below, //the same will be done for new columns in this->resize(). for (int i = 0; i < childCount(); i++) child(i)->setUndoAware(false); columnOffset = this->resize(importMode, colNameList, actualCols); // resize the spreadsheet if (importMode == AbstractFileFilter::Replace) { clear(); setRowCount(actualRows); } else { if (rowCount() < actualRows) setRowCount(actualRows); } if (columnMode.size() < actualCols) { qWarning("columnMode[] size is too small! Giving up."); return -1; } dataContainer.resize(actualCols); for (int n = 0; n < actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) Column* column = this->child(columnOffset+n); DEBUG(" column " << n << " columnMode = " << columnMode[n]); column->setColumnMode(columnMode[n]); //in the most cases the first imported column is meant to be used as x-data. //Other columns provide mostly y-data or errors. //TODO: this has to be configurable for the user in the import widget, //it should be possible to specify x-error plot designation, etc. AbstractColumn::PlotDesignation desig = (n == 0) ? AbstractColumn::X : AbstractColumn::Y; column->setPlotDesignation(desig); switch (columnMode[n]) { case AbstractColumn::Numeric: { auto* vector = static_cast*>(column->data()); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { auto* vector = static_cast*>(column->data()); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { auto* vector = static_cast*>(column->data()); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Month: case AbstractColumn::Day: case AbstractColumn::DateTime: { auto* vector = static_cast* >(column->data()); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } } } // QDEBUG("dataPointers =" << dataPointers); DEBUG("Spreadsheet::prepareImport() DONE"); return columnOffset; } /*! resize data source to cols columns returns column offset depending on import mode */ int Spreadsheet::resize(AbstractFileFilter::ImportMode mode, QStringList colNameList, int cols) { DEBUG("Spreadsheet::resize()") QDEBUG(" column name list = " << colNameList) // name additional columns for (int k = colNameList.size(); k < cols; k++ ) colNameList.append( "Column " + QString::number(k+1) ); int columnOffset = 0; //indexes the "start column" in the spreadsheet. Starting from this column the data will be imported. Column* newColumn = nullptr; if (mode == AbstractFileFilter::Append) { columnOffset = childCount(); for (int n = 0; n < cols; n++ ) { newColumn = new Column(colNameList.at(n), AbstractColumn::Numeric); newColumn->setUndoAware(false); addChildFast(newColumn); } } else if (mode == AbstractFileFilter::Prepend) { Column* firstColumn = child(0); for (int n = 0; n < cols; n++ ) { newColumn = new Column(colNameList.at(n), AbstractColumn::Numeric); newColumn->setUndoAware(false); insertChildBeforeFast(newColumn, firstColumn); } } else if (mode == AbstractFileFilter::Replace) { //replace completely the previous content of the data source with the content to be imported. int columns = childCount(); if (columns > cols) { //there're more columns in the data source then required -> remove the superfluous columns for (int i = 0; i < columns-cols; i++) removeChild(child(0)); } else { //create additional columns if needed for (int i = columns; i < cols; i++) { newColumn = new Column(colNameList.at(i), AbstractColumn::Numeric); newColumn->setUndoAware(false); addChildFast(newColumn); } } //rename the columns that are already available and suppress the dataChanged signal for them for (int i = 0; i < childCount(); i++) { if (mode == AbstractFileFilter::Replace) { child(i)->setSuppressDataChangedSignal(true); emit child(i)->reset(child(i)); } child(i)->setName(colNameList.at(i)); // Force aspectDescriptionChanged is send, because otherwise the column // will not connected again to the curves (project.cpp, descriptionChanged) // it is not that expensive to call it twice (first time in setName, but // only if the name changed) child(i)->aspectDescriptionChanged(child(i)); } } return columnOffset; } void Spreadsheet::finalizeImport(int columnOffset, int startColumn, int endColumn, const QString& dateTimeFormat, AbstractFileFilter::ImportMode importMode) { DEBUG("Spreadsheet::finalizeImport()"); //determine the dependent plots QVector plots; if (importMode == AbstractFileFilter::Replace) { for (int n = startColumn; n <= endColumn; n++) { Column* column = this->column(columnOffset + n - startColumn); column->addUsedInPlots(plots); } //suppress retransform in the dependent plots for (auto* plot : plots) plot->setSuppressDataChangedSignal(true); } // set the comments for each of the columns if datasource is a spreadsheet const int rows = rowCount(); for (int n = startColumn; n <= endColumn; n++) { Column* column = this->column(columnOffset + n - startColumn); DEBUG(" column " << n << " of type " << column->columnMode()); QString comment; switch (column->columnMode()) { case AbstractColumn::Numeric: comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); break; case AbstractColumn::Integer: comment = i18np("integer data, %1 element", "integer data, %1 elements", rows); break; case AbstractColumn::Text: comment = i18np("text data, %1 element", "text data, %1 elements", rows); break; case AbstractColumn::Month: comment = i18np("month data, %1 element", "month data, %1 elements", rows); break; case AbstractColumn::Day: comment = i18np("day data, %1 element", "day data, %1 elements", rows); break; case AbstractColumn::DateTime: comment = i18np("date and time data, %1 element", "date and time data, %1 elements", rows); // set same datetime format in column auto* filter = static_cast(column->outputFilter()); filter->setFormat(dateTimeFormat); } column->setComment(comment); if (importMode == AbstractFileFilter::Replace) { column->setSuppressDataChangedSignal(false); column->setChanged(); } } if (importMode == AbstractFileFilter::Replace) { //retransform the dependent plots for (auto* plot : plots) { plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } //make the spreadsheet and all its children undo aware again setUndoAware(true); for (int i = 0; i < childCount(); i++) child(i)->setUndoAware(true); if (m_model != nullptr) m_model->suppressSignals(false); if (m_partView != nullptr && m_view != nullptr) m_view->resizeHeader(); DEBUG("Spreadsheet::finalizeImport() DONE"); } diff --git a/src/backend/spreadsheet/Spreadsheet.h b/src/backend/spreadsheet/Spreadsheet.h index 2fa284312..56b6e91eb 100644 --- a/src/backend/spreadsheet/Spreadsheet.h +++ b/src/backend/spreadsheet/Spreadsheet.h @@ -1,130 +1,130 @@ /*************************************************************************** File : Spreadsheet.h Project : LabPlot Description : Aspect providing a spreadsheet table with column logic -------------------------------------------------------------------- Copyright : (C) 2010-2017 Alexander Semke(alexander.semke@web.de) Copyright : (C) 2006-2008 Tilman Benkert (thzs@gmx.net) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef SPREADSHEET_H #define SPREADSHEET_H #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/ColumnStringIO.h" class AbstractFileFilter; class SpreadsheetView; class SpreadsheetModel; template class QVector; class Spreadsheet : public AbstractDataSource { Q_OBJECT public: explicit Spreadsheet(const QString& name, bool loading = false, AspectType type = AspectType::Spreadsheet); QIcon icon() const override; QMenu* createContextMenu() override; QWidget* view() const override; bool exportView() const override; bool printView() override; bool printPreview() const override; void setModel(SpreadsheetModel*); SpreadsheetModel* model(); int columnCount() const; int columnCount(AbstractColumn::PlotDesignation) const; Column* column(int index) const; Column* column(const QString&) const; int rowCount() const; void removeRows(int first, int count); void insertRows(int before, int count); void removeColumns(int first, int count); void insertColumns(int before, int count); int colX(int col); int colY(int col); QString text(int row, int col) const; void copy(Spreadsheet* other); void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*, bool preview) override; void setColumnSelectedInView(int index, bool selected); // used from model to inform dock void emitRowCountChanged() { emit rowCountChanged(rowCount()); } void emitColumnCountChanged() { emit columnCountChanged(columnCount()); } void registerShortcuts() override; void unregisterShortcuts() override; //data import - int prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode, + int prepareImport(std::vector& dataContainer, AbstractFileFilter::ImportMode, int rows, int cols, QStringList colNameList, QVector) override; void finalizeImport(int columnOffset, int startColumn , int endColumn, const QString& dateTimeFormat, AbstractFileFilter::ImportMode) override; int resize(AbstractFileFilter::ImportMode, QStringList colNameList, int cols); public slots: void appendRows(int); void appendRow(); void appendColumns(int); void appendColumn(); void prependColumns(int); void setColumnCount(int); void setRowCount(int); void clear(); void clearMasks(); void moveColumn(int from, int to); void sortColumns(Column* leading, QVector, bool ascending); private: void init(); SpreadsheetModel* m_model{nullptr}; protected: mutable SpreadsheetView* m_view{nullptr}; private slots: void childSelected(const AbstractAspect*) override; void childDeselected(const AbstractAspect*) override; signals: void requestProjectContextMenu(QMenu*); void columnSelected(int); void columnDeselected(int); // for spreadsheet dock void rowCountChanged(int); void columnCountChanged(int); }; #endif diff --git a/src/kdefrontend/datasources/ImportSQLDatabaseWidget.cpp b/src/kdefrontend/datasources/ImportSQLDatabaseWidget.cpp index fe668e431..d16662147 100644 --- a/src/kdefrontend/datasources/ImportSQLDatabaseWidget.cpp +++ b/src/kdefrontend/datasources/ImportSQLDatabaseWidget.cpp @@ -1,489 +1,489 @@ /*************************************************************************** File : ImportSQLDatabaseWidget.cpp Project : LabPlot Description : Datapicker -------------------------------------------------------------------- Copyright : (C) 2016 by Ankit Wagadre (wagadre.ankit@gmail.com) Copyright : (C) 2016-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "ImportSQLDatabaseWidget.h" #include "DatabaseManagerDialog.h" #include "DatabaseManagerWidget.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/lib/macros.h" #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_KF5_SYNTAX_HIGHLIGHTING #include #include #include #endif ImportSQLDatabaseWidget::ImportSQLDatabaseWidget(QWidget* parent) : QWidget(parent) { ui.setupUi(this); ui.cbImportFrom->addItem(i18n("Table")); ui.cbImportFrom->addItem(i18n("Custom query")); ui.bDatabaseManager->setIcon(QIcon::fromTheme("network-server-database")); ui.bDatabaseManager->setToolTip(i18n("Manage connections")); ui.twPreview->setEditTriggers(QAbstractItemView::NoEditTriggers); ui.cbNumberFormat->addItems(AbstractFileFilter::numberFormats()); ui.cbDateTimeFormat->addItems(AbstractColumn::dateTimeFormats()); #ifdef HAVE_KF5_SYNTAX_HIGHLIGHTING m_highlighter = new KSyntaxHighlighting::SyntaxHighlighter(ui.teQuery->document()); m_highlighter->setDefinition(m_repository.definitionForName("SQL")); m_highlighter->setTheme( (palette().color(QPalette::Base).lightness() < 128) ? m_repository.defaultTheme(KSyntaxHighlighting::Repository::DarkTheme) : m_repository.defaultTheme(KSyntaxHighlighting::Repository::LightTheme) ); #endif m_configPath = QStandardPaths::standardLocations(QStandardPaths::AppDataLocation).constFirst() + "sql_connections"; connect( ui.cbConnection, SIGNAL(currentIndexChanged(int)), SLOT(connectionChanged()) ); connect( ui.cbImportFrom, SIGNAL(currentIndexChanged(int)), SLOT(importFromChanged(int)) ); connect( ui.bDatabaseManager, SIGNAL(clicked()), this, SLOT(showDatabaseManager()) ); connect( ui.lwTables, SIGNAL(currentRowChanged(int)), this, SLOT(refreshPreview()) ); connect( ui.bRefreshPreview, SIGNAL(clicked()), this, SLOT(refreshPreview()) ); //defer the loading of settings a bit in order to show the dialog prior to blocking the GUI in refreshPreview() QTimer::singleShot( 100, this, SLOT(loadSettings()) ); } void ImportSQLDatabaseWidget::loadSettings() { m_initializing = true; //read available connections readConnections(); //load last used connection and other settings KConfigGroup config(KSharedConfig::openConfig(), "ImportSQLDatabaseWidget"); ui.cbConnection->setCurrentIndex(ui.cbConnection->findText(config.readEntry("Connection", ""))); ui.cbImportFrom->setCurrentIndex(config.readEntry("ImportFrom", 0)); importFromChanged(ui.cbImportFrom->currentIndex()); ui.cbNumberFormat->setCurrentIndex(config.readEntry("NumberFormat", (int)QLocale::AnyLanguage)); ui.cbDateTimeFormat->setCurrentItem(config.readEntry("DateTimeFormat", "yyyy-dd-MM hh:mm:ss:zzz")); QList defaultSizes; defaultSizes << 100 << 100; ui.splitterMain->setSizes(config.readEntry("SplitterMainSizes", defaultSizes)); ui.splitterPreview->setSizes(config.readEntry("SplitterPreviewSizes", defaultSizes)); //TODO m_initializing = false; //all settings loaded -> trigger the selection of the last used connection in order to get the data preview connectionChanged(); } ImportSQLDatabaseWidget::~ImportSQLDatabaseWidget() { // save current settings KConfigGroup config(KSharedConfig::openConfig(), "ImportSQLDatabaseWidget"); config.writeEntry("Connection", ui.cbConnection->currentText()); config.writeEntry("ImportFrom", ui.cbImportFrom->currentIndex()); config.writeEntry("NumberFormat", ui.cbNumberFormat->currentIndex()); config.writeEntry("DateTimeFormat", ui.cbDateTimeFormat->currentText()); config.writeEntry("SplitterMainSizes", ui.splitterMain->sizes()); config.writeEntry("SplitterPreviewSizes", ui.splitterPreview->sizes()); //TODO } /*! * in case the import from a table is selected, returns the currently selected database table. * returns empty string otherwise. */ QString ImportSQLDatabaseWidget::selectedTable() const { if (ui.cbImportFrom->currentIndex() == 0) { if (ui.lwTables->currentItem()) return ui.lwTables->currentItem()->text(); } return QString(); } /*! returns \c true if a working connections was selected and a table (or custom query) is provided and ready to be imported. returns \c false otherwise. */ bool ImportSQLDatabaseWidget::isValid() const { return m_valid; } /*! returns \c true if the selected table or the result of a custom query contains numeric data only. returns \c false otherwise. */ bool ImportSQLDatabaseWidget::isNumericData() const { return m_numeric; } /*! loads all available saved connections */ void ImportSQLDatabaseWidget::readConnections() { DEBUG("ImportSQLDatabaseWidget: reading available connections"); KConfig config(m_configPath, KConfig::SimpleConfig); for (const auto& name : config.groupList()) ui.cbConnection->addItem(name); } void ImportSQLDatabaseWidget::connectionChanged() { if (m_initializing) return; QDEBUG("ImportSQLDatabaseWidget: connecting to " + ui.cbConnection->currentText()); //clear the previously shown content ui.teQuery->clear(); ui.lwTables->clear(); ui.twPreview->clear(); ui.twPreview->setColumnCount(0); ui.twPreview->setRowCount(0); if (ui.cbConnection->currentIndex() == -1) return; //connection name was changed, determine the current connections settings KConfig config(m_configPath, KConfig::SimpleConfig); KConfigGroup group = config.group(ui.cbConnection->currentText()); //close and remove the previous connection, if available if (m_db.isOpen()) { m_db.close(); QSqlDatabase::removeDatabase(m_db.driverName()); } //open the selected connection const QString& driver = group.readEntry("Driver"); m_db = QSqlDatabase::addDatabase(driver); const QString& dbName = group.readEntry("DatabaseName"); if (DatabaseManagerWidget::isFileDB(driver)) { if (!QFile::exists(dbName)) { KMessageBox::error(this, i18n("Couldn't find the database file '%1'. Please check the connection settings.", dbName), i18n("Connection Failed")); setInvalid(); return; } else m_db.setDatabaseName(dbName); } else if (DatabaseManagerWidget::isODBC(driver)) { if (group.readEntry("CustomConnectionEnabled", false)) m_db.setDatabaseName(group.readEntry("CustomConnectionString")); else m_db.setDatabaseName(dbName); } else { m_db.setDatabaseName(dbName); m_db.setHostName( group.readEntry("HostName") ); m_db.setPort( group.readEntry("Port", 0) ); m_db.setUserName( group.readEntry("UserName") ); m_db.setPassword( group.readEntry("Password") ); } WAIT_CURSOR; if (!m_db.open()) { RESET_CURSOR; KMessageBox::error(this, i18n("Failed to connect to the database '%1'. Please check the connection settings.", ui.cbConnection->currentText()) + QLatin1String("\n\n") + m_db.lastError().databaseText(), i18n("Connection Failed")); setInvalid(); return; } //show all available database tables if (m_db.tables().size()) { ui.lwTables->addItems(m_db.tables()); ui.lwTables->setCurrentRow(0); for (int i = 0; i < ui.lwTables->count(); ++i) ui.lwTables->item(i)->setIcon(QIcon::fromTheme("view-form-table")); } else setInvalid(); RESET_CURSOR; } void ImportSQLDatabaseWidget::refreshPreview() { if (!ui.lwTables->currentItem()) { setInvalid(); return; } WAIT_CURSOR; ui.twPreview->clear(); //execute the current query (select on a table or a custom query) const QString& query = currentQuery(true); if (query.isEmpty()) { RESET_CURSOR; setInvalid(); return; } QSqlQuery q; q.prepare(currentQuery(true)); q.setForwardOnly(true); q.exec(); if (!q.isActive() || !q.next()) { // check if query was successful and got to first record RESET_CURSOR; if (!q.lastError().databaseText().isEmpty()) KMessageBox::error(this, q.lastError().databaseText(), i18n("Unable to Execute Query")); setInvalid(); return; } //resize the table to the number of columns (=number of fields in the result set) m_cols = q.record().count(); ui.twPreview->setColumnCount(m_cols); //determine the names and the data type (column modes) of the table columns. //check whether we have numerical data only by checking the data types of the first record. m_columnNames.clear(); m_columnModes.clear(); bool numeric = true; const auto numberFormat = (QLocale::Language)ui.cbNumberFormat->currentIndex(); const QString& dateTimeFormat = ui.cbDateTimeFormat->currentText(); // ui.twPreview->setRowCount(1); //add the first row for the check boxes for (int i = 0; i < m_cols; ++i) { //name m_columnNames << q.record().fieldName(i); //value and type const QString valueString = q.record().value(i).toString(); AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); m_columnModes << mode; if (mode != AbstractColumn::Numeric) numeric = false; //header item QTableWidgetItem* item = new QTableWidgetItem(m_columnNames[i] + QLatin1String(" {") + ENUM_TO_STRING(AbstractColumn, ColumnMode, mode) + QLatin1String("}")); item->setTextAlignment(Qt::AlignLeft); item->setIcon(AbstractColumn::iconForMode(mode)); ui.twPreview->setHorizontalHeaderItem(i, item); //create checked items // QTableWidgetItem* itemChecked = new QTableWidgetItem(); // itemChecked->setCheckState(Qt::Checked); // ui.twPreview->setItem(0, i, itemChecked); } //preview the data const bool customQuery = (ui.cbImportFrom->currentIndex() != 0); int row = 0; do { for (int col = 0; col < m_cols; ++col) { ui.twPreview->setRowCount(row+1); ui.twPreview->setItem(row, col, new QTableWidgetItem(q.value(col).toString()) ); } row++; //in case a custom query is executed, check whether the row number limit is reached if (customQuery && row >= ui.sbPreviewLines->value()) break; } while (q.next()); ui.twPreview->horizontalHeader()->resizeSections(QHeaderView::ResizeToContents); setValid(); if (numeric != m_numeric) { m_numeric = numeric; emit stateChanged(); } RESET_CURSOR; } void ImportSQLDatabaseWidget::importFromChanged(int index) { if (index == 0) { //import from a table ui.gbQuery->hide(); ui.lwTables->show(); } else { //import the result set of a custom query ui.gbQuery->show(); ui.lwTables->hide(); ui.twPreview->clear(); } refreshPreview(); } void ImportSQLDatabaseWidget::read(AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { if (!dataSource) return; WAIT_CURSOR; //execute the current query (select on a table or a custom query) QSqlQuery q; // q.setForwardOnly(true); //TODO: crashes most probably because of q.last() and q.first() below q.prepare(currentQuery()); if (!q.exec() || !q.isActive()) { RESET_CURSOR; if (!q.lastError().databaseText().isEmpty()) KMessageBox::error(this, q.lastError().databaseText(), i18n("Unable to Execute Query")); setInvalid(); return; } //determine the number of rows/records to read q.last(); const int rows = q.at()+1; q.first(); // pointers to the actual data containers //columnOffset indexes the "start column" in the datasource. Data will be imported starting from this column. - QVector dataContainer; + std::vector dataContainer; int columnOffset = dataSource->prepareImport(dataContainer, importMode, rows, m_cols, m_columnNames, m_columnModes); //number and DateTime formatting const QString& dateTimeFormat = ui.cbDateTimeFormat->currentText(); const QLocale numberFormat = QLocale((QLocale::Language)ui.cbNumberFormat->currentIndex()); //read the data int row = 0; do { for (int col = 0; col < m_cols; ++col) { const QString valueString = q.record().value(col).toString(); // set value depending on data type switch (m_columnModes[col]) { case AbstractColumn::Numeric: { bool isNumber; const double value = numberFormat.toDouble(valueString, &isNumber); static_cast*>(dataContainer[col])->operator[](row) = (isNumber ? value : NAN); break; } case AbstractColumn::Integer: { bool isNumber; const int value = numberFormat.toInt(valueString, &isNumber); static_cast*>(dataContainer[col])->operator[](row) = (isNumber ? value : NAN); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(dataContainer[col])->operator[](row) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(dataContainer[col])->operator[](row) = valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } row++; emit completed(100 * row/rows); } while (q.next()); DEBUG(" Read " << row << " rows"); dataSource->finalizeImport(columnOffset, 1, m_cols, dateTimeFormat, importMode); RESET_CURSOR; } QString ImportSQLDatabaseWidget::currentQuery(bool preview) { QString query; const bool customQuery = (ui.cbImportFrom->currentIndex() != 0); if ( !customQuery ) { const QString& tableName = ui.lwTables->currentItem()->text(); if (!preview) { query = QLatin1String("SELECT * FROM ") + tableName; } else { //preview the content of the currently selected table const QString& driver = m_db.driverName(); const QString& limit = QString::number(ui.sbPreviewLines->value()); if ( (driver == QLatin1String("QSQLITE3")) || (driver == QLatin1String("QSQLITE")) || (driver == QLatin1String("QMYSQL3")) || (driver == QLatin1String("QMYSQL")) || (driver == QLatin1String("QPSQL")) ) query = QLatin1String("SELECT * FROM ") + tableName + QLatin1String(" LIMIT ") + limit; else if (driver == QLatin1String("QOCI")) query = QLatin1String("SELECT * FROM ") + tableName + QLatin1String(" ROWNUM<=") + limit; else if (driver == QLatin1String("QDB2")) query = QLatin1String("SELECT * FROM ") + tableName + QLatin1String(" FETCH FIRST ") + limit + QLatin1String(" ROWS ONLY"); else if (driver == QLatin1String("QIBASE")) query = QLatin1String("SELECT * FROM ") + tableName + QLatin1String(" ROWS ") + limit; else //for ODBC the DBMS is not known and it's not clear what syntax to use -> select all rows query = QLatin1String("SELECT * FROM ") + tableName; } } else { //preview the result of a custom query query = ui.teQuery->toPlainText().simplified(); } return query; } /*! shows the database manager where the connections are created and edited. The selected connection is selected in the connection combo box in this widget. **/ void ImportSQLDatabaseWidget::showDatabaseManager() { DatabaseManagerDialog* dlg = new DatabaseManagerDialog(this, ui.cbConnection->currentText()); if (dlg->exec() == QDialog::Accepted) { //re-read the available connections to be in sync with the changes in DatabaseManager m_initializing = true; ui.cbConnection->clear(); readConnections(); //select the connection the user has selected in DatabaseManager const QString& conn = dlg->connection(); ui.cbConnection->setCurrentIndex(ui.cbConnection->findText(conn)); m_initializing = false; connectionChanged(); } delete dlg; } void ImportSQLDatabaseWidget::setInvalid() { if (m_valid) { ui.twPreview->setColumnCount(0); ui.twPreview->setRowCount(0); m_valid = false; emit stateChanged(); } } void ImportSQLDatabaseWidget::setValid() { if (!m_valid) { m_valid = true; emit stateChanged(); } }