diff --git a/src/backend/datasources/AbstractDataSource.h b/src/backend/datasources/AbstractDataSource.h index 606948f86..5032be7b7 100644 --- a/src/backend/datasources/AbstractDataSource.h +++ b/src/backend/datasources/AbstractDataSource.h @@ -1,53 +1,53 @@ /*************************************************************************** File : AbstractDataSource.h Project : LabPlot Description : Interface for data sources -------------------------------------------------------------------- Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2015 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ABSTRACTDATASOURCE_H #define ABSTRACTDATASOURCE_H #include "backend/core/AbstractPart.h" #include "backend/core/AbstractScriptingEngine.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" #include class QStringList; class AbstractDataSource : public AbstractPart, public scripted { Q_OBJECT public: AbstractDataSource(AbstractScriptingEngine *engine, const QString& name); ~AbstractDataSource() override {} void clear(); virtual int prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode, int actualRows, int actualCols, QStringList colNameList = QStringList(), QVector = QVector()) = 0; - virtual void finalizeImport(int columnOffset = 0, int startColumn = 0, int endColumn = 0, + virtual void finalizeImport(int columnOffset = 0, int startColumn = 0, int endColumn = 0, int numRows = 0, const QString& dateTimeFormat = QString(), AbstractFileFilter::ImportMode importMode = AbstractFileFilter::Replace) = 0; }; #endif // ABSTRACTDATASOURCE_H diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index a09011dc8..bffc1e740 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,1637 +1,1637 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2018 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include "backend/core/Project.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" #include "backend/lib/macros.h" #include "backend/lib/trace.h" #include #include #include #include #include /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() {} /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource* dataSource) { d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { return d->readFromLiveDevice(device, dataSource, from); } /*! reads the content of the file \c fileName. */ void AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } QVector AsciiFilter::preview(QIODevice &device) { return d->preview(device); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE" << "2xSPACE" << "3xSPACE" << "4xSPACE" << "2xTAB"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } QString AsciiFilter::fileInfoString(const QString& fileName) { QString info(i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName))); info += QLatin1String("
"); info += i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); return info; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " to determine number of lines"); return 0; } if (!device.canReadLine()) return -1; size_t lineCount = 0; while (!device.atEnd()) { device.readLine(); lineCount++; } //TODO: wc is much faster but not portable /* QElapsedTimer myTimer; myTimer.start(); QProcess wc; wc.start(QString("wc"), QStringList() << "-l" << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) lineCount = wc.readLine().split(' ')[0].toInt(); lineCount++; // last line not counted DEBUG(" Elapsed time counting lines : " << myTimer.elapsed() << " ms"); */ return lineCount; } /*! returns the number of lines in the device \c device and 0 if sequential. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice &device) { if (device.isSequential()) return 0; if (!device.canReadLine()) DEBUG("WARNING in AsciiFilter::lineNumber(): device cannot 'readLine()' but using it anyway."); size_t lineCount = 0; device.seek(0); while (!device.atEnd()) { device.readLine(); lineCount++; } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool AsciiFilter::NaNValueToZeroEnabled() const { if (d->nanValue == 0) return true; return false; } void AsciiFilter::setRemoveQuotesEnabled(bool b) { d->removeQuotesEnabled = b; } bool AsciiFilter::removeQuotesEnabled() const { return d->removeQuotesEnabled; } void AsciiFilter::setVectorNames(const QString& s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner), commentCharacter("#"), separatingCharacter("auto"), numberFormat(QLocale::C), autoModeEnabled(true), headerEnabled(true), skipEmptyParts(false), simplifyWhitespacesEnabled(true), nanValue(NAN), removeQuotesEnabled(false), createIndexEnabled(false), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_actualStartRow(1), m_actualRows(0), m_actualCols(0), m_prepared(false), m_columnOffset(0) { } /*! * get a single line from device */ QStringList AsciiFilterPrivate::getLineString(QIODevice& device) { QString line; do { // skip comment lines in data lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::getLineString(): device cannot 'readLine()' but using it anyway."); // line = device.readAll(); line = device.readLine(); } while (line.startsWith(commentCharacter)); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); DEBUG("data line : \'" << line.toStdString() << '\''); QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); //TODO: remove quotes here? QDEBUG("data line, parsed: " << lineStringList); return lineStringList; } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("AsciiFilterPrivate::prepareDeviceToRead(): is sequential = " << device.isSequential() << ", can readLine = " << device.canReadLine()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; ///////////////////////////////////////////////////////////////// // Find first data line (ignoring comment lines) DEBUG(" Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { QString line; if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); line = device.readLine(); DEBUG(" line = " << line.toStdString()); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } } // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); firstLine = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive); m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); QDEBUG("first line: " << firstLineStringList); DEBUG("headerEnabled: " << headerEnabled); //optionally, remove potential spaces in the first line if (simplifyWhitespacesEnabled) { for (int i = 0; i < firstLineStringList.size(); ++i) firstLineStringList[i] = firstLineStringList[i].simplified(); } if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); m_actualStartRow = startRow + 1; } else m_actualStartRow = startRow; // set range to read if (endColumn == -1) { if (headerEnabled || vectorNames.size() == 0) endColumn = firstLineStringList.size(); // last column else //number of vector names provided in the import dialog (not more than the maximal number of columns in the file) endColumn = qMin(vectorNames.size(), firstLineStringList.size()); } if (createIndexEnabled) { vectorNames.prepend(i18n("Index")); endColumn++; } m_actualCols = endColumn - startColumn + 1; //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ ///////////////////////////////////////////////////////////////// // parse first data line to determine data type for each column if (!device.isSequential()) firstLineStringList = getLineString(device); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Integer; col = 1; } for (auto& valueString: firstLineStringList) { // parse columns available in first data line if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } // parsing more lines to better determine data types for (unsigned int i = 0; i < m_dataTypeLines; ++i) { if (device.atEnd()) // EOF reached break; firstLineStringList = getLineString(device); if (createIndexEnabled) col = 1; else col = 0; for (auto& valueString: firstLineStringList) { if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (col == m_actualCols) break; AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); // numeric: integer -> numeric if (mode == AbstractColumn::Numeric && columnModes[col] == AbstractColumn::Integer) columnModes[col] = mode; // text: non text -> text if (mode == AbstractColumn::Text && columnModes[col] != AbstractColumn::Text) columnModes[col] = mode; col++; } } QDEBUG("column modes = " << columnModes); // ATTENTION: This resets the position in the device to 0 m_actualRows = (int)AsciiFilter::lineNumber(device); const int actualEndRow = (endRow == -1 || endRow > m_actualRows) ? m_actualRows : endRow; m_actualRows = actualEndRow - m_actualStartRow + 1; DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << m_actualStartRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0 && !device.isSequential()) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode)); KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode); } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { DEBUG("AsciiFilterPrivate::readFromLiveDevice(): bytes available = " << device.bytesAvailable() << ", from = " << from); if (!(device.bytesAvailable() > 0)) { DEBUG(" No new data available"); return 0; } LiveDataSource* spreadsheet = dynamic_cast(dataSource); if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe) if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return 0; if (!m_prepared) { DEBUG("Preparing .."); switch (spreadsheet->sourceType()) { case LiveDataSource::SourceType::FileOrPipe: { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return 0; } break; } case LiveDataSource::SourceType::NetworkTcpSocket: case LiveDataSource::SourceType::NetworkUdpSocket: case LiveDataSource::SourceType::LocalSocket: case LiveDataSource::SourceType::SerialPort: m_actualRows = 1; if (createIndexEnabled) { m_actualCols = 2; columnModes << AbstractColumn::Integer << AbstractColumn::Numeric; vectorNames << i18n("Index") << i18n("Value"); } else { m_actualCols = 1; columnModes << AbstractColumn::Numeric; vectorNames << i18n("Value"); } QDEBUG(" vector names = " << vectorNames); } // prepare import for spreadsheet spreadsheet->setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); DEBUG(" data source resized to col: " << m_actualCols); DEBUG(" data source rowCount: " << spreadsheet->rowCount()); //columns in a file data source don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } int keepNValues = spreadsheet->keepNValues(); if (keepNValues == 0) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(keepNValues); m_actualRows = keepNValues; } m_dataContainer.resize(m_actualCols); DEBUG(" Setting data .."); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } DEBUG("Prepared!"); } qint64 bytesread = 0; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif LiveDataSource::ReadingType readingType; if (!m_prepared) { readingType = LiveDataSource::ReadingType::TillEnd; } else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = LiveDataSource::ReadingType::TillEnd; //if we read the whole file we just start from the beginning of it //and read till end else if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) readingType = LiveDataSource::ReadingType::TillEnd; else readingType = spreadsheet->readingType(); } DEBUG(" reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType)); //move to the last read position, from == total bytes read //since the other source types are sequencial we cannot seek on them if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); DEBUG(" bytes available = " << device.bytesAvailable()); //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleSizeNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != LiveDataSource::ReadingType::TillEnd) newData.resize(spreadsheet->sampleSize()); int newDataIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif while (!device.atEnd()) { DEBUG(" reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType)); DEBUG(" source type = " << ENUM_TO_STRING(LiveDataSource, SourceType, spreadsheet->sourceType())); if (readingType != LiveDataSource::ReadingType::TillEnd) { switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData[newDataIdx++] = device.readAll(); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData[newDataIdx++] = device.read(device.bytesAvailable()); break; case LiveDataSource::SourceType::FileOrPipe: case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::readFromLiveDevice(): device cannot 'readLine()' but using it anyway."); newData[newDataIdx++] = device.readLine(); } } else { // ReadingType::TillEnd switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData.push_back(device.readAll()); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData.push_back(device.read(device.bytesAvailable())); break; case LiveDataSource::SourceType::FileOrPipe: case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::readFromLiveDevice(): device cannot 'readLine()' but using it anyway."); newData.push_back(device.readLine()); } } newLinesTillEnd++; if (readingType != LiveDataSource::ReadingType::TillEnd) { newLinesForSampleSizeNotTillEnd++; //for Continuous reading and FromEnd we read sample rate number of lines if possible //here TillEnd and Whole file behave the same if (newLinesForSampleSizeNotTillEnd == spreadsheet->sampleSize()) break; } } QDEBUG(" data read: " << newData); } //now we reset the readingType if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = spreadsheet->readingType(); //we had less new lines than the sample size specified if (readingType != LiveDataSource::ReadingType::TillEnd) QDEBUG("Removed empty lines: " << newData.removeAll("")); //back to the last read position before counting when reading from files if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); int currentRow = 0; // indexes the position in the vector(column) int linesToRead = 0; int keepNValues = spreadsheet->keepNValues(); DEBUG("Increase row count"); if (m_prepared) { //increase row count if we don't have a fixed size //but only after the preparation step if (keepNValues == 0) { if (readingType != LiveDataSource::ReadingType::TillEnd) m_actualRows += qMin(newData.size(), spreadsheet->sampleSize()); else { //we don't increase it if we reread the whole file, we reset it if (!(spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)) m_actualRows += newData.size(); else m_actualRows = newData.size(); } //appending if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) linesToRead = m_actualRows; else linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; } else { // fixed size if (readingType == LiveDataSource::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; //TODO after reading we should skip the next data lines //because it's TillEnd actually } else linesToRead = newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinuouslyFixed or FromEnd, WholeFile is disabled linesToRead = qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } DEBUG(" actual rows = " << m_actualRows); if (linesToRead == 0) return 0; } else { linesToRead = newLinesTillEnd; if (headerEnabled) --m_actualRows; } DEBUG(" lines to read = " << linesToRead); //TODO: check other source types if (spreadsheet->sourceType() == LiveDataSource::SourceType::NetworkUdpSocket) { if (m_actualRows < linesToRead) { DEBUG(" SET actual rows to " << linesToRead); m_actualRows = linesToRead; } } //new rows/resize columns if we don't have a fixed size //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize if (keepNValues == 0) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { //when we have a fixed size we have to pop sampleSize number of lines if specified //here popping, setting currentRow if (!m_prepared) { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); } else { if (readingType == LiveDataSource::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) { currentRow = 0; } else { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } } else { //we read max sample rate number of lines when the reading mode //is ContinuouslyFixed or FromEnd currentRow = m_actualRows - qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet qDebug() << "reading from line" << currentRow << " till end" << newLinesTillEnd; qDebug() << "Lines to read:" << linesToRead <<", actual rows:" << m_actualRows << ", actual cols:" << m_actualCols; newDataIdx = 0; if (readingType == LiveDataSource::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->sampleSize()) newDataIdx = newData.size() - spreadsheet->sampleSize(); //since we skip a couple of lines, we need to count those bytes too for (int i = 0; i < newDataIdx; ++i) bytesread += newData.at(i).size(); } } qDebug() << "newDataIdx: " << newDataIdx; //TODO static int indexColumnIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif int row = 0; if (readingType == LiveDataSource::ReadingType::TillEnd || (readingType == LiveDataSource::ReadingType::ContinuousFixed)) { if (headerEnabled) { if (!m_prepared) { row = 1; bytesread += newData.at(0).size(); } } } if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { if (readingType == LiveDataSource::ReadingType::WholeFile) { if (headerEnabled) { row = 1; bytesread += newData.at(0).size(); } } } for (; row < linesToRead; ++row) { DEBUG(" row = " << row); QString line; if (readingType == LiveDataSource::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(row); //when we read the whole file we don't care about the previous position //so we don't have to count those bytes if (readingType != LiveDataSource::ReadingType::WholeFile) { if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { bytesread += line.size(); } } //qDebug() << "line bytes: " << line.size() << " line: " << line; if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList; // only FileOrPipe support multiple columns if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); else lineStringList << line; QDEBUG(" line = " << lineStringList << ", separator = \'" << m_separator << "\'"); if (createIndexEnabled) { if (spreadsheet->keepNValues() == 0) lineStringList.prepend(QString::number(currentRow)); else lineStringList.prepend(QString::number(indexColumnIdx++)); } QDEBUG(" column modes = " << columnModes); for (int n = 0; n < m_actualCols; ++n) { DEBUG(" actual col = " << n); if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); DEBUG(" value string = " << valueString.toStdString()); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { DEBUG(" Numeric"); bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::Integer: { DEBUG(" Integer"); bool isNumber; const int value = locale.toInt(valueString, &isNumber); DEBUG(" container size = " << m_dataContainer.size() << ", current row = " << currentRow); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { DEBUG(" missing columns in this line"); switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); const Project* project = spreadsheet->project(); QVector curves = project->children(AbstractAspect::Recursive); QVector plots; for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); //determine the plots where the column is consumed for (const auto* curve: curves) { if (curve->xColumn() == column || curve->yColumn() == column) { CartesianPlot* plot = dynamic_cast(curve->parentAspect()); if (plots.indexOf(plot) == -1) { plots << plot; plot->setSuppressDataChangedSignal(true); } } } column->setChanged(); } //loop over all affected plots and retransform them for (auto* plot: plots) { plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } m_prepared = true; return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return; } // matrix data has only one column mode (which is not text) if (dynamic_cast(dataSource)) { auto mode = columnModes[0]; if (mode == AbstractColumn::Text) mode = AbstractColumn::Numeric; for (auto& c: columnModes) if (c != mode) c = mode; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; //skip data lines, if required DEBUG(" Skipping " << m_actualStartRow - 1 << " lines"); for (int i = 0; i < m_actualStartRow - 1; ++i) device.readLine(); DEBUG(" Reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); ++i) { QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; - QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); //prepend the index if required //TODO: come up maybe with a solution with adding the index inside of the loop below, //without conversion to string, prepending to the list and then conversion back to integer. if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); // remove left white spaces if (skipEmptyParts) { for (int n = 0; n < lineStringList.size(); ++n) { QString valueString = lineStringList.at(n); if (!QString::compare(valueString, " ")) { lineStringList.removeAt(n); n--; } } } for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } + DEBUG(" Read " << currentRow << " lines"); - dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, dateTimeFormat, importMode); + dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, currentRow, dateTimeFormat, importMode); } /*! * preview for special devices (local/UDP/TCP socket or serial port) */ QVector AsciiFilterPrivate::preview(QIODevice &device) { DEBUG("AsciiFilterPrivate::preview(): bytesAvailable = " << device.bytesAvailable() << ", isSequential = " << device.isSequential()); QVector dataStrings; if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return dataStrings; } if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return dataStrings; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif int linesToRead = 0; QVector newData; while (!device.atEnd()) { if (device.canReadLine()) newData.push_back(device.readLine()); else // UDP fails otherwise newData.push_back(device.readAll()); linesToRead++; } QDEBUG(" data = " << newData); if (linesToRead == 0) return dataStrings; int col = 0; int colMax = newData.at(0).size(); if (createIndexEnabled) colMax++; columnModes.resize(colMax); if (createIndexEnabled) { columnModes[0] = AbstractColumn::ColumnMode::Integer; col = 1; vectorNames.prepend(i18n("Index")); } vectorNames.append(i18n("Value")); QDEBUG(" vector names = " << vectorNames); for (const auto& valueString: newData.at(0).split(' ', QString::SkipEmptyParts)) { if (col == colMax) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } for (int i = 0; i < linesToRead; ++i) { QString line = newData.at(i); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(' ', QString::SkipEmptyParts); if (createIndexEnabled) lineStringList.prepend(QString::number(i)); QStringList lineString; for (int n = 0; n < lineStringList.size(); ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 16); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; // set column names for preview if (!headerEnabled) { int start = 0; if (createIndexEnabled) start = 1; for (int i=start;iwriteStartElement( "asciiFilter"); writer->writeAttribute( "commentCharacter", d->commentCharacter); writer->writeAttribute( "separatingCharacter", d->separatingCharacter); writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); writer->writeAttribute( "nanValue", QString::number(d->nanValue)); writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled)); writer->writeAttribute( "startRow", QString::number(d->startRow)); writer->writeAttribute( "endRow", QString::number(d->endRow)); writer->writeAttribute( "startColumn", QString::number(d->startColumn)); writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("commentCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("commentCharacter").toString()); else d->commentCharacter = str; str = attribs.value("separatingCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("separatingCharacter").toString()); else d->separatingCharacter = str; str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("createIndex").toString()); else d->createIndexEnabled = str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("autoMode").toString()); else d->autoModeEnabled = str.toInt(); str = attribs.value("header").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("header").toString()); else d->headerEnabled = str.toInt(); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty str = attribs.value("simplifyWhitespaces").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("simplifyWhitespaces").toString()); else d->simplifyWhitespacesEnabled = str.toInt(); str = attribs.value("nanValue").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("nanValue").toString()); else d->nanValue = str.toDouble(); str = attribs.value("removeQuotes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("removeQuotes").toString()); else d->removeQuotesEnabled = str.toInt(); str = attribs.value("skipEmptyParts").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipEmptyParts").toString()); else d->skipEmptyParts = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startRow").toString()); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endRow").toString()); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startColumn").toString()); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endColumn").toString()); else d->endColumn = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/BinaryFilter.cpp b/src/backend/datasources/filters/BinaryFilter.cpp index fd857b24b..b17e7c107 100644 --- a/src/backend/datasources/filters/BinaryFilter.cpp +++ b/src/backend/datasources/filters/BinaryFilter.cpp @@ -1,616 +1,616 @@ /*************************************************************************** File : BinaryFilter.cpp Project : LabPlot Description : Binary I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/BinaryFilter.h" #include "backend/datasources/filters/BinaryFilterPrivate.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include /*! \class BinaryFilter \brief Manages the import/export of data organized as columns (vectors) from/to a binary file. \ingroup datasources */ BinaryFilter::BinaryFilter():AbstractFileFilter(), d(new BinaryFilterPrivate(this)) {} BinaryFilter::~BinaryFilter() {} /*! reads the content of the file \c fileName. */ void BinaryFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } /*! reads the content of the device \c device. */ void BinaryFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } QVector BinaryFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void BinaryFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! returns the list of all predefined data formats. */ QStringList BinaryFilter::dataTypes() { return (QStringList()<<"int8 (8 bit signed integer)"<<"int16 (16 bit signed integer)"<<"int32 (32 bit signed integer)"<<"int64 (64 bit signed integer)" <<"uint8 (8 bit unsigned integer)"<<"uint16 (16 bit unsigned integer)"<<"uint32 (32 bit unsigned integer)"<<"uint64 (64 bit unsigned integer)" <<"real32 (single precision floats)"<<"real64 (double precision floats)"); } /*! returns the list of all predefined byte order. */ QStringList BinaryFilter::byteOrders() { return (QStringList() << "Little endian" << "Big endian"); } /*! returns the size of the predefined data types */ int BinaryFilter::dataSize(BinaryFilter::DataType type) { int sizes[] = {1,2,4,8,1,2,4,8,4,8}; return sizes[(int)type]; } /*! returns the number of rows (length of vectors) in the file \c fileName. */ size_t BinaryFilter::rowNumber(const QString& fileName, const size_t vectors, const BinaryFilter::DataType type) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) return 0; size_t rows = 0; while (!device.atEnd()) { // one row for (size_t i = 0; i < vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(type); ++j) device.read(1); } rows++; } return rows; } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void BinaryFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void BinaryFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void BinaryFilter::setVectors(const size_t v) { d->vectors = v; } size_t BinaryFilter::vectors() const { return d->vectors; } void BinaryFilter::setDataType(const BinaryFilter::DataType t) { d->dataType = t; } BinaryFilter::DataType BinaryFilter::dataType() const { return d->dataType; } void BinaryFilter::setByteOrder(const BinaryFilter::ByteOrder b) { d->byteOrder = b; } BinaryFilter::ByteOrder BinaryFilter::byteOrder() const { return d->byteOrder; } void BinaryFilter::setSkipStartBytes(const size_t s) { d->skipStartBytes = s; } size_t BinaryFilter::skipStartBytes() const { return d->skipStartBytes; } void BinaryFilter::setStartRow(const int s) { d->startRow = s; } int BinaryFilter::startRow() const { return d->startRow; } void BinaryFilter::setEndRow(const int e) { d->endRow = e; } int BinaryFilter::endRow() const { return d->endRow; } void BinaryFilter::setSkipBytes(const size_t s) { d->skipBytes = s; } size_t BinaryFilter::skipBytes() const { return d->skipBytes; } void BinaryFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void BinaryFilter::setAutoModeEnabled(bool b) { d->autoModeEnabled = b; } bool BinaryFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } //##################################################################### //################### Private implementation ########################## //##################################################################### BinaryFilterPrivate::BinaryFilterPrivate(BinaryFilter* owner) : q(owner), vectors(2), dataType(BinaryFilter::INT8), byteOrder(BinaryFilter::LittleEndian), startRow(1), endRow(-1), numRows(0), skipStartBytes(0), skipBytes(0), createIndexEnabled(false), autoModeEnabled(true), m_actualRows(0), m_actualCols(0) { } /*! reads the content of the device \c device to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ void BinaryFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("readDataFromFile()"); KFilterDev device(fileName); numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); if (! device.open(QIODevice::ReadOnly)) { DEBUG(" could not open file " << fileName.toStdString()); return; } readDataFromDevice(device, dataSource, importMode); } /*! * returns 1 if the current read position in the device is at the end and 0 otherwise. */ int BinaryFilterPrivate::prepareStreamToRead(QDataStream& in) { DEBUG("prepareStreamToRead()"); if (byteOrder == BinaryFilter::BigEndian) in.setByteOrder(QDataStream::BigEndian); else if (byteOrder == BinaryFilter::LittleEndian) in.setByteOrder(QDataStream::LittleEndian); // catch case that skipStartBytes or startRow is bigger than file if (skipStartBytes >= BinaryFilter::dataSize(dataType) * vectors * numRows || startRow > (int)numRows) return 1; // skip bytes at start for (size_t i = 0; i < skipStartBytes; ++i) { qint8 tmp; in >> tmp; } // skip until start row for (size_t i = 0; i < (startRow-1) * vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(dataType); ++j) { qint8 tmp; in >> tmp; } } // set range of rows if (endRow == -1) m_actualRows = (int)numRows - startRow + 1; else if (endRow > (int)numRows - startRow + 1) m_actualRows = (int)numRows; else m_actualRows = endRow - startRow + 1; m_actualCols = (int)vectors; DEBUG("numRows = " << numRows); DEBUG("endRow = " << endRow); DEBUG("actual rows = " << m_actualRows); DEBUG("actual cols = " << m_actualCols); return 0; } /*! reads \c lines lines of the device \c device and return as string for preview. */ QVector BinaryFilterPrivate::preview(const QString& fileName, int lines) { DEBUG("BinaryFilterPrivate::preview( " << fileName.toStdString() << ", " << lines << ")"); QVector dataStrings; KFilterDev device(fileName); if (! device.open(QIODevice::ReadOnly)) return dataStrings << (QStringList() << i18n("could not open device")); numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); QDataStream in(&device); const int deviceError = prepareStreamToRead(in); if(deviceError) return dataStrings << (QStringList() << i18n("data selection empty")); //TODO: support other modes columnModes.resize(m_actualCols); //TODO: use given names QStringList vectorNames; if (createIndexEnabled) vectorNames.prepend(i18n("Index")); if (lines == -1) lines = m_actualRows; // read data //TODO: use ColumnMode ? DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(m_actualRows, lines); ++i) { QStringList lineString; //prepend the index if required if (createIndexEnabled) lineString << QString::number(i+1); for (int n = 0; n < m_actualCols; ++n) { switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT16: { qint16 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT32: { qint32 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT64: { qint64 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT8: { quint8 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT16: { quint16 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT32: { quint32 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT64: { quint64 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::REAL32: { float value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::REAL64: { double value; in >> value; lineString << QString::number(value); break; } } } dataStrings << lineString; emit q->completed(100*i/m_actualRows); } return dataStrings; } /*! reads the content of the file \c fileName to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ void BinaryFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("BinaryFilterPrivate::readDataFromDevice()"); QDataStream in(&device); const int deviceError = prepareStreamToRead(in); if (deviceError) { dataSource->clear(); DEBUG("device error"); return; } if (createIndexEnabled) m_actualCols++; QVector dataContainer; int columnOffset = 0; //TODO: support other modes columnModes.resize(m_actualCols); //TODO: use given names QStringList vectorNames; if (createIndexEnabled) { vectorNames.prepend(i18n("Index")); columnModes[0] = AbstractColumn::Integer; } columnOffset = dataSource->prepareImport(dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); if (lines == -1) lines = m_actualRows; // start column int startColumn = 0; if (createIndexEnabled) startColumn++; // read data //TODO: use ColumnMode ? DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(m_actualRows, lines); ++i) { DEBUG("reading row " << i); //prepend the index if required if (createIndexEnabled) static_cast*>(dataContainer[0])->operator[](i) = i+1; for (int n = startColumn; n < m_actualCols; ++n) { DEBUG("reading column " << n); switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT16: { qint16 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT32: { qint32 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT64: { qint64 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT8: { quint8 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT16: { quint16 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT32: { quint32 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT64: { quint64 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::REAL32: { float value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::REAL64: { double value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } } } if (m_actualRows > 0) emit q->completed(100*i/m_actualRows); } - dataSource->finalizeImport(columnOffset, 1, m_actualCols, "", importMode); + dataSource->finalizeImport(columnOffset, 1, m_actualCols, m_actualRows, "", importMode); } /*! writes the content of \c dataSource to the file \c fileName. */ void BinaryFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void BinaryFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("binaryFilter"); writer->writeAttribute("vectors", QString::number(d->vectors) ); writer->writeAttribute("dataType", QString::number(d->dataType) ); writer->writeAttribute("byteOrder", QString::number(d->byteOrder) ); writer->writeAttribute("autoMode", QString::number(d->autoModeEnabled) ); writer->writeAttribute("startRow", QString::number(d->startRow) ); writer->writeAttribute("endRow", QString::number(d->endRow) ); writer->writeAttribute("skipStartBytes", QString::number(d->skipStartBytes) ); writer->writeAttribute("skipBytes", QString::number(d->skipBytes) ); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled) ); writer->writeEndElement(); } /*! Loads from XML. */ bool BinaryFilter::load(XmlStreamReader* reader) { KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); // read attributes QString str = attribs.value("vectors").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("vectors").toString()); else d->vectors = (size_t)str.toULong(); str = attribs.value("dataType").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("dataType").toString()); else d->dataType = (BinaryFilter::DataType) str.toInt(); str = attribs.value("byteOrder").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("byteOrder").toString()); else d->byteOrder = (BinaryFilter::ByteOrder) str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("autoMode").toString()); else d->autoModeEnabled = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startRow").toString()); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endRow").toString()); else d->endRow = str.toInt(); str = attribs.value("skipStartBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipStartBytes").toString()); else d->skipStartBytes = (size_t)str.toULong(); str = attribs.value("skipBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipBytes").toString()); else d->skipBytes = (size_t)str.toULong(); str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("createIndex").toString()); else d->createIndexEnabled = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/FITSFilter.cpp b/src/backend/datasources/filters/FITSFilter.cpp index f9bc55d6a..a2ab4f4a4 100644 --- a/src/backend/datasources/filters/FITSFilter.cpp +++ b/src/backend/datasources/filters/FITSFilter.cpp @@ -1,1637 +1,1637 @@ /*************************************************************************** File : FITSFilter.cpp Project : LabPlot Description : FITS I/O-filter -------------------------------------------------------------------- Copyright : (C) 2016 by Fabian Kristof (fkristofszabolcs@gmail.com) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "FITSFilter.h" #include "FITSFilterPrivate.h" #include "backend/core/column/Column.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/datatypes/Double2StringFilter.h" #include "backend/matrix/MatrixModel.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/matrix/Matrix.h" #include "commonfrontend/matrix/MatrixView.h" #include #include #include /*! \class FITSFilter * \brief Manages the import/export of data from/to a FITS file. * \since 2.2.0 * \ingroup datasources */ FITSFilter::FITSFilter():AbstractFileFilter(), d(new FITSFilterPrivate(this)) {} FITSFilter::~FITSFilter() {} void FITSFilter::readDataFromFile(const QString &fileName, AbstractDataSource *dataSource, AbstractFileFilter::ImportMode importMode) { d->readCHDU(fileName, dataSource, importMode); } QVector FITSFilter::readChdu(const QString &fileName, bool* okToMatrix, int lines) { return d->readCHDU(fileName, NULL, AbstractFileFilter::Replace, okToMatrix, lines); } void FITSFilter::write(const QString &fileName, AbstractDataSource *dataSource) { d->writeCHDU(fileName, dataSource); } void FITSFilter::addNewKeyword(const QString &filename, const QList &keywords) { d->addNewKeyword(filename, keywords); } void FITSFilter::updateKeywords(const QString &fileName, const QList& originals, const QVector& updates) { d->updateKeywords(fileName, originals, updates); } void FITSFilter::deleteKeyword(const QString &fileName, const QList& keywords) { d->deleteKeyword(fileName, keywords); } void FITSFilter::addKeywordUnit(const QString &fileName, const QList &keywords) { d->addKeywordUnit(fileName, keywords); } void FITSFilter::removeExtensions(const QStringList &extensions) { d->removeExtensions(extensions); } void FITSFilter::parseHeader(const QString &fileName, QTableWidget *headerEditTable, bool readKeys, const QList &keys) { d->parseHeader(fileName, headerEditTable, readKeys, keys); } void FITSFilter::parseExtensions(const QString &fileName, QTreeWidget *tw, bool checkPrimary) { d->parseExtensions(fileName, tw, checkPrimary); } QList FITSFilter::chduKeywords(const QString &fileName) { return d->chduKeywords(fileName); } void FITSFilter::loadFilterSettings(const QString& fileName) { Q_UNUSED(fileName) } void FITSFilter::saveFilterSettings(const QString& fileName) const { Q_UNUSED(fileName) } /*! * \brief contains the {StandardKeywords \ MandatoryKeywords} keywords * \return A list of keywords */ QStringList FITSFilter::standardKeywords() { return QStringList() << QLatin1String("(blank)") << QLatin1String("CROTA") << QLatin1String("EQUINOX") << QLatin1String("NAXIS") << QLatin1String("TBCOL") << QLatin1String("TUNIT") << QLatin1String("AUTHOR") << QLatin1String("CRPIX") << QLatin1String("EXTEND") << QLatin1String("OBJECT") << QLatin1String("TDIM") << QLatin1String("TZERO") << QLatin1String("BITPIX") << QLatin1String("CRVAL") << QLatin1String("EXTLEVEL") << QLatin1String("OBSERVER") << QLatin1String("TDISP") << QLatin1String("XTENSION") << QLatin1String("BLANK") << QLatin1String("CTYPE") << QLatin1String("EXTNAME") << QLatin1String("ORIGIN") << QLatin1String("TELESCOP") << QLatin1String("BLOCKED") << QLatin1String("DATAMAX") << QLatin1String("EXTVER") << QLatin1String("BSCALE") << QLatin1String("DATAMIN") << QLatin1String("PSCAL") << QLatin1String("TFORM") << QLatin1String("BUNIT") << QLatin1String("DATE") << QLatin1String("GROUPS") << QLatin1String("PTYPE") << QLatin1String("THEAP") << QLatin1String("BZERO") << QLatin1String("DATE-OBS") << QLatin1String("HISTORY") << QLatin1String("PZERO") << QLatin1String("TNULL") << QLatin1String("CDELT") << QLatin1String("INSTRUME") << QLatin1String("REFERENC") << QLatin1String("TSCAL") << QLatin1String("COMMENT") << QLatin1String("EPOCH") << QLatin1String("NAXIS") << QLatin1String("SIMPLE") << QLatin1String("TTYPE"); } /*! * \brief Returns a list of keywords, that are mandatory for an image extension of a FITS file * see: * https://archive.stsci.edu/fits/fits_standard/node64.html * \return A list of keywords */ QStringList FITSFilter::mandatoryImageExtensionKeywords() { return QStringList() << QLatin1String("XTENSION") << QLatin1String("BITPIX") << QLatin1String("NAXIS") << QLatin1String("PCOUNT") << QLatin1String("GCOUNT") << QLatin1String("END"); } /*! * \brief Returns a list of keywords, that are mandatory for a table extension (ascii or bintable) * of a FITS file * see: * https://archive.stsci.edu/fits/fits_standard/node58.html * https://archive.stsci.edu/fits/fits_standard/node68.html * \return A list of keywords */ QStringList FITSFilter::mandatoryTableExtensionKeywords() { return QStringList() << QLatin1String("XTENSION") << QLatin1String("BITPIX") << QLatin1String("NAXIS") << QLatin1String("NAXIS1") << QLatin1String("NAXIS2") << QLatin1String("PCOUNT") << QLatin1String("GCOUNT") << QLatin1String("TFIELDS") << QLatin1String("END"); } /*! * \brief Returns a list of strings that represent units which are used for autocompletion when adding * keyword units to keywords * \return A list of strings that represent units */ QStringList FITSFilter::units() { return QStringList() << QLatin1String("m (Metre)") << QLatin1String("kg (Kilogram)") << QLatin1String("s (Second)") << QString("M☉ (Solar mass)") << QLatin1String("AU (Astronomical unit") << QLatin1String("l.y (Light year)") << QLatin1String("km (Kilometres") << QLatin1String("pc (Parsec)") << QLatin1String("K (Kelvin)") << QLatin1String("mol (Mole)") << QLatin1String("cd (Candela)"); } /*! * \brief Sets the startColumn to \a column * \param column the column to be set */ void FITSFilter::setStartColumn(const int column) { d->startColumn = column; } /*! * \brief Returns startColumn * \return The startColumn */ int FITSFilter::startColumn() const { return d->startColumn; } /*! * \brief Sets the endColumn to \a column * \param column the column to be set */ void FITSFilter::setEndColumn(const int column) { d->endColumn = column; } /*! * \brief Returns endColumn * \return The endColumn */ int FITSFilter::endColumn() const { return d->endColumn; } /*! * \brief Sets the startRow to \a row * \param row the row to be set */ void FITSFilter::setStartRow(const int row) { d->startRow = row; } /*! * \brief Returns startRow * \return The startRow */ int FITSFilter::startRow() const { return d->startRow; } /*! * \brief Sets the endRow to \a row * \param row the row to be set */ void FITSFilter::setEndRow(const int row) { d->endRow = row; } /*! * \brief Returns endRow * \return The endRow */ int FITSFilter::endRow() const { return d->endRow; } /*! * \brief Sets commentsAsUnits to \a commentsAsUnits * * This is used when spreadsheets are exported to FITS table extensions and comments are used as the * units of the table's columns. * \param commentsAsUnits */ void FITSFilter::setCommentsAsUnits(const bool commentsAsUnits) { d->commentsAsUnits = commentsAsUnits; } /*! * \brief Sets exportTo to \a exportTo * * This is used to decide whether the container should be exported to a FITS image or a FITS table * For an image \a exportTo should be 0, for a table 1 * \param exportTo */ void FITSFilter::setExportTo(const int exportTo) { d->exportTo = exportTo; } QString FITSFilter::fileInfoString(const QString& fileName) { const int imagesCount = FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("IMAGES")).size(); QString info(i18n("Images: %1", QString::number(imagesCount ))); info += QLatin1String("
"); const int tablesCount = FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("TABLES")).size(); info += i18n("Tables: %1", QString::number(tablesCount)); return info; } //##################################################################### //################### Private implementation ########################## //##################################################################### FITSFilterPrivate::FITSFilterPrivate(FITSFilter* owner) : q(owner), startRow(-1), endRow(-1), startColumn(-1), endColumn(-1), commentsAsUnits(false), exportTo(0) { #ifdef HAVE_FITS m_fitsFile = 0; #endif } /*! * \brief Read the current header data unit from file \a filename in data source \a dataSource in \a importMode import mode * \param fileName the name of the file to be read * \param dataSource the data source to be filled * \param importMode */ QVector FITSFilterPrivate::readCHDU(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, bool* okToMatrix, int lines) { DEBUG("FITSFilterPrivate::readCHDU() file name = " << fileName.toStdString()); QVector dataStrings; #ifdef HAVE_FITS int status = 0; if(fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status)) { DEBUG(" ERROR opening file " << fileName.toStdString()); printError(status); return dataStrings; } int chduType; if (fits_get_hdu_type(m_fitsFile, &chduType, &status)) { printError(status); return dataStrings; } long actualRows; int actualCols; int columnOffset = 0; bool noDataSource = (dataSource == NULL); if(chduType == IMAGE_HDU) { DEBUG("IMAGE_HDU"); int maxdim = 2; int bitpix; int naxis; long naxes[2]; if (fits_get_img_param(m_fitsFile, maxdim, &bitpix, &naxis, naxes, &status)) { printError(status); return dataStrings; } if (naxis == 0) return dataStrings; actualRows = naxes[1]; actualCols = naxes[0]; if (lines == -1) lines = actualRows; else { if (lines > actualRows) lines = actualRows; } if (endRow != -1) { if (!noDataSource) lines = endRow; } if (endColumn != -1) actualCols = endColumn; if (noDataSource) dataStrings.reserve(lines); int i = 0; int j = 0; if (startRow != 1) i = startRow; if (startColumn != 1) j = startColumn; const int jstart = j; //TODO: support other modes QVector columnModes; columnModes.resize(actualCols - j); QStringList vectorNames; QVector dataContainer; if (!noDataSource) { dataContainer.reserve(actualCols - j); columnOffset = dataSource->prepareImport(dataContainer, importMode, lines - i, actualCols - j, vectorNames, columnModes); } long pixelCount = lines * actualCols; double* data = new double[pixelCount]; if (!data) { qDebug() << "Not enough memory for data"; return dataStrings; } if (fits_read_img(m_fitsFile, TDOUBLE, 1, pixelCount, NULL, data, NULL, &status)) { printError(status); return dataStrings << (QStringList() << QString("Error")); } int ii = 0; DEBUG(" Import " << lines << " lines"); for (; i < lines; ++i) { int jj = 0; QStringList line; line.reserve(actualCols - j); for (; j < actualCols; ++j) { if (noDataSource) line << QString::number(data[i*naxes[0] +j]); else static_cast*>(dataContainer[jj++])->operator[](ii) = data[i* naxes[0] + j]; } dataStrings << line; j = jstart; ii++; } delete[] data; if (dataSource) - dataSource->finalizeImport(columnOffset, 1, actualCols, "", importMode); + dataSource->finalizeImport(columnOffset, 1, actualCols, lines, "", importMode); fits_close_file(m_fitsFile, &status); return dataStrings; } else if ((chduType == ASCII_TBL) || (chduType == BINARY_TBL)) { DEBUG("ASCII_TBL or BINARY_TBL"); if (endColumn != -1) actualCols = endColumn; else fits_get_num_cols(m_fitsFile, &actualCols, &status); if (endRow != -1) actualRows = endRow; else fits_get_num_rows(m_fitsFile, &actualRows, &status); QStringList columnNames; QList columnsWidth; QStringList columnUnits; columnUnits.reserve(actualCols); columnsWidth.reserve(actualCols); columnNames.reserve(actualCols); int colWidth; char keyword[FLEN_KEYWORD]; char value[FLEN_VALUE]; int col = 1; if (startColumn != 1) { if (startColumn != 0) col = startColumn; } for (; col <=actualCols; ++col) { status = 0; fits_make_keyn("TTYPE", col, keyword, &status); fits_read_key(m_fitsFile, TSTRING, keyword, value, NULL, &status); columnNames.append(QLatin1String(value)); fits_make_keyn("TUNIT", col, keyword, &status); fits_read_key(m_fitsFile, TSTRING, keyword, value, NULL, &status); columnUnits.append(QLatin1String(value)); fits_get_col_display_width(m_fitsFile, col, &colWidth, &status); columnsWidth.append(colWidth); } status = 0; if (lines == -1) lines = actualRows; else if (lines > actualRows) lines = actualRows; if (endRow != -1) lines = endRow; QVector stringDataPointers; QVector numericDataPointers; QList columnNumericTypes; int startCol = 0; if (startColumn != 1) startCol = startColumn; int startRrow = 0; if (startRow != 1) startRrow = startRow; columnNumericTypes.reserve(actualCols); int datatype; int c = 1; if (startColumn != 1) { if (startColumn != 0) c = startColumn; } QList matrixNumericColumnIndices; for (; c <= actualCols; ++c) { fits_get_coltype(m_fitsFile, c, &datatype, NULL, NULL, &status); switch (datatype) { case TSTRING: columnNumericTypes.append(false); break; case TSHORT: columnNumericTypes.append(true); break; case TLONG: columnNumericTypes.append(true); break; case TFLOAT: columnNumericTypes.append(true); break; case TDOUBLE: columnNumericTypes.append(true); break; case TLOGICAL: columnNumericTypes.append(false); break; case TBIT: columnNumericTypes.append(true); break; case TBYTE: columnNumericTypes.append(true); break; case TCOMPLEX: columnNumericTypes.append(true); break; default: columnNumericTypes.append(false); break; } if ((datatype != TSTRING) && (datatype != TLOGICAL)) matrixNumericColumnIndices.append(c); } if (noDataSource) *okToMatrix = matrixNumericColumnIndices.isEmpty() ? false : true; if (!noDataSource) { DEBUG("HAS DataSource"); Spreadsheet* spreadsheet = dynamic_cast(dataSource); if(spreadsheet) { numericDataPointers.reserve(actualCols - startCol); stringDataPointers.reserve(actualCols - startCol); spreadsheet->setUndoAware(false); columnOffset = spreadsheet->resize(importMode, columnNames, actualCols - startCol); if (importMode == AbstractFileFilter::Replace) { spreadsheet->clear(); spreadsheet->setRowCount(lines - startRrow); } else { if (spreadsheet->rowCount() < (lines - startRrow)) spreadsheet->setRowCount(lines - startRrow); } DEBUG("Reading columns ..."); for (int n = 0; n < actualCols - startCol; ++n) { if (columnNumericTypes.at(n)) { spreadsheet->column(columnOffset+ n)->setColumnMode(AbstractColumn::Numeric); QVector* datap = static_cast* >(spreadsheet->column(columnOffset+n)->data()); numericDataPointers.push_back(datap); if (importMode == AbstractFileFilter::Replace) datap->clear(); } else { spreadsheet->column(columnOffset+ n)->setColumnMode(AbstractColumn::Text); QStringList* list = static_cast(spreadsheet->column(columnOffset+n)->data()); stringDataPointers.push_back(list); if (importMode == AbstractFileFilter::Replace) list->clear(); } } DEBUG(" ... DONE"); stringDataPointers.squeeze(); } else { numericDataPointers.reserve(matrixNumericColumnIndices.size()); columnOffset = dataSource->prepareImport(numericDataPointers, importMode, lines - startRrow, matrixNumericColumnIndices.size()); } numericDataPointers.squeeze(); } char* array = new char[1000]; //TODO: why 1000? int row = 1; if (startRow != 1) { if (startRow != 0) row = startRow; } int coll = 1; if (startColumn != 1) { if (startColumn != 0) coll = startColumn; } bool isMatrix = false; if (dynamic_cast(dataSource)) { isMatrix = true; coll = matrixNumericColumnIndices.first(); actualCols = matrixNumericColumnIndices.last(); if (importMode == AbstractFileFilter::Replace) { for (auto* col: numericDataPointers) static_cast*>(col)->clear(); } } for (; row <= lines; ++row) { int numericixd = 0; int stringidx = 0; QStringList line; line.reserve(actualCols-coll); for (int col = coll; col <= actualCols; ++col) { if (isMatrix) { if (!matrixNumericColumnIndices.contains(col)) continue; } if(fits_read_col_str(m_fitsFile, col, row, 1, 1, NULL, &array, NULL, &status)) printError(status); if (!noDataSource) { const QString& str = QString::fromLatin1(array); if (str.isEmpty()) { if (columnNumericTypes.at(col - 1)) static_cast*>(numericDataPointers[numericixd++])->push_back(0); else stringDataPointers[stringidx++]->append(QLatin1String("NULL")); } else { if (columnNumericTypes.at(col - 1)) static_cast*>(numericDataPointers[numericixd++])->push_back(str.toDouble()); else { if (!stringDataPointers.isEmpty()) stringDataPointers[stringidx++]->operator<<(str.simplified()); } } } else { QString tmpColstr = QString::fromLatin1(array); tmpColstr = tmpColstr.simplified(); if (tmpColstr.isEmpty()) line << QLatin1String("NULL"); else line << tmpColstr; } } dataStrings << line; } delete[] array; if (!noDataSource) - dataSource->finalizeImport(columnOffset, 1, actualCols, "", importMode); + dataSource->finalizeImport(columnOffset, 1, actualCols, lines, "", importMode); fits_close_file(m_fitsFile, &status); return dataStrings; } else qDebug() << "Incorrect header type"; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(importMode) Q_UNUSED(okToMatrix) Q_UNUSED(lines) #endif return dataStrings; } /*! * \brief Export from data source \a dataSource to file \a fileName * \param fileName the name of the file to be exported to * \param dataSource the datasource whose data is exported */ void FITSFilterPrivate::writeCHDU(const QString &fileName, AbstractDataSource *dataSource) { #ifdef HAVE_FITS if (!fileName.endsWith(QLatin1String(".fits"))) return; int status = 0; bool existed = false; if (!QFile::exists(fileName)) { if (fits_create_file(&m_fitsFile, fileName.toLatin1(), &status)) { printError(status); qDebug() << fileName; return; } } else { if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } else existed = true; } Matrix* const matrix = dynamic_cast(dataSource); if (matrix) { //FITS image if (exportTo == 0) { long naxes[2] = { matrix->columnCount(), matrix->rowCount() }; if (fits_create_img(m_fitsFile, FLOAT_IMG, 2, naxes, &status)) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); return; } const long nelem = naxes[0] * naxes[1]; double* const array = new double[nelem]; const QVector >* const data = static_cast>*>(matrix->data()); for (int col = 0; col < naxes[0]; ++col) for (int row = 0; row < naxes[1]; ++row) array[row * naxes[0] + col] = data->at(row).at(col); if (fits_write_img(m_fitsFile, TDOUBLE, 1, nelem, array, &status )) { printError(status); status = 0; } fits_close_file(m_fitsFile, &status); delete[] array; //FITS table } else { const int nrows = matrix->rowCount(); const int tfields = matrix->columnCount(); QVector columnNames; columnNames.resize(tfields); columnNames.reserve(tfields); QVector tform; tform.resize(tfields); tform.reserve(tfields); QVector tunit; tunit.resize(tfields); tunit.reserve(tfields); //TODO: mode const QVector>* const matrixData = static_cast>*>(matrix->data()); QVector column; const MatrixModel* matrixModel = static_cast(matrix->view())->model(); const int precision = matrix->precision(); for (int i = 0; i < tfields; ++i) { column = matrixData->at(i); const QString& columnName = matrixModel->headerData(i, Qt::Horizontal).toString(); columnNames[i] = new char[columnName.size()]; strcpy(columnNames[i], columnName.toLatin1().data()); tunit[i] = new char[1]; strcpy(tunit[i], ""); int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (matrix->text(row, i).size() > maxSize) maxSize = matrix->text(row, i).size(); } QString tformn; if (precision > 0) { tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".") + QString::number(precision); } else tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".0"); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); } //TODO extension name containing[] ? if (fits_create_tbl(m_fitsFile, ASCII_TBL, nrows, tfields, columnNames.data(), tform.data(), tunit.data(), matrix->name().toLatin1().data(),&status )) { printError(status); qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); double* columnNumeric = new double[nrows]; for (int col = 1; col <= tfields; ++col) { column = matrixData->at(col-1); for (int r = 0; r < column.size(); ++r) columnNumeric[r] = column.at(r); fits_write_col(m_fitsFile, TDOUBLE, col, 1, 1, nrows, columnNumeric, &status); if (status) { printError(status); delete[] columnNumeric; status = 0; if (!existed) { QFile file(fileName); file.remove(); } fits_close_file(m_fitsFile, &status); return; } } delete[] columnNumeric; fits_close_file(m_fitsFile, &status); } return; } Spreadsheet* const spreadsheet = dynamic_cast(dataSource); if (spreadsheet) { //FITS image if (exportTo == 0) { int maxRowIdx = -1; //don't export lots of empty lines if all of those contain nans // TODO: option? for (int c = 0; c < spreadsheet->columnCount(); ++c) { const Column* const col = spreadsheet->column(c); int currMaxRoxIdx = -1; for (int r = col->rowCount(); r >= 0; --r) { if (col->isValid(r)) { currMaxRoxIdx = r; break; } } if (currMaxRoxIdx > maxRowIdx) { maxRowIdx = currMaxRoxIdx; } } long naxes[2] = { spreadsheet->columnCount(), maxRowIdx + 1}; if (fits_create_img(m_fitsFile, FLOAT_IMG, 2, naxes, &status)) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } const long nelem = naxes[0] * naxes[1]; double* array = new double[nelem]; for (int row = 0; row < naxes[1]; ++row) { for (int col = 0; col < naxes[0]; ++col) array[row * naxes[0] + col] = spreadsheet->column(col)->valueAt(row); } if (fits_write_img(m_fitsFile, TDOUBLE, 1, nelem, array, &status )) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } fits_close_file(m_fitsFile, &status); delete[] array; } else { const int nrows = spreadsheet->rowCount(); const int tfields = spreadsheet->columnCount(); QVector columnNames; columnNames.resize(tfields); columnNames.reserve(tfields); QVector tform; tform.resize(tfields); tform.reserve(tfields); QVector tunit; tunit.resize(tfields); tunit.reserve(tfields); for (int i = 0; i < tfields; ++i) { const Column* const column = spreadsheet->column(i); columnNames[i] = new char[column->name().size()]; strcpy(columnNames[i], column->name().toLatin1().data()); if (commentsAsUnits) { tunit[i] = new char[column->comment().size()]; strcpy(tunit[i], column->comment().toLatin1().constData()); } else { tunit[i] = new char[2]; strcpy(tunit[i], ""); } switch (column->columnMode()) { case AbstractColumn::Numeric: { int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (QString::number(column->valueAt(row)).size() > maxSize) maxSize = QString::number(column->valueAt(row)).size(); } const Double2StringFilter* const filter = static_cast(column->outputFilter()); bool decimals = false; for (int ii = 0; ii < nrows; ++ii) { bool ok; QString cell = column->asStringColumn()->textAt(ii); double val = cell.toDouble(&ok); if (cell.size() > QString::number(val).size() + 1) { decimals = true; break; } } QString tformn; if (decimals) { int maxStringSize = -1; for (int row = 0; row < nrows; ++row) { if (column->asStringColumn()->textAt(row).size() > maxStringSize) maxStringSize = column->asStringColumn()->textAt(row).size(); } const int diff = abs(maxSize - maxStringSize); maxSize+= diff; tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".") + QString::number(filter->numDigits()); } else tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".0"); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); break; } case AbstractColumn::Text: { int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (column->textAt(row).size() > maxSize) maxSize = column->textAt(row).size(); } const QString& tformn = QLatin1String("A") + QString::number(maxSize); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); break; } case AbstractColumn::Integer: //TODO case AbstractColumn::DateTime: case AbstractColumn::Day: case AbstractColumn::Month: break; } } //TODO extension name containing[] ? if (fits_create_tbl(m_fitsFile, ASCII_TBL, nrows, tfields, columnNames.data(), tform.data(), tunit.data(), spreadsheet->name().toLatin1().data(),&status )) { printError(status); qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); QVector column; column.resize(nrows); column.reserve(nrows); double* columnNumeric = new double[nrows]; bool hadTextColumn = false; for (int col = 1; col <= tfields; ++col) { const Column* c = spreadsheet->column(col-1); AbstractColumn::ColumnMode columnMode = c->columnMode(); if (columnMode == AbstractColumn::Numeric) { for (int row = 0; row < nrows; ++row) columnNumeric[row] = c->valueAt(row); fits_write_col(m_fitsFile, TDOUBLE, col, 1, 1, nrows, columnNumeric, &status); if (status) { printError(status); delete[] columnNumeric; status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } } else { hadTextColumn = true; for (int row = 0; row < nrows; ++row) { column[row] = new char[c->textAt(row).size()]; strcpy(column[row], c->textAt(row).toLatin1().data()); } fits_write_col(m_fitsFile, TSTRING, col, 1, 1, nrows, column.data(), &status); if (status) { printError(status); qDeleteAll(column); status = 0; fits_close_file(m_fitsFile, &status); return; } } } delete[] columnNumeric; if (hadTextColumn) qDeleteAll(column); status = 0; fits_close_file(m_fitsFile, &status); } } #else Q_UNUSED(fileName) Q_UNUSED(dataSource) #endif } /*! * \brief Return a map of the available extensions names in file \a filename * The keys of the map are the extension types, the values are the names * \param fileName the name of the FITS file to be analyzed */ QMultiMap FITSFilterPrivate::extensionNames(const QString& fileName) { DEBUG("FITSFilterPrivate::extensionNames() file name = " << fileName.toStdString()); #ifdef HAVE_FITS QMultiMap extensions; int status = 0; fitsfile* fitsFile = 0; if (fits_open_file(&fitsFile, fileName.toLatin1(), READONLY, &status )) return QMultiMap(); int hduCount; if (fits_get_num_hdus(fitsFile, &hduCount, &status)) return QMultiMap(); int imageCount = 0; int asciiTableCount = 0; int binaryTableCount = 0; for (int currentHDU = 1; (currentHDU <= hduCount) && !status; ++currentHDU) { int hduType; status = 0; fits_get_hdu_type(fitsFile, &hduType, &status); switch (hduType) { case IMAGE_HDU: imageCount++; break; case ASCII_TBL: asciiTableCount++; break; case BINARY_TBL: binaryTableCount++; break; } char* keyVal = new char[FLEN_VALUE]; QString extName; if (!fits_read_keyword(fitsFile,"EXTNAME", keyVal, NULL, &status)) { extName = QLatin1String(keyVal); extName = extName.mid(1, extName.length() -2).simplified(); } else { status = 0; if (!fits_read_keyword(fitsFile,"HDUNAME", keyVal, NULL, &status)) { extName = QLatin1String(keyVal); extName = extName.mid(1, extName.length() -2).simplified(); } else { status = 0; switch (hduType) { case IMAGE_HDU: if (imageCount == 1) extName = i18n("Primary header"); else extName = i18n("IMAGE #%1", imageCount); break; case ASCII_TBL: extName = i18n("ASCII_TBL #%1", asciiTableCount); break; case BINARY_TBL: extName = i18n("BINARY_TBL #%1", binaryTableCount); break; } } } delete[] keyVal; status = 0; extName = extName.trimmed(); switch (hduType) { case IMAGE_HDU: extensions.insert(QLatin1String("IMAGES"), extName); break; case ASCII_TBL: extensions.insert(QLatin1String("TABLES"), extName); break; case BINARY_TBL: extensions.insert(QLatin1String("TABLES"), extName); break; } fits_movrel_hdu(fitsFile, 1, NULL, &status); } if (status == END_OF_FILE) status = 0; fits_close_file(fitsFile, &status); return extensions; #else Q_UNUSED(fileName) return QMultiMap(); #endif } /*! * \brief Prints the error text corresponding to the status code \a status * \param status the status code of the error */ void FITSFilterPrivate::printError(int status) const { #ifdef HAVE_FITS if (status) { char errorText[FLEN_ERRMSG]; fits_get_errstatus(status, errorText ); qDebug() << QLatin1String(errorText); } #else Q_UNUSED(status) #endif } /*! * \brief Add the keywords \a keywords to the current header unit * \param keywords the keywords to be added * \param fileName the name of the FITS file (extension) in which the keywords are added */ void FITSFilterPrivate::addNewKeyword(const QString& fileName, const QList& keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const FITSFilter::Keyword& keyword: keywords) { status = 0; if (!keyword.key.compare(QLatin1String("COMMENT"))) { if (fits_write_comment(m_fitsFile, keyword.value.toLatin1(), &status)) printError(status); } else if (!keyword.key.compare(QLatin1String("HISTORY"))) { if (fits_write_history(m_fitsFile, keyword.value.toLatin1(), &status)) printError(status); } else if (!keyword.key.compare(QLatin1String("DATE"))) { if (fits_write_date(m_fitsFile, &status)) printError(status); } else { int ok = 0; if (keyword.key.length() <= FLEN_KEYWORD) { ok++; if (keyword.value.length() <= FLEN_VALUE) { ok++; if(keyword.comment.length() <= FLEN_COMMENT) ok++; } } if (ok == 3) { bool ok; double val = keyword.value.toDouble(&ok); if (ok) { if (fits_write_key(m_fitsFile, TDOUBLE, keyword.key.toLatin1().data(), &val, keyword.comment.toLatin1().data(), &status)) printError(status); } else { if (fits_write_key(m_fitsFile, TSTRING, keyword.key.toLatin1().data(), keyword.value.toLatin1().data(), keyword.comment.toLatin1().data(), &status)) printError(status); } } else if ( ok == 2) { //comment too long } else if ( ok == 1) { //value too long } else { //keyword too long } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(keywords) Q_UNUSED(fileName) #endif } /*! * \brief Update keywords in the current header unit * \param fileName The name of the FITS file (extension) in which the keywords will be updated * \param originals The original keywords of the FITS file (extension) * \param updates The keywords that contain the updated values */ void FITSFilterPrivate::updateKeywords(const QString& fileName, const QList& originals, const QVector& updates) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } FITSFilter::Keyword updatedKeyword; FITSFilter::Keyword originalKeyword; FITSFilter::KeywordUpdate keywordUpdate; for (int i = 0; i < updates.size(); ++i) { updatedKeyword = updates.at(i); originalKeyword = originals.at(i); keywordUpdate = originals.at(i).updates; if (keywordUpdate.keyUpdated && keywordUpdate.valueUpdated && keywordUpdate.commentUpdated) { if (updatedKeyword.isEmpty()) { if (fits_delete_key(m_fitsFile, originalKeyword.key.toLatin1(), &status)) { printError(status); status = 0; } continue; } } if (!updatedKeyword.key.isEmpty()) { if (fits_modify_name(m_fitsFile, originalKeyword.key.toLatin1(), updatedKeyword.key.toLatin1(), &status )) { printError(status); status = 0; } } if (!updatedKeyword.value.isEmpty()) { bool ok; int intValue; double doubleValue; bool updated = false; doubleValue = updatedKeyword.value.toDouble(&ok); if (ok) { if (fits_update_key(m_fitsFile,TDOUBLE, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), &doubleValue, NULL, &status)) printError(status); else updated = true; } if (!updated) { intValue = updatedKeyword.value.toInt(&ok); if (ok) { if (fits_update_key(m_fitsFile,TINT, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), &intValue, NULL, &status)) printError(status); else updated = true; } } if (!updated) { if (fits_update_key(m_fitsFile,TSTRING, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), updatedKeyword.value.toLatin1().data(), NULL, &status)) printError(status); } } else { if (keywordUpdate.valueUpdated) { if (fits_update_key_null(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), NULL, &status)) { printError(status); status = 0; } } } if (!updatedKeyword.comment.isEmpty()) { if (fits_modify_comment(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), updatedKeyword.comment.toLatin1().data(), &status)) { printError(status); status = 0; } } else { if (keywordUpdate.commentUpdated) { if (fits_modify_comment(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), QString("").toLatin1().data(), &status)) { printError(status); status = 0; } } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(originals) Q_UNUSED(updates) #endif } /*! * \brief Delete the keywords \a keywords from the current header unit * \param fileName the name of the FITS file (extension) in which the keywords will be deleted. * \param keywords the keywords to deleted */ void FITSFilterPrivate::deleteKeyword(const QString& fileName, const QList &keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const auto& keyword : keywords) { if (!keyword.key.isEmpty()) { status = 0; if (fits_delete_key(m_fitsFile, keyword.key.toLatin1(), &status)) printError(status); } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(keywords) Q_UNUSED(fileName) #endif } /*! * \brief FITSFilterPrivate::addKeywordUnit * \param fileName the FITS file (extension) in which the keyword units are updated/added * \param keywords the keywords whose units were modified/added */ void FITSFilterPrivate::addKeywordUnit(const QString &fileName, const QList &keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for(const FITSFilter::Keyword& keyword : keywords) { if (keyword.updates.unitUpdated) { if (fits_write_key_unit(m_fitsFile, keyword.key.toLatin1(), keyword.unit.toLatin1().data(), &status)) { printError(status); status = 0; } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(keywords) #endif } /*! * \brief Remove extensions from a FITS file * \param extensions The extensions to be removed from the FITS file */ void FITSFilterPrivate::removeExtensions(const QStringList &extensions) { #ifdef HAVE_FITS int status = 0; for (const auto& ext : extensions) { status = 0; if (fits_open_file(&m_fitsFile, ext.toLatin1(), READWRITE, &status )) { printError(status); continue; } if (fits_delete_hdu(m_fitsFile, NULL, &status)) printError(status); status = 0; fits_close_file(m_fitsFile, &status); } #else Q_UNUSED(extensions) #endif } /*! * \brief Returns a list of keywords in the current header of \a fileName * \param fileName the name of the FITS file (extension) to be opened * \return A list of keywords */ QList FITSFilterPrivate::chduKeywords(const QString& fileName) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status )) { printError(status); return QList (); } int numberOfKeys; if (fits_get_hdrspace(m_fitsFile, &numberOfKeys, NULL, &status)) { printError(status); return QList (); } QList keywords; keywords.reserve(numberOfKeys); char* key = new char[FLEN_KEYWORD]; char* value = new char[FLEN_VALUE]; char* comment = new char[FLEN_COMMENT]; char* unit = new char[FLEN_VALUE]; for (int i = 1; i <= numberOfKeys; ++i) { QStringList recordValues; FITSFilter::Keyword keyword; if (fits_read_keyn(m_fitsFile, i, key, value, comment, &status)) { printError(status); status = 0; continue; } fits_read_key_unit(m_fitsFile, key, unit, &status); recordValues << QLatin1String(key) << QLatin1String(value) << QLatin1String(comment) << QLatin1String(unit); keyword.key = recordValues[0].simplified(); keyword.value = recordValues[1].simplified(); keyword.comment = recordValues[2].simplified(); keyword.unit = recordValues[3].simplified(); keywords.append(keyword); } delete[] key; delete[] value; delete[] comment; delete[] unit; fits_close_file(m_fitsFile, &status); return keywords; #else Q_UNUSED(fileName) return QList(); #endif } /*! * \brief Builds the table \a headerEditTable from the keywords \a keys * \param fileName The name of the FITS file from which the keys are read if \a readKeys is \c true * \param headerEditTable The table to be built * \param readKeys It's used to determine whether the keywords are provided or they should be read from * file \a fileName * \param keys The keywords that are provided if the keywords were read already. */ void FITSFilterPrivate::parseHeader(const QString &fileName, QTableWidget *headerEditTable, bool readKeys, const QList& keys) { #ifdef HAVE_FITS QList keywords; if (readKeys) keywords = chduKeywords(fileName); else keywords = keys; headerEditTable->setRowCount(keywords.size()); QTableWidgetItem* item; for (int i = 0; i < keywords.size(); ++i) { const FITSFilter::Keyword& keyword = keywords.at(i); const bool mandatory = FITSFilter::mandatoryImageExtensionKeywords().contains(keyword.key) || FITSFilter::mandatoryTableExtensionKeywords().contains(keyword.key); item = new QTableWidgetItem(keyword.key); const QString& itemText = item->text(); const bool notEditableKey = mandatory || itemText.contains(QLatin1String("TFORM")) || itemText.contains(QLatin1String("TTYPE")) || itemText.contains(QLatin1String("TUNIT")) || itemText.contains(QLatin1String("TDISP")) || itemText.contains(QLatin1String("TBCOL")) || itemText.contains(QLatin1String("TZERO")); const bool notEditableValue= mandatory || itemText.contains(QLatin1String("TFORM")) || itemText.contains(QLatin1String("TDISP")) || itemText.contains(QLatin1String("TBCOL")) || itemText.contains(QLatin1String("TZERO")); if (notEditableKey) item->setFlags(item->flags() & ~Qt::ItemIsEditable); else item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 0, item ); item = new QTableWidgetItem(keyword.value); if (notEditableValue) item->setFlags(item->flags() & ~Qt::ItemIsEditable); else item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 1, item ); QString commentFieldText; if (!keyword.unit.isEmpty()) { if (keyword.updates.unitUpdated) { const QString& comment = keyword.comment.right( keyword.comment.size() - keyword.comment.indexOf(QChar(']'))-1); commentFieldText = QLatin1String("[") + keyword.unit + QLatin1String("] ") + comment; } else { if (keyword.comment.at(0) == QLatin1Char('[')) commentFieldText = keyword.comment; else commentFieldText = QLatin1String("[") + keyword.unit + QLatin1String("] ") + keyword.comment; } } else commentFieldText = keyword.comment; item = new QTableWidgetItem(commentFieldText); item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 2, item ); } headerEditTable->resizeColumnsToContents(); #else Q_UNUSED(fileName) Q_UNUSED(headerEditTable) Q_UNUSED(readKeys) Q_UNUSED(keys) #endif } /*! * \brief Helper function to return the value of the key \a key * \param fileName The name of the FITS file (extension) in which the keyword with key \a key should exist * \param key The key of the keyword whose value it's returned * \return The value of the keyword as a string */ const QString FITSFilterPrivate::valueOf(const QString& fileName, const char *key) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status )) { printError(status); return QString (); } char* keyVal = new char[FLEN_VALUE]; QString keyValue; if (!fits_read_keyword(m_fitsFile, key, keyVal, NULL, &status)) { keyValue = QLatin1String(keyVal); keyValue = keyValue.simplified(); } else { printError(status); delete[] keyVal; fits_close_file(m_fitsFile, &status); return QString(); } delete[] keyVal; status = 0; fits_close_file(m_fitsFile, &status); return keyValue; #else Q_UNUSED(fileName) Q_UNUSED(key) return QString(); #endif } /*! * \brief Build the extensions tree from FITS file (extension) \a fileName * \param fileName The name of the FITS file to be opened * \param tw The QTreeWidget to be built * \param checkPrimary Used to determine whether the tree will be used for import or the header edit, * if it's \c true and if the primary array it's empty, then the item won't be added to the tree */ void FITSFilterPrivate::parseExtensions(const QString &fileName, QTreeWidget *tw, bool checkPrimary) { DEBUG("FITSFilterPrivate::parseExtensions()"); #ifdef HAVE_FITS const QMultiMap& extensions = extensionNames(fileName); const QStringList& imageExtensions = extensions.values(QLatin1String("IMAGES")); const QStringList& tableExtensions = extensions.values(QLatin1String("TABLES")); QTreeWidgetItem* root = tw->invisibleRootItem(); //TODO: fileName may contain any data type: check if it's a FITS file QTreeWidgetItem* treeNameItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << fileName); root->addChild(treeNameItem); treeNameItem->setExpanded(true); QTreeWidgetItem* imageExtensionItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << i18n("Images")); imageExtensionItem->setFlags(imageExtensionItem->flags() & ~Qt::ItemIsSelectable ); QString primaryHeaderNaxis = valueOf(fileName, "NAXIS"); const int naxis = primaryHeaderNaxis.toInt(); bool noImage = false; for (const QString& ext : imageExtensions) { QTreeWidgetItem* treeItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << ext); if (ext == i18n("Primary header")) { if (checkPrimary && naxis == 0) continue; } imageExtensionItem->addChild(treeItem); } if (imageExtensionItem->childCount() > 0) { treeNameItem->addChild(imageExtensionItem); imageExtensionItem->setIcon(0,QIcon::fromTheme("view-preview")); imageExtensionItem->setExpanded(true); imageExtensionItem->child(0)->setSelected(true); tw->setCurrentItem(imageExtensionItem->child(0)); } else noImage = true; if (tableExtensions.size() > 0) { QTreeWidgetItem* tableExtensionItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << i18n("Tables")); tableExtensionItem->setFlags(tableExtensionItem->flags() & ~Qt::ItemIsSelectable ); for (const QString& ext : tableExtensions) { QTreeWidgetItem* treeItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << ext); tableExtensionItem->addChild(treeItem); } if (tableExtensionItem->childCount() > 0) { treeNameItem->addChild(tableExtensionItem); tableExtensionItem->setIcon(0,QIcon::fromTheme("x-office-spreadsheet")); tableExtensionItem->setExpanded(true); if (noImage) { tableExtensionItem->child(0)->setSelected(true); tw->setCurrentItem(tableExtensionItem->child(0)); } } } #else Q_UNUSED(fileName) Q_UNUSED(tw) Q_UNUSED(checkPrimary) #endif DEBUG("FITSFilterPrivate::parseExtensions() DONE"); } /*! * \brief FITSFilterPrivate::~FITSFilterPrivate */ FITSFilterPrivate::~FITSFilterPrivate() { } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void FITSFilter::save(QXmlStreamWriter * writer) const { Q_UNUSED(writer) } /*! Loads from XML. */ bool FITSFilter::load(XmlStreamReader * loader) { Q_UNUSED(loader) return false; } diff --git a/src/backend/datasources/filters/HDF5Filter.cpp b/src/backend/datasources/filters/HDF5Filter.cpp index 3e86e1382..353de199b 100644 --- a/src/backend/datasources/filters/HDF5Filter.cpp +++ b/src/backend/datasources/filters/HDF5Filter.cpp @@ -1,1715 +1,1715 @@ /*************************************************************************** File : HDF5Filter.cpp Project : LabPlot Description : HDF5 I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ /* TODO: * Feature: implement missing data types and ranks * Performance: only fill dataPointer or dataStrings (not both) */ #include "backend/datasources/filters/HDF5Filter.h" #include "backend/datasources/filters/HDF5FilterPrivate.h" #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include /*! \class HDF5Filter \brief Manages the import/export of data from/to a HDF5 file. \ingroup datasources */ HDF5Filter::HDF5Filter():AbstractFileFilter(), d(new HDF5FilterPrivate(this)) {} HDF5Filter::~HDF5Filter() {} /*! parses the content of the file \c fileName. */ void HDF5Filter::parse(const QString & fileName, QTreeWidgetItem* rootItem) { d->parse(fileName, rootItem); } /*! reads the content of the data set \c dataSet from file \c fileName. */ QVector HDF5Filter::readCurrentDataSet(const QString& fileName, AbstractDataSource* dataSource, bool &ok, AbstractFileFilter::ImportMode importMode, int lines) { return d->readCurrentDataSet(fileName, dataSource, ok, importMode, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ void HDF5Filter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { d->readDataFromFile(fileName, dataSource, mode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void HDF5Filter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void HDF5Filter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void HDF5Filter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void HDF5Filter::setCurrentDataSetName(const QString& ds) { d->currentDataSetName = ds; } const QString HDF5Filter::currentDataSetName() const { return d->currentDataSetName; } void HDF5Filter::setStartRow(const int s) { d->startRow = s; } int HDF5Filter::startRow() const { return d->startRow; } void HDF5Filter::setEndRow(const int e) { d->endRow = e; } int HDF5Filter::endRow() const { return d->endRow; } void HDF5Filter::setStartColumn(const int c) { d->startColumn = c; } int HDF5Filter::startColumn() const { return d->startColumn; } void HDF5Filter::setEndColumn(const int c) { d->endColumn = c; } int HDF5Filter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### HDF5FilterPrivate::HDF5FilterPrivate(HDF5Filter* owner) : q(owner),currentDataSetName(""),startRow(1), endRow(-1), startColumn(1), endColumn(-1) { #ifdef HAVE_HDF5 m_status = 0; #endif } #ifdef HAVE_HDF5 void HDF5FilterPrivate::handleError(int err, const QString& function, const QString& arg) { #ifdef NDEBUG Q_UNUSED(err) Q_UNUSED(function) Q_UNUSED(arg) #else if (err < 0) DEBUG("ERROR " << err << ":" << function.toStdString() << "() - " << arg.toStdString()); #endif } QString HDF5FilterPrivate::translateHDF5Order(H5T_order_t o) { QString order; switch (o) { case H5T_ORDER_LE: order = "LE"; break; case H5T_ORDER_BE: order = "BE"; break; case H5T_ORDER_VAX: order = "VAX"; break; case H5T_ORDER_MIXED: order = "MIXED"; break; case H5T_ORDER_NONE: order = "NONE"; break; case H5T_ORDER_ERROR: order = "ERROR"; break; } return order; } QString HDF5FilterPrivate::translateHDF5Type(hid_t t) { QString type; if (H5Tequal(t, H5T_STD_I8LE) || H5Tequal(t, H5T_STD_I8BE)) type = "CHAR"; else if (H5Tequal(t, H5T_STD_U8LE) || H5Tequal(t, H5T_STD_U8BE)) type = "UCHAR"; else if (H5Tequal(t, H5T_STD_I16LE) || H5Tequal(t, H5T_STD_I16BE)) type = "SHORT"; else if (H5Tequal(t, H5T_STD_U16LE) || H5Tequal(t, H5T_STD_U16BE)) type = "USHORT"; else if (H5Tequal(t, H5T_STD_I32LE) || H5Tequal(t, H5T_STD_I32BE)) type = "INT"; else if (H5Tequal(t, H5T_STD_U32LE) || H5Tequal(t, H5T_STD_U32BE)) type = "UINT"; else if (H5Tequal(t, H5T_NATIVE_LONG)) type = "LONG"; else if (H5Tequal(t, H5T_NATIVE_ULONG)) type = "ULONG"; else if (H5Tequal(t, H5T_STD_I64LE) || H5Tequal(t, H5T_STD_I64BE)) type = "LLONG"; else if (H5Tequal(t, H5T_STD_U64LE) || H5Tequal(t, H5T_STD_U64BE)) type = "ULLONG"; else if (H5Tequal(t, H5T_IEEE_F32LE) || H5Tequal(t, H5T_IEEE_F32BE)) type = "FLOAT"; else if (H5Tequal(t, H5T_IEEE_F64LE) || H5Tequal(t, H5T_IEEE_F64BE)) type = "DOUBLE"; else if (H5Tequal(t, H5T_NATIVE_LDOUBLE)) type = "LDOUBLE"; else type = "UNKNOWN"; return type; } QString HDF5FilterPrivate::translateHDF5Class(H5T_class_t c) { QString dclass; switch (c) { case H5T_INTEGER: dclass = "INTEGER"; break; case H5T_FLOAT: dclass = "FLOAT"; break; case H5T_STRING: dclass = "STRING"; break; case H5T_BITFIELD: dclass = "BITFIELD"; break; case H5T_OPAQUE: dclass = "OPAQUE"; break; case H5T_COMPOUND: dclass = "COMPOUND"; break; case H5T_ARRAY: dclass = "ARRAY"; break; case H5T_ENUM: dclass = "ENUM"; break; case H5T_REFERENCE: dclass = "REFERENCE"; break; case H5T_VLEN: dclass = "VLEN"; break; case H5T_TIME: dclass = "TIME"; break; case H5T_NCLASSES: dclass = "NCLASSES"; break; case H5T_NO_CLASS: dclass = "NOCLASS"; break; } return dclass; } QStringList HDF5FilterPrivate::readHDF5Compound(hid_t tid) { size_t typeSize = H5Tget_size(tid); QString line; line += QLatin1String("COMPOUND(") + QString::number(typeSize) + QLatin1String(") : ("); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); for (int i=0; i < members; ++i) { H5T_class_t mclass = H5Tget_member_class(tid, i); handleError((int)mclass, "H5Tget_member_class"); hid_t mtype = H5Tget_member_type(tid, i); handleError((int)mtype, "H5Tget_member_type"); size_t size = H5Tget_size(mtype); handleError((int)size, "H5Tget_size"); QString typeString = translateHDF5Class(mclass); if (mclass == H5T_INTEGER || mclass == H5T_FLOAT) typeString = translateHDF5Type(mtype); line += H5Tget_member_name(tid, i) + QLatin1String("[") + typeString + QLatin1String("(") + QString::number(size) + QLatin1String(")]"); if (i == members-1) line += QLatin1String(")"); else line += QLatin1String(","); m_status = H5Tclose(mtype); handleError(m_status, "H5Tclose"); } QStringList dataString; dataString << line; return dataString; } template QStringList HDF5FilterPrivate::readHDF5Data1D(hid_t dataset, hid_t type, int rows, int lines, void* dataContainer) { DEBUG("readHDF5Data1D() rows =" << rows << "lines =" << lines); QStringList dataString; // we read all rows of data T* data = new T[rows]; m_status = H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Dread"); DEBUG(" startRow =" << startRow << "endRow =" << endRow); // DEBUG("dataContainer =" << dataContainer); for (int i = startRow-1; i < qMin(endRow, lines+startRow-1); ++i) { if (dataContainer) // read to data source static_cast*>(dataContainer)->operator[](i-startRow+1) = data[i]; else // for preview dataString << QString::number(static_cast(data[i])); } delete[] data; return dataString; } QStringList HDF5FilterPrivate::readHDF5CompoundData1D(hid_t dataset, hid_t tid, int rows, int lines, QVector& dataContainer) { DEBUG("HDF5FilterPrivate::readHDF5CompoundData1D()"); DEBUG(" dataContainer size = " << dataContainer.size()); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); DEBUG(" # members = " << members); QStringList dataString; if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) dataString << QLatin1String("("); dataContainer.resize(members); // avoid "index out of range" for preview } for (int m = 0; m < members; ++m) { hid_t mtype = H5Tget_member_type(tid, m); handleError((int)mtype, "H5Tget_member_type"); size_t msize = H5Tget_size(mtype); handleError((int)msize, "H5Tget_size"); hid_t ctype = H5Tcreate(H5T_COMPOUND, msize); handleError((int)ctype, "H5Tcreate"); m_status = H5Tinsert(ctype, H5Tget_member_name(tid, m), 0, mtype); handleError(m_status, "H5Tinsert"); QStringList mdataString; if (H5Tequal(mtype, H5T_STD_I8LE) || H5Tequal(mtype, H5T_STD_I8BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 2: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 4: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 8: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; } } else if (H5Tequal(mtype, H5T_STD_U8LE) || H5Tequal(mtype, H5T_STD_U8BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 2: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 4: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 8: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; } } else if (H5Tequal(mtype, H5T_STD_I16LE) || H5Tequal(mtype, H5T_STD_I16BE) || H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U16LE) || H5Tequal(mtype, H5T_STD_U16BE) || H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_I32LE) || H5Tequal(mtype, H5T_STD_I32BE) || H5Tequal(mtype, H5T_NATIVE_INT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U32LE) || H5Tequal(mtype, H5T_STD_U32BE) || H5Tequal(mtype, H5T_NATIVE_UINT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_LONG)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_ULONG)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_I64LE) || H5Tequal(mtype, H5T_STD_I64BE) || H5Tequal(mtype, H5T_NATIVE_LLONG)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U64LE) || H5Tequal(mtype, H5T_STD_U64BE) || H5Tequal(mtype, H5T_NATIVE_ULLONG)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_IEEE_F32LE) || H5Tequal(mtype, H5T_IEEE_F32BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_IEEE_F64LE) || H5Tequal(mtype, H5T_IEEE_F64BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_LDOUBLE)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else { if (dataContainer[m]) { for (int i = startRow-1; i < qMin(endRow, lines+startRow-1); ++i) static_cast*>(dataContainer[m])->operator[](i-startRow+1) = 0; } else { for (int i = 0; i < qMin(rows, lines); ++i) mdataString << QLatin1String("_"); } H5T_class_t mclass = H5Tget_member_class(tid, m); handleError((int)mclass, "H5Tget_member_class"); DEBUG("unsupported type of class " << translateHDF5Class(mclass).toStdString()); } if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) { dataString[i] += mdataString[i]; if (m < members-1) dataString[i] += QLatin1String(","); } } H5Tclose(ctype); } if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) dataString[i] += QLatin1String(")"); } return dataString; } template QVector HDF5FilterPrivate::readHDF5Data2D(hid_t dataset, hid_t type, int rows, int cols, int lines, QVector& dataPointer) { DEBUG("readHDF5Data2D() rows = " << rows << ", cols =" << cols << ", lines =" << lines); QVector dataStrings; T** data = (T**) malloc(rows*sizeof(T*)); data[0] = (T*) malloc(cols*rows*sizeof(T)); for (int i = 1; i < rows; ++i) data[i] = data[0]+i*cols; m_status = H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0][0]); handleError(m_status,"H5Dread"); for (int i = 0; i < qMin(rows, lines); ++i) { QStringList line; line.reserve(cols); for (int j = 0; j < cols; ++j) { if (dataPointer[0]) static_cast*>(dataPointer[j-startColumn+1])->operator[](i-startRow+1) = data[i][j]; else line << QString::number(static_cast(data[i][j])); } dataStrings << line; } free(data[0]); free(data); QDEBUG(dataStrings); return dataStrings; } QVector HDF5FilterPrivate::readHDF5CompoundData2D(hid_t dataset, hid_t tid, int rows, int cols, int lines) { DEBUG("readHDF5CompoundData2D() rows =" << rows << "cols =" << cols << "lines =" << lines); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); DEBUG(" # members =" << members); QVector dataStrings; for (int i = 0; i < qMin(rows, lines); ++i) { QStringList lineStrings; for (int j = 0; j < cols; ++j) lineStrings << QLatin1String("("); dataStrings << lineStrings; } //QStringList* data = new QStringList[members]; for (int m = 0; m < members; ++m) { hid_t mtype = H5Tget_member_type(tid, m); handleError((int)mtype, "H5Tget_member_type"); size_t msize = H5Tget_size(mtype); handleError((int)msize, "H5Tget_size"); hid_t ctype = H5Tcreate(H5T_COMPOUND, msize); handleError((int)ctype, "H5Tcreate"); m_status = H5Tinsert(ctype, H5Tget_member_name(tid, m), 0, mtype); handleError(m_status, "H5Tinsert"); // dummy container for all data columns // initially contains one pointer set to NULL QVector dummy(1, nullptr); QVector mdataStrings; if (H5Tequal(mtype, H5T_STD_I8LE) || H5Tequal(mtype, H5T_STD_I8BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 2: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 4: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 8: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; } } else if (H5Tequal(mtype, H5T_STD_U8LE) || H5Tequal(mtype, H5T_STD_U8BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 2: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 4: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 8: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; } } else if (H5Tequal(mtype, H5T_STD_I16LE) || H5Tequal(mtype, H5T_STD_I16BE)|| H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U16LE) || H5Tequal(mtype, H5T_STD_U16BE) || H5Tequal(mtype, H5T_NATIVE_USHORT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_I32LE) || H5Tequal(mtype, H5T_STD_I32BE) || H5Tequal(mtype, H5T_NATIVE_INT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U32LE) || H5Tequal(mtype, H5T_STD_U32BE) || H5Tequal(mtype, H5T_NATIVE_UINT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_LONG)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_ULONG)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_I64LE) || H5Tequal(mtype, H5T_STD_I64BE) || H5Tequal(mtype, H5T_NATIVE_LLONG)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U64LE) || H5Tequal(mtype, H5T_STD_U64BE) || H5Tequal(mtype, H5T_NATIVE_ULLONG)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_IEEE_F32LE) || H5Tequal(mtype, H5T_IEEE_F32BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_IEEE_F64LE) || H5Tequal(mtype, H5T_IEEE_F64BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_LDOUBLE)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else { for (int i = 0; i < qMin(rows, lines); ++i) { QStringList lineString; for (int j = 0; j < cols; ++j) lineString << QLatin1String("_"); mdataStrings << lineString; } #ifndef NDEBUG H5T_class_t mclass = H5Tget_member_class(tid, m); #endif DEBUG("unsupported class " << translateHDF5Class(mclass).toStdString()); } m_status = H5Tclose(ctype); handleError(m_status, "H5Tclose"); for (int i = 0; i < qMin(rows, lines); i++) { for (int j = 0; j < cols; j++) { dataStrings[i][j] += mdataStrings[i][j]; if (m < members-1) dataStrings[i][j] += QLatin1String(","); } } } for (int i = 0; i < qMin(rows, lines); ++i) { for (int j = 0; j < cols; ++j) dataStrings[i][j] += QLatin1String(")"); } QDEBUG("dataStrings =" << dataStrings); return dataStrings; } QStringList HDF5FilterPrivate::readHDF5Attr(hid_t aid) { QStringList attr; char name[MAXNAMELENGTH]; m_status = H5Aget_name(aid, MAXNAMELENGTH, name); handleError(m_status, "H5Aget_name"); attr << QString(name); // DEBUG(" name =" << QString(name)); hid_t aspace = H5Aget_space(aid); // the dimensions of the attribute data handleError((int)aspace, "H5Aget_space"); hid_t atype = H5Aget_type(aid); handleError((int)atype, "H5Aget_type"); hid_t aclass = H5Tget_class(atype); handleError((int)aclass, "H5Aget_class"); if (aclass == H5T_STRING) { char buf[MAXSTRINGLENGTH]; // buffer to read attr value hid_t amem = H5Tget_native_type(atype, H5T_DIR_ASCEND); handleError((int)amem, "H5Tget_native_type"); m_status = H5Aread(aid, amem, buf); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString(buf); m_status = H5Tclose(amem); handleError(m_status, "H5Tclose"); } else if (aclass == H5T_INTEGER) { if (H5Tequal(atype, H5T_STD_I8LE)) { qint8 value; m_status = H5Aread(aid, H5T_STD_I8LE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I8BE)) { qint8 value; m_status = H5Aread(aid, H5T_STD_I8BE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: { qint8 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 2: { qint16 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 4: { qint32 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 8: { qint64 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } default: DEBUG("unknown size " << sizeof(H5T_NATIVE_CHAR) << " of H5T_NATIVE_CHAR"); return QStringList(""); } } else if (H5Tequal(atype, H5T_STD_U8LE)) { uint8_t value; m_status = H5Aread(aid, H5T_STD_U8LE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U8BE)) { uint8_t value; m_status = H5Aread(aid, H5T_STD_U8BE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: { uint8_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 2: { uint16_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 4: { uint32_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 8: { uint64_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } default: DEBUG("unknown size " << sizeof(H5T_NATIVE_UCHAR) << " of H5T_NATIVE_UCHAR"); return QStringList(""); } } else if (H5Tequal(atype, H5T_STD_I16LE) || H5Tequal(atype, H5T_STD_I16BE) || H5Tequal(atype, H5T_NATIVE_SHORT)) { short value; m_status = H5Aread(aid, H5T_NATIVE_SHORT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U16LE) || H5Tequal(atype, H5T_STD_U16BE) || H5Tequal(atype, H5T_NATIVE_USHORT)) { unsigned short value; m_status = H5Aread(aid, H5T_NATIVE_USHORT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I32LE) || H5Tequal(atype, H5T_STD_I32BE) || H5Tequal(atype, H5T_NATIVE_INT)) { int value; m_status = H5Aread(aid, H5T_NATIVE_INT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U32LE) || H5Tequal(atype, H5T_STD_U32BE) || H5Tequal(atype, H5T_NATIVE_UINT)) { unsigned int value; m_status = H5Aread(aid, H5T_NATIVE_UINT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_LONG)) { long value; m_status = H5Aread(aid, H5T_NATIVE_LONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_ULONG)) { unsigned long value; m_status = H5Aread(aid, H5T_NATIVE_ULONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I64LE) || H5Tequal(atype, H5T_STD_I64BE) || H5Tequal(atype, H5T_NATIVE_LLONG)) { long long value; m_status = H5Aread(aid, H5T_NATIVE_LLONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U64LE) || H5Tequal(atype, H5T_STD_U64BE) || H5Tequal(atype, H5T_NATIVE_ULLONG)) { unsigned long long value; m_status = H5Aread(aid, H5T_NATIVE_ULLONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else attr<<" (unknown integer)"; } else if (aclass == H5T_FLOAT) { if (H5Tequal(atype, H5T_IEEE_F32LE) || H5Tequal(atype, H5T_IEEE_F32BE)) { float value; m_status = H5Aread(aid, H5T_NATIVE_FLOAT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_IEEE_F64LE) || H5Tequal(atype, H5T_IEEE_F64BE)) { double value; m_status = H5Aread(aid, H5T_NATIVE_DOUBLE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_LDOUBLE)) { long double value; m_status = H5Aread(aid, H5T_NATIVE_LDOUBLE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number((double)value); } else attr<<" (unknown float)"; } m_status = H5Tclose(atype); handleError(m_status, "H5Tclose"); m_status = H5Sclose(aspace); handleError(m_status, "H5Sclose"); return attr; } QStringList HDF5FilterPrivate::scanHDF5Attrs(hid_t oid) { QStringList attrList; int numAttr = H5Aget_num_attrs(oid); handleError(numAttr, "H5Aget_num_attrs"); DEBUG("number of attr =" << numAttr); for (int i = 0; i < numAttr; ++i) { hid_t aid = H5Aopen_idx(oid, i); handleError((int)aid, "H5Aopen_idx"); attrList << readHDF5Attr(aid); if (i < numAttr-1) attrList << QLatin1String(", "); m_status = H5Aclose(aid); handleError(m_status, "H5Aclose"); } return attrList; } QStringList HDF5FilterPrivate::readHDF5DataType(hid_t tid) { H5T_class_t typeClass = H5Tget_class(tid); handleError((int)typeClass, "H5Tget_class"); QStringList typeProps; QString typeString = translateHDF5Class(typeClass); if (typeClass == H5T_INTEGER || typeClass == H5T_FLOAT) typeString = translateHDF5Type(tid); typeProps<setIcon(0, QIcon::fromTheme("accessories-calculator")); dataTypeItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(dataTypeItem); } void HDF5FilterPrivate::scanHDF5DataSet(hid_t did, char *dataSetName, QTreeWidgetItem* parentItem) { QString attr = scanHDF5Attrs(did).join(""); char link[MAXNAMELENGTH]; m_status = H5Iget_name(did, link, MAXNAMELENGTH); handleError(m_status, "H5Iget_name"); QStringList dataSetProps; hsize_t size = H5Dget_storage_size(did); handleError((int)size, "H5Dget_storage_size"); hid_t datatype = H5Dget_type(did); handleError((int)datatype, "H5Dget_type"); size_t typeSize = H5Tget_size(datatype); handleError((int)typeSize, "H5Dget_size"); dataSetProps << readHDF5DataType(datatype); hid_t dataspace = H5Dget_space(did); int rank = H5Sget_simple_extent_ndims(dataspace); handleError(rank, "H5Sget_simple_extent_ndims"); unsigned int rows = 1, cols = 1, regs = 1; if (rank == 1) { hsize_t dims_out[1]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else if (rank == 2) { hsize_t dims_out[2]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; cols = dims_out[1]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String("x") << QString::number(cols) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else if (rank == 3) { hsize_t dims_out[3]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; cols = dims_out[1]; regs = dims_out[2]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String("x") << QString::number(cols) << QLatin1String("x") << QString::number(regs) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else dataSetProps << QLatin1String(", ") << i18n("rank %1 not supported yet", rank); hid_t pid = H5Dget_create_plist(did); handleError((int)pid, "H5Dget_create_plist"); dataSetProps << ", " << readHDF5PropertyList(pid).join(""); QTreeWidgetItem* dataSetItem = new QTreeWidgetItem(QStringList()<setIcon(0, QIcon::fromTheme("x-office-spreadsheet")); for (int i = 0; i < dataSetItem->columnCount(); ++i) { if (rows > 0 && cols > 0 && regs > 0) { dataSetItem->setBackground(i, QColor(192,255,192)); dataSetItem->setForeground(i, Qt::black); dataSetItem->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable); } else dataSetItem->setFlags(Qt::NoItemFlags); } parentItem->addChild(dataSetItem); } void HDF5FilterPrivate::scanHDF5Link(hid_t gid, char *linkName, QTreeWidgetItem* parentItem) { char target[MAXNAMELENGTH]; m_status = H5Gget_linkval(gid, linkName, MAXNAMELENGTH, target) ; handleError(m_status, "H5Gget_linkval"); QTreeWidgetItem* linkItem = new QTreeWidgetItem(QStringList() << QString(linkName) << i18n("symbolic link") << i18n("link to %1", QFile::decodeName(target))); linkItem->setIcon(0, QIcon::fromTheme("emblem-symbolic-link")); linkItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(linkItem); } void HDF5FilterPrivate::scanHDF5Group(hid_t gid, char *groupName, QTreeWidgetItem* parentItem) { DEBUG("HDF5FilterPrivate::scanHDF5Group()"); //check for hard link H5G_stat_t statbuf; m_status = H5Gget_objinfo(gid, ".", true, &statbuf); handleError(m_status, "H5Gget_objinfo"); if (statbuf.nlink > 1) { if (m_multiLinkList.contains(statbuf.objno[0])) { QTreeWidgetItem* objectItem = new QTreeWidgetItem(QStringList()<setIcon(0, QIcon::fromTheme("link")); objectItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(objectItem); return; } else { m_multiLinkList.append(statbuf.objno[0]); DEBUG(" group multiple links: "<setIcon(0, QIcon::fromTheme("folder")); groupItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(groupItem); hsize_t numObj; m_status = H5Gget_num_objs(gid, &numObj); handleError(m_status, "H5Gget_num_objs"); for (unsigned int i = 0; i < numObj; ++i) { char memberName[MAXNAMELENGTH]; m_status = H5Gget_objname_by_idx(gid, (hsize_t)i, memberName, (size_t)MAXNAMELENGTH ); handleError(m_status, "H5Gget_objname_by_idx"); int otype = H5Gget_objtype_by_idx(gid, (size_t)i ); handleError(otype, "H5Gget_objtype_by_idx"); switch (otype) { case H5G_LINK: { scanHDF5Link(gid, memberName, groupItem); break; } case H5G_GROUP: { hid_t grpid = H5Gopen(gid, memberName, H5P_DEFAULT); handleError((int)grpid, "H5Gopen"); scanHDF5Group(grpid, memberName, groupItem); m_status = H5Gclose(grpid); handleError(m_status, "H5Gclose"); break; } case H5G_DATASET: { hid_t dsid = H5Dopen(gid, memberName, H5P_DEFAULT); handleError((int)dsid, "H5Dopen"); scanHDF5DataSet(dsid, memberName, groupItem); m_status = H5Dclose(dsid); handleError(m_status, "H5Dclose"); break; } case H5G_TYPE: { hid_t tid = H5Topen(gid, memberName, H5P_DEFAULT); handleError((int)tid, "H5Topen"); scanHDF5DataType(tid, memberName, groupItem); m_status = H5Tclose(tid); handleError(m_status, "H5Tclose"); break; } default: QTreeWidgetItem* objectItem = new QTreeWidgetItem(QStringList() << QString(memberName) << i18n("unknown")); objectItem->setFlags(Qt::ItemIsEnabled); groupItem->addChild(objectItem); break; } } } #endif /*! parses the content of the file \c fileName and fill the tree using rootItem. */ void HDF5FilterPrivate::parse(const QString& fileName, QTreeWidgetItem* rootItem) { DEBUG("HDF5FilterPrivate::parse()"); #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); DEBUG("fileName = " << bafileName.data()); // check file type first htri_t isHdf5 = H5Fis_hdf5(bafileName.data()); if (isHdf5 == 0) { DEBUG(bafileName.data() << " is not a HDF5 file! Giving up."); return; } if (isHdf5 < 0) { DEBUG("H5Fis_hdf5() failed on " << bafileName.data() << "! Giving up."); return; } // open file hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); handleError((int)file, "H5Fopen", fileName); if (file < 0) { DEBUG("Opening file " << bafileName.data() << " failed! Giving up."); return; } char rootName[] = "/"; hid_t group = H5Gopen(file, rootName, H5P_DEFAULT); handleError((int)group, "H5Gopen", rootName); // CRASHES multiLinkList.clear(); scanHDF5Group(group, rootName, rootItem); m_status = H5Gclose(group); handleError(m_status, "H5Gclose", ""); m_status = H5Fclose(file); handleError(m_status, "H5Fclose", ""); #else DEBUG("HDF5 not available"); Q_UNUSED(fileName) Q_UNUSED(rootItem) #endif } /*! reads the content of the date set in the file \c fileName to a string (for preview) or to the data source. */ QVector HDF5FilterPrivate::readCurrentDataSet(const QString& fileName, AbstractDataSource* dataSource, bool &ok, AbstractFileFilter::ImportMode mode, int lines) { DEBUG("HDF5Filter::readCurrentDataSet()"); QVector dataStrings; if (currentDataSetName.isEmpty()) { //return QString("No data set selected").replace(' ',QChar::Nbsp); ok = false; return dataStrings << (QStringList() << i18n("No data set selected")); } DEBUG(" current data set =" << currentDataSetName.toStdString()); #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); handleError((int)file, "H5Fopen", fileName); QByteArray badataSet = currentDataSetName.toLatin1(); hid_t dataset = H5Dopen2(file, badataSet.data(), H5P_DEFAULT); handleError((int)file, "H5Dopen2", currentDataSetName); // Get datatype and dataspace hid_t dtype = H5Dget_type(dataset); handleError((int)dtype, "H5Dget_type"); H5T_class_t dclass = H5Tget_class(dtype); handleError((int)dclass, "H5Dget_class"); size_t typeSize = H5Tget_size(dtype); handleError((int)(typeSize-1), "H5Dget_size"); hid_t dataspace = H5Dget_space(dataset); handleError((int)dataspace, "H5Dget_space"); int rank = H5Sget_simple_extent_ndims(dataspace); handleError(rank, "H5Dget_simple_extent_ndims"); DEBUG(" rank =" << rank); int columnOffset = 0; // offset to import data int actualRows = 0, actualCols = 0; // rows and cols to read // dataContainer is used to store the data read from the dataSource // it contains the pointers of all columns // initially there is one pointer set to nullptr // check for dataContainer[0] != nullptr to decide if dataSource can be used QVector dataContainer(1, nullptr); // rank= 0: single value, 1: vector, 2: matrix, 3: 3D data, ... switch (rank) { case 0: { // single value actualCols = 1; switch (dclass) { case H5T_STRING: { char* data = (char *) malloc(typeSize * sizeof(char)); hid_t memtype = H5Tcopy(H5T_C_S1); handleError((int)memtype, "H5Tcopy"); m_status = H5Tset_size(memtype, typeSize); handleError(m_status, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Tread"); dataStrings << (QStringList() << data); free(data); break; } case H5T_INTEGER: case H5T_FLOAT: case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_COMPOUND: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataStrings << (QStringList() << i18n("rank 0 not implemented yet for type %1", translateHDF5Class(dclass))); qDebug() << dataStrings; } default: break; } break; } case 1: { // vector hsize_t size, maxSize; m_status = H5Sget_simple_extent_dims(dataspace, &size, &maxSize); handleError(m_status, "H5Sget_simple_extent_dims"); int rows = size; if (endRow == -1) endRow = rows; if (lines == -1) lines = endRow; actualRows = endRow-startRow+1; actualCols = 1; #ifndef NDEBUG H5T_order_t order = H5Tget_order(dtype); handleError((int)order, "H5Sget_order"); #endif QDEBUG(translateHDF5Class(dclass) << '(' << typeSize << ')' << translateHDF5Order(order) << ", rows:" << rows << " max:" << maxSize); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); QStringList dataString; // data saved in a list switch (dclass) { case H5T_STRING: { DEBUG("rank 1 H5T_STRING"); hid_t memtype = H5Tcopy(H5T_C_S1); handleError((int)memtype, "H5Tcopy"); char** data = (char **) malloc(rows * sizeof (char *)); if (H5Tis_variable_str(dtype)) { m_status = H5Tset_size(memtype, H5T_VARIABLE); handleError((int)memtype, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Dread"); } else { data[0] = (char *) malloc(rows * typeSize * sizeof (char)); for (int i = 1; i < rows; ++i) data[i] = data[0] + i * typeSize; m_status = H5Tset_size(memtype, typeSize); handleError((int)memtype, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data[0]); handleError(m_status, "H5Dread"); } for (int i = startRow-1; i < qMin(endRow, lines+startRow-1); ++i) dataString << data[i]; free(data); break; } case H5T_INTEGER: { if (H5Tequal(dtype, H5T_STD_I8LE)) dataString = readHDF5Data1D(dataset, H5T_STD_I8LE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I8BE)) dataString = readHDF5Data1D(dataset, H5T_STD_I8BE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 2: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 4: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 8: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; } } else if (H5Tequal(dtype, H5T_STD_U8LE)) dataString = readHDF5Data1D(dataset, H5T_STD_U8LE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U8BE)) dataString = readHDF5Data1D(dataset, H5T_STD_U8BE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 2: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 4: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 8: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; } } else if (H5Tequal(dtype, H5T_STD_I16LE) || H5Tequal(dtype, H5T_STD_I16BE) || H5Tequal(dtype, H5T_NATIVE_SHORT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_SHORT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U16LE) || H5Tequal(dtype, H5T_STD_U16BE) || H5Tequal(dtype, H5T_NATIVE_USHORT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_USHORT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I32LE) || H5Tequal(dtype, H5T_STD_I32BE) || H5Tequal(dtype, H5T_NATIVE_INT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_INT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U32LE) || H5Tequal(dtype, H5T_STD_U32BE) || H5Tequal(dtype, H5T_NATIVE_UINT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_UINT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_LONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_ULONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_ULONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I64LE) || H5Tequal(dtype, H5T_STD_I64BE) || H5Tequal(dtype, H5T_NATIVE_LLONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LLONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U64LE) || H5Tequal(dtype, H5T_STD_U64BE) || H5Tequal(dtype, H5T_NATIVE_ULLONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_ULLONG, rows, lines, dataContainer[0]); else { ok = false; dataString = (QStringList() << i18n("unsupported integer type for rank 1")); qDebug() << dataString; } break; } case H5T_FLOAT: { if (H5Tequal(dtype, H5T_IEEE_F32LE) || H5Tequal(dtype, H5T_IEEE_F32BE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_FLOAT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_IEEE_F64LE) || H5Tequal(dtype, H5T_IEEE_F64BE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_DOUBLE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LDOUBLE, rows, lines, dataContainer[0]); else { ok = false; dataString = (QStringList() << i18n("unsupported float type for rank 1")); qDebug() << dataString; } break; } case H5T_COMPOUND: { int members = H5Tget_nmembers(dtype); handleError(members, "H5Tget_nmembers"); if (dataSource) { // re-create data pointer dataContainer.clear(); dataSource->prepareImport(dataContainer, mode, actualRows, members, vectorNames, columnModes); } else dataStrings << readHDF5Compound(dtype); dataString = readHDF5CompoundData1D(dataset, dtype, rows, lines, dataContainer); break; } case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataString = (QStringList() << i18n("rank 1 not implemented yet for type %1", translateHDF5Class(dclass))); qDebug() << dataString; } default: break; } if (dataSource == NULL) { QDEBUG("dataString =" << dataString); for (int i = 0; i < qMin(rows, lines); ++i) dataStrings << (QStringList() << dataString[i]); } break; } case 2: { // matrix hsize_t dims_out[2]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, NULL); handleError(m_status, "H5Sget_simple_extent_dims"); int rows = dims_out[0]; int cols = dims_out[1]; if (endRow == -1) endRow=rows; if (lines == -1) lines = endRow; if (endColumn == -1) endColumn = cols; actualRows = endRow-startRow+1; actualCols = endColumn-startColumn+1; #ifndef NDEBUG H5T_order_t order = H5Tget_order(dtype); handleError((int)order, "H5Tget_order"); #endif QDEBUG(translateHDF5Class(dclass) << '(' << typeSize << ')' << translateHDF5Order(order) << "," << rows << "x" << cols); DEBUG("startRow/endRow" << startRow << endRow); DEBUG("startColumn/endColumn" << startColumn << endColumn); DEBUG("actual rows/cols" << actualRows << actualCols); DEBUG("lines" << lines); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); // read data switch (dclass) { case H5T_INTEGER: { if (H5Tequal(dtype, H5T_STD_I8LE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_I8LE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I8BE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_I8BE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 2: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 4: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 8: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; } } else if (H5Tequal(dtype, H5T_STD_U8LE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_U8LE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U8BE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_U8BE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 2: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 4: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 8: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; } } else if (H5Tequal(dtype, H5T_STD_I16LE) || H5Tequal(dtype, H5T_STD_I16BE) || H5Tequal(dtype, H5T_NATIVE_SHORT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_SHORT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U16LE) || H5Tequal(dtype, H5T_STD_U16BE) || H5Tequal(dtype, H5T_NATIVE_USHORT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_USHORT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I32LE) || H5Tequal(dtype, H5T_STD_I32BE) || H5Tequal(dtype, H5T_NATIVE_INT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_INT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U32LE) || H5Tequal(dtype, H5T_STD_U32BE) || H5Tequal(dtype, H5T_NATIVE_UINT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UINT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_LONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_ULONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_ULONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I64LE) || H5Tequal(dtype, H5T_STD_I64BE) || H5Tequal(dtype, H5T_NATIVE_LLONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LLONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U64LE) || H5Tequal(dtype, H5T_STD_U64BE) || H5Tequal(dtype, H5T_NATIVE_ULLONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_ULLONG, rows, cols, lines, dataContainer); else { ok=false; dataStrings << (QStringList() << i18n("unsupported integer type for rank 2")); qDebug() << dataStrings; } break; } case H5T_FLOAT: { if (H5Tequal(dtype, H5T_IEEE_F32LE) || H5Tequal(dtype, H5T_IEEE_F32BE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_FLOAT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_IEEE_F64LE) || H5Tequal(dtype, H5T_IEEE_F64BE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_DOUBLE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LDOUBLE, rows, cols, lines, dataContainer); else { ok = false; dataStrings << (QStringList() << i18n("unsupported float type for rank 2")); qDebug() << dataStrings; } break; } case H5T_COMPOUND: { dataStrings << readHDF5Compound(dtype); qDebug() << dataStrings; dataStrings << readHDF5CompoundData2D(dataset,dtype,rows,cols,lines); break; } case H5T_STRING: { // TODO: implement this ok = false; dataStrings << (QStringList() << i18n("rank 2 not implemented yet for type %1, size = %2", translateHDF5Class(dclass), typeSize)); qDebug() << dataStrings; break; } case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataStrings << (QStringList() << i18n("rank 2 not implemented yet for type %1", translateHDF5Class(dclass))); qDebug() << dataStrings; } default: break; } break; } default: { // 3D or more data ok = false; dataStrings << (QStringList() << i18n("rank %1 not implemented yet for type %2", rank, translateHDF5Class(dclass))); qDebug() << dataStrings; } } m_status = H5Sclose(dataspace); handleError(m_status, "H5Sclose"); m_status = H5Tclose(dtype); handleError(m_status, "H5Tclose"); m_status = H5Dclose(dataset); handleError(m_status, "H5Dclose"); m_status = H5Fclose(file); handleError(m_status, "H5Fclose"); if (!dataSource) return dataStrings; - dataSource->finalizeImport(columnOffset, 1, actualCols, "", mode); + dataSource->finalizeImport(columnOffset, 1, actualCols, -1, "", mode); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(mode) Q_UNUSED(lines) #endif return dataStrings; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void HDF5FilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { DEBUG("HDF5Filter::read()"); if (currentDataSetName.isEmpty()) { DEBUG("No data set selected"); return; } bool ok = true; readCurrentDataSet(fileName, dataSource, ok, mode); } /*! writes the content of \c dataSource to the file \c fileName. */ void HDF5FilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: writing HDF5 not implemented yet } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void HDF5Filter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("hdfFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool HDF5Filter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/datasources/filters/JsonFilter.cpp b/src/backend/datasources/filters/JsonFilter.cpp index e0252bced..ac6647c40 100644 --- a/src/backend/datasources/filters/JsonFilter.cpp +++ b/src/backend/datasources/filters/JsonFilter.cpp @@ -1,835 +1,835 @@ /*************************************************************************** File : JsonFilter.cpp Project : LabPlot Description : JSON I/O-filter. -------------------------------------------------------------------- -------------------------------------------------------------------- Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/JsonFilter.h" #include "backend/datasources/filters/JsonFilterPrivate.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include #include #include #include /*! \class JsonFilter \brief Manages the import/export of data from/to a file formatted using JSON. \ingroup datasources */ JsonFilter::JsonFilter() : AbstractFileFilter(), d(new JsonFilterPrivate(this)) {} JsonFilter::~JsonFilter() {} /*! reads the content of the device \c device. */ void JsonFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } /*! reads the content of the file \c fileName. */ void JsonFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } QVector JsonFilter::preview(const QString& fileName) { return d->preview(fileName); } QVector JsonFilter::preview(QIODevice& device) { return d->preview(device); } QVector JsonFilter::preview(QJsonDocument& doc) { return d->preview(doc); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void JsonFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void JsonFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void JsonFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list of all predefined data types. */ QStringList JsonFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } /*! returns the list of all predefined data row types. */ QStringList JsonFilter::dataRowTypes() { return (QStringList() << "Array" << "Object"); } void JsonFilter::setDataRowType(QJsonValue::Type type) { d->rowType = type; } QJsonValue::Type JsonFilter::dataRowType() const { return d->rowType; } void JsonFilter::setModelRows(QVector rows){ d->modelRows = rows; } QVector JsonFilter::modelRows() const { return d->modelRows; } void JsonFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString JsonFilter::dateTimeFormat() const { return d->dateTimeFormat; } void JsonFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language JsonFilter::numberFormat() const { return d->numberFormat; } void JsonFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool JsonFilter::NaNValueToZeroEnabled() const { if (d->nanValue == 0) return true; return false; } void JsonFilter::setCreateIndexEnabled(bool b){ d->createIndexEnabled = b; } void JsonFilter::setParseRowsName(bool b) { d->parseRowsName = b; } void JsonFilter::setVectorNames(const QString& s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList JsonFilter::vectorNames() const { return d->vectorNames; } QVector JsonFilter::columnModes() { return d->columnModes; } void JsonFilter::setStartRow(const int r) { d->startRow = r; } int JsonFilter::startRow() const { return d->startRow; } void JsonFilter::setEndRow(const int r) { d->endRow = r; } int JsonFilter::endRow() const { return d->endRow; } void JsonFilter::setStartColumn(const int c) { d->startColumn = c; } int JsonFilter::startColumn() const { return d->startColumn; } void JsonFilter::setEndColumn(const int c) { d->endColumn = c; } int JsonFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### JsonFilterPrivate::JsonFilterPrivate(JsonFilter* owner) : q(owner), model(new QJsonModel()), containerType(JsonFilter::Object), rowType(QJsonValue::Object), numberFormat(QLocale::C), createIndexEnabled(false), parseRowsName(false), vectorNames(), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_actualRows(0), m_actualCols(0), m_prepared(false), m_columnOffset(0) { } //TODO: delete model from memory /*! returns 1 if row is invalid and 0 otherwise. */ int JsonFilterPrivate::checkRow(QJsonValueRef value, int& countCols) { switch(rowType){ //TODO: implement other value types case QJsonValue::Array: { QJsonArray row = value.toArray(); if(row.isEmpty()) return 1; countCols = (countCols == -1 || countCols > row.count()) ? row.count() : countCols; break; } case QJsonValue::Object: { QJsonObject row = value.toObject(); if(row.isEmpty()) return 1; countCols = (countCols == -1 || countCols > row.count()) ? row.count() : countCols; break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return 1; } return 0; } /*! returns -1 if a parse error has occurred, 1 if the current row type not supported and 0 otherwise. */ int JsonFilterPrivate::parseColumnModes(QJsonValue row, QString rowName) { columnModes.resize(m_actualCols); int colIndexInContainer = startColumn - 1; for(int i = 0; i < m_actualCols; ++i){ if((createIndexEnabled || parseRowsName) && i == 0){ if(createIndexEnabled) columnModes[i] = AbstractColumn::Integer; if(parseRowsName) columnModes[i + createIndexEnabled] = AbstractFileFilter::columnMode(rowName, dateTimeFormat, numberFormat); i = i + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue columnValue; switch (rowType) { //TODO: implement other value types case QJsonValue::Array: { QJsonArray arr = row.toArray(); if(arr.count() < colIndexInContainer + 1) return -1; columnValue = *(row.toArray().begin() + colIndexInContainer); break; } case QJsonValue::Object: { QJsonObject obj = row.toObject(); if(obj.count() < colIndexInContainer + 1) return -1; if(vectorNames.count() == 0) vectorNames = row.toObject().keys(); columnValue = *(row.toObject().begin() + colIndexInContainer); break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return 1; } switch (columnValue.type()) { case QJsonValue::Double: columnModes[i] = AbstractColumn::Numeric; break; case QJsonValue::String: columnModes[i] = AbstractFileFilter::columnMode(columnValue.toString(), dateTimeFormat, numberFormat); break; case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return -1; } colIndexInContainer++; } if(parseRowsName) vectorNames.prepend("row name"); if(createIndexEnabled) vectorNames.prepend("index"); return 0; } void JsonFilterPrivate::setEmptyValue(int column, int row){ switch (columnModes[column]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[column])->operator[](row) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[column])->operator[](row) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[column])->operator[](row) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[column])->operator[](row) = ""; break; case AbstractColumn::Month: case AbstractColumn::Day: break; } } void JsonFilterPrivate::setValueFromString(int column, int row, QString valueString){ QLocale locale(numberFormat); switch (columnModes[column]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[column])->operator[](row) = isNumber ? value : nanValue; break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[column])->operator[](row) = isNumber ? value : 0; break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[column])->operator[](row) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[column])->operator[](row) = valueString; break; case AbstractColumn::Month: case AbstractColumn::Day: break; } } /*! returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end */ int JsonFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("device is sequential = " << device.isSequential()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; QJsonParseError err; QJsonDocument doc = QJsonDocument::fromJson(device.readAll(), &err); if(err.error != QJsonParseError::NoError || doc.isEmpty()) return 1; if(prepareDocumentToRead(doc) != 0) return 2; // reset to start of file if (!device.isSequential()) device.seek(0); return 0; } /*! returns 2 if a parse error has occurred and 0 otherwise. */ int JsonFilterPrivate::prepareDocumentToRead(const QJsonDocument& doc) { model->loadJson(doc); if(modelRows.isEmpty()) m_preparedDoc = doc; else { QModelIndex index; for(auto it = modelRows.begin(); it != modelRows.end(); ++it){ index = model->index(*it, 0, index); } m_preparedDoc = model->genJsonByIndex(index); } if(!m_preparedDoc.isEmpty()){ if(m_preparedDoc.isArray()) containerType = JsonFilter::Array; else if(m_preparedDoc.isObject()) containerType = JsonFilter::Object; else return 2; } else return 2; int countRows = 0; int countCols = -1; QJsonValue firstRow; QString firstRowName = ""; parseRowsName = parseRowsName && rowType == QJsonValue::Object; switch(containerType) { case JsonFilter::Array: { QJsonArray arr = m_preparedDoc.array(); if(arr.count() < startRow) return 2; int endRowOffset = (endRow == -1 || endRow > arr.count()) ? arr.count() : endRow; firstRow = *(arr.begin() + (startRow - 1)); for(QJsonArray::iterator it = arr.begin() + (startRow - 1); it != arr.begin() + endRowOffset; ++it) { if(checkRow(*it, countCols) != 0) return 2; countRows++; } break; } case JsonFilter::Object: { QJsonObject obj = m_preparedDoc.object(); if(obj.count() < startRow) return 2; int startRowOffset = startRow - 1; int endRowOffset = (endRow == -1 || endRow > obj.count()) ? obj.count() : endRow; firstRow = *(obj.begin() + startRowOffset); firstRowName = (obj.begin() + startRowOffset).key(); for(QJsonObject::iterator it = obj.begin() + startRowOffset; it != obj.begin() + endRowOffset; ++it) { if(checkRow(*it, countCols) != 0) return 2; countRows++; } break; } } if(endColumn == -1 || endColumn > countCols) endColumn = countCols; m_actualRows = countRows; m_actualCols = endColumn - startColumn + 1 + createIndexEnabled + parseRowsName; if(parseColumnModes(firstRow, firstRowName) != 0) return 2; DEBUG("start/end column: = " << startColumn << ' ' << endColumn); DEBUG("start/end rows = " << startRow << ' ' << endRow); DEBUG("actual cols/rows = " << m_actualCols << ' ' << m_actualRows); return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode); } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { if(!m_prepared) { const int deviceError = prepareDeviceToRead(device); if(deviceError != 0){ DEBUG("Device error = " << deviceError); return; } //TODO: support other modes and vector names m_prepared = true; } importData(dataSource, importMode, lines); } /*! reads the content of document \c doc to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::readDataFromDocument(const QJsonDocument& doc, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { if(!m_prepared) { const int docError = prepareDocumentToRead(doc); if(docError != 0){ DEBUG("Document parse error = " << docError); return; } //TODO: support other modes and vector names m_prepared = true; } importData(dataSource, importMode, lines); } /*! import the content of document \c m_preparedDoc to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::importData(AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { Q_UNUSED(lines) m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); int rowOffset = startRow - 1; DEBUG("reading " << m_actualRows << " lines"); for(int i = 0; i < m_actualRows; ++i) { QString rowName; QJsonValue row; switch (containerType) { case JsonFilter::Array: row = *(m_preparedDoc.array().begin() + rowOffset + i); break; case JsonFilter::Object: rowName = (m_preparedDoc.object().begin() + rowOffset + i).key(); row = *(m_preparedDoc.object().begin() + rowOffset + i); break; } int colIndex = 0; for(int n = 0; n < m_actualCols; ++n) { if((createIndexEnabled || parseRowsName) && n == 0) { if(createIndexEnabled) static_cast*>(m_dataContainer[n])->operator[](i) = i + 1; if(parseRowsName) setValueFromString(n + createIndexEnabled, i, rowName); n = n + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue value; switch(rowType){ //TODO: implement other value types case QJsonValue::Array: { value = *(row.toArray().begin() + colIndex); break; } case QJsonValue::Object: { value = *(row.toObject().begin() + colIndex); break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: break; } switch(value.type()) { case QJsonValue::Double: if(columnModes[n] == AbstractColumn::Numeric) static_cast*>(m_dataContainer[n])->operator[](i) = value.toDouble(); else setEmptyValue(n, i + startRow - 1); break; case QJsonValue::String: setValueFromString(n, i, value.toString()); break; case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: setEmptyValue(n, i + startRow - 1); break; } colIndex++; } emit q->completed(100 * i/m_actualRows); } //TODO: fix (startColumn + m_actualCols - 1) - dataSource->finalizeImport(m_columnOffset, startColumn, startColumn + m_actualCols - 1, dateTimeFormat, importMode); + dataSource->finalizeImport(m_columnOffset, startColumn, startColumn + m_actualCols - 1, m_actualRows, dateTimeFormat, importMode); } /*! generates the preview for the file \c fileName. */ QVector JsonFilterPrivate::preview(const QString& fileName) { KFilterDev device(fileName); return preview(device); } /*! generates the preview for device \c device. */ QVector JsonFilterPrivate::preview(QIODevice &device) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return QVector(); } return preview(); } /*! generates the preview for document \c doc. */ QVector JsonFilterPrivate::preview(QJsonDocument &doc) { if(prepareDocumentToRead(doc) != 0) return QVector(); return preview(); } /*! generates the preview for document \c m_preparedDoc. */ QVector JsonFilterPrivate::preview() { QVector dataStrings; int rowOffset = startRow - 1; DEBUG("reading " << m_actualRows << " lines"); for(int i = 0; i < m_actualRows; ++i) { QString rowName; QJsonValue row; switch (containerType) { case JsonFilter::Object: rowName = (m_preparedDoc.object().begin() + rowOffset + i).key(); row = *(m_preparedDoc.object().begin() + rowOffset + i); break; case JsonFilter::Array: row = *(m_preparedDoc.array().begin() + rowOffset + i); break; } QStringList lineString; int colIndex = 0; for(int n = 0; n < m_actualCols; ++n) { if((createIndexEnabled || parseRowsName) && n == 0) { if(createIndexEnabled) lineString += QString::number(i + 1); if(parseRowsName) lineString += rowName; n = n + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue value; switch(rowType){ case QJsonValue::Object: { value = *(row.toObject().begin() + colIndex); break; } case QJsonValue::Array: { value = *(row.toArray().begin() + colIndex); break; } //TODO: implement other value types case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: break; } switch(value.type()) { case QJsonValue::Double: if(columnModes[n] == AbstractColumn::Numeric) lineString += QString::number(value.toDouble(), 'g', 16); else lineString += lineString += QLatin1String(""); break; case QJsonValue::String: { //TODO: add parsing string before appending lineString += value.toString(); break; } case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: lineString += QLatin1String(""); break; } colIndex++; } dataStrings << lineString; emit q->completed(100 * i/m_actualRows); } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void JsonFilterPrivate::write(const QString& fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to json file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void JsonFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("jsonFilter"); writer->writeAttribute("rowType", QString::number(d->rowType)); writer->writeAttribute("dateTimeFormat", d->dateTimeFormat); writer->writeAttribute("numberFormat", QString::number(d->numberFormat)); writer->writeAttribute("createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute("parseRowsName", QString::number(d->parseRowsName)); writer->writeAttribute("nanValue", QString::number(d->nanValue)); writer->writeAttribute("startRow", QString::number(d->startRow)); writer->writeAttribute("endRow", QString::number(d->endRow)); writer->writeAttribute("startColumn", QString::number(d->startColumn)); writer->writeAttribute("endColumn", QString::number(d->endColumn)); QStringList list; for(auto it = modelRows().begin(); it != modelRows().end(); ++it){ list.append(QString::number(*it)); } writer->writeAttribute("modelRows", list.join(';')); writer->writeEndElement(); DEBUG("JsonFilter save params"); } /*! Loads from XML. */ bool JsonFilter::load(XmlStreamReader* reader) { QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("rowType").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'rowType'")); else d->rowType = static_cast(str.toInt()); str = attribs.value("dateTimeFormat").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'dateTimeFormat'")); else d->dateTimeFormat = str; str = attribs.value("numberFormat").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'numberFormat'")); else d->numberFormat = static_cast(str.toInt()); str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); str = attribs.value("parseRowsName").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'parseRowsName'")); else d->parseRowsName = str.toInt(); str = attribs.value("nanValue").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'nanValue'")); else d->nanValue = str.toDouble(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startColumn'")); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endColumn'")); else d->endColumn = str.toInt(); QStringList list = attribs.value("modelRows").toString().split(';'); if (list.isEmpty()) reader->raiseWarning(attributeWarning.arg("'modelRows'")); else{ d->modelRows = QVector(); for(auto it = list.begin(); it !=list.end(); ++it) d->modelRows.append((*it).toInt()); } DEBUG("JsonFilter load params"); return true; } diff --git a/src/backend/datasources/filters/NetCDFFilter.cpp b/src/backend/datasources/filters/NetCDFFilter.cpp index 8e0c7d8db..22daa2d62 100644 --- a/src/backend/datasources/filters/NetCDFFilter.cpp +++ b/src/backend/datasources/filters/NetCDFFilter.cpp @@ -1,690 +1,690 @@ /*************************************************************************** File : NetCDFFilter.cpp Project : LabPlot Description : NetCDF I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/NetCDFFilter.h" #include "backend/datasources/filters/NetCDFFilterPrivate.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/core/column/Column.h" #include #include /*! \class NetCDFFilter \brief Manages the import/export of data from/to a NetCDF file. \ingroup datasources */ NetCDFFilter::NetCDFFilter():AbstractFileFilter(), d(new NetCDFFilterPrivate(this)) {} NetCDFFilter::~NetCDFFilter() {} /*! parses the content of the file \c ileName. */ void NetCDFFilter::parse(const QString & fileName, QTreeWidgetItem* rootItem) { d->parse(fileName, rootItem); } /*! reads the content of the selected attribute from file \c fileName. */ QString NetCDFFilter::readAttribute(const QString & fileName, const QString & name, const QString & varName) { return d->readAttribute(fileName, name, varName); } /*! reads the content of the current variable from file \c fileName. */ QVector NetCDFFilter::readCurrentVar(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { return d->readCurrentVar(fileName, dataSource, importMode, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ void NetCDFFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { d->readDataFromFile(fileName, dataSource, mode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void NetCDFFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void NetCDFFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void NetCDFFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void NetCDFFilter::setCurrentVarName(const QString& ds) { d->currentVarName = ds; } const QString NetCDFFilter::currentVarName() const { return d->currentVarName; } void NetCDFFilter::setStartRow(const int s) { d->startRow = s; } int NetCDFFilter::startRow() const { return d->startRow; } void NetCDFFilter::setEndRow(const int e) { d->endRow = e; } int NetCDFFilter::endRow() const { return d->endRow; } void NetCDFFilter::setStartColumn(const int c) { d->startColumn=c; } int NetCDFFilter::startColumn() const { return d->startColumn; } void NetCDFFilter::setEndColumn(const int c) { d->endColumn = c; } int NetCDFFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### NetCDFFilterPrivate::NetCDFFilterPrivate(NetCDFFilter* owner) : q(owner), startRow(1), endRow(-1), startColumn(1), endColumn(-1) { #ifdef HAVE_NETCDF m_status = 0; #endif } #ifdef HAVE_NETCDF void NetCDFFilterPrivate::handleError(int err, const QString& function) { if (err != NC_NOERR) qDebug() << "NETCDF ERROR:" << function << "() - " << nc_strerror(m_status); } QString NetCDFFilterPrivate::translateDataType(nc_type type) { QString typeString; switch (type) { case NC_BYTE: typeString = "BYTE"; break; case NC_UBYTE: typeString = "UBYTE"; break; case NC_CHAR: typeString = "CHAR"; break; case NC_SHORT: typeString = "SHORT"; break; case NC_USHORT: typeString = "USHORT"; break; case NC_INT: typeString = "INT"; break; case NC_UINT: typeString = "UINT"; break; case NC_INT64: typeString = "INT64"; break; case NC_UINT64: typeString = "UINT64"; break; case NC_FLOAT: typeString = "FLOAT"; break; case NC_DOUBLE: typeString = "DOUBLE"; break; case NC_STRING: typeString = "STRING"; break; default: typeString = "UNKNOWN"; } return typeString; } QString NetCDFFilterPrivate::scanAttrs(int ncid, int varid, int attid, QTreeWidgetItem* parentItem) { char name[NC_MAX_NAME + 1]; int nattr, nstart = 0; if (attid == -1) { m_status = nc_inq_varnatts(ncid, varid, &nattr); handleError(m_status, "nc_inq_varnatts"); } else { nstart = attid; nattr = attid+1; } nc_type type; size_t len; QStringList valueString; for (int i = nstart; i < nattr; i++) { valueString.clear(); m_status = nc_inq_attname(ncid, varid, i, name); handleError(m_status, "nc_inq_attname"); m_status = nc_inq_att(ncid, varid, name, &type, &len); handleError(m_status, "nc_inq_att"); QDEBUG(" attr" << i+1 << "name/type/len =" << name << translateDataType(type) << len); //read attribute switch (type) { case NC_BYTE: { signed char *value = (signed char *)malloc(len*sizeof(signed char)); m_status = nc_get_att_schar(ncid, varid, name, value); handleError(m_status, "nc_get_att_schar"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_UBYTE: { unsigned char *value = (unsigned char *)malloc(len*sizeof(unsigned char)); m_status = nc_get_att_uchar(ncid, varid, name, value); handleError(m_status, "nc_get_att_uchar"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_CHAR: { char *value = (char *)malloc((len+1)*sizeof(char)); m_status = nc_get_att_text(ncid, varid, name, value); handleError(m_status, "nc_get_att_text"); value[len] = 0; valueString << value; free(value); break; } case NC_SHORT: { short *value = (short *)malloc(len*sizeof(short)); m_status = nc_get_att_short(ncid, varid, name, value); handleError(m_status, "nc_get_att_short"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_USHORT: { unsigned short *value = (unsigned short *)malloc(len*sizeof(unsigned short)); m_status = nc_get_att_ushort(ncid, varid, name, value); handleError(m_status, "nc_get_att_ushort"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_INT: { int *value = (int *)malloc(len*sizeof(int)); m_status = nc_get_att_int(ncid, varid, name, value); handleError(m_status, "nc_get_att_int"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_UINT: { unsigned int *value = (unsigned int *)malloc(len*sizeof(unsigned int)); m_status = nc_get_att_uint(ncid, varid, name, value); handleError(m_status, "nc_get_att_uint"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_INT64: { long long *value = (long long *)malloc(len*sizeof(long long)); m_status = nc_get_att_longlong(ncid, varid, name, value); handleError(m_status, "nc_get_att_longlong"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_UINT64: { unsigned long long *value = (unsigned long long *)malloc(len*sizeof(unsigned long long)); m_status = nc_get_att_ulonglong(ncid, varid, name, value); handleError(m_status, "nc_get_att_ulonglong"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_FLOAT: { float *value = (float *)malloc(len*sizeof(float)); m_status = nc_get_att_float(ncid, varid, name, value); handleError(m_status, "nc_get_att_float"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } case NC_DOUBLE: { double *value = (double *)malloc(len*sizeof(double)); m_status = nc_get_att_double(ncid, varid, name, value); handleError(m_status, "nc_get_att_double"); for (unsigned int l = 0; l < len; l++) valueString << QString::number(value[l]); free(value); break; } default: valueString << "not supported"; } if (parentItem != NULL) { QString typeName; if (varid == NC_GLOBAL) typeName = i18n("global attribute"); else { char varName[NC_MAX_NAME + 1]; m_status = nc_inq_varname(ncid, varid, varName); typeName = i18n("%1 attribute", QString(varName)); } QStringList props; props << translateDataType(type) << " (" << QString::number(len) << ")"; QTreeWidgetItem *attrItem = new QTreeWidgetItem(QStringList() << QString(name) << typeName << props.join("") << valueString.join(", ")); attrItem->setIcon(0, QIcon::fromTheme("accessories-calculator")); attrItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(attrItem); } } return valueString.join("\n"); } void NetCDFFilterPrivate::scanDims(int ncid, int ndims, QTreeWidgetItem* parentItem) { int ulid; m_status = nc_inq_unlimdim(ncid, &ulid); handleError(m_status, "nc_inq_att"); char name[NC_MAX_NAME + 1]; size_t len; for (int i = 0; i < ndims; i++) { m_status = nc_inq_dim(ncid, i, name, &len); handleError(m_status, "nc_inq_att"); DEBUG(" dim" << i+1 << ": name/len =" << name << len); QStringList props; props<setIcon(0, QIcon::fromTheme("accessories-calculator")); attrItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(attrItem); } } void NetCDFFilterPrivate::scanVars(int ncid, int nvars, QTreeWidgetItem* parentItem) { char name[NC_MAX_NAME + 1]; nc_type type; int ndims, nattrs; int dimids[NC_MAX_VAR_DIMS]; for (int i = 0; i < nvars; i++) { m_status = nc_inq_var(ncid, i, name, &type, &ndims, dimids, &nattrs); handleError(m_status, "nc_inq_att"); QDEBUG(" var" << i+1 << ": name/type=" << name << translateDataType(type)); DEBUG(" ndims/nattr" << ndims << nattrs); QStringList props; props << translateDataType(type); char dname[NC_MAX_NAME + 1]; size_t dlen; props << "("; for (int j = 0; j < ndims; j++) { m_status = nc_inq_dim(ncid, dimids[j], dname, &dlen); if (j != 0) props << "x"; props << QString::number(dlen); } props << ")"; QTreeWidgetItem *varItem = new QTreeWidgetItem(QStringList() << QString(name) << i18n("variable") << props.join("")<<""); varItem->setIcon(0, QIcon::fromTheme("x-office-spreadsheet")); varItem->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable); // highlight item for (int c = 0; c < varItem->columnCount(); c++) { varItem->setBackground(c, QColor(192, 255, 192)); varItem->setForeground(c, Qt::black); } parentItem->addChild(varItem); scanAttrs(ncid, i, -1, varItem); } } #endif /*! parses the content of the file \c fileName and fill the tree using rootItem. */ void NetCDFFilterPrivate::parse(const QString & fileName, QTreeWidgetItem* rootItem) { #ifdef HAVE_NETCDF QByteArray bafileName = fileName.toLatin1(); int ncid; m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); int ndims, nvars, nattr, uldid; m_status = nc_inq(ncid, &ndims, &nvars, &nattr, &uldid); handleError(m_status, "nc_inq"); DEBUG(" nattr/ndims/nvars =" << nattr << ndims << nvars); QTreeWidgetItem *attrItem = new QTreeWidgetItem(QStringList() << QString(i18n("Attributes"))); attrItem->setIcon(0,QIcon::fromTheme("folder")); attrItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(attrItem); scanAttrs(ncid, NC_GLOBAL, -1, attrItem); QTreeWidgetItem *dimItem = new QTreeWidgetItem(QStringList() << QString(i18n("Dimensions"))); dimItem->setIcon(0, QIcon::fromTheme("folder")); dimItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(dimItem); scanDims(ncid, ndims, dimItem); QTreeWidgetItem *varItem = new QTreeWidgetItem(QStringList() << QString(i18n("Variables"))); varItem->setIcon(0, QIcon::fromTheme("folder")); varItem->setFlags(Qt::ItemIsEnabled); rootItem->addChild(varItem); scanVars(ncid, nvars, varItem); #else Q_UNUSED(fileName) Q_UNUSED(rootItem) #endif } QString NetCDFFilterPrivate::readAttribute(const QString & fileName, const QString & name, const QString & varName) { #ifdef HAVE_NETCDF int ncid; QByteArray bafileName = fileName.toLatin1(); m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); // get varid int varid; if (varName == "global") varid = NC_GLOBAL; else { QByteArray bavarName = varName.toLatin1(); m_status = nc_inq_varid(ncid, bavarName.data(), &varid); handleError(m_status, "nc_inq_varid"); } // attribute 'name' int attid; QByteArray baName = name.toLatin1(); m_status = nc_inq_attid(ncid, varid, baName.data(), &attid); handleError(m_status, "nc_inq_attid"); return scanAttrs(ncid, varid, attid); #else Q_UNUSED(fileName) Q_UNUSED(name) Q_UNUSED(varName) return QString(); #endif } /*! reads the content of the variable in the file \c fileName to a string (for preview) or to the data source. */ QVector NetCDFFilterPrivate::readCurrentVar(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode, int lines) { QVector dataStrings; if (currentVarName.isEmpty()) return dataStrings << (QStringList() << i18n("No variable selected")); QDEBUG(" current variable =" << currentVarName); #ifdef HAVE_NETCDF int ncid; QByteArray bafileName = fileName.toLatin1(); m_status = nc_open(bafileName.data(), NC_NOWRITE, &ncid); handleError(m_status, "nc_open"); int varid; QByteArray baVarName = currentVarName.toLatin1(); m_status = nc_inq_varid(ncid, baVarName.data(), &varid); handleError(m_status, "nc_inq_varid"); int ndims; nc_type type; m_status = nc_inq_varndims(ncid, varid, &ndims); handleError(m_status, "nc_inq_varndims"); m_status = nc_inq_vartype(ncid, varid, &type); handleError(m_status, "nc_inq_type"); int* dimids = (int *) malloc(ndims * sizeof(int)); m_status = nc_inq_vardimid(ncid, varid, dimids); handleError(m_status, "nc_inq_vardimid"); int actualRows = 0, actualCols = 0; int columnOffset = 0; QVector dataContainer; switch (ndims) { case 0: dataStrings << (QStringList() << i18n("zero dimensions")); qDebug() << dataStrings; break; case 1: { size_t size; m_status = nc_inq_dimlen(ncid, dimids[0], &size); handleError(m_status, "nc_inq_dimlen"); if (endRow == -1) endRow = (int)size; if (lines == -1) lines = endRow; actualRows = endRow-startRow+1; actualCols = 1; DEBUG("start/end row" << startRow << endRow); DEBUG("act rows/cols" << actualRows << actualCols); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); double* data = nullptr; if (dataSource) data = static_cast*>(dataContainer[0])->data(); else data = new double[(unsigned int)actualRows]; size_t start = (size_t)(startRow-1), count = (size_t)actualRows; m_status = nc_get_vara_double(ncid, varid, &start, &count, data); handleError(m_status, "nc_get_vara_double"); if (!dataSource) { for (int i = 0; i < qMin(actualRows, lines); i++) dataStrings << (QStringList() << QString::number(data[i])); delete[] data; } break; } case 2: { size_t rows, cols; m_status = nc_inq_dimlen(ncid, dimids[0], &rows); handleError(m_status, "nc_inq_dimlen"); m_status = nc_inq_dimlen(ncid, dimids[1], &cols); handleError(m_status, "nc_inq_dimlen"); if (endRow == -1) endRow = (int)rows; if (lines == -1) lines = endRow; if (endColumn == -1) endColumn = (int)cols; actualRows = endRow-startRow+1; actualCols = endColumn-startColumn+1; DEBUG("dim =" << rows << "x" << cols); DEBUG("startRow/endRow:" << startRow << endRow); DEBUG("startColumn/endColumn:" << startColumn << endColumn); DEBUG("actual rows/cols:" << actualRows << actualCols); DEBUG("lines:" << lines); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); //TODO: use given names? QStringList vectorNames; if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); double** data = (double**) malloc(rows * sizeof(double*)); data[0] = (double*)malloc( cols * rows * sizeof(double) ); for (unsigned int i = 1; i < rows; i++) data[i] = data[0] + i*cols; m_status = nc_get_var_double(ncid, varid, &data[0][0]); handleError(m_status, "nc_get_var_double"); for (int i = 0; i < qMin((int)rows, lines); i++) { QStringList line; for (size_t j = 0; j < cols; j++) { if (dataContainer[0]) static_cast*>(dataContainer[(int)(j-(size_t)startColumn+1)])->operator[](i-startRow+1) = data[i][(int)j]; else line << QString::number(data[i][j]); } dataStrings << line; emit q->completed(100*i/actualRows); } free(data[0]); free(data); break; } default: dataStrings << (QStringList() << i18n("%1 dimensional data of type %2 not supported yet", ndims, translateDataType(type))); qDebug() << dataStrings; } free(dimids); if (dataSource) - dataSource->finalizeImport(columnOffset, 1, actualCols, "", mode); + dataSource->finalizeImport(columnOffset, 1, actualCols, -1, "", mode); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(mode) Q_UNUSED(lines) #endif return dataStrings; } /*! reads the content of the current selected variable from file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ QVector NetCDFFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { QVector dataStrings; if (currentVarName.isEmpty()) { DEBUG(" No variable selected"); return dataStrings; } DEBUG(" current variable =" << currentVarName.toStdString()); return readCurrentVar(fileName, dataSource, mode); } /*! writes the content of \c dataSource to the file \c fileName. */ void NetCDFFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: not implemented yet } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void NetCDFFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("netcdfFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool NetCDFFilter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/datasources/filters/NgspiceRawAsciiFilter.cpp b/src/backend/datasources/filters/NgspiceRawAsciiFilter.cpp index fd1400835..0eb03f99f 100644 --- a/src/backend/datasources/filters/NgspiceRawAsciiFilter.cpp +++ b/src/backend/datasources/filters/NgspiceRawAsciiFilter.cpp @@ -1,393 +1,393 @@ /*************************************************************************** File : NgspiceRawAsciiFilter.cpp Project : LabPlot Description : Ngspice RAW ASCII filter -------------------------------------------------------------------- Copyright : (C) 2018 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/datasources/filters/NgspiceRawAsciiFilter.h" #include "backend/datasources/filters/NgspiceRawAsciiFilterPrivate.h" #include "backend/lib/trace.h" #include /*! \class NgspiceRawAsciiFilter \brief Import of data stored in Ngspice's raw formant, ASCCI version of it. \ingroup datasources */ NgspiceRawAsciiFilter::NgspiceRawAsciiFilter() : AbstractFileFilter(), d(new NgspiceRawAsciiFilterPrivate(this)) {} NgspiceRawAsciiFilter::~NgspiceRawAsciiFilter() {} bool NgspiceRawAsciiFilter::isNgspiceAsciiFile(const QString& fileName) { QFile file(fileName); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { DEBUG("Failed to open the file " << fileName.toStdString()); return false; } QString line = file.readLine(); if (!line.startsWith(QLatin1String("Title:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("Date:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("Plotname:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("Flags:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("No. Variables:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("No. Points:"))) return false; line = file.readLine(); if (!line.startsWith(QLatin1String("Variables:"))) return false; return true; } QString NgspiceRawAsciiFilter::fileInfoString(const QString& fileName) { QFile file(fileName); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) return QString(); QString info; while (!file.atEnd()) { QString line = file.readLine(); if (line.simplified() == QLatin1String("Values:")) break; if (!info.isEmpty()) info += QLatin1String("
"); info += line; } return info; } /*! reads the content of the file \c fileName. */ void NgspiceRawAsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } QVector NgspiceRawAsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void NgspiceRawAsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /*! loads the predefined filter settings for \c filterName */ void NgspiceRawAsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void NgspiceRawAsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } void NgspiceRawAsciiFilter::setStartRow(const int r) { d->startRow = r; } int NgspiceRawAsciiFilter::startRow() const { return d->startRow; } void NgspiceRawAsciiFilter::setEndRow(const int r) { d->endRow = r; } int NgspiceRawAsciiFilter::endRow() const { return d->endRow; } void NgspiceRawAsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int NgspiceRawAsciiFilter::startColumn() const { return d->startColumn; } void NgspiceRawAsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int NgspiceRawAsciiFilter::endColumn() const { return d->endColumn; } QStringList NgspiceRawAsciiFilter::vectorNames() const { return d->vectorNames; } QVector NgspiceRawAsciiFilter::columnModes() { return d->columnModes; } //##################################################################### //################### Private implementation ########################## //##################################################################### NgspiceRawAsciiFilterPrivate::NgspiceRawAsciiFilterPrivate(NgspiceRawAsciiFilter* owner) : q(owner), startRow(1), endRow(-1), startColumn(1), endColumn(-1) { } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void NgspiceRawAsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("NgspiceRawAsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode)); QFile file(fileName); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { DEBUG("Failed to open the file " << fileName.toStdString()); return; } //skip the first four lines in the header file.readLine(); file.readLine(); file.readLine(); file.readLine(); //number of variables QString line = file.readLine(); const int vars = line.right(line.length() - 15).toInt(); //remove the "No. Variables: " sub-string //number of points line = file.readLine(); const int points = line.right(line.length() - 12).toInt(); //remove the "No. Points: " sub-string //add names of the variables vectorNames.clear(); columnModes.clear(); file.readLine(); for (int i = 0; i points) ? points : endRow; const int actualRows = actualEndRow - startRow + 1; const int actualCols = hasComplexValues ? vars*2 : vars; const int columnOffset = dataSource->prepareImport(m_dataContainer, importMode, actualRows, actualCols, vectorNames, columnModes); //skip data lines, if required DEBUG(" Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { for (int j = 0; j < vars; ++j) file.readLine(); file.readLine(); //skip the empty line after each value block } //read the data points QStringList lineString; int currentRow = 0; // indexes the position in the vector(column) QLocale locale(QLocale::C); bool isNumber(false); for (int i = 0; i < actualEndRow; ++i) { lineString.clear(); for (int j = 0; j < vars; ++j) { line = file.readLine(); QStringList tokens = line.split(QLatin1Char('\t')); QString valueString = tokens.at(1).simplified(); //string containing the value(s) if (hasComplexValues) { QStringList realImgTokens = valueString.split(QLatin1Char(',')); if (realImgTokens.size() == 2) { //sanity check to make sure we really have both parts //real part double value = locale.toDouble(realImgTokens.at(0), &isNumber); static_cast*>(m_dataContainer[2*j])->operator[](currentRow) = (isNumber ? value : NAN); //imaginary part value = locale.toDouble(realImgTokens.at(1), &isNumber); static_cast*>(m_dataContainer[2*j+1])->operator[](currentRow) = (isNumber ? value : NAN); } } else { const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[j])->operator[](currentRow) = (isNumber ? value : NAN); } } file.readLine(); //skip the empty line after each value block currentRow++; emit q->completed(100 * currentRow/actualRows); } - dataSource->finalizeImport(columnOffset, startColumn, endColumn, "", importMode); + dataSource->finalizeImport(columnOffset, startColumn, endColumn, currentRow, "", importMode); } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector NgspiceRawAsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; QFile file(fileName); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) { DEBUG("Failed to open the file " << fileName.toStdString()); return dataStrings; } //skip the first four lines in the header file.readLine(); file.readLine(); file.readLine(); file.readLine(); //number of variables QString line = file.readLine(); const int vars = line.right(line.length() - 15).toInt(); //remove the "No. Variables: " sub-string //number of points line = file.readLine(); const int points = line.right(line.length() - 12).toInt(); //remove the "No. Points: " sub-string //add names of the variables vectorNames.clear(); columnModes.clear(); file.readLine(); for (int i = 0; i #ifdef HAVE_ZIP #include #include #endif #include #include #include #include #include #include ROOTFilter::ROOTFilter():AbstractFileFilter(), d(new ROOTFilterPrivate) {} ROOTFilter::~ROOTFilter() {} void ROOTFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } void ROOTFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } void ROOTFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } void ROOTFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } void ROOTFilter::setCurrentHistogram(const QString& histogram) { d->currentHistogram = histogram; } const QString ROOTFilter::currentHistogram() const { return d->currentHistogram; } QStringList ROOTFilter::listHistograms(const QString& fileName) { return d->listHistograms(fileName); } QVector ROOTFilter::previewCurrentHistogram(const QString& fileName, int first, int last) { return d->previewCurrentHistogram(fileName, first, last); } int ROOTFilter::binsInCurrentHistogram(const QString& fileName) { return d->binsInCurrentHistogram(fileName); } void ROOTFilter::setStartBin(const int s) { d->startBin = s; } int ROOTFilter::startBin() const { return d->startBin; } void ROOTFilter::setEndBin(const int e) { d->endBin = e; } int ROOTFilter::endBin() const { return d->endBin; } void ROOTFilter::setColumns(const int columns) { d->columns = columns; } int ROOTFilter::columns() const { return d->columns; } void ROOTFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("rootFilter"); writer->writeAttribute("startBin", QString::number(d->startBin) ); writer->writeAttribute("endBin", QString::number(d->endBin) ); writer->writeAttribute("columns", QString::number(d->columns) ); writer->writeEndElement(); } bool ROOTFilter::load(XmlStreamReader* reader) { QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); // read attributes QString str = attribs.value("startBin").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startBin'")); else d->startBin = str.toInt(); str = attribs.value("endBin").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endBin'")); else d->endBin = str.toInt(); str = attribs.value("columns").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'columns'")); else d->columns = str.toInt(); return true; } /**************** ROOTFilterPrivate implementation *******************/ ROOTFilterPrivate::ROOTFilterPrivate() {} void ROOTFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("ROOTFilterPrivate::readDataFromFile()"); setFile(fileName); auto bins = readHistogram(); const int nbins = (int)bins.size(); // skip underflow and overflow bins by default int first = qMax(startBin, 0); int last = qMin(endBin, nbins - 1); QStringList colNames = createHeaders(); QVector dataContainer; const int columnOffset = dataSource->prepareImport(dataContainer, importMode, last - first + 1, colNames.size(), colNames, QVector(colNames.size(), AbstractColumn::Numeric)); // read data DEBUG("reading " << first - last + 1 << " lines"); int c = 0; if (columns & ROOTFilter::Center) { QVector& container = *static_cast*>(dataContainer[c++]); for (int i = first; i <= last; ++i) container[i - first] = (i > 0 && i < nbins - 1) ? 0.5 * (bins[i].lowedge + bins[i + 1].lowedge) : i == 0 ? bins.front().lowedge // -infinity : -bins.front().lowedge; // +infinity } if (columns & ROOTFilter::Low) { QVector& container = *static_cast*>(dataContainer[c++]); for (int i = first; i <= last; ++i) container[i - first] = bins[i].lowedge; } if (columns & ROOTFilter::Content) { QVector& container = *static_cast*>(dataContainer[c++]); for (int i = first; i <= last; ++i) container[i - first] = bins[i].content; } if (columns & ROOTFilter::Error) { QVector& container = *static_cast*>(dataContainer[c++]); for (int i = first; i <= last; ++i) container[i - first] = std::sqrt(bins[i].sumw2); } - dataSource->finalizeImport(columnOffset, 0, colNames.size() - 1, QString(), importMode); + dataSource->finalizeImport(columnOffset, 0, colNames.size() - 1, -1, QString(), importMode); } void ROOTFilterPrivate::write(const QString& fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } QStringList ROOTFilterPrivate::listHistograms(const QString& fileName) { setFile(fileName); QStringList histList; for (const auto& hist : currentROOTHist->listHistograms()) { histList << QString::fromStdString(hist); } return histList; } QVector ROOTFilterPrivate::previewCurrentHistogram(const QString& fileName, int first, int last) { DEBUG("previewCurrentHistogram()"); setFile(fileName); auto bins = readHistogram(); const int nbins = (int)bins.size(); last = qMin(nbins - 1, last); QVector preview(qMax(last - first + 2, 1)); preview.last() = createHeaders(); // read data DEBUG("reading " << preview.size() << " lines"); if (columns & ROOTFilter::Center) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number( (i > 0 && i < nbins - 1) ? 0.5 * (bins[i].lowedge + bins[i + 1].lowedge) : i == 0 ? bins.front().lowedge // -infinity : -bins.front().lowedge); // +infinity } if (columns & ROOTFilter::Low) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(bins[i].lowedge); } if (columns & ROOTFilter::Content) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(bins[i].content); } if (columns & ROOTFilter::Error) { for (int i = first; i <= last; ++i) preview[i - first] << QString::number(std::sqrt(bins[i].sumw2)); } return preview; } int ROOTFilterPrivate::binsInCurrentHistogram(const QString& fileName) { setFile(fileName); QStringList nameindex = currentHistogram.split(';'); bool ok = nameindex.size() > 1; int cycle = ok ? nameindex.last().toInt(&ok) : 1; if (ok) { nameindex.removeLast(); } else { cycle = 1; } return currentROOTHist->histogramBins(nameindex.join(';').toStdString(), cycle); } QStringList ROOTFilterPrivate::createHeaders() { QStringList colNames; if (columns & ROOTFilter::Center) colNames << i18n("Bin Center"); if (columns & ROOTFilter::Low) colNames << i18n("Low Edge"); if (columns & ROOTFilter::Content) colNames << i18n("Content"); if (columns & ROOTFilter::Error) colNames << i18n("Error"); return colNames; } void ROOTFilterPrivate::setFile(const QString& fileName) { if (!currentROOTHist || fileName != currentFile) { currentFile = fileName; currentROOTHist.reset(new ROOTHist(fileName.toStdString())); } } std::vector ROOTFilterPrivate::readHistogram() { QStringList nameindex = currentHistogram.split(';'); bool ok = nameindex.size() > 1; int cycle = ok ? nameindex.last().toInt(&ok) : 1; if (ok) { nameindex.removeLast(); } else { cycle = 1; } return currentROOTHist->readHistogram(nameindex.join(';').toStdString(), cycle); } /******************** ROOTHist implementation ************************/ namespace ROOTHistHelpers { /// Read value from stream template T read(std::ifstream& is) { union { T val; char buf[sizeof(T)]; } r; for (size_t i = 0; i < sizeof(T); ++i) { r.buf[sizeof(T) - i - 1] = is.get(); } return r.val; } /// Read value from buffer template T read(char*& s) { union { T val; char buf[sizeof(T)]; } r; for (size_t i = 0; i < sizeof(T); ++i) { r.buf[sizeof(T) - i - 1] = *(s++); } return r.val; } /// Get version of ROOT object, obtain number of bytes in object short Version(char*& buffer, size_t& count) { count = read(buffer); short version = (count & 0x40000000) ? read(buffer) : read(buffer -= 4); count &= (count & 0x40000000) ? (~0x40000000) : 0; return version; } /// Get version of ROOT object short Version(char*& buffer) { size_t c; return Version(buffer, c); } /// Skip ROOT object void Skip(char*& buffer, const size_t& n) { for (size_t i = 0; i < n; ++i) { char* nbuf = buffer + 4; size_t count; Version(buffer, count); buffer = nbuf + count; } } /// Skip TObject header void SkipObject(char*& buffer) { Version(buffer); buffer += 8; } /// Get TString std::string String(char*& buffer) { size_t s = *(buffer++); if (s == 0) return std::string(); else { if (s == 0xFF) s = read(buffer); buffer += s; return std::string(buffer - s, buffer); } } } using namespace ROOTHistHelpers; ROOTHist::ROOTHist(const std::string& filename) : filename(filename) { // The file structure is described in root/io/io/src/TFile.cxx std::ifstream is(filename, std::ifstream::binary); std::string root(4, 0); is.read(const_cast(root.data()), 4); if (root != "root") return; is.seekg(8); // skip version int pos = read(is); is.seekg(16); int lastpos = read(is); is.seekg(33); compression = read(is); compression = compression > 0 ? compression : 0; while (is.good() && pos < lastpos) { is.seekg(pos); size_t lcdata = read(is); is.seekg(2, is.cur); // short version = read(is); size_t ldata = read(is); is.seekg(4, is.cur); size_t lkey = read(is); short cycle = read(is); is.seekg(8, is.cur); std::string cname(read(is), 0); is.read(&cname[0], cname.size()); if (cname.size() == 4 && cname.substr(0, 3) == "TH1") { KeyBuffer::ContentType type; switch (cname[3]) { case 'D': type = KeyBuffer::ContentType::Double; break; case 'F': type = KeyBuffer::ContentType::Float; break; case 'I': type = KeyBuffer::ContentType::Int; break; case 'S': type = KeyBuffer::ContentType::Short; break; case 'C': type = KeyBuffer::ContentType::Byte; break; default: type = KeyBuffer::ContentType::Invalid; break; } if (type) { std::string name(read(is), 0); is.read(&name[0], name.size()); std::string title(read(is), 0); is.read(&title[0], title.size()); auto it = histkeys.end(); // see root/io/io/src/TKey.cxx for reference if (compression && ldata > 256) { # ifdef HAVE_ZIP // Default: compression level // ZLIB: 100 + compression level // LZ4: 400 + compression level if (compression / 100 <= 1 || compression / 100 == 4) { // see root/core/zip/src/RZip.cxx -> R__unzip std::string lib(2, 0); is.read(&lib[0], 2); char method = is.get(); size_t chcdata = is.get(); chcdata |= (is.get() << 8); chcdata |= (is.get() << 16); size_t chdata = is.get(); chdata |= (is.get() << 8); chdata |= (is.get() << 16); if (chcdata == lcdata - lkey - 9 && chdata == ldata) { if (lib == "ZL" && method == Z_DEFLATED) { it = histkeys.emplace(name + ";" + std::to_string(cycle), KeyBuffer{name, title, cycle, type, KeyBuffer::zlib, pos + lkey + 9, chcdata, chdata, 0} ).first; } else if (lib == "L4" && method == LZ4_versionNumber() / (100 * 100)) { it = histkeys.emplace(name + ";" + std::to_string(cycle), KeyBuffer{name, title, cycle, type, KeyBuffer::lz4, pos + lkey + 9 + 8, chcdata - 8, chdata, 0} ).first; } } } # endif } else { it = histkeys.emplace(name + ";" + std::to_string(cycle), KeyBuffer{name, title, cycle, type, KeyBuffer::none, pos + lkey, ldata, ldata, 0} ).first; } if (it != histkeys.end()) readNBins(it->second); } } pos += (int)lcdata; } } std::vector ROOTHist::listHistograms() const { std::vector l; for (auto& n : histkeys) { l.emplace_back(n.first); } return l; } std::vector ROOTHist::readHistogram(const std::string& name, int cycle) { auto it = histkeys.find(name + ";" + std::to_string(cycle)); if (it == histkeys.end()) return std::vector(); std::string buffer = data(it->second); if (!buffer.empty()) { // The object structure can be retrieved from TFile::GetStreamerInfoList(). // Every ROOT object contains a version number which may include the byte count // for the object. The latter is currently assumed to be present to skip unused // objects. No checks are performed. The corresponding ROOT code is quite nested // but the actual readout is straight forward. // Files for reference: // root/io/io/src/TBufferFile.cxx -> ReadVersion, ReadTString, ReadObjectAny // root/core/cont/src/TArrayD.cxx -> Streamer // root/hist/hist/src/TH1.cxx -> Streamer char *buf = &buffer[0]; size_t count; Version(buf, count); // TH1(D/F/I/S/C) char* dbuf = buf + 4; Version(buf, count); // TH1 dbuf += count; Skip(buf, 4); // skip TNamed, TAttLine, TAttFill, TAttMarker std::vector r(read(buf)); // fNcells if (r.size() < 3) return std::vector(); r.front().lowedge = -std::numeric_limits::infinity(); // x-Axis char* nbuf = buf + 4; Version(buf, count); // TAxis nbuf += count; Skip(buf, 2); // skip TNamed, TAttAxis int nbins = read(buf); double xmin = read(buf); double xmax = read(buf); const size_t nborders = read(buf); if (nborders == r.size() - 1) { for (size_t i = 0; i < nborders; ++i) { r[i + 1].lowedge = read(buf); } } else { buf += sizeof(double) * nbins; const double scale = (xmax - xmin) / static_cast(nbins); for (size_t i = 0; i < r.size() - 1; ++i) { r[i + 1].lowedge = static_cast(i) * scale + xmin; } } buf = nbuf; Skip(buf, 2); // skip y-Axis and z-Axis; buf += 68; // skip 2 shorts and 8 doubles buf += read(buf) * 8; // skip fContour array if (static_cast(read(buf)) == r.size()) { for (auto& b : r) b.sumw2 = read(buf); // always double } buf = dbuf; // skip to contents of TH1(D/F/I/S/C) if (static_cast(read(buf)) == r.size()) { switch (it->second.type) { case KeyBuffer::ContentType::Double: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Float: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Int: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Short: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Byte: for (auto& b : r) b.content = read(buf); break; case KeyBuffer::ContentType::Invalid: // never reached default: break; } } return r; } else return std::vector(); } void ROOTHist::readNBins(ROOTHist::KeyBuffer& kbuffer) { std::string buffer = data(kbuffer); if (!buffer.empty()) { // The object structure can be retrieved from TFile::GetStreamerInfoList(). // Every ROOT object contains a version number which may include the byte count // for the object. The latter is currently assumed to be present to skip unused // objects. No checks are performed. The corresponding ROOT code is quite nested // but the actual readout is straight forward. // Files for reference: // root/io/io/src/TBufferFile.cxx -> ReadVersion, ReadTString, ReadObjectAny // root/core/cont/src/TArrayD.cxx -> Streamer // root/hist/hist/src/TH1.cxx -> Streamer char *buf = &buffer[0]; size_t count; Version(buf, count); // TH1(D/F/I/S/C) char* dbuf = buf + 4; Version(buf, count); // TH1 dbuf += count; Skip(buf, 4); // skip TNamed, TAttLine, TAttFill, TAttMarker kbuffer.nbins = read(buf); // fNcells } } std::string ROOTHist::data(const ROOTHist::KeyBuffer& buffer) const { std::ifstream is(filename, std::ifstream::binary); return data(buffer, is); } std::string ROOTHist::data(const ROOTHist::KeyBuffer& buffer, std::ifstream& is) const { std::string data(buffer.count, 0); is.seekg(buffer.start); if (buffer.compression == KeyBuffer::none) { is.read(&data[0], buffer.count); return data; #ifdef HAVE_ZIP } else if (buffer.compression == KeyBuffer::zlib) { std::string cdata(buffer.compressed_count, 0); is.read(&cdata[0], buffer.compressed_count); size_t luncomp = buffer.count; if (uncompress((Bytef *)data.data(), &luncomp, (Bytef *)cdata.data(), cdata.size()) == Z_OK && data.size() == luncomp) return data; } else { std::string cdata(buffer.compressed_count, 0); is.read(&cdata[0], buffer.compressed_count); if (LZ4_decompress_safe(cdata.data(), const_cast(data.data()), buffer.compressed_count, buffer.count) == static_cast(buffer.count)) return data; #endif } return std::string(); } diff --git a/src/backend/matrix/Matrix.cpp b/src/backend/matrix/Matrix.cpp index aafc9a4ba..7cb27947c 100644 --- a/src/backend/matrix/Matrix.cpp +++ b/src/backend/matrix/Matrix.cpp @@ -1,1289 +1,1290 @@ /*************************************************************************** File : Matrix.cpp Project : Matrix Description : Spreadsheet with a MxN matrix data model -------------------------------------------------------------------- Copyright : (C) 2008-2009 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2015-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017-2018 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "Matrix.h" #include "MatrixPrivate.h" #include "matrixcommands.h" #include "backend/matrix/MatrixModel.h" #include "backend/core/Folder.h" #include "backend/lib/commandtemplates.h" #include "backend/lib/XmlStreamReader.h" #include "commonfrontend/matrix/MatrixView.h" #include "kdefrontend/spreadsheet/ExportSpreadsheetDialog.h" #include #include #include #include #include #include #include /*! This class manages matrix based data (i.e., mathematically a MxN matrix with M rows, N columns). This data is typically used to for 3D plots. The values of the matrix are stored as generic values. Each column of the matrix is stored in a QVector objects. \ingroup backend */ Matrix::Matrix(AbstractScriptingEngine* engine, int rows, int cols, const QString& name, const AbstractColumn::ColumnMode mode) : AbstractDataSource(engine, name), d(new MatrixPrivate(this, mode)), m_model(nullptr), m_view(nullptr) { //set initial number of rows and columns appendColumns(cols); appendRows(rows); init(); } Matrix::Matrix(AbstractScriptingEngine* engine, const QString& name, bool loading, const AbstractColumn::ColumnMode mode) : AbstractDataSource(engine, name), d(new MatrixPrivate(this, mode)), m_model(nullptr), m_view(nullptr) { if (!loading) init(); } Matrix::~Matrix() { delete d; } void Matrix::init() { KConfig config; KConfigGroup group = config.group("Matrix"); //matrix dimension int rows = group.readEntry("RowCount", 10); int cols = group.readEntry("ColumnCount", 10); appendRows(rows); appendColumns(cols); //mapping to logical x- and y-coordinates d->xStart = group.readEntry("XStart", 0.0); d->xEnd = group.readEntry("XEnd", 1.0); d->yStart = group.readEntry("YStart", 0.0); d->yEnd = group.readEntry("YEnd", 1.0); //format QByteArray formatba = group.readEntry("NumericFormat", "f").toLatin1(); d->numericFormat = *formatba.data(); d->precision = group.readEntry("Precision", 3); d->headerFormat = (Matrix::HeaderFormat)group.readEntry("HeaderFormat", (int)Matrix::HeaderRowsColumns); } /*! Returns an icon to be used for decorating my views. */ QIcon Matrix::icon() const { return QIcon::fromTheme("labplot-matrix"); } /*! Returns a new context menu. The caller takes ownership of the menu. */ QMenu* Matrix::createContextMenu() { QMenu* menu = AbstractPart::createContextMenu(); emit requestProjectContextMenu(menu); return menu; } QWidget* Matrix::view() const { if (!m_partView) { m_view= new MatrixView(const_cast(this)); m_partView = m_view; m_model = m_view->model(); } return m_partView; } bool Matrix::exportView() const { ExportSpreadsheetDialog* dlg = new ExportSpreadsheetDialog(m_view); dlg->setFileName(name()); dlg->setMatrixMode(true); //TODO FITS filter to decide if it can be exported to both dlg->setExportTo(QStringList() << i18n("FITS image") << i18n("FITS table")); if (m_view->selectedColumnCount() == 0) { dlg->setExportSelection(false); } bool ret; if ( (ret = (dlg->exec() == QDialog::Accepted)) ) { const QString path = dlg->path(); WAIT_CURSOR; if (dlg->format() == ExportSpreadsheetDialog::LaTeX) { const bool verticalHeader = dlg->matrixVerticalHeader(); const bool horizontalHeader = dlg->matrixHorizontalHeader(); const bool latexHeader = dlg->exportHeader(); const bool gridLines = dlg->gridLines(); const bool entire = dlg->entireSpreadheet(); const bool captions = dlg->captions(); m_view->exportToLaTeX(path, verticalHeader, horizontalHeader, latexHeader, gridLines, entire, captions); } else if (dlg->format() == ExportSpreadsheetDialog::FITS) { const int exportTo = dlg->exportToFits(); m_view->exportToFits(path, exportTo ); } else { const QString separator = dlg->separator(); m_view->exportToFile(path, separator); } RESET_CURSOR; } delete dlg; return ret; } bool Matrix::printView() { QPrinter printer; QPrintDialog* dlg = new QPrintDialog(&printer, m_view); bool ret; dlg->setWindowTitle(i18nc("@title:window", "Print Matrix")); if ( (ret = (dlg->exec() == QDialog::Accepted)) ) m_view->print(&printer); delete dlg; return ret; } bool Matrix::printPreview() const { QPrintPreviewDialog* dlg = new QPrintPreviewDialog(m_view); connect(dlg, &QPrintPreviewDialog::paintRequested, m_view, &MatrixView::print); return dlg->exec(); } //############################################################################## //########################## getter methods ################################## //############################################################################## void* Matrix::data() const { return d->data; } BASIC_D_READER_IMPL(Matrix, AbstractColumn::ColumnMode, mode, mode) BASIC_D_READER_IMPL(Matrix, int, rowCount, rowCount) BASIC_D_READER_IMPL(Matrix, int, columnCount, columnCount) BASIC_D_READER_IMPL(Matrix, double, xStart, xStart) BASIC_D_READER_IMPL(Matrix, double, xEnd, xEnd) BASIC_D_READER_IMPL(Matrix, double, yStart, yStart) BASIC_D_READER_IMPL(Matrix, double, yEnd, yEnd) BASIC_D_READER_IMPL(Matrix, char, numericFormat, numericFormat) BASIC_D_READER_IMPL(Matrix, int, precision, precision) BASIC_D_READER_IMPL(Matrix, Matrix::HeaderFormat, headerFormat, headerFormat) CLASS_D_READER_IMPL(Matrix, QString, formula, formula) void Matrix::setSuppressDataChangedSignal(bool b) { if (m_model) m_model->setSuppressDataChangedSignal(b); } void Matrix::setChanged() { if (m_model) m_model->setChanged(); } //############################################################################## //################# setter methods and undo commands ########################## //############################################################################## void Matrix::setRowCount(int count) { if (count == d->rowCount) return; const int diff = count - d->rowCount; if (diff > 0) appendRows(diff); else if (diff < 0) removeRows(rowCount() + diff, -diff); } void Matrix::setColumnCount(int count) { if (count == d->columnCount) return; const int diff = count - columnCount(); if (diff > 0) appendColumns(diff); else if (diff < 0) removeColumns(columnCount() + diff, -diff); } STD_SETTER_CMD_IMPL_F_S(Matrix, SetXStart, double, xStart, updateViewHeader) void Matrix::setXStart(double xStart) { if (xStart != d->xStart) exec(new MatrixSetXStartCmd(d, xStart, ki18n("%1: x-start changed"))); } STD_SETTER_CMD_IMPL_F_S(Matrix, SetXEnd, double, xEnd, updateViewHeader) void Matrix::setXEnd(double xEnd) { if (xEnd != d->xEnd) exec(new MatrixSetXEndCmd(d, xEnd, ki18n("%1: x-end changed"))); } STD_SETTER_CMD_IMPL_F_S(Matrix, SetYStart, double, yStart, updateViewHeader) void Matrix::setYStart(double yStart) { if (yStart != d->yStart) exec(new MatrixSetYStartCmd(d, yStart, ki18n("%1: y-start changed"))); } STD_SETTER_CMD_IMPL_F_S(Matrix, SetYEnd, double, yEnd, updateViewHeader) void Matrix::setYEnd(double yEnd) { if (yEnd != d->yEnd) exec(new MatrixSetYEndCmd(d, yEnd, ki18n("%1: y-end changed"))); } STD_SETTER_CMD_IMPL_S(Matrix, SetNumericFormat, char, numericFormat) void Matrix::setNumericFormat(char format) { if (format != d->numericFormat) exec(new MatrixSetNumericFormatCmd(d, format, ki18n("%1: numeric format changed"))); } STD_SETTER_CMD_IMPL_S(Matrix, SetPrecision, int, precision) void Matrix::setPrecision(int precision) { if (precision != d->precision) exec(new MatrixSetPrecisionCmd(d, precision, ki18n("%1: precision changed"))); } //TODO: make this undoable? void Matrix::setHeaderFormat(Matrix::HeaderFormat format) { d->headerFormat = format; m_model->updateHeader(); if (m_view) m_view->resizeHeaders(); emit headerFormatChanged(format); } //columns void Matrix::insertColumns(int before, int count) { if (count < 1 || before < 0 || before > columnCount()) return; WAIT_CURSOR; exec(new MatrixInsertColumnsCmd(d, before, count)); RESET_CURSOR; } void Matrix::appendColumns(int count) { insertColumns(columnCount(), count); } void Matrix::removeColumns(int first, int count) { if (count < 1 || first < 0 || first+count > columnCount()) return; WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixRemoveColumnsCmd(d, first, count)); break; case AbstractColumn::Text: exec(new MatrixRemoveColumnsCmd(d, first, count)); break; case AbstractColumn::Integer: exec(new MatrixRemoveColumnsCmd(d, first, count)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixRemoveColumnsCmd(d, first, count)); break; } RESET_CURSOR; } void Matrix::clearColumn(int c) { WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixClearColumnCmd(d, c)); break; case AbstractColumn::Text: exec(new MatrixClearColumnCmd(d, c)); break; case AbstractColumn::Integer: exec(new MatrixClearColumnCmd(d, c)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixClearColumnCmd(d, c)); break; } RESET_CURSOR; } //rows void Matrix::insertRows(int before, int count) { if (count < 1 || before < 0 || before > rowCount()) return; WAIT_CURSOR; exec(new MatrixInsertRowsCmd(d, before, count)); RESET_CURSOR; } void Matrix::appendRows(int count) { insertRows(rowCount(), count); } void Matrix::removeRows(int first, int count) { if (count < 1 || first < 0 || first+count > rowCount()) return; WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixRemoveRowsCmd(d, first, count)); break; case AbstractColumn::Text: exec(new MatrixRemoveRowsCmd(d, first, count)); break; case AbstractColumn::Integer: exec(new MatrixRemoveRowsCmd(d, first, count)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixRemoveRowsCmd(d, first, count)); break; } RESET_CURSOR; } void Matrix::clearRow(int r) { switch (d->mode) { case AbstractColumn::Numeric: for (int c = 0; c < columnCount(); ++c) exec(new MatrixSetCellValueCmd(d, r, c, 0.0)); break; case AbstractColumn::Text: for (int c = 0; c < columnCount(); ++c) exec(new MatrixSetCellValueCmd(d, r, c, QString())); break; case AbstractColumn::Integer: for (int c = 0; c < columnCount(); ++c) exec(new MatrixSetCellValueCmd(d, r, c, 0)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int c = 0; c < columnCount(); ++c) exec(new MatrixSetCellValueCmd(d, r, c, QDateTime())); break; } } //! Return the value in the given cell (needs explicit instantiation) template T Matrix::cell(int row, int col) const { return d->cell(row, col); } template double Matrix::cell(int row, int col) const; template int Matrix::cell(int row, int col) const; template QDateTime Matrix::cell(int row, int col) const; template QString Matrix::cell(int row, int col) const; //! Return the text displayed in the given cell (needs explicit instantiation) template QString Matrix::text(int row, int col) { return QLocale().toString(cell(row,col)); } // special cases template <> QString Matrix::text(int row, int col) { return QLocale().toString(cell(row,col), d->numericFormat, d->precision); } template <> QString Matrix::text(int row, int col) { return cell(row,col); } template QString Matrix::text(int row, int col); template QString Matrix::text(int row, int col); //! Set the value of the cell (needs explicit instantiation) template void Matrix::setCell(int row, int col, T value) { if(row < 0 || row >= rowCount()) return; if(col < 0 || col >= columnCount()) return; exec(new MatrixSetCellValueCmd(d, row, col, value)); } template void Matrix::setCell(int row, int col, double value); template void Matrix::setCell(int row, int col, int value); template void Matrix::setCell(int row, int col, QDateTime value); template void Matrix::setCell(int row, int col, QString value); void Matrix::clearCell(int row, int col) { switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixSetCellValueCmd(d, row, col, 0.0)); break; case AbstractColumn::Text: exec(new MatrixSetCellValueCmd(d, row, col, QString())); break; case AbstractColumn::Integer: exec(new MatrixSetCellValueCmd(d, row, col, 0)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixSetCellValueCmd(d, row, col, QDateTime())); break; } } void Matrix::setDimensions(int rows, int cols) { if( (rows < 0) || (cols < 0 ) || (rows == rowCount() && cols == columnCount()) ) return; WAIT_CURSOR; beginMacro(i18n("%1: set matrix size to %2x%3", name(), rows, cols)); int col_diff = cols - columnCount(); if (col_diff > 0) insertColumns(columnCount(), col_diff); else if (col_diff < 0) removeColumns(columnCount() + col_diff, -col_diff); int row_diff = rows - rowCount(); if(row_diff > 0) appendRows(row_diff); else if (row_diff < 0) removeRows(rowCount() + row_diff, -row_diff); endMacro(); RESET_CURSOR; } void Matrix::copy(Matrix* other) { WAIT_CURSOR; beginMacro(i18n("%1: copy %2", name(), other->name())); int rows = other->rowCount(); int columns = other->columnCount(); setDimensions(rows, columns); for (int i=0; irowHeight(i)); for (int i=0; icolumnWidth(i)); d->suppressDataChange = true; switch (d->mode) { case AbstractColumn::Numeric: for (int i = 0; i < columns; i++) setColumnCells(i, 0, rows-1, other->columnCells(i, 0, rows-1)); break; case AbstractColumn::Text: for (int i = 0; i < columns; i++) setColumnCells(i, 0, rows-1, other->columnCells(i, 0, rows-1)); break; case AbstractColumn::Integer: for (int i = 0; i < columns; i++) setColumnCells(i, 0, rows-1, other->columnCells(i, 0, rows-1)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int i = 0; i < columns; i++) setColumnCells(i, 0, rows-1, other->columnCells(i, 0, rows-1)); break; } setCoordinates(other->xStart(), other->xEnd(), other->yStart(), other->yEnd()); setNumericFormat(other->numericFormat()); setPrecision(other->precision()); d->formula = other->formula(); d->suppressDataChange = false; emit dataChanged(0, 0, rows-1, columns-1); if (m_view) m_view->adjustHeaders(); endMacro(); RESET_CURSOR; } //! Duplicate the matrix inside its folder void Matrix::duplicate() { Matrix* matrix = new Matrix(0, rowCount(), columnCount(), name()); matrix->copy(this); if (folder()) folder()->addChild(matrix); } void Matrix::addRows() { if (!m_view) return; WAIT_CURSOR; int count = m_view->selectedRowCount(false); beginMacro(i18np("%1: add %2 rows", "%1: add %2 rows", name(), count)); exec(new MatrixInsertRowsCmd(d, rowCount(), count)); endMacro(); RESET_CURSOR; } void Matrix::addColumns() { if (!m_view) return; WAIT_CURSOR; int count = m_view->selectedRowCount(false); beginMacro(i18np("%1: add %2 column", "%1: add %2 columns", name(), count)); exec(new MatrixInsertColumnsCmd(d, columnCount(), count)); endMacro(); RESET_CURSOR; } void Matrix::setCoordinates(double x1, double x2, double y1, double y2) { exec(new MatrixSetCoordinatesCmd(d, x1, x2, y1, y2)); } void Matrix::setFormula(const QString& formula) { exec(new MatrixSetFormulaCmd(d, formula)); } //! This method should only be called by the view. /** This method does not change the view, it only changes the * values that are saved when the matrix is saved. The view * has to take care of reading and applying these values */ void Matrix::setRowHeight(int row, int height) { d->setRowHeight(row, height); } //! This method should only be called by the view. /** This method does not change the view, it only changes the * values that are saved when the matrix is saved. The view * has to take care of reading and applying these values */ void Matrix::setColumnWidth(int col, int width) { d->setColumnWidth(col, width); } int Matrix::rowHeight(int row) const { return d->rowHeight(row); } int Matrix::columnWidth(int col) const { return d->columnWidth(col); } //! Return the values in the given cells as vector template QVector Matrix::columnCells(int col, int first_row, int last_row) { return d->columnCells(col, first_row, last_row); } //! Set the values in the given cells from a double vector template void Matrix::setColumnCells(int col, int first_row, int last_row, const QVector& values) { WAIT_CURSOR; exec(new MatrixSetColumnCellsCmd(d, col, first_row, last_row, values)); RESET_CURSOR; } //! Return the values in the given cells as vector (needs explicit instantiation) template QVector Matrix::rowCells(int row, int first_column, int last_column) { return d->rowCells(row, first_column, last_column); } template QVector Matrix::rowCells(int row, int first_column, int last_column); template QVector Matrix::rowCells(int row, int first_column, int last_column); template QVector Matrix::rowCells(int row, int first_column, int last_column); template QVector Matrix::rowCells(int row, int first_column, int last_column); //! Set the values in the given cells from a double vector template void Matrix::setRowCells(int row, int first_column, int last_column, const QVector& values) { WAIT_CURSOR; exec(new MatrixSetRowCellsCmd(d, row, first_column, last_column, values)); RESET_CURSOR; } void Matrix::setData(void* data) { bool isEmpty = false; switch (d->mode) { case AbstractColumn::Numeric: if (static_cast>*>(data)->isEmpty()) isEmpty = true; break; case AbstractColumn::Text: if (static_cast>*>(data)->isEmpty()) isEmpty = true; break; case AbstractColumn::Integer: if (static_cast>*>(data)->isEmpty()) isEmpty = true; break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: if (static_cast>*>(data)->isEmpty()) isEmpty = true; break; } if (!isEmpty) exec(new MatrixReplaceValuesCmd(d, data)); } //############################################################################## //######################### Public slots ##################################### //############################################################################## //! Clear the whole matrix (i.e. reset all cells) void Matrix::clear() { WAIT_CURSOR; beginMacro(i18n("%1: clear", name())); switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixClearCmd(d)); break; case AbstractColumn::Text: exec(new MatrixClearCmd(d)); break; case AbstractColumn::Integer: exec(new MatrixClearCmd(d)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixClearCmd(d)); break; } endMacro(); RESET_CURSOR; } void Matrix::transpose() { WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixTransposeCmd(d)); break; case AbstractColumn::Text: exec(new MatrixTransposeCmd(d)); break; case AbstractColumn::Integer: exec(new MatrixTransposeCmd(d)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixTransposeCmd(d)); break; } RESET_CURSOR; } void Matrix::mirrorHorizontally() { WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixMirrorHorizontallyCmd(d)); break; case AbstractColumn::Text: exec(new MatrixMirrorHorizontallyCmd(d)); break; case AbstractColumn::Integer: exec(new MatrixMirrorHorizontallyCmd(d)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixMirrorHorizontallyCmd(d)); break; } RESET_CURSOR; } void Matrix::mirrorVertically() { WAIT_CURSOR; switch (d->mode) { case AbstractColumn::Numeric: exec(new MatrixMirrorVerticallyCmd(d)); break; case AbstractColumn::Text: exec(new MatrixMirrorVerticallyCmd(d)); break; case AbstractColumn::Integer: exec(new MatrixMirrorVerticallyCmd(d)); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: exec(new MatrixMirrorVerticallyCmd(d)); break; } RESET_CURSOR; } //############################################################################## //###################### Private implementation ############################### //############################################################################## MatrixPrivate::MatrixPrivate(Matrix* owner, const AbstractColumn::ColumnMode m) : q(owner), data(0), mode(m), rowCount(0), columnCount(0), suppressDataChange(false) { switch (mode) { case AbstractColumn::Numeric: data = new QVector>(); break; case AbstractColumn::Text: data = new QVector>(); break; case AbstractColumn::Month: case AbstractColumn::Day: case AbstractColumn::DateTime: data = new QVector>(); break; case AbstractColumn::Integer: data = new QVector>(); break; } } MatrixPrivate::~MatrixPrivate() { if (data) { switch (mode) { case AbstractColumn::Numeric: delete static_cast>*>(data); break; case AbstractColumn::Text: delete static_cast>*>(data); break; case AbstractColumn::Integer: delete static_cast>*>(data); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: delete static_cast>*>(data); break; } } } void MatrixPrivate::updateViewHeader() { q->m_view->model()->updateHeader(); } /*! Insert \count columns before column number \c before */ void MatrixPrivate::insertColumns(int before, int count) { Q_ASSERT(before >= 0); Q_ASSERT(before <= columnCount); emit q->columnsAboutToBeInserted(before, count); switch (mode) { case AbstractColumn::Numeric: for (int i = 0; i < count; i++) { static_cast>*>(data)->insert(before+i, QVector(rowCount)); columnWidths.insert(before+i, 0); } break; case AbstractColumn::Text: for (int i = 0; i < count; i++) { static_cast>*>(data)->insert(before+i, QVector(rowCount)); columnWidths.insert(before+i, 0); } break; case AbstractColumn::Integer: for (int i = 0; i < count; i++) { static_cast>*>(data)->insert(before+i, QVector(rowCount)); columnWidths.insert(before+i, 0); } break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int i = 0; i < count; i++) { static_cast>*>(data)->insert(before+i, QVector(rowCount)); columnWidths.insert(before+i, 0); } break; } columnCount += count; emit q->columnsInserted(before, count); } /*! Remove \c count columns starting with column index \c first */ void MatrixPrivate::removeColumns(int first, int count) { emit q->columnsAboutToBeRemoved(first, count); Q_ASSERT(first >= 0); Q_ASSERT(first + count <= columnCount); switch (mode) { case AbstractColumn::Numeric: (static_cast>*>(data))->remove(first, count); break; case AbstractColumn::Text: (static_cast>*>(data))->remove(first, count); break; case AbstractColumn::Integer: (static_cast>*>(data))->remove(first, count); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: (static_cast>*>(data))->remove(first, count); break; } for (int i = 0; i < count; i++) columnWidths.remove(first); columnCount -= count; emit q->columnsRemoved(first, count); } /*! Insert \c count rows before row with the index \c before */ void MatrixPrivate::insertRows(int before, int count) { emit q->rowsAboutToBeInserted(before, count); Q_ASSERT(before >= 0); Q_ASSERT(before <= rowCount); switch (mode) { case AbstractColumn::Numeric: for (int col = 0; col < columnCount; col++) for (int i = 0; i < count; i++) (static_cast>*>(data))->operator[](col).insert(before+i, 0.0); break; case AbstractColumn::Text: for (int col = 0; col < columnCount; col++) for (int i = 0; i < count; i++) (static_cast>*>(data))->operator[](col).insert(before+i, QString()); break; case AbstractColumn::Integer: for (int col = 0; col < columnCount; col++) for (int i = 0; i < count; i++) (static_cast>*>(data))->operator[](col).insert(before+i, 0); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int col = 0; col < columnCount; col++) for (int i = 0; i < count; i++) (static_cast>*>(data))->operator[](col).insert(before+i, QDateTime()); } for(int i=0; irowsInserted(before, count); } /*! Remove \c count columns starting from the column with index \c first */ void MatrixPrivate::removeRows(int first, int count) { emit q->rowsAboutToBeRemoved(first, count); Q_ASSERT(first >= 0); Q_ASSERT(first+count <= rowCount); switch (mode) { case AbstractColumn::Numeric: for (int col = 0; col < columnCount; col++) (static_cast>*>(data))->operator[](col).remove(first, count); break; case AbstractColumn::Text: for (int col = 0; col < columnCount; col++) (static_cast>*>(data))->operator[](col).remove(first, count); break; case AbstractColumn::Integer: for (int col = 0; col < columnCount; col++) (static_cast>*>(data))->operator[](col).remove(first, count); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int col = 0; col < columnCount; col++) (static_cast>*>(data))->operator[](col).remove(first, count); break; } for (int i = 0; i < count; i++) rowHeights.remove(first); rowCount -= count; emit q->rowsRemoved(first, count); } //! Fill column with zeroes void MatrixPrivate::clearColumn(int col) { switch (mode) { case AbstractColumn::Numeric: static_cast>*>(data)->operator[](col).fill(0.0); break; case AbstractColumn::Text: static_cast>*>(data)->operator[](col).fill(QString()); break; case AbstractColumn::Integer: static_cast>*>(data)->operator[](col).fill(0); break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: static_cast>*>(data)->operator[](col).fill(QDateTime()); break; } if (!suppressDataChange) emit q->dataChanged(0, col, rowCount-1, col); } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## void Matrix::save(QXmlStreamWriter* writer) const { writer->writeStartElement("matrix"); writeBasicAttributes(writer); writeCommentElement(writer); //formula writer->writeStartElement("formula"); writer->writeCharacters(d->formula); writer->writeEndElement(); //format writer->writeStartElement("format"); writer->writeAttribute("mode", QString::number(d->mode)); writer->writeAttribute("headerFormat", QString::number(d->headerFormat)); writer->writeAttribute("numericFormat", QString(QChar(d->numericFormat))); writer->writeAttribute("precision", QString::number(d->precision)); writer->writeEndElement(); //dimensions writer->writeStartElement("dimension"); writer->writeAttribute("columns", QString::number(d->columnCount)); writer->writeAttribute("rows", QString::number(d->rowCount)); writer->writeAttribute("x_start", QString::number(d->xStart)); writer->writeAttribute("x_end", QString::number(d->xEnd)); writer->writeAttribute("y_start", QString::number(d->yStart)); writer->writeAttribute("y_end", QString::number(d->yEnd)); writer->writeEndElement(); //vector with row heights writer->writeStartElement("row_heights"); const char* data = reinterpret_cast(d->rowHeights.constData()); int size = d->rowHeights.size() * sizeof(int); writer->writeCharacters(QByteArray::fromRawData(data,size).toBase64()); writer->writeEndElement(); //vector with column widths writer->writeStartElement("column_widths"); data = reinterpret_cast(d->columnWidths.constData()); size = d->columnWidths.size()*sizeof(int); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); //columns switch (d->mode) { case AbstractColumn::Numeric: size = d->rowCount*sizeof(double); for (int i = 0; i < d->columnCount; ++i) { data = reinterpret_cast(static_cast>*>(d->data)->at(i).constData()); writer->writeStartElement("column"); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); } break; case AbstractColumn::Text: size = d->rowCount*sizeof(QString); for (int i = 0; i < d->columnCount; ++i) { data = reinterpret_cast(static_cast>*>(d->data)->at(i).constData()); writer->writeStartElement("column"); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); } break; case AbstractColumn::Integer: size = d->rowCount*sizeof(int); for (int i = 0; i < d->columnCount; ++i) { data = reinterpret_cast(static_cast>*>(d->data)->at(i).constData()); writer->writeStartElement("column"); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); } break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: size = d->rowCount*sizeof(QDateTime); for (int i = 0; i < d->columnCount; ++i) { data = reinterpret_cast(static_cast>*>(d->data)->at(i).constData()); writer->writeStartElement("column"); writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); writer->writeEndElement(); } break; } writer->writeEndElement(); // "matrix" } bool Matrix::load(XmlStreamReader* reader, bool preview) { if (!readBasicAttributes(reader)) return false; KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs; QString str; // read child elements while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement() && reader->name() == "matrix") break; if (!reader->isStartElement()) continue; if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if(!preview && reader->name() == "formula") { d->formula = reader->text().toString().trimmed(); } else if (!preview && reader->name() == "format") { attribs = reader->attributes(); str = attribs.value("mode").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("mode").toString()); else d->mode = AbstractColumn::ColumnMode(str.toInt()); str = attribs.value("headerFormat").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("headerFormat").toString()); else d->headerFormat = Matrix::HeaderFormat(str.toInt()); str = attribs.value("numericFormat").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("numericFormat").toString()); else { QByteArray formatba = str.toLatin1(); d->numericFormat = *formatba.data(); } str = attribs.value("precision").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("precision").toString()); else d->precision = str.toInt(); } else if (!preview && reader->name() == "dimension") { attribs = reader->attributes(); str = attribs.value("columns").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("columns").toString()); else d->columnCount = str.toInt(); str = attribs.value("rows").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("rows").toString()); else d->rowCount = str.toInt(); str = attribs.value("x_start").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("x_start").toString()); else d->xStart = str.toDouble(); str = attribs.value("x_end").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("x_end").toString()); else d->xEnd = str.toDouble(); str = attribs.value("y_start").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("y_start").toString()); else d->yStart = str.toDouble(); str = attribs.value("y_end").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.subs("y_end").toString()); else d->yEnd = str.toDouble(); } else if (!preview && reader->name() == "row_heights") { reader->readNext(); QString content = reader->text().toString().trimmed(); QByteArray bytes = QByteArray::fromBase64(content.toAscii()); int count = bytes.size()/sizeof(int); d->rowHeights.resize(count); memcpy(d->rowHeights.data(), bytes.data(), count*sizeof(int)); } else if (!preview && reader->name() == "column_widths") { reader->readNext(); QString content = reader->text().toString().trimmed(); QByteArray bytes = QByteArray::fromBase64(content.toAscii()); int count = bytes.size()/sizeof(int); d->columnWidths.resize(count); memcpy(d->columnWidths.data(), bytes.data(), count*sizeof(int)); } else if (!preview && reader->name() == "column") { //TODO: parallelize reading of columns? reader->readNext(); QString content = reader->text().toString().trimmed(); QByteArray bytes = QByteArray::fromBase64(content.toAscii()); switch (d->mode) { case AbstractColumn::Numeric: { int count = bytes.size()/sizeof(double); QVector column; column.resize(count); memcpy(column.data(), bytes.data(), count*sizeof(double)); static_cast>*>(d->data)->append(column); break; } case AbstractColumn::Text: { int count = bytes.size()/sizeof(QString); QVector column; column.resize(count); //TODO: warning (GCC8): writing to an object of type 'class QDateTime' with no trivial copy-assignment; use copy-assignment or copy-initialization instead memcpy(column.data(), bytes.data(), count*sizeof(QString)); static_cast>*>(d->data)->append(column); break; } case AbstractColumn::Integer: { int count = bytes.size()/sizeof(int); QVector column; column.resize(count); memcpy(column.data(), bytes.data(), count*sizeof(int)); static_cast>*>(d->data)->append(column); break; } case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: { int count = bytes.size()/sizeof(QDateTime); QVector column; column.resize(count); //TODO: warning (GCC8): writing to an object of type 'class QDateTime' with no trivial copy-assignment; use copy-assignment or copy-initialization instead memcpy(column.data(), bytes.data(), count*sizeof(QDateTime)); static_cast>*>(d->data)->append(column); break; } } } else { // unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } return true; } //############################################################################## //######################## Data Import ####################################### //############################################################################## int Matrix::prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode mode, int actualRows, int actualCols, QStringList colNameList, QVector columnMode) { QDEBUG("prepareImport() rows =" << actualRows << " cols =" << actualCols); Q_UNUSED(colNameList); int columnOffset = 0; setUndoAware(false); setSuppressDataChangedSignal(true); // resize the matrix if (mode == AbstractFileFilter::Replace) { clear(); setDimensions(actualRows, actualCols); } else { if (rowCount() < actualRows) setDimensions(actualRows, actualCols); else setDimensions(rowCount(), actualCols); } // data() returns a void* which is a pointer to a matrix of any data type (see ColumnPrivate.cpp) dataContainer.resize(actualCols); switch (columnMode[0]) { // only columnMode[0] is used case AbstractColumn::Numeric: for (int n = 0; n < actualCols; n++) { QVector* vector = &(static_cast>*>(data())->operator[](n)); vector->reserve(actualRows); vector->resize(actualRows); dataContainer[n] = static_cast(vector); } d->mode = AbstractColumn::Numeric; break; case AbstractColumn::Integer: for (int n = 0; n < actualCols; n++) { QVector* vector = &(static_cast>*>(data())->operator[](n)); vector->reserve(actualRows); vector->resize(actualRows); dataContainer[n] = static_cast(vector); } d->mode = AbstractColumn::Integer; break; case AbstractColumn::Text: for (int n = 0; n < actualCols; n++) { QVector* vector = &(static_cast>*>(data())->operator[](n)); vector->reserve(actualRows); vector->resize(actualRows); dataContainer[n] = static_cast(vector); } d->mode = AbstractColumn::Text; break; case AbstractColumn::Day: case AbstractColumn::Month: case AbstractColumn::DateTime: for (int n = 0; n < actualCols; n++) { QVector* vector = &(static_cast>*>(data())->operator[](n)); vector->reserve(actualRows); vector->resize(actualRows); dataContainer[n] = static_cast(vector); } d->mode = AbstractColumn::DateTime; break; } return columnOffset; } -void Matrix::finalizeImport(int columnOffset, int startColumn, int endColumn, const QString& dateTimeFormat, AbstractFileFilter::ImportMode importMode) { +void Matrix::finalizeImport(int columnOffset, int startColumn, int endColumn, int numRows, const QString& dateTimeFormat, AbstractFileFilter::ImportMode importMode) { DEBUG("Matrix::finalizeImport()"); Q_UNUSED(columnOffset); Q_UNUSED(startColumn); Q_UNUSED(endColumn); + Q_UNUSED(numRows); Q_UNUSED(dateTimeFormat); Q_UNUSED(importMode); setSuppressDataChangedSignal(false); setChanged(); setUndoAware(true); DEBUG("Matrix::finalizeImport() DONE"); } diff --git a/src/backend/matrix/Matrix.h b/src/backend/matrix/Matrix.h index e81ef1de4..969316e75 100644 --- a/src/backend/matrix/Matrix.h +++ b/src/backend/matrix/Matrix.h @@ -1,175 +1,175 @@ /*************************************************************************** File : Matrix.h Project : Matrix Description : Spreadsheet with a MxN matrix data model -------------------------------------------------------------------- Copyright : (C) 2008-2009 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2015-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef MATRIX_H #define MATRIX_H #include "backend/datasources/AbstractDataSource.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/lib/macros.h" class MatrixPrivate; class MatrixModel; class MatrixView; class Matrix : public AbstractDataSource { Q_OBJECT Q_ENUMS(HeaderFormat) public: enum HeaderFormat {HeaderRowsColumns, HeaderValues, HeaderRowsColumnsValues}; Matrix(AbstractScriptingEngine* engine, const QString& name, bool loading = false, const AbstractColumn::ColumnMode = AbstractColumn::Numeric); Matrix(AbstractScriptingEngine* engine, int rows, int cols, const QString& name, const AbstractColumn::ColumnMode = AbstractColumn::Numeric); ~Matrix() override; QIcon icon() const override; QMenu* createContextMenu() override; QWidget* view() const override; bool exportView() const override; bool printView() override; bool printPreview() const override; void* data() const; void setData(void*); BASIC_D_ACCESSOR_DECL(AbstractColumn::ColumnMode, mode, Mode) BASIC_D_ACCESSOR_DECL(int, rowCount, RowCount) BASIC_D_ACCESSOR_DECL(int, columnCount, ColumnCount) BASIC_D_ACCESSOR_DECL(char, numericFormat, NumericFormat) BASIC_D_ACCESSOR_DECL(int, precision, Precision) BASIC_D_ACCESSOR_DECL(HeaderFormat, headerFormat, HeaderFormat) BASIC_D_ACCESSOR_DECL(double, xStart, XStart) BASIC_D_ACCESSOR_DECL(double, xEnd, XEnd) BASIC_D_ACCESSOR_DECL(double, yStart, YStart) BASIC_D_ACCESSOR_DECL(double, yEnd, YEnd) CLASS_D_ACCESSOR_DECL(QString, formula, Formula) void setSuppressDataChangedSignal(bool); void setChanged(); int rowHeight(int row) const; void setRowHeight(int row, int height); int columnWidth(int col) const; void setColumnWidth(int col, int width); void setDimensions(int rows, int cols); void setCoordinates(double x1, double x2, double y1, double y2); void insertColumns(int before, int count); void appendColumns(int count); void removeColumns(int first, int count); void clearColumn(int); void insertRows(int before, int count); void appendRows(int count); void removeRows(int first, int count); void clearRow(int); template T cell(int row, int col) const; template QString text(int row, int col); template void setCell(int row, int col, T value); void clearCell(int row, int col); template QVector columnCells(int col, int first_row, int last_row); template void setColumnCells(int col, int first_row, int last_row, const QVector& values); template QVector rowCells(int row, int first_column, int last_column); template void setRowCells(int row, int first_column, int last_column, const QVector& values); void copy(Matrix* other); void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*, bool preview) override; int prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode, int rows, int cols, QStringList colNameList, QVector) override; - void finalizeImport(int columnOffset, int startColumn, int endColumn, + void finalizeImport(int columnOffset, int startColumn, int endColumn, int numRows, const QString& dateTimeFormat, AbstractFileFilter::ImportMode) override; typedef MatrixPrivate Private; public slots: void clear(); void transpose(); void mirrorVertically(); void mirrorHorizontally(); void addColumns(); void addRows(); void duplicate(); signals: void requestProjectContextMenu(QMenu*); void columnsAboutToBeInserted(int before, int count); void columnsInserted(int first, int count); void columnsAboutToBeRemoved(int first, int count); void columnsRemoved(int first, int count); void rowsAboutToBeInserted(int before, int count); void rowsInserted(int first, int count); void rowsAboutToBeRemoved(int first, int count); void rowsRemoved(int first, int count); void dataChanged(int top, int left, int bottom, int right); void coordinatesChanged(); friend class MatrixInsertRowsCmd; friend class MatrixInsertColumnsCmd; void rowCountChanged(int); void columnCountChanged(int); friend class MatrixSetXStartCmd; friend class MatrixSetXEndCmd; friend class MatrixSetYStartCmd; friend class MatrixSetYEndCmd; void xStartChanged(double); void xEndChanged(double); void yStartChanged(double); void yEndChanged(double); friend class MatrixSetNumericFormatCmd; friend class MatrixSetPrecisionCmd; void numericFormatChanged(char); void precisionChanged(int); void headerFormatChanged(Matrix::HeaderFormat); private: void init(); MatrixPrivate* const d; mutable MatrixModel* m_model; mutable MatrixView* m_view; friend class MatrixPrivate; template friend class MatrixRemoveColumnsCmd; template friend class MatrixRemoveRowsCmd; }; #endif diff --git a/src/backend/spreadsheet/Spreadsheet.cpp b/src/backend/spreadsheet/Spreadsheet.cpp index 836168d6a..cb58db110 100644 --- a/src/backend/spreadsheet/Spreadsheet.cpp +++ b/src/backend/spreadsheet/Spreadsheet.cpp @@ -1,943 +1,945 @@ /*************************************************************************** File : Spreadsheet.cpp Project : LabPlot Description : Aspect providing a spreadsheet table with column logic -------------------------------------------------------------------- Copyright : (C) 2006-2008 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2006-2009 Knut Franke (knut.franke@gmx.de) Copyright : (C) 2012-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "Spreadsheet.h" #include "backend/core/AspectPrivate.h" #include "backend/core/AbstractAspect.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/datatypes/DateTime2StringFilter.h" #include "commonfrontend/spreadsheet/SpreadsheetView.h" #include #include #include #include #include /*! \class Spreadsheet \brief Aspect providing a spreadsheet table with column logic. Spreadsheet is a container object for columns with no data of its own. By definition, it's columns are all of its children inheriting from class Column. Thus, the basic API is already defined by AbstractAspect (managing the list of columns, notification of column insertion/removal) and Column (changing and monitoring state of the actual data). Spreadsheet stores a pointer to its primary view of class SpreadsheetView. SpreadsheetView calls the Spreadsheet API but Spreadsheet only notifies SpreadsheetView by signals without calling its API directly. This ensures a maximum independence of UI and backend. SpreadsheetView can be easily replaced by a different class. User interaction is completely handled in SpreadsheetView and translated into Spreadsheet API calls (e.g., when a user edits a cell this will be handled by the delegate of SpreadsheetView and Spreadsheet will not know whether a script or a user changed the data.). All actions, menus etc. for the user interaction are handled SpreadsheetView, e.g., via a context menu. Selections are also handled by SpreadsheetView. The view itself is created by the first call to view(); \ingroup backend */ Spreadsheet::Spreadsheet(AbstractScriptingEngine* engine, const QString& name, bool loading) : AbstractDataSource(engine, name), m_view(nullptr) { if (!loading) init(); } /*! initializes the spreadsheet with the default number of columns and rows */ void Spreadsheet::init() { KConfigGroup group = KSharedConfig::openConfig()->group(QLatin1String("Spreadsheet")); const int columns = group.readEntry(QLatin1String("ColumnCount"), 2); const int rows = group.readEntry(QLatin1String("RowCount"), 100); for (int i = 0; i < columns; i++) { Column* new_col = new Column(QString::number(i+1), AbstractColumn::Numeric); new_col->setPlotDesignation(i == 0 ? AbstractColumn::X : AbstractColumn::Y); addChild(new_col); } setRowCount(rows); } /*! Constructs a primary view on me. This method may be called multiple times during the life time of an Aspect, or it might not get called at all. Aspects must not depend on the existence of a view for their operation. */ QWidget* Spreadsheet::view() const { if (!m_partView) { m_view = new SpreadsheetView(const_cast(this)); m_partView = m_view; } return m_partView; } bool Spreadsheet::exportView() const { return m_view->exportView(); } bool Spreadsheet::printView() { return m_view->printView(); } bool Spreadsheet::printPreview() const { return m_view->printPreview(); } /*! Returns the maximum number of rows in the spreadsheet. */ int Spreadsheet::rowCount() const { int col_rows, result = 0; for (auto* col: children()) if ((col_rows = col->rowCount()) > result) result = col_rows; return result; } void Spreadsheet::removeRows(int first, int count) { if( count < 1 || first < 0 || first+count > rowCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: remove 1 row", "%1: remove %2 rows", name(), count) ); for (auto* col: children(IncludeHidden)) col->removeRows(first, count); endMacro(); RESET_CURSOR; } void Spreadsheet::insertRows(int before, int count) { if( count < 1 || before < 0 || before > rowCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: insert 1 row", "%1: insert %2 rows", name(), count) ); for (auto* col: children(IncludeHidden)) col->insertRows(before, count); endMacro(); RESET_CURSOR; } void Spreadsheet::appendRows(int count) { insertRows(rowCount(), count); } void Spreadsheet::appendRow() { insertRows(rowCount(), 1); } void Spreadsheet::appendColumns(int count) { insertColumns(columnCount(), count); } void Spreadsheet::appendColumn() { insertColumns(columnCount(), 1); } void Spreadsheet::prependColumns(int count) { insertColumns(0, count); } /*! Sets the number of rows of the spreadsheet to \c new_size */ void Spreadsheet::setRowCount(int new_size) { int current_size = rowCount(); if (new_size > current_size) insertRows(current_size, new_size-current_size); if (new_size < current_size && new_size >= 0) removeRows(new_size, current_size-new_size); } /*! Returns the column with the number \c index. Shallow wrapper around \sa AbstractAspect::child() - see there for caveat. */ Column* Spreadsheet::column(int index) const { return child(index); } /*! Returns the column with the name \c name. */ Column* Spreadsheet::column(const QString &name) const { return child(name); } /*! Returns the total number of columns in the spreadsheet. */ int Spreadsheet::columnCount() const { return childCount(); } /*! Returns the number of columns matching the given designation. */ int Spreadsheet::columnCount(AbstractColumn::PlotDesignation pd) const { int count = 0; for (auto* col: children()) if (col->plotDesignation() == pd) count++; return count; } void Spreadsheet::removeColumns(int first, int count) { if( count < 1 || first < 0 || first+count > columnCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: remove 1 column", "%1: remove %2 columns", name(), count) ); for (int i = 0; i < count; i++) child(first)->remove(); endMacro(); RESET_CURSOR; } void Spreadsheet::insertColumns(int before, int count) { WAIT_CURSOR; beginMacro( i18np("%1: insert 1 column", "%1: insert %2 columns", name(), count) ); Column * before_col = column(before); int rows = rowCount(); for (int i = 0; i < count; i++) { Column * new_col = new Column(QString::number(i+1), AbstractColumn::Numeric); new_col->setPlotDesignation(AbstractColumn::Y); new_col->insertRows(0, rows); insertChildBefore(new_col, before_col); } endMacro(); RESET_CURSOR; } /*! Sets the number of columns to \c new_size */ void Spreadsheet::setColumnCount(int new_size) { int old_size = columnCount(); if ( old_size == new_size || new_size < 0 ) return; if (new_size < old_size) removeColumns(new_size, old_size-new_size); else insertColumns(old_size, new_size-old_size); } /*! Clears the whole spreadsheet. */ void Spreadsheet::clear() { WAIT_CURSOR; beginMacro(i18n("%1: clear", name())); for (auto* col: children()) col->clear(); endMacro(); RESET_CURSOR; } /*! Clears all mask in the spreadsheet. */ void Spreadsheet::clearMasks() { WAIT_CURSOR; beginMacro(i18n("%1: clear all masks", name())); for (auto* col: children()) col->clearMasks(); endMacro(); RESET_CURSOR; } /*! Returns a new context menu. The caller takes ownership of the menu. */ QMenu* Spreadsheet::createContextMenu() { QMenu* menu = AbstractPart::createContextMenu(); Q_ASSERT(menu); emit requestProjectContextMenu(menu); return menu; } void Spreadsheet::moveColumn(int from, int to) { Column* col = child(from); beginMacro(i18n("%1: move column %2 from position %3 to %4.", name(), col->name(), from+1, to+1)); col->remove(); insertChildBefore(col, child(to)); endMacro(); } void Spreadsheet::copy(Spreadsheet* other) { WAIT_CURSOR; beginMacro(i18n("%1: copy %2", name(), other->name())); for (auto* col: children()) col->remove(); for (auto* src_col: other->children()) { Column * new_col = new Column(src_col->name(), src_col->columnMode()); new_col->copy(src_col); new_col->setPlotDesignation(src_col->plotDesignation()); QVector< Interval > masks = src_col->maskedIntervals(); for (const auto& iv: masks) new_col->setMasked(iv); QVector< Interval > formulas = src_col->formulaIntervals(); for (const auto& iv: formulas) new_col->setFormula(iv, src_col->formula(iv.start())); new_col->setWidth(src_col->width()); addChild(new_col); } setComment(other->comment()); endMacro(); RESET_CURSOR; } // FIXME: replace index-based API with Column*-based one /*! Determines the corresponding X column. */ int Spreadsheet::colX(int col) { for(int i = col-1; i >= 0; i--) { if (column(i)->plotDesignation() == AbstractColumn::X) return i; } int cols = columnCount(); for(int i = col+1; i < cols; i++) { if (column(i)->plotDesignation() == AbstractColumn::X) return i; } return -1; } /*! Determines the corresponding Y column. */ int Spreadsheet::colY(int col) { int cols = columnCount(); if (column(col)->plotDesignation() == AbstractColumn::XError || column(col)->plotDesignation() == AbstractColumn::YError) { // look to the left first for(int i=col-1; i>=0; i--) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } for(int i=col+1; iplotDesignation() == AbstractColumn::Y) return i; } } else { // look to the right first for(int i=col+1; iplotDesignation() == AbstractColumn::Y) return i; } for(int i=col-1; i>=0; i--) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } } return -1; } /*! Sorts the given list of column. If 'leading' is a null pointer, each column is sorted separately. */ void Spreadsheet::sortColumns(Column* leading, QVector cols, bool ascending) { if(cols.isEmpty()) return; // the normal QPair comparison does not work properly with descending sorting // thefore we use our own compare functions class CompareFunctions { public: static bool doubleLess(const QPair& a, const QPair& b) { return a.first < b.first; } static bool doubleGreater(const QPair& a, const QPair& b) { return a.first > b.first; } static bool integerLess(const QPair& a, const QPair& b) { return a.first < b.first; } static bool integerGreater(const QPair& a, const QPair& b) { return a.first > b.first; } static bool QStringLess(const QPair& a, const QPair& b) { return a < b; } static bool QStringGreater(const QPair& a, const QPair& b) { return a > b; } static bool QDateTimeLess(const QPair& a, const QPair& b) { return a < b; } static bool QDateTimeGreater(const QPair& a, const QPair& b) { return a > b; } }; WAIT_CURSOR; beginMacro(i18n("%1: sort columns", name())); if(leading == 0) { // sort separately for (auto* col: cols) { switch (col->columnMode()) { case AbstractColumn::Numeric: { int rows = col->rowCount(); QVector< QPair > map; for(int j=0; j(col->valueAt(j), j)); if(ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::Integer: { int rows = col->rowCount(); QVector< QPair > map; for (int j = 0; j < rows; j++) map.append(QPair(col->valueAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator> it(map); Column* temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::Text: { int rows = col->rowCount(); QVector> map; for (int j = 0; j < rows; j++) map.append(QPair(col->textAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringGreater); QVectorIterator< QPair > it(map); Column* temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { int rows = col->rowCount(); QVector< QPair > map; for(int j=0; j(col->dateTimeAt(j), j)); if(ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeGreater); QVectorIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } } } } else { // sort with leading column switch (leading->columnMode()) { case AbstractColumn::Numeric: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->valueAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator> it(map); for (auto* col: cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::Integer: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->valueAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::integerLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::integerGreater); QVectorIterator> it(map); for (auto* col: cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::Text: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->textAt(i), i)); if(ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringGreater); QVectorIterator> it(map); for (auto* col: cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->dateTimeAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeGreater); QVectorIterator> it(map); for (auto* col: cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } } } endMacro(); RESET_CURSOR; } // end of sortColumns() /*! Returns an icon to be used for decorating my views. */ QIcon Spreadsheet::icon() const { return QIcon::fromTheme("labplot-spreadsheet"); } /*! Returns the text displayed in the given cell. */ QString Spreadsheet::text(int row, int col) const { Column* c = column(col); if(!c) return QString(); return c->asStringColumn()->textAt(row); } /*! * This slot is, indirectly, called when a child of \c Spreadsheet (i.e. column) was selected in \c ProjectExplorer. * Emits the signal \c columnSelected that is handled in \c SpreadsheetView. */ void Spreadsheet::childSelected(const AbstractAspect* aspect) { const Column* column = qobject_cast(aspect); if (column) { int index = indexOfChild(column); emit columnSelected(index); } } /*! * This slot is, indirectly, called when a child of \c Spreadsheet (i.e. column) was deselected in \c ProjectExplorer. * Emits the signal \c columnDeselected that is handled in \c SpreadsheetView. */ void Spreadsheet::childDeselected(const AbstractAspect* aspect) { const Column* column = qobject_cast(aspect); if (column) { int index = indexOfChild(column); emit columnDeselected(index); } } /*! * Emits the signal to select or to deselect the column number \c index in the project explorer, * if \c selected=true or \c selected=false, respectively. * The signal is handled in \c AspectTreeModel and forwarded to the tree view in \c ProjectExplorer. * This function is called in \c SpreadsheetView upon selection changes. */ void Spreadsheet::setColumnSelectedInView(int index, bool selected) { if (selected) { emit childAspectSelectedInView(child(index)); //deselect the spreadsheet in the project explorer, if a child (column) was selected. //prevents unwanted multiple selection with spreadsheet (if it was selected before). emit childAspectDeselectedInView(this); } else emit childAspectDeselectedInView(child(index)); } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void Spreadsheet::save(QXmlStreamWriter* writer) const { writer->writeStartElement("spreadsheet"); writeBasicAttributes(writer); writeCommentElement(writer); //columns for (auto* col: children(IncludeHidden)) col->save(writer); writer->writeEndElement(); // "spreadsheet" } /*! Loads from XML. */ bool Spreadsheet::load(XmlStreamReader* reader, bool preview) { if (!readBasicAttributes(reader)) return false; // read child elements while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement()) break; if (reader->isStartElement()) { if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if(reader->name() == "column") { Column* column = new Column(""); if (!column->load(reader, preview)) { delete column; setColumnCount(0); return false; } addChildFast(column); } else { // unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } } return !reader->hasError(); } //############################################################################## //######################## Data Import ####################################### //############################################################################## int Spreadsheet::prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode importMode, int actualRows, int actualCols, QStringList colNameList, QVector columnMode) { DEBUG("Spreadsheet::prepareImport()"); DEBUG(" resize spreadsheet to rows = " << actualRows << "and cols = " << actualCols); int columnOffset = 0; setUndoAware(false); //make the available columns undo unaware before we resize and rename them below, //the same will be done for new columns in this->resize(). for (int i = 0; i < childCount(); i++) child(i)->setUndoAware(false); columnOffset = this->resize(importMode, colNameList, actualCols); // resize the spreadsheet if (importMode == AbstractFileFilter::Replace) { clear(); setRowCount(actualRows); } else { if (rowCount() < actualRows) setRowCount(actualRows); } if (columnMode.size() < actualCols) { qWarning("columnMode[] size is too small! Giving up."); return -1; } dataContainer.resize(actualCols); for (int n = 0; n < actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) Column* column = this->child(columnOffset+n); DEBUG(" column " << n << " columnMode = " << columnMode[n]); column->setColumnMode(columnMode[n]); //in the most cases the first imported column is meant to be used as x-data. //Other columns provide mostly y-data or errors. //TODO: this has to be configurable for the user in the import widget, //it should be possible to specify x-error plot designation, etc. AbstractColumn::PlotDesignation desig = (n == 0) ? AbstractColumn::X : AbstractColumn::Y; column->setPlotDesignation(desig); switch (columnMode[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast*>(column->data()); - vector->reserve(actualRows); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast*>(column->data()); - vector->reserve(actualRows); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(column->data()); - vector->reserve(actualRows); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Month: case AbstractColumn::Day: case AbstractColumn::DateTime: { QVector* vector = static_cast* >(column->data()); - vector->reserve(actualRows); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } } } // QDEBUG("dataPointers =" << dataPointers); DEBUG("Spreadsheet::prepareImport() DONE"); return columnOffset; } /*! resize data source to cols columns returns column offset depending on import mode */ int Spreadsheet::resize(AbstractFileFilter::ImportMode mode, QStringList colNameList, int cols) { // name additional columns for (int k = colNameList.size(); k < cols; k++ ) colNameList.append( "Column " + QString::number(k+1) ); int columnOffset = 0; //indexes the "start column" in the spreadsheet. Starting from this column the data will be imported. Column* newColumn = 0; if (mode == AbstractFileFilter::Append) { columnOffset = childCount(); for (int n = 0; n < cols; n++ ) { newColumn = new Column(colNameList.at(n), AbstractColumn::Numeric); newColumn->setUndoAware(false); addChildFast(newColumn); } } else if (mode == AbstractFileFilter::Prepend) { Column* firstColumn = child(0); for (int n = 0; n < cols; n++ ) { newColumn = new Column(colNameList.at(n), AbstractColumn::Numeric); newColumn->setUndoAware(false); insertChildBeforeFast(newColumn, firstColumn); } } else if (mode == AbstractFileFilter::Replace) { //replace completely the previous content of the data source with the content to be imported. int columns = childCount(); if (columns > cols) { //there're more columns in the data source then required -> remove the superfluous columns for (int i = 0; i < columns-cols; i++) removeChild(child(0)); } else { //create additional columns if needed for (int i = columns; i < cols; i++) { newColumn = new Column(colNameList.at(i), AbstractColumn::Numeric); newColumn->setUndoAware(false); addChildFast(newColumn); } } //rename the columns that are already available and supress the dataChanged signal for them for (int i = 0; i < childCount(); i++) { if (mode == AbstractFileFilter::Replace) child(i)->setSuppressDataChangedSignal(true); child(i)->setName(colNameList.at(i)); } } return columnOffset; } -void Spreadsheet::finalizeImport(int columnOffset, int startColumn, int endColumn, const QString& dateTimeFormat, AbstractFileFilter::ImportMode importMode) { +void Spreadsheet::finalizeImport(int columnOffset, int startColumn, int endColumn, int numRows, const QString& dateTimeFormat, AbstractFileFilter::ImportMode importMode) { DEBUG("Spreadsheet::finalizeImport()"); + // shrink the spreadsheet if needed + if (numRows > 0 && numRows != rowCount()) { + if (importMode == AbstractFileFilter::Replace) + setRowCount(numRows); + } + // set the comments for each of the columns if datasource is a spreadsheet const int rows = rowCount(); for (int n = startColumn; n <= endColumn; n++) { Column* column = this->column(columnOffset + n - startColumn); //DEBUG(" column " << n << " of type " << column->columnMode()); QString comment; switch (column->columnMode()) { case AbstractColumn::Numeric: comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); break; case AbstractColumn::Integer: comment = i18np("integer data, %1 element", "integer data, %1 elements", rows); break; case AbstractColumn::Text: comment = i18np("text data, %1 element", "text data, %1 elements", rows); break; case AbstractColumn::Month: comment = i18np("month data, %1 element", "month data, %1 elements", rows); break; case AbstractColumn::Day: comment = i18np("day data, %1 element", "day data, %1 elements", rows); break; case AbstractColumn::DateTime: comment = i18np("date and time data, %1 element", "date and time data, %1 elements", rows); // set same datetime format in column DateTime2StringFilter* filter = static_cast(column->outputFilter()); filter->setFormat(dateTimeFormat); } column->setComment(comment); if (importMode == AbstractFileFilter::Replace) { column->setSuppressDataChangedSignal(false); column->setChanged(); } } //make the spreadsheet and all its children undo aware again setUndoAware(true); for (int i = 0; i < childCount(); i++) child(i)->setUndoAware(true); if (m_partView != nullptr) m_view->resizeHeader(); DEBUG("Spreadsheet::finalizeImport() DONE"); } diff --git a/src/backend/spreadsheet/Spreadsheet.h b/src/backend/spreadsheet/Spreadsheet.h index 223da7b8c..355eb1bb7 100644 --- a/src/backend/spreadsheet/Spreadsheet.h +++ b/src/backend/spreadsheet/Spreadsheet.h @@ -1,120 +1,120 @@ /*************************************************************************** File : Spreadsheet.h Project : LabPlot Description : Aspect providing a spreadsheet table with column logic -------------------------------------------------------------------- Copyright : (C) 2010-2017 Alexander Semke(alexander.semke@web.de) Copyright : (C) 2006-2008 Tilman Benkert (thzs@gmx.net) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef SPREADSHEET_H #define SPREADSHEET_H #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/ColumnStringIO.h" class AbstractFileFilter; class SpreadsheetView; template class QVector; class Spreadsheet : public AbstractDataSource { Q_OBJECT public: Spreadsheet(AbstractScriptingEngine* engine, const QString& name, bool loading = false); QIcon icon() const override; QMenu* createContextMenu() override; QWidget* view() const override; bool exportView() const override; bool printView() override; bool printPreview() const override; int columnCount() const; int columnCount(AbstractColumn::PlotDesignation) const; Column* column(int index) const; Column* column(const QString&) const; int rowCount() const; void removeRows(int first, int count); void insertRows(int before, int count); void removeColumns(int first, int count); void insertColumns(int before, int count); int colX(int col); int colY(int col); QString text(int row, int col) const; void copy(Spreadsheet* other); void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*, bool preview) override; void setColumnSelectedInView(int index, bool selected); // used from model to inform dock void emitRowCountChanged() { emit rowCountChanged(rowCount()); } void emitColumnCountChanged() { emit columnCountChanged(columnCount()); } //data import int prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode, int rows, int cols, QStringList colNameList, QVector) override; - void finalizeImport(int columnOffset, int startColumn , int endColumn, + void finalizeImport(int columnOffset, int startColumn , int endColumn, int numRows, const QString& dateTimeFormat, AbstractFileFilter::ImportMode) override; int resize(AbstractFileFilter::ImportMode, QStringList colNameList, int cols); public slots: void appendRows(int); void appendRow(); void appendColumns(int); void appendColumn(); void prependColumns(int); void setColumnCount(int); void setRowCount(int); void clear(); void clearMasks(); void moveColumn(int from, int to); void sortColumns(Column* leading, QVector, bool ascending); private: void init(); mutable SpreadsheetView* m_view; private slots: void childSelected(const AbstractAspect*) override; void childDeselected(const AbstractAspect*) override; signals: void requestProjectContextMenu(QMenu*); void columnSelected(int); void columnDeselected(int); // for spreadsheet dock void rowCountChanged(int); void columnCountChanged(int); }; #endif diff --git a/src/kdefrontend/datasources/ImportSQLDatabaseWidget.cpp b/src/kdefrontend/datasources/ImportSQLDatabaseWidget.cpp index 0cf0ff397..a22579696 100644 --- a/src/kdefrontend/datasources/ImportSQLDatabaseWidget.cpp +++ b/src/kdefrontend/datasources/ImportSQLDatabaseWidget.cpp @@ -1,448 +1,449 @@ /*************************************************************************** File : ImportSQLDatabaseWidget.cpp Project : LabPlot Description : Datapicker -------------------------------------------------------------------- Copyright : (C) 2016 by Ankit Wagadre (wagadre.ankit@gmail.com) Copyright : (C) 2016-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "ImportSQLDatabaseWidget.h" #include "DatabaseManagerDialog.h" #include "DatabaseManagerWidget.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/lib/macros.h" #include #include #include #include #include #ifdef HAVE_KF5_SYNTAX_HIGHLIGHTING #include #include #include #endif #include #include ImportSQLDatabaseWidget::ImportSQLDatabaseWidget(QWidget* parent) : QWidget(parent), m_cols(0), m_rows(0), m_databaseTreeModel(0), m_initializing(0), m_valid(false), m_numeric(false) { ui.setupUi(this); ui.cbImportFrom->addItem(i18n("Table")); ui.cbImportFrom->addItem(i18n("Custom query")); ui.bDatabaseManager->setIcon(QIcon::fromTheme("network-server-database")); ui.twPreview->setEditTriggers(QAbstractItemView::NoEditTriggers); ui.cbNumberFormat->addItems(AbstractFileFilter::numberFormats()); ui.cbDateTimeFormat->addItems(AbstractColumn::dateTimeFormats()); #ifdef HAVE_KF5_SYNTAX_HIGHLIGHTING m_highlighter = new KSyntaxHighlighting::SyntaxHighlighter(ui.teQuery->document()); m_highlighter->setDefinition(m_repository.definitionForName("SQL")); m_highlighter->setTheme( (palette().color(QPalette::Base).lightness() < 128) ? m_repository.defaultTheme(KSyntaxHighlighting::Repository::DarkTheme) : m_repository.defaultTheme(KSyntaxHighlighting::Repository::LightTheme) ); #endif m_configPath = QStandardPaths::standardLocations(QStandardPaths::AppDataLocation).constFirst() + "sql_connections"; connect( ui.cbConnection, SIGNAL(currentIndexChanged(int)), SLOT(connectionChanged()) ); connect( ui.cbImportFrom, SIGNAL(currentIndexChanged(int)), SLOT(importFromChanged(int)) ); connect( ui.bDatabaseManager, SIGNAL(clicked()), this, SLOT(showDatabaseManager()) ); connect( ui.lwTables, SIGNAL(currentRowChanged(int)), this, SLOT(refreshPreview()) ); connect( ui.bRefreshPreview, SIGNAL(clicked()), this, SLOT(refreshPreview()) ); //defer the loading of settings a bit in order to show the dialog prior to blocking the GUI in refreshPreview() QTimer::singleShot( 100, this, SLOT(loadSettings()) ); } void ImportSQLDatabaseWidget::loadSettings() { m_initializing = true; //read available connections readConnections(); //load last used connection and other settings KConfigGroup config(KSharedConfig::openConfig(), "ImportSQLDatabaseWidget"); ui.cbConnection->setCurrentIndex(ui.cbConnection->findText(config.readEntry("Connection", ""))); ui.cbImportFrom->setCurrentIndex(config.readEntry("ImportFrom", 0)); importFromChanged(ui.cbImportFrom->currentIndex()); ui.cbNumberFormat->setCurrentIndex(config.readEntry("NumberFormat", (int)QLocale::AnyLanguage)); ui.cbDateTimeFormat->setCurrentItem(config.readEntry("DateTimeFormat", "yyyy-dd-MM hh:mm:ss:zzz")); QList defaultSizes; defaultSizes << 100 << 100; ui.splitterMain->setSizes(config.readEntry("SplitterMainSizes", defaultSizes)); ui.splitterPreview->setSizes(config.readEntry("SplitterPreviewSizes", defaultSizes)); //TODO m_initializing = false; //all settings loaded -> trigger the selection of the last used connection in order to get the data preview connectionChanged(); } ImportSQLDatabaseWidget::~ImportSQLDatabaseWidget() { // save current settings KConfigGroup config(KSharedConfig::openConfig(), "ImportSQLDatabaseWidget"); config.writeEntry("Connection", ui.cbConnection->currentText()); config.writeEntry("ImportFrom", ui.cbImportFrom->currentIndex()); config.writeEntry("NumberFormat", ui.cbNumberFormat->currentText()); config.writeEntry("DateTimeFormat", ui.cbDateTimeFormat->currentText()); config.writeEntry("SplitterMainSizes", ui.splitterMain->sizes()); config.writeEntry("SplitterPreviewSizes", ui.splitterPreview->sizes()); //TODO } /*! * in case the import from a table is selected, returns the currently selected database table. * returns empty string otherwise. */ QString ImportSQLDatabaseWidget::selectedTable() const { if (ui.cbImportFrom->currentIndex() == 0) { if (ui.lwTables->currentItem()) return ui.lwTables->currentItem()->text(); } return QString(); } /*! returns \c true if a working connections was selected and a table (or custom query) is provided and ready to be imported. returns \c false otherwise. */ bool ImportSQLDatabaseWidget::isValid() const { return m_valid; } /*! returns \c true if the selected table or the result of a custom query contains numeric data only. returns \c false otherwise. */ bool ImportSQLDatabaseWidget::isNumericData() const { return m_numeric; } /*! loads all available saved connections */ void ImportSQLDatabaseWidget::readConnections() { DEBUG("ImportSQLDatabaseWidget: reading available connections"); KConfig config(m_configPath, KConfig::SimpleConfig); for (const auto& name : config.groupList()) ui.cbConnection->addItem(name); } void ImportSQLDatabaseWidget::connectionChanged() { if (m_initializing) return; QDEBUG("ImportSQLDatabaseWidget: connecting to " + ui.cbConnection->currentText()); //clear the previously shown content ui.teQuery->clear(); ui.lwTables->clear(); ui.twPreview->clear(); if (ui.cbConnection->currentIndex() == -1) return; //connection name was changed, determine the current connections settings KConfig config(m_configPath, KConfig::SimpleConfig); KConfigGroup group = config.group(ui.cbConnection->currentText()); //open the selected connection const QString driver = group.readEntry("Driver"); m_db = QSqlDatabase::addDatabase(driver); m_db.setDatabaseName( group.readEntry("DatabaseName") ); if (!DatabaseManagerWidget::isFileDB(driver)) { m_db.setHostName( group.readEntry("HostName") ); m_db.setPort( group.readEntry("Port", 0) ); m_db.setUserName( group.readEntry("UserName") ); m_db.setPassword( group.readEntry("Password") ); } if (!m_db.open()) { KMessageBox::error(this, i18n("Failed to connect to the database '%1'. Please check the connection settings.", ui.cbConnection->currentText()), i18n("Connection Failed")); setInvalid(); return; } //show all available database tables if (m_db.tables().size()) { ui.lwTables->addItems(m_db.tables()); ui.lwTables->setCurrentRow(0); for (int i = 0; i < ui.lwTables->count(); ++i) ui.lwTables->item(i)->setIcon(QIcon::fromTheme("view-form-table")); } else setInvalid(); } void ImportSQLDatabaseWidget::refreshPreview() { if (!ui.lwTables->currentItem()) { setInvalid(); return; } WAIT_CURSOR; ui.twPreview->clear(); //execute the current query (select on a table or a custom query) const QString& query = currentQuery(true); if (query.isEmpty()) { RESET_CURSOR; setInvalid(); return; } QSqlQuery q; q.prepare(currentQuery(true)); q.setForwardOnly(true); q.exec(); if (!q.isActive()) { RESET_CURSOR; if (!q.lastError().databaseText().isEmpty()) KMessageBox::error(this, q.lastError().databaseText(), i18n("Unable to Execute Query")); setInvalid(); return; } //resize the table to the number of columns (=number of fields in the result set) m_cols = q.record().count(); ui.twPreview->setColumnCount(m_cols); //determine the names and the data type (column modes) of the table columns. //check whether we have numerical data only by checking the data types of the first record. m_columnNames.clear(); m_columnModes.clear(); bool numeric = true; QLocale::Language numberFormat = (QLocale::Language)ui.cbNumberFormat->currentIndex(); const QString& dateTimeFormat = ui.cbDateTimeFormat->currentText(); q.next(); //go to the first record // ui.twPreview->setRowCount(1); //add the first row for the check boxes for (int i = 0; i < m_cols; ++i) { //name m_columnNames << q.record().fieldName(i); //value and type const QString valueString = q.record().value(i).toString(); AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); m_columnModes << mode; if (mode != AbstractColumn::Numeric) numeric = false; //header item QTableWidgetItem* item = new QTableWidgetItem(m_columnNames[i] + QLatin1String(" {") + ENUM_TO_STRING(AbstractColumn, ColumnMode, mode) + QLatin1String("}")); item->setTextAlignment(Qt::AlignLeft); item->setIcon(AbstractColumn::iconForMode(mode)); ui.twPreview->setHorizontalHeaderItem(i, item); //create checked items // QTableWidgetItem* itemChecked = new QTableWidgetItem(); // itemChecked->setCheckState(Qt::Checked); // ui.twPreview->setItem(0, i, itemChecked); } //preview the data const bool customQuery = (ui.cbImportFrom->currentIndex() != 0); int row = 0; do { for(int col = 0; col < m_cols; ++col) { ui.twPreview->setRowCount(row+1); ui.twPreview->setItem(row, col, new QTableWidgetItem(q.value(col).toString()) ); } row++; //in case a custom query is executed, check whether the row number limit is reached if (customQuery && row >= ui.sbPreviewLines->value()) break; } while (q.next()); ui.twPreview->horizontalHeader()->resizeSections(QHeaderView::ResizeToContents); setValid(); if (numeric != m_numeric) { m_numeric = numeric; emit stateChanged(); } RESET_CURSOR; } void ImportSQLDatabaseWidget::importFromChanged(int index) { if (index==0) { //import from a table ui.gbQuery->hide(); ui.lwTables->show(); } else { //import the result set of a custom query ui.gbQuery->show(); ui.lwTables->hide(); ui.twPreview->clear(); } refreshPreview(); } void ImportSQLDatabaseWidget::read(AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { if (!dataSource) return; WAIT_CURSOR; //execute the current query (select on a table or a custom query) QSqlQuery q; // q.setForwardOnly(true); //TODO: crashes most probably because of q.last() and q.first() below q.prepare(currentQuery()); if (!q.exec() || !q.isActive()) { RESET_CURSOR; if (!q.lastError().databaseText().isEmpty()) KMessageBox::error(this, q.lastError().databaseText(), i18n("Unable to Execute Query")); setInvalid(); return; } //determine the number of rows/records to read q.last(); const int rows = q.at()+1; q.first(); // pointers to the actual data containers //columnOffset indexes the "start column" in the datasource. Data will be imported starting from this column. QVector dataContainer; int columnOffset = dataSource->prepareImport(dataContainer, importMode, rows, m_cols, m_columnNames, m_columnModes); //number and DateTime formatting const QString& dateTimeFormat = ui.cbDateTimeFormat->currentText(); const QLocale numberFormat = QLocale((QLocale::Language)ui.cbNumberFormat->currentIndex()); //read the data int row = 0; do { for(int col = 0; col < m_cols; ++col) { const QString valueString = q.record().value(col).toString(); // set value depending on data type switch (m_columnModes[col]) { case AbstractColumn::Numeric: { bool isNumber; const double value = numberFormat.toDouble(valueString, &isNumber); static_cast*>(dataContainer[col])->operator[](row) = (isNumber ? value : NAN); break; } case AbstractColumn::Integer: { bool isNumber; const int value = numberFormat.toInt(valueString, &isNumber); static_cast*>(dataContainer[col])->operator[](row) = (isNumber ? value : NAN); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(dataContainer[col])->operator[](row) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(dataContainer[col])->operator[](row) = valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } row++; emit completed(100 * row/rows); } while (q.next()); + DEBUG(" Read " << row << " rows"); - dataSource->finalizeImport(columnOffset, 1, m_cols, dateTimeFormat, importMode); + dataSource->finalizeImport(columnOffset, 1, m_cols, row, dateTimeFormat, importMode); RESET_CURSOR; } QString ImportSQLDatabaseWidget::currentQuery(bool preview) { QString query; const bool customQuery = (ui.cbImportFrom->currentIndex() != 0); if ( !customQuery ) { const QString& tableName = ui.lwTables->currentItem()->text(); if (!preview) { query = QLatin1String("SELECT * FROM ") + tableName; } else { //preview the content of the currently selected table const QString& driver = m_db.driverName(); const QString& limit = QString::number(ui.sbPreviewLines->value()); if ( (driver == QLatin1String("QSQLITE3")) || (driver == QLatin1String("QMYSQL3")) || (driver == QLatin1String("QPSQL")) || (driver == QLatin1String("QSQLITE")) || (driver == QLatin1String("QMYSQL")) ) query = QLatin1String("SELECT * FROM ") + tableName + QLatin1String(" LIMIT ") + limit; else if (driver == QLatin1String("QOCI")) query = QLatin1String("SELECT * FROM ") + tableName + QLatin1String(" ROWNUM<=") + limit; else if (driver == QLatin1String("QDB2")) query = QLatin1String("SELECT * FROM ") + tableName + QLatin1String(" FETCH FIRST ") + limit + QLatin1String(" ROWS ONLY"); else if (driver == QLatin1String("QIBASE")) query = QLatin1String("SELECT * FROM ") + tableName + QLatin1String(" ROWS ") + limit; else query = QLatin1String("SELECT TOP ") + limit + QLatin1String(" * FROM ") + tableName; } } else { //preview the result of a custom query query = ui.teQuery->toPlainText().simplified(); } return query; } /*! shows the database manager where the connections are created and edited. The selected connection is selected in the connection combo box in this widget. **/ void ImportSQLDatabaseWidget::showDatabaseManager() { DatabaseManagerDialog* dlg = new DatabaseManagerDialog(this, ui.cbConnection->currentText()); if (dlg->exec() == QDialog::Accepted) { //re-read the available connections to be in sync with the changes in DatabaseManager ui.cbConnection->clear(); readConnections(); //select the connection the user has selected in DataabaseManager QString conn = dlg->connection(); ui.cbConnection->setCurrentIndex(ui.cbConnection->findText(conn)); } delete dlg; } void ImportSQLDatabaseWidget::setInvalid() { if (m_valid) { m_valid = false; emit stateChanged(); } } void ImportSQLDatabaseWidget::setValid() { if (!m_valid) { m_valid = true; emit stateChanged(); } } diff --git a/tests/import_export/ASCII/AsciiFilterTest.cpp b/tests/import_export/ASCII/AsciiFilterTest.cpp index c00604328..693b99ad1 100644 --- a/tests/import_export/ASCII/AsciiFilterTest.cpp +++ b/tests/import_export/ASCII/AsciiFilterTest.cpp @@ -1,342 +1,342 @@ /*************************************************************************** File : AsciiFilterTest.cpp Project : LabPlot Description : Tests for the ascii filter -------------------------------------------------------------------- Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "AsciiFilterTest.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/spreadsheet/Spreadsheet.h" void AsciiFilterTest::initTestCase() { const QString currentDir = __FILE__; m_dataDir = currentDir.left(currentDir.lastIndexOf(QDir::separator())) + QDir::separator() + QLatin1String("data") + QDir::separator(); // needed in order to have the signals triggered by SignallingUndoCommand, see LabPlot.cpp //TODO: redesign/remove this qRegisterMetaType("const AbstractAspect*"); qRegisterMetaType("const AbstractColumn*"); } //############################################################################## //################# handling of empty and sparse files ######################## //############################################################################## void AsciiFilterTest::testEmptyFileAppend() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const int rowCount = spreadsheet.rowCount(); const int colCount = spreadsheet.columnCount(); const QString fileName = m_dataDir + "empty_file.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Append; filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), rowCount); QCOMPARE(spreadsheet.columnCount(), colCount); } void AsciiFilterTest::testEmptyFilePrepend() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const int rowCount = spreadsheet.rowCount(); const int colCount = spreadsheet.columnCount(); const QString fileName = m_dataDir + "empty_file.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Prepend; filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), rowCount); QCOMPARE(spreadsheet.columnCount(), colCount); } void AsciiFilterTest::testEmptyFileReplace() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const int rowCount = spreadsheet.rowCount(); const int colCount = spreadsheet.columnCount(); const QString fileName = m_dataDir + "empty_file.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), rowCount); QCOMPARE(spreadsheet.columnCount(), colCount); } void AsciiFilterTest::testEmptyLines01() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "empty_lines_01.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter("auto"); filter.setHeaderEnabled(true); filter.readDataFromFile(fileName, &spreadsheet, mode); - QCOMPARE(spreadsheet.rowCount(), 7); //TODO: there are only 3 lines, we need an option to skip empty lines + QCOMPARE(spreadsheet.rowCount(), 3); QCOMPARE(spreadsheet.columnCount(), 3); QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("y")); QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("values")); QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::Integer); QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::Integer); QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::Integer); QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); QCOMPARE(spreadsheet.column(1)->integerAt(0), 2); QCOMPARE(spreadsheet.column(1)->integerAt(1), 4); QCOMPARE(spreadsheet.column(1)->integerAt(2), 9); QCOMPARE(spreadsheet.column(2)->integerAt(0), 10); QCOMPARE(spreadsheet.column(2)->integerAt(1), 40); QCOMPARE(spreadsheet.column(2)->integerAt(2), 90); } void AsciiFilterTest::testSparseFile01() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "sparse_file_01.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(","); filter.setHeaderEnabled(true); filter.setSimplifyWhitespacesEnabled(true); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 3); QCOMPARE(spreadsheet.columnCount(), 3); QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("N")); QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Col1")); QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("Col2")); QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::Integer); QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::Integer); QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::Integer); QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); QCOMPARE(spreadsheet.column(1)->integerAt(0), 1); QCOMPARE(spreadsheet.column(1)->integerAt(1), 0); QCOMPARE(spreadsheet.column(1)->integerAt(2), 1); QCOMPARE(spreadsheet.column(2)->integerAt(0), 2); QCOMPARE(spreadsheet.column(2)->integerAt(1), 2); QCOMPARE(spreadsheet.column(2)->integerAt(2), 0); } void AsciiFilterTest::testSparseFile02() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "sparse_file_02.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(","); filter.setNaNValueToZero(false); filter.setSimplifyWhitespacesEnabled(true); filter.setSkipEmptyParts(false); filter.setHeaderEnabled(true); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 3); QCOMPARE(spreadsheet.columnCount(), 3); QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("N")); QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Col1")); QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("Col2")); QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::Integer); QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::Numeric); QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::Numeric); QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.); QCOMPARE((bool)std::isnan(spreadsheet.column(1)->valueAt(1)), true); QCOMPARE(spreadsheet.column(1)->valueAt(2), 1.); QCOMPARE(spreadsheet.column(2)->valueAt(0), 2.); QCOMPARE(spreadsheet.column(2)->valueAt(1), 2.); QCOMPARE((bool)std::isnan(spreadsheet.column(2)->valueAt(2)), true); } void AsciiFilterTest::testSparseFile03() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "sparse_file_03.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(","); filter.setNaNValueToZero(true); filter.setSimplifyWhitespacesEnabled(true); filter.setSkipEmptyParts(true); filter.setHeaderEnabled(true); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 4); QCOMPARE(spreadsheet.columnCount(), 3); QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("N")); QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("Col1")); QCOMPARE(spreadsheet.column(2)->name(), QLatin1String("Col2")); QCOMPARE(spreadsheet.column(0)->columnMode(), AbstractColumn::Integer); QCOMPARE(spreadsheet.column(1)->columnMode(), AbstractColumn::Numeric); QCOMPARE(spreadsheet.column(2)->columnMode(), AbstractColumn::Numeric); QCOMPARE(spreadsheet.column(0)->integerAt(0), 1); QCOMPARE(spreadsheet.column(0)->integerAt(1), 2); QCOMPARE(spreadsheet.column(0)->integerAt(2), 3); QCOMPARE(spreadsheet.column(0)->integerAt(3), 3); QCOMPARE(spreadsheet.column(1)->valueAt(0), 1.); QCOMPARE(spreadsheet.column(1)->valueAt(1), 2.); QCOMPARE(spreadsheet.column(1)->valueAt(2), 1.); QCOMPARE(spreadsheet.column(1)->valueAt(3), 0.); QCOMPARE(spreadsheet.column(2)->valueAt(0), 2.); QCOMPARE(spreadsheet.column(2)->valueAt(1), 0.); QCOMPARE(spreadsheet.column(2)->valueAt(2), 0.); QCOMPARE(spreadsheet.column(2)->valueAt(3), 0.); } //############################################################################## //################################ header handling ############################ //############################################################################## void AsciiFilterTest::testHeader01() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "separator_semicolon.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(";"); filter.setHeaderEnabled(false); filter.setVectorNames(""); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 3); QCOMPARE(spreadsheet.columnCount(), 2); } void AsciiFilterTest::testHeader02() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "separator_semicolon.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(";"); filter.setHeaderEnabled(true); filter.setVectorNames(""); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 2);//out of 3 rows one row is used for the column names (header) QCOMPARE(spreadsheet.columnCount(), 2); QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("1")); //TODO: we start with the names "1" and "2" in the spreadsheet and try to rename them to "1" and "1" (names coming from the file) //-> the second column with the name "2" will be renamed to "3" because of the current logic in AbstractAspect::uniqueNameFor(). QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("3")); } void AsciiFilterTest::testHeader03() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "separator_semicolon.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(";"); filter.setHeaderEnabled(false); filter.setVectorNames("x"); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 3); QCOMPARE(spreadsheet.columnCount(), 1); //one column name was specified, we import only one column QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); } void AsciiFilterTest::testHeader04() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "separator_semicolon.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(";"); filter.setHeaderEnabled(false); filter.setVectorNames("x"); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 3); QCOMPARE(spreadsheet.columnCount(), 1); //one column name was specified -> we import only one column QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); } void AsciiFilterTest::testHeader05() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "separator_semicolon.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(";"); filter.setHeaderEnabled(false); filter.setVectorNames("x y"); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 3); QCOMPARE(spreadsheet.columnCount(), 2); //two names were specified -> we import two columns QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("y")); } void AsciiFilterTest::testHeader06() { Spreadsheet spreadsheet(0, "test", false); AsciiFilter filter; const QString fileName = m_dataDir + "separator_semicolon.txt"; AbstractFileFilter::ImportMode mode = AbstractFileFilter::Replace; filter.setSeparatingCharacter(";"); filter.setHeaderEnabled(false); filter.setVectorNames("x y z"); filter.readDataFromFile(fileName, &spreadsheet, mode); QCOMPARE(spreadsheet.rowCount(), 3); QCOMPARE(spreadsheet.columnCount(), 2); //thee names were specified, but there're only two columns in the file -> we import only two columns QCOMPARE(spreadsheet.column(0)->name(), QLatin1String("x")); QCOMPARE(spreadsheet.column(1)->name(), QLatin1String("y")); } //############################################################################## //##################### handling of different separators ###################### //############################################################################## QTEST_MAIN(AsciiFilterTest)