diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index 14d8be4ff..2baf6d682 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,1653 +1,1660 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2018 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include "backend/core/Project.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" #include "backend/lib/macros.h" #include "backend/lib/trace.h" #include #include #include #include #include /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() {} /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource* dataSource) { d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { return d->readFromLiveDevice(device, dataSource, from); } /*! reads the content of the file \c fileName. */ QVector AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromFile(fileName, dataSource, importMode, lines); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } QVector AsciiFilter::preview(QIODevice &device) { return d->preview(device); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE" << "2xSPACE" << "3xSPACE" << "4xSPACE" << "2xTAB"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } +QString AsciiFilter::fileInfoString(const QString& fileName) { + QString info(i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName))); + info += QLatin1String("
"); + info += i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); + return info; +} + /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " to determine number of lines"); return 0; } if (!device.canReadLine()) return -1; size_t lineCount = 0; while (!device.atEnd()) { device.readLine(); lineCount++; } //TODO: wc is much faster but not portable /* QElapsedTimer myTimer; myTimer.start(); QProcess wc; wc.start(QString("wc"), QStringList() << "-l" << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) lineCount = wc.readLine().split(' ')[0].toInt(); lineCount++; // last line not counted DEBUG(" Elapsed time counting lines : " << myTimer.elapsed() << " ms"); */ return lineCount; } /*! returns the number of lines in the device \c device and 0 if sequential. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice &device) { if (device.isSequential()) return 0; if (!device.canReadLine()) DEBUG("WARNING in AsciiFilter::lineNumber(): device cannot 'readLine()' but using it anyway."); size_t lineCount = 0; device.seek(0); while (!device.atEnd()) { device.readLine(); lineCount++; } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool AsciiFilter::NaNValueToZeroEnabled() const { if (d->nanValue == 0) return true; return false; } void AsciiFilter::setRemoveQuotesEnabled(bool b) { d->removeQuotesEnabled = b; } bool AsciiFilter::removeQuotesEnabled() const { return d->removeQuotesEnabled; } void AsciiFilter::setVectorNames(const QString& s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner), commentCharacter("#"), separatingCharacter("auto"), numberFormat(QLocale::C), autoModeEnabled(true), headerEnabled(true), skipEmptyParts(false), simplifyWhitespacesEnabled(true), nanValue(NAN), removeQuotesEnabled(false), createIndexEnabled(false), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_actualStartRow(1), m_actualRows(0), m_actualCols(0), m_prepared(false), m_columnOffset(0) { } /*! * get a single line from device */ QStringList AsciiFilterPrivate::getLineString(QIODevice& device) { QString line; do { // skip comment lines in data lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::getLineString(): device cannot 'readLine()' but using it anyway."); // line = device.readAll(); line = device.readLine(); } while (line.startsWith(commentCharacter)); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); DEBUG("data line : \'" << line.toStdString() << '\''); QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); //TODO: remove quotes here? QDEBUG("data line, parsed: " << lineStringList); return lineStringList; } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("AsciiFilterPrivate::prepareDeviceToRead(): is sequential = " << device.isSequential() << ", can readLine = " << device.canReadLine()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; ///////////////////////////////////////////////////////////////// // Find first data line (ignoring comment lines) DEBUG(" Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { QString line; if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); line = device.readLine(); DEBUG(" line = " << line.toStdString()); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } //TOOD: this logic seems to be wrong. If the user asks to read from line startRow, we should start here independent of any comments if (line.startsWith(commentCharacter)) // ignore commented lines before startRow i--; } // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); firstLine = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive); m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); QDEBUG("first line: " << firstLineStringList); DEBUG("headerEnabled = " << headerEnabled); //optionally, remove potential spaces in the first line if (simplifyWhitespacesEnabled) { for (int i = 0; i < firstLineStringList.size(); ++i) firstLineStringList[i] = firstLineStringList[i].simplified(); } if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); m_actualStartRow = startRow + 1; } else m_actualStartRow = startRow; // set range to read if (endColumn == -1) { if (headerEnabled || vectorNames.size() == 0) endColumn = firstLineStringList.size(); // last column else //number of vector names provided in the import dialog (not more than the maximal number of columns in the file) endColumn = qMin(vectorNames.size(), firstLineStringList.size()); } if (createIndexEnabled) { vectorNames.prepend(i18n("Index")); endColumn++; } m_actualCols = endColumn - startColumn + 1; //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ ///////////////////////////////////////////////////////////////// // parse first data line to determine data type for each column if (!device.isSequential()) firstLineStringList = getLineString(device); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Integer; col = 1; } for (auto& valueString: firstLineStringList) { // parse columns available in first data line if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } // parsing more lines to better determine data types for (unsigned int i = 0; i < m_dataTypeLines; ++i) { firstLineStringList = getLineString(device); if (createIndexEnabled) col = 1; else col = 0; for (auto& valueString: firstLineStringList) { if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (col == m_actualCols) break; AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); // numeric: integer -> numeric if (mode == AbstractColumn::Numeric && columnModes[col] == AbstractColumn::Integer) columnModes[col] = mode; // text: non text -> text if (mode == AbstractColumn::Text && columnModes[col] != AbstractColumn::Text) columnModes[col] = mode; col++; } } QDEBUG("column modes = " << columnModes); // ATTENTION: This resets the position in the device to 0 m_actualRows = (int)AsciiFilter::lineNumber(device); // reset to start of file //TODO: seems to be redundant since it's already done in the lineNumber() call above if (!device.isSequential()) device.seek(0); ///////////////////////////////////////////////////////////////// int actualEndRow = endRow; DEBUG("endRow(actualEndRow) = " << endRow << ", m_actualRows = " << m_actualRows); if (endRow == -1 || endRow > m_actualRows) actualEndRow = m_actualRows; if (m_actualRows > actualEndRow) m_actualRows = actualEndRow; DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << m_actualStartRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header incl. start rows): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0 && !device.isSequential()) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode, lines); } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { DEBUG("AsciiFilterPrivate::readFromLiveDevice(): bytes available = " << device.bytesAvailable() << ", from = " << from); if (!(device.bytesAvailable() > 0)) { DEBUG(" No new data available"); return 0; } LiveDataSource* spreadsheet = dynamic_cast(dataSource); if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe) if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return 0; if (!m_prepared) { DEBUG("Preparing .."); switch (spreadsheet->sourceType()) { case LiveDataSource::SourceType::FileOrPipe: { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return 0; } break; } case LiveDataSource::SourceType::NetworkTcpSocket: case LiveDataSource::SourceType::NetworkUdpSocket: case LiveDataSource::SourceType::LocalSocket: case LiveDataSource::SourceType::SerialPort: m_actualRows = 1; if (createIndexEnabled) { m_actualCols = 2; columnModes << AbstractColumn::Integer << AbstractColumn::Numeric; vectorNames << i18n("Index") << i18n("Value"); } else { m_actualCols = 1; columnModes << AbstractColumn::Numeric; vectorNames << i18n("Value"); } QDEBUG(" vector names = " << vectorNames); } // prepare import for spreadsheet spreadsheet->setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); DEBUG(" data source resized to col: " << m_actualCols); DEBUG(" data source rowCount: " << spreadsheet->rowCount()); //columns in a file data source don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } int keepNValues = spreadsheet->keepNValues(); if (keepNValues == 0) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(keepNValues); m_actualRows = keepNValues; } m_dataContainer.resize(m_actualCols); DEBUG(" Setting data .."); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } DEBUG("Prepared!"); } qint64 bytesread = 0; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif LiveDataSource::ReadingType readingType; if (!m_prepared) { readingType = LiveDataSource::ReadingType::TillEnd; } else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = LiveDataSource::ReadingType::TillEnd; //if we read the whole file we just start from the beginning of it //and read till end else if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) readingType = LiveDataSource::ReadingType::TillEnd; else readingType = spreadsheet->readingType(); } DEBUG(" reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType)); //move to the last read position, from == total bytes read //since the other source types are sequencial we cannot seek on them if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); DEBUG(" bytes available = " << device.bytesAvailable()); //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleSizeNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != LiveDataSource::ReadingType::TillEnd) newData.resize(spreadsheet->sampleSize()); int newDataIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif while (!device.atEnd()) { DEBUG(" reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType)); DEBUG(" source type = " << ENUM_TO_STRING(LiveDataSource, SourceType, spreadsheet->sourceType())); if (readingType != LiveDataSource::ReadingType::TillEnd) { switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData[newDataIdx++] = device.readAll(); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData[newDataIdx++] = device.read(device.bytesAvailable()); break; case LiveDataSource::SourceType::FileOrPipe: case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::readFromLiveDevice(): device cannot 'readLine()' but using it anyway."); newData[newDataIdx++] = device.readLine(); } } else { // ReadingType::TillEnd switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData.push_back(device.readAll()); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData.push_back(device.read(device.bytesAvailable())); break; case LiveDataSource::SourceType::FileOrPipe: case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::readFromLiveDevice(): device cannot 'readLine()' but using it anyway."); newData.push_back(device.readLine()); } } newLinesTillEnd++; if (readingType != LiveDataSource::ReadingType::TillEnd) { newLinesForSampleSizeNotTillEnd++; //for Continuous reading and FromEnd we read sample rate number of lines if possible //here TillEnd and Whole file behave the same if (newLinesForSampleSizeNotTillEnd == spreadsheet->sampleSize()) break; } } QDEBUG(" data read: " << newData); } //now we reset the readingType if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = spreadsheet->readingType(); //we had less new lines than the sample size specified if (readingType != LiveDataSource::ReadingType::TillEnd) QDEBUG("Removed empty lines: " << newData.removeAll("")); //back to the last read position before counting when reading from files if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); int currentRow = 0; // indexes the position in the vector(column) int linesToRead = 0; int keepNValues = spreadsheet->keepNValues(); DEBUG("Increase row count"); if (m_prepared) { //increase row count if we don't have a fixed size //but only after the preparation step if (keepNValues == 0) { if (readingType != LiveDataSource::ReadingType::TillEnd) m_actualRows += qMin(newData.size(), spreadsheet->sampleSize()); else { //we don't increase it if we reread the whole file, we reset it if (!(spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)) m_actualRows += newData.size(); else m_actualRows = newData.size(); } //appending if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) linesToRead = m_actualRows; else linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; } else { // fixed size if (readingType == LiveDataSource::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; //TODO after reading we should skip the next data lines //because it's TillEnd actually } else linesToRead = newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinuouslyFixed or FromEnd, WholeFile is disabled linesToRead = qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } DEBUG(" actual row = " << m_actualRows); if (linesToRead == 0) return 0; } else { linesToRead = newLinesTillEnd; if (headerEnabled) --m_actualRows; } DEBUG(" lines to read = " << linesToRead); //TODO: check other source types if (spreadsheet->sourceType() == LiveDataSource::SourceType::NetworkUdpSocket) { if (m_actualRows < linesToRead) { DEBUG(" SET actual rows to " << linesToRead); m_actualRows = linesToRead; } } //new rows/resize columns if we don't have a fixed size //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize if (keepNValues == 0) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { //when we have a fixed size we have to pop sampleSize number of lines if specified //here popping, setting currentRow if (!m_prepared) { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); } else { if (readingType == LiveDataSource::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) { currentRow = 0; } else { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } } else { //we read max sample rate number of lines when the reading mode //is ContinuouslyFixed or FromEnd currentRow = m_actualRows - qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet qDebug() << "reading from line" << currentRow << " till end" << newLinesTillEnd; qDebug() << "Lines to read:" << linesToRead <<", actual rows:" << m_actualRows << ", actual cols:" << m_actualCols; newDataIdx = 0; if (readingType == LiveDataSource::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->sampleSize()) newDataIdx = newData.size() - spreadsheet->sampleSize(); //since we skip a couple of lines, we need to count those bytes too for (int i = 0; i < newDataIdx; ++i) bytesread += newData.at(i).size(); } } qDebug() << "newDataIdx: " << newDataIdx; //TODO static int indexColumnIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif int row = 0; if (readingType == LiveDataSource::ReadingType::TillEnd || (readingType == LiveDataSource::ReadingType::ContinuousFixed)) { if (headerEnabled) { if (!m_prepared) { row = 1; bytesread += newData.at(0).size(); } } } if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { if (readingType == LiveDataSource::ReadingType::WholeFile) { if (headerEnabled) { row = 1; bytesread += newData.at(0).size(); } } } for (; row < linesToRead; ++row) { DEBUG(" row = " << row); QString line; if (readingType == LiveDataSource::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(row); //when we read the whole file we don't care about the previous position //so we don't have to count those bytes if (readingType != LiveDataSource::ReadingType::WholeFile) { if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { bytesread += line.size(); } } //qDebug() << "line bytes: " << line.size() << " line: " << line; if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList; // only FileOrPipe support multiple columns if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); else lineStringList << line; QDEBUG(" line = " << lineStringList << ", separator = \'" << m_separator << "\'"); if (createIndexEnabled) { if (spreadsheet->keepNValues() == 0) lineStringList.prepend(QString::number(currentRow)); else lineStringList.prepend(QString::number(indexColumnIdx++)); } QDEBUG(" column modes = " << columnModes); for (int n = 0; n < m_actualCols; ++n) { DEBUG(" actual col = " << n); if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); DEBUG(" value string = " << valueString.toStdString()); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { DEBUG(" Numeric"); bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::Integer: { DEBUG(" Integer"); bool isNumber; const int value = locale.toInt(valueString, &isNumber); DEBUG(" container size = " << m_dataContainer.size() << ", current row = " << currentRow); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { DEBUG(" missing columns in this line"); switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); const Project* project = spreadsheet->project(); QVector curves = project->children(AbstractAspect::Recursive); QVector plots; for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); //determine the plots where the column is consumed for (const auto* curve: curves) { if (curve->xColumn() == column || curve->yColumn() == column) { CartesianPlot* plot = dynamic_cast(curve->parentAspect()); if (plots.indexOf(plot) == -1) { plots << plot; plot->setSuppressDataChangedSignal(true); } } } column->setChanged(); } //loop over all affected plots and retransform them for (auto* plot: plots) { plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } m_prepared = true; return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return; } // matrix data has only one column mode (which is not text) if (dynamic_cast(dataSource)) { auto mode = columnModes[0]; if (mode == AbstractColumn::Text) mode = AbstractColumn::Numeric; for (auto& c: columnModes) if (c != mode) c = mode; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows - m_actualStartRow + 1, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); ++i) { QString line = device.readLine(); // skip start lines if (m_actualStartRow > 1) { m_actualStartRow--; continue; } line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); //prepend the index if required //TODO: come up maybe with a solution with adding the index inside of the loop below, //without conversion to string, prepending to the list and then conversion back to integer. if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); // remove left white spaces if (skipEmptyParts) { for (int n = 0; n < lineStringList.size(); ++n) { QString valueString = lineStringList.at(n); if (!QString::compare(valueString, " ")) { lineStringList.removeAt(n); n--; } } } for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, dateTimeFormat, importMode); } /*! * preview for special devices (local/UDP/TCP socket or serial port) */ QVector AsciiFilterPrivate::preview(QIODevice &device) { DEBUG("AsciiFilterPrivate::preview(): bytesAvailable = " << device.bytesAvailable() << ", isSequential = " << device.isSequential()); QVector dataStrings; if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return dataStrings; } if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return dataStrings; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif int linesToRead = 0; QVector newData; while (!device.atEnd()) { if (device.canReadLine()) newData.push_back(device.readLine()); else // UDP fails otherwise newData.push_back(device.readAll()); linesToRead++; } QDEBUG(" data = " << newData); if (linesToRead == 0) return dataStrings; int col = 0; int colMax = newData.at(0).size(); if (createIndexEnabled) colMax++; columnModes.resize(colMax); if (createIndexEnabled) { columnModes[0] = AbstractColumn::ColumnMode::Integer; col = 1; vectorNames.prepend(i18n("Index")); } vectorNames.append(i18n("Value")); QDEBUG(" vector names = " << vectorNames); for (const auto& valueString: newData.at(0).split(' ', QString::SkipEmptyParts)) { if (col == colMax) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } for (int i = 0; i < linesToRead; ++i) { QString line = newData.at(i); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(' ', QString::SkipEmptyParts); if (createIndexEnabled) lineStringList.prepend(QString::number(i)); QStringList lineString; for (int n = 0; n < lineStringList.size(); ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 16); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; // set column names for preview if (!headerEnabled) { int start = 0; if (createIndexEnabled) start = 1; for (int i=start;i 1) { m_actualStartRow--; continue; } line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); QDEBUG(" line = " << lineStringList); //prepend index if required if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); QStringList lineString; for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); //DEBUG(" valueString = " << valueString.toStdString()); if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces continue; // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 15); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to ascii file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement( "asciiFilter"); writer->writeAttribute( "commentCharacter", d->commentCharacter); writer->writeAttribute( "separatingCharacter", d->separatingCharacter); writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); writer->writeAttribute( "nanValue", QString::number(d->nanValue)); writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled)); writer->writeAttribute( "startRow", QString::number(d->startRow)); writer->writeAttribute( "endRow", QString::number(d->endRow)); writer->writeAttribute( "startColumn", QString::number(d->startColumn)); writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "asciiFilter") { reader->raiseError(i18n("no ascii filter element found")); return false; } KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("commentCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("commentCharacter").toString()); else d->commentCharacter = str; str = attribs.value("separatingCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("separatingCharacter").toString()); else d->separatingCharacter = str; str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("createIndex").toString()); else d->createIndexEnabled = str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("autoMode").toString()); else d->autoModeEnabled = str.toInt(); str = attribs.value("header").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("header").toString()); else d->headerEnabled = str.toInt(); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty str = attribs.value("simplifyWhitespaces").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("simplifyWhitespaces").toString()); else d->simplifyWhitespacesEnabled = str.toInt(); str = attribs.value("nanValue").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("nanValue").toString()); else d->nanValue = str.toDouble(); str = attribs.value("removeQuotes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("removeQuotes").toString()); else d->removeQuotesEnabled = str.toInt(); str = attribs.value("skipEmptyParts").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("skipEmptyParts").toString()); else d->skipEmptyParts = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startRow").toString()); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endRow").toString()); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("startColumn").toString()); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("endColumn").toString()); else d->endColumn = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/AsciiFilter.h b/src/backend/datasources/filters/AsciiFilter.h index 8263c9c3d..7fd2b2d11 100644 --- a/src/backend/datasources/filters/AsciiFilter.h +++ b/src/backend/datasources/filters/AsciiFilter.h @@ -1,115 +1,116 @@ /*************************************************************************** File : AsciiFilter.h Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTER_H #define ASCIIFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" class QStringList; class QIODevice; class AsciiFilterPrivate; class QAbstractSocket; class AsciiFilter : public AbstractFileFilter { Q_OBJECT public: AsciiFilter(); ~AsciiFilter() override; static QStringList separatorCharacters(); static QStringList commentCharacters(); static QStringList dataTypes(); static QStringList predefinedFilters(); + static QString fileInfoString(const QString&); static int columnNumber(const QString& fileName, const QString& separator = QString()); static size_t lineNumber(const QString& fileName); static size_t lineNumber(QIODevice&); // calculate number of lines if device supports it // read data from any device void readDataFromDevice(QIODevice& device, AbstractDataSource*, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*dataSource); qint64 readFromLiveDevice(QIODevice& device, AbstractDataSource*, qint64 from = -1); // overloaded function to read from file QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; void write(const QString& fileName, AbstractDataSource*) override; QVector preview(const QString& fileName, int lines); QVector preview(QIODevice& device); void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setCommentCharacter(const QString&); QString commentCharacter() const; void setSeparatingCharacter(const QString&); QString separatingCharacter() const; void setDateTimeFormat(const QString&); QString dateTimeFormat() const; void setNumberFormat(QLocale::Language); QLocale::Language numberFormat() const; void setAutoModeEnabled(const bool); bool isAutoModeEnabled() const; void setHeaderEnabled(const bool); bool isHeaderEnabled() const; void setSkipEmptyParts(const bool); bool skipEmptyParts() const; void setSimplifyWhitespacesEnabled(const bool); bool simplifyWhitespacesEnabled() const; void setNaNValueToZero(const bool); bool NaNValueToZeroEnabled() const; void setRemoveQuotesEnabled(const bool); bool removeQuotesEnabled() const; void setCreateIndexEnabled(const bool); void setVectorNames(const QString&); QStringList vectorNames() const; QVector columnModes(); void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class AsciiFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/FITSFilter.cpp b/src/backend/datasources/filters/FITSFilter.cpp index 86fb9c690..6a799f5fc 100644 --- a/src/backend/datasources/filters/FITSFilter.cpp +++ b/src/backend/datasources/filters/FITSFilter.cpp @@ -1,1636 +1,1638 @@ /*************************************************************************** File : FITSFilter.cpp Project : LabPlot Description : FITS I/O-filter -------------------------------------------------------------------- Copyright : (C) 2016 by Fabian Kristof (fkristofszabolcs@gmail.com) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "FITSFilter.h" #include "FITSFilterPrivate.h" #include "backend/core/column/Column.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/datatypes/Double2StringFilter.h" #include "backend/matrix/MatrixModel.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/matrix/Matrix.h" #include "commonfrontend/matrix/MatrixView.h" #include #include #include /*! \class FITSFilter * \brief Manages the import/export of data from/to a FITS file. * \since 2.2.0 * \ingroup datasources */ FITSFilter::FITSFilter():AbstractFileFilter(), d(new FITSFilterPrivate(this)) {} FITSFilter::~FITSFilter() {} QVector FITSFilter::readDataFromFile(const QString &fileName, AbstractDataSource *dataSource, AbstractFileFilter::ImportMode importMode, int lines) { Q_UNUSED(lines); return d->readCHDU(fileName, dataSource, importMode); } QVector FITSFilter::readChdu(const QString &fileName, bool* okToMatrix, int lines) { return d->readCHDU(fileName, NULL, AbstractFileFilter::Replace, okToMatrix, lines); } void FITSFilter::write(const QString &fileName, AbstractDataSource *dataSource) { d->writeCHDU(fileName, dataSource); } void FITSFilter::addNewKeyword(const QString &filename, const QList &keywords) { d->addNewKeyword(filename, keywords); } void FITSFilter::updateKeywords(const QString &fileName, const QList& originals, const QVector& updates) { d->updateKeywords(fileName, originals, updates); } void FITSFilter::deleteKeyword(const QString &fileName, const QList& keywords) { d->deleteKeyword(fileName, keywords); } void FITSFilter::addKeywordUnit(const QString &fileName, const QList &keywords) { d->addKeywordUnit(fileName, keywords); } void FITSFilter::removeExtensions(const QStringList &extensions) { d->removeExtensions(extensions); } void FITSFilter::parseHeader(const QString &fileName, QTableWidget *headerEditTable, bool readKeys, const QList &keys) { d->parseHeader(fileName, headerEditTable, readKeys, keys); } void FITSFilter::parseExtensions(const QString &fileName, QTreeWidget *tw, bool checkPrimary) { d->parseExtensions(fileName, tw, checkPrimary); } QList FITSFilter::chduKeywords(const QString &fileName) { return d->chduKeywords(fileName); } void FITSFilter::loadFilterSettings(const QString& fileName) { Q_UNUSED(fileName) } void FITSFilter::saveFilterSettings(const QString& fileName) const { Q_UNUSED(fileName) } /*! * \brief contains the {StandardKeywords \ MandatoryKeywords} keywords * \return A list of keywords */ QStringList FITSFilter::standardKeywords() { return QStringList() << QLatin1String("(blank)") << QLatin1String("CROTA") << QLatin1String("EQUINOX") << QLatin1String("NAXIS") << QLatin1String("TBCOL") << QLatin1String("TUNIT") << QLatin1String("AUTHOR") << QLatin1String("CRPIX") << QLatin1String("EXTEND") << QLatin1String("OBJECT") << QLatin1String("TDIM") << QLatin1String("TZERO") << QLatin1String("BITPIX") << QLatin1String("CRVAL") << QLatin1String("EXTLEVEL") << QLatin1String("OBSERVER") << QLatin1String("TDISP") << QLatin1String("XTENSION") << QLatin1String("BLANK") << QLatin1String("CTYPE") << QLatin1String("EXTNAME") << QLatin1String("ORIGIN") << QLatin1String("TELESCOP") << QLatin1String("BLOCKED") << QLatin1String("DATAMAX") << QLatin1String("EXTVER") << QLatin1String("BSCALE") << QLatin1String("DATAMIN") << QLatin1String("PSCAL") << QLatin1String("TFORM") << QLatin1String("BUNIT") << QLatin1String("DATE") << QLatin1String("GROUPS") << QLatin1String("PTYPE") << QLatin1String("THEAP") << QLatin1String("BZERO") << QLatin1String("DATE-OBS") << QLatin1String("HISTORY") << QLatin1String("PZERO") << QLatin1String("TNULL") << QLatin1String("CDELT") << QLatin1String("INSTRUME") << QLatin1String("REFERENC") << QLatin1String("TSCAL") << QLatin1String("COMMENT") << QLatin1String("EPOCH") << QLatin1String("NAXIS") << QLatin1String("SIMPLE") << QLatin1String("TTYPE"); } /*! * \brief Returns a list of keywords, that are mandatory for an image extension of a FITS file * see: * https://archive.stsci.edu/fits/fits_standard/node64.html * \return A list of keywords */ QStringList FITSFilter::mandatoryImageExtensionKeywords() { return QStringList() << QLatin1String("XTENSION") << QLatin1String("BITPIX") << QLatin1String("NAXIS") << QLatin1String("PCOUNT") << QLatin1String("GCOUNT") << QLatin1String("END"); } /*! * \brief Returns a list of keywords, that are mandatory for a table extension (ascii or bintable) * of a FITS file * see: * https://archive.stsci.edu/fits/fits_standard/node58.html * https://archive.stsci.edu/fits/fits_standard/node68.html * \return A list of keywords */ QStringList FITSFilter::mandatoryTableExtensionKeywords() { return QStringList() << QLatin1String("XTENSION") << QLatin1String("BITPIX") << QLatin1String("NAXIS") << QLatin1String("NAXIS1") << QLatin1String("NAXIS2") << QLatin1String("PCOUNT") << QLatin1String("GCOUNT") << QLatin1String("TFIELDS") << QLatin1String("END"); } /*! * \brief Returns a list of strings that represent units which are used for autocompletion when adding * keyword units to keywords * \return A list of strings that represent units */ QStringList FITSFilter::units() { return QStringList() << QLatin1String("m (Metre)") << QLatin1String("kg (Kilogram)") << QLatin1String("s (Second)") << QString("M☉ (Solar mass)") << QLatin1String("AU (Astronomical unit") << QLatin1String("l.y (Light year)") << QLatin1String("km (Kilometres") << QLatin1String("pc (Parsec)") << QLatin1String("K (Kelvin)") << QLatin1String("mol (Mole)") << QLatin1String("cd (Candela)"); } /*! * \brief Sets the startColumn to \a column * \param column the column to be set */ void FITSFilter::setStartColumn(const int column) { d->startColumn = column; } /*! * \brief Returns startColumn * \return The startColumn */ int FITSFilter::startColumn() const { return d->startColumn; } /*! * \brief Sets the endColumn to \a column * \param column the column to be set */ void FITSFilter::setEndColumn(const int column) { d->endColumn = column; } /*! * \brief Returns endColumn * \return The endColumn */ int FITSFilter::endColumn() const { return d->endColumn; } /*! * \brief Sets the startRow to \a row * \param row the row to be set */ void FITSFilter::setStartRow(const int row) { d->startRow = row; } /*! * \brief Returns startRow * \return The startRow */ int FITSFilter::startRow() const { return d->startRow; } /*! * \brief Sets the endRow to \a row * \param row the row to be set */ void FITSFilter::setEndRow(const int row) { d->endRow = row; } /*! * \brief Returns endRow * \return The endRow */ int FITSFilter::endRow() const { return d->endRow; } /*! * \brief Sets commentsAsUnits to \a commentsAsUnits * * This is used when spreadsheets are exported to FITS table extensions and comments are used as the * units of the table's columns. * \param commentsAsUnits */ void FITSFilter::setCommentsAsUnits(const bool commentsAsUnits) { d->commentsAsUnits = commentsAsUnits; } /*! * \brief Sets exportTo to \a exportTo * * This is used to decide whether the container should be exported to a FITS image or a FITS table * For an image \a exportTo should be 0, for a table 1 * \param exportTo */ void FITSFilter::setExportTo(const int exportTo) { d->exportTo = exportTo; } +QString FITSFilter::fileInfoString(const QString& fileName) { + const int imagesCount = FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("IMAGES")).size(); + QString info(i18n("Images: %1", QString::number(imagesCount ))); -int FITSFilter::imagesCount(const QString &fileName) { - return FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("IMAGES")).size(); -} + info += QLatin1String("
"); -int FITSFilter::tablesCount(const QString &fileName) { - return FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("TABLES")).size(); -} + const int tablesCount = FITSFilterPrivate::extensionNames(fileName).values(QLatin1String("TABLES")).size(); + info += i18n("Tables: %1", QString::number(tablesCount)); + return info; +} //##################################################################### //################### Private implementation ########################## //##################################################################### FITSFilterPrivate::FITSFilterPrivate(FITSFilter* owner) : q(owner), startRow(-1), endRow(-1), startColumn(-1), endColumn(-1), commentsAsUnits(false), exportTo(0) { #ifdef HAVE_FITS m_fitsFile = 0; #endif } /*! * \brief Read the current header data unit from file \a filename in data source \a dataSource in \a importMode import mode * \param fileName the name of the file to be read * \param dataSource the data source to be filled * \param importMode */ QVector FITSFilterPrivate::readCHDU(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, bool* okToMatrix, int lines) { DEBUG("FITSFilterPrivate::readCHDU() file name = " << fileName.toStdString()); QVector dataStrings; #ifdef HAVE_FITS int status = 0; if(fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status)) { DEBUG(" ERROR opening file " << fileName.toStdString()); printError(status); return dataStrings; } int chduType; if (fits_get_hdu_type(m_fitsFile, &chduType, &status)) { printError(status); return dataStrings; } long actualRows; int actualCols; int columnOffset = 0; bool noDataSource = (dataSource == NULL); if(chduType == IMAGE_HDU) { DEBUG("IMAGE_HDU"); int maxdim = 2; int bitpix; int naxis; long naxes[2]; if (fits_get_img_param(m_fitsFile, maxdim, &bitpix, &naxis, naxes, &status)) { printError(status); return dataStrings; } if (naxis == 0) return dataStrings; actualRows = naxes[1]; actualCols = naxes[0]; if (lines == -1) lines = actualRows; else { if (lines > actualRows) lines = actualRows; } if (endRow != -1) { if (!noDataSource) lines = endRow; } if (endColumn != -1) actualCols = endColumn; if (noDataSource) dataStrings.reserve(lines); int i = 0; int j = 0; if (startRow != 1) i = startRow; if (startColumn != 1) j = startColumn; const int jstart = j; //TODO: support other modes QVector columnModes; columnModes.resize(actualCols - j); QStringList vectorNames; QVector dataContainer; if (!noDataSource) { dataContainer.reserve(actualCols - j); columnOffset = dataSource->prepareImport(dataContainer, importMode, lines - i, actualCols - j, vectorNames, columnModes); } long pixelCount = lines * actualCols; double* data = new double[pixelCount]; if (!data) { qDebug() << "Not enough memory for data"; return dataStrings; } if (fits_read_img(m_fitsFile, TDOUBLE, 1, pixelCount, NULL, data, NULL, &status)) { printError(status); return dataStrings << (QStringList() << QString("Error")); } int ii = 0; DEBUG(" Import " << lines << " lines"); for (; i < lines; ++i) { int jj = 0; QStringList line; line.reserve(actualCols - j); for (; j < actualCols; ++j) { if (noDataSource) line << QString::number(data[i*naxes[0] +j]); else static_cast*>(dataContainer[jj++])->operator[](ii) = data[i* naxes[0] + j]; } dataStrings << line; j = jstart; ii++; } delete[] data; if (dataSource) dataSource->finalizeImport(columnOffset, 1, actualCols, "", importMode); fits_close_file(m_fitsFile, &status); return dataStrings; } else if ((chduType == ASCII_TBL) || (chduType == BINARY_TBL)) { DEBUG("ASCII_TBL or BINARY_TBL"); if (endColumn != -1) actualCols = endColumn; else fits_get_num_cols(m_fitsFile, &actualCols, &status); if (endRow != -1) actualRows = endRow; else fits_get_num_rows(m_fitsFile, &actualRows, &status); QStringList columnNames; QList columnsWidth; QStringList columnUnits; columnUnits.reserve(actualCols); columnsWidth.reserve(actualCols); columnNames.reserve(actualCols); int colWidth; char keyword[FLEN_KEYWORD]; char value[FLEN_VALUE]; int col = 1; if (startColumn != 1) { if (startColumn != 0) col = startColumn; } for (; col <=actualCols; ++col) { status = 0; fits_make_keyn("TTYPE", col, keyword, &status); fits_read_key(m_fitsFile, TSTRING, keyword, value, NULL, &status); columnNames.append(QLatin1String(value)); fits_make_keyn("TUNIT", col, keyword, &status); fits_read_key(m_fitsFile, TSTRING, keyword, value, NULL, &status); columnUnits.append(QLatin1String(value)); fits_get_col_display_width(m_fitsFile, col, &colWidth, &status); columnsWidth.append(colWidth); } status = 0; if (lines == -1) lines = actualRows; else if (lines > actualRows) lines = actualRows; if (endRow != -1) lines = endRow; QVector stringDataPointers; QVector numericDataPointers; QList columnNumericTypes; int startCol = 0; if (startColumn != 1) startCol = startColumn; int startRrow = 0; if (startRow != 1) startRrow = startRow; columnNumericTypes.reserve(actualCols); int datatype; int c = 1; if (startColumn != 1) { if (startColumn != 0) c = startColumn; } QList matrixNumericColumnIndices; for (; c <= actualCols; ++c) { fits_get_coltype(m_fitsFile, c, &datatype, NULL, NULL, &status); switch (datatype) { case TSTRING: columnNumericTypes.append(false); break; case TSHORT: columnNumericTypes.append(true); break; case TLONG: columnNumericTypes.append(true); break; case TFLOAT: columnNumericTypes.append(true); break; case TDOUBLE: columnNumericTypes.append(true); break; case TLOGICAL: columnNumericTypes.append(false); break; case TBIT: columnNumericTypes.append(true); break; case TBYTE: columnNumericTypes.append(true); break; case TCOMPLEX: columnNumericTypes.append(true); break; default: columnNumericTypes.append(false); break; } if ((datatype != TSTRING) && (datatype != TLOGICAL)) matrixNumericColumnIndices.append(c); } if (noDataSource) *okToMatrix = matrixNumericColumnIndices.isEmpty() ? false : true; if (!noDataSource) { DEBUG("HAS DataSource"); Spreadsheet* spreadsheet = dynamic_cast(dataSource); if(spreadsheet) { numericDataPointers.reserve(actualCols - startCol); stringDataPointers.reserve(actualCols - startCol); spreadsheet->setUndoAware(false); columnOffset = spreadsheet->resize(importMode, columnNames, actualCols - startCol); if (importMode == AbstractFileFilter::Replace) { spreadsheet->clear(); spreadsheet->setRowCount(lines - startRrow); } else { if (spreadsheet->rowCount() < (lines - startRrow)) spreadsheet->setRowCount(lines - startRrow); } DEBUG("Reading columns ..."); for (int n = 0; n < actualCols - startCol; ++n) { if (columnNumericTypes.at(n)) { spreadsheet->column(columnOffset+ n)->setColumnMode(AbstractColumn::Numeric); QVector* datap = static_cast* >(spreadsheet->column(columnOffset+n)->data()); numericDataPointers.push_back(datap); if (importMode == AbstractFileFilter::Replace) datap->clear(); } else { spreadsheet->column(columnOffset+ n)->setColumnMode(AbstractColumn::Text); QStringList* list = static_cast(spreadsheet->column(columnOffset+n)->data()); stringDataPointers.push_back(list); if (importMode == AbstractFileFilter::Replace) list->clear(); } } DEBUG(" ... DONE"); stringDataPointers.squeeze(); } else { numericDataPointers.reserve(matrixNumericColumnIndices.size()); columnOffset = dataSource->prepareImport(numericDataPointers, importMode, lines - startRrow, matrixNumericColumnIndices.size()); } numericDataPointers.squeeze(); } char* array = new char[1000]; //TODO: why 1000? int row = 1; if (startRow != 1) { if (startRow != 0) row = startRow; } int coll = 1; if (startColumn != 1) { if (startColumn != 0) coll = startColumn; } bool isMatrix = false; if (dynamic_cast(dataSource)) { isMatrix = true; coll = matrixNumericColumnIndices.first(); actualCols = matrixNumericColumnIndices.last(); if (importMode == AbstractFileFilter::Replace) { for (auto* col: numericDataPointers) static_cast*>(col)->clear(); } } for (; row <= lines; ++row) { int numericixd = 0; int stringidx = 0; QStringList line; line.reserve(actualCols-coll); for (int col = coll; col <= actualCols; ++col) { if (isMatrix) { if (!matrixNumericColumnIndices.contains(col)) continue; } if(fits_read_col_str(m_fitsFile, col, row, 1, 1, NULL, &array, NULL, &status)) printError(status); if (!noDataSource) { const QString& str = QString::fromLatin1(array); if (str.isEmpty()) { if (columnNumericTypes.at(col - 1)) static_cast*>(numericDataPointers[numericixd++])->push_back(0); else stringDataPointers[stringidx++]->append(QLatin1String("NULL")); } else { if (columnNumericTypes.at(col - 1)) static_cast*>(numericDataPointers[numericixd++])->push_back(str.toDouble()); else { if (!stringDataPointers.isEmpty()) stringDataPointers[stringidx++]->operator<<(str.simplified()); } } } else { QString tmpColstr = QString::fromLatin1(array); tmpColstr = tmpColstr.simplified(); if (tmpColstr.isEmpty()) line << QLatin1String("NULL"); else line << tmpColstr; } } dataStrings << line; } delete[] array; if (!noDataSource) dataSource->finalizeImport(columnOffset, 1, actualCols, "", importMode); fits_close_file(m_fitsFile, &status); return dataStrings; } else qDebug() << "Incorrect header type"; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(importMode) Q_UNUSED(okToMatrix) Q_UNUSED(lines) #endif return dataStrings; } /*! * \brief Export from data source \a dataSource to file \a fileName * \param fileName the name of the file to be exported to * \param dataSource the datasource whose data is exported */ void FITSFilterPrivate::writeCHDU(const QString &fileName, AbstractDataSource *dataSource) { #ifdef HAVE_FITS if (!fileName.endsWith(QLatin1String(".fits"))) return; int status = 0; bool existed = false; if (!QFile::exists(fileName)) { if (fits_create_file(&m_fitsFile, fileName.toLatin1(), &status)) { printError(status); qDebug() << fileName; return; } } else { if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } else existed = true; } Matrix* const matrix = dynamic_cast(dataSource); if (matrix) { //FITS image if (exportTo == 0) { long naxes[2] = { matrix->columnCount(), matrix->rowCount() }; if (fits_create_img(m_fitsFile, FLOAT_IMG, 2, naxes, &status)) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); return; } const long nelem = naxes[0] * naxes[1]; double* const array = new double[nelem]; const QVector >* const data = static_cast>*>(matrix->data()); for (int col = 0; col < naxes[0]; ++col) for (int row = 0; row < naxes[1]; ++row) array[row * naxes[0] + col] = data->at(row).at(col); if (fits_write_img(m_fitsFile, TDOUBLE, 1, nelem, array, &status )) { printError(status); status = 0; } fits_close_file(m_fitsFile, &status); delete[] array; //FITS table } else { const int nrows = matrix->rowCount(); const int tfields = matrix->columnCount(); QVector columnNames; columnNames.resize(tfields); columnNames.reserve(tfields); QVector tform; tform.resize(tfields); tform.reserve(tfields); QVector tunit; tunit.resize(tfields); tunit.reserve(tfields); //TODO: mode const QVector>* const matrixData = static_cast>*>(matrix->data()); QVector column; const MatrixModel* matrixModel = static_cast(matrix->view())->model(); const int precision = matrix->precision(); for (int i = 0; i < tfields; ++i) { column = matrixData->at(i); const QString& columnName = matrixModel->headerData(i, Qt::Horizontal).toString(); columnNames[i] = new char[columnName.size()]; strcpy(columnNames[i], columnName.toLatin1().data()); tunit[i] = new char[1]; strcpy(tunit[i], ""); int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (matrix->text(row, i).size() > maxSize) maxSize = matrix->text(row, i).size(); } QString tformn; if (precision > 0) { tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".") + QString::number(precision); } else tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".0"); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); } //TODO extension name containing[] ? if (fits_create_tbl(m_fitsFile, ASCII_TBL, nrows, tfields, columnNames.data(), tform.data(), tunit.data(), matrix->name().toLatin1().data(),&status )) { printError(status); qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); double* columnNumeric = new double[nrows]; for (int col = 1; col <= tfields; ++col) { column = matrixData->at(col-1); for (int r = 0; r < column.size(); ++r) columnNumeric[r] = column.at(r); fits_write_col(m_fitsFile, TDOUBLE, col, 1, 1, nrows, columnNumeric, &status); if (status) { printError(status); delete[] columnNumeric; status = 0; if (!existed) { QFile file(fileName); file.remove(); } fits_close_file(m_fitsFile, &status); return; } } delete[] columnNumeric; fits_close_file(m_fitsFile, &status); } return; } Spreadsheet* const spreadsheet = dynamic_cast(dataSource); if (spreadsheet) { //FITS image if (exportTo == 0) { int maxRowIdx = -1; //don't export lots of empty lines if all of those contain nans // TODO: option? for (int c = 0; c < spreadsheet->columnCount(); ++c) { const Column* const col = spreadsheet->column(c); int currMaxRoxIdx = -1; for (int r = col->rowCount(); r >= 0; --r) { if (col->isValid(r)) { currMaxRoxIdx = r; break; } } if (currMaxRoxIdx > maxRowIdx) { maxRowIdx = currMaxRoxIdx; } } long naxes[2] = { spreadsheet->columnCount(), maxRowIdx + 1}; if (fits_create_img(m_fitsFile, FLOAT_IMG, 2, naxes, &status)) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } const long nelem = naxes[0] * naxes[1]; double* array = new double[nelem]; for (int row = 0; row < naxes[1]; ++row) { for (int col = 0; col < naxes[0]; ++col) array[row * naxes[0] + col] = spreadsheet->column(col)->valueAt(row); } if (fits_write_img(m_fitsFile, TDOUBLE, 1, nelem, array, &status )) { printError(status); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } fits_close_file(m_fitsFile, &status); delete[] array; } else { const int nrows = spreadsheet->rowCount(); const int tfields = spreadsheet->columnCount(); QVector columnNames; columnNames.resize(tfields); columnNames.reserve(tfields); QVector tform; tform.resize(tfields); tform.reserve(tfields); QVector tunit; tunit.resize(tfields); tunit.reserve(tfields); for (int i = 0; i < tfields; ++i) { const Column* const column = spreadsheet->column(i); columnNames[i] = new char[column->name().size()]; strcpy(columnNames[i], column->name().toLatin1().data()); if (commentsAsUnits) { tunit[i] = new char[column->comment().size()]; strcpy(tunit[i], column->comment().toLatin1().constData()); } else { tunit[i] = new char[2]; strcpy(tunit[i], ""); } switch (column->columnMode()) { case AbstractColumn::Numeric: { int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (QString::number(column->valueAt(row)).size() > maxSize) maxSize = QString::number(column->valueAt(row)).size(); } const Double2StringFilter* const filter = static_cast(column->outputFilter()); bool decimals = false; for (int ii = 0; ii < nrows; ++ii) { bool ok; QString cell = column->asStringColumn()->textAt(ii); double val = cell.toDouble(&ok); if (cell.size() > QString::number(val).size() + 1) { decimals = true; break; } } QString tformn; if (decimals) { int maxStringSize = -1; for (int row = 0; row < nrows; ++row) { if (column->asStringColumn()->textAt(row).size() > maxStringSize) maxStringSize = column->asStringColumn()->textAt(row).size(); } const int diff = abs(maxSize - maxStringSize); maxSize+= diff; tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".") + QString::number(filter->numDigits()); } else tformn = QLatin1String("F")+ QString::number(maxSize) + QLatin1String(".0"); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); break; } case AbstractColumn::Text: { int maxSize = -1; for (int row = 0; row < nrows; ++row) { if (column->textAt(row).size() > maxSize) maxSize = column->textAt(row).size(); } const QString& tformn = QLatin1String("A") + QString::number(maxSize); tform[i] = new char[tformn.size()]; strcpy(tform[i], tformn.toLatin1().data()); break; } case AbstractColumn::Integer: //TODO case AbstractColumn::DateTime: case AbstractColumn::Day: case AbstractColumn::Month: break; } } //TODO extension name containing[] ? if (fits_create_tbl(m_fitsFile, ASCII_TBL, nrows, tfields, columnNames.data(), tform.data(), tunit.data(), spreadsheet->name().toLatin1().data(),&status )) { printError(status); qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } qDeleteAll(tform); qDeleteAll(tunit); qDeleteAll(columnNames); QVector column; column.resize(nrows); column.reserve(nrows); double* columnNumeric = new double[nrows]; bool hadTextColumn = false; for (int col = 1; col <= tfields; ++col) { const Column* c = spreadsheet->column(col-1); AbstractColumn::ColumnMode columnMode = c->columnMode(); if (columnMode == AbstractColumn::Numeric) { for (int row = 0; row < nrows; ++row) columnNumeric[row] = c->valueAt(row); fits_write_col(m_fitsFile, TDOUBLE, col, 1, 1, nrows, columnNumeric, &status); if (status) { printError(status); delete[] columnNumeric; status = 0; fits_close_file(m_fitsFile, &status); if (!existed) { QFile file(fileName); file.remove(); } return; } } else { hadTextColumn = true; for (int row = 0; row < nrows; ++row) { column[row] = new char[c->textAt(row).size()]; strcpy(column[row], c->textAt(row).toLatin1().data()); } fits_write_col(m_fitsFile, TSTRING, col, 1, 1, nrows, column.data(), &status); if (status) { printError(status); qDeleteAll(column); status = 0; fits_close_file(m_fitsFile, &status); return; } } } delete[] columnNumeric; if (hadTextColumn) qDeleteAll(column); status = 0; fits_close_file(m_fitsFile, &status); } } #else Q_UNUSED(fileName) Q_UNUSED(dataSource) #endif } /*! * \brief Return a map of the available extensions names in file \a filename * The keys of the map are the extension types, the values are the names * \param fileName the name of the FITS file to be analyzed */ QMultiMap FITSFilterPrivate::extensionNames(const QString& fileName) { DEBUG("FITSFilterPrivate::extensionNames() file name = " << fileName.toStdString()); #ifdef HAVE_FITS QMultiMap extensions; int status = 0; fitsfile* fitsFile = 0; if (fits_open_file(&fitsFile, fileName.toLatin1(), READONLY, &status )) return QMultiMap(); int hduCount; if (fits_get_num_hdus(fitsFile, &hduCount, &status)) return QMultiMap(); int imageCount = 0; int asciiTableCount = 0; int binaryTableCount = 0; for (int currentHDU = 1; (currentHDU <= hduCount) && !status; ++currentHDU) { int hduType; status = 0; fits_get_hdu_type(fitsFile, &hduType, &status); switch (hduType) { case IMAGE_HDU: imageCount++; break; case ASCII_TBL: asciiTableCount++; break; case BINARY_TBL: binaryTableCount++; break; } char* keyVal = new char[FLEN_VALUE]; QString extName; if (!fits_read_keyword(fitsFile,"EXTNAME", keyVal, NULL, &status)) { extName = QLatin1String(keyVal); extName = extName.mid(1, extName.length() -2).simplified(); } else { status = 0; if (!fits_read_keyword(fitsFile,"HDUNAME", keyVal, NULL, &status)) { extName = QLatin1String(keyVal); extName = extName.mid(1, extName.length() -2).simplified(); } else { status = 0; switch (hduType) { case IMAGE_HDU: if (imageCount == 1) extName = i18n("Primary header"); else extName = i18n("IMAGE #%1", imageCount); break; case ASCII_TBL: extName = i18n("ASCII_TBL #%1", asciiTableCount); break; case BINARY_TBL: extName = i18n("BINARY_TBL #%1", binaryTableCount); break; } } } delete[] keyVal; status = 0; extName = extName.trimmed(); switch (hduType) { case IMAGE_HDU: extensions.insert(QLatin1String("IMAGES"), extName); break; case ASCII_TBL: extensions.insert(QLatin1String("TABLES"), extName); break; case BINARY_TBL: extensions.insert(QLatin1String("TABLES"), extName); break; } fits_movrel_hdu(fitsFile, 1, NULL, &status); } if (status == END_OF_FILE) status = 0; fits_close_file(fitsFile, &status); return extensions; #else Q_UNUSED(fileName) return QMultiMap(); #endif } /*! * \brief Prints the error text corresponding to the status code \a status * \param status the status code of the error */ void FITSFilterPrivate::printError(int status) const { #ifdef HAVE_FITS if (status) { char errorText[FLEN_ERRMSG]; fits_get_errstatus(status, errorText ); qDebug() << QLatin1String(errorText); } #else Q_UNUSED(status) #endif } /*! * \brief Add the keywords \a keywords to the current header unit * \param keywords the keywords to be added * \param fileName the name of the FITS file (extension) in which the keywords are added */ void FITSFilterPrivate::addNewKeyword(const QString& fileName, const QList& keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const FITSFilter::Keyword& keyword: keywords) { status = 0; if (!keyword.key.compare(QLatin1String("COMMENT"))) { if (fits_write_comment(m_fitsFile, keyword.value.toLatin1(), &status)) printError(status); } else if (!keyword.key.compare(QLatin1String("HISTORY"))) { if (fits_write_history(m_fitsFile, keyword.value.toLatin1(), &status)) printError(status); } else if (!keyword.key.compare(QLatin1String("DATE"))) { if (fits_write_date(m_fitsFile, &status)) printError(status); } else { int ok = 0; if (keyword.key.length() <= FLEN_KEYWORD) { ok++; if (keyword.value.length() <= FLEN_VALUE) { ok++; if(keyword.comment.length() <= FLEN_COMMENT) ok++; } } if (ok == 3) { bool ok; double val = keyword.value.toDouble(&ok); if (ok) { if (fits_write_key(m_fitsFile, TDOUBLE, keyword.key.toLatin1().data(), &val, keyword.comment.toLatin1().data(), &status)) printError(status); } else { if (fits_write_key(m_fitsFile, TSTRING, keyword.key.toLatin1().data(), keyword.value.toLatin1().data(), keyword.comment.toLatin1().data(), &status)) printError(status); } } else if ( ok == 2) { //comment too long } else if ( ok == 1) { //value too long } else { //keyword too long } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(keywords) Q_UNUSED(fileName) #endif } /*! * \brief Update keywords in the current header unit * \param fileName The name of the FITS file (extension) in which the keywords will be updated * \param originals The original keywords of the FITS file (extension) * \param updates The keywords that contain the updated values */ void FITSFilterPrivate::updateKeywords(const QString& fileName, const QList& originals, const QVector& updates) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } FITSFilter::Keyword updatedKeyword; FITSFilter::Keyword originalKeyword; FITSFilter::KeywordUpdate keywordUpdate; for (int i = 0; i < updates.size(); ++i) { updatedKeyword = updates.at(i); originalKeyword = originals.at(i); keywordUpdate = originals.at(i).updates; if (keywordUpdate.keyUpdated && keywordUpdate.valueUpdated && keywordUpdate.commentUpdated) { if (updatedKeyword.isEmpty()) { if (fits_delete_key(m_fitsFile, originalKeyword.key.toLatin1(), &status)) { printError(status); status = 0; } continue; } } if (!updatedKeyword.key.isEmpty()) { if (fits_modify_name(m_fitsFile, originalKeyword.key.toLatin1(), updatedKeyword.key.toLatin1(), &status )) { printError(status); status = 0; } } if (!updatedKeyword.value.isEmpty()) { bool ok; int intValue; double doubleValue; bool updated = false; doubleValue = updatedKeyword.value.toDouble(&ok); if (ok) { if (fits_update_key(m_fitsFile,TDOUBLE, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), &doubleValue, NULL, &status)) printError(status); else updated = true; } if (!updated) { intValue = updatedKeyword.value.toInt(&ok); if (ok) { if (fits_update_key(m_fitsFile,TINT, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), &intValue, NULL, &status)) printError(status); else updated = true; } } if (!updated) { if (fits_update_key(m_fitsFile,TSTRING, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), updatedKeyword.value.toLatin1().data(), NULL, &status)) printError(status); } } else { if (keywordUpdate.valueUpdated) { if (fits_update_key_null(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), NULL, &status)) { printError(status); status = 0; } } } if (!updatedKeyword.comment.isEmpty()) { if (fits_modify_comment(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), updatedKeyword.comment.toLatin1().data(), &status)) { printError(status); status = 0; } } else { if (keywordUpdate.commentUpdated) { if (fits_modify_comment(m_fitsFile, keywordUpdate.keyUpdated ? updatedKeyword.key.toLatin1() : originalKeyword.key.toLatin1(), QString("").toLatin1().data(), &status)) { printError(status); status = 0; } } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(originals) Q_UNUSED(updates) #endif } /*! * \brief Delete the keywords \a keywords from the current header unit * \param fileName the name of the FITS file (extension) in which the keywords will be deleted. * \param keywords the keywords to deleted */ void FITSFilterPrivate::deleteKeyword(const QString& fileName, const QList &keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for (const auto& keyword : keywords) { if (!keyword.key.isEmpty()) { status = 0; if (fits_delete_key(m_fitsFile, keyword.key.toLatin1(), &status)) printError(status); } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(keywords) Q_UNUSED(fileName) #endif } /*! * \brief FITSFilterPrivate::addKeywordUnit * \param fileName the FITS file (extension) in which the keyword units are updated/added * \param keywords the keywords whose units were modified/added */ void FITSFilterPrivate::addKeywordUnit(const QString &fileName, const QList &keywords) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READWRITE, &status )) { printError(status); return; } for(const FITSFilter::Keyword& keyword : keywords) { if (keyword.updates.unitUpdated) { if (fits_write_key_unit(m_fitsFile, keyword.key.toLatin1(), keyword.unit.toLatin1().data(), &status)) { printError(status); status = 0; } } } status = 0; fits_close_file(m_fitsFile, &status); #else Q_UNUSED(fileName) Q_UNUSED(keywords) #endif } /*! * \brief Remove extensions from a FITS file * \param extensions The extensions to be removed from the FITS file */ void FITSFilterPrivate::removeExtensions(const QStringList &extensions) { #ifdef HAVE_FITS int status = 0; for (const auto& ext : extensions) { status = 0; if (fits_open_file(&m_fitsFile, ext.toLatin1(), READWRITE, &status )) { printError(status); continue; } if (fits_delete_hdu(m_fitsFile, NULL, &status)) printError(status); status = 0; fits_close_file(m_fitsFile, &status); } #else Q_UNUSED(extensions) #endif } /*! * \brief Returns a list of keywords in the current header of \a fileName * \param fileName the name of the FITS file (extension) to be opened * \return A list of keywords */ QList FITSFilterPrivate::chduKeywords(const QString& fileName) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status )) { printError(status); return QList (); } int numberOfKeys; if (fits_get_hdrspace(m_fitsFile, &numberOfKeys, NULL, &status)) { printError(status); return QList (); } QList keywords; keywords.reserve(numberOfKeys); char* key = new char[FLEN_KEYWORD]; char* value = new char[FLEN_VALUE]; char* comment = new char[FLEN_COMMENT]; char* unit = new char[FLEN_VALUE]; for (int i = 1; i <= numberOfKeys; ++i) { QStringList recordValues; FITSFilter::Keyword keyword; if (fits_read_keyn(m_fitsFile, i, key, value, comment, &status)) { printError(status); status = 0; continue; } fits_read_key_unit(m_fitsFile, key, unit, &status); recordValues << QLatin1String(key) << QLatin1String(value) << QLatin1String(comment) << QLatin1String(unit); keyword.key = recordValues[0].simplified(); keyword.value = recordValues[1].simplified(); keyword.comment = recordValues[2].simplified(); keyword.unit = recordValues[3].simplified(); keywords.append(keyword); } delete[] key; delete[] value; delete[] comment; delete[] unit; fits_close_file(m_fitsFile, &status); return keywords; #else Q_UNUSED(fileName) return QList(); #endif } /*! * \brief Builds the table \a headerEditTable from the keywords \a keys * \param fileName The name of the FITS file from which the keys are read if \a readKeys is \c true * \param headerEditTable The table to be built * \param readKeys It's used to determine whether the keywords are provided or they should be read from * file \a fileName * \param keys The keywords that are provided if the keywords were read already. */ void FITSFilterPrivate::parseHeader(const QString &fileName, QTableWidget *headerEditTable, bool readKeys, const QList& keys) { #ifdef HAVE_FITS QList keywords; if (readKeys) keywords = chduKeywords(fileName); else keywords = keys; headerEditTable->setRowCount(keywords.size()); QTableWidgetItem* item; for (int i = 0; i < keywords.size(); ++i) { const FITSFilter::Keyword& keyword = keywords.at(i); const bool mandatory = FITSFilter::mandatoryImageExtensionKeywords().contains(keyword.key) || FITSFilter::mandatoryTableExtensionKeywords().contains(keyword.key); item = new QTableWidgetItem(keyword.key); const QString& itemText = item->text(); const bool notEditableKey = mandatory || itemText.contains(QLatin1String("TFORM")) || itemText.contains(QLatin1String("TTYPE")) || itemText.contains(QLatin1String("TUNIT")) || itemText.contains(QLatin1String("TDISP")) || itemText.contains(QLatin1String("TBCOL")) || itemText.contains(QLatin1String("TZERO")); const bool notEditableValue= mandatory || itemText.contains(QLatin1String("TFORM")) || itemText.contains(QLatin1String("TDISP")) || itemText.contains(QLatin1String("TBCOL")) || itemText.contains(QLatin1String("TZERO")); if (notEditableKey) item->setFlags(item->flags() & ~Qt::ItemIsEditable); else item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 0, item ); item = new QTableWidgetItem(keyword.value); if (notEditableValue) item->setFlags(item->flags() & ~Qt::ItemIsEditable); else item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 1, item ); QString commentFieldText; if (!keyword.unit.isEmpty()) { if (keyword.updates.unitUpdated) { const QString& comment = keyword.comment.right( keyword.comment.size() - keyword.comment.indexOf(QChar(']'))-1); commentFieldText = QLatin1String("[") + keyword.unit + QLatin1String("] ") + comment; } else { if (keyword.comment.at(0) == QLatin1Char('[')) commentFieldText = keyword.comment; else commentFieldText = QLatin1String("[") + keyword.unit + QLatin1String("] ") + keyword.comment; } } else commentFieldText = keyword.comment; item = new QTableWidgetItem(commentFieldText); item->setFlags(Qt::ItemIsEditable | Qt::ItemIsSelectable | Qt::ItemIsEnabled); headerEditTable->setItem(i, 2, item ); } headerEditTable->resizeColumnsToContents(); #else Q_UNUSED(fileName) Q_UNUSED(headerEditTable) Q_UNUSED(readKeys) Q_UNUSED(keys) #endif } /*! * \brief Helper function to return the value of the key \a key * \param fileName The name of the FITS file (extension) in which the keyword with key \a key should exist * \param key The key of the keyword whose value it's returned * \return The value of the keyword as a string */ const QString FITSFilterPrivate::valueOf(const QString& fileName, const char *key) { #ifdef HAVE_FITS int status = 0; if (fits_open_file(&m_fitsFile, fileName.toLatin1(), READONLY, &status )) { printError(status); return QString (); } char* keyVal = new char[FLEN_VALUE]; QString keyValue; if (!fits_read_keyword(m_fitsFile, key, keyVal, NULL, &status)) { keyValue = QLatin1String(keyVal); keyValue = keyValue.simplified(); } else { printError(status); delete[] keyVal; fits_close_file(m_fitsFile, &status); return QString(); } delete[] keyVal; status = 0; fits_close_file(m_fitsFile, &status); return keyValue; #else Q_UNUSED(fileName) Q_UNUSED(key) return QString(); #endif } /*! * \brief Build the extensions tree from FITS file (extension) \a fileName * \param fileName The name of the FITS file to be opened * \param tw The QTreeWidget to be built * \param checkPrimary Used to determine whether the tree will be used for import or the header edit, * if it's \c true and if the primary array it's empty, then the item won't be added to the tree */ void FITSFilterPrivate::parseExtensions(const QString &fileName, QTreeWidget *tw, bool checkPrimary) { DEBUG("FITSFilterPrivate::parseExtensions()"); #ifdef HAVE_FITS const QMultiMap& extensions = extensionNames(fileName); const QStringList& imageExtensions = extensions.values(QLatin1String("IMAGES")); const QStringList& tableExtensions = extensions.values(QLatin1String("TABLES")); QTreeWidgetItem* root = tw->invisibleRootItem(); //TODO: fileName may contain any data type: check if it's a FITS file QTreeWidgetItem* treeNameItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << fileName); root->addChild(treeNameItem); treeNameItem->setExpanded(true); QTreeWidgetItem* imageExtensionItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << i18n("Images")); imageExtensionItem->setFlags(imageExtensionItem->flags() & ~Qt::ItemIsSelectable ); QString primaryHeaderNaxis = valueOf(fileName, "NAXIS"); const int naxis = primaryHeaderNaxis.toInt(); bool noImage = false; for (const QString& ext : imageExtensions) { QTreeWidgetItem* treeItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << ext); if (ext == i18n("Primary header")) { if (checkPrimary && naxis == 0) continue; } imageExtensionItem->addChild(treeItem); } if (imageExtensionItem->childCount() > 0) { treeNameItem->addChild(imageExtensionItem); imageExtensionItem->setIcon(0,QIcon::fromTheme("view-preview")); imageExtensionItem->setExpanded(true); imageExtensionItem->child(0)->setSelected(true); tw->setCurrentItem(imageExtensionItem->child(0)); } else noImage = true; if (tableExtensions.size() > 0) { QTreeWidgetItem* tableExtensionItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << i18n("Tables")); tableExtensionItem->setFlags(tableExtensionItem->flags() & ~Qt::ItemIsSelectable ); for (const QString& ext : tableExtensions) { QTreeWidgetItem* treeItem = new QTreeWidgetItem((QTreeWidgetItem*)0, QStringList() << ext); tableExtensionItem->addChild(treeItem); } if (tableExtensionItem->childCount() > 0) { treeNameItem->addChild(tableExtensionItem); tableExtensionItem->setIcon(0,QIcon::fromTheme("x-office-spreadsheet")); tableExtensionItem->setExpanded(true); if (noImage) { tableExtensionItem->child(0)->setSelected(true); tw->setCurrentItem(tableExtensionItem->child(0)); } } } #else Q_UNUSED(fileName) Q_UNUSED(tw) Q_UNUSED(checkPrimary) #endif DEBUG("FITSFilterPrivate::parseExtensions() DONE"); } /*! * \brief FITSFilterPrivate::~FITSFilterPrivate */ FITSFilterPrivate::~FITSFilterPrivate() { } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void FITSFilter::save(QXmlStreamWriter * writer) const { Q_UNUSED(writer) } /*! Loads from XML. */ bool FITSFilter::load(XmlStreamReader * loader) { Q_UNUSED(loader) return false; } diff --git a/src/backend/datasources/filters/FITSFilter.h b/src/backend/datasources/filters/FITSFilter.h index 3458ac3b0..b0c70ff41 100644 --- a/src/backend/datasources/filters/FITSFilter.h +++ b/src/backend/datasources/filters/FITSFilter.h @@ -1,119 +1,118 @@ /*************************************************************************** File : FITSFilter.h Project : LabPlot Description : FITS I/O-filter -------------------------------------------------------------------- Copyright : (C) 2016 by Fabian Kristof (fkristofszabolcs@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef FITSFILTER_H #define FITSFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include #include #include #include #include class FITSFilterPrivate; class FITSHeaderEditWidget; class FITSFilter : public AbstractFileFilter { Q_OBJECT public: FITSFilter(); ~FITSFilter() override; QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; void write(const QString& fileName, AbstractDataSource*) override; QVector readChdu(const QString& fileName, bool *okToMatrix = nullptr, int lines = -1); void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; struct KeywordUpdate { KeywordUpdate() : keyUpdated(false), valueUpdated(false), commentUpdated(false), unitUpdated(false) {} bool keyUpdated; bool valueUpdated; bool commentUpdated; bool unitUpdated; }; struct Keyword { Keyword(const QString& key, const QString& value, const QString& comment): key(key), value(value), comment(comment) {} Keyword() {} QString key; QString value; QString comment; QString unit; bool operator==(const Keyword& other) const { return other.key == key && other.value == value && other.comment == comment; } bool isEmpty() const { return key.isEmpty() && value.isEmpty() && comment.isEmpty(); } KeywordUpdate updates; }; - static int imagesCount(const QString& fileName); - static int tablesCount(const QString& fileName); + static QString fileInfoString(const QString&); void updateKeywords(const QString& fileName, const QList& originals, const QVector& updates); void addNewKeyword(const QString& filename, const QList& keywords); void addKeywordUnit(const QString& fileName, const QList& keywords); void deleteKeyword(const QString& fileName, const QList& keywords); void removeExtensions(const QStringList& extensions); void parseHeader(const QString &fileName, QTableWidget* headerEditTable, bool readKeys = true, const QList &keys = QList()); void parseExtensions(const QString& fileName, QTreeWidget* tw, bool checkPrimary = false); QList chduKeywords(const QString &fileName); static QStringList standardKeywords(); static QStringList mandatoryImageExtensionKeywords(); static QStringList mandatoryTableExtensionKeywords(); static QStringList units(); void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void setCommentsAsUnits(const bool); void setExportTo(const int); private: std::unique_ptr const d; friend class FITSFilterPrivate; }; #endif // FITSFILTER_H diff --git a/src/kdefrontend/datasources/FileInfoDialog.cpp b/src/kdefrontend/datasources/FileInfoDialog.cpp index c212b1d23..23a1dbb89 100644 --- a/src/kdefrontend/datasources/FileInfoDialog.cpp +++ b/src/kdefrontend/datasources/FileInfoDialog.cpp @@ -1,191 +1,203 @@ /*************************************************************************** File : FileInfoDialog.cpp Project : LabPlot Description : import file data dialog -------------------------------------------------------------------- Copyright : (C) 2009-2018 by Alexander Semke (alexander.semke@web.de) Copyright : (C) 2015-2016 Stefan-Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "FileInfoDialog.h" #include "backend/datasources/LiveDataSource.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/FITSFilter.h" #include #include #include #include #include #include #include #include #include /*! \class ImportWidget \brief Provides a dialog containing the information about the files to be imported. \ingroup kdefrontend */ FileInfoDialog::FileInfoDialog(QWidget* parent) : QDialog(parent) { m_textEditWidget.setReadOnly(true); m_textEditWidget.setLineWrapMode(QTextEdit::NoWrap); QVBoxLayout* layout = new QVBoxLayout(this); layout->addWidget(&m_textEditWidget); QDialogButtonBox *buttonBox = new QDialogButtonBox(QDialogButtonBox::Ok); connect(buttonBox, &QDialogButtonBox::rejected, this, &FileInfoDialog::reject); connect(buttonBox, &QDialogButtonBox::accepted, this, &FileInfoDialog::accept); layout->addWidget(buttonBox); setWindowIcon(QIcon::fromTheme("help-about")); setWindowTitle(i18nc("@title:window", "File Information")); setAttribute(Qt::WA_DeleteOnClose); setLayout(layout); QTimer::singleShot(0, this, &FileInfoDialog::loadSettings); } void FileInfoDialog::loadSettings() { //restore saved settings KConfigGroup conf(KSharedConfig::openConfig(), "FileInfoDialog"); KWindowConfig::restoreWindowSize(windowHandle(), conf); } FileInfoDialog::~FileInfoDialog() { //save current settings KConfigGroup conf(KSharedConfig::openConfig(), "FileInfoDialog"); KWindowConfig::saveWindowSize(windowHandle(), conf); } void FileInfoDialog::setFiles(QStringList& files) { QString infoString; for (const auto& fileName: files) { if(fileName.isEmpty()) continue; if (!infoString.isEmpty()) infoString += "


"; infoString += fileInfoString(fileName); } m_textEditWidget.document()->setHtml(infoString); } /*! returns a string containing the general information about the file \c name and some content specific information (number of columns and lines for ASCII, color-depth for images etc.). */ QString FileInfoDialog::fileInfoString(const QString& name) const { QString infoString; QFileInfo fileInfo; QString fileTypeString; QIODevice *file = new QFile(name); QString fileName; #ifdef Q_OS_WIN if (name.at(1) != QLatin1Char(':')) fileName = QDir::homePath() + name; else fileName = name; #else if (name.at(0) != QDir::separator()) fileName = QDir::homePath() + QDir::separator() + name; else fileName = name; #endif if(file==0) file = new QFile(fileName); if (file->open(QIODevice::ReadOnly)) { QStringList infoStrings; + infoStrings << "" + fileName + "
"; // file type and type specific information about the file #ifdef Q_OS_LINUX QProcess *proc = new QProcess(); QStringList args; args<<"-b"<start( "file", args); if(proc->waitForReadyRead(1000) == false) infoStrings << i18n("Could not open file %1 for reading.", fileName); else { fileTypeString = proc->readLine(); if( fileTypeString.contains(i18n("cannot open")) ) fileTypeString=""; else { fileTypeString.remove(fileTypeString.length()-1,1); // remove '\n' } } infoStrings << i18n("File type: %1", fileTypeString); #endif - //TODO depending on the file type, generate additional information about the file: - //Number of lines for ASCII, color-depth for images etc. Use the specific filters here. - // port the old labplot1.6 code. - if( fileTypeString.contains("ASCII")) { - infoStrings << "
"; - //TODO: consider choosen separator - infoStrings << i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName)); - - infoStrings << i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); - } - infoString += infoStrings.join("
"); - -#ifdef HAVE_FITS - if (fileName.endsWith(QLatin1String(".fits"))) { - infoStrings << i18n("Images: %1", QString::number(FITSFilter::imagesCount(fileName) )); - infoStrings << i18n("Tables: %1", QString::number(FITSFilter::tablesCount(fileName) )); + //depending on the file type, generate additional information about the file: + infoStrings << "
"; + AbstractFileFilter::FileType fileType = AbstractFileFilter::fileType(fileName); + switch(fileType) { + case AbstractFileFilter::Ascii: + infoStrings << AsciiFilter::fileInfoString(fileName); + break; + case AbstractFileFilter::Binary: + //TODO infoStrings << BinaryFilter::fileInfoString(fileName); + break; + case AbstractFileFilter::Image: + //TODO infoStrings << ImageFilter::fileInfoString(fileName); + break; + case AbstractFileFilter::HDF5: + //TODO infoStrings << HDF5Filter::fileInfoString(fileName); + break; + case AbstractFileFilter::NETCDF: + //TODO infoStrings << NETCDFFilter::fileInfoString(fileName); + break; + case AbstractFileFilter::FITS: + infoStrings << FITSFilter::fileInfoString(fileName); + break; + case AbstractFileFilter::ROOT: + //TODO infoStrings << ROOTFilter::fileInfoString(fileName); + break; + case AbstractFileFilter::NgspiceRawAscii: + //TODO infoStrings << NgspiceRawAsciiFilter::fileInfoString(fileName); + break; } -#endif //general information about the file - infoStrings << "

"; - infoStrings << "" + fileName + "
"; + infoStrings << "
"; fileInfo.setFile(fileName); infoStrings << i18n("Readable: %1", fileInfo.isReadable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Writable: %1", fileInfo.isWritable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Executable: %1", fileInfo.isExecutable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Created: %1", fileInfo.created().toString()); infoStrings << i18n("Last modified: %1", fileInfo.lastModified().toString()); infoStrings << i18n("Last read: %1", fileInfo.lastRead().toString()); infoStrings << i18n("Owner: %1", fileInfo.owner()); infoStrings << i18n("Group: %1", fileInfo.group()); infoStrings << i18n("Size: %1", i18np("%1 cByte", "%1 cBytes", fileInfo.size())); + infoString += infoStrings.join("
"); } else infoString += i18n("Could not open file %1 for reading.", fileName); return infoString; }