diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index 4ac2facb0..386ec3ece 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,1449 +1,1465 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2017 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include "backend/core/Project.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" #include "backend/lib/macros.h" #include "backend/lib/trace.h" #include #include #include #include #include #include /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() {} /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource * dataSource) { d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { return d->readFromLiveDevice(device, dataSource, from); } /*! reads the content of the file \c fileName. */ QVector AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromFile(fileName, dataSource, importMode, lines); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } QVector AsciiFilter::preview(QIODevice &device) { return d->preview(device); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; i++) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of lines"); return 0; } size_t lineCount = 0; while (!device.atEnd()) { device.readLine(); lineCount++; } //TODO: wc is much faster but not portable /* QElapsedTimer myTimer; myTimer.start(); QProcess wc; wc.start(QString("wc"), QStringList() << "-l" << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) lineCount = wc.readLine().split(' ')[0].toInt(); lineCount++; // last line not counted DEBUG(" Elapsed time counting lines : " << myTimer.elapsed() << " ms"); */ return lineCount; } /*! returns the number of lines in the device \c device or 0 if not available. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice &device) { // device.hasReadLine() always returns 0 for KFilterDev! if (device.isSequential()) return 0; size_t lineCount = 0; device.seek(0); while (!device.atEnd()) { device.readLine(); lineCount++; } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool AsciiFilter::NaNValueToZeroEnabled() const { if (d->nanValue == 0) return true; return false; } +void AsciiFilter::setRemoveQuotesEnabled(bool b) { + d->removeQuotesEnabled = b; +} +bool AsciiFilter::removeQuotesEnabled() const { + return d->removeQuotesEnabled; +} + void AsciiFilter::setVectorNames(const QString s) { d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner), commentCharacter("#"), separatingCharacter("auto"), autoModeEnabled(true), headerEnabled(true), skipEmptyParts(false), simplifyWhitespacesEnabled(true), nanValue(NAN), + removeQuotesEnabled(false), createIndexEnabled(false), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_prepared(false), m_columnOffset(0) { } QStringList AsciiFilterPrivate::getLineString(QIODevice& device) { QString line; do { // skip comment lines in data lines line = device.readLine(); } while (line.startsWith(commentCharacter)); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); DEBUG("data line : \'" << line.toStdString() << '\''); QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); + //TODO: remove quotes here? QDEBUG("data line, parsed: " << lineStringList); return lineStringList; } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; DEBUG("device is sequential = " << device.isSequential()); ///////////////////////////////////////////////////////////////// // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines firstLine = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, QString::SkipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, QString::SkipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); DEBUG("headerEnabled = " << headerEnabled); if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); startRow++; } // set range to read if (endColumn == -1) { if (headerEnabled) endColumn = firstLineStringList.size(); // last column else endColumn = vectorNames.size(); //number of vector names provided in the import dialog } if (createIndexEnabled) { vectorNames.prepend("index"); endColumn++; } m_actualCols = endColumn - startColumn + 1; //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ - // ATTENTION: This resets the position to start of file + // ATTENTION: This resets the position in the device to 0 m_actualRows = AsciiFilter::lineNumber(device); ///////////////////////////////////////////////////////////////// // Find first data line (ignoring comment lines) DEBUG("Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { QString line = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } if (line.startsWith(commentCharacter)) // ignore commented lines before startRow i--; } // parse first data line to determine data type for each column if (!device.isSequential()) firstLineStringList = getLineString(device); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Integer; col = 1; } for (const auto& valueString: firstLineStringList) { // parse columns available in first data line if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } // parsing more lines to better determine data types for (unsigned int i = 0; i < m_dataTypeLines; i++) { firstLineStringList = getLineString(device); if (createIndexEnabled) col = 1; else col = 0; for (const auto& valueString: firstLineStringList) { if (col == m_actualCols) break; AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); // numeric: integer -> numeric if (mode == AbstractColumn::Numeric && columnModes[col] == AbstractColumn::Integer) columnModes[col] = mode; // text: non text -> text if (mode == AbstractColumn::Text && columnModes[col] != AbstractColumn::Text) columnModes[col] = mode; col++; } } QDEBUG("column modes = " << columnModes); // reset to start of file if (!device.isSequential()) device.seek(0); ///////////////////////////////////////////////////////////////// int actualEndRow = endRow; DEBUG("endRow = " << endRow); if (endRow == -1 || endRow > m_actualRows) actualEndRow = m_actualRows; if (m_actualRows > actualEndRow) m_actualRows = actualEndRow; DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << startRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header incl. start rows): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0 && !device.isSequential()) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode, lines); } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return 0; } Q_ASSERT(dataSource != nullptr); LiveDataSource* spreadsheet = dynamic_cast(dataSource); if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe) if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return 0; if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return 0; } // prepare import for spreadsheet spreadsheet->setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); qDebug() << "fds resized to col: " << m_actualCols; qDebug() << "fds rowCount: " << spreadsheet->rowCount(); //columns in a file data source don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } //also here we need a cheaper version of this if (!spreadsheet->keepLastValues()) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(spreadsheet->keepNvalues()); m_actualRows = spreadsheet->keepNvalues(); } m_dataContainer.resize(m_actualCols); for (int n = 0; n < m_actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } qDebug() << "prepared!"; } qint64 bytesread = 0; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif LiveDataSource::ReadingType readingType; if (!m_prepared) readingType = LiveDataSource::ReadingType::TillEnd; else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = LiveDataSource::ReadingType::TillEnd; else readingType = spreadsheet->readingType(); } //move to the last read position, from == total bytes read //since the other source types are sequencial we cannot seek on them if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); qDebug() <<"available bytes: " << device.bytesAvailable(); //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleRateNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != LiveDataSource::ReadingType::TillEnd) { newData.reserve(spreadsheet->sampleRate()); newData.resize(spreadsheet->sampleRate()); } int newDataIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif while (!device.atEnd()) { if (readingType != LiveDataSource::ReadingType::TillEnd) newData[newDataIdx++] = device.readLine(); else newData.push_back(device.readLine()); newLinesTillEnd++; if (readingType != LiveDataSource::ReadingType::TillEnd) { newLinesForSampleRateNotTillEnd++; //for Continous reading and FromEnd we read sample rate number of lines if possible if (newLinesForSampleRateNotTillEnd == spreadsheet->sampleRate()) break; } } } //now we reset the readingType if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = spreadsheet->readingType(); //we had less new lines than the sample rate specified if (readingType != LiveDataSource::ReadingType::TillEnd) qDebug() << "Removed empty lines: " << newData.removeAll(""); //back to the last read position before counting when reading from files if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); int currentRow; // indexes the position in the vector(column) int linesToRead; if (!m_prepared) linesToRead = newLinesTillEnd; if (m_prepared) { //increase row count if we don't have a fixed size //but only after the preparation step if (!spreadsheet->keepLastValues()) { if (readingType != LiveDataSource::ReadingType::TillEnd) m_actualRows += qMin(newData.size(), spreadsheet->sampleRate()); else m_actualRows += newData.size(); } //fixed size if (spreadsheet->keepLastValues()) { if (readingType == LiveDataSource::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; //TODO after reading we should skip the next data lines //because it's TillEnd actually } else linesToRead = newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinouslyFixed or FromEnd linesToRead = qMin(spreadsheet->sampleRate(), newLinesTillEnd); } } else { //appending linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; } } if (m_prepared && (linesToRead == 0)) return 0; //new rows/resize columns if we don't have a fixed size //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize if (!spreadsheet->keepLastValues()) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { //when we have a fixed size we have to pop sampleRate number of lines if specified //here popping, setting currentRow if (!m_prepared) currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); else { if (readingType == LiveDataSource::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinouslyFixed or FromEnd currentRow = m_actualRows - qMin(spreadsheet->sampleRate(), newLinesTillEnd); } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet qDebug() << "reading from line: " << currentRow << " lines till end: " << newLinesTillEnd; qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows; newDataIdx = 0; if (readingType == LiveDataSource::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->sampleRate()) newDataIdx = newData.size() - spreadsheet->sampleRate(); //since we skip a couple of lines, we need to count those bytes too for (int i = 0; i < newDataIdx; ++i) bytesread += newData.at(i).size(); } } qDebug() << "newDataIdx: " << newDataIdx; //TODO static int indexColumnIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif for (int i = 0; i < linesToRead; ++i) { QString line; if (readingType == LiveDataSource::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(i); if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) bytesread += line.size(); //qDebug() << "line bytes: " << line.size() << " line: " << line; //qDebug() << "reading in row: " << currentRow; if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); if (createIndexEnabled) { if (spreadsheet->keepLastValues()) lineStringList.prepend(QString::number(indexColumnIdx++)); else lineStringList.prepend(QString::number(currentRow)); } for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); //qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); //qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); const Project* project = spreadsheet->project(); QVector curves = project->children(AbstractAspect::Recursive); QVector plots; for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); //determine the plots where the column is consumed for (const auto* curve: curves) { if (curve->xColumn() == column || curve->yColumn() == column) { CartesianPlot* plot = dynamic_cast(curve->parentAspect()); if (plots.indexOf(plot) == -1) { plots << plot; plot->setSuppressDataChangedSignal(true); } } } column->setChanged(); } //loop over all affected plots and retransform them for (auto* plot: plots) { //TODO setting this back to true triggers again a lot of retransforms in the plot (one for each curve). // plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } m_prepared = true; return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); Q_ASSERT(dataSource != nullptr); if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) DEBUG("Device error = " << deviceError); if (deviceError == 1 && importMode == AbstractFileFilter::Replace && dataSource) dataSource->clear(); if (deviceError) return; // matrix data has only one column mode (which is not text) if (dynamic_cast(dataSource)) { auto mode = columnModes[0]; (mode == AbstractColumn::Text) ? mode = AbstractColumn::Numeric : 0; for (auto& c: columnModes) (c != mode) ? c = mode : 0; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows - startRow + 1, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); i++) { QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend the index if required //TODO: come up maybe with a solution with adding the index inside of the loop below, //without conversion to string, prepending to the list and then conversion back to integer. if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, dateTimeFormat, importMode); } QVector AsciiFilterPrivate::preview(QIODevice &device) { QVector dataStrings; if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return dataStrings; } if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return dataStrings; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif int linesToRead = 0; QVector newData; while (!device.atEnd()) { newData.push_back(device.readLine()); linesToRead++; } if (linesToRead == 0) return dataStrings; int col = 0; int colMax = newData.at(0).size(); if (createIndexEnabled) colMax++; columnModes.resize(colMax); if (createIndexEnabled) { columnModes[0] = AbstractColumn::ColumnMode::Integer; col = 1; vectorNames.prepend("index"); } if (headerEnabled) { int i = 0; if (createIndexEnabled) i = 1; for (; i < vectorNames.size(); ++i) vectorNames[i] = "Column " + QString::number(i); } for (const auto& valueString: newData.at(0).split(' ', QString::SkipEmptyParts)) { if (col == colMax) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } for (int i = 0; i < linesToRead; ++i) { QString line = newData.at(i); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(' ', QString::SkipEmptyParts); if (createIndexEnabled) lineStringList.prepend(QString::number(i)); QStringList lineString; for (int n = 0; n < lineStringList.size(); n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); i++) { QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend index if required if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); QStringList lineString; for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to ascii file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement( "asciiFilter"); writer->writeAttribute( "commentCharacter", d->commentCharacter); writer->writeAttribute( "separatingCharacter", d->separatingCharacter); writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); writer->writeAttribute( "nanValue", QString::number(d->nanValue)); + writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled)); writer->writeAttribute( "startRow", QString::number(d->startRow)); writer->writeAttribute( "endRow", QString::number(d->endRow)); writer->writeAttribute( "startColumn", QString::number(d->startColumn)); writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "asciiFilter") { reader->raiseError(i18n("no ascii filter element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("commentCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'commentCharacter'")); else d->commentCharacter = str; str = attribs.value("separatingCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'separatingCharacter'")); else d->separatingCharacter = str; str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'autoMode'")); else d->autoModeEnabled = str.toInt(); str = attribs.value("header").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'header'")); else d->headerEnabled = str.toInt(); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty str = attribs.value("simplifyWhitespaces").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'simplifyWhitespaces'")); else d->simplifyWhitespacesEnabled = str.toInt(); str = attribs.value("nanValue").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'nanValue'")); else d->nanValue = str.toDouble(); + str = attribs.value("removeQuotes").toString(); + if (str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'removeQuotes'")); + else + d->removeQuotesEnabled = str.toInt(); + str = attribs.value("skipEmptyParts").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipEmptyParts'")); else d->skipEmptyParts = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startColumn'")); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endColumn'")); else d->endColumn = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/AsciiFilter.h b/src/backend/datasources/filters/AsciiFilter.h index 74a3bd9a9..f0d988c4e 100644 --- a/src/backend/datasources/filters/AsciiFilter.h +++ b/src/backend/datasources/filters/AsciiFilter.h @@ -1,113 +1,115 @@ /*************************************************************************** File : AsciiFilter.h Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTER_H #define ASCIIFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" class QStringList; class QIODevice; class AsciiFilterPrivate; class QAbstractSocket; class AsciiFilter : public AbstractFileFilter { Q_OBJECT public: AsciiFilter(); ~AsciiFilter(); static QStringList separatorCharacters(); static QStringList commentCharacters(); static QStringList dataTypes(); static QStringList predefinedFilters(); static int columnNumber(const QString& fileName, const QString& separator = QString()); static size_t lineNumber(const QString& fileName); static size_t lineNumber(QIODevice&); // calculate number of lines if device supports it // read data from any device void readDataFromDevice(QIODevice& device, AbstractDataSource*, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*dataSource); qint64 readFromLiveDevice(QIODevice& device, AbstractDataSource*, qint64 from = -1); // overloaded function to read from file QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); QVector preview(QIODevice& device); void loadFilterSettings(const QString&); void saveFilterSettings(const QString&) const; void setCommentCharacter(const QString&); QString commentCharacter() const; void setSeparatingCharacter(const QString&); QString separatingCharacter() const; void setDateTimeFormat(const QString&); QString dateTimeFormat() const; void setNumberFormat(QLocale::Language); QLocale::Language numberFormat() const; void setAutoModeEnabled(const bool); bool isAutoModeEnabled() const; void setHeaderEnabled(const bool); bool isHeaderEnabled() const; void setSkipEmptyParts(const bool); bool skipEmptyParts() const; void setSimplifyWhitespacesEnabled(const bool); bool simplifyWhitespacesEnabled() const; void setNaNValueToZero(const bool); bool NaNValueToZeroEnabled() const; + void setRemoveQuotesEnabled(const bool); + bool removeQuotesEnabled() const; void setCreateIndexEnabled(const bool); void setVectorNames(const QString); QStringList vectorNames() const; QVector columnModes(); void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; virtual void save(QXmlStreamWriter*) const; virtual bool load(XmlStreamReader*); private: std::unique_ptr const d; friend class AsciiFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/AsciiFilterPrivate.h b/src/backend/datasources/filters/AsciiFilterPrivate.h index 8323e5889..ae0aff2be 100644 --- a/src/backend/datasources/filters/AsciiFilterPrivate.h +++ b/src/backend/datasources/filters/AsciiFilterPrivate.h @@ -1,84 +1,85 @@ /*************************************************************************** File : AsciiFilterPrivate.h Project : LabPlot Description : Private implementation class for AsciiFilter. -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTERPRIVATE_H #define ASCIIFILTERPRIVATE_H class KFilterDev; class AbstractDataSource; class AbstractColumn; class AsciiFilterPrivate { public: explicit AsciiFilterPrivate(AsciiFilter*); QStringList getLineString(QIODevice&); int prepareDeviceToRead(QIODevice&); void readDataFromDevice(QIODevice&, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); qint64 readFromLiveDevice(QIODevice&, AbstractDataSource*, qint64 from = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); QVector preview(QIODevice& device); const AsciiFilter* q; QString commentCharacter; QString separatingCharacter; QString dateTimeFormat; QLocale::Language numberFormat; bool autoModeEnabled; bool headerEnabled; bool skipEmptyParts; bool simplifyWhitespacesEnabled; double nanValue; + bool removeQuotesEnabled; bool createIndexEnabled; QStringList vectorNames; QVector columnModes; int startRow; int endRow; int startColumn; int endColumn; private: static const unsigned int m_dataTypeLines = 10; // lines to read for determining data types QString m_separator; int m_actualRows; int m_actualCols; int m_prepared; int m_columnOffset; // indexes the "start column" in the datasource. Data will be imported starting from this column. QVector m_dataContainer; // pointers to the actual data containers }; #endif diff --git a/src/commonfrontend/worksheet/WorksheetView.h b/src/commonfrontend/worksheet/WorksheetView.h index 90f07a6e2..e81157a0c 100644 --- a/src/commonfrontend/worksheet/WorksheetView.h +++ b/src/commonfrontend/worksheet/WorksheetView.h @@ -1,272 +1,272 @@ /*************************************************************************** File : WorksheetView.h Project : LabPlot Description : Worksheet view -------------------------------------------------------------------- Copyright : (C) 2009-2016 by Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef WORKSHEETVIEW_H #define WORKSHEETVIEW_H #include #include "backend/worksheet/Worksheet.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include class QMenu; class QToolBar; class QToolButton; class QWheelEvent; class QTimeLine; class AbstractAspect; class WorksheetElement; class WorksheetView : public QGraphicsView { Q_OBJECT public: explicit WorksheetView(Worksheet* worksheet); enum ExportFormat {Pdf, Svg, Png}; enum GridStyle {NoGrid, LineGrid, DotGrid}; enum ExportArea {ExportBoundingBox, ExportSelection, ExportWorksheet}; struct GridSettings { GridStyle style; QColor color; int horizontalSpacing; int verticalSpacing; float opacity; }; void setScene(QGraphicsScene*); void exportToFile(const QString&, const ExportFormat, const ExportArea, const bool, const int); void exportToClipboard(); void setIsClosing(); void setIsBeingPresented(bool presenting); private: enum MouseMode {SelectionMode, NavigationMode, ZoomSelectionMode}; enum CartesianPlotActionMode {ApplyActionToSelection, ApplyActionToAll}; void initActions(); void initMenus(); void processResize(); - void drawForeground(QPainter*, const QRectF&); - void drawBackground(QPainter*, const QRectF&); + void drawForeground(QPainter*, const QRectF&) override; + void drawBackground(QPainter*, const QRectF&) override; void drawBackgroundItems(QPainter*, const QRectF&); bool isPlotAtPos(const QPoint&) const; CartesianPlot* plotAt(const QPoint&) const; void exportPaint(QPainter* painter, const QRectF& targetRect, const QRectF& sourceRect, const bool); void cartesianPlotAdd(CartesianPlot*, QAction*); //events virtual void resizeEvent(QResizeEvent*) override; virtual void contextMenuEvent(QContextMenuEvent*) override; virtual void wheelEvent(QWheelEvent*) override; virtual void mousePressEvent(QMouseEvent*) override; virtual void mouseReleaseEvent(QMouseEvent*) override; virtual void mouseMoveEvent(QMouseEvent*) override; virtual void keyPressEvent(QKeyEvent*) override; virtual void dragEnterEvent(QDragEnterEvent*) override; virtual void dragMoveEvent(QDragMoveEvent*) override; virtual void dropEvent(QDropEvent*) override; Worksheet* m_worksheet; MouseMode m_mouseMode; CartesianPlotActionMode m_cartesianPlotActionMode; CartesianPlot::MouseMode m_cartesianPlotMouseMode; bool m_selectionBandIsShown; QPoint m_selectionStart; QPoint m_selectionEnd; int magnificationFactor; QGraphicsPixmapItem* m_magnificationWindow; GridSettings m_gridSettings; QList m_selectedItems; bool m_suppressSelectionChangedEvent; WorksheetElement* lastAddedWorksheetElement; QTimeLine* m_fadeInTimeLine; QTimeLine* m_fadeOutTimeLine; bool m_isClosing; bool m_menusInitialized; //Menus QMenu* m_addNewMenu; QMenu* m_addNewCartesianPlotMenu; QMenu* m_zoomMenu; QMenu* m_magnificationMenu; QMenu* m_layoutMenu; QMenu* m_gridMenu; QMenu* m_themeMenu; QMenu* m_viewMouseModeMenu; QMenu* m_cartesianPlotMenu; QMenu* m_cartesianPlotMouseModeMenu; QMenu* m_cartesianPlotAddNewMenu; QMenu* m_cartesianPlotZoomMenu; QMenu* m_cartesianPlotActionModeMenu; QMenu* m_dataManipulationMenu; QToolButton* tbNewCartesianPlot; QToolButton* tbZoom; QToolButton* tbMagnification; QAction* currentZoomAction; QAction* currentMagnificationAction; //Actions QAction* selectAllAction; QAction* deleteAction; QAction* backspaceAction; QAction* zoomInViewAction; QAction* zoomOutViewAction; QAction* zoomOriginAction; QAction* zoomFitPageHeightAction; QAction* zoomFitPageWidthAction; QAction* zoomFitSelectionAction; QAction* navigationModeAction; QAction* zoomSelectionModeAction; QAction* selectionModeAction; QAction* addCartesianPlot1Action; QAction* addCartesianPlot2Action; QAction* addCartesianPlot3Action; QAction* addCartesianPlot4Action; QAction* addTextLabelAction; QAction* addHistogram; QAction* addBarChartPlot; QAction* verticalLayoutAction; QAction* horizontalLayoutAction; QAction* gridLayoutAction; QAction* breakLayoutAction; QAction* noGridAction; QAction* denseLineGridAction; QAction* sparseLineGridAction; QAction* denseDotGridAction; QAction* sparseDotGridAction; QAction* customGridAction; QAction* snapToGridAction; QAction* noMagnificationAction; QAction* twoTimesMagnificationAction; QAction* threeTimesMagnificationAction; QAction* fourTimesMagnificationAction; QAction* fiveTimesMagnificationAction; QAction* showPresenterMode; //Actions for cartesian plots QAction* cartesianPlotApplyToSelectionAction; QAction* cartesianPlotApplyToAllAction; QAction* cartesianPlotSelectionModeAction; QAction* cartesianPlotZoomSelectionModeAction; QAction* cartesianPlotZoomXSelectionModeAction; QAction* cartesianPlotZoomYSelectionModeAction; QAction* addCurveAction; QAction* addEquationCurveAction; QAction* addDataOperationCurveAction; QAction* addDataReductionCurveAction; QAction* addDifferentiationCurveAction; QAction* addIntegrationCurveAction; QAction* addInterpolationCurveAction; QAction* addSmoothCurveAction; QAction* addFitCurveAction; QAction* addFourierFilterCurveAction; QAction* addFourierTransformCurveAction; QAction* addHorizontalAxisAction; QAction* addVerticalAxisAction; QAction* addLegendAction; QAction* addCustomPointAction; QAction* scaleAutoXAction; QAction* scaleAutoYAction; QAction* scaleAutoAction; QAction* zoomInAction; QAction* zoomOutAction; QAction* zoomInXAction; QAction* zoomOutXAction; QAction* zoomInYAction; QAction* zoomOutYAction; QAction* shiftLeftXAction; QAction* shiftRightXAction; QAction* shiftUpYAction; QAction* shiftDownYAction; // Analysis menu QAction* addDataOperationAction; QAction* addDataReductionAction; QAction* addDifferentiationAction; QAction* addIntegrationAction; QAction* addInterpolationAction; QAction* addSmoothAction; QAction* addFitAction; QAction* addFourierFilterAction; QAction* addFourierTransformAction; public slots: void createContextMenu(QMenu*); void createAnalysisMenu(QMenu*); void fillToolBar(QToolBar*); void fillCartesianPlotToolBar(QToolBar*); void print(QPrinter*); void selectItem(QGraphicsItem*); void presenterMode(); private slots: void addNew(QAction*); void aspectAboutToBeRemoved(const AbstractAspect*); void selectAllElements(); void deleteElement(); void mouseModeChanged(QAction*); void useViewSizeRequested(); void changeZoom(QAction*); void magnificationChanged(QAction*); void changeLayout(QAction*); void changeGrid(QAction*); void changeSnapToGrid(); void deselectItem(QGraphicsItem*); void selectionChanged(); void updateBackground(); void layoutChanged(Worksheet::Layout); void fadeIn(qreal); void fadeOut(qreal); //SLOTs for cartesian plots void cartesianPlotActionModeChanged(QAction*); void cartesianPlotMouseModeChanged(QAction*); void cartesianPlotNavigationChanged(QAction*); void cartesianPlotAddNew(QAction*); void handleCartesianPlotActions(); signals: void statusInfo(const QString&); }; #endif diff --git a/src/kdefrontend/datasources/AsciiOptionsWidget.cpp b/src/kdefrontend/datasources/AsciiOptionsWidget.cpp index d35146054..32908f08a 100644 --- a/src/kdefrontend/datasources/AsciiOptionsWidget.cpp +++ b/src/kdefrontend/datasources/AsciiOptionsWidget.cpp @@ -1,188 +1,191 @@ /*************************************************************************** File : AsciiOptionsWidget.h Project : LabPlot Description : widget providing options for the import of ascii data -------------------------------------------------------------------- Copyright : (C) 2009-2017 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "AsciiOptionsWidget.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/lib/macros.h" #include #include #include /*! \class AsciiOptionsWidget \brief Widget providing options for the import of ascii data \ingroup kdefrontend */ AsciiOptionsWidget::AsciiOptionsWidget(QWidget* parent) : QWidget(parent) { ui.setupUi(parent); ui.cbSeparatingCharacter->addItems(AsciiFilter::separatorCharacters()); ui.cbCommentCharacter->addItems(AsciiFilter::commentCharacters()); ui.cbNumberFormat->addItems(AbstractFileFilter::numberFormats()); ui.cbDateTimeFormat->addItems(AbstractColumn::dateTimeFormats()); const QString textNumberFormatShort = i18n("This option determines how the imported strings have to be converted to numbers."); const QString textNumberFormat = textNumberFormatShort + "

" + i18n( "For 'C Format', a period is used for the decimal point character and comma is used for the thousands group separator. " "Valid number representations are:" "
    " "
  • 1234.56
  • " "
  • 1,234.56
  • " "
  • etc.
  • " "
" "When using 'System locale', the system settings will be used. " "E.g., for the German local the valid number representations are:" "
    " "
  • 1234,56
  • " "
  • 1.234,56
  • " "
  • etc.
  • " "
" ); ui.lNumberFormat->setToolTip(textNumberFormatShort); ui.lNumberFormat->setWhatsThis(textNumberFormat); ui.cbNumberFormat->setToolTip(textNumberFormatShort); ui.cbNumberFormat->setWhatsThis(textNumberFormat); const QString textDateTimeFormatShort = i18n("This option determines how the imported strings have to be converted to calendar date, i.e. year, month, and day numbers in the Gregorian calendar and to time."); const QString textDateTimeFormat = textDateTimeFormatShort + "

" + i18n( "Expressions that may be used for the date part of format string:" "" "" "" "" "" "" "" "" "" "" "" "
dthe day as number without a leading zero (1 to 31).
ddthe day as number with a leading zero (01 to 31).
dddthe abbreviated localized day name (e.g. 'Mon' to 'Sun'). Uses the system locale to localize the name.
ddddthe long localized day name (e.g. 'Monday' to 'Sunday'). Uses the system locale to localize the name.
Mthe month as number without a leading zero (1 to 12).
MMthe month as number with a leading zero (01 to 12).
MMMthe abbreviated localized month name (e.g. 'Jan' to 'Dec'). Uses the system locale to localize the name.
MMMMthe long localized month name (e.g. 'January' to 'December'). Uses the system locale to localize the name.
yythe year as two digit number (00 to 99).
yyyythe year as four digit number. If the year is negative, a minus sign is prepended in addition.


" "Expressions that may be used for the time part of the format string:" "" "" "" "" "" "" "" "" "" "" "" "" "" "
hthe hour without a leading zero (0 to 23 or 1 to 12 if AM/PM display)
hhthe hour with a leading zero (00 to 23 or 01 to 12 if AM/PM display)
Hthe hour without a leading zero (0 to 23, even with AM/PM display)
HHthe hour with a leading zero (00 to 23, even with AM/PM display)
mthe minute without a leading zero (0 to 59)
mmthe minute with a leading zero (00 to 59)
sthe second without a leading zero (0 to 59)
ssthe second with a leading zero (00 to 59)
zthe milliseconds without leading zeroes (0 to 999)
zzzthe milliseconds with leading zeroes (000 to 999)
AP or Ainterpret as an AM/PM time. AP must be either 'AM' or 'PM'.
ap or aInterpret as an AM/PM time. ap must be either 'am' or 'pm'.


" "Examples are:" "" "" "" "" "
dd.MM.yyyy20.07.1969
ddd MMMM d yySun July 20 69
'The day is' ddddThe day is Sunday
"); ui.lDateTimeFormat->setToolTip(textDateTimeFormatShort); ui.lDateTimeFormat->setWhatsThis(textDateTimeFormat); ui.cbDateTimeFormat->setToolTip(textDateTimeFormatShort); ui.cbDateTimeFormat->setWhatsThis(textDateTimeFormat); connect(ui.chbHeader, SIGNAL(stateChanged(int)), SLOT(headerChanged(int))); } void AsciiOptionsWidget::showAsciiHeaderOptions(bool b) { DEBUG("AsciiOptionsWidget::showAsciiHeaderOptions(" << b << ")"); ui.chbHeader->setVisible(b); ui.lVectorNames->setVisible(b); ui.kleVectorNames->setVisible(b); } /*! enables a text field for the vector names if the option "Use the first row..." was not selected. Disables it otherwise. */ void AsciiOptionsWidget::headerChanged(int state) { DEBUG("AsciiOptionsWidget::headerChanged(" << state << ")"); if (state == Qt::Checked) { ui.kleVectorNames->setEnabled(false); ui.lVectorNames->setEnabled(false); } else { ui.kleVectorNames->setEnabled(true); ui.lVectorNames->setEnabled(true); } } void AsciiOptionsWidget::applyFilterSettings(AsciiFilter* filter) const { Q_ASSERT(filter); filter->setCommentCharacter( ui.cbCommentCharacter->currentText() ); filter->setSeparatingCharacter( ui.cbSeparatingCharacter->currentText() ); filter->setNumberFormat( QLocale::Language(ui.cbNumberFormat->currentIndex()) ); filter->setDateTimeFormat(ui.cbDateTimeFormat->currentText()); filter->setCreateIndexEnabled( ui.chbCreateIndex->isChecked() ); filter->setSimplifyWhitespacesEnabled( ui.chbSimplifyWhitespaces->isChecked() ); filter->setNaNValueToZero( ui.chbConvertNaNToZero->isChecked() ); + filter->setRemoveQuotesEnabled( ui.chbRemoveQuotes->isChecked() ); filter->setSkipEmptyParts( ui.chbSkipEmptyParts->isChecked() ); filter->setVectorNames( ui.kleVectorNames->text() ); filter->setHeaderEnabled( ui.chbHeader->isChecked() ); } void AsciiOptionsWidget::loadSettings() const { KConfigGroup conf(KSharedConfig::openConfig(), "ImportAscii"); //TODO: check if this works (character gets currentItem?) ui.cbCommentCharacter->setCurrentItem(conf.readEntry("CommentCharacter", "#")); ui.cbSeparatingCharacter->setCurrentItem(conf.readEntry("SeparatingCharacter", "auto")); ui.cbNumberFormat->setCurrentIndex(conf.readEntry("NumberFormat", (int)QLocale::AnyLanguage)); ui.cbDateTimeFormat->setCurrentItem(conf.readEntry("DateTimeFormat", "yyyy-MM-dd hh:mm:ss.zzz")); ui.chbCreateIndex->setChecked(conf.readEntry("CreateIndex", false)); ui.chbSimplifyWhitespaces->setChecked(conf.readEntry("SimplifyWhitespaces", true)); ui.chbConvertNaNToZero->setChecked(conf.readEntry("ConvertNaNToZero", false)); + ui.chbRemoveQuotes->setChecked(conf.readEntry("RemoveQuotes", false)); ui.chbSkipEmptyParts->setChecked(conf.readEntry("SkipEmptyParts", false)); ui.chbHeader->setChecked(conf.readEntry("UseFirstRow", true)); ui.kleVectorNames->setText(conf.readEntry("Names", "")); } void AsciiOptionsWidget::saveSettings() { KConfigGroup conf(KSharedConfig::openConfig(), "ImportAscii"); conf.writeEntry("CommentCharacter", ui.cbCommentCharacter->currentText()); conf.writeEntry("SeparatingCharacter", ui.cbSeparatingCharacter->currentText()); conf.writeEntry("NumberFormat", ui.cbNumberFormat->currentIndex()); conf.writeEntry("DateTimeFormat", ui.cbDateTimeFormat->currentText()); conf.writeEntry("CreateIndex", ui.chbCreateIndex->isChecked()); conf.writeEntry("SimplifyWhitespaces", ui.chbSimplifyWhitespaces->isChecked()); conf.writeEntry("ConvertNaNToZero", ui.chbConvertNaNToZero->isChecked()); + conf.writeEntry("RemoveQuotes", ui.chbRemoveQuotes->isChecked()); conf.writeEntry("SkipEmptyParts", ui.chbSkipEmptyParts->isChecked()); conf.writeEntry("UseFirstRow", ui.chbHeader->isChecked()); conf.writeEntry("Names", ui.kleVectorNames->text()); } diff --git a/src/kdefrontend/ui/datasources/asciioptionswidget.ui b/src/kdefrontend/ui/datasources/asciioptionswidget.ui index 5f16fdd4f..dc8eddf7f 100644 --- a/src/kdefrontend/ui/datasources/asciioptionswidget.ui +++ b/src/kdefrontend/ui/datasources/asciioptionswidget.ui @@ -1,226 +1,233 @@ AsciiOptionsWidget 0 0 469 499 0 0 Skip empty parts Use the first row to name the vectors false DateTime format true Vector names: Removes the whitespaces from the start and the end, and replaces each sequence of internal whitespaces with a single space. Simplify whitespaces true true Qt::Vertical QSizePolicy::Fixed 20 18 Number format true Vector names, space separated. E.g. "x y" true Qt::Vertical QSizePolicy::Expanding 20 44 0 0 true Qt::Vertical QSizePolicy::Fixed 20 18 Separating character 0 0 true Comment character Qt::Horizontal 91 20 Convert NaN to 0 false Create index column + + + + Remove Quotes + + + KComboBox QComboBox
kcombobox.h