diff --git a/src/backend/core/column/Column.cpp b/src/backend/core/column/Column.cpp index 1b42da01e..95d3faac5 100644 --- a/src/backend/core/column/Column.cpp +++ b/src/backend/core/column/Column.cpp @@ -1,1182 +1,1182 @@ /*************************************************************************** File : Column.cpp Project : LabPlot Description : Aspect that manages a column -------------------------------------------------------------------- Copyright : (C) 2007-2009 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2013-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/core/column/Column.h" #include "backend/core/column/ColumnPrivate.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/column/columncommands.h" #include "backend/core/Project.h" #include "backend/lib/XmlStreamReader.h" #include "backend/core/datatypes/String2DateTimeFilter.h" #include "backend/core/datatypes/DateTime2StringFilter.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" extern "C" { #include } #include #include #include #include #include #include /** * \class Column * \brief Aspect that manages a column * * This class represents a column, i.e., (mathematically) a 1D vector of * values with a header. It provides a public reading and (undo aware) writing * interface as defined in AbstractColumn. A column * can have one of currently three data types: double, QString, or * QDateTime. The string representation of the values can differ depending * on the mode of the column. * * Column inherits from AbstractAspect and is intended to be a child * of the corresponding Spreadsheet in the aspect hierarchy. Columns don't * have a view as they are intended to be displayed inside a spreadsheet. */ Column::Column(const QString& name, AbstractColumn::ColumnMode mode) : AbstractColumn(name), d(new ColumnPrivate(this, mode)) { init(); } /** * \brief Common part of ctors */ void Column::init() { m_string_io = new ColumnStringIO(this); d->inputFilter()->input(0, m_string_io); d->outputFilter()->input(0, this); d->inputFilter()->setHidden(true); d->outputFilter()->setHidden(true); addChild(d->inputFilter()); addChild(d->outputFilter()); m_suppressDataChangedSignal = false; m_usedInActionGroup = new QActionGroup(this); connect(m_usedInActionGroup, &QActionGroup::triggered, this, &Column::navigateTo); } Column::~Column() { delete m_string_io; delete d; } QMenu* Column::createContextMenu() { QMenu* menu = AbstractAspect::createContextMenu(); QAction* firstAction = menu->actions().at(1); //add actions available in SpreadsheetView emit requestProjectContextMenu(menu); //"Used in" menu containing all curves where the column is used QMenu* usedInMenu = new QMenu(i18n("Used in")); usedInMenu->setIcon(QIcon::fromTheme("go-next-view")); //remove previously added actions for (auto* action: m_usedInActionGroup->actions()) m_usedInActionGroup->removeAction(action); //add curves where the column is currently in use QVector curves = project()->children(AbstractAspect::Recursive); for (const auto* curve: curves) { if (curve->dataSourceType() == XYCurve::DataSourceSpreadsheet && (curve->xColumn() == this || curve->yColumn() == this) ) { QAction* action = new QAction(curve->icon(), curve->name(), m_usedInActionGroup); action->setData(curve->path()); usedInMenu->addAction(action); } } menu->insertSeparator(firstAction); menu->insertMenu(firstAction, usedInMenu); menu->insertSeparator(firstAction); return menu; } void Column::navigateTo(QAction* action) { project()->navigateTo(action->data().toString()); } /*! * */ void Column::setSuppressDataChangedSignal(bool b) { m_suppressDataChangedSignal = b; } /** * \brief Set the column mode * * This sets the column mode and, if * necessary, converts it to another datatype. */ void Column::setColumnMode(AbstractColumn::ColumnMode mode) { if (mode == columnMode()) return; DEBUG("Column::setColumnMode()"); beginMacro(i18n("%1: change column type", name())); auto* old_input_filter = d->inputFilter(); auto* old_output_filter = d->outputFilter(); exec(new ColumnSetModeCmd(d, mode)); if (d->inputFilter() != old_input_filter) { removeChild(old_input_filter); addChild(d->inputFilter()); d->inputFilter()->input(0, m_string_io); } if (d->outputFilter() != old_output_filter) { removeChild(old_output_filter); addChild(d->outputFilter()); d->outputFilter()->input(0, this); } endMacro(); DEBUG("Column::setColumnMode() DONE"); } void Column::setColumnModeFast(AbstractColumn::ColumnMode mode) { if (mode == columnMode()) return; auto* old_input_filter = d->inputFilter(); auto* old_output_filter = d->outputFilter(); exec(new ColumnSetModeCmd(d, mode)); if (d->inputFilter() != old_input_filter) { removeChild(old_input_filter); addChildFast(d->inputFilter()); d->inputFilter()->input(0, m_string_io); } if (d->outputFilter() != old_output_filter) { removeChild(old_output_filter); addChildFast(d->outputFilter()); d->outputFilter()->input(0, this); } } /** * \brief Copy another column of the same type * * This function will return false if the data type * of 'other' is not the same as the type of 'this'. * Use a filter to convert a column to another type. */ bool Column::copy(const AbstractColumn* other) { Q_CHECK_PTR(other); if (other->columnMode() != columnMode()) return false; exec(new ColumnFullCopyCmd(d, other)); return true; } /** * \brief Copies a part of another column of the same type * * This function will return false if the data type * of 'other' is not the same as the type of 'this'. * \param other pointer to the column to copy * \param src_start first row to copy in the column to copy * \param dest_start first row to copy in * \param num_rows the number of rows to copy */ bool Column::copy(const AbstractColumn* source, int source_start, int dest_start, int num_rows) { Q_CHECK_PTR(source); if (source->columnMode() != columnMode()) return false; exec(new ColumnPartialCopyCmd(d, source, source_start, dest_start, num_rows)); return true; } /** * \brief Insert some empty (or initialized with zero) rows */ void Column::handleRowInsertion(int before, int count) { AbstractColumn::handleRowInsertion(before, count); exec(new ColumnInsertRowsCmd(d, before, count)); if (!m_suppressDataChangedSignal) emit dataChanged(this); setStatisticsAvailable(false); } /** * \brief Remove 'count' rows starting from row 'first' */ void Column::handleRowRemoval(int first, int count) { AbstractColumn::handleRowRemoval(first, count); exec(new ColumnRemoveRowsCmd(d, first, count)); if (!m_suppressDataChangedSignal) emit dataChanged(this); setStatisticsAvailable(false); } /** * \brief Set the column plot designation */ void Column::setPlotDesignation(AbstractColumn::PlotDesignation pd) { if (pd != plotDesignation()) exec(new ColumnSetPlotDesignationCmd(d, pd)); } /** * \brief Get width */ int Column::width() const { return d->width(); } /** * \brief Set width */ void Column::setWidth(int value) { d->setWidth(value); } /** * \brief Clear the whole column */ void Column::clear() { exec(new ColumnClearCmd(d)); } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// //! \name Formula related functions //@{ //////////////////////////////////////////////////////////////////////////////// /** * \brief Returns the formula used to generate column values */ QString Column:: formula() const { return d->formula(); } const QStringList& Column::formulaVariableNames() const { return d->formulaVariableNames(); } const QStringList& Column::formulaVariableColumnPathes() const { return d->formulaVariableColumnPathes(); } /** * \brief Sets the formula used to generate column values */ void Column::setFormula(const QString& formula, const QStringList& variableNames, const QStringList& columnPathes) { exec(new ColumnSetGlobalFormulaCmd(d, formula, variableNames, columnPathes)); } /** * \brief Set a formula string for an interval of rows */ void Column::setFormula(Interval i, QString formula) { exec(new ColumnSetFormulaCmd(d, i, formula)); } /** * \brief Overloaded function for convenience */ void Column::setFormula(int row, QString formula) { setFormula(Interval(row, row), formula); } /** * \brief Clear all formulas */ void Column::clearFormulas() { exec(new ColumnClearFormulasCmd(d)); } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// //! \name type specific functions //@{ //////////////////////////////////////////////////////////////////////////////// /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Text */ void Column::setTextAt(int row, const QString& new_value) { DEBUG("Column::setTextAt()"); setStatisticsAvailable(false); exec(new ColumnSetTextCmd(d, row, new_value)); } /** * \brief Replace a range of values * * Use this only when columnMode() is Text */ void Column::replaceTexts(int first, const QVector& new_values) { DEBUG("Column::replaceTexts()"); if (!new_values.isEmpty()) { //TODO: do we really need this check? setStatisticsAvailable(false); exec(new ColumnReplaceTextsCmd(d, first, new_values)); } } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void Column::setDateAt(int row, const QDate& new_value) { setStatisticsAvailable(false); setDateTimeAt(row, QDateTime(new_value, timeAt(row))); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void Column::setTimeAt(int row, const QTime& new_value) { setStatisticsAvailable(false); setDateTimeAt(row, QDateTime(dateAt(row), new_value)); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void Column::setDateTimeAt(int row, const QDateTime& new_value) { setStatisticsAvailable(false); exec(new ColumnSetDateTimeCmd(d, row, new_value)); } /** * \brief Replace a range of values * * Use this only when columnMode() is DateTime, Month or Day */ void Column::replaceDateTimes(int first, const QVector& new_values) { if (!new_values.isEmpty()) { setStatisticsAvailable(false); exec(new ColumnReplaceDateTimesCmd(d, first, new_values)); } } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Numeric */ void Column::setValueAt(int row, double new_value) { // DEBUG("Column::setValueAt()"); setStatisticsAvailable(false); exec(new ColumnSetValueCmd(d, row, new_value)); } /** * \brief Replace a range of values * * Use this only when columnMode() is Numeric */ void Column::replaceValues(int first, const QVector& new_values) { DEBUG("Column::replaceValues()"); if (!new_values.isEmpty()) { setStatisticsAvailable(false); exec(new ColumnReplaceValuesCmd(d, first, new_values)); } } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Integer */ void Column::setIntegerAt(int row, int new_value) { DEBUG("Column::setIntegerAt()"); setStatisticsAvailable(false); exec(new ColumnSetIntegerCmd(d, row, new_value)); } /** * \brief Replace a range of values * * Use this only when columnMode() is Integer */ void Column::replaceInteger(int first, const QVector& new_values) { DEBUG("Column::replaceInteger()"); if (!new_values.isEmpty()) { setStatisticsAvailable(false); exec(new ColumnReplaceIntegersCmd(d, first, new_values)); } } void Column::setStatisticsAvailable(bool available) { d->statisticsAvailable = available; } bool Column::statisticsAvailable() const { return d->statisticsAvailable; } const Column::ColumnStatistics& Column::statistics() { if (!statisticsAvailable()) calculateStatistics(); return d->statistics; } void Column::calculateStatistics() { d->statistics = ColumnStatistics(); ColumnStatistics& statistics = d->statistics; // TODO: support other data types? QVector* rowValues = reinterpret_cast*>(data()); - int notNanCount = 0; + size_t notNanCount = 0; double val; double columnSum = 0.0; double columnProduct = 1.0; double columnSumNeg = 0.0; double columnSumSquare = 0.0; statistics.minimum = INFINITY; statistics.maximum = -INFINITY; QMap frequencyOfValues; QVector rowData; rowData.reserve(rowValues->size()); for (int row = 0; row < rowValues->size(); ++row) { val = rowValues->value(row); if (std::isnan(val) || isMasked(row)) continue; if (val < statistics.minimum) statistics.minimum = val; if (val > statistics.maximum) statistics.maximum = val; columnSum+= val; columnSumNeg += (1.0 / val); columnSumSquare += pow(val, 2.0); columnProduct *= val; if (frequencyOfValues.contains(val)) frequencyOfValues.operator [](val)++; else frequencyOfValues.insert(val, 1); ++notNanCount; rowData.push_back(val); } if (notNanCount == 0) { setStatisticsAvailable(true); return; } if (rowData.size() < rowValues->size()) rowData.squeeze(); statistics.arithmeticMean = columnSum / notNanCount; statistics.geometricMean = pow(columnProduct, 1.0 / notNanCount); statistics.harmonicMean = notNanCount / columnSumNeg; statistics.contraharmonicMean = columnSumSquare / columnSum; double columnSumVariance = 0; double columnSumMeanDeviation = 0.0; double columnSumMedianDeviation = 0.0; double sumForCentralMoment_r3 = 0.0; double sumForCentralMoment_r4 = 0.0; gsl_sort(rowData.data(), 1, notNanCount); statistics.median = (notNanCount%2) ? rowData.at((notNanCount-1)/2) : (rowData.at((notNanCount-1)/2) + rowData.at(notNanCount/2))/2.0; QVector absoluteMedianList; absoluteMedianList.reserve(notNanCount); absoluteMedianList.resize(notNanCount); int idx = 0; for(int row = 0; row < rowValues->size(); ++row) { val = rowValues->value(row); if (std::isnan(val) || isMasked(row) ) continue; columnSumVariance += pow(val - statistics.arithmeticMean, 2.0); sumForCentralMoment_r3 += pow(val - statistics.arithmeticMean, 3.0); sumForCentralMoment_r4 += pow(val - statistics.arithmeticMean, 4.0); columnSumMeanDeviation += fabs( val - statistics.arithmeticMean ); absoluteMedianList[idx] = fabs(val - statistics.median); columnSumMedianDeviation += absoluteMedianList[idx]; idx++; } statistics.meanDeviationAroundMedian = columnSumMedianDeviation / notNanCount; statistics.medianDeviation = (notNanCount%2) ? absoluteMedianList.at((notNanCount-1)/2) : (absoluteMedianList.at((notNanCount-1)/2) + absoluteMedianList.at(notNanCount/2))/2.0; const double centralMoment_r3 = sumForCentralMoment_r3 / notNanCount; const double centralMoment_r4 = sumForCentralMoment_r4 / notNanCount; statistics.variance = columnSumVariance / notNanCount; statistics.standardDeviation = sqrt(statistics.variance); statistics.skewness = centralMoment_r3 / pow(statistics.standardDeviation, 3.0); statistics.kurtosis = (centralMoment_r4 / pow(statistics.standardDeviation, 4.0)) - 3.0; statistics.meanDeviation = columnSumMeanDeviation / notNanCount; double entropy = 0.0; for (const auto& v: frequencyOfValues.values()) { const double frequencyNorm = static_cast(v) / notNanCount; entropy += (frequencyNorm * log2(frequencyNorm)); } statistics.entropy = -entropy; setStatisticsAvailable(true); } ////////////////////////////////////////////////////////////////////////////////////////////// void* Column::data() const { return d->data(); } //TODO: support all data types /** * \brief Return the content of row 'row'. * * Use this only when columnMode() is Text */ QString Column::textAt(int row) const { return d->textAt(row); } /** * \brief Return the date part of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QDate Column::dateAt(int row) const { return d->dateAt(row); } /** * \brief Return the time part of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QTime Column::timeAt(int row) const { return d->timeAt(row); } /** * \brief Return the QDateTime in row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QDateTime Column::dateTimeAt(int row) const { return d->dateTimeAt(row); } /** * \brief Return the double value in row 'row' */ double Column::valueAt(int row) const { return d->valueAt(row); } /** * \brief Return the int value in row 'row' */ int Column::integerAt(int row) const { return d->integerAt(row); } /* * call this function if the data of the column was changed directly via the data()-pointer * and not via the setValueAt() in order to emit the dataChanged-signal. * This is used e.g. in \c XYFitCurvePrivate::recalculate() */ void Column::setChanged() { if (!m_suppressDataChangedSignal) emit dataChanged(this); setStatisticsAvailable(false); } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// /** * \brief Return an icon to be used for decorating the views and spreadsheet column headers */ QIcon Column::icon() const { return iconForMode(columnMode()); } //////////////////////////////////////////////////////////////////////////////////////////////////// //! \name serialize/deserialize //@{ //////////////////////////////////////////////////////////////////////////////////////////////////// /** * \brief Save the column as XML */ void Column::save(QXmlStreamWriter* writer) const { writer->writeStartElement("column"); writeBasicAttributes(writer); writer->writeAttribute("designation", QString::number(plotDesignation())); writer->writeAttribute("mode", QString::number(columnMode())); writer->writeAttribute("width", QString::number(width())); //save the formula used to generate column values, if available if (!formula().isEmpty() ) { writer->writeStartElement("formula"); writer->writeTextElement("text", formula()); writer->writeStartElement("variableNames"); for (auto name: formulaVariableNames()) writer->writeTextElement("name", name); writer->writeEndElement(); writer->writeStartElement("columnPathes"); for (auto path: formulaVariableColumnPathes()) writer->writeTextElement("path", path); writer->writeEndElement(); writer->writeEndElement(); } writeCommentElement(writer); writer->writeStartElement("input_filter"); d->inputFilter()->save(writer); writer->writeEndElement(); writer->writeStartElement("output_filter"); d->outputFilter()->save(writer); writer->writeEndElement(); XmlWriteMask(writer); //TODO: formula in cells is not implemented yet // QList< Interval > formulas = formulaIntervals(); // foreach(const Interval& interval, formulas) { // writer->writeStartElement("formula"); // writer->writeAttribute("start_row", QString::number(interval.start())); // writer->writeAttribute("end_row", QString::number(interval.end())); // writer->writeCharacters(formula(interval.start())); // writer->writeEndElement(); // } int i; switch(columnMode()) { case AbstractColumn::Numeric: { const char* data = reinterpret_cast(static_cast< QVector* >(d->data())->constData()); - int size = d->rowCount() * sizeof(double); - writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); + size_t size = d->rowCount() * sizeof(double); + writer->writeCharacters(QByteArray::fromRawData(data, (int)size).toBase64()); break; } case AbstractColumn::Integer: { const char* data = reinterpret_cast(static_cast< QVector* >(d->data())->constData()); - int size = d->rowCount() * sizeof(int); - writer->writeCharacters(QByteArray::fromRawData(data, size).toBase64()); + size_t size = d->rowCount() * sizeof(int); + writer->writeCharacters(QByteArray::fromRawData(data, (int)size).toBase64()); break; } case AbstractColumn::Text: for (i = 0; i < rowCount(); ++i) { writer->writeStartElement("row"); writer->writeAttribute("index", QString::number(i)); writer->writeCharacters(textAt(i)); writer->writeEndElement(); } break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: for (i = 0; i < rowCount(); ++i) { writer->writeStartElement("row"); writer->writeAttribute("index", QString::number(i)); writer->writeCharacters(dateTimeAt(i).toString("yyyy-dd-MM hh:mm:ss:zzz")); writer->writeEndElement(); } break; } writer->writeEndElement(); // "column" } //TODO: extra header class DecodeColumnTask : public QRunnable { public: DecodeColumnTask(ColumnPrivate* priv, const QString& content) { m_private = priv; m_content = content; }; void run() { QByteArray bytes = QByteArray::fromBase64(m_content.toAscii()); if (m_private->columnMode() == AbstractColumn::Numeric) { - QVector* data = new QVector(bytes.size()/sizeof(double)); + QVector* data = new QVector(bytes.size()/(int)sizeof(double)); memcpy(data->data(), bytes.data(), bytes.size()); m_private->replaceData(data); } else { - QVector* data = new QVector(bytes.size()/sizeof(int)); + QVector* data = new QVector(bytes.size()/(int)sizeof(int)); memcpy(data->data(), bytes.data(), bytes.size()); m_private->replaceData(data); } } private: ColumnPrivate* m_private; QString m_content; }; /** * \brief Load the column from XML */ bool Column::load(XmlStreamReader* reader, bool preview) { if (reader->isStartElement() && reader->name() != "column") { reader->raiseError(i18n("no column element found")); return false; } if (!readBasicAttributes(reader)) return false; QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("designation").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'designation'")); else d->setPlotDesignation( AbstractColumn::PlotDesignation(str.toInt()) ); str = attribs.value("mode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'mode'")); else setColumnModeFast( AbstractColumn::ColumnMode(str.toInt()) ); str = attribs.value("width").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'width'")); else d->setWidth(str.toInt()); // read child elements while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement()) break; if (reader->isStartElement()) { bool ret_val = true; if (reader->name() == "comment") ret_val = readCommentElement(reader); else if (reader->name() == "input_filter") ret_val = XmlReadInputFilter(reader); else if (reader->name() == "output_filter") ret_val = XmlReadOutputFilter(reader); else if (reader->name() == "mask") ret_val = XmlReadMask(reader); else if (reader->name() == "formula") ret_val = XmlReadFormula(reader); else if (reader->name() == "row") ret_val = XmlReadRow(reader); else { // unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } if (!ret_val) return false; } if (!preview) { QString content = reader->text().toString().trimmed(); if (!content.isEmpty() && ( columnMode() == AbstractColumn::Numeric || columnMode() == AbstractColumn::Integer)) { DecodeColumnTask* task = new DecodeColumnTask(d, content); QThreadPool::globalInstance()->start(task); } } } return !reader->error(); } /** * \brief Read XML input filter element */ bool Column::XmlReadInputFilter(XmlStreamReader* reader) { Q_ASSERT(reader->isStartElement() && reader->name() == "input_filter"); if (!reader->skipToNextTag()) return false; if (!d->inputFilter()->load(reader, false)) return false; if (!reader->skipToNextTag()) return false; Q_ASSERT(reader->isEndElement() && reader->name() == "input_filter"); return true; } /** * \brief Read XML output filter element */ bool Column::XmlReadOutputFilter(XmlStreamReader* reader) { Q_ASSERT(reader->isStartElement() && reader->name() == "output_filter"); if (!reader->skipToNextTag()) return false; if (!d->outputFilter()->load(reader, false)) return false; if (!reader->skipToNextTag()) return false; Q_ASSERT(reader->isEndElement() && reader->name() == "output_filter"); return true; } /** * \brief Read XML formula element */ bool Column::XmlReadFormula(XmlStreamReader* reader) { QString formula; QStringList variableNames; QStringList columnPathes; while (reader->readNext()) { if (reader->isEndElement()) break; if (reader->name() == "text") formula = reader->readElementText(); else if (reader->name() == "variableNames") { while (reader->readNext()) { if (reader->name() == "variableNames" && reader->isEndElement()) break; if (reader->isStartElement()) variableNames << reader->readElementText(); } } else if (reader->name() == "columnPathes") { while (reader->readNext()) { if (reader->name() == "columnPathes" && reader->isEndElement()) break; if (reader->isStartElement()) columnPathes << reader->readElementText(); } } } setFormula(formula, variableNames, columnPathes); return true; } //TODO: read cell formula, not implemented yet // bool Column::XmlReadFormula(XmlStreamReader* reader) // { // Q_ASSERT(reader->isStartElement() && reader->name() == "formula"); // // bool ok1, ok2; // int start, end; // start = reader->readAttributeInt("start_row", &ok1); // end = reader->readAttributeInt("end_row", &ok2); // if(!ok1 || !ok2) // { // reader->raiseError(i18n("invalid or missing start or end row")); // return false; // } // setFormula(Interval(start,end), reader->readElementText()); // // return true; // } /** * \brief Read XML row element */ bool Column::XmlReadRow(XmlStreamReader* reader) { Q_ASSERT(reader->isStartElement() && reader->name() == "row"); // QXmlStreamAttributes attribs = reader->attributes(); bool ok; int index = reader->readAttributeInt("index", &ok); if (!ok) { reader->raiseError(i18n("invalid or missing row index")); return false; } QString str = reader->readElementText(); switch (columnMode()) { case AbstractColumn::Numeric: { double value = str.toDouble(&ok); if(!ok) { reader->raiseError(i18n("invalid row value")); return false; } setValueAt(index, value); break; } case AbstractColumn::Integer: { int value = str.toInt(&ok); if(!ok) { reader->raiseError(i18n("invalid row value")); return false; } setIntegerAt(index, value); break; } case AbstractColumn::Text: setTextAt(index, str); break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: QDateTime date_time = QDateTime::fromString(str,"yyyy-dd-MM hh:mm:ss:zzz"); setDateTimeAt(index, date_time); break; } return true; } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// /** * \brief Return whether the object is read-only */ bool Column::isReadOnly() const { return false; } /** * \brief Return the column mode * * This function is mostly used by spreadsheets but can also be used * by plots. The column mode specifies how to interpret * the values in the column additional to the data type. */ AbstractColumn::ColumnMode Column::columnMode() const { return d->columnMode(); } /** * \brief Return the data vector size * * This returns the number of rows that actually contain data. * Rows beyond this can be masked etc. but should be ignored by filters, * plots etc. */ int Column::rowCount() const { return d->rowCount(); } /** * \brief Return the column plot designation */ AbstractColumn::PlotDesignation Column::plotDesignation() const { return d->plotDesignation(); } AbstractSimpleFilter* Column::outputFilter() const { return d->outputFilter(); } /** * \brief Return a wrapper column object used for String I/O. */ ColumnStringIO* Column::asStringColumn() const { return m_string_io; } //////////////////////////////////////////////////////////////////////////////// //! \name IntervalAttribute related functions //@{ //////////////////////////////////////////////////////////////////////////////// /** * \brief Return the formula associated with row 'row' */ QString Column::formula(int row) const { return d->formula(row); } /** * \brief Return the intervals that have associated formulas * * This can be used to make a list of formulas with their intervals. * Here is some example code: * * \code * QStringList list; * QList< Interval > intervals = my_column.formulaIntervals(); * foreach(Interval interval, intervals) * list << QString(interval.toString() + ": " + my_column.formula(interval.start())); * \endcode */ QList< Interval > Column::formulaIntervals() const { return d->formulaIntervals(); } void Column::handleFormatChange() { DEBUG("Column::handleFormatChange() mode = " << ENUM_TO_STRING(AbstractColumn, ColumnMode, columnMode())); if (columnMode() == AbstractColumn::DateTime) { auto* input_filter = static_cast(d->inputFilter()); auto* output_filter = static_cast(d->outputFilter()); DEBUG("change format " << input_filter->format().toStdString() << " to " << output_filter->format().toStdString()); input_filter->setFormat(output_filter->format()); } emit aspectDescriptionChanged(this); // the icon for the type changed if (!m_suppressDataChangedSignal) emit dataChanged(this); // all cells must be repainted setStatisticsAvailable(false); DEBUG("Column::handleFormatChange() DONE"); } /*! * calculates the minimal value in the column. * for \c count = 0, the minimum of all elements is returned. * for \c count > 0, the minimum of the first \count elements is returned. * for \c count = 0, the minimum of the last \count elements is returned. */ double Column::minimum(int count) const { double min = INFINITY; if (count == 0 && statisticsAvailable()) min = const_cast(this)->statistics().minimum; else { ColumnMode mode = columnMode(); int start, end; if (count == 0) { start = 0; end = rowCount(); } else if (count > 0) { start = 0; end = qMin(rowCount(), count); } else { start = qMax(rowCount() + count, 0); end = rowCount(); } switch (mode) { case Numeric: { QVector* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const double val = vec->at(row); if (std::isnan(val)) continue; if (val < min) min = val; } break; } case Integer: { QVector* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const int val = vec->at(row); if (val < min) min = val; } break; } case Text: case DateTime: case Day: case Month: default: break; } } return min; } /*! * calculates the maximal value in the column. * for \c count = 0, the maximum of all elements is returned. * for \c count > 0, the maximum of the first \count elements is returned. * for \c count = 0, the maximum of the last \count elements is returned. */ double Column::maximum(int count) const { double max = -INFINITY; if (count == 0 && statisticsAvailable()) max = const_cast(this)->statistics().maximum; else { ColumnMode mode = columnMode(); int start, end; if (count == 0) { start = 0; end = rowCount(); } else if (count > 0) { start = 0; end = qMin(rowCount(), count); } else { start = qMax(rowCount() + count, 0); end = rowCount(); } switch (mode) { case Numeric: { QVector* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const double val = vec->at(row); if (std::isnan(val)) continue; if (val > max) max = val; } break; } case Integer: { QVector* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const int val = vec->at(row); if (val > max) max = val; } break; } case Text: case DateTime: case Day: case Month: default: break; } } return max; } diff --git a/src/backend/datapicker/Segments.cpp b/src/backend/datapicker/Segments.cpp index 68cb0c514..7407bfb76 100644 --- a/src/backend/datapicker/Segments.cpp +++ b/src/backend/datapicker/Segments.cpp @@ -1,293 +1,293 @@ /*************************************************************************** File : Segments.cpp Project : LabPlot Description : Contains methods to trace curve of image/plot -------------------------------------------------------------------- Copyright : (C) 2015 by Ankit Wagadre (wagadre.ankit@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "Segments.h" #include "backend/datapicker/Segment.h" #include "backend/datapicker/ImageEditor.h" #include #include #include /** * \class Segments * \brief container to open image/plot. * * this class is a container for all of the segment. a segment is a set * of linked lines that run along a curve in the original image. the * main complication is that curves in the original image cross each other * and other things like grid lines. we rely on the user to link * multiple segments together to get points along the entire curve length * * \ingroup datapicker */ Segments::Segments(DatapickerImage* image): m_image(image) { } /*! segments are built when the original image is loaded. they start out hidden and remain so until showSegments is called */ void Segments::makeSegments(QImage &imageProcessed) { // QElapsedTimer timer; // timer.start(); clearSegments(); const int width = imageProcessed.width(); const int height = imageProcessed.height(); // for each new column of pixels, loop through the runs. a run is defined as // one or more colored pixels that are all touching, with one uncolored pixel or the // image boundary at each end of the set. for each set in the current column, count // the number of runs it touches in the adjacent (left and right) columns. here is // the pseudocode: // if ((L > 1) || (R > 1)) // "this run is at a branch point so ignore the set" // else // if (L == 0) // "this run is the start of a new segment" // else // "this run is appended to the segment on the left bool* lastBool = new bool [height]; bool* currBool = new bool [height]; bool* nextBool = new bool [height]; - Segment** lastSegment = new Segment* [height]; - Segment** currSegment = new Segment* [height]; + Segment** lastSegment = new Segment* [(size_t)height]; + Segment** currSegment = new Segment* [(size_t)height]; loadSegment(lastSegment, height); //initialize one column of boolean flags using the pixels of the specified column for (int y = 0; y < height; ++y) { lastBool[y] = false; currBool[y] = ImageEditor::processedPixelIsOn(imageProcessed, 0, y); nextBool[y] = ImageEditor::processedPixelIsOn(imageProcessed, 1, y); } for (int x = 0; x < width; ++x) { matchRunsToSegments(x, height, lastBool, lastSegment, currBool, currSegment, nextBool); // get ready for next column for (int y = 0; y < height; ++y) { lastBool[y] = currBool[y]; currBool[y] = nextBool[y]; } if (x + 1 < width) { for (int y = 0; y < height; ++y) nextBool[y] = ImageEditor::processedPixelIsOn(imageProcessed, x+1, y); } scrollSegment(lastSegment, currSegment, height); } delete[] lastBool; delete[] currBool; delete[] nextBool; delete[] lastSegment; delete[] currSegment; // qDebug() << "Made segments in " << timer.elapsed() << "ms"; } /*! scroll the segment pointers of the right column into the left column */ void Segments::scrollSegment(Segment** left, Segment** right, int height) { for (int y = 0; y < height; ++y) left [y] = right [y]; } /*! identify the runs in a column, and connect them to segments */ void Segments::matchRunsToSegments(int x, int height, bool* lastBool, Segment** lastSegment, bool* currBool, Segment** currSegment, bool* nextBool) { loadSegment(currSegment, height); int yStart = 0; bool inRun = false; for (int y = 0; y < height; ++y) { if (!inRun && currBool [y]) { inRun = true; yStart = y; } if ((y + 1 >= height) || !currBool [y + 1]) { if (inRun) finishRun(lastBool, nextBool, lastSegment, currSegment, x, yStart, y, height); inRun = false; } } removeUnneededLines(lastSegment, currSegment, height); } /*! remove unneeded lines belonging to segments that just finished in the previous column. */ void Segments::removeUnneededLines(Segment** lastSegment, Segment** currSegment, int height) { Segment* segLast = 0; for (int yLast = 0; yLast < height; ++yLast) { if (lastSegment [yLast] && (lastSegment [yLast] != segLast)) { segLast = lastSegment [yLast]; bool found = false; for (int yCur = 0; yCur < height; ++yCur) if (segLast == currSegment [yCur]) { found = true; break; } if (!found) { if (segLast->length < m_image->minSegmentLength()) { // remove whole segment since it is too short m_image->scene()->removeItem(segLast->graphicsItem()); segments.removeOne(segLast); } } } } } /*! initialize one column of segment pointers */ void Segments::loadSegment(Segment** columnSegment, int height) { for (int y = 0; y < height; ++y) columnSegment [y] = 0; } void Segments::clearSegments() { for(auto* seg : segments) m_image->scene()->removeItem(seg->graphicsItem()); segments.clear(); } /*! set segments visible */ void Segments::setSegmentsVisible(bool on) { for(auto* seg : segments) seg->setVisible(on); } void Segments::setAcceptHoverEvents(bool on) { for (auto* seg : segments) { QGraphicsItem *item = seg->graphicsItem(); item->setAcceptHoverEvents(on); item->setFlag(QGraphicsItem::ItemIsSelectable, on); } } /*! process a run of pixels. if there are fewer than two adjacent pixel runs on either side, this run will be added to an existing segment, or the start of a new segment */ void Segments::finishRun(bool* lastBool, bool* nextBool, Segment** lastSegment, Segment** currSegment, int x, int yStart, int yStop, int height) { // count runs that touch on the left if (adjacentRuns(lastBool, yStart, yStop, height) > 1) return; // count runs that touch on the right if (adjacentRuns(nextBool, yStart, yStop, height) > 1) return; Segment* seg; int y = (int) ((yStart + yStop) / 2); if (adjacentSegments(lastSegment, yStart, yStop, height) == 0) { seg = new Segment(m_image); QLine* line = new QLine(QPoint(x, y),QPoint( x, y)); seg->path.append(line); seg->yLast = y; segments.append(seg); } else { // this is the continuation of an existing segment seg = adjacentSegment(lastSegment, yStart, yStop, height); QLine* line = new QLine(QPoint(x - 1, seg->yLast),QPoint( x, y)); seg->length += abs(1 + (seg->yLast - y)*(seg->yLast - y)); seg->path.append(line); seg->yLast = y; } seg->retransform(); for (int y = yStart; y <= yStop; ++y) currSegment [y] = seg; } /*! return the number of runs adjacent to the pixels from yStart to yStop (inclusive) */ int Segments::adjacentRuns(bool* columnBool, int yStart, int yStop, int height) { int runs = 0; bool inRun = false; for (int y = yStart - 1; y <= yStop + 1; ++y) { if ((0 <= y) && (y < height)) { if (!inRun && columnBool [y]) { inRun = true; ++runs; } else if (inRun && !columnBool [y]) inRun = false; } } return runs; } /*! find the single segment pointer among the adjacent pixels from yStart-1 to yStop+1 */ Segment* Segments::adjacentSegment(Segment** lastSegment, int yStart, int yStop, int height) { for (int y = yStart - 1; y <= yStop + 1; ++y) { if ((0 <= y) && (y < height)) if (lastSegment [y]) return lastSegment [y]; } return 0; } /*! return the number of segments adjacent to the pixels from yStart to yStop (inclusive) */ int Segments::adjacentSegments(Segment** lastSegment, int yStart, int yStop, int height) { int count = 0; bool inSegment = false; for (int y = yStart - 1; y <= yStop + 1; ++y) { if ((0 <= y) && (y < height)) { if (!inSegment && lastSegment [y]) { inSegment = true; ++count; } else if (inSegment && !lastSegment [y]) inSegment = false; } } return count; } diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index ab947473f..5ba3808ef 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,1481 +1,1481 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2017 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include "backend/core/Project.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" #include "backend/lib/macros.h" #include "backend/lib/trace.h" #include #include #include #include #include /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() {} /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource * dataSource) { d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { return d->readFromLiveDevice(device, dataSource, from); } /*! reads the content of the file \c fileName. */ QVector AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromFile(fileName, dataSource, importMode, lines); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } QVector AsciiFilter::preview(QIODevice &device) { return d->preview(device); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of lines"); return 0; } size_t lineCount = 0; while (!device.atEnd()) { device.readLine(); lineCount++; } //TODO: wc is much faster but not portable /* QElapsedTimer myTimer; myTimer.start(); QProcess wc; wc.start(QString("wc"), QStringList() << "-l" << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) lineCount = wc.readLine().split(' ')[0].toInt(); lineCount++; // last line not counted DEBUG(" Elapsed time counting lines : " << myTimer.elapsed() << " ms"); */ return lineCount; } /*! returns the number of lines in the device \c device or 0 if not available. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice &device) { // device.hasReadLine() always returns 0 for KFilterDev! if (device.isSequential()) return 0; size_t lineCount = 0; device.seek(0); while (!device.atEnd()) { device.readLine(); lineCount++; } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool AsciiFilter::NaNValueToZeroEnabled() const { if (d->nanValue == 0) return true; return false; } void AsciiFilter::setRemoveQuotesEnabled(bool b) { d->removeQuotesEnabled = b; } bool AsciiFilter::removeQuotesEnabled() const { return d->removeQuotesEnabled; } void AsciiFilter::setVectorNames(const QString s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner), commentCharacter("#"), separatingCharacter("auto"), numberFormat(QLocale::C), autoModeEnabled(true), headerEnabled(true), skipEmptyParts(false), simplifyWhitespacesEnabled(true), nanValue(NAN), removeQuotesEnabled(false), createIndexEnabled(false), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_actualRows(0), m_actualCols(0), m_prepared(false), m_columnOffset(0) { } QStringList AsciiFilterPrivate::getLineString(QIODevice& device) { QString line; do { // skip comment lines in data lines line = device.readLine(); } while (line.startsWith(commentCharacter)); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); DEBUG("data line : \'" << line.toStdString() << '\''); QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //TODO: remove quotes here? QDEBUG("data line, parsed: " << lineStringList); return lineStringList; } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("device is sequential = " << device.isSequential()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; ///////////////////////////////////////////////////////////////// // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines firstLine = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, QString::SkipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, QString::SkipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); DEBUG("headerEnabled = " << headerEnabled); //optionally, remove potential spaces in the first line if (simplifyWhitespacesEnabled) { for (int i = 0; i < firstLineStringList.size(); ++i) firstLineStringList[i] = firstLineStringList[i].simplified(); } if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); startRow++; } // set range to read if (endColumn == -1) { if (headerEnabled || vectorNames.size() == 0) endColumn = firstLineStringList.size(); // last column else //number of vector names provided in the import dialog (not more than the maximal number of columns in the file) endColumn = qMin(vectorNames.size(), firstLineStringList.size()); } if (createIndexEnabled) { vectorNames.prepend("index"); endColumn++; } m_actualCols = endColumn - startColumn + 1; //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ // ATTENTION: This resets the position in the device to 0 - m_actualRows = AsciiFilter::lineNumber(device); + m_actualRows = (int)AsciiFilter::lineNumber(device); ///////////////////////////////////////////////////////////////// // Find first data line (ignoring comment lines) DEBUG("Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { QString line = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } if (line.startsWith(commentCharacter)) // ignore commented lines before startRow i--; } // parse first data line to determine data type for each column if (!device.isSequential()) firstLineStringList = getLineString(device); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Integer; col = 1; } for (const auto& valueString: firstLineStringList) { // parse columns available in first data line if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } // parsing more lines to better determine data types for (unsigned int i = 0; i < m_dataTypeLines; ++i) { firstLineStringList = getLineString(device); if (createIndexEnabled) col = 1; else col = 0; for (const auto& valueString: firstLineStringList) { if (col == m_actualCols) break; AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); // numeric: integer -> numeric if (mode == AbstractColumn::Numeric && columnModes[col] == AbstractColumn::Integer) columnModes[col] = mode; // text: non text -> text if (mode == AbstractColumn::Text && columnModes[col] != AbstractColumn::Text) columnModes[col] = mode; col++; } } QDEBUG("column modes = " << columnModes); // reset to start of file if (!device.isSequential()) device.seek(0); ///////////////////////////////////////////////////////////////// int actualEndRow = endRow; DEBUG("endRow = " << endRow); if (endRow == -1 || endRow > m_actualRows) actualEndRow = m_actualRows; if (m_actualRows > actualEndRow) m_actualRows = actualEndRow; DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << startRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header incl. start rows): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0 && !device.isSequential()) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode, lines); } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return 0; } Q_ASSERT(dataSource != nullptr); LiveDataSource* spreadsheet = dynamic_cast(dataSource); if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe) if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return 0; if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return 0; } // prepare import for spreadsheet spreadsheet->setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); qDebug() << "fds resized to col: " << m_actualCols; qDebug() << "fds rowCount: " << spreadsheet->rowCount(); //columns in a file data source don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } //also here we need a cheaper version of this if (!spreadsheet->keepLastValues()) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(spreadsheet->keepNvalues()); m_actualRows = spreadsheet->keepNvalues(); } m_dataContainer.resize(m_actualCols); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } qDebug() << "prepared!"; } qint64 bytesread = 0; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif LiveDataSource::ReadingType readingType; if (!m_prepared) readingType = LiveDataSource::ReadingType::TillEnd; else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = LiveDataSource::ReadingType::TillEnd; else readingType = spreadsheet->readingType(); } //move to the last read position, from == total bytes read //since the other source types are sequencial we cannot seek on them if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); qDebug() <<"available bytes: " << device.bytesAvailable(); //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleRateNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != LiveDataSource::ReadingType::TillEnd) { newData.reserve(spreadsheet->sampleRate()); newData.resize(spreadsheet->sampleRate()); } int newDataIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif while (!device.atEnd()) { if (readingType != LiveDataSource::ReadingType::TillEnd) newData[newDataIdx++] = device.readLine(); else newData.push_back(device.readLine()); newLinesTillEnd++; if (readingType != LiveDataSource::ReadingType::TillEnd) { newLinesForSampleRateNotTillEnd++; //for Continous reading and FromEnd we read sample rate number of lines if possible if (newLinesForSampleRateNotTillEnd == spreadsheet->sampleRate()) break; } } } //now we reset the readingType if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = spreadsheet->readingType(); //we had less new lines than the sample rate specified if (readingType != LiveDataSource::ReadingType::TillEnd) qDebug() << "Removed empty lines: " << newData.removeAll(""); //back to the last read position before counting when reading from files if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); int currentRow; // indexes the position in the vector(column) int linesToRead; if (!m_prepared) linesToRead = newLinesTillEnd; if (m_prepared) { //increase row count if we don't have a fixed size //but only after the preparation step if (!spreadsheet->keepLastValues()) { if (readingType != LiveDataSource::ReadingType::TillEnd) m_actualRows += qMin(newData.size(), spreadsheet->sampleRate()); else m_actualRows += newData.size(); } //fixed size if (spreadsheet->keepLastValues()) { if (readingType == LiveDataSource::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; //TODO after reading we should skip the next data lines //because it's TillEnd actually } else linesToRead = newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinouslyFixed or FromEnd linesToRead = qMin(spreadsheet->sampleRate(), newLinesTillEnd); } } else { //appending linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; } } if (m_prepared && (linesToRead == 0)) return 0; //new rows/resize columns if we don't have a fixed size //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize if (!spreadsheet->keepLastValues()) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { //when we have a fixed size we have to pop sampleRate number of lines if specified //here popping, setting currentRow if (!m_prepared) currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); else { if (readingType == LiveDataSource::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinouslyFixed or FromEnd currentRow = m_actualRows - qMin(spreadsheet->sampleRate(), newLinesTillEnd); } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet qDebug() << "reading from line: " << currentRow << " lines till end: " << newLinesTillEnd; qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows; newDataIdx = 0; if (readingType == LiveDataSource::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->sampleRate()) newDataIdx = newData.size() - spreadsheet->sampleRate(); //since we skip a couple of lines, we need to count those bytes too for (int i = 0; i < newDataIdx; ++i) bytesread += newData.at(i).size(); } } qDebug() << "newDataIdx: " << newDataIdx; //TODO static int indexColumnIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif for (int i = 0; i < linesToRead; ++i) { QString line; if (readingType == LiveDataSource::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(i); if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) bytesread += line.size(); //qDebug() << "line bytes: " << line.size() << " line: " << line; //qDebug() << "reading in row: " << currentRow; if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); if (createIndexEnabled) { if (spreadsheet->keepLastValues()) lineStringList.prepend(QString::number(indexColumnIdx++)); else lineStringList.prepend(QString::number(currentRow)); } for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); //qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); //qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); const Project* project = spreadsheet->project(); QVector curves = project->children(AbstractAspect::Recursive); QVector plots; for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); //determine the plots where the column is consumed for (const auto* curve: curves) { if (curve->xColumn() == column || curve->yColumn() == column) { CartesianPlot* plot = dynamic_cast(curve->parentAspect()); if (plots.indexOf(plot) == -1) { plots << plot; plot->setSuppressDataChangedSignal(true); } } } column->setChanged(); } //loop over all affected plots and retransform them for (auto* plot: plots) { //TODO setting this back to true triggers again a lot of retransforms in the plot (one for each curve). // plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } m_prepared = true; return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); Q_ASSERT(dataSource != nullptr); if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return; } // matrix data has only one column mode (which is not text) if (dynamic_cast(dataSource)) { auto mode = columnModes[0]; (mode == AbstractColumn::Text) ? mode = AbstractColumn::Numeric : 0; for (auto& c: columnModes) (c != mode) ? c = mode : 0; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows - startRow + 1, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); ++i) { QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend the index if required //TODO: come up maybe with a solution with adding the index inside of the loop below, //without conversion to string, prepending to the list and then conversion back to integer. if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = ""; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, dateTimeFormat, importMode); } QVector AsciiFilterPrivate::preview(QIODevice &device) { QVector dataStrings; if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return dataStrings; } if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return dataStrings; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif int linesToRead = 0; QVector newData; while (!device.atEnd()) { newData.push_back(device.readLine()); linesToRead++; } if (linesToRead == 0) return dataStrings; int col = 0; int colMax = newData.at(0).size(); if (createIndexEnabled) colMax++; columnModes.resize(colMax); if (createIndexEnabled) { columnModes[0] = AbstractColumn::ColumnMode::Integer; col = 1; vectorNames.prepend("index"); } if (headerEnabled) { int i = 0; if (createIndexEnabled) i = 1; for (; i < vectorNames.size(); ++i) vectorNames[i] = "Column " + QString::number(i); } for (const auto& valueString: newData.at(0).split(' ', QString::SkipEmptyParts)) { if (col == colMax) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } for (int i = 0; i < linesToRead; ++i) { QString line = newData.at(i); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(' ', QString::SkipEmptyParts); if (createIndexEnabled) lineStringList.prepend(QString::number(i)); QStringList lineString; for (int n = 0; n < lineStringList.size(); ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); ++i) { QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend index if required if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); QStringList lineString; for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QRegExp("[\"\']")); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String(""); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to ascii file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement( "asciiFilter"); writer->writeAttribute( "commentCharacter", d->commentCharacter); writer->writeAttribute( "separatingCharacter", d->separatingCharacter); writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); writer->writeAttribute( "nanValue", QString::number(d->nanValue)); writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled)); writer->writeAttribute( "startRow", QString::number(d->startRow)); writer->writeAttribute( "endRow", QString::number(d->endRow)); writer->writeAttribute( "startColumn", QString::number(d->startColumn)); writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "asciiFilter") { reader->raiseError(i18n("no ascii filter element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("commentCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'commentCharacter'")); else d->commentCharacter = str; str = attribs.value("separatingCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'separatingCharacter'")); else d->separatingCharacter = str; str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'autoMode'")); else d->autoModeEnabled = str.toInt(); str = attribs.value("header").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'header'")); else d->headerEnabled = str.toInt(); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty str = attribs.value("simplifyWhitespaces").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'simplifyWhitespaces'")); else d->simplifyWhitespacesEnabled = str.toInt(); str = attribs.value("nanValue").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'nanValue'")); else d->nanValue = str.toDouble(); str = attribs.value("removeQuotes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'removeQuotes'")); else d->removeQuotesEnabled = str.toInt(); str = attribs.value("skipEmptyParts").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipEmptyParts'")); else d->skipEmptyParts = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startColumn'")); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endColumn'")); else d->endColumn = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/BinaryFilter.cpp b/src/backend/datasources/filters/BinaryFilter.cpp index fbba520f0..df026a3e0 100644 --- a/src/backend/datasources/filters/BinaryFilter.cpp +++ b/src/backend/datasources/filters/BinaryFilter.cpp @@ -1,619 +1,619 @@ /*************************************************************************** File : BinaryFilter.cpp Project : LabPlot Description : Binary I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/BinaryFilter.h" #include "backend/datasources/filters/BinaryFilterPrivate.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include /*! \class BinaryFilter \brief Manages the import/export of data organized as columns (vectors) from/to a binary file. \ingroup datasources */ BinaryFilter::BinaryFilter():AbstractFileFilter(), d(new BinaryFilterPrivate(this)) {} BinaryFilter::~BinaryFilter() {} /*! reads the content of the file \c fileName. */ QVector BinaryFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromFile(fileName, dataSource, importMode, lines); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } /*! reads the content of the device \c device. */ void BinaryFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } QVector BinaryFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void BinaryFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! returns the list of all predefined data formats. */ QStringList BinaryFilter::dataTypes() { return (QStringList()<<"int8 (8 bit signed integer)"<<"int16 (16 bit signed integer)"<<"int32 (32 bit signed integer)"<<"int64 (64 bit signed integer)" <<"uint8 (8 bit unsigned integer)"<<"uint16 (16 bit unsigned integer)"<<"uint32 (32 bit unsigned integer)"<<"uint64 (64 bit unsigned integer)" <<"real32 (single precision floats)"<<"real64 (double precision floats)"); } /*! returns the list of all predefined byte order. */ QStringList BinaryFilter::byteOrders() { return (QStringList() << "Little endian" << "Big endian"); } /*! returns the size of the predefined data types */ int BinaryFilter::dataSize(BinaryFilter::DataType type) { int sizes[] = {1,2,4,8,1,2,4,8,4,8}; return sizes[(int)type]; } /*! returns the number of rows (length of vectors) in the file \c fileName. */ size_t BinaryFilter::rowNumber(const QString& fileName, const size_t vectors, const BinaryFilter::DataType type) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) return 0; size_t rows = 0; while (!device.atEnd()) { // one row for (size_t i = 0; i < vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(type); ++j) device.read(1); } rows++; } return rows; } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void BinaryFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void BinaryFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void BinaryFilter::setVectors(const size_t v) { d->vectors = v; } size_t BinaryFilter::vectors() const { return d->vectors; } void BinaryFilter::setDataType(const BinaryFilter::DataType t) { d->dataType = t; } BinaryFilter::DataType BinaryFilter::dataType() const { return d->dataType; } void BinaryFilter::setByteOrder(const BinaryFilter::ByteOrder b) { d->byteOrder = b; } BinaryFilter::ByteOrder BinaryFilter::byteOrder() const { return d->byteOrder; } void BinaryFilter::setSkipStartBytes(const size_t s) { d->skipStartBytes = s; } size_t BinaryFilter::skipStartBytes() const { return d->skipStartBytes; } void BinaryFilter::setStartRow(const int s) { d->startRow = s; } int BinaryFilter::startRow() const { return d->startRow; } void BinaryFilter::setEndRow(const int e) { d->endRow = e; } int BinaryFilter::endRow() const { return d->endRow; } void BinaryFilter::setSkipBytes(const size_t s) { d->skipBytes = s; } size_t BinaryFilter::skipBytes() const { return d->skipBytes; } void BinaryFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void BinaryFilter::setAutoModeEnabled(bool b) { d->autoModeEnabled = b; } bool BinaryFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } //##################################################################### //################### Private implementation ########################## //##################################################################### BinaryFilterPrivate::BinaryFilterPrivate(BinaryFilter* owner) : q(owner), vectors(2), dataType(BinaryFilter::INT8), byteOrder(BinaryFilter::LittleEndian), startRow(1), endRow(-1), skipStartBytes(0), skipBytes(0), createIndexEnabled(false), autoModeEnabled(true) { } /*! reads the content of the device \c device to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ void BinaryFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("readDataFromFile()"); KFilterDev device(fileName); numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); if (! device.open(QIODevice::ReadOnly)) { DEBUG(" could not open file " << fileName.toStdString()); return; } readDataFromDevice(device, dataSource, importMode, lines); } /*! * returns 1 if the current read position in the device is at the end and 0 otherwise. */ int BinaryFilterPrivate::prepareStreamToRead(QDataStream& in) { DEBUG("prepareStreamToRead()"); if (byteOrder == BinaryFilter::BigEndian) in.setByteOrder(QDataStream::BigEndian); else if (byteOrder == BinaryFilter::LittleEndian) in.setByteOrder(QDataStream::LittleEndian); // catch case that skipStartBytes or startRow is bigger than file if (skipStartBytes >= BinaryFilter::dataSize(dataType) * vectors * numRows || startRow > (int)numRows) return 1; // skip bytes at start for (size_t i = 0; i < skipStartBytes; ++i) { qint8 tmp; in >> tmp; } // skip until start row - for (int i = 0; i < (startRow-1) * vectors; ++i) { + for (size_t i = 0; i < (startRow-1) * vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(dataType); ++j) { qint8 tmp; in >> tmp; } } // set range of rows if (endRow == -1) m_actualRows = numRows - startRow + 1; else if (endRow > (int)numRows - startRow + 1) m_actualRows = numRows; else m_actualRows = endRow - startRow + 1; m_actualCols = vectors; DEBUG("numRows = " << numRows); DEBUG("endRow = " << endRow); DEBUG("actual rows = " << m_actualRows); DEBUG("actual cols = " << m_actualCols); return 0; } /*! reads \c lines lines of the device \c device and return as string for preview. */ QVector BinaryFilterPrivate::preview(const QString& fileName, int lines) { DEBUG("BinaryFilterPrivate::preview( " << fileName.toStdString() << ", " << lines << ")"); QVector dataStrings; KFilterDev device(fileName); if (! device.open(QIODevice::ReadOnly)) return dataStrings << (QStringList() << i18n("could not open device")); numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); QDataStream in(&device); const int deviceError = prepareStreamToRead(in); if(deviceError) return dataStrings << (QStringList() << i18n("data selection empty")); //TODO: support other modes columnModes.resize(m_actualCols); //TODO: use given names QStringList vectorNames; if (createIndexEnabled) vectorNames.prepend("index"); if (lines == -1) lines = m_actualRows; // read data //TODO: use ColumnMode ? DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(m_actualRows, lines); ++i) { QStringList lineString; //prepend the index if required if (createIndexEnabled) lineString << QString::number(i+1); for (int n = 0; n < m_actualCols; ++n) { switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT16: { qint16 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT32: { qint32 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::INT64: { qint64 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT8: { quint8 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT16: { quint16 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT32: { quint32 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::UINT64: { quint64 value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::REAL32: { float value; in >> value; lineString << QString::number(value); break; } case BinaryFilter::REAL64: { double value; in >> value; lineString << QString::number(value); break; } } } dataStrings << lineString; emit q->completed(100*i/m_actualRows); } return dataStrings; } /*! reads the content of the file \c fileName to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ void BinaryFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("BinaryFilterPrivate::readDataFromDevice()"); QDataStream in(&device); const int deviceError = prepareStreamToRead(in); if (deviceError) { dataSource->clear(); DEBUG("device error"); return; } if (createIndexEnabled) m_actualCols++; QVector dataContainer; int columnOffset = 0; //TODO: support other modes columnModes.resize(m_actualCols); //TODO: use given names QStringList vectorNames; if (createIndexEnabled) { vectorNames.prepend("index"); columnModes[0] = AbstractColumn::Integer; } columnOffset = dataSource->prepareImport(dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); if (lines == -1) lines = m_actualRows; // start column int startColumn = 0; if (createIndexEnabled) startColumn++; // read data //TODO: use ColumnMode ? DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(m_actualRows, lines); ++i) { DEBUG("reading row " << i); //prepend the index if required if (createIndexEnabled) static_cast*>(dataContainer[0])->operator[](i) = i+1; for (int n = startColumn; n < m_actualCols; ++n) { DEBUG("reading column " << n); switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT16: { qint16 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT32: { qint32 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::INT64: { qint64 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT8: { quint8 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT16: { quint16 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT32: { quint32 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::UINT64: { quint64 value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::REAL32: { float value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } case BinaryFilter::REAL64: { double value; in >> value; static_cast*>(dataContainer[n])->operator[](i) = value; break; } } } if (m_actualRows > 0) emit q->completed(100*i/m_actualRows); } dataSource->finalizeImport(columnOffset, 1, m_actualCols, "", importMode); } /*! writes the content of \c dataSource to the file \c fileName. */ void BinaryFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void BinaryFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("binaryFilter"); writer->writeAttribute("vectors", QString::number(d->vectors) ); writer->writeAttribute("dataType", QString::number(d->dataType) ); writer->writeAttribute("byteOrder", QString::number(d->byteOrder) ); writer->writeAttribute("autoMode", QString::number(d->autoModeEnabled) ); writer->writeAttribute("startRow", QString::number(d->startRow) ); writer->writeAttribute("endRow", QString::number(d->endRow) ); writer->writeAttribute("skipStartBytes", QString::number(d->skipStartBytes) ); writer->writeAttribute("skipBytes", QString::number(d->skipBytes) ); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled) ); writer->writeEndElement(); } /*! Loads from XML. */ bool BinaryFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "binaryFilter") { reader->raiseError(i18n("no binary filter element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); // read attributes QString str = attribs.value("vectors").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'vectors'")); else d->vectors = str.toInt(); str = attribs.value("dataType").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'dataType'")); else d->dataType = (BinaryFilter::DataType) str.toInt(); str = attribs.value("byteOrder").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'byteOrder'")); else d->byteOrder = (BinaryFilter::ByteOrder) str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'autoMode'")); else d->autoModeEnabled = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("skipStartBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipStartBytes'")); else d->skipStartBytes = str.toInt(); str = attribs.value("skipBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipBytes'")); else d->skipBytes = str.toInt(); str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/BinaryFilterPrivate.h b/src/backend/datasources/filters/BinaryFilterPrivate.h index 145d7a633..fccb921ef 100644 --- a/src/backend/datasources/filters/BinaryFilterPrivate.h +++ b/src/backend/datasources/filters/BinaryFilterPrivate.h @@ -1,69 +1,69 @@ /*************************************************************************** File : BinaryFilterPrivate.h Project : LabPlot Description : Private implementation class for BinaryFilter. -------------------------------------------------------------------- Copyright : (C) 2015-2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef BINARYFILTERPRIVATE_H #define BINARYFILTERPRIVATE_H #include class AbstractDataSource; class AbstractColumn; class BinaryFilterPrivate { public: explicit BinaryFilterPrivate(BinaryFilter*); int prepareStreamToRead(QDataStream&); void readDataFromDevice(QIODevice& device, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); const BinaryFilter* q; - int vectors; + size_t vectors; BinaryFilter::DataType dataType; BinaryFilter::ByteOrder byteOrder; QVector columnModes; int startRow; // start row (value*vectors) to read (can be -1) int endRow; // end row to (value*vectors) read (can be -1) size_t numRows; // number of rows size_t skipStartBytes; // bytes to skip at start size_t skipBytes; // bytes to skip after each value bool createIndexEnabled; // if create index column bool autoModeEnabled; private: int m_actualRows; int m_actualCols; }; #endif diff --git a/src/backend/worksheet/plots/cartesian/XYInterpolationCurve.cpp b/src/backend/worksheet/plots/cartesian/XYInterpolationCurve.cpp index a83f8e273..6abda0811 100644 --- a/src/backend/worksheet/plots/cartesian/XYInterpolationCurve.cpp +++ b/src/backend/worksheet/plots/cartesian/XYInterpolationCurve.cpp @@ -1,619 +1,619 @@ /*************************************************************************** File : XYInterpolationCurve.cpp Project : LabPlot Description : A xy-curve defined by an interpolation -------------------------------------------------------------------- Copyright : (C) 2016 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 20016-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ /*! \class XYInterpolationCurve \brief A xy-curve defined by an interpolation \ingroup worksheet */ #include "XYInterpolationCurve.h" #include "XYInterpolationCurvePrivate.h" #include "CartesianCoordinateSystem.h" #include "backend/core/column/Column.h" #include "backend/lib/commandtemplates.h" #include "backend/lib/macros.h" #include "backend/gsl/errors.h" extern "C" { #include #include #include "backend/nsl/nsl_diff.h" #include "backend/nsl/nsl_int.h" } #include #include #include XYInterpolationCurve::XYInterpolationCurve(const QString& name) : XYCurve(name, new XYInterpolationCurvePrivate(this)) { init(); } XYInterpolationCurve::XYInterpolationCurve(const QString& name, XYInterpolationCurvePrivate* dd) : XYCurve(name, dd) { init(); } XYInterpolationCurve::~XYInterpolationCurve() { //no need to delete the d-pointer here - it inherits from QGraphicsItem //and is deleted during the cleanup in QGraphicsScene } void XYInterpolationCurve::init() { Q_D(XYInterpolationCurve); //TODO: read from the saved settings for XYInterpolationCurve? d->lineType = XYCurve::Line; d->symbolsStyle = Symbol::NoSymbols; } void XYInterpolationCurve::recalculate() { Q_D(XYInterpolationCurve); d->recalculate(); } /*! Returns an icon to be used in the project explorer. */ QIcon XYInterpolationCurve::icon() const { return QIcon::fromTheme("labplot-xy-interpolation-curve"); } //############################################################################## //########################## getter methods ################################## //############################################################################## BASIC_SHARED_D_READER_IMPL(XYInterpolationCurve, const AbstractColumn*, xDataColumn, xDataColumn) BASIC_SHARED_D_READER_IMPL(XYInterpolationCurve, const AbstractColumn*, yDataColumn, yDataColumn) const QString& XYInterpolationCurve::xDataColumnPath() const { Q_D(const XYInterpolationCurve); return d->xDataColumnPath; } const QString& XYInterpolationCurve::yDataColumnPath() const { Q_D(const XYInterpolationCurve); return d->yDataColumnPath; } BASIC_SHARED_D_READER_IMPL(XYInterpolationCurve, XYInterpolationCurve::InterpolationData, interpolationData, interpolationData) const XYInterpolationCurve::InterpolationResult& XYInterpolationCurve::interpolationResult() const { Q_D(const XYInterpolationCurve); return d->interpolationResult; } //############################################################################## //################# setter methods and undo commands ########################## //############################################################################## STD_SETTER_CMD_IMPL_S(XYInterpolationCurve, SetXDataColumn, const AbstractColumn*, xDataColumn) void XYInterpolationCurve::setXDataColumn(const AbstractColumn* column) { Q_D(XYInterpolationCurve); if (column != d->xDataColumn) { exec(new XYInterpolationCurveSetXDataColumnCmd(d, column, i18n("%1: assign x-data"))); handleSourceDataChanged(); if (column) { connect(column, SIGNAL(dataChanged(const AbstractColumn*)), this, SLOT(handleSourceDataChanged())); //TODO disconnect on undo } } } STD_SETTER_CMD_IMPL_S(XYInterpolationCurve, SetYDataColumn, const AbstractColumn*, yDataColumn) void XYInterpolationCurve::setYDataColumn(const AbstractColumn* column) { Q_D(XYInterpolationCurve); if (column != d->yDataColumn) { exec(new XYInterpolationCurveSetYDataColumnCmd(d, column, i18n("%1: assign y-data"))); handleSourceDataChanged(); if (column) { connect(column, SIGNAL(dataChanged(const AbstractColumn*)), this, SLOT(handleSourceDataChanged())); //TODO disconnect on undo } } } STD_SETTER_CMD_IMPL_F_S(XYInterpolationCurve, SetInterpolationData, XYInterpolationCurve::InterpolationData, interpolationData, recalculate); void XYInterpolationCurve::setInterpolationData(const XYInterpolationCurve::InterpolationData& interpolationData) { Q_D(XYInterpolationCurve); exec(new XYInterpolationCurveSetInterpolationDataCmd(d, interpolationData, i18n("%1: set options and perform the interpolation"))); } //############################################################################## //######################### Private implementation ############################# //############################################################################## XYInterpolationCurvePrivate::XYInterpolationCurvePrivate(XYInterpolationCurve* owner) : XYCurvePrivate(owner), xDataColumn(0), yDataColumn(0), xColumn(0), yColumn(0), xVector(0), yVector(0), q(owner) { } XYInterpolationCurvePrivate::~XYInterpolationCurvePrivate() { //no need to delete xColumn and yColumn, they are deleted //when the parent aspect is removed } // ... // see XYFitCurvePrivate void XYInterpolationCurvePrivate::recalculate() { QElapsedTimer timer; timer.start(); //create interpolation result columns if not available yet, clear them otherwise if (!xColumn) { xColumn = new Column("x", AbstractColumn::Numeric); yColumn = new Column("y", AbstractColumn::Numeric); xVector = static_cast* >(xColumn->data()); yVector = static_cast* >(yColumn->data()); xColumn->setHidden(true); q->addChild(xColumn); yColumn->setHidden(true); q->addChild(yColumn); q->setUndoAware(false); q->setXColumn(xColumn); q->setYColumn(yColumn); q->setUndoAware(true); } else { xVector->clear(); yVector->clear(); } // clear the previous result interpolationResult = XYInterpolationCurve::InterpolationResult(); //determine the data source columns const AbstractColumn* tmpXDataColumn = 0; const AbstractColumn* tmpYDataColumn = 0; if (dataSourceType == XYCurve::DataSourceSpreadsheet) { //spreadsheet columns as data source tmpXDataColumn = xDataColumn; tmpYDataColumn = yDataColumn; } else { //curve columns as data source tmpXDataColumn = dataSourceCurve->xColumn(); tmpYDataColumn = dataSourceCurve->yColumn(); } if (!tmpXDataColumn || !tmpYDataColumn) { emit (q->dataChanged()); sourceDataChangedSinceLastRecalc = false; return; } //check column sizes if (tmpXDataColumn->rowCount() != tmpYDataColumn->rowCount()) { interpolationResult.available = true; interpolationResult.valid = false; interpolationResult.status = i18n("Number of x and y data points must be equal."); emit (q->dataChanged()); sourceDataChangedSinceLastRecalc = false; return; } //copy all valid data point for the interpolation to temporary vectors QVector xdataVector; QVector ydataVector; double xmin; double xmax; if (interpolationData.autoRange) { xmin = tmpXDataColumn->minimum(); xmax = tmpXDataColumn->maximum(); } else { xmin = interpolationData.xRange.first(); xmax = interpolationData.xRange.last(); } for (int row=0; rowrowCount(); ++row) { //only copy those data where _all_ values (for x and y, if given) are valid if (!std::isnan(tmpXDataColumn->valueAt(row)) && !std::isnan(tmpYDataColumn->valueAt(row)) && !tmpXDataColumn->isMasked(row) && !tmpYDataColumn->isMasked(row)) { // only when inside given range if (tmpXDataColumn->valueAt(row) >= xmin && tmpXDataColumn->valueAt(row) <= xmax) { xdataVector.append(tmpXDataColumn->valueAt(row)); ydataVector.append(tmpYDataColumn->valueAt(row)); } } } //number of data points to interpolate const unsigned int n = xdataVector.size(); if (n < 2) { interpolationResult.available = true; interpolationResult.valid = false; interpolationResult.status = i18n("Not enough data points available."); emit (q->dataChanged()); sourceDataChangedSinceLastRecalc = false; return; } double* xdata = xdataVector.data(); double* ydata = ydataVector.data(); // interpolation settings const nsl_interp_type type = interpolationData.type; const nsl_interp_pch_variant variant = interpolationData.variant; const double tension = interpolationData.tension; const double continuity = interpolationData.continuity; const double bias = interpolationData.bias; const nsl_interp_evaluate evaluate = interpolationData.evaluate; const unsigned int npoints = interpolationData.npoints; DEBUG("type:"<data(), yVector->data(), npoints); break; case nsl_interp_evaluate_second_derivative: nsl_diff_second_deriv_second_order(xVector->data(), yVector->data(), npoints); break; case nsl_interp_evaluate_integral: nsl_int_trapezoid(xVector->data(), yVector->data(), npoints, 0); break; } } // check values for (unsigned int i = 0; i < npoints; i++) { if ((*yVector)[i] > CartesianScale::LIMIT_MAX) (*yVector)[i] = CartesianScale::LIMIT_MAX; else if ((*yVector)[i] < CartesianScale::LIMIT_MIN) (*yVector)[i] = CartesianScale::LIMIT_MIN; } gsl_spline_free(spline); gsl_interp_accel_free(acc); /////////////////////////////////////////////////////////// //write the result interpolationResult.available = true; interpolationResult.valid = true; interpolationResult.status = gslErrorToString(status); interpolationResult.elapsedTime = timer.elapsed(); //redraw the curve emit (q->dataChanged()); sourceDataChangedSinceLastRecalc = false; } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## //! Save as XML void XYInterpolationCurve::save(QXmlStreamWriter* writer) const { Q_D(const XYInterpolationCurve); writer->writeStartElement("xyInterpolationCurve"); //write xy-curve information XYCurve::save(writer); //write xy-interpolation-curve specific information // interpolation data writer->writeStartElement("interpolationData"); WRITE_COLUMN(d->xDataColumn, xDataColumn); WRITE_COLUMN(d->yDataColumn, yDataColumn); writer->writeAttribute( "autoRange", QString::number(d->interpolationData.autoRange) ); writer->writeAttribute( "xRangeMin", QString::number(d->interpolationData.xRange.first()) ); writer->writeAttribute( "xRangeMax", QString::number(d->interpolationData.xRange.last()) ); writer->writeAttribute( "type", QString::number(d->interpolationData.type) ); writer->writeAttribute( "variant", QString::number(d->interpolationData.variant) ); writer->writeAttribute( "tension", QString::number(d->interpolationData.tension) ); writer->writeAttribute( "continuity", QString::number(d->interpolationData.continuity) ); writer->writeAttribute( "bias", QString::number(d->interpolationData.bias) ); writer->writeAttribute( "npoints", QString::number(d->interpolationData.npoints) ); writer->writeAttribute( "pointsMode", QString::number(d->interpolationData.pointsMode) ); writer->writeAttribute( "evaluate", QString::number(d->interpolationData.evaluate) ); writer->writeEndElement();// interpolationData // interpolation results (generated columns) writer->writeStartElement("interpolationResult"); writer->writeAttribute( "available", QString::number(d->interpolationResult.available) ); writer->writeAttribute( "valid", QString::number(d->interpolationResult.valid) ); writer->writeAttribute( "status", d->interpolationResult.status ); writer->writeAttribute( "time", QString::number(d->interpolationResult.elapsedTime) ); //save calculated columns if available if (d->xColumn) { d->xColumn->save(writer); d->yColumn->save(writer); } writer->writeEndElement(); //"interpolationResult" writer->writeEndElement(); //"xyInterpolationCurve" } //! Load from XML bool XYInterpolationCurve::load(XmlStreamReader* reader, bool preview) { Q_D(XYInterpolationCurve); if (!reader->isStartElement() || reader->name() != "xyInterpolationCurve") { reader->raiseError(i18n("no xy interpolation curve element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs; QString str; while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement() && reader->name() == "xyInterpolationCurve") break; if (!reader->isStartElement()) continue; if (reader->name() == "xyCurve") { if ( !XYCurve::load(reader, preview) ) return false; } else if (!preview && reader->name() == "interpolationData") { attribs = reader->attributes(); READ_COLUMN(xDataColumn); READ_COLUMN(yDataColumn); READ_INT_VALUE("autoRange", interpolationData.autoRange, bool); READ_DOUBLE_VALUE("xRangeMin", interpolationData.xRange.first()); READ_DOUBLE_VALUE("xRangeMax", interpolationData.xRange.last()); READ_INT_VALUE("type", interpolationData.type, nsl_interp_type); READ_INT_VALUE("variant", interpolationData.variant, nsl_interp_pch_variant); READ_DOUBLE_VALUE("tension", interpolationData.tension); READ_DOUBLE_VALUE("continuity", interpolationData.continuity); READ_DOUBLE_VALUE("bias", interpolationData.bias); READ_INT_VALUE("npoints", interpolationData.npoints, int); READ_INT_VALUE("pointsMode", interpolationData.pointsMode, XYInterpolationCurve::PointsMode); READ_INT_VALUE("evaluate", interpolationData.evaluate, nsl_interp_evaluate); } else if (!preview && reader->name() == "interpolationResult") { attribs = reader->attributes(); READ_INT_VALUE("available", interpolationResult.available, int); READ_INT_VALUE("valid", interpolationResult.valid, int); READ_STRING_VALUE("status", interpolationResult.status); READ_INT_VALUE("time", interpolationResult.elapsedTime, int); } else if (reader->name() == "column") { Column* column = new Column("", AbstractColumn::Numeric); if (!column->load(reader, preview)) { delete column; return false; } if (column->name()=="x") d->xColumn = column; else if (column->name()=="y") d->yColumn = column; } } if (preview) return true; // wait for data to be read before using the pointers QThreadPool::globalInstance()->waitForDone(); if (d->xColumn && d->yColumn) { d->xColumn->setHidden(true); addChild(d->xColumn); d->yColumn->setHidden(true); addChild(d->yColumn); d->xVector = static_cast* >(d->xColumn->data()); d->yVector = static_cast* >(d->yColumn->data()); setUndoAware(false); XYCurve::d_ptr->xColumn = d->xColumn; XYCurve::d_ptr->yColumn = d->yColumn; setUndoAware(true); } return true; }