diff --git a/src/backend/core/column/Column.cpp b/src/backend/core/column/Column.cpp index ec6540ee9..86f56c8ce 100644 --- a/src/backend/core/column/Column.cpp +++ b/src/backend/core/column/Column.cpp @@ -1,1322 +1,1322 @@ /*************************************************************************** File : Column.cpp Project : LabPlot Description : Aspect that manages a column -------------------------------------------------------------------- Copyright : (C) 2007-2009 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2013-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/core/column/Column.h" #include "backend/core/column/ColumnPrivate.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/column/columncommands.h" #include "backend/core/Project.h" #include "backend/lib/XmlStreamReader.h" #include "backend/core/datatypes/String2DateTimeFilter.h" #include "backend/core/datatypes/DateTime2StringFilter.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" #include "backend/worksheet/plots/cartesian/XYAnalysisCurve.h" extern "C" { #include } #include #include #include #include #include #include /** * \class Column * \brief Aspect that manages a column * * This class represents a column, i.e., (mathematically) a 1D vector of * values with a header. It provides a public reading and (undo aware) writing * interface as defined in AbstractColumn. A column * can have one of currently three data types: double, QString, or * QDateTime. The string representation of the values can differ depending * on the mode of the column. * * Column inherits from AbstractAspect and is intended to be a child * of the corresponding Spreadsheet in the aspect hierarchy. Columns don't * have a view as they are intended to be displayed inside a spreadsheet. */ Column::Column(const QString& name, ColumnMode mode) : AbstractColumn(name, AspectType::Column), d(new ColumnPrivate(this, mode)) { init(); } /** * \brief Common part of ctors */ void Column::init() { m_string_io = new ColumnStringIO(this); d->inputFilter()->input(0, m_string_io); d->outputFilter()->input(0, this); d->inputFilter()->setHidden(true); d->outputFilter()->setHidden(true); addChild(d->inputFilter()); addChild(d->outputFilter()); m_suppressDataChangedSignal = false; m_usedInActionGroup = new QActionGroup(this); connect(m_usedInActionGroup, &QActionGroup::triggered, this, &Column::navigateTo); } Column::~Column() { delete m_string_io; delete d; } QMenu* Column::createContextMenu() { QMenu* menu = AbstractAspect::createContextMenu(); QAction* firstAction{nullptr}; //insert after "rename" and "delete" actions, if available. //MQTTTopic columns don't have these actions if (menu->actions().size() > 1) firstAction = menu->actions().at(1); //add actions available in SpreadsheetView //TODO: we don't need to add anything from the view for MQTTTopic columns. //at the moment it's ok to check to the null pointer for firstAction here. //later, once we have some actions in the menu also for MQTT topics we'll //need to explicitely to dynamic_cast for MQTTTopic if (firstAction) emit requestProjectContextMenu(menu); //"Used in" menu containing all curves where the column is used QMenu* usedInMenu = new QMenu(i18n("Used in")); usedInMenu->setIcon(QIcon::fromTheme("go-next-view")); //remove previously added actions for (auto* action : m_usedInActionGroup->actions()) m_usedInActionGroup->removeAction(action); //add curves where the column is currently in use QVector curves = project()->children(AbstractAspect::Recursive); for (const auto* curve : curves) { bool used = false; const auto* analysisCurve = dynamic_cast(curve); if (analysisCurve) { if (analysisCurve->dataSourceType() == XYAnalysisCurve::DataSourceSpreadsheet && (analysisCurve->xDataColumn() == this || analysisCurve->yDataColumn() == this || analysisCurve->y2DataColumn() == this) ) used = true; } else { if (curve->xColumn() == this || curve->yColumn() == this) used = true; } if (used) { QAction* action = new QAction(curve->icon(), curve->name(), m_usedInActionGroup); action->setData(curve->path()); usedInMenu->addAction(action); } } if (firstAction) menu->insertSeparator(firstAction); menu->insertMenu(firstAction, usedInMenu); menu->insertSeparator(firstAction); return menu; } void Column::navigateTo(QAction* action) { project()->navigateTo(action->data().toString()); } /*! * */ void Column::setSuppressDataChangedSignal(bool b) { m_suppressDataChangedSignal = b; } /** * \brief Set the column mode * * This sets the column mode and, if * necessary, converts it to another datatype. */ void Column::setColumnMode(AbstractColumn::ColumnMode mode) { if (mode == columnMode()) return; DEBUG("Column::setColumnMode()"); beginMacro(i18n("%1: change column type", name())); auto* old_input_filter = d->inputFilter(); auto* old_output_filter = d->outputFilter(); exec(new ColumnSetModeCmd(d, mode)); if (d->inputFilter() != old_input_filter) { removeChild(old_input_filter); addChild(d->inputFilter()); d->inputFilter()->input(0, m_string_io); } if (d->outputFilter() != old_output_filter) { removeChild(old_output_filter); addChild(d->outputFilter()); d->outputFilter()->input(0, this); } endMacro(); DEBUG("Column::setColumnMode() DONE"); } void Column::setColumnModeFast(AbstractColumn::ColumnMode mode) { if (mode == columnMode()) return; auto* old_input_filter = d->inputFilter(); auto* old_output_filter = d->outputFilter(); exec(new ColumnSetModeCmd(d, mode)); if (d->inputFilter() != old_input_filter) { removeChild(old_input_filter); addChildFast(d->inputFilter()); d->inputFilter()->input(0, m_string_io); } if (d->outputFilter() != old_output_filter) { removeChild(old_output_filter); addChildFast(d->outputFilter()); d->outputFilter()->input(0, this); } } bool Column::isDraggable() const { return true; } QVector Column::dropableOn() const { return QVector{AspectType::CartesianPlot}; } /** * \brief Copy another column of the same type * * This function will return false if the data type * of 'other' is not the same as the type of 'this'. * Use a filter to convert a column to another type. */ bool Column::copy(const AbstractColumn* other) { Q_CHECK_PTR(other); if (other->columnMode() != columnMode()) return false; exec(new ColumnFullCopyCmd(d, other)); return true; } /** * \brief Copies a part of another column of the same type * * This function will return false if the data type * of 'other' is not the same as the type of 'this'. * \param other pointer to the column to copy * \param src_start first row to copy in the column to copy * \param dest_start first row to copy in * \param num_rows the number of rows to copy */ bool Column::copy(const AbstractColumn* source, int source_start, int dest_start, int num_rows) { Q_CHECK_PTR(source); if (source->columnMode() != columnMode()) return false; exec(new ColumnPartialCopyCmd(d, source, source_start, dest_start, num_rows)); return true; } /** * \brief Insert some empty (or initialized with zero) rows */ void Column::handleRowInsertion(int before, int count) { AbstractColumn::handleRowInsertion(before, count); exec(new ColumnInsertRowsCmd(d, before, count)); if (!m_suppressDataChangedSignal) emit dataChanged(this); d->statisticsAvailable = false; d->hasValuesAvailable = false; d->propertiesAvailable = false; } /** * \brief Remove 'count' rows starting from row 'first' */ void Column::handleRowRemoval(int first, int count) { AbstractColumn::handleRowRemoval(first, count); exec(new ColumnRemoveRowsCmd(d, first, count)); if (!m_suppressDataChangedSignal) emit dataChanged(this); d->statisticsAvailable = false; d->hasValuesAvailable = false; d->propertiesAvailable = false; } /** * \brief Set the column plot designation */ void Column::setPlotDesignation(AbstractColumn::PlotDesignation pd) { if (pd != plotDesignation()) exec(new ColumnSetPlotDesignationCmd(d, pd)); } /** * \brief Get width */ int Column::width() const { return d->width(); } /** * \brief Set width */ void Column::setWidth(int value) { d->setWidth(value); } /** * \brief Clear the whole column */ void Column::clear() { exec(new ColumnClearCmd(d)); } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// //! \name Formula related functions //@{ //////////////////////////////////////////////////////////////////////////////// /** * \brief Returns the formula used to generate column values */ QString Column:: formula() const { return d->formula(); } const QStringList& Column::formulaVariableNames() const { return d->formulaVariableNames(); } const QVector& Column::formulaVariableColumns() const { return d->formulaVariableColumns(); } const QStringList& Column::formulaVariableColumnPaths() const { return d->formulaVariableColumnPaths(); } bool Column::formulaAutoUpdate() const { return d->formulaAutoUpdate(); } /** * \brief Sets the formula used to generate column values */ void Column::setFormula(const QString& formula, const QStringList& variableNames, const QVector& columns, bool autoUpdate) { exec(new ColumnSetGlobalFormulaCmd(d, formula, variableNames, columns, autoUpdate)); } /*! * in case the cell values are calculated via a global column formula, * updates the values on data changes in all the dependent changes in the * "variable columns". */ void Column::updateFormula() { d->updateFormula(); } /** * \brief Set a formula string for an interval of rows */ void Column::setFormula(const Interval& i, const QString& formula) { exec(new ColumnSetFormulaCmd(d, i, formula)); } /** * \brief Overloaded function for convenience */ void Column::setFormula(int row, const QString& formula) { setFormula(Interval(row, row), formula); } /** * \brief Clear all formulas */ void Column::clearFormulas() { exec(new ColumnClearFormulasCmd(d)); } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// //! \name type specific functions //@{ //////////////////////////////////////////////////////////////////////////////// /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Text */ void Column::setTextAt(int row, const QString& new_value) { DEBUG("Column::setTextAt()"); d->statisticsAvailable = false; d->propertiesAvailable = false; exec(new ColumnSetTextCmd(d, row, new_value)); } /** * \brief Replace a range of values * * Use this only when columnMode() is Text */ void Column::replaceTexts(int first, const QVector& new_values) { DEBUG("Column::replaceTexts()"); if (!new_values.isEmpty()) { //TODO: do we really need this check? d->statisticsAvailable = false; d->propertiesAvailable = false; exec(new ColumnReplaceTextsCmd(d, first, new_values)); } } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void Column::setDateAt(int row, QDate new_value) { d->statisticsAvailable = false; d->propertiesAvailable = false; setDateTimeAt(row, QDateTime(new_value, timeAt(row))); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void Column::setTimeAt(int row, QTime new_value) { d->statisticsAvailable = false; d->propertiesAvailable = false; setDateTimeAt(row, QDateTime(dateAt(row), new_value)); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void Column::setDateTimeAt(int row, const QDateTime& new_value) { d->statisticsAvailable = false; d->propertiesAvailable = false; exec(new ColumnSetDateTimeCmd(d, row, new_value)); } /** * \brief Replace a range of values * * Use this only when columnMode() is DateTime, Month or Day */ void Column::replaceDateTimes(int first, const QVector& new_values) { if (!new_values.isEmpty()) { d->statisticsAvailable = false; d->propertiesAvailable = false; exec(new ColumnReplaceDateTimesCmd(d, first, new_values)); } } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Numeric */ void Column::setValueAt(int row, const double new_value) { // DEBUG("Column::setValueAt()"); d->statisticsAvailable = false; d->hasValuesAvailable = false; d->propertiesAvailable = false; exec(new ColumnSetValueCmd(d, row, new_value)); } /** * \brief Replace a range of values * * Use this only when columnMode() is Numeric */ void Column::replaceValues(int first, const QVector& new_values) { DEBUG("Column::replaceValues()"); if (!new_values.isEmpty()) { d->statisticsAvailable = false; d->hasValuesAvailable = false; d->propertiesAvailable = false; exec(new ColumnReplaceValuesCmd(d, first, new_values)); } } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Integer */ void Column::setIntegerAt(int row, const qint64 new_value) { DEBUG("Column::setIntegerAt()"); d->statisticsAvailable = false; d->hasValuesAvailable = false; d->propertiesAvailable = false; exec(new ColumnSetIntegerCmd(d, row, new_value)); } /** * \brief Replace a range of values * * Use this only when columnMode() is Integer */ void Column::replaceInteger(int first, const QVector& new_values) { DEBUG("Column::replaceInteger()"); if (!new_values.isEmpty()) { d->statisticsAvailable = false; d->hasValuesAvailable = false; d->propertiesAvailable = false; exec(new ColumnReplaceIntegersCmd(d, first, new_values)); } } /*! * \brief Column::properties * Returns the column properties of this curve (monoton increasing, monoton decreasing, ... ) * \see AbstractColumn::properties */ AbstractColumn::Properties Column::properties() const{ if (!d->propertiesAvailable) d->updateProperties(); return d->properties; } const Column::ColumnStatistics& Column::statistics() const { if (!d->statisticsAvailable) calculateStatistics(); return d->statistics; } void Column::calculateStatistics() const { d->statistics = ColumnStatistics(); ColumnStatistics& statistics = d->statistics; // TODO: support other data types? auto* rowValues = reinterpret_cast*>(data()); size_t notNanCount = 0; double val; double columnSum = 0.0; double columnProduct = 1.0; double columnSumNeg = 0.0; double columnSumSquare = 0.0; statistics.minimum = INFINITY; statistics.maximum = -INFINITY; QMap frequencyOfValues; QVector rowData; rowData.reserve(rowValues->size()); for (int row = 0; row < rowValues->size(); ++row) { val = rowValues->value(row); if (std::isnan(val) || isMasked(row)) continue; if (val < statistics.minimum) statistics.minimum = val; if (val > statistics.maximum) statistics.maximum = val; columnSum += val; columnSumNeg += (1.0 / val); columnSumSquare += pow(val, 2.0); columnProduct *= val; if (frequencyOfValues.contains(val)) frequencyOfValues.operator [](val)++; else frequencyOfValues.insert(val, 1); ++notNanCount; rowData.push_back(val); } if (notNanCount == 0) { d->statisticsAvailable = true; return; } if (rowData.size() < rowValues->size()) rowData.squeeze(); statistics.arithmeticMean = columnSum / notNanCount; statistics.geometricMean = pow(columnProduct, 1.0 / notNanCount); statistics.harmonicMean = notNanCount / columnSumNeg; statistics.contraharmonicMean = columnSumSquare / columnSum; double columnSumVariance = 0; double columnSumMeanDeviation = 0.0; double columnSumMedianDeviation = 0.0; double sumForCentralMoment_r3 = 0.0; double sumForCentralMoment_r4 = 0.0; gsl_sort(rowData.data(), 1, notNanCount); statistics.median = (notNanCount%2) ? rowData.at((int)((notNanCount-1)/2)) : (rowData.at((int)((notNanCount-1)/2)) + rowData.at((int)(notNanCount/2)))/2.0; QVector absoluteMedianList; absoluteMedianList.reserve((int)notNanCount); absoluteMedianList.resize((int)notNanCount); int idx = 0; for (int row = 0; row < rowValues->size(); ++row) { val = rowValues->value(row); if (std::isnan(val) || isMasked(row) ) continue; columnSumVariance += pow(val - statistics.arithmeticMean, 2.0); sumForCentralMoment_r3 += pow(val - statistics.arithmeticMean, 3.0); sumForCentralMoment_r4 += pow(val - statistics.arithmeticMean, 4.0); columnSumMeanDeviation += fabs( val - statistics.arithmeticMean ); absoluteMedianList[idx] = fabs(val - statistics.median); columnSumMedianDeviation += absoluteMedianList[idx]; idx++; } statistics.meanDeviationAroundMedian = columnSumMedianDeviation / notNanCount; statistics.medianDeviation = (notNanCount%2) ? absoluteMedianList.at((int)((notNanCount-1)/2)) : (absoluteMedianList.at((int)((notNanCount-1)/2)) + absoluteMedianList.at((int)(notNanCount/2)))/2.0; const double centralMoment_r3 = sumForCentralMoment_r3 / notNanCount; const double centralMoment_r4 = sumForCentralMoment_r4 / notNanCount; statistics.variance = columnSumVariance / notNanCount; statistics.standardDeviation = sqrt(statistics.variance); statistics.skewness = centralMoment_r3 / pow(statistics.standardDeviation, 3.0); statistics.kurtosis = (centralMoment_r4 / pow(statistics.standardDeviation, 4.0)) - 3.0; statistics.meanDeviation = columnSumMeanDeviation / notNanCount; double entropy = 0.0; for (const auto& v : frequencyOfValues) { const double frequencyNorm = static_cast(v) / notNanCount; entropy += (frequencyNorm * log2(frequencyNorm)); } statistics.entropy = -entropy; d->statisticsAvailable = true; } ////////////////////////////////////////////////////////////////////////////////////////////// void* Column::data() const { return d->data(); } /*! * return \c true if the column has numeric values, \false otherwise. */ bool Column::hasValues() const { if (d->hasValuesAvailable) return d->hasValues; bool foundValues = false; if (columnMode() == AbstractColumn::Numeric) { for (int row = 0; row < rowCount(); ++row) { if (!std::isnan(valueAt(row))) { foundValues = true; break; } } } else if (columnMode() == AbstractColumn::Integer) { //integer column has always valid values foundValues = true; } else if (columnMode() == AbstractColumn::DateTime) { for (int row = 0; row < rowCount(); ++row) { if (dateTimeAt(row).isValid()) { foundValues = true; break; } } } d->hasValues = foundValues; d->hasValuesAvailable = true; return d->hasValues; } //TODO: support all data types /** * \brief Return the content of row 'row'. * * Use this only when columnMode() is Text */ QString Column::textAt(int row) const { return d->textAt(row); } /** * \brief Return the date part of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QDate Column::dateAt(int row) const { return d->dateAt(row); } /** * \brief Return the time part of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QTime Column::timeAt(int row) const { return d->timeAt(row); } /** * \brief Return the QDateTime in row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QDateTime Column::dateTimeAt(int row) const { return d->dateTimeAt(row); } /** * \brief Return the double value in row 'row' */ double Column::valueAt(int row) const { return d->valueAt(row); } /** * \brief Return the integer value in row 'row' */ qint64 Column::integerAt(int row) const { return d->integerAt(row); } /* * call this function if the data of the column was changed directly via the data()-pointer * and not via the setValueAt() in order to emit the dataChanged-signal. * This is used e.g. in \c XYFitCurvePrivate::recalculate() */ void Column::setChanged() { d->propertiesAvailable = false; if (!m_suppressDataChangedSignal) emit dataChanged(this); d->statisticsAvailable = false; d->hasValuesAvailable = false; } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// /** * \brief Return an icon to be used for decorating the views and spreadsheet column headers */ QIcon Column::icon() const { return iconForMode(columnMode()); } //////////////////////////////////////////////////////////////////////////////////////////////////// //! \name serialize/deserialize //@{ //////////////////////////////////////////////////////////////////////////////////////////////////// /** * \brief Save the column as XML */ void Column::save(QXmlStreamWriter* writer) const { writer->writeStartElement("column"); writeBasicAttributes(writer); writer->writeAttribute("rows", QString::number(rowCount())); writer->writeAttribute("designation", QString::number(plotDesignation())); writer->writeAttribute("mode", QString::number(columnMode())); writer->writeAttribute("width", QString::number(width())); //save the formula used to generate column values, if available if (!formula().isEmpty() ) { writer->writeStartElement("formula"); writer->writeAttribute("autoUpdate", QString::number(d->formulaAutoUpdate())); writer->writeTextElement("text", formula()); writer->writeStartElement("variableNames"); for (const auto& name : formulaVariableNames()) writer->writeTextElement("name", name); writer->writeEndElement(); writer->writeStartElement("columnPathes"); for (const auto path : formulaVariableColumnPaths()) writer->writeTextElement("path", path); writer->writeEndElement(); writer->writeEndElement(); } writeCommentElement(writer); writer->writeStartElement("input_filter"); d->inputFilter()->save(writer); writer->writeEndElement(); writer->writeStartElement("output_filter"); d->outputFilter()->save(writer); writer->writeEndElement(); XmlWriteMask(writer); //TODO: formula in cells is not implemented yet // QVector< Interval > formulas = formulaIntervals(); // foreach(const Interval& interval, formulas) { // writer->writeStartElement("formula"); // writer->writeAttribute("start_row", QString::number(interval.start())); // writer->writeAttribute("end_row", QString::number(interval.end())); // writer->writeCharacters(formula(interval.start())); // writer->writeEndElement(); // } int i; switch (columnMode()) { case AbstractColumn::Numeric: { const char* data = reinterpret_cast(static_cast< QVector* >(d->data())->constData()); size_t size = d->rowCount() * sizeof(double); writer->writeCharacters(QByteArray::fromRawData(data, (int)size).toBase64()); break; } case AbstractColumn::Integer: { const char* data = reinterpret_cast(static_cast< QVector* >(d->data())->constData()); size_t size = d->rowCount() * sizeof(qint64); writer->writeCharacters(QByteArray::fromRawData(data, (int)size).toBase64()); break; } case AbstractColumn::Text: for (i = 0; i < rowCount(); ++i) { writer->writeStartElement("row"); writer->writeAttribute("index", QString::number(i)); writer->writeCharacters(textAt(i)); writer->writeEndElement(); } break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: for (i = 0; i < rowCount(); ++i) { writer->writeStartElement("row"); writer->writeAttribute("index", QString::number(i)); writer->writeCharacters(dateTimeAt(i).toString("yyyy-dd-MM hh:mm:ss:zzz")); writer->writeEndElement(); } break; } writer->writeEndElement(); // "column" } //TODO: extra header class DecodeColumnTask : public QRunnable { public: DecodeColumnTask(ColumnPrivate* priv, const QString& content) { m_private = priv; m_content = content; }; void run() override { QByteArray bytes = QByteArray::fromBase64(m_content.toLatin1()); if (m_private->columnMode() == AbstractColumn::Numeric) { auto* data = new QVector(bytes.size()/(int)sizeof(double)); memcpy(data->data(), bytes.data(), bytes.size()); m_private->replaceData(data); } else { - auto* data = new QVector(bytes.size()/(int)sizeof(int)); + auto* data = new QVector(bytes.size()/(int)sizeof(qint64)); memcpy(data->data(), bytes.data(), bytes.size()); m_private->replaceData(data); } } private: ColumnPrivate* m_private; QString m_content; }; /** * \brief Load the column from XML */ bool Column::load(XmlStreamReader* reader, bool preview) { if (!readBasicAttributes(reader)) return false; KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("rows").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("rows").toString()); else d->resizeTo(str.toInt()); str = attribs.value("designation").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("designation").toString()); else d->setPlotDesignation( AbstractColumn::PlotDesignation(str.toInt()) ); str = attribs.value("mode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("mode").toString()); else setColumnModeFast( AbstractColumn::ColumnMode(str.toInt()) ); str = attribs.value("width").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.subs("width").toString()); else d->setWidth(str.toInt()); // read child elements while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement()) break; if (reader->isStartElement()) { bool ret_val = true; if (reader->name() == "comment") ret_val = readCommentElement(reader); else if (reader->name() == "input_filter") ret_val = XmlReadInputFilter(reader); else if (reader->name() == "output_filter") ret_val = XmlReadOutputFilter(reader); else if (reader->name() == "mask") ret_val = XmlReadMask(reader); else if (reader->name() == "formula") ret_val = XmlReadFormula(reader); else if (reader->name() == "row") ret_val = XmlReadRow(reader); else { // unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } if (!ret_val) return false; } if (!preview) { QString content = reader->text().toString().trimmed(); if (!content.isEmpty() && ( columnMode() == AbstractColumn::Numeric || columnMode() == AbstractColumn::Integer)) { auto* task = new DecodeColumnTask(d, content); QThreadPool::globalInstance()->start(task); } } } return !reader->error(); } void Column::finalizeLoad() { d->finalizeLoad(); } /** * \brief Read XML input filter element */ bool Column::XmlReadInputFilter(XmlStreamReader* reader) { Q_ASSERT(reader->isStartElement() == true && reader->name() == "input_filter"); if (!reader->skipToNextTag()) return false; if (!d->inputFilter()->load(reader, false)) return false; if (!reader->skipToNextTag()) return false; Q_ASSERT(reader->isEndElement() == true && reader->name() == "input_filter"); return true; } /** * \brief Read XML output filter element */ bool Column::XmlReadOutputFilter(XmlStreamReader* reader) { Q_ASSERT(reader->isStartElement() == true && reader->name() == "output_filter"); if (!reader->skipToNextTag()) return false; if (!d->outputFilter()->load(reader, false)) return false; if (!reader->skipToNextTag()) return false; Q_ASSERT(reader->isEndElement() == true && reader->name() == "output_filter"); return true; } /** * \brief Read XML formula element */ bool Column::XmlReadFormula(XmlStreamReader* reader) { QString formula; QStringList variableNames; QStringList columnPathes; bool autoUpdate = reader->attributes().value("autoUpdate").toInt(); while (reader->readNext()) { if (reader->isEndElement()) break; if (reader->name() == "text") formula = reader->readElementText(); else if (reader->name() == "variableNames") { while (reader->readNext()) { if (reader->name() == "variableNames" && reader->isEndElement()) break; if (reader->isStartElement()) variableNames << reader->readElementText(); } } else if (reader->name() == "columnPathes") { while (reader->readNext()) { if (reader->name() == "columnPathes" && reader->isEndElement()) break; if (reader->isStartElement()) columnPathes << reader->readElementText(); } } } d->setFormula(formula, variableNames, columnPathes, autoUpdate); return true; } //TODO: read cell formula, not implemented yet // bool Column::XmlReadFormula(XmlStreamReader* reader) // { // Q_ASSERT(reader->isStartElement() && reader->name() == "formula"); // // bool ok1, ok2; // int start, end; // start = reader->readAttributeInt("start_row", &ok1); // end = reader->readAttributeInt("end_row", &ok2); // if (!ok1 || !ok2) // { // reader->raiseError(i18n("invalid or missing start or end row")); // return false; // } // setFormula(Interval(start,end), reader->readElementText()); // // return true; // } /** * \brief Read XML row element */ bool Column::XmlReadRow(XmlStreamReader* reader) { Q_ASSERT(reader->isStartElement() == true && reader->name() == "row"); // QXmlStreamAttributes attribs = reader->attributes(); bool ok; int index = reader->readAttributeInt("index", &ok); if (!ok) { reader->raiseError(i18n("invalid or missing row index")); return false; } QString str = reader->readElementText(); switch (columnMode()) { case AbstractColumn::Numeric: { double value = str.toDouble(&ok); if (!ok) { reader->raiseError(i18n("invalid row value")); return false; } setValueAt(index, value); break; } case AbstractColumn::Integer: { qint64 value = str.toLongLong(&ok); if (!ok) { reader->raiseError(i18n("invalid row value")); return false; } setIntegerAt(index, value); break; } case AbstractColumn::Text: setTextAt(index, str); break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: QDateTime date_time = QDateTime::fromString(str,"yyyy-dd-MM hh:mm:ss:zzz"); setDateTimeAt(index, date_time); break; } return true; } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// /** * \brief Return whether the object is read-only */ bool Column::isReadOnly() const { return false; } /** * \brief Return the column mode * * This function is mostly used by spreadsheets but can also be used * by plots. The column mode specifies how to interpret * the values in the column additional to the data type. */ AbstractColumn::ColumnMode Column::columnMode() const { return d->columnMode(); } /** * \brief Return the data vector size * * This returns the number of rows that actually contain data. * Rows beyond this can be masked etc. but should be ignored by filters, * plots etc. */ int Column::rowCount() const { return d->rowCount(); } /** * \brief Return the column plot designation */ AbstractColumn::PlotDesignation Column::plotDesignation() const { return d->plotDesignation(); } AbstractSimpleFilter* Column::outputFilter() const { return d->outputFilter(); } /** * \brief Return a wrapper column object used for String I/O. */ ColumnStringIO* Column::asStringColumn() const { return m_string_io; } //////////////////////////////////////////////////////////////////////////////// //! \name IntervalAttribute related functions //@{ //////////////////////////////////////////////////////////////////////////////// /** * \brief Return the formula associated with row 'row' */ QString Column::formula(int row) const { return d->formula(row); } /** * \brief Return the intervals that have associated formulas * * This can be used to make a list of formulas with their intervals. * Here is some example code: * * \code * QStringList list; * QVector< Interval > intervals = my_column.formulaIntervals(); * foreach(Interval interval, intervals) * list << QString(interval.toString() + ": " + my_column.formula(interval.start())); * \endcode */ QVector< Interval > Column::formulaIntervals() const { return d->formulaIntervals(); } void Column::handleFormatChange() { DEBUG("Column::handleFormatChange() mode = " << ENUM_TO_STRING(AbstractColumn, ColumnMode, columnMode())); if (columnMode() == AbstractColumn::DateTime) { auto* input_filter = static_cast(d->inputFilter()); auto* output_filter = static_cast(d->outputFilter()); DEBUG("change format " << input_filter->format().toStdString() << " to " << output_filter->format().toStdString()); input_filter->setFormat(output_filter->format()); } emit aspectDescriptionChanged(this); // the icon for the type changed if (!m_suppressDataChangedSignal) emit dataChanged(this); // all cells must be repainted d->statisticsAvailable = false; d->hasValuesAvailable = false; d->propertiesAvailable = false; DEBUG("Column::handleFormatChange() DONE"); } /*! * calculates the minimal value in the column. * for \c count = 0, the minimum of all elements is returned. * for \c count > 0, the minimum of the first \count elements is returned. * for \c count < 0, the minimum of the last \count elements is returned. */ double Column::minimum(int count) const { double min = INFINITY; if (count == 0 && d->statisticsAvailable) min = const_cast(this)->statistics().minimum; else { ColumnMode mode = columnMode(); int start, end; if (count == 0) { start = 0; end = rowCount(); } else if (count > 0) { start = 0; end = qMin(rowCount(), count); } else { start = qMax(rowCount() + count, 0); end = rowCount(); } switch (mode) { case Numeric: { auto* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const double val = vec->at(row); if (std::isnan(val)) continue; if (val < min) min = val; } break; } case Integer: { auto* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const qint64 val = vec->at(row); if (val < min) min = val; } break; } case Text: break; case DateTime: { auto* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const qint64 val = vec->at(row).toMSecsSinceEpoch(); if (val < min) min = val; } break; } case Day: case Month: default: break; } } return min; } /*! * calculates the maximal value in the column. * for \c count = 0, the maximum of all elements is returned. * for \c count > 0, the maximum of the first \count elements is returned. * for \c count < 0, the maximum of the last \count elements is returned. */ double Column::maximum(int count) const { double max = -INFINITY; if (count == 0 && d->statisticsAvailable) max = const_cast(this)->statistics().maximum; else { ColumnMode mode = columnMode(); int start, end; if (count == 0) { start = 0; end = rowCount(); } else if (count > 0) { start = 0; end = qMin(rowCount(), count); } else { start = qMax(rowCount() + count, 0); end = rowCount(); } switch (mode) { case Numeric: { auto* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const double val = vec->at(row); if (std::isnan(val)) continue; if (val > max) max = val; } break; } case Integer: { auto* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const qint64 val = vec->at(row); if (val > max) max = val; } break; } case Text: break; case DateTime: { auto* vec = static_cast*>(data()); for (int row = start; row < end; ++row) { const qint64 val = vec->at(row).toMSecsSinceEpoch(); if (val > max) max = val; } break; } case Day: case Month: default: break; } } return max; } diff --git a/src/backend/core/column/ColumnPrivate.cpp b/src/backend/core/column/ColumnPrivate.cpp index e6564fb47..a696f577e 100644 --- a/src/backend/core/column/ColumnPrivate.cpp +++ b/src/backend/core/column/ColumnPrivate.cpp @@ -1,1486 +1,1485 @@ /*************************************************************************** File : ColumnPrivate.cpp Project : AbstractColumn Description : Private data class of Column -------------------------------------------------------------------- Copyright : (C) 2007-2008 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2012-2019 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "ColumnPrivate.h" #include "ColumnStringIO.h" #include "Column.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/core/datatypes/filter.h" #include "backend/gsl/ExpressionParser.h" ColumnPrivate::ColumnPrivate(Column* owner, AbstractColumn::ColumnMode mode) : m_column_mode(mode), m_owner(owner) { Q_ASSERT(owner != nullptr); switch (mode) { case AbstractColumn::Numeric: m_input_filter = new String2DoubleFilter(); m_output_filter = new Double2StringFilter(); m_data = new QVector(); break; case AbstractColumn::Integer: m_input_filter = new String2IntegerFilter(); m_output_filter = new Integer2StringFilter(); - m_data = new QVector(); + m_data = new QVector(); break; case AbstractColumn::Text: m_input_filter = new SimpleCopyThroughFilter(); m_output_filter = new SimpleCopyThroughFilter(); m_data = new QStringList(); break; case AbstractColumn::DateTime: m_input_filter = new String2DateTimeFilter(); m_output_filter = new DateTime2StringFilter(); m_data = new QVector(); break; case AbstractColumn::Month: m_input_filter = new String2MonthFilter(); m_output_filter = new DateTime2StringFilter(); static_cast(m_output_filter)->setFormat("MMMM"); m_data = new QVector(); break; case AbstractColumn::Day: m_input_filter = new String2DayOfWeekFilter(); m_output_filter = new DateTime2StringFilter(); static_cast(m_output_filter)->setFormat("dddd"); m_data = new QVector(); break; } connect(m_output_filter, &AbstractSimpleFilter::formatChanged, m_owner, &Column::handleFormatChange); m_input_filter->setName("InputFilter"); m_output_filter->setName("OutputFilter"); } /** * \brief Special ctor (to be called from Column only!) */ ColumnPrivate::ColumnPrivate(Column* owner, AbstractColumn::ColumnMode mode, void* data) : m_column_mode(mode), m_data(data), m_owner(owner) { switch (mode) { case AbstractColumn::Numeric: m_input_filter = new String2DoubleFilter(); m_output_filter = new Double2StringFilter(); connect(static_cast(m_output_filter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Integer: m_input_filter = new String2IntegerFilter(); m_output_filter = new Integer2StringFilter(); connect(static_cast(m_output_filter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Text: m_input_filter = new SimpleCopyThroughFilter(); m_output_filter = new SimpleCopyThroughFilter(); break; case AbstractColumn::DateTime: m_input_filter = new String2DateTimeFilter(); m_output_filter = new DateTime2StringFilter(); connect(static_cast(m_output_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Month: m_input_filter = new String2MonthFilter(); m_output_filter = new DateTime2StringFilter(); static_cast(m_output_filter)->setFormat("MMMM"); connect(static_cast(m_output_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Day: m_input_filter = new String2DayOfWeekFilter(); m_output_filter = new DateTime2StringFilter(); static_cast(m_output_filter)->setFormat("dddd"); connect(static_cast(m_output_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; } m_input_filter->setName("InputFilter"); m_output_filter->setName("OutputFilter"); } ColumnPrivate::~ColumnPrivate() { if (!m_data) return; switch (m_column_mode) { case AbstractColumn::Numeric: delete static_cast*>(m_data); break; case AbstractColumn::Integer: delete static_cast*>(m_data); break; case AbstractColumn::Text: delete static_cast*>(m_data); break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: delete static_cast*>(m_data); break; } } AbstractColumn::ColumnMode ColumnPrivate::columnMode() const { return m_column_mode; } /** * \brief Set the column mode * * This sets the column mode and, if * necessary, converts it to another datatype. * Remark: setting the mode back to undefined (the * initial value) is not supported. */ void ColumnPrivate::setColumnMode(AbstractColumn::ColumnMode mode) { DEBUG("ColumnPrivate::setColumnMode() " << ENUM_TO_STRING(AbstractColumn, ColumnMode, m_column_mode) << " -> " << ENUM_TO_STRING(AbstractColumn, ColumnMode, mode)) if (mode == m_column_mode) return; void* old_data = m_data; // remark: the deletion of the old data will be done in the dtor of a command AbstractSimpleFilter* filter = nullptr, *new_in_filter = nullptr, *new_out_filter = nullptr; bool filter_is_temporary = false; // it can also become outputFilter(), which we may not delete here Column* temp_col = nullptr; emit m_owner->modeAboutToChange(m_owner); // determine the conversion filter and allocate the new data vector switch (m_column_mode) { // old mode case AbstractColumn::Numeric: { disconnect(static_cast(m_output_filter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); switch (mode) { case AbstractColumn::Numeric: break; case AbstractColumn::Integer: filter = new Double2IntegerFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast*>(old_data))); m_data = new QVector(); break; case AbstractColumn::Text: filter = outputFilter(); filter_is_temporary = false; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data))); m_data = new QVector(); break; case AbstractColumn::DateTime: filter = new Double2DateTimeFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data))); m_data = new QVector(); break; case AbstractColumn::Month: filter = new Double2MonthFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data))); m_data = new QVector(); break; case AbstractColumn::Day: filter = new Double2DayOfWeekFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data))); m_data = new QVector(); break; } // switch(mode) break; } case AbstractColumn::Integer: { disconnect(static_cast(m_output_filter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); switch (mode) { case AbstractColumn::Integer: break; case AbstractColumn::Numeric: filter = new Integer2DoubleFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast*>(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::Text: filter = outputFilter(); filter_is_temporary = false; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::DateTime: filter = new Integer2DateTimeFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::Month: filter = new Integer2MonthFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::Day: filter = new Integer2DayOfWeekFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data)), m_column_mode); m_data = new QVector(); break; } // switch(mode) break; } case AbstractColumn::Text: { switch (mode) { case AbstractColumn::Text: break; case AbstractColumn::Numeric: filter = new String2DoubleFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast*>(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::Integer: filter = new String2IntegerFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast*>(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::DateTime: filter = new String2DateTimeFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast*>(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::Month: filter = new String2MonthFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast*>(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::Day: filter = new String2DayOfWeekFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast*>(old_data)), m_column_mode); m_data = new QVector(); break; } // switch(mode) break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { disconnect(static_cast(m_output_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); switch (mode) { case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: break; case AbstractColumn::Text: filter = outputFilter(); filter_is_temporary = false; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data)), m_column_mode); m_data = new QStringList(); break; case AbstractColumn::Numeric: if (m_column_mode == AbstractColumn::Month) filter = new Month2DoubleFilter(); else if (m_column_mode == AbstractColumn::Day) filter = new DayOfWeek2DoubleFilter(); else filter = new DateTime2DoubleFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data)), m_column_mode); m_data = new QVector(); break; case AbstractColumn::Integer: if (m_column_mode == AbstractColumn::Month) filter = new Month2IntegerFilter(); else if (m_column_mode == AbstractColumn::Day) filter = new DayOfWeek2IntegerFilter(); else filter = new DateTime2IntegerFilter(); filter_is_temporary = true; temp_col = new Column("temp_col", *(static_cast< QVector* >(old_data)), m_column_mode); m_data = new QVector(); break; } // switch(mode) break; } } // determine the new input and output filters switch (mode) { // new mode case AbstractColumn::Numeric: new_in_filter = new String2DoubleFilter(); new_out_filter = new Double2StringFilter(); connect(static_cast(new_out_filter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Integer: new_in_filter = new String2IntegerFilter(); new_out_filter = new Integer2StringFilter(); connect(static_cast(new_out_filter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Text: new_in_filter = new SimpleCopyThroughFilter(); new_out_filter = new SimpleCopyThroughFilter(); break; case AbstractColumn::DateTime: new_in_filter = new String2DateTimeFilter(); new_out_filter = new DateTime2StringFilter(); connect(static_cast(new_out_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Month: new_in_filter = new String2MonthFilter(); new_out_filter = new DateTime2StringFilter(); static_cast(new_out_filter)->setFormat("MMMM"); DEBUG(" Month out_filter format: " << static_cast(new_out_filter)->format().toStdString()); connect(static_cast(new_out_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Day: new_in_filter = new String2DayOfWeekFilter(); new_out_filter = new DateTime2StringFilter(); static_cast(new_out_filter)->setFormat("dddd"); connect(static_cast(new_out_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; } // switch(mode) m_column_mode = mode; new_in_filter->setName("InputFilter"); new_out_filter->setName("OutputFilter"); m_input_filter = new_in_filter; m_output_filter = new_out_filter; m_input_filter->input(0, m_owner->m_string_io); m_output_filter->input(0, m_owner); m_input_filter->setHidden(true); m_output_filter->setHidden(true); if (temp_col) { // if temp_col == 0, only the input/output filters need to be changed // copy the filtered, i.e. converted, column (mode is orig mode) DEBUG(" temp_col column mode = " << ENUM_TO_STRING(AbstractColumn, ColumnMode, temp_col->columnMode())); filter->input(0, temp_col); DEBUG(" filter->output size = " << filter->output(0)->rowCount()); copy(filter->output(0)); delete temp_col; } if (filter_is_temporary) delete filter; emit m_owner->modeChanged(m_owner); DEBUG("ColumnPrivate::setColumnMode() DONE"); } /** * \brief Replace all mode related members * * Replace column mode, data type, data pointer and filters directly */ void ColumnPrivate::replaceModeData(AbstractColumn::ColumnMode mode, void* data, AbstractSimpleFilter* in_filter, AbstractSimpleFilter* out_filter) { DEBUG("ColumnPrivate::replaceModeData()"); emit m_owner->modeAboutToChange(m_owner); // disconnect formatChanged() switch (m_column_mode) { case AbstractColumn::Numeric: disconnect(static_cast(m_output_filter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Integer: disconnect(static_cast(m_output_filter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Text: break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: disconnect(static_cast(m_output_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; } m_column_mode = mode; m_data = data; in_filter->setName("InputFilter"); out_filter->setName("OutputFilter"); m_input_filter = in_filter; m_output_filter = out_filter; m_input_filter->input(0, m_owner->m_string_io); m_output_filter->input(0, m_owner); // connect formatChanged() switch (m_column_mode) { case AbstractColumn::Numeric: connect(static_cast(m_output_filter), &Double2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Integer: connect(static_cast(m_output_filter), &Integer2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; case AbstractColumn::Text: break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: connect(static_cast(m_output_filter), &DateTime2StringFilter::formatChanged, m_owner, &Column::handleFormatChange); break; } emit m_owner->modeChanged(m_owner); } /** * \brief Replace data pointer */ void ColumnPrivate::replaceData(void* data) { DEBUG("ColumnPrivate::replaceData()"); emit m_owner->dataAboutToChange(m_owner); m_data = data; if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /** * \brief Copy another column of the same type * * This function will return false if the data type * of 'other' is not the same as the type of 'this'. * Use a filter to convert a column to another type. */ bool ColumnPrivate::copy(const AbstractColumn* other) { DEBUG("ColumnPrivate::copy(other)"); if (other->columnMode() != columnMode()) return false; DEBUG(" mode = " << ENUM_TO_STRING(AbstractColumn, ColumnMode, columnMode())); int num_rows = other->rowCount(); DEBUG(" rows " << num_rows); emit m_owner->dataAboutToChange(m_owner); resizeTo(num_rows); // copy the data switch (m_column_mode) { case AbstractColumn::Numeric: { double* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; ++i) ptr[i] = other->valueAt(i); break; } case AbstractColumn::Integer: { qint64* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; ++i) ptr[i] = other->integerAt(i); break; } case AbstractColumn::Text: { for (int i = 0; i < num_rows; ++i) static_cast*>(m_data)->replace(i, other->textAt(i)); break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { for (int i = 0; i < num_rows; ++i) static_cast*>(m_data)->replace(i, other->dateTimeAt(i)); break; } } if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); return true; } /** * \brief Copies a part of another column of the same type * * This function will return false if the data type * of 'other' is not the same as the type of 'this'. * \param other pointer to the column to copy * \param src_start first row to copy in the column to copy * \param dest_start first row to copy in * \param num_rows the number of rows to copy */ bool ColumnPrivate::copy(const AbstractColumn* source, int source_start, int dest_start, int num_rows) { DEBUG("ColumnPrivate::copy()"); if (source->columnMode() != m_column_mode) return false; if (num_rows == 0) return true; emit m_owner->dataAboutToChange(m_owner); if (dest_start + num_rows > rowCount()) resizeTo(dest_start + num_rows); // copy the data switch (m_column_mode) { case AbstractColumn::Numeric: { double* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; i++) ptr[dest_start+i] = source->valueAt(source_start + i); break; } case AbstractColumn::Integer: { qint64* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; i++) ptr[dest_start+i] = source->integerAt(source_start + i); break; } case AbstractColumn::Text: for (int i = 0; i < num_rows; i++) static_cast*>(m_data)->replace(dest_start+i, source->textAt(source_start + i)); break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: for (int i = 0; i < num_rows; i++) static_cast*>(m_data)->replace(dest_start+i, source->dateTimeAt(source_start + i)); break; } if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); return true; } /** * \brief Copy another column of the same type * * This function will return false if the data type * of 'other' is not the same as the type of 'this'. * Use a filter to convert a column to another type. */ bool ColumnPrivate::copy(const ColumnPrivate* other) { if (other->columnMode() != m_column_mode) return false; int num_rows = other->rowCount(); emit m_owner->dataAboutToChange(m_owner); resizeTo(num_rows); // copy the data switch (m_column_mode) { case AbstractColumn::Numeric: { double* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; ++i) ptr[i] = other->valueAt(i); break; } case AbstractColumn::Integer: { qint64* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; ++i) ptr[i] = other->integerAt(i); break; } case AbstractColumn::Text: for (int i = 0; i < num_rows; ++i) static_cast*>(m_data)->replace(i, other->textAt(i)); break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: for (int i = 0; i < num_rows; ++i) static_cast*>(m_data)->replace(i, other->dateTimeAt(i)); break; } if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); return true; } /** * \brief Copies a part of another column of the same type * * This function will return false if the data type * of 'other' is not the same as the type of 'this'. * \param other pointer to the column to copy * \param src_start first row to copy in the column to copy * \param dest_start first row to copy in * \param num_rows the number of rows to copy */ bool ColumnPrivate::copy(const ColumnPrivate* source, int source_start, int dest_start, int num_rows) { if (source->columnMode() != m_column_mode) return false; if (num_rows == 0) return true; emit m_owner->dataAboutToChange(m_owner); if (dest_start + num_rows > rowCount()) resizeTo(dest_start + num_rows); // copy the data switch (m_column_mode) { case AbstractColumn::Numeric: { double* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; ++i) ptr[dest_start+i] = source->valueAt(source_start + i); break; } case AbstractColumn::Integer: { qint64* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; ++i) ptr[dest_start+i] = source->integerAt(source_start + i); break; } case AbstractColumn::Text: for (int i = 0; i < num_rows; ++i) static_cast*>(m_data)->replace(dest_start+i, source->textAt(source_start + i)); break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: for (int i = 0; i *>(m_data)->replace(dest_start+i, source->dateTimeAt(source_start + i)); break; } if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); return true; } /** * \brief Return the data vector size * * This returns the number of rows that actually contain data. * Rows beyond this can be masked etc. but should be ignored by filters, * plots etc. */ int ColumnPrivate::rowCount() const { switch (m_column_mode) { case AbstractColumn::Numeric: return static_cast*>(m_data)->size(); case AbstractColumn::Integer: return static_cast*>(m_data)->size(); case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: return static_cast*>(m_data)->size(); case AbstractColumn::Text: return static_cast*>(m_data)->size(); } return 0; } /** * \brief Resize the vector to the specified number of rows * * Since selecting and masking rows higher than the * real internal number of rows is supported, this * does not change the interval attributes. Also * no signal is emitted. If the new rows are filled * with values AbstractColumn::dataChanged() * must be emitted. */ void ColumnPrivate::resizeTo(int new_size) { int old_size = rowCount(); DEBUG("ColumnPrivate::resizeTo() " << old_size << " -> " << new_size); if (new_size == old_size) return; switch (m_column_mode) { case AbstractColumn::Numeric: { auto* numeric_data = static_cast*>(m_data); numeric_data->insert(numeric_data->end(), new_size - old_size, NAN); break; } case AbstractColumn::Integer: { auto* numeric_data = static_cast*>(m_data); numeric_data->insert(numeric_data->end(), new_size - old_size, 0); break; } case AbstractColumn::Text: { int new_rows = new_size - old_size; if (new_rows > 0) { for (int i = 0; i < new_rows; ++i) static_cast*>(m_data)->append(QString()); } else { for (int i = 0; i < -new_rows; ++i) static_cast*>(m_data)->removeLast(); } break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { int new_rows = new_size - old_size; if (new_rows > 0) { for (int i = 0; i < new_rows; ++i) static_cast*>(m_data)->append(QDateTime()); } else { for (int i = 0; i < -new_rows; ++i) static_cast*>(m_data)->removeLast(); } break; } } } /** * \brief Insert some empty (or initialized with zero) rows */ void ColumnPrivate::insertRows(int before, int count) { if (count == 0) return; m_formulas.insertRows(before, count); if (before <= rowCount()) { switch (m_column_mode) { case AbstractColumn::Numeric: static_cast*>(m_data)->insert(before, count, NAN); break; case AbstractColumn::Integer: static_cast*>(m_data)->insert(before, count, 0); break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: for (int i = 0; i < count; ++i) static_cast*>(m_data)->insert(before, QDateTime()); break; case AbstractColumn::Text: for (int i = 0; i < count; ++i) static_cast*>(m_data)->insert(before, QString()); break; } } } /** * \brief Remove 'count' rows starting from row 'first' */ void ColumnPrivate::removeRows(int first, int count) { if (count == 0) return; m_formulas.removeRows(first, count); if (first < rowCount()) { int corrected_count = count; if (first + count > rowCount()) corrected_count = rowCount() - first; switch (m_column_mode) { case AbstractColumn::Numeric: static_cast*>(m_data)->remove(first, corrected_count); break; case AbstractColumn::Integer: static_cast*>(m_data)->remove(first, corrected_count); break; case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: for (int i = 0; i < corrected_count; ++i) static_cast*>(m_data)->removeAt(first); break; case AbstractColumn::Text: for (int i = 0; i < corrected_count; ++i) static_cast*>(m_data)->removeAt(first); break; } } } //! Return the column name QString ColumnPrivate::name() const { return m_owner->name(); } /** * \brief Return the column plot designation */ AbstractColumn::PlotDesignation ColumnPrivate::plotDesignation() const { return m_plot_designation; } /** * \brief Set the column plot designation */ void ColumnPrivate::setPlotDesignation(AbstractColumn::PlotDesignation pd) { emit m_owner->plotDesignationAboutToChange(m_owner); m_plot_designation = pd; emit m_owner->plotDesignationChanged(m_owner); } /** * \brief Get width */ int ColumnPrivate::width() const { return m_width; } /** * \brief Set width */ void ColumnPrivate::setWidth(int value) { m_width = value; } /** * \brief Return the data pointer */ void* ColumnPrivate::data() const { return m_data; } /** * \brief Return the input filter (for string -> data type conversion) */ AbstractSimpleFilter *ColumnPrivate::inputFilter() const { return m_input_filter; } /** * \brief Return the output filter (for data type -> string conversion) */ AbstractSimpleFilter *ColumnPrivate::outputFilter() const { return m_output_filter; } //////////////////////////////////////////////////////////////////////////////// //! \name Formula related functions //@{ //////////////////////////////////////////////////////////////////////////////// /** * \brief Return the formula last used to generate data for the column */ QString ColumnPrivate::formula() const { return m_formula; } bool ColumnPrivate::formulaAutoUpdate() const { return m_formulaAutoUpdate; } /** * \brief Sets the formula used to generate column values */ void ColumnPrivate::setFormula(const QString& formula, const QStringList& variableNames, const QVector& variableColumns, bool autoUpdate) { m_formula = formula; m_formulaVariableNames = variableNames; m_formulaVariableColumns = variableColumns; m_formulaAutoUpdate = autoUpdate; for (auto connection: m_connectionsUpdateFormula) disconnect(connection); m_formulaVariableColumnPaths.clear(); if (autoUpdate) { for (auto column : variableColumns) { m_formulaVariableColumnPaths << column->path(); m_connectionsUpdateFormula << connect(column, &Column::dataChanged, m_owner, &Column::updateFormula); connect(column->parentAspect(), &AbstractAspect::aspectAboutToBeRemoved, this, &ColumnPrivate::formulaVariableColumnRemoved); connect(column->parentAspect(), &AbstractAspect::aspectAdded, this, &ColumnPrivate::formulaVariableColumnAdded); } } } /*! * called after the import of the project was done and all columns were loaded in \sa Project::load() * to establish the required slot-signal connections for the formula update */ void ColumnPrivate::finalizeLoad() { if (m_formulaAutoUpdate) { for (auto column : m_formulaVariableColumns) { m_connectionsUpdateFormula << connect(column, &Column::dataChanged, m_owner, &Column::updateFormula); connect(column->parentAspect(), &AbstractAspect::aspectAboutToBeRemoved, this, &ColumnPrivate::formulaVariableColumnRemoved); connect(column->parentAspect(), &AbstractAspect::aspectAdded, this, &ColumnPrivate::formulaVariableColumnAdded); } } } /*! * helper function used in \c Column::load() to set parameters read from the xml file. * \param variableColumnPathes is used to restore the pointers to columns from pathes * after the project was loaded in Project::load(). */ void ColumnPrivate::setFormula(const QString& formula, const QStringList& variableNames, const QStringList& variableColumnPaths, bool autoUpdate) { m_formula = formula; m_formulaVariableNames = variableNames; m_formulaVariableColumnPaths = variableColumnPaths; m_formulaAutoUpdate = autoUpdate; } const QStringList& ColumnPrivate::formulaVariableNames() const { return m_formulaVariableNames; } const QVector& ColumnPrivate::formulaVariableColumns() const { return m_formulaVariableColumns; } const QStringList& ColumnPrivate::formulaVariableColumnPaths() const { return m_formulaVariableColumnPaths; } /*! * \sa FunctionValuesDialog::generate() */ void ColumnPrivate::updateFormula() { //determine variable names and the data vectors of the specified columns QVector*> xVectors; QVector*> xNewVectors; int maxRowCount = 0; bool valid = true; for (auto column : m_formulaVariableColumns) { if (!column) { valid = false; break; } if (column->columnMode() == AbstractColumn::Integer) { //convert integers to doubles first auto* xVector = new QVector(column->rowCount()); for (int i = 0; irowCount(); ++i) xVector->operator[](i) = column->valueAt(i); xNewVectors << xVector; xVectors << xVector; } else xVectors << static_cast* >(column->data()); if (column->rowCount() > maxRowCount) maxRowCount = column->rowCount(); } if (valid) { //resize the spreadsheet if one of the data vectors from //other spreadsheet(s) has more elements than the parent spreadsheet Spreadsheet* spreadsheet = dynamic_cast(m_owner->parentAspect()); Q_ASSERT(spreadsheet); if (spreadsheet->rowCount() < maxRowCount) spreadsheet->setRowCount(maxRowCount); //create new vector for storing the calculated values //the vectors with the variable data can be smaller then the result vector. So, not all values in the result vector might get initialized. //->"clean" the result vector first QVector new_data(rowCount(), NAN); //evaluate the expression for f(x_1, x_2, ...) and write the calculated values into a new vector. ExpressionParser* parser = ExpressionParser::getInstance(); parser->evaluateCartesian(m_formula, m_formulaVariableNames, xVectors, &new_data); replaceValues(0, new_data); // initialize remaining rows with NAN int remainingRows = rowCount() - maxRowCount; if (remainingRows > 0) { QVector emptyRows(remainingRows, NAN); replaceValues(maxRowCount, emptyRows); } } else { QVector new_data(rowCount(), NAN); replaceValues(0, new_data); } //delete help vectors created for the conversion from int to double for (auto* vector : xNewVectors) delete vector; } void ColumnPrivate::formulaVariableColumnRemoved(const AbstractAspect* aspect) { const Column* column = dynamic_cast(aspect); //TODO: why is const_cast requried here?!? int index = m_formulaVariableColumns.indexOf(const_cast(column)); if (index != -1) { m_formulaVariableColumns[index] = nullptr; updateFormula(); } } void ColumnPrivate::formulaVariableColumnAdded(const AbstractAspect* aspect) { int index = m_formulaVariableColumnPaths.indexOf(aspect->path()); if (index != -1) { const Column* column = dynamic_cast(aspect); m_formulaVariableColumns[index] = const_cast(column); updateFormula(); } } /** * \brief Return the formula associated with row 'row' */ QString ColumnPrivate::formula(int row) const { return m_formulas.value(row); } /** * \brief Return the intervals that have associated formulas * * This can be used to make a list of formulas with their intervals. * Here is some example code: * * \code * QStringList list; * QVector< Interval > intervals = my_column.formulaIntervals(); * foreach(Interval interval, intervals) * list << QString(interval.toString() + ": " + my_column.formula(interval.start())); * \endcode */ QVector< Interval > ColumnPrivate::formulaIntervals() const { return m_formulas.intervals(); } /** * \brief Set a formula string for an interval of rows */ void ColumnPrivate::setFormula(Interval i, QString formula) { m_formulas.setValue(i, formula); } /** * \brief Overloaded function for convenience */ void ColumnPrivate::setFormula(int row, QString formula) { setFormula(Interval(row,row), formula); } /** * \brief Clear all formulas */ void ColumnPrivate::clearFormulas() { m_formulas.clear(); } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// //! \name type specific functions //@{ //////////////////////////////////////////////////////////////////////////////// /** * \brief Return the content of row 'row'. * * Use this only when columnMode() is Text */ QString ColumnPrivate::textAt(int row) const { if (m_column_mode != AbstractColumn::Text) return QString(); return static_cast*>(m_data)->value(row); } /** * \brief Return the date part of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QDate ColumnPrivate::dateAt(int row) const { if (m_column_mode != AbstractColumn::DateTime && m_column_mode != AbstractColumn::Month && m_column_mode != AbstractColumn::Day) return QDate{}; return dateTimeAt(row).date(); } /** * \brief Return the time part of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QTime ColumnPrivate::timeAt(int row) const { if (m_column_mode != AbstractColumn::DateTime && m_column_mode != AbstractColumn::Month && m_column_mode != AbstractColumn::Day) return QTime{}; return dateTimeAt(row).time(); } /** * \brief Return the QDateTime in row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ QDateTime ColumnPrivate::dateTimeAt(int row) const { if (m_column_mode != AbstractColumn::DateTime && m_column_mode != AbstractColumn::Month && m_column_mode != AbstractColumn::Day) return QDateTime(); return static_cast*>(m_data)->value(row); } /** * \brief Return the double value in row 'row' for columns with type Numeric and Integer. * This function has to be used everywhere where the exact type (double or qint64) is not relevant for numerical calculations. * For cases where the integer value is needed without any implicit conversions, \sa integerAt() has to be used. */ double ColumnPrivate::valueAt(int row) const { if (m_column_mode == AbstractColumn::Numeric) return static_cast*>(m_data)->value(row, NAN); else if (m_column_mode == AbstractColumn::Integer) return static_cast*>(m_data)->value(row, 0); else return NAN; } /** * \brief Return the integer value in row 'row' */ qint64 ColumnPrivate::integerAt(int row) const { if (m_column_mode != AbstractColumn::Integer) return 0; return static_cast*>(m_data)->value(row, 0); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Text */ void ColumnPrivate::setTextAt(int row, const QString& new_value) { if (m_column_mode != AbstractColumn::Text) return; emit m_owner->dataAboutToChange(m_owner); if (row >= rowCount()) resizeTo(row+1); static_cast*>(m_data)->replace(row, new_value); if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /** * \brief Replace a range of values * * Use this only when columnMode() is Text */ void ColumnPrivate::replaceTexts(int first, const QVector& new_values) { if (m_column_mode != AbstractColumn::Text) return; emit m_owner->dataAboutToChange(m_owner); int num_rows = new_values.size(); if (first + num_rows > rowCount()) resizeTo(first + num_rows); for (int i = 0; i < num_rows; ++i) static_cast*>(m_data)->replace(first+i, new_values.at(i)); if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void ColumnPrivate::setDateAt(int row, QDate new_value) { if (m_column_mode != AbstractColumn::DateTime && m_column_mode != AbstractColumn::Month && m_column_mode != AbstractColumn::Day) return; setDateTimeAt(row, QDateTime(new_value, timeAt(row))); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void ColumnPrivate::setTimeAt(int row, QTime new_value) { if (m_column_mode != AbstractColumn::DateTime && m_column_mode != AbstractColumn::Month && m_column_mode != AbstractColumn::Day) return; setDateTimeAt(row, QDateTime(dateAt(row), new_value)); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is DateTime, Month or Day */ void ColumnPrivate::setDateTimeAt(int row, const QDateTime& new_value) { if (m_column_mode != AbstractColumn::DateTime && m_column_mode != AbstractColumn::Month && m_column_mode != AbstractColumn::Day) return; emit m_owner->dataAboutToChange(m_owner); if (row >= rowCount()) resizeTo(row+1); static_cast< QVector* >(m_data)->replace(row, new_value); if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /** * \brief Replace a range of values * * Use this only when columnMode() is DateTime, Month or Day */ void ColumnPrivate::replaceDateTimes(int first, const QVector& new_values) { if (m_column_mode != AbstractColumn::DateTime && m_column_mode != AbstractColumn::Month && m_column_mode != AbstractColumn::Day) return; emit m_owner->dataAboutToChange(m_owner); int num_rows = new_values.size(); if (first + num_rows > rowCount()) resizeTo(first + num_rows); for (int i = 0; i < num_rows; ++i) static_cast*>(m_data)->replace(first+i, new_values.at(i)); if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Numeric */ void ColumnPrivate::setValueAt(int row, double new_value) { // DEBUG("ColumnPrivate::setValueAt()"); if (m_column_mode != AbstractColumn::Numeric) return; emit m_owner->dataAboutToChange(m_owner); if (row >= rowCount()) resizeTo(row+1); static_cast*>(m_data)->replace(row, new_value); if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /** * \brief Replace a range of values * * Use this only when columnMode() is Numeric */ void ColumnPrivate::replaceValues(int first, const QVector& new_values) { DEBUG("ColumnPrivate::replaceValues()"); if (m_column_mode != AbstractColumn::Numeric) return; emit m_owner->dataAboutToChange(m_owner); int num_rows = new_values.size(); if (first + num_rows > rowCount()) resizeTo(first + num_rows); double* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; ++i) ptr[first+i] = new_values.at(i); if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /** * \brief Set the content of row 'row' * * Use this only when columnMode() is Integer */ void ColumnPrivate::setIntegerAt(int row, qint64 new_value) { DEBUG("ColumnPrivate::setIntegerAt()"); if (m_column_mode != AbstractColumn::Integer) return; emit m_owner->dataAboutToChange(m_owner); if (row >= rowCount()) resizeTo(row + 1); static_cast*>(m_data)->replace(row, new_value); if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /** * \brief Replace a range of values * * Use this only when columnMode() is Integer */ void ColumnPrivate::replaceInteger(int first, const QVector& new_values) { DEBUG("ColumnPrivate::replaceInteger()"); if (m_column_mode != AbstractColumn::Integer) return; emit m_owner->dataAboutToChange(m_owner); int num_rows = new_values.size(); if (first + num_rows > rowCount()) resizeTo(first + num_rows); qint64* ptr = static_cast*>(m_data)->data(); for (int i = 0; i < num_rows; ++i) ptr[first+i] = new_values.at(i); if (!m_owner->m_suppressDataChangedSignal) emit m_owner->dataChanged(m_owner); } /*! * Updates the properties. Will be called, when data in the column changed. * The properies will be used to speed up some algorithms. * See where variable properties will be used. */ void ColumnPrivate::updateProperties() { // TODO: for double Properties::Constant will never be used. Use an epsilon (difference smaller than epsilon is zero) if (rowCount() == 0) { properties = AbstractColumn::Properties::No; propertiesAvailable = true; return; } double prevValue = NAN; - int prevValueInt = 0; + qint64 prevValueInt = 0; qint64 prevValueDatetime = 0; if (m_column_mode == AbstractColumn::Integer) prevValueInt = integerAt(0); else if (m_column_mode == AbstractColumn::Numeric) prevValue = valueAt(0); else if (m_column_mode == AbstractColumn::DateTime || m_column_mode == AbstractColumn::Month || m_column_mode == AbstractColumn::Day) prevValueDatetime = dateTimeAt(0).toMSecsSinceEpoch(); else { properties = AbstractColumn::Properties::No; propertiesAvailable = true; return; } int monotonic_decreasing = -1; int monotonic_increasing = -1; double value; - int valueInt; + qint64 valueInt; qint64 valueDateTime; for (int row = 1; row < rowCount(); row++) { - if (m_column_mode == AbstractColumn::Integer) { valueInt = integerAt(row); // check monotonic increasing if (valueInt >= prevValueInt && monotonic_increasing < 0) monotonic_increasing = 1; else if (valueInt < prevValueInt && monotonic_increasing >= 0) monotonic_increasing = 0; // else: nothing // check monotonic decreasing if (valueInt <= prevValueInt && monotonic_decreasing < 0) monotonic_decreasing = 1; else if (valueInt > prevValueInt && monotonic_decreasing >= 0) monotonic_decreasing = 0; prevValueInt = valueInt; } else if (m_column_mode == AbstractColumn::Numeric) { value = valueAt(row); // check monotonic increasing if (value >= prevValue && monotonic_increasing < 0) monotonic_increasing = 1; else if (value < prevValue || std::isnan(value)) { monotonic_increasing = 0; if (monotonic_decreasing == 0) break; } // else: nothing // check monotonic decreasing if (value <= prevValue && monotonic_decreasing < 0) monotonic_decreasing = 1; else if (value > prevValue || std::isnan(value)) { monotonic_decreasing = 0; if (monotonic_increasing == 0) break; } prevValue = value; } else if (m_column_mode == AbstractColumn::DateTime || m_column_mode == AbstractColumn::Month || m_column_mode == AbstractColumn::Day) { valueDateTime = dateTimeAt(row).toMSecsSinceEpoch(); // check monotonic increasing if (valueDateTime >= prevValueDatetime && monotonic_increasing < 0) monotonic_increasing = 1; else if (valueDateTime < prevValueDatetime) monotonic_increasing = 0; // else: nothing // check monotonic decreasing if (valueDateTime <= prevValueDatetime && monotonic_decreasing < 0) monotonic_decreasing = 1; else if (valueDateTime > prevValueDatetime) monotonic_decreasing = 0; prevValueDatetime = valueDateTime; } } properties = AbstractColumn::Properties::No; if (monotonic_increasing > 0 && monotonic_decreasing > 0) properties = AbstractColumn::Properties::Constant; else if (monotonic_decreasing > 0) properties = AbstractColumn::Properties::MonotonicDecreasing; else if (monotonic_increasing > 0) properties = AbstractColumn::Properties::MonotonicIncreasing; propertiesAvailable = true; } //////////////////////////////////////////////////////////////////////////////// //@} //////////////////////////////////////////////////////////////////////////////// /** * \brief Return the interval attribute representing the formula strings */ IntervalAttribute ColumnPrivate::formulaAttribute() const { return m_formulas; } /** * \brief Replace the interval attribute for the formula strings */ void ColumnPrivate::replaceFormulas(const IntervalAttribute& formulas) { m_formulas = formulas; } diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index 6c2f2ee1c..ade77f576 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,2633 +1,2633 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2018 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2019 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include "backend/core/Project.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/worksheet/plots/cartesian/CartesianPlot.h" #include "backend/worksheet/plots/cartesian/XYCurve.h" #include "backend/lib/macros.h" #include "backend/lib/trace.h" #ifdef HAVE_MQTT #include "backend/datasources/MQTTClient.h" #include "backend/datasources/MQTTTopic.h" #endif #include #include #include #if defined(Q_OS_LINUX) || defined(Q_OS_BSD4) #include #include #endif /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(Ascii), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() = default; /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource* dataSource) { d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { return d->readFromLiveDevice(device, dataSource, from); } #ifdef HAVE_MQTT QVector AsciiFilter::preview(const QString& message) { return d->preview(message); } /*! reads the content of a message received by the topic. */ void AsciiFilter::readMQTTTopic(const QString& message, AbstractDataSource* dataSource) { d->readMQTTTopic(message, dataSource); } /*! Returns the statistical data, that the MQTTTopic needs for the will message. */ QString AsciiFilter::MQTTColumnStatistics(const MQTTTopic* topic) const { return d->MQTTColumnStatistics(topic); } /*! Returns the column mode of the last column (the value column of the MQTTTopic). */ AbstractColumn::ColumnMode AsciiFilter::MQTTColumnMode() const { return d->MQTTColumnMode(); } /*! After the MQTTTopic is loaded, prepares the filter for reading. */ void AsciiFilter::setPreparedForMQTT(bool prepared, MQTTTopic* topic, const QString& separator) { d->setPreparedForMQTT(prepared, topic, separator); } #endif /*! returns the separator used by the filter. */ QString AsciiFilter::separator() const { return d->separator(); } /*! returns the separator used by the filter. */ int AsciiFilter::isPrepared() { return d->isPrepared(); } /*! reads the content of the file \c fileName. */ void AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { d->readDataFromFile(fileName, dataSource, importMode); } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } QVector AsciiFilter::preview(QIODevice& device) { return d->preview(device); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE" << "2xSPACE" << "3xSPACE" << "4xSPACE" << "2xTAB"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } QString AsciiFilter::fileInfoString(const QString& fileName) { QString info(i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName))); info += QLatin1String("
"); info += i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); return info; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " to determine number of lines"); return 0; } // if (!device.canReadLine()) // return -1; size_t lineCount = 0; #if defined(Q_OS_LINUX) || defined(Q_OS_BSD4) //on linux and BSD use wc, if available, which is much faster than counting lines in the file if (device.compressionType() == KCompressionDevice::None && !QStandardPaths::findExecutable(QLatin1String("wc")).isEmpty()) { QProcess wc; wc.start(QLatin1String("wc"), QStringList() << QLatin1String("-l") << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) { QString line(wc.readLine()); // wc on macOS has leading spaces: use SkipEmptyParts lineCount = line.split(' ', QString::SkipEmptyParts)[0].toInt(); } return lineCount; } #endif while (!device.atEnd()) { device.readLine(); lineCount++; } return lineCount; } /*! returns the number of lines in the device \c device and 0 if sequential. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice& device) const { if (device.isSequential()) return 0; // if (!device.canReadLine()) // DEBUG("WARNING in AsciiFilter::lineNumber(): device cannot 'readLine()' but using it anyway."); size_t lineCount = 0; device.seek(0); if (d->readingFile) lineCount = lineNumber(d->readingFileName); else { while (!device.atEnd()) { device.readLine(); lineCount++; } } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } bool AsciiFilter::createIndexEnabled() const { return d->createIndexEnabled; } void AsciiFilter::setCreateTimestampEnabled(bool b) { d->createTimestampEnabled = b; } bool AsciiFilter::createTimestampEnabled() const { return d->createTimestampEnabled; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool AsciiFilter::NaNValueToZeroEnabled() const { return (d->nanValue == 0); } void AsciiFilter::setRemoveQuotesEnabled(bool b) { d->removeQuotesEnabled = b; } bool AsciiFilter::removeQuotesEnabled() const { return d->removeQuotesEnabled; } void AsciiFilter::setVectorNames(const QString& s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner) { } /*! * get a single line from device */ QStringList AsciiFilterPrivate::getLineString(QIODevice& device) { QString line; do { // skip comment lines in data lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::getLineString(): device cannot 'readLine()' but using it anyway."); // line = device.readAll(); line = device.readLine(); } while (line.startsWith(commentCharacter)); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); DEBUG("data line : \'" << line.toStdString() << '\''); QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); //TODO: remove quotes here? QDEBUG("data line, parsed: " << lineStringList); return lineStringList; } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("AsciiFilterPrivate::prepareDeviceToRead(): is sequential = " << device.isSequential() << ", can readLine = " << device.canReadLine()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; ///////////////////////////////////////////////////////////////// // Find first data line (ignoring comment lines) DEBUG(" Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { QString line; if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); line = device.readLine(); DEBUG(" line = " << line.toStdString()); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } } // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines if (!device.canReadLine()) DEBUG("WARNING in AsciiFilterPrivate::prepareDeviceToRead(): device cannot 'readLine()' but using it anyway."); if (device.atEnd()) { DEBUG("device at end! Giving up."); if (device.isSequential()) break; else return 1; } firstLine = device.readLine(); } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (removeQuotesEnabled) firstLine = firstLine.remove(QLatin1Char('"')); //TODO: this doesn't work, the split below introduces whitespaces again // if (simplifyWhitespacesEnabled) // firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive); m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); QDEBUG("first line: " << firstLineStringList); DEBUG("headerEnabled: " << headerEnabled); //optionally, remove potential spaces in the first line //TODO: this part should be obsolete actually if we do firstLine = firstLine.simplified(); above... if (simplifyWhitespacesEnabled) { for (int i = 0; i < firstLineStringList.size(); ++i) firstLineStringList[i] = firstLineStringList[i].simplified(); } //in GUI in AsciiOptionsWidget we start counting from 1, subtract 1 here to start from zero m_actualStartRow = startRow - 1; if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); ++m_actualStartRow; } // set range to read if (endColumn == -1) { if (headerEnabled || vectorNames.size() == 0) endColumn = firstLineStringList.size(); // last column else //number of vector names provided in the import dialog (not more than the maximal number of columns in the file) endColumn = qMin(vectorNames.size(), firstLineStringList.size()); } if (endColumn < startColumn) m_actualCols = 0; else m_actualCols = endColumn - startColumn + 1; if (createIndexEnabled) { vectorNames.prepend(i18n("Index")); m_actualCols++; } //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ ///////////////////////////////////////////////////////////////// // parse first data line to determine data type for each column // if the first line was already parsed as the header, read the next line if (headerEnabled && !device.isSequential()) firstLineStringList = getLineString(device); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Integer; col = 1; } for (auto& valueString : firstLineStringList) { // parse columns available in first data line if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } // parsing more lines to better determine data types for (unsigned int i = 0; i < m_dataTypeLines; ++i) { if (device.atEnd()) // EOF reached break; firstLineStringList = getLineString(device); createIndexEnabled ? col = 1 : col = 0; for (auto& valueString : firstLineStringList) { if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); if (col == m_actualCols) break; AbstractColumn::ColumnMode mode = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); // numeric: integer -> numeric if (mode == AbstractColumn::Numeric && columnModes[col] == AbstractColumn::Integer) columnModes[col] = mode; // text: non text -> text if (mode == AbstractColumn::Text && columnModes[col] != AbstractColumn::Text) columnModes[col] = mode; col++; } } QDEBUG("column modes = " << columnModes); // ATTENTION: This resets the position in the device to 0 m_actualRows = (int)q->lineNumber(device); const int actualEndRow = (endRow == -1 || endRow > m_actualRows) ? m_actualRows : endRow; if (actualEndRow > m_actualStartRow) m_actualRows = actualEndRow - m_actualStartRow + 1; else m_actualRows = 0; DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << m_actualStartRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0 && !device.isSequential()) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode)); //dirty hack: set readingFile and readingFileName in order to know in lineNumber(QIODevice) //that we're reading from a file and to benefit from much faster wc on linux //TODO: redesign the APIs and remove this later readingFile = true; readingFileName = fileName; KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode); readingFile = false; } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from) { DEBUG("AsciiFilterPrivate::readFromLiveDevice(): bytes available = " << device.bytesAvailable() << ", from = " << from); if (device.bytesAvailable() <= 0) { DEBUG(" No new data available"); return 0; } //TODO: may be also a matrix? auto* spreadsheet = dynamic_cast(dataSource); if (spreadsheet->sourceType() != LiveDataSource::SourceType::FileOrPipe) if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return 0; if (!m_prepared) { DEBUG(" Preparing .."); switch (spreadsheet->sourceType()) { case LiveDataSource::SourceType::FileOrPipe: { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG(" Device error = " << deviceError); return 0; } break; } case LiveDataSource::SourceType::NetworkTcpSocket: case LiveDataSource::SourceType::NetworkUdpSocket: case LiveDataSource::SourceType::LocalSocket: case LiveDataSource::SourceType::SerialPort: m_actualRows = 1; if (createIndexEnabled) { m_actualCols = 2; columnModes << AbstractColumn::Integer << AbstractColumn::Numeric; vectorNames << i18n("Index") << i18n("Value"); } else { m_actualCols = 1; columnModes << AbstractColumn::Numeric; vectorNames << i18n("Value"); } QDEBUG(" vector names = " << vectorNames); case LiveDataSource::SourceType::MQTT: break; } // prepare import for spreadsheet spreadsheet->setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); DEBUG(" data source resized to col: " << m_actualCols); DEBUG(" data source rowCount: " << spreadsheet->rowCount()); //columns in a file data source don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } int keepNValues = spreadsheet->keepNValues(); if (keepNValues == 0) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(keepNValues); m_actualRows = keepNValues; } m_dataContainer.resize(m_actualCols); DEBUG(" Setting data .."); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } DEBUG(" Prepared!"); } qint64 bytesread = 0; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif LiveDataSource::ReadingType readingType; if (!m_prepared) { readingType = LiveDataSource::ReadingType::TillEnd; } else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = LiveDataSource::ReadingType::TillEnd; //if we read the whole file we just start from the beginning of it //and read till end else if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) readingType = LiveDataSource::ReadingType::TillEnd; else readingType = spreadsheet->readingType(); } DEBUG(" Reading type = " << ENUM_TO_STRING(LiveDataSource, ReadingType, readingType)); //move to the last read position, from == total bytes read //since the other source types are sequential we cannot seek on them if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleSizeNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != LiveDataSource::ReadingType::TillEnd) newData.resize(spreadsheet->sampleSize()); int newDataIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif DEBUG(" source type = " << ENUM_TO_STRING(LiveDataSource, SourceType, spreadsheet->sourceType())); while (!device.atEnd()) { if (readingType != LiveDataSource::ReadingType::TillEnd) { switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData[newDataIdx++] = device.readAll(); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData[newDataIdx++] = device.read(device.bytesAvailable()); break; case LiveDataSource::SourceType::FileOrPipe: newData.push_back(device.readLine()); break; case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: newData[newDataIdx++] = device.read(device.bytesAvailable()); case LiveDataSource::SourceType::MQTT: break; } } else { // ReadingType::TillEnd switch (spreadsheet->sourceType()) { // different sources need different read methods case LiveDataSource::SourceType::LocalSocket: newData.push_back(device.readAll()); break; case LiveDataSource::SourceType::NetworkUdpSocket: newData.push_back(device.read(device.bytesAvailable())); break; case LiveDataSource::SourceType::FileOrPipe: newData.push_back(device.readLine()); break; case LiveDataSource::SourceType::NetworkTcpSocket: //TODO: check serial port case LiveDataSource::SourceType::SerialPort: newData.push_back(device.read(device.bytesAvailable())); case LiveDataSource::SourceType::MQTT: break; } } newLinesTillEnd++; if (readingType != LiveDataSource::ReadingType::TillEnd) { newLinesForSampleSizeNotTillEnd++; //for Continuous reading and FromEnd we read sample rate number of lines if possible //here TillEnd and Whole file behave the same if (newLinesForSampleSizeNotTillEnd == spreadsheet->sampleSize()) break; } } QDEBUG(" data read: " << newData); } //now we reset the readingType if (spreadsheet->readingType() == LiveDataSource::ReadingType::FromEnd) readingType = spreadsheet->readingType(); //we had less new lines than the sample size specified if (readingType != LiveDataSource::ReadingType::TillEnd) QDEBUG(" Removed empty lines: " << newData.removeAll(QString())); //back to the last read position before counting when reading from files if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) device.seek(from); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); int currentRow = 0; // indexes the position in the vector(column) int linesToRead = 0; int keepNValues = spreadsheet->keepNValues(); DEBUG(" Increase row count. keepNValues = " << keepNValues); if (m_prepared) { //increase row count if we don't have a fixed size //but only after the preparation step if (keepNValues == 0) { DEBUG(" keep All values"); if (readingType != LiveDataSource::ReadingType::TillEnd) m_actualRows += qMin(newData.size(), spreadsheet->sampleSize()); else { //we don't increase it if we reread the whole file, we reset it if (!(spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile)) m_actualRows += newData.size(); else m_actualRows = newData.size(); } //appending if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) linesToRead = m_actualRows; else linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; } else { // fixed size DEBUG(" keep " << keepNValues << " values"); if (readingType == LiveDataSource::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; //TODO after reading we should skip the next data lines //because it's TillEnd actually } else linesToRead = newLinesTillEnd; } else { //we read max sample size number of lines when the reading mode //is ContinuouslyFixed or FromEnd, WholeFile is disabled linesToRead = qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } DEBUG(" actual rows = " << m_actualRows); if (linesToRead == 0) return 0; } else { // not prepared linesToRead = newLinesTillEnd; if (headerEnabled) --m_actualRows; } DEBUG(" lines to read = " << linesToRead); if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe || spreadsheet->sourceType() == LiveDataSource::SourceType::NetworkUdpSocket) { if (m_actualRows < linesToRead) { DEBUG(" SET lines to read to " << m_actualRows); linesToRead = m_actualRows; } } //new rows/resize columns if we don't have a fixed size //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize if (keepNValues == 0) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { // fixed size //when we have a fixed size we have to pop sampleSize number of lines if specified //here popping, setting currentRow if (!m_prepared) { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); } else { if (readingType == LiveDataSource::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) { currentRow = 0; } else { if (spreadsheet->readingType() == LiveDataSource::ReadingType::WholeFile) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } } else { //we read max sample size number of lines when the reading mode //is ContinuouslyFixed or FromEnd currentRow = m_actualRows - qMin(spreadsheet->sampleSize(), newLinesTillEnd); } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif // enable data change signal for (int col = 0; col < m_actualCols; ++col) spreadsheet->child(col)->setSuppressDataChangedSignal(false); for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { - QVector* vector = static_cast* >(spreadsheet->child(col)->data()); + QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet DEBUG(" Reading from line " << currentRow << " till end line " << newLinesTillEnd); DEBUG(" Lines to read:" << linesToRead <<", actual rows:" << m_actualRows << ", actual cols:" << m_actualCols); newDataIdx = 0; if (readingType == LiveDataSource::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->sampleSize()) newDataIdx = newData.size() - spreadsheet->sampleSize(); //since we skip a couple of lines, we need to count those bytes too for (int i = 0; i < newDataIdx; ++i) bytesread += newData.at(i).size(); } } DEBUG(" newDataIdx: " << newDataIdx); static int indexColumnIdx = 1; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif int row = 0; if (readingType == LiveDataSource::ReadingType::TillEnd || (readingType == LiveDataSource::ReadingType::ContinuousFixed)) { if (headerEnabled) { if (!m_prepared) { row = 1; bytesread += newData.at(0).size(); } } } if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { if (readingType == LiveDataSource::ReadingType::WholeFile) { if (headerEnabled) { row = 1; bytesread += newData.at(0).size(); } } } QLocale locale(numberFormat); for (; row < linesToRead; ++row) { DEBUG("\n Reading row " << row + 1 << " of " << linesToRead); QString line; if (readingType == LiveDataSource::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(row); //when we read the whole file we don't care about the previous position //so we don't have to count those bytes if (readingType != LiveDataSource::ReadingType::WholeFile) { if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) { bytesread += line.size(); } } DEBUG(" Line bytes: " << line.size() << " line: " << line.toStdString()); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QStringList lineStringList; // only FileOrPipe support multiple columns if (spreadsheet->sourceType() == LiveDataSource::SourceType::FileOrPipe) lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); else lineStringList << line; QDEBUG(" line = " << lineStringList << ", separator = \'" << m_separator << "\'"); if (createIndexEnabled) { if (spreadsheet->keepNValues() == 0) lineStringList.prepend(QString::number(currentRow + 1)); else lineStringList.prepend(QString::number(indexColumnIdx++)); } QDEBUG(" column modes = " << columnModes); for (int n = 0; n < m_actualCols; ++n) { DEBUG(" actual col = " << n); if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); DEBUG(" value string = " << valueString.toStdString()); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { DEBUG(" Numeric"); bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::Integer: { DEBUG(" Integer"); bool isNumber; - const int value = locale.toInt(valueString, &isNumber); + const qint64 value = locale.toLongLong(valueString, &isNumber); DEBUG(" container size = " << m_dataContainer.size() << ", current row = " << currentRow); - static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); - qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); + static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); + qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { DEBUG(" missing columns in this line"); switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: - static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; + static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow).clear(); break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); const Project* project = spreadsheet->project(); QVector curves = project->children(AbstractAspect::Recursive); QVector plots; for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); //determine the plots where the column is consumed for (const auto* curve : curves) { if (curve->xColumn() == column || curve->yColumn() == column) { auto* plot = dynamic_cast(curve->parentAspect()); if (plots.indexOf(plot) == -1) { plots << plot; plot->setSuppressDataChangedSignal(true); } } } column->setChanged(); } //loop over all affected plots and retransform them for (auto* plot : plots) { plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } else m_prepared = true; DEBUG("AsciiFilterPrivate::readFromLiveDevice() DONE"); return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); if (!m_prepared) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return; } // matrix data has only one column mode if (dynamic_cast(dataSource)) { auto mode = columnModes[0]; //TODO: remove this when Matrix supports text type if (mode == AbstractColumn::Text) mode = AbstractColumn::Numeric; for (auto& c : columnModes) if (c != mode) c = mode; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; //skip data lines, if required DEBUG(" Skipping " << m_actualStartRow << " lines"); for (int i = 0; i < m_actualStartRow; ++i) device.readLine(); DEBUG(" Reading " << qMin(lines, m_actualRows) << " lines, " << m_actualCols << " columns"); if (qMin(lines, m_actualRows) == 0 || m_actualCols == 0) return; for (int i = 0; i < qMin(lines, m_actualRows); ++i) { QString line = device.readLine(); // remove any newline line.remove(QLatin1Char('\n')); line.remove(QLatin1Char('\r')); if (simplifyWhitespacesEnabled) line = line.simplified(); if (removeQuotesEnabled) line.remove(QLatin1Char('"')); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); // remove left white spaces if (skipEmptyParts) { for (int n = 0; n < lineStringList.size(); ++n) { QString valueString = lineStringList.at(n); if (!QString::compare(valueString, " ")) { lineStringList.removeAt(n); n--; } } } for (int n = 0; n < m_actualCols; ++n) { // index column if required if (n == 0 && createIndexEnabled) { - static_cast*>(m_dataContainer[0])->operator[](currentRow) = i + 1; + static_cast*>(m_dataContainer[0])->operator[](currentRow) = i + 1; continue; } //column counting starts with 1, substract 1 as well as another 1 for the index column if required int col = createIndexEnabled ? n + startColumn - 2: n + startColumn - 1; if (col < lineStringList.size()) { QString valueString = lineStringList.at(col); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; - const int value = locale.toInt(valueString, &isNumber); - static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); + const qint64 value = locale.toLongLong(valueString, &isNumber); + static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: { QVector* colData = static_cast*>(m_dataContainer[n]); colData->operator[](currentRow) = valueString; break; } case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: - static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; + static_cast*>(m_dataContainer[n])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow).clear(); break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } DEBUG(" Read " << currentRow << " lines"); dataSource->finalizeImport(m_columnOffset, startColumn, startColumn + m_actualCols - 1, currentRow, dateTimeFormat, importMode); } /*! * preview for special devices (local/UDP/TCP socket or serial port) */ QVector AsciiFilterPrivate::preview(QIODevice &device) { DEBUG("AsciiFilterPrivate::preview(): bytesAvailable = " << device.bytesAvailable() << ", isSequential = " << device.isSequential()); QVector dataStrings; if (!(device.bytesAvailable() > 0)) { DEBUG("No new data available"); return dataStrings; } if (device.isSequential() && device.bytesAvailable() < (int)sizeof(quint16)) return dataStrings; #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif int linesToRead = 0; QVector newData; //TODO: serial port "read(nBytes)"? while (!device.atEnd()) { if (device.canReadLine()) newData.push_back(device.readLine()); else // UDP fails otherwise newData.push_back(device.readAll()); linesToRead++; } QDEBUG(" data = " << newData); if (linesToRead == 0) return dataStrings; int col = 0; int colMax = newData.at(0).size(); if (createIndexEnabled) colMax++; columnModes.resize(colMax); if (createIndexEnabled) { columnModes[0] = AbstractColumn::ColumnMode::Integer; col = 1; vectorNames.prepend(i18n("Index")); } vectorNames.append(i18n("Value")); QDEBUG(" vector names = " << vectorNames); for (const auto& valueString : newData.at(0).split(' ', QString::SkipEmptyParts)) { if (col == colMax) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } for (int i = 0; i < linesToRead; ++i) { QString line = newData.at(i); // remove any newline line = line.remove('\n'); line = line.remove('\r'); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(' ', QString::SkipEmptyParts); if (createIndexEnabled) lineStringList.prepend(QString::number(i + 1)); QStringList lineString; for (int n = 0; n < lineStringList.size(); ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 16); break; } case AbstractColumn::Integer: { bool isNumber; - const int value = locale.toInt(valueString, &isNumber); + const qint64 value = locale.toLongLong(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QString(); } dataStrings << lineString; } return dataStrings; } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; //dirty hack: set readingFile and readingFileName in order to know in lineNumber(QIODevice) //that we're reading from a file and to benefit from much faster wc on linux //TODO: redesign the APIs and remove this later readingFile = true; readingFileName = fileName; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); readingFile = false; if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; // set column names for preview if (!headerEnabled) { int start = 0; if (createIndexEnabled) start = 1; for (int i = start; i < m_actualCols; i++) vectorNames << "Column " + QString::number(i + 1); } QDEBUG(" column names = " << vectorNames); //skip data lines, if required DEBUG(" Skipping " << m_actualStartRow << " lines"); for (int i = 0; i < m_actualStartRow; ++i) device.readLine(); DEBUG(" Generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); ++i) { QString line = device.readLine(); // remove any newline line = line.remove('\n'); line = line.remove('\r'); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; const QStringList& lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); QDEBUG(" line = " << lineStringList); QStringList lineString; for (int n = 0; n < m_actualCols; ++n) { // index column if required if (n == 0 && createIndexEnabled) { lineString += QString::number(i + 1); continue; } //column counting starts with 1, substract 1 as well as another 1 for the index column if required int col = createIndexEnabled ? n + startColumn - 2: n + startColumn - 1; if (col < lineStringList.size()) { QString valueString = lineStringList.at(col); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); //DEBUG(" valueString = " << valueString.toStdString()); if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces continue; // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 15); break; } case AbstractColumn::Integer: { bool isNumber; - const int value = locale.toInt(valueString, &isNumber); + const qint64 value = locale.toLongLong(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QString(); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to ascii file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement( "asciiFilter"); writer->writeAttribute( "commentCharacter", d->commentCharacter); writer->writeAttribute( "separatingCharacter", d->separatingCharacter); writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute( "createTimestamp", QString::number(d->createTimestampEnabled)); writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); writer->writeAttribute( "nanValue", QString::number(d->nanValue)); writer->writeAttribute( "removeQuotes", QString::number(d->removeQuotesEnabled)); writer->writeAttribute( "startRow", QString::number(d->startRow)); writer->writeAttribute( "endRow", QString::number(d->endRow)); writer->writeAttribute( "startColumn", QString::number(d->startColumn)); writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str; READ_STRING_VALUE("commentCharacter", commentCharacter); READ_STRING_VALUE("separatingCharacter", separatingCharacter); READ_INT_VALUE("createIndex", createIndexEnabled, bool); READ_INT_VALUE("createTimestamp", createTimestampEnabled, bool); READ_INT_VALUE("autoMode", autoModeEnabled, bool); READ_INT_VALUE("header", headerEnabled, bool); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty READ_INT_VALUE("simplifyWhitespaces", simplifyWhitespacesEnabled, bool); READ_DOUBLE_VALUE("nanValue", nanValue); READ_INT_VALUE("removeQuotes", removeQuotesEnabled, bool); READ_INT_VALUE("skipEmptyParts", skipEmptyParts, bool); READ_INT_VALUE("startRow", startRow, int); READ_INT_VALUE("endRow", endRow, int); READ_INT_VALUE("startColumn", startColumn, int); READ_INT_VALUE("endColumn", endColumn, int); return true; } int AsciiFilterPrivate::isPrepared() { return m_prepared; } #ifdef HAVE_MQTT int AsciiFilterPrivate::prepareToRead(const QString& message) { QStringList lines = message.split('\n'); if (lines.isEmpty()) return 1; // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine = lines.at(0); if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, (QString::SplitBehavior)skipEmptyParts); } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("2xTAB"), "\t\t", Qt::CaseInsensitive); m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("2xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("3xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("4xSPACE"), QLatin1String(" "), Qt::CaseInsensitive); m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, (QString::SplitBehavior)skipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); QDEBUG("first line: " << firstLineStringList); //all columns are read plus the optional column for the index and for the timestamp m_actualCols = firstLineStringList.size() + int(createIndexEnabled) + int(createTimestampEnabled); //column names: //when reading the message strings for different topics, it's not possible to specify vector names //since the different topics can have different content and different number of columns/vectors //->we always set the vector names here to fixed values vectorNames.clear(); columnModes.clear(); //add index column if (createIndexEnabled) { vectorNames << i18n("index"); columnModes << AbstractColumn::Integer; } //add timestamp column if (createTimestampEnabled) { vectorNames << i18n("timestamp"); columnModes << AbstractColumn::DateTime; } //parse the first data line to determine data type for each column int i = 1; for (auto& valueString : firstLineStringList) { if (simplifyWhitespacesEnabled) valueString = valueString.simplified(); if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); vectorNames << i18n("value %1").arg(i); columnModes << AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); ++i; } m_actualStartRow = startRow; m_actualRows = lines.size(); QDEBUG("column modes = " << columnModes); DEBUG("actual cols/rows (w/o header): " << m_actualCols << ' ' << m_actualRows); return 0; } /*! * generates the preview for the string \s message. */ QVector AsciiFilterPrivate::preview(const QString& message) { QVector dataStrings; prepareToRead(message); //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data QStringList lines = message.split('\n'); int i = 0; for (auto line : lines) { if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; const QStringList& lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); QDEBUG(" line = " << lineStringList); QStringList lineString; // index column if required if (createIndexEnabled) lineString += QString::number(i + 1); // timestamp column if required if (createTimestampEnabled) lineString += QDateTime::currentDateTime().toString(); int offset = int(createIndexEnabled) + int(createTimestampEnabled); for (int n = 0; n < m_actualCols - offset; ++n) { if (n < lineStringList.size()) { QString valueString = lineStringList.at(n); //DEBUG(" valueString = " << valueString.toStdString()); if (skipEmptyParts && !QString::compare(valueString, " ")) // handle left white spaces continue; // set value depending on data type switch (columnModes[n+offset]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : nanValue, 'g', 15); break; } case AbstractColumn::Integer: { bool isNumber; - const int value = locale.toInt(valueString, &isNumber); + const qint64 value = locale.toLongLong(valueString, &isNumber); lineString += QString::number(isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QString(); } ++i; dataStrings << lineString; } return dataStrings; } /*! * \brief Returns the statistical data that is needed by the topic for its MQTTClient's will message * \param topic */ QString AsciiFilterPrivate::MQTTColumnStatistics(const MQTTTopic* topic) const { Column* const tempColumn = topic->child(m_actualCols - 1); QString statistics; QVector willStatistics = topic->mqttClient()->willStatistics(); //Add every statistical data to the string, the flag of which is set true for (int i = 0; i <= willStatistics.size(); i++) { if (willStatistics[i]) { switch (static_cast(i) ) { case MQTTClient::WillStatisticsType::ArithmeticMean: statistics += QLatin1String("Arithmetic mean: ") + QString::number(tempColumn->statistics().arithmeticMean) + "\n"; break; case MQTTClient::WillStatisticsType::ContraharmonicMean: statistics += QLatin1String("Contraharmonic mean: ") + QString::number(tempColumn->statistics().contraharmonicMean) + "\n"; break; case MQTTClient::WillStatisticsType::Entropy: statistics += QLatin1String("Entropy: ") + QString::number(tempColumn->statistics().entropy) + "\n"; break; case MQTTClient::WillStatisticsType::GeometricMean: statistics += QLatin1String("Geometric mean: ") + QString::number(tempColumn->statistics().geometricMean) + "\n"; break; case MQTTClient::WillStatisticsType::HarmonicMean: statistics += QLatin1String("Harmonic mean: ") + QString::number(tempColumn->statistics().harmonicMean) + "\n"; break; case MQTTClient::WillStatisticsType::Kurtosis: statistics += QLatin1String("Kurtosis: ") + QString::number(tempColumn->statistics().kurtosis) + "\n"; break; case MQTTClient::WillStatisticsType::Maximum: statistics += QLatin1String("Maximum: ") + QString::number(tempColumn->statistics().maximum) + "\n"; break; case MQTTClient::WillStatisticsType::MeanDeviation: statistics += QLatin1String("Mean deviation: ") + QString::number(tempColumn->statistics().meanDeviation) + "\n"; break; case MQTTClient::WillStatisticsType::MeanDeviationAroundMedian: statistics += QLatin1String("Mean deviation around median: ") + QString::number(tempColumn->statistics().meanDeviationAroundMedian) + "\n"; break; case MQTTClient::WillStatisticsType::Median: statistics += QLatin1String("Median: ") + QString::number(tempColumn->statistics().median) + "\n"; break; case MQTTClient::WillStatisticsType::MedianDeviation: statistics += QLatin1String("Median deviation: ") + QString::number(tempColumn->statistics().medianDeviation) + "\n"; break; case MQTTClient::WillStatisticsType::Minimum: statistics += QLatin1String("Minimum: ") + QString::number(tempColumn->statistics().minimum) + "\n"; break; case MQTTClient::WillStatisticsType::Skewness: statistics += QLatin1String("Skewness: ") + QString::number(tempColumn->statistics().skewness) + "\n"; break; case MQTTClient::WillStatisticsType::StandardDeviation: statistics += QLatin1String("Standard deviation: ") + QString::number(tempColumn->statistics().standardDeviation) + "\n"; break; case MQTTClient::WillStatisticsType::Variance: statistics += QLatin1String("Variance: ") + QString::number(tempColumn->statistics().variance) + "\n"; break; case MQTTClient::WillStatisticsType::NoStatistics: default: break; } } } return statistics; } AbstractColumn::ColumnMode AsciiFilterPrivate::MQTTColumnMode() const { return columnModes[m_actualCols - 1]; } /*! * \brief reads the content of a message received by the topic. * Uses the settings defined in the MQTTTopic's MQTTClient * \param message * \param topic * \param dataSource */ void AsciiFilterPrivate::readMQTTTopic(const QString& message, AbstractDataSource* dataSource) { //If the message is empty, there is nothing to do if (message.isEmpty()) { DEBUG("No new data available"); return; } MQTTTopic* spreadsheet = dynamic_cast(dataSource); const int keepNValues = spreadsheet->mqttClient()->keepNValues(); if (!m_prepared) { qDebug()<<"Start preparing filter for: " << spreadsheet->topicName(); //Prepare the filter const int mqttPrepareError = prepareToRead(message); if (mqttPrepareError != 0) { DEBUG("Mqtt Prepare Error = " << mqttPrepareError); qDebug()<setUndoAware(false); spreadsheet->resize(AbstractFileFilter::Replace, vectorNames, m_actualCols); qDebug() << "fds resized to col: " << m_actualCols; qDebug() << "fds rowCount: " << spreadsheet->rowCount(); //columns in a MQTTTopic don't have any manual changes. //make the available columns undo unaware and suppress the "data changed" signal. //data changes will be propagated via an explicit Column::setChanged() call once new data was read. for (int i = 0; i < spreadsheet->childCount(); i++) { spreadsheet->child(i)->setUndoAware(false); spreadsheet->child(i)->setSuppressDataChangedSignal(true); } if (keepNValues == 0) spreadsheet->setRowCount(m_actualRows > 1 ? m_actualRows : 1); else { spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues()); m_actualRows = spreadsheet->mqttClient()->keepNValues(); } m_dataContainer.resize(m_actualCols); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportTotal: "); #endif MQTTClient::ReadingType readingType; if (!m_prepared) { //if filter is not prepared we read till the end readingType = MQTTClient::ReadingType::TillEnd; } else { //we have to read all the data when reading from end //so we set readingType to TillEnd if (static_cast (spreadsheet->mqttClient()->readingType()) == MQTTClient::ReadingType::FromEnd) readingType = MQTTClient::ReadingType::TillEnd; else readingType = spreadsheet->mqttClient()->readingType(); } //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleSizeNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (readingType != MQTTClient::ReadingType::TillEnd) { newData.reserve(spreadsheet->mqttClient()->sampleSize()); newData.resize(spreadsheet->mqttClient()->sampleSize()); } int newDataIdx = 0; //TODO: bool sampleSizeReached = false; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportReadingFromFile: "); #endif QStringList newDataList = message.split(QRegExp("\n|\r\n|\r"), QString::SkipEmptyParts); for (auto& line : newDataList) { newData.push_back(line); newLinesTillEnd++; if (readingType != MQTTClient::ReadingType::TillEnd) { newLinesForSampleSizeNotTillEnd++; //for Continuous reading and FromEnd we read sample rate number of lines if possible if (newLinesForSampleSizeNotTillEnd == spreadsheet->mqttClient()->sampleSize()) { //TODO: sampleSizeReached = true; break; } } } } qDebug()<<"Processing message done"; //now we reset the readingType if (spreadsheet->mqttClient()->readingType() == MQTTClient::ReadingType::FromEnd) readingType = static_cast(spreadsheet->mqttClient()->readingType()); //we had less new lines than the sample rate specified if (readingType != MQTTClient::ReadingType::TillEnd) qDebug() << "Removed empty lines: " << newData.removeAll(QString()); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); if (m_prepared ) { if (keepNValues == 0) m_actualRows = spreadsheetRowCountBeforeResize; else { //if the keepNValues changed since the last read we have to manage the columns accordingly if (m_actualRows != spreadsheet->mqttClient()->keepNValues()) { if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues()); qDebug()<<"rowcount set to: " << spreadsheet->mqttClient()->keepNValues(); } //Calculate the difference between the old and new keepNValues int rowDiff = 0; if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) rowDiff = m_actualRows - spreadsheet->mqttClient()->keepNValues(); if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) rowDiff = spreadsheet->mqttClient()->keepNValues() - m_actualRows; for (int n = 0; n < columnModes.size(); ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); m_dataContainer[n] = static_cast(vector); //if the keepNValues got smaller then we move the last keepNValues count of data //in the first keepNValues places if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) { for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) { static_cast*>(m_dataContainer[n])->operator[] (i) = static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); } } //if the keepNValues got bigger we move the existing values to the last m_actualRows positions //then fill the remaining lines with NaN if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { vector->reserve( spreadsheet->mqttClient()->keepNValues()); vector->resize( spreadsheet->mqttClient()->keepNValues()); for (int i = 1; i <= m_actualRows; i++) { static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); } for (int i = 0; i < rowDiff; i++) static_cast*>(m_dataContainer[n])->operator[](i) = nanValue; } break; } case AbstractColumn::Integer: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); m_dataContainer[n] = static_cast(vector); //if the keepNValues got smaller then we move the last keepNValues count of data //in the first keepNValues places if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) { for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) { - static_cast*>(m_dataContainer[n])->operator[] (i) = - static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); + static_cast*>(m_dataContainer[n])->operator[] (i) = + static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); } } //if the keepNValues got bigger we move the existing values to the last m_actualRows positions //then fill the remaining lines with 0 if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { vector->reserve( spreadsheet->mqttClient()->keepNValues()); vector->resize( spreadsheet->mqttClient()->keepNValues()); for (int i = 1; i <= m_actualRows; i++) { - static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = - static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); + static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = + static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); } for (int i = 0; i < rowDiff; i++) - static_cast*>(m_dataContainer[n])->operator[](i) = 0; + static_cast*>(m_dataContainer[n])->operator[](i) = 0; } break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); m_dataContainer[n] = static_cast(vector); //if the keepNValues got smaller then we move the last keepNValues count of data //in the first keepNValues places if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) { for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) { static_cast*>(m_dataContainer[n])->operator[] (i) = static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); } } //if the keepNValues got bigger we move the existing values to the last m_actualRows positions //then fill the remaining lines with empty lines if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { vector->reserve( spreadsheet->mqttClient()->keepNValues()); vector->resize( spreadsheet->mqttClient()->keepNValues()); for (int i = 1; i <= m_actualRows; i++) { static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); } for (int i = 0; i < rowDiff; i++) static_cast*>(m_dataContainer[n])->operator[](i).clear(); } break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); m_dataContainer[n] = static_cast(vector); //if the keepNValues got smaller then we move the last keepNValues count of data //in the first keepNValues places if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) { for (int i = 0; i < spreadsheet->mqttClient()->keepNValues(); i++) { static_cast*>(m_dataContainer[n])->operator[] (i) = static_cast*>(m_dataContainer[n])->operator[](m_actualRows - spreadsheet->mqttClient()->keepNValues() + i); } } //if the keepNValues got bigger we move the existing values to the last m_actualRows positions //then fill the remaining lines with null datetime if (m_actualRows < spreadsheet->mqttClient()->keepNValues()) { vector->reserve( spreadsheet->mqttClient()->keepNValues()); vector->resize( spreadsheet->mqttClient()->keepNValues()); for (int i = 1; i <= m_actualRows; i++) { static_cast*>(m_dataContainer[n])->operator[] (spreadsheet->mqttClient()->keepNValues() - i) = static_cast*>(m_dataContainer[n])->operator[](spreadsheet->mqttClient()->keepNValues() - i - rowDiff); } for (int i = 0; i < rowDiff; i++) static_cast*>(m_dataContainer[n])->operator[](i) = QDateTime(); } break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } //if the keepNValues got smaller resize the spreadsheet if (m_actualRows > spreadsheet->mqttClient()->keepNValues()) spreadsheet->setRowCount(spreadsheet->mqttClient()->keepNValues()); //set the new row count m_actualRows = spreadsheet->mqttClient()->keepNValues(); qDebug()<<"actual rows: "<mqttClient()->sampleSize()); else { m_actualRows += newData.size(); } } //fixed size if (keepNValues != 0) { if (readingType == MQTTClient::ReadingType::TillEnd) { //we had more lines than the fixed size, so we read m_actualRows number of lines if (newLinesTillEnd > m_actualRows) { linesToRead = m_actualRows; } else linesToRead = newLinesTillEnd; } else { //we read max sample size number of lines when the reading mode //is ContinuouslyFixed or FromEnd if (spreadsheet->mqttClient()->sampleSize() <= spreadsheet->mqttClient()->keepNValues()) linesToRead = qMin(spreadsheet->mqttClient()->sampleSize(), newLinesTillEnd); else linesToRead = qMin(spreadsheet->mqttClient()->keepNValues(), newLinesTillEnd); } } else linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; if (linesToRead == 0) return; } else { if (keepNValues != 0) linesToRead = newLinesTillEnd > m_actualRows ? m_actualRows : newLinesTillEnd; else linesToRead = newLinesTillEnd; } qDebug()<<"linestoread = " << linesToRead; //new rows/resize columns if we don't have a fixed size if (keepNValues == 0) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportResizing: "); #endif if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); if (!m_prepared) currentRow = 0; else { // indexes the position in the vector(column) currentRow = spreadsheetRowCountBeforeResize; } // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { //when we have a fixed size we have to pop sampleSize number of lines if specified //here popping, setting currentRow if (!m_prepared) currentRow = m_actualRows - qMin(newLinesTillEnd, m_actualRows); else { if (readingType == MQTTClient::ReadingType::TillEnd) { if (newLinesTillEnd > m_actualRows) currentRow = 0; else currentRow = m_actualRows - newLinesTillEnd; } else { //we read max sample rate number of lines when the reading mode //is ContinuouslyFixed or FromEnd currentRow = m_actualRows - linesToRead; } } if (m_prepared) { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportPopping: "); #endif for (int row = 0; row < linesToRead; ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Integer: { - QVector* vector = static_cast* >(spreadsheet->child(col)->data()); + QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } } // from the last row we read the new data in the spreadsheet qDebug() << "reading from line: " << currentRow << " lines till end: " << newLinesTillEnd; qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows; newDataIdx = 0; //From end means that we read the last sample size amount of data if (readingType == MQTTClient::ReadingType::FromEnd) { if (m_prepared) { if (newData.size() > spreadsheet->mqttClient()->sampleSize()) newDataIdx = newData.size() - spreadsheet->mqttClient()->sampleSize(); } } qDebug() << "newDataIdx: " << newDataIdx; //read the data static int indexColumnIdx = 0; { #ifdef PERFTRACE_LIVE_IMPORT PERFTRACE("AsciiLiveDataImportFillingContainers: "); #endif int row = 0; QLocale locale(numberFormat); for (; row < linesToRead; ++row) { QString line; if (readingType == MQTTClient::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(row); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) continue; //add index if required int offset = 0; if (createIndexEnabled) { int index = (keepNValues != 0) ? indexColumnIdx++ : currentRow; static_cast*>(m_dataContainer[0])->operator[](currentRow) = index; ++offset; } //add current timestamp if required if (createTimestampEnabled) { static_cast*>(m_dataContainer[offset])->operator[](currentRow) = QDateTime::currentDateTime(); ++offset; } //parse the columns QStringList lineStringList = line.split(m_separator, (QString::SplitBehavior)skipEmptyParts); qDebug()<<"########################################################################"; qDebug()<*>(m_dataContainer[col])->operator[](currentRow) = (isNumber ? value : nanValue); break; } case AbstractColumn::Integer: { bool isNumber; - const int value = locale.toInt(valueString, &isNumber); - static_cast*>(m_dataContainer[col])->operator[](currentRow) = (isNumber ? value : 0); + const qint64 value = locale.toLongLong(valueString, &isNumber); + static_cast*>(m_dataContainer[col])->operator[](currentRow) = (isNumber ? value : 0); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[col])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: if (removeQuotesEnabled) valueString.remove(QLatin1Char('"')); static_cast*>(m_dataContainer[col])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { DEBUG(" missing columns in this line"); switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[col])->operator[](currentRow) = nanValue; break; case AbstractColumn::Integer: - static_cast*>(m_dataContainer[col])->operator[](currentRow) = 0; + static_cast*>(m_dataContainer[col])->operator[](currentRow) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[col])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[col])->operator[](currentRow).clear(); break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } } if (m_prepared) { //notify all affected columns and plots about the changes PERFTRACE("AsciiLiveDataImport, notify affected columns and plots"); const Project* project = spreadsheet->project(); QVector curves = project->children(AbstractAspect::Recursive); QVector plots; for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); //determine the plots where the column is consumed for (const auto* curve : curves) { if (curve->xColumn() == column || curve->yColumn() == column) { CartesianPlot* plot = dynamic_cast(curve->parentAspect()); if (plots.indexOf(plot) == -1) { plots << plot; plot->setSuppressDataChangedSignal(true); } } } column->setChanged(); } //loop over all affected plots and retransform them for (auto* const plot : plots) { //TODO setting this back to true triggers again a lot of retransforms in the plot (one for each curve). // plot->setSuppressDataChangedSignal(false); plot->dataChanged(); } } else m_prepared = true; DEBUG("AsciiFilterPrivate::readFromMQTTTopic() DONE"); } /*! * \brief After the MQTTTopic was loaded, the filter is prepared for reading * \param prepared * \param topic * \param separator */ void AsciiFilterPrivate::setPreparedForMQTT(bool prepared, MQTTTopic* topic, const QString& separator) { m_prepared = prepared; //If originally it was prepared we have to restore the settings if (prepared) { m_separator = separator; m_actualCols = endColumn - startColumn + 1; m_actualRows = topic->rowCount(); //set the column modes columnModes.resize(topic->columnCount()); for (int i = 0; i < topic->columnCount(); ++i) { columnModes[i] = topic->column(i)->columnMode(); } //set the data containers m_dataContainer.resize(m_actualCols); for (int n = 0; n < m_actualCols; ++n) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) topic->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(topic->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { - QVector* vector = static_cast* >(topic->child(n)->data()); + QVector* vector = static_cast* >(topic->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(topic->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(topic->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } #endif /*! * \brief Returns the separator used by the filter * \return */ QString AsciiFilterPrivate::separator() const { return m_separator; }