diff --git a/src/backend/datasources/filters/JsonFilter.cpp b/src/backend/datasources/filters/JsonFilter.cpp index cb093a583..1b63e1464 100644 --- a/src/backend/datasources/filters/JsonFilter.cpp +++ b/src/backend/datasources/filters/JsonFilter.cpp @@ -1,811 +1,840 @@ +/*************************************************************************** + File : JsonFilter.cpp + Project : LabPlot + Description : JSON I/O-filter. + -------------------------------------------------------------------- + -------------------------------------------------------------------- + Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) + + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software * + * Foundation, Inc., 51 Franklin Street, Fifth Floor, * + * Boston, MA 02110-1301 USA * + * * + ***************************************************************************/ + #include "backend/datasources/filters/JsonFilter.h" #include "backend/datasources/filters/JsonFilterPrivate.h" #include "backend/datasources/AbstractDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include #include #include #include /*! \class JsonFilter \brief Manages the import/export of data from/to a file formatted using JSON. \ingroup datasources */ JsonFilter::JsonFilter() : AbstractFileFilter(), d(new JsonFilterPrivate(this)) {} JsonFilter::~JsonFilter() {} /*! reads the content of the device \c device. */ void JsonFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } /*! reads the content of the file \c fileName. */ QVector JsonFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromFile(fileName, dataSource, importMode, lines); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector JsonFilter::preview(const QString& fileName) { return d->preview(fileName); } QVector JsonFilter::preview(QIODevice& device) { return d->preview(device); } QVector JsonFilter::preview(QJsonDocument& doc) { return d->preview(doc); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void JsonFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void JsonFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void JsonFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list of all predefined data types. */ QStringList JsonFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; ++i) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } /*! returns the list of all predefined data row types. */ QStringList JsonFilter::dataRowTypes() { return (QStringList() << "Array" << "Object"); } void JsonFilter::setDataRowType(QJsonValue::Type type) { d->rowType = type; } QJsonValue::Type JsonFilter::dataRowType() const { return d->rowType; } void JsonFilter::setModelRows(QVector rows){ d->modelRows = rows; } QVector JsonFilter::modelRows() const { return d->modelRows; } void JsonFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString JsonFilter::dateTimeFormat() const { return d->dateTimeFormat; } void JsonFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language JsonFilter::numberFormat() const { return d->numberFormat; } void JsonFilter::setNaNValueToZero(bool b) { if (b) d->nanValue = 0; else d->nanValue = NAN; } bool JsonFilter::NaNValueToZeroEnabled() const { if (d->nanValue == 0) return true; return false; } void JsonFilter::setCreateIndexEnabled(bool b){ d->createIndexEnabled = b; } void JsonFilter::setParseRowsName(bool b) { d->parseRowsName = b; } void JsonFilter::setVectorNames(const QString& s) { d->vectorNames.clear(); if (!s.simplified().isEmpty()) d->vectorNames = s.simplified().split(' '); } QStringList JsonFilter::vectorNames() const { return d->vectorNames; } QVector JsonFilter::columnModes() { return d->columnModes; } void JsonFilter::setStartRow(const int r) { d->startRow = r; } int JsonFilter::startRow() const { return d->startRow; } void JsonFilter::setEndRow(const int r) { d->endRow = r; } int JsonFilter::endRow() const { return d->endRow; } void JsonFilter::setStartColumn(const int c) { d->startColumn = c; } int JsonFilter::startColumn() const { return d->startColumn; } void JsonFilter::setEndColumn(const int c) { d->endColumn = c; } int JsonFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### JsonFilterPrivate::JsonFilterPrivate(JsonFilter* owner) : q(owner), model(new QJsonModel()), containerType(JsonFilter::Object), rowType(QJsonValue::Object), numberFormat(QLocale::C), createIndexEnabled(false), parseRowsName(false), vectorNames(), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_actualRows(0), m_actualCols(0), m_prepared(false), m_columnOffset(0) { } //TODO: delete model from memory /*! returns 1 if row is invalid and 0 otherwise. */ int JsonFilterPrivate::checkRow(QJsonValueRef value, int& countCols) { switch(rowType){ //TODO: implement other value types case QJsonValue::Array: { QJsonArray row = value.toArray(); if(row.isEmpty()) return 1; countCols = (countCols == -1 || countCols > row.count()) ? row.count() : countCols; break; } case QJsonValue::Object: { QJsonObject row = value.toObject(); if(row.isEmpty()) return 1; countCols = (countCols == -1 || countCols > row.count()) ? row.count() : countCols; break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return 1; } return 0; } /*! returns -1 if a parse error has occurred, 1 if the current row type not supported and 0 otherwise. */ int JsonFilterPrivate::parseColumnModes(QJsonValue row, QString rowName) { columnModes.resize(m_actualCols); int colIndexInContainer = startColumn - 1; for(int i = 0; i < m_actualCols; ++i){ if((createIndexEnabled || parseRowsName) && i == 0){ if(createIndexEnabled) columnModes[i] = AbstractColumn::Integer; if(parseRowsName) columnModes[i + createIndexEnabled] = AbstractFileFilter::columnMode(rowName, dateTimeFormat, numberFormat); i = i + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue columnValue; switch (rowType) { //TODO: implement other value types case QJsonValue::Array: { QJsonArray arr = row.toArray(); if(arr.count() < colIndexInContainer + 1) return -1; columnValue = *(row.toArray().begin() + colIndexInContainer); break; } case QJsonValue::Object: { QJsonObject obj = row.toObject(); if(obj.count() < colIndexInContainer + 1) return -1; if(vectorNames.count() == 0) vectorNames = row.toObject().keys(); columnValue = *(row.toObject().begin() + colIndexInContainer); break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return 1; } switch (columnValue.type()) { case QJsonValue::Double: columnModes[i] = AbstractColumn::Numeric; break; case QJsonValue::String: columnModes[i] = AbstractFileFilter::columnMode(columnValue.toString(), dateTimeFormat, numberFormat); break; case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: return -1; } colIndexInContainer++; } if(parseRowsName) vectorNames.prepend("row name"); if(createIndexEnabled) vectorNames.prepend("index"); return 0; } void JsonFilterPrivate::setEmptyValue(int column, int row){ switch (columnModes[column]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[column])->operator[](row) = nanValue; break; case AbstractColumn::Integer: static_cast*>(m_dataContainer[column])->operator[](row) = 0; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[column])->operator[](row) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[column])->operator[](row) = ""; break; case AbstractColumn::Month: case AbstractColumn::Day: break; } } void JsonFilterPrivate::setValueFromString(int column, int row, QString valueString){ QLocale locale(numberFormat); switch (columnModes[column]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[column])->operator[](row) = isNumber ? value : nanValue; break; } case AbstractColumn::Integer: { bool isNumber; const int value = locale.toInt(valueString, &isNumber); static_cast*>(m_dataContainer[column])->operator[](row) = isNumber ? value : 0; break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[column])->operator[](row) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[column])->operator[](row) = valueString; break; case AbstractColumn::Month: case AbstractColumn::Day: break; } } /*! returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end */ int JsonFilterPrivate::prepareDeviceToRead(QIODevice& device) { DEBUG("device is sequential = " << device.isSequential()); if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd() && !device.isSequential()) // empty file return 1; QJsonParseError err; QJsonDocument doc = QJsonDocument::fromJson(device.readAll(), &err); if(err.error != QJsonParseError::NoError || doc.isEmpty()) return 1; if(prepareDocumentToRead(doc) != 0) return 2; // reset to start of file if (!device.isSequential()) device.seek(0); return 0; } /*! returns 2 if a parse error has occurred and 0 otherwise. */ int JsonFilterPrivate::prepareDocumentToRead(const QJsonDocument& doc) { model->loadJson(doc); if(modelRows.isEmpty()) m_preparedDoc = doc; else { QModelIndex index; for(auto it = modelRows.begin(); it != modelRows.end(); ++it){ index = model->index(*it, 0, index); } m_preparedDoc = model->genJsonByIndex(index); } if(!m_preparedDoc.isEmpty()){ if(m_preparedDoc.isArray()) containerType = JsonFilter::Array; else if(m_preparedDoc.isObject()) containerType = JsonFilter::Object; else return 2; } else return 2; int countRows = 0; int countCols = -1; QJsonValue firstRow; QString firstRowName = ""; parseRowsName = parseRowsName && rowType == QJsonValue::Object; switch(containerType) { case JsonFilter::Array: { QJsonArray arr = m_preparedDoc.array(); if(arr.count() < startRow) return 2; int endRowOffset = (endRow == -1 || endRow > arr.count()) ? arr.count() : endRow; firstRow = *(arr.begin() + (startRow - 1)); for(QJsonArray::iterator it = arr.begin() + (startRow - 1); it != arr.begin() + endRowOffset; ++it) { if(checkRow(*it, countCols) != 0) return 2; countRows++; } break; } case JsonFilter::Object: { QJsonObject obj = m_preparedDoc.object(); if(obj.count() < startRow) return 2; int startRowOffset = startRow - 1; int endRowOffset = (endRow == -1 || endRow > obj.count()) ? obj.count() : endRow; firstRow = *(obj.begin() + startRowOffset); firstRowName = (obj.begin() + startRowOffset).key(); for(QJsonObject::iterator it = obj.begin() + startRowOffset; it != obj.begin() + endRowOffset; ++it) { if(checkRow(*it, countCols) != 0) return 2; countRows++; } break; } } if(endColumn == -1 || endColumn > countCols) endColumn = countCols; m_actualRows = countRows; m_actualCols = endColumn - startColumn + 1 + createIndexEnabled + parseRowsName; if(parseColumnModes(firstRow, firstRowName) != 0) return 2; DEBUG("start/end column: = " << startColumn << ' ' << endColumn); DEBUG("start/end rows = " << startRow << ' ' << endRow); DEBUG("actual cols/rows = " << m_actualCols << ' ' << m_actualRows); return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode, lines); } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { if(!m_prepared) { const int deviceError = prepareDeviceToRead(device); if(deviceError != 0){ DEBUG("Device error = " << deviceError); return; } //TODO: support other modes and vector names m_prepared = true; } importData(dataSource, importMode, lines); } /*! reads the content of document \c doc to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::readDataFromDocument(const QJsonDocument& doc, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { if(!m_prepared) { const int docError = prepareDocumentToRead(doc); if(docError != 0){ DEBUG("Document parse error = " << docError); return; } //TODO: support other modes and vector names m_prepared = true; } importData(dataSource, importMode, lines); } /*! import the content of document \c m_preparedDoc to the data source \c dataSource. Uses the settings defined in the data source. */ void JsonFilterPrivate::importData(AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { Q_UNUSED(lines) m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); int rowOffset = startRow - 1; DEBUG("reading " << m_actualRows << " lines"); for(int i = 0; i < m_actualRows; ++i) { QString rowName; QJsonValue row; switch (containerType) { case JsonFilter::Array: row = *(m_preparedDoc.array().begin() + rowOffset + i); break; case JsonFilter::Object: rowName = (m_preparedDoc.object().begin() + rowOffset + i).key(); row = *(m_preparedDoc.object().begin() + rowOffset + i); break; } int colIndex = 0; for(int n = 0; n < m_actualCols; ++n) { if((createIndexEnabled || parseRowsName) && n == 0) { if(createIndexEnabled) static_cast*>(m_dataContainer[n])->operator[](i) = i + 1; if(parseRowsName) setValueFromString(n + createIndexEnabled, i, rowName); n = n + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue value; switch(rowType){ //TODO: implement other value types case QJsonValue::Array: { value = *(row.toArray().begin() + colIndex); break; } case QJsonValue::Object: { value = *(row.toObject().begin() + colIndex); break; } case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: break; } switch(value.type()) { case QJsonValue::Double: if(columnModes[n] == AbstractColumn::Numeric) static_cast*>(m_dataContainer[n])->operator[](i) = value.toDouble(); else setEmptyValue(n, i + startRow - 1); break; case QJsonValue::String: setValueFromString(n, i, value.toString()); break; case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: setEmptyValue(n, i + startRow - 1); break; } colIndex++; } emit q->completed(100 * i/m_actualRows); } //TODO: fix (startColumn + m_actualCols - 1) dataSource->finalizeImport(m_columnOffset, startColumn, startColumn + m_actualCols - 1, dateTimeFormat, importMode); } /*! generates the preview for the file \c fileName. */ QVector JsonFilterPrivate::preview(const QString& fileName) { KFilterDev device(fileName); return preview(device); } /*! generates the preview for device \c device. */ QVector JsonFilterPrivate::preview(QIODevice &device) { const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return QVector(); } return preview(); } /*! generates the preview for document \c doc. */ QVector JsonFilterPrivate::preview(QJsonDocument &doc) { if(prepareDocumentToRead(doc) != 0) return QVector(); return preview(); } /*! generates the preview for document \c m_preparedDoc. */ QVector JsonFilterPrivate::preview() { QVector dataStrings; int rowOffset = startRow - 1; DEBUG("reading " << m_actualRows << " lines"); for(int i = 0; i < m_actualRows; ++i) { QString rowName; QJsonValue row; switch (containerType) { case JsonFilter::Object: rowName = (m_preparedDoc.object().begin() + rowOffset + i).key(); row = *(m_preparedDoc.object().begin() + rowOffset + i); break; case JsonFilter::Array: row = *(m_preparedDoc.array().begin() + rowOffset + i); break; } QStringList lineString; int colIndex = 0; for(int n = 0; n < m_actualCols; ++n) { if((createIndexEnabled || parseRowsName) && n == 0) { if(createIndexEnabled) lineString += QString::number(i + 1); if(parseRowsName) lineString += rowName; n = n + createIndexEnabled + parseRowsName - 1; continue; } QJsonValue value; switch(rowType){ case QJsonValue::Object: { value = *(row.toObject().begin() + colIndex); break; } case QJsonValue::Array: { value = *(row.toArray().begin() + colIndex); break; } //TODO: implement other value types case QJsonValue::Double: case QJsonValue::String: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: break; } switch(value.type()) { case QJsonValue::Double: if(columnModes[n] == AbstractColumn::Numeric) lineString += QString::number(value.toDouble(), 'g', 16); else lineString += lineString += QLatin1String(""); break; case QJsonValue::String: { //TODO: add parsing string before appending lineString += value.toString(); break; } case QJsonValue::Array: case QJsonValue::Object: case QJsonValue::Bool: case QJsonValue::Null: case QJsonValue::Undefined: lineString += QLatin1String(""); break; } colIndex++; } dataStrings << lineString; emit q->completed(100 * i/m_actualRows); } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void JsonFilterPrivate::write(const QString& fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to json file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void JsonFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("jsonFilter"); writer->writeAttribute("rowType", QString::number(d->rowType)); writer->writeAttribute("dateTimeFormat", d->dateTimeFormat); writer->writeAttribute("numberFormat", QString::number(d->numberFormat)); writer->writeAttribute("createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute("parseRowsName", QString::number(d->parseRowsName)); writer->writeAttribute("nanValue", QString::number(d->nanValue)); writer->writeAttribute("startRow", QString::number(d->startRow)); writer->writeAttribute("endRow", QString::number(d->endRow)); writer->writeAttribute("startColumn", QString::number(d->startColumn)); writer->writeAttribute("endColumn", QString::number(d->endColumn)); QStringList list; for(auto it = modelRows().begin(); it != modelRows().end(); ++it){ list.append(QString::number(*it)); } writer->writeAttribute("modelRows", list.join(';')); writer->writeEndElement(); DEBUG("JsonFilter save params"); } /*! Loads from XML. */ bool JsonFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "jsonFilter") { reader->raiseError(i18n("no json filter element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("rowType").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'rowType'")); else d->rowType = static_cast(str.toInt()); str = attribs.value("dateTimeFormat").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'dateTimeFormat'")); else d->dateTimeFormat = str; str = attribs.value("numberFormat").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'numberFormat'")); else d->numberFormat = static_cast(str.toInt()); str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); str = attribs.value("parseRowsName").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'parseRowsName'")); else d->parseRowsName = str.toInt(); str = attribs.value("nanValue").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'nanValue'")); else d->nanValue = str.toDouble(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startColumn'")); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endColumn'")); else d->endColumn = str.toInt(); QStringList list = attribs.value("modelRows").toString().split(';'); if (list.isEmpty()) reader->raiseWarning(attributeWarning.arg("'modelRows'")); else{ d->modelRows = QVector(); for(auto it = list.begin(); it !=list.end(); ++it) d->modelRows.append((*it).toInt()); } DEBUG("JsonFilter load params"); return true; } \ No newline at end of file diff --git a/src/backend/datasources/filters/JsonFilter.h b/src/backend/datasources/filters/JsonFilter.h index 53e4033b1..76c6e818b 100644 --- a/src/backend/datasources/filters/JsonFilter.h +++ b/src/backend/datasources/filters/JsonFilter.h @@ -1,77 +1,106 @@ +/*************************************************************************** + File : JsonFilter.h + Project : LabPlot + Description : JSON I/O-filter. + -------------------------------------------------------------------- + -------------------------------------------------------------------- + Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) + + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software * + * Foundation, Inc., 51 Franklin Street, Fifth Floor, * + * Boston, MA 02110-1301 USA * + * * + ***************************************************************************/ + #ifndef JSONFILTER_H #define JSONFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" #include #include class QStringList; class QIODevice; class QJsonDocument; class QJsonModel; class JsonFilterPrivate; class JsonFilter : public AbstractFileFilter { Q_OBJECT public: enum DataContainerType {Array, Object}; JsonFilter(); ~JsonFilter() override; static QStringList dataTypes(); static QStringList dataRowTypes(); // read data from any device void readDataFromDevice(QIODevice& device, AbstractDataSource*, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); // overloaded function to read from file QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1) override; void write(const QString& fileName, AbstractDataSource*) override; QVector preview(const QString& fileName); QVector preview(QIODevice& device); QVector preview(QJsonDocument& doc); void loadFilterSettings(const QString&) override; void saveFilterSettings(const QString&) const override; void setDataRowType(const QJsonValue::Type); QJsonValue::Type dataRowType() const; void setModelRows(const QVector); QVector modelRows() const; void setDateTimeFormat(const QString&); QString dateTimeFormat() const; void setNumberFormat(QLocale::Language); QLocale::Language numberFormat() const; void setNaNValueToZero(const bool); bool NaNValueToZeroEnabled() const; void setCreateIndexEnabled(const bool); void setParseRowsName(const bool); void setVectorNames(const QString&); QStringList vectorNames() const; QVector columnModes(); void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; void save(QXmlStreamWriter*) const override; bool load(XmlStreamReader*) override; private: std::unique_ptr const d; friend class JsonFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/JsonFilterPrivate.h b/src/backend/datasources/filters/JsonFilterPrivate.h index 04db4fdff..fc48061d9 100644 --- a/src/backend/datasources/filters/JsonFilterPrivate.h +++ b/src/backend/datasources/filters/JsonFilterPrivate.h @@ -1,68 +1,97 @@ +/*************************************************************************** + File : JsonFilterPrivate.h + Project : LabPlot + Description : Private implementation class for JsonFilter. + -------------------------------------------------------------------- + -------------------------------------------------------------------- + Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) + + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software * + * Foundation, Inc., 51 Franklin Street, Fifth Floor, * + * Boston, MA 02110-1301 USA * + * * + ***************************************************************************/ + #ifndef JSONFILTERPRIVATE_H #define JSONFILTERPRIVATE_H #include #include "QJsonModel.h" class KFilterDev; class AbstractDataSource; class AbstractColumn; class JsonFilterPrivate { public: JsonFilterPrivate (JsonFilter* owner); int checkRow(QJsonValueRef value, int &countCols); int parseColumnModes(QJsonValue row, QString rowName = ""); void setEmptyValue(int column, int row); void setValueFromString(int column, int row, QString value); int prepareDeviceToRead(QIODevice&); int prepareDocumentToRead(const QJsonDocument&); void readDataFromDevice(QIODevice& device, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readDataFromDocument(const QJsonDocument& doc, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void importData(AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName); QVector preview(QIODevice& device); QVector preview(QJsonDocument& doc); QVector preview(); const JsonFilter* q; QJsonModel* model; JsonFilter::DataContainerType containerType; QJsonValue::Type rowType; QVector modelRows; QString dateTimeFormat; QLocale::Language numberFormat; double nanValue; bool createIndexEnabled; bool parseRowsName; QStringList vectorNames; QVector columnModes; int startRow; // start row int endRow; // end row int startColumn; // start column int endColumn; // end column private: int m_actualRows; int m_actualCols; int m_prepared; int m_columnOffset; // indexes the "start column" in the datasource. Data will be imported starting from this column. QVector m_dataContainer; // pointers to the actual data containers (columns). QJsonDocument m_preparedDoc; // parsed Json document }; #endif diff --git a/src/kdefrontend/datasources/JsonOptionsWidget.cpp b/src/kdefrontend/datasources/JsonOptionsWidget.cpp index ab99817e3..61e7a0d93 100644 --- a/src/kdefrontend/datasources/JsonOptionsWidget.cpp +++ b/src/kdefrontend/datasources/JsonOptionsWidget.cpp @@ -1,166 +1,195 @@ +/*************************************************************************** + File : JsonOptionsWidget.cpp + Project : LabPlot + Description : Widget providing options for the import of json data. + -------------------------------------------------------------------- + -------------------------------------------------------------------- + Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) + + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software * + * Foundation, Inc., 51 Franklin Street, Fifth Floor, * + * Boston, MA 02110-1301 USA * + * * + ***************************************************************************/ + #include "ImportFileWidget.h" #include "JsonOptionsWidget.h" #include "backend/datasources/filters/QJsonModel.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/datasources/filters/JsonFilter.h" #include #include #include #include /*! \class JsonOptionsWidget \brief Widget providing options for the import of json data \ingroup kdefrontend */ JsonOptionsWidget::JsonOptionsWidget(QWidget* parent, ImportFileWidget* fileWidget) : QWidget(parent), m_fileWidget(fileWidget), m_model(new QJsonModel()) { ui.setupUi(parent); ui.cbNumberFormat->addItems(AbstractFileFilter::numberFormats()); ui.cbDateTimeFormat->addItems(AbstractColumn::dateTimeFormats()); setTooltips(); } void JsonOptionsWidget::applyFilterSettings(JsonFilter* filter, const QModelIndex& index) const { Q_ASSERT(filter); filter->setModelRows(getIndexRows(index)); filter->setNumberFormat( QLocale::Language(ui.cbNumberFormat->currentIndex())); filter->setDateTimeFormat(ui.cbDateTimeFormat->currentText()); filter->setCreateIndexEnabled(ui.chbCreateIndex->isChecked()); filter->setNaNValueToZero(ui.chbConvertNaNToZero->isChecked()); filter->setParseRowsName(ui.chbParseRowsName->isChecked()); //TODO: change this after implementation other row types filter->setDataRowType(QJsonValue::Array); if(!index.isValid()) return; QJsonTreeItem *item = static_cast(index.internalPointer()); if(item->childCount() < 1) return; filter->setDataRowType(item->child(0)->type()); } void JsonOptionsWidget::clearModel() { m_model->clear(); } void JsonOptionsWidget::loadSettings() const { KConfigGroup conf(KSharedConfig::openConfig(), "ImportJson"); ui.cbNumberFormat->setCurrentIndex(conf.readEntry("NumberFormat", (int)QLocale::AnyLanguage)); ui.cbDateTimeFormat->setCurrentItem(conf.readEntry("DateTimeFormat", "yyyy-MM-dd hh:mm:ss.zzz")); ui.chbCreateIndex->setChecked(conf.readEntry("CreateIndex", false)); ui.chbConvertNaNToZero->setChecked(conf.readEntry("ConvertNaNToZero", false)); ui.chbParseRowsName->setChecked(conf.readEntry("ParseRowsName", false)); } void JsonOptionsWidget::saveSettings() { KConfigGroup conf(KSharedConfig::openConfig(), "ImportJson"); conf.writeEntry("NumberFormat", ui.cbNumberFormat->currentIndex()); conf.writeEntry("DateTimeFormat", ui.cbDateTimeFormat->currentText()); conf.writeEntry("CreateIndex", ui.chbCreateIndex->isChecked()); conf.writeEntry("ConvertNaNToZero", ui.chbConvertNaNToZero->isChecked()); conf.writeEntry("ParseRowsName", ui.chbParseRowsName->isChecked()); } void JsonOptionsWidget::loadDocument(QString filename) { if(m_filename == filename) return; else m_filename = filename; KFilterDev device(m_filename); if (!device.open(QIODevice::ReadOnly)) return; if (device.atEnd() && !device.isSequential()) // empty file return; if(!m_model->loadJson(device.readAll())) m_model->clear(); } QJsonModel* JsonOptionsWidget::model() { return m_model; } void JsonOptionsWidget::setTooltips() { const QString textNumberFormatShort = i18n("This option determines how the imported strings have to be converted to numbers."); const QString textNumberFormat = textNumberFormatShort + "

" + i18n( "For 'C Format', a period is used for the decimal point character and comma is used for the thousands group separator. " "Valid number representations are:" "
    " "
  • 1234.56
  • " "
  • 1,234.56
  • " "
  • etc.
  • " "
" "When using 'System locale', the system settings will be used. " "E.g., for the German local the valid number representations are:" "
    " "
  • 1234,56
  • " "
  • 1.234,56
  • " "
  • etc.
  • " "
" ); ui.lNumberFormat->setToolTip(textNumberFormatShort); ui.lNumberFormat->setWhatsThis(textNumberFormat); ui.cbNumberFormat->setToolTip(textNumberFormatShort); ui.cbNumberFormat->setWhatsThis(textNumberFormat); const QString textDateTimeFormatShort = i18n("This option determines how the imported strings have to be converted to calendar date, i.e. year, month, and day numbers in the Gregorian calendar and to time."); const QString textDateTimeFormat = textDateTimeFormatShort + "

" + i18n( "Expressions that may be used for the date part of format string:" "" "" "" "" "" "" "" "" "" "" "" "
dthe day as number without a leading zero (1 to 31).
ddthe day as number with a leading zero (01 to 31).
dddthe abbreviated localized day name (e.g. 'Mon' to 'Sun'). Uses the system locale to localize the name.
ddddthe long localized day name (e.g. 'Monday' to 'Sunday'). Uses the system locale to localize the name.
Mthe month as number without a leading zero (1 to 12).
MMthe month as number with a leading zero (01 to 12).
MMMthe abbreviated localized month name (e.g. 'Jan' to 'Dec'). Uses the system locale to localize the name.
MMMMthe long localized month name (e.g. 'January' to 'December'). Uses the system locale to localize the name.
yythe year as two digit number (00 to 99).
yyyythe year as four digit number. If the year is negative, a minus sign is prepended in addition.


" "Expressions that may be used for the time part of the format string:" "" "" "" "" "" "" "" "" "" "" "" "" "" "
hthe hour without a leading zero (0 to 23 or 1 to 12 if AM/PM display)
hhthe hour with a leading zero (00 to 23 or 01 to 12 if AM/PM display)
Hthe hour without a leading zero (0 to 23, even with AM/PM display)
HHthe hour with a leading zero (00 to 23, even with AM/PM display)
mthe minute without a leading zero (0 to 59)
mmthe minute with a leading zero (00 to 59)
sthe second without a leading zero (0 to 59)
ssthe second with a leading zero (00 to 59)
zthe milliseconds without leading zeroes (0 to 999)
zzzthe milliseconds with leading zeroes (000 to 999)
AP or Ainterpret as an AM/PM time. AP must be either 'AM' or 'PM'.
ap or aInterpret as an AM/PM time. ap must be either 'am' or 'pm'.


" "Examples are:" "" "" "" "" "
dd.MM.yyyy20.07.1969
ddd MMMM d yySun July 20 69
'The day is' ddddThe day is Sunday
"); ui.lDateTimeFormat->setToolTip(textDateTimeFormatShort); ui.lDateTimeFormat->setWhatsThis(textDateTimeFormat); ui.cbDateTimeFormat->setToolTip(textDateTimeFormatShort); ui.cbDateTimeFormat->setWhatsThis(textDateTimeFormat); } QVector JsonOptionsWidget::getIndexRows(const QModelIndex &index) const { QVector rows; QModelIndex current = index; while(current.isValid()){ rows.prepend(current.row()); current = current.parent(); } return rows; } \ No newline at end of file diff --git a/src/kdefrontend/datasources/JsonOptionsWidget.h b/src/kdefrontend/datasources/JsonOptionsWidget.h index e179e34e0..43540b09e 100644 --- a/src/kdefrontend/datasources/JsonOptionsWidget.h +++ b/src/kdefrontend/datasources/JsonOptionsWidget.h @@ -1,34 +1,63 @@ +/*************************************************************************** + File : JsonOptionsWidget.h + Project : LabPlot + Description : Widget providing options for the import of json data. + -------------------------------------------------------------------- + -------------------------------------------------------------------- + Copyright : (C) 2018 Andrey Cygankov (craftplace.ms@gmail.com) + + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software * + * Foundation, Inc., 51 Franklin Street, Fifth Floor, * + * Boston, MA 02110-1301 USA * + * * + ***************************************************************************/ + #ifndef JSONOPTIONSWIDGET_H #define JSONOPTIONSWIDGET_H #include #include "ui_jsonoptionswidget.h" class ImportFileWidget; class JsonFilter; class QJsonModel; class QJsonTreeItem; class JsonOptionsWidget : public QWidget { Q_OBJECT public: explicit JsonOptionsWidget(QWidget*, ImportFileWidget*); void applyFilterSettings(JsonFilter*, const QModelIndex&) const; void clearModel(); void loadSettings() const; void saveSettings(); void loadDocument(QString filename); QJsonModel* model(); private: void setTooltips(); QVector getIndexRows(const QModelIndex&) const; QString m_filename; Ui::JsonOptionsWidget ui; ImportFileWidget* m_fileWidget; QJsonModel* m_model; }; #endif